WORKER PROXY WIP

.ci/scripts/prepare_old_deps.sh

@@ -31,6 +31,35 @@ sed -i \
    -e '/systemd/d' \
    pyproject.toml
 
+# Use poetry to do the installation. This ensures that the versions are all mutually
+# compatible (as far the package metadata declares, anyway); pip's package resolver
+# is more lax.
+#
+# Rather than `poetry install --no-dev`, we drop all dev dependencies and the dev-docs
+# group from the toml file. This means we don't have to ensure compatibility between
+# old deps and dev tools.
+
+pip install toml wheel
+
+REMOVE_DEV_DEPENDENCIES="
+import toml
+with open('pyproject.toml', 'r') as f:
+    data = toml.loads(f.read())
+
+del data['tool']['poetry']['dev-dependencies']
+del data['tool']['poetry']['group']['dev-docs']
+
+with open('pyproject.toml', 'w') as f:
+    toml.dump(data, f)
+"
+python3 -c "$REMOVE_DEV_DEPENDENCIES"
+
+pip install poetry==1.3.2
+poetry lock
+
 echo "::group::Patched pyproject.toml"
 cat pyproject.toml
 echo "::endgroup::"
+echo "::group::Lockfile after patch"
+cat poetry.lock
+echo "::endgroup::"

.github/workflows/docs-pr-netlify.yaml

@@ -22,7 +22,7 @@ jobs:
           path: book
 
       - name: 📤 Deploy to Netlify
-        uses: matrix-org/netlify-pr-preview@v2
+        uses: matrix-org/netlify-pr-preview@v1
         with:
           path: book
           owner: ${{ github.event.workflow_run.head_repository.owner.login }}

.github/workflows/tests.yml

@@ -107,15 +107,14 @@ jobs:
         uses: dtolnay/rust-toolchain@1.58.1
       - uses: Swatinem/rust-cache@v2
 
-      # Cribbed from
-      # https://github.com/AustinScola/mypy-cache-github-action/blob/85ea4f2972abed39b33bd02c36e341b28ca59213/src/restore.ts#L10-L17
+      # NB: I have two concerns with this action:
+      # 1. We occasionally see odd mypy problems that aren't reproducible
+      #    locally with clean caches. I suspect some dodgy caching behaviour.
+      # 2. The action uses GHA machinery that's deprecated
+      #    (https://github.com/AustinScola/mypy-cache-github-action/issues/277)
+      # It may be simpler to use actions/cache ourselves to restore .mypy_cache.
       - name: Restore/persist mypy's cache
-        uses: actions/cache@v3
-        with:
-          path: |
-            .mypy_cache
-          key: mypy-cache-${{ github.context.sha }}
-          restore-keys: mypy-cache-
+        uses: AustinScola/mypy-cache-github-action@df56268388422ee282636ee2c7a9cc55ec644a41
 
       - name: Run mypy
         run: poetry run mypy
@@ -314,33 +313,41 @@ jobs:
       # There aren't wheels for some of the older deps, so we need to install
       # their build dependencies
       - run: |
-          sudo apt update
           sudo apt-get -qq install build-essential libffi-dev python-dev \
-            libxml2-dev libxslt-dev xmlsec1 zlib1g-dev libjpeg-dev libwebp-dev
+          libxml2-dev libxslt-dev xmlsec1 zlib1g-dev libjpeg-dev libwebp-dev
 
       - uses: actions/setup-python@v4
         with:
           python-version: '3.7'
 
+      # Calculating the old-deps actually takes a bunch of time, so we cache the
+      # pyproject.toml / poetry.lock. We need to cache pyproject.toml as
+      # otherwise the `poetry install` step will error due to the poetry.lock
+      # file being outdated.
+      #
+      # This caches the output of `Prepare old deps`, which should generate the
+      # same `pyproject.toml` and `poetry.lock` for a given `pyproject.toml` input.
+      - uses: actions/cache@v3
+        id: cache-poetry-old-deps
+        name: Cache poetry.lock
+        with:
+          path: |
+            poetry.lock
+            pyproject.toml
+          key: poetry-old-deps2-${{ hashFiles('pyproject.toml') }}
       - name: Prepare old deps
         if: steps.cache-poetry-old-deps.outputs.cache-hit != 'true'
         run: .ci/scripts/prepare_old_deps.sh
 
-      # Note: we install using `pip` here, not poetry. `poetry install` ignores the
-      # build-system section (https://github.com/python-poetry/poetry/issues/6154), but
-      # we explicitly want to test that you can `pip install` using the oldest version
-      # of poetry-core and setuptools-rust.
-      - run: pip install .[all,test]
+      # We only now install poetry so that `setup-python-poetry` caches the
+      # right poetry.lock's dependencies.
+      - uses: matrix-org/setup-python-poetry@v1
+        with:
+          python-version: '3.7'
+          poetry-version: "1.3.2"
+          extras: "all test"
 
-      # We nuke the local copy, as we've installed synapse into the virtualenv
-      # (rather than use an editable install, which we no longer support). If we
-      # don't do this then python can't find the native lib.
-      - run: rm -rf synapse/
-
-      # Sanity check we can import/run Synapse
-      - run: python -m synapse.app.homeserver --help
-
-      - run: python -m twisted.trial -j6 tests
+      - run: poetry run trial -j6 tests
       - name: Dump logs
         # Logs are most useful when the command fails, always include them.
         if: ${{ always() }}

CHANGES.md

@@ -1,96 +1,3 @@
-Synapse 1.84.0 (2023-05-23)
-===========================
-
-The `worker_replication_*` configuration settings have been deprecated in favour of configuring the main process consistently with other instances in the `instance_map`. The deprecated settings will be removed in Synapse v1.88.0, but changing your configuration in advance is recommended. See the [upgrade notes](https://github.com/matrix-org/synapse/blob/release-v1.84/docs/upgrade.md#upgrading-to-v1840) for more information.
-
-Bugfixes
---------
-
-- Fix a bug introduced in Synapse 1.84.0rc1 where errors during startup were not reported correctly on Python < 3.10. ([\#15599](https://github.com/matrix-org/synapse/issues/15599))
-
-
-Synapse 1.84.0rc1 (2023-05-16)
-==============================
-
-Features
---------
-
-- Add an option to prevent media downloads from configured domains. ([\#15197](https://github.com/matrix-org/synapse/issues/15197))
-- Add `forget_rooms_on_leave` config option to automatically forget rooms when users leave them or are removed from them. ([\#15224](https://github.com/matrix-org/synapse/issues/15224))
-- Add redis TLS configuration options. ([\#15312](https://github.com/matrix-org/synapse/issues/15312))
-- Add a config option to delay push notifications by a random amount, to discourage time-based profiling. ([\#15516](https://github.com/matrix-org/synapse/issues/15516))
-- Stabilize support for [MSC2659](https://github.com/matrix-org/matrix-spec-proposals/pull/2659): application service ping endpoint. Contributed by Tulir @ Beeper. ([\#15528](https://github.com/matrix-org/synapse/issues/15528))
-- Implement [MSC4009](https://github.com/matrix-org/matrix-spec-proposals/pull/4009) to expand the supported characters in Matrix IDs. ([\#15536](https://github.com/matrix-org/synapse/issues/15536))
-- Advertise support for Matrix 1.6 on `/_matrix/client/versions`. ([\#15559](https://github.com/matrix-org/synapse/issues/15559))
-- Print full error and stack-trace of any exception that occurs during startup/initialization. ([\#15569](https://github.com/matrix-org/synapse/issues/15569))
-
-
-Bugfixes
---------
-
-- Don't fail on federation over TOR where SRV queries are not supported. Contributed by Zdzichu. ([\#15523](https://github.com/matrix-org/synapse/issues/15523))
-- Experimental support for [MSC4010](https://github.com/matrix-org/matrix-spec-proposals/pull/4010) which rejects setting the `"m.push_rules"` via account data. ([\#15554](https://github.com/matrix-org/synapse/issues/15554), [\#15555](https://github.com/matrix-org/synapse/issues/15555))
-- Fix a long-standing bug where an invalid membership event could cause an internal server error. ([\#15564](https://github.com/matrix-org/synapse/issues/15564))
-- Require at least poetry-core v1.1.0. ([\#15566](https://github.com/matrix-org/synapse/issues/15566), [\#15571](https://github.com/matrix-org/synapse/issues/15571))
-
-
-Deprecations and Removals
--------------------------
-
-- Remove need for `worker_replication_*` based settings in worker configuration yaml by placing this data directly on the `instance_map` instead. ([\#15491](https://github.com/matrix-org/synapse/issues/15491))
-
-
-Updates to the Docker image
----------------------------
-
-- Add pkg-config package to Stage 0 to be able to build Dockerfile on ppc64le architecture. ([\#15567](https://github.com/matrix-org/synapse/issues/15567))
-
-
-Improved Documentation
-----------------------
-
-- Clarify documentation of the "Create or modify account" Admin API. ([\#15544](https://github.com/matrix-org/synapse/issues/15544))
-- Fix path to the `statistics/database/rooms` admin API in documentation. ([\#15560](https://github.com/matrix-org/synapse/issues/15560))
-- Update and improve Mastodon Single Sign-On documentation. ([\#15587](https://github.com/matrix-org/synapse/issues/15587))
-
-
-Internal Changes
-----------------
-
-- Use oEmbed to generate URL previews for YouTube Shorts. ([\#15025](https://github.com/matrix-org/synapse/issues/15025))
-- Create new `Client` for use with HTTP Replication between workers. Contributed by Jason Little. ([\#15470](https://github.com/matrix-org/synapse/issues/15470))
-- Bump pyicu from 2.10.2 to 2.11. ([\#15509](https://github.com/matrix-org/synapse/issues/15509))
-- Remove references to supporting per-user flag for [MSC2654](https://github.com/matrix-org/matrix-spec-proposals/pull/2654). ([\#15522](https://github.com/matrix-org/synapse/issues/15522))
-- Don't use a trusted key server when running the demo scripts. ([\#15527](https://github.com/matrix-org/synapse/issues/15527))
-- Speed up rebuilding of the user directory for local users. ([\#15529](https://github.com/matrix-org/synapse/issues/15529))
-- Speed up deleting of old rows in `event_push_actions`. ([\#15531](https://github.com/matrix-org/synapse/issues/15531))
-- Install the `xmlsec` and `mdbook` packages and switch back to the upstream [cachix/devenv](https://github.com/cachix/devenv) repo in the nix development environment. ([\#15532](https://github.com/matrix-org/synapse/issues/15532), [\#15533](https://github.com/matrix-org/synapse/issues/15533), [\#15545](https://github.com/matrix-org/synapse/issues/15545))
-- Implement [MSC3987](https://github.com/matrix-org/matrix-spec-proposals/pull/3987) by removing `"dont_notify"` from the list of actions in default push rules. ([\#15534](https://github.com/matrix-org/synapse/issues/15534))
-- Move various module API callback registration methods to a dedicated class. ([\#15535](https://github.com/matrix-org/synapse/issues/15535))
-- Proxy `/user/devices` federation queries to application services for [MSC3984](https://github.com/matrix-org/matrix-spec-proposals/pull/3984). ([\#15539](https://github.com/matrix-org/synapse/issues/15539))
-- Factor out an `is_mine_server_name` method. ([\#15542](https://github.com/matrix-org/synapse/issues/15542))
-- Allow running Complement tests using [podman](https://podman.io/) by adding a `PODMAN` environment variable to `scripts-dev/complement.sh`. ([\#15543](https://github.com/matrix-org/synapse/issues/15543))
-- Bump serde from 1.0.160 to 1.0.162. ([\#15548](https://github.com/matrix-org/synapse/issues/15548))
-- Bump types-setuptools from 67.6.0.5 to 67.7.0.1. ([\#15549](https://github.com/matrix-org/synapse/issues/15549))
-- Bump sentry-sdk from 1.19.1 to 1.22.1. ([\#15550](https://github.com/matrix-org/synapse/issues/15550))
-- Bump ruff from 0.0.259 to 0.0.265. ([\#15551](https://github.com/matrix-org/synapse/issues/15551))
-- Bump hiredis from 2.2.2 to 2.2.3. ([\#15552](https://github.com/matrix-org/synapse/issues/15552))
-- Bump types-requests from 2.29.0.0 to 2.30.0.0. ([\#15553](https://github.com/matrix-org/synapse/issues/15553))
-- Add `org.matrix.msc3981` info to `/_matrix/client/versions`. ([\#15558](https://github.com/matrix-org/synapse/issues/15558))
-- Declare unstable support for [MSC3391](https://github.com/matrix-org/matrix-spec-proposals/pull/3391) under `/_matrix/client/versions` if the experimental implementation is enabled. ([\#15562](https://github.com/matrix-org/synapse/issues/15562))
-- Implement [MSC3821](https://github.com/matrix-org/matrix-spec-proposals/pull/3821) to update the redaction rules. ([\#15563](https://github.com/matrix-org/synapse/issues/15563))
-- Implement updated redaction rules from [MSC3389](https://github.com/matrix-org/matrix-spec-proposals/pull/3389). ([\#15565](https://github.com/matrix-org/synapse/issues/15565))
-- Allow `pip install` to use setuptools_rust 1.6.0 when building Synapse. ([\#15570](https://github.com/matrix-org/synapse/issues/15570))
-- Deal with upcoming Github Actions deprecations. ([\#15576](https://github.com/matrix-org/synapse/issues/15576))
-- Export `run_as_background_process` from the module API. ([\#15577](https://github.com/matrix-org/synapse/issues/15577))
-- Update build system requirements to allow building with poetry-core==1.6.0. ([\#15588](https://github.com/matrix-org/synapse/issues/15588))
-- Bump serde from 1.0.162 to 1.0.163. ([\#15589](https://github.com/matrix-org/synapse/issues/15589))
-- Bump phonenumbers from 8.13.7 to 8.13.11. ([\#15590](https://github.com/matrix-org/synapse/issues/15590))
-- Bump types-psycopg2 from 2.9.21.9 to 2.9.21.10. ([\#15591](https://github.com/matrix-org/synapse/issues/15591))
-- Bump types-commonmark from 0.9.2.2 to 0.9.2.3. ([\#15592](https://github.com/matrix-org/synapse/issues/15592))
-- Bump types-setuptools from 67.7.0.1 to 67.7.0.2. ([\#15594](https://github.com/matrix-org/synapse/issues/15594))
-
-
 Synapse 1.83.0 (2023-05-09)
 ===========================
 

Cargo.lock

@@ -323,18 +323,18 @@ checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd"
 
 [[package]]
 name = "serde"
-version = "1.0.163"
+version = "1.0.162"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2113ab51b87a539ae008b5c6c02dc020ffa39afd2d83cffcb3f4eb2722cebec2"
+checksum = "71b2f6e1ab5c2b98c05f0f35b236b22e8df7ead6ffbf51d7808da7f8817e7ab6"
 dependencies = [
  "serde_derive",
 ]
 
 [[package]]
 name = "serde_derive"
-version = "1.0.163"
+version = "1.0.162"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8c805777e3930c8883389c602315a24224bcc738b63905ef87cd1420353ea93e"
+checksum = "a2a0814352fd64b58489904a44ea8d90cb1a91dcb6b4f5ebabc32c8318e93cb6"
 dependencies = [
  "proc-macro2",
  "quote",

changelog.d/10428.removal

@@ -1 +0,0 @@
-Remove the old version of the R30 (30-day retained users) phone-home metric.

changelog.d/15025.misc

@@ -0,0 +1 @@
+Use oEmbed to generate URL previews for YouTube Shorts.

changelog.d/15197.feature

@@ -0,0 +1 @@
+Add an option to prevent media downloads from configured domains.

changelog.d/15224.feature

@@ -0,0 +1 @@
+Add `forget_rooms_on_leave` config option to automatically forget rooms when users leave them or are removed from them.

changelog.d/15388.feature

@@ -1 +0,0 @@
-Stable support for [MSC3882](https://github.com/matrix-org/matrix-spec-proposals/pull/3882) to allow an existing device/session to generate a login token for use on a new device/session.

changelog.d/15437.misc

@@ -0,0 +1 @@
+Make the `thread_id` column on `event_push_actions`, `event_push_actions_staging`, and `event_push_summary` non-null.

changelog.d/15464.bugfix

@@ -1 +0,0 @@
-Fix a long-standing bug where setting the read marker could fail when using message retention. Contributed by Nick @ Beeper (@fizzadar).

changelog.d/15470.misc

@@ -0,0 +1 @@
+Create new `Client` for use with HTTP Replication between workers. Contributed by Jason Little.

changelog.d/15509.misc

@@ -0,0 +1 @@
+Bump pyicu from 2.10.2 to 2.11.

changelog.d/15516.feature

@@ -0,0 +1 @@
+Add a config option to delay push notifications by a random amount, to discourage time-based profiling.

changelog.d/15522.misc

@@ -0,0 +1 @@
+Remove references to supporting per-user flag for [MSC2654](https://github.com/matrix-org/matrix-spec-proposals/pull/2654) (#15522).

changelog.d/15523.bugfix

@@ -0,0 +1 @@
+Don't fail on federation over TOR where SRV queries are not supported. Contributed by Zdzichu.

changelog.d/15527.misc

@@ -0,0 +1 @@
+Don't use a trusted key server when running the demo scripts.

changelog.d/15528.feature

@@ -0,0 +1 @@
+Stabilize support for [MSC2659](https://github.com/matrix-org/matrix-spec-proposals/pull/2659): application service ping endpoint. Contributed by Tulir @ Beeper.

changelog.d/15529.misc

@@ -0,0 +1 @@
+Speed up rebuilding of the user directory for local users.

changelog.d/15531.misc

@@ -0,0 +1 @@
+Speed up deleting of old rows in `event_push_actions`.

changelog.d/15532.misc

@@ -0,0 +1 @@
+Install the `xmlsec` and `mdbook` packages and switch back to the upstream [cachix/devenv](https://github.com/cachix/devenv) repo in the nix development environment.

changelog.d/15533.misc

@@ -0,0 +1 @@
+Install the `xmlsec` and `mdbook` packages and switch back to the upstream [cachix/devenv](https://github.com/cachix/devenv) repo in the nix development environment.

changelog.d/15534.misc

@@ -0,0 +1 @@
+Implement [MSC3987](https://github.com/matrix-org/matrix-spec-proposals/pull/3987) by removing `"dont_notify"` from the list of actions in default push rules.

changelog.d/15535.misc

@@ -0,0 +1 @@
+Move various module API callback registration methods to a dedicated class.

changelog.d/15536.feature

@@ -0,0 +1 @@
+Implement [MSC4009](https://github.com/matrix-org/matrix-spec-proposals/pull/4009) to expand the supported characters in Matrix IDs.

changelog.d/15537.misc

@@ -1 +0,0 @@
-Add not null constraint to column full_user_id of tables profiles and user_filters.

changelog.d/15539.misc

@@ -0,0 +1 @@
+Proxy `/user/devices` federation queries to application services for [MSC3984](https://github.com/matrix-org/matrix-spec-proposals/pull/3984).

changelog.d/15542.misc

@@ -0,0 +1 @@
+Factor out an `is_mine_server_name` method.

changelog.d/15543.misc

@@ -0,0 +1 @@
+Allow running Complement tests using [podman](https://podman.io/) by adding a `PODMAN` environment variable to `scripts-dev/complement.sh`.

changelog.d/15544.doc

@@ -0,0 +1 @@
+Clarify documentation of the "Create or modify account" Admin API.

changelog.d/15545.misc

@@ -0,0 +1 @@
+ Install the `xmlsec` and `mdbook` packages and switch back to the upstream [cachix/devenv](https://github.com/cachix/devenv) repo in the nix development environment.

changelog.d/15548.misc

@@ -0,0 +1 @@
+Bump serde from 1.0.160 to 1.0.162.

changelog.d/15549.misc

@@ -0,0 +1 @@
+Bump types-setuptools from 67.6.0.5 to 67.7.0.1.

changelog.d/15550.misc

@@ -0,0 +1 @@
+Bump sentry-sdk from 1.19.1 to 1.22.1.

changelog.d/15551.misc

@@ -0,0 +1 @@
+Bump ruff from 0.0.259 to 0.0.265.

changelog.d/15552.misc

@@ -0,0 +1 @@
+Bump hiredis from 2.2.2 to 2.2.3.

changelog.d/15553.misc

@@ -0,0 +1 @@
+Bump types-requests from 2.29.0.0 to 2.30.0.0.

changelog.d/15554.bugfix

@@ -0,0 +1 @@
+Experimental support for [MSC4010](https://github.com/matrix-org/matrix-spec-proposals/pull/4010) which rejects setting the `"m.push_rules"` via account data.

changelog.d/15555.bugfix

@@ -0,0 +1 @@
+Experimental support for [MSC4010](https://github.com/matrix-org/matrix-spec-proposals/pull/4010) which rejects setting the `"m.push_rules"` via account data.

changelog.d/15560.doc

@@ -0,0 +1 @@
+Fix path to the `statistics/database/rooms` admin API in documentation.

changelog.d/15578.misc

@@ -1 +0,0 @@
-Allow connecting to HTTP Replication Endpoints by using `worker_name` when constructing the request.

changelog.d/15599.bugfix

@@ -1 +0,0 @@
-Print full error and stack-trace of any exception that occurs during startup/initialization.

changelog.d/15601.bugfix

@@ -1 +0,0 @@
-Fix a long-standing bug where the `url_preview_url_blacklist` configuration setting was not applied to oEmbed or image URLs found while previewing a URL.

changelog.d/15602.misc

@@ -1 +0,0 @@
-Run mypy type checking with the minimum supported Python version to catch new usage that isn't backwards-compatible.

changelog.d/15604.misc

@@ -1 +0,0 @@
-Fix subscriptable type usage in Python <3.9.

changelog.d/15606.misc

@@ -1 +0,0 @@
-Update internal terminology.

changelog.d/15607.bugfix

@@ -1 +0,0 @@
-Fix a long-standing bug where filters with multiple backslashes were rejected.

changelog.d/15610.misc

@@ -1 +0,0 @@
-Instrument `state` and `state_group` storage-related operations to better picture what's happening when tracing.

changelog.d/15611.feature

@@ -1 +0,0 @@
-Add a new admin API to create a new device for a user.

changelog.d/15613.doc

@@ -1 +0,0 @@
-Warn users that at least 3.75GB of space is needed for the nix Synapse development environment.

changelog.d/15614.bugfix

@@ -1 +0,0 @@
-Fix a bug introduced in Synapse 1.82.0 where the error message displayed when validation of the `app_service_config_files` config option fails would be incorrectly formatted.

changelog.d/15615.misc

@@ -1 +0,0 @@
-Re-type config paths in `ConfigError`s to be `StrSequence`s instead of `Iterable[str]`s.

changelog.d/15620.misc

@@ -1 +0,0 @@
-Update internal terminology.

changelog.d/15621.misc

@@ -1 +0,0 @@
-Update Mutual Rooms (MSC2666) implementation to match new proposal text.

changelog.d/15624.bugfix

@@ -1 +0,0 @@
-Fix a long-standing bug where deactivated users were still able to login using the custom `org.matrix.login.jwt` login type (if enabled).

changelog.d/15625.misc

@@ -1 +0,0 @@
-Remove the unstable identifiers from faster joins ([MSC3706](https://github.com/matrix-org/matrix-spec-proposals/pull/3706).

changelog.d/15626.misc

@@ -1 +0,0 @@
-Fix the olddeps CI.

changelog.d/15630.misc

@@ -1 +0,0 @@
-Fix two memory leaks in `trial` test runs.

changelog.d/15633.misc

@@ -1 +0,0 @@
-Trace how many new events from the backfill response we need to process.

changelog.d/15634.bugfix

@@ -1 +0,0 @@
-Fix a long-standing bug where deactivated users were able to login in uncommon situations.

changelog.d/15636.misc

@@ -1 +0,0 @@
-Remove duplicate timestamp from test logs (`_trial_temp/test.log`).

changelog.d/15639.misc

@@ -1 +0,0 @@
-Bump types-setuptools from 67.7.0.2 to 67.8.0.0.

changelog.d/15640.misc

@@ -1 +0,0 @@
-Bump types-pillow from 9.5.0.2 to 9.5.0.4.

changelog.d/15641.misc

@@ -1 +0,0 @@
-Bump sphinx from 6.1.3 to 6.2.1.

changelog.d/15642.misc

@@ -1 +0,0 @@
-Bump furo from 2023.3.27 to 2023.5.20.

changelog.d/15643.misc

@@ -1 +0,0 @@
-Bump pygithub from 1.58.1 to 1.58.2.

changelog.d/15646.misc

@@ -1 +0,0 @@
-Limit the size of the `HomeServerConfig` cache in trial test runs.

changelog.d/15647.bugfix

@@ -1 +0,0 @@
-Instrument `state` and `state_group` storage-related operations to better picture what's happening when tracing.

changelog.d/15648.doc

@@ -1 +0,0 @@
-Remove outdated comment from the generated and sample homeserver log configs.

changelog.d/15651.misc

@@ -1 +0,0 @@
-Bump requests from 2.28.2 to 2.31.0.

changelog.d/15658.misc

@@ -1 +0,0 @@
-Improve type hints.

changelog.d/15670.removal

@@ -1 +0,0 @@
-Remove support for unstable revisions of [MSC3882](https://github.com/matrix-org/matrix-spec-proposals/pull/3882).

contrib/docker_compose_workers/README.md

@@ -70,10 +70,6 @@ redis:
   port: 6379
   # dbid:  <redis_logical_db_id>
   # password: <secret_password>  
-  # use_tls: True
-  # certificate_file: <path_to_certificate>
-  # private_key_file: <path_to_private_key>
-  # ca_file: <path_to_ca_certificate>
 ```
 
 This assumes that your Redis service is called `redis` in your Docker Compose file.

debian/changelog

@@ -1,15 +1,3 @@
-matrix-synapse-py3 (1.84.0) stable; urgency=medium
-
-  * New Synapse release 1.84.0.
-
- -- Synapse Packaging team <packages@matrix.org>  Tue, 23 May 2023 10:57:22 +0100
-
-matrix-synapse-py3 (1.84.0~rc1) stable; urgency=medium
-
-  * New Synapse release 1.84.0rc1.
-
- -- Synapse Packaging team <packages@matrix.org>  Tue, 16 May 2023 11:12:02 +0100
-
 matrix-synapse-py3 (1.83.0) stable; urgency=medium
 
   * New Synapse release 1.83.0.

docker/Dockerfile

@@ -37,7 +37,7 @@ RUN \
   --mount=type=cache,target=/var/cache/apt,sharing=locked \
   --mount=type=cache,target=/var/lib/apt,sharing=locked \
   apt-get update -qq && apt-get install -yqq \
-  build-essential curl git libffi-dev libssl-dev pkg-config \
+  build-essential curl git libffi-dev libssl-dev \
   && rm -rf /var/lib/apt/lists/*
 
 # Install rust and ensure its in the PATH.

docker/conf-workers/worker.yaml.j2

@@ -6,6 +6,10 @@
 worker_app: "{{ app }}"
 worker_name: "{{ name }}"
 
+# The replication listener on the main synapse process.
+worker_replication_host: 127.0.0.1
+worker_replication_http_port: 9093
+
 worker_listeners:
   - type: http
     port: {{ port }}

docker/configure_workers_and_start.py

@@ -69,9 +69,6 @@ import yaml
 from jinja2 import Environment, FileSystemLoader
 
 MAIN_PROCESS_HTTP_LISTENER_PORT = 8080
-MAIN_PROCESS_INSTANCE_NAME = "main"
-MAIN_PROCESS_LOCALHOST_ADDRESS = "127.0.0.1"
-MAIN_PROCESS_REPLICATION_PORT = 9093
 
 # A simple name used as a placeholder in the WORKERS_CONFIG below. This will be replaced
 # during processing with the name of the worker.
@@ -722,8 +719,8 @@ def generate_worker_files(
     # shared config file.
     listeners = [
         {
-            "port": MAIN_PROCESS_REPLICATION_PORT,
-            "bind_address": MAIN_PROCESS_LOCALHOST_ADDRESS,
+            "port": 9093,
+            "bind_address": "127.0.0.1",
             "type": "http",
             "resources": [{"names": ["replication"]}],
         }
@@ -873,14 +870,6 @@ def generate_worker_files(
 
     workers_in_use = len(requested_worker_types) > 0
 
-    # If there are workers, add the main process to the instance_map too.
-    if workers_in_use:
-        instance_map = shared_config.setdefault("instance_map", {})
-        instance_map[MAIN_PROCESS_INSTANCE_NAME] = {
-            "host": MAIN_PROCESS_LOCALHOST_ADDRESS,
-            "port": MAIN_PROCESS_REPLICATION_PORT,
-        }
-
     # Shared homeserver config
     convert(
         "/conf/shared.yaml.j2",

docs/admin_api/user_admin_api.md

@@ -813,33 +813,6 @@ The following fields are returned in the JSON response body:
 
 - `total` - Total number of user's devices.
 
-### Create a device
-
-Creates a new device for a specific `user_id` and `device_id`. Does nothing if the `device_id` 
-exists already.
-
-The API is:
-
-```
-POST /_synapse/admin/v2/users/<user_id>/devices
-
-{
-  "device_id": "QBUAZIFURK"
-}
-```
-
-An empty JSON dict is returned.
-
-**Parameters**
-
-The following parameters should be set in the URL:
-
-- `user_id` - fully qualified: for example, `@user:server.com`.
-
-The following fields are required in the JSON request body:
-
-- `device_id` - The device ID to create.
-
 ### Delete multiple devices
 Deletes the given devices for a specific `user_id`, and invalidates
 any access token associated with them.

docs/modules/password_auth_provider_callbacks.md

@@ -46,9 +46,6 @@ instead.
 
 If the authentication is unsuccessful, the module must return `None`.
 
-Note that the user is not automatically registered, the `register_user(..)` method of
-the [module API](writing_a_module.html) can be used to lazily create users.
-
 If multiple modules register an auth checker for the same login type but with different
 fields, Synapse will refuse to start.
 

docs/openid.md

@@ -569,7 +569,7 @@ You should receive a response similar to the following. Make sure to save it.
 {"client_id":"someclientid_123","client_secret":"someclientsecret_123","id":"12345","name":"my_synapse_app","redirect_uri":"https://[synapse_public_baseurl]/_synapse/client/oidc/callback","website":null,"vapid_key":"somerandomvapidkey_123"}
 ```
 
-As the Synapse login mechanism needs an attribute to uniquely identify users, and Mastodon's endpoint does not return a `sub` property, an alternative `subject_template` has to be set. Your Synapse configuration should include the following:
+As the Synapse login mechanism needs an attribute to uniquely identify users, and Mastodon's endpoint does not return a `sub` property, an alternative `subject_claim` has to be set. Your Synapse configuration should include the following:
 
 ```yaml
 oidc_providers:
@@ -585,9 +585,7 @@ oidc_providers:
     scopes: ["read"]
     user_mapping_provider:
       config:
-        subject_template: "{{ user.id }}"
-        localpart_template: "{{ user.username }}"
-        display_name_template: "{{ user.display_name }}"
+        subject_claim: "id"
 ```
 
 Note that the fields `client_id` and `client_secret` are taken from the CURL response above.

docs/replication.md

@@ -30,6 +30,12 @@ minimal.
 
 See [the TCP replication documentation](tcp_replication.md).
 
+### The Slaved DataStore
+
+There are read-only version of the synapse storage layer in
+`synapse/replication/slave/storage` that use the response of the
+replication API to invalidate their caches.
+
 ### The TCP Replication Module
 Information about how the tcp replication module is structured, including how
 the classes interact, can be found in

docs/sample_log_config.yaml

@@ -68,7 +68,9 @@ root:
     # Write logs to the `buffer` handler, which will buffer them together in memory,
     # then write them to a file.
     #
-    # Replace "buffer" with "console" to log to stderr instead.
+    # Replace "buffer" with "console" to log to stderr instead. (Note that you'll
+    # also need to update the configuration for the `twisted` logger above, in
+    # this case.)
     #
     handlers: [buffer]
 

docs/systemd-with-workers/workers/generic_worker.yaml

@@ -1,6 +1,10 @@
 worker_app: synapse.app.generic_worker
 worker_name: generic_worker1
 
+# The replication listener on the main synapse process.
+worker_replication_host: 127.0.0.1
+worker_replication_http_port: 9093
+
 worker_listeners:
   - type: http
     port: 8083

docs/upgrade.md

@@ -88,95 +88,6 @@ process, for example:
     dpkg -i matrix-synapse-py3_1.3.0+stretch1_amd64.deb
     ```
 
-# Upgrading to v1.84.0
-
-## Deprecation of `worker_replication_*` configuration settings
-
-When using workers,
-
-* `worker_replication_host`
-* `worker_replication_http_port`
-* `worker_replication_http_tls`
- 
-should now be removed from individual worker YAML configurations and the main process should instead be added to the `instance_map`
-in the shared YAML configuration, using the name `main`.
-
-The old `worker_replication_*` settings are now considered deprecated and are expected to be removed in Synapse v1.88.0.
-
-
-### Example change
-
-#### Before:
-
-Shared YAML
-```yaml
-instance_map:
-  generic_worker1:
-    host: localhost
-    port: 5678
-    tls: false
-```
-
-Worker YAML
-```yaml
-worker_app: synapse.app.generic_worker
-worker_name: generic_worker1
-
-worker_replication_host: localhost
-worker_replication_http_port: 3456
-worker_replication_http_tls: false
-
-worker_listeners:
-  - type: http
-    port: 1234
-    resources:
-      - names: [client, federation]
-  - type: http
-    port: 5678
-    resources:
-      - names: [replication]
-
-worker_log_config: /etc/matrix-synapse/generic-worker-log.yaml
-```
-
-
-#### After:
-
-Shared YAML
-```yaml
-instance_map:
-  main:
-    host: localhost
-    port: 3456
-    tls: false
-  generic_worker1:
-    host: localhost
-    port: 5678
-    tls: false
-```
-
-Worker YAML
-```yaml
-worker_app: synapse.app.generic_worker
-worker_name: generic_worker1
-
-worker_listeners:
-  - type: http
-    port: 1234
-    resources:
-      - names: [client, federation]
-  - type: http
-    port: 5678
-    resources:
-      - names: [replication]
-
-worker_log_config: /etc/matrix-synapse/generic-worker-log.yaml
-
-```
-Notes: 
-* `tls` is optional but mirrors the functionality of `worker_replication_http_tls`
-
-
 # Upgrading to v1.81.0
 
 ## Application service path & authentication deprecations

docs/usage/administration/monitoring/reporting_homeserver_usage_statistics.md

@@ -42,6 +42,11 @@ The following statistics are sent to the configured reporting endpoint:
 | `daily_e2ee_messages`      | int    | The number of (state) events with the type `m.room.encrypted` seen in the last 24 hours.                                                                                                                                                                                                        |
 | `daily_sent_messages`      | int    | The number of (state) events sent by a local user with the type `m.room.message` seen in the last 24 hours.                                                                                                                                                                                     |
 | `daily_sent_e2ee_messages` | int    | The number of (state) events sent by a local user with the type `m.room.encrypted` seen in the last 24 hours.                                                                                                                                                                                   |
+| `r30_users_all`            | int    | The number of 30 day retained users, defined as users who have created their accounts more than 30 days ago, where they were last seen at most 30 days ago and where those two timestamps are over 30 days apart. Includes clients that do not fit into the below r30 client types.             |
+| `r30_users_android`        | int    | The number of 30 day retained users, as defined above. Filtered only to clients with "Android" in the user agent string.                                                                                                                                                                        |
+| `r30_users_ios`            | int    | The number of 30 day retained users, as defined above. Filtered only to clients with "iOS" in the user agent string.                                                                                                                                                                            |
+| `r30_users_electron`       | int    | The number of 30 day retained users, as defined above. Filtered only to clients with "Electron" in the user agent string.                                                                                                                                                                       |
+| `r30_users_web`            | int    | The number of 30 day retained users, as defined above. Filtered only to clients with "Mozilla" or "Gecko" in the user agent string.                                                                                                                                                             |
 | `r30v2_users_all`          | int    | The number of 30 day retained users, with a revised algorithm. Defined as users that appear more than once in the past 60 days, and have more than 30 days between the most and least recent appearances in the past 60 days. Includes clients that do not fit into the below r30 client types. |
 | `r30v2_users_android`      | int    | The number of 30 day retained users, as defined above. Filtered only to clients with ("riot" or "element") and "android" (case-insensitive) in the user agent string.                                                                                                                           |
 | `r30v2_users_ios`          | int    | The number of 30 day retained users, as defined above. Filtered only to clients with ("riot" or "element") and "ios" (case-insensitive) in the user agent string.                                                                                                                               |

docs/usage/configuration/config_documentation.md

@@ -2570,49 +2570,7 @@ Example configuration:
 ```yaml
 nonrefreshable_access_token_lifetime: 24h
 ```
----
-### `ui_auth`
 
-The amount of time to allow a user-interactive authentication session to be active.
-
-This defaults to 0, meaning the user is queried for their credentials
-before every action, but this can be overridden to allow a single
-validation to be re-used.  This weakens the protections afforded by
-the user-interactive authentication process, by allowing for multiple
-(and potentially different) operations to use the same validation session.
-
-This is ignored for potentially "dangerous" operations (including
-deactivating an account, modifying an account password, adding a 3PID,
-and minting additional login tokens).
-
-Use the `session_timeout` sub-option here to change the time allowed for credential validation.
-
-Example configuration:
-```yaml
-ui_auth:
-    session_timeout: "15s"
-```
----
-### `login_via_existing_session`
-
-Matrix supports the ability of an existing session to mint a login token for
-another client.
-
-Synapse disables this by default as it has security ramifications.
-
-The duration of time the generated token is valid for can be configured with the
-`token_timeout` sub-option.
-
-User-interactive authentication is required when this is enabled unless the
-`require_ui_auth` sub-option is set to `False`.
-
-Example configuration:
-```yaml
-login_via_existing_session:
-    enabled: true
-    require_ui_auth: false
-    token_timeout: "5m"
-```
 ---
 ## Metrics
 Config options related to metrics.
@@ -3457,6 +3415,28 @@ password_config:
       require_uppercase: true
 ```
 ---
+### `ui_auth`
+
+The amount of time to allow a user-interactive authentication session to be active.
+
+This defaults to 0, meaning the user is queried for their credentials
+before every action, but this can be overridden to allow a single
+validation to be re-used.  This weakens the protections afforded by
+the user-interactive authentication process, by allowing for multiple
+(and potentially different) operations to use the same validation session.
+
+This is ignored for potentially "dangerous" operations (including
+deactivating an account, modifying an account password, and
+adding a 3PID).
+
+Use the `session_timeout` sub-option here to change the time allowed for credential validation.
+
+Example configuration:
+```yaml
+ui_auth:
+    session_timeout: "15s"
+```
+---
 ## Push
 Configuration settings related to push notifications
 
@@ -3904,20 +3884,15 @@ federation_sender_instances:
 ### `instance_map`
 
 When using workers this should be a map from [`worker_name`](#worker_name) to the
-HTTP replication listener of the worker, if configured, and to the main process.
+HTTP replication listener of the worker, if configured.
 Each worker declared under [`stream_writers`](../../workers.md#stream-writers) needs
 a HTTP replication listener, and that listener should be included in the `instance_map`.
-The main process also needs an entry on the `instance_map`, and it should be listed under
-`main` **if even one other worker exists**. Ensure the port matches with what is declared 
-inside the `listener` block for a `replication` listener.
-
+(The main process also needs an HTTP replication listener, but it should not be
+listed in the `instance_map`.)
 
 Example configuration:
 ```yaml
 instance_map:
-  main:
-    host: localhost
-    port: 8030
   worker1:
     host: localhost
     port: 8034
@@ -4001,16 +3976,9 @@ This setting has the following sub-options:
    localhost and 6379
 * `password`: Optional password if configured on the Redis instance.
 * `dbid`: Optional redis dbid if needs to connect to specific redis logical db.
-* `use_tls`: Whether to use tls connection. Defaults to false.
-* `certificate_file`: Optional path to the certificate file
-* `private_key_file`: Optional path to the private key file
-* `ca_file`: Optional path to the CA certificate file. Use this one or:
-* `ca_path`: Optional path to the folder containing the CA certificate file
 
   _Added in Synapse 1.78.0._
 
-  _Changed in Synapse 1.84.0: Added use\_tls, certificate\_file, private\_key\_file, ca\_file and ca\_path attributes_
-
 Example configuration:
 ```yaml
 redis:
@@ -4019,10 +3987,6 @@ redis:
   port: 6379
   password: <secret_password>
   dbid: <dbid>
-  #use_tls: True
-  #certificate_file: <path_to_the_certificate_file>
-  #private_key_file: <path_to_the_private_key_file>
-  #ca_file: <path_to_the_ca_certificate_file>
 ```
 ---
 ## Individual worker configuration
@@ -4060,7 +4024,6 @@ worker_name: generic_worker1
 ```
 ---
 ### `worker_replication_host`
-*Deprecated as of version 1.84.0. Place `host` under `main` entry on the [`instance_map`](#instance_map) in your shared yaml configuration instead.*
 
 The HTTP replication endpoint that it should talk to on the main Synapse process.
 The main Synapse process defines this with a `replication` resource in
@@ -4072,7 +4035,6 @@ worker_replication_host: 127.0.0.1
 ```
 ---
 ### `worker_replication_http_port`
-*Deprecated as of version 1.84.0. Place `port` under `main` entry on the [`instance_map`](#instance_map) in your shared yaml configuration instead.*
 
 The HTTP replication port that it should talk to on the main Synapse process.
 The main Synapse process defines this with a `replication` resource in
@@ -4084,7 +4046,6 @@ worker_replication_http_port: 9093
 ```
 ---
 ### `worker_replication_http_tls`
-*Deprecated as of version 1.84.0. Place `tls` under `main` entry on the [`instance_map`](#instance_map) in your shared yaml configuration instead.*
 
 Whether TLS should be used for talking to the HTTP replication port on the main
 Synapse process.
@@ -4110,9 +4071,9 @@ A worker can handle HTTP requests. To do so, a `worker_listeners` option
 must be declared, in the same way as the [`listeners` option](#listeners)
 in the shared config.
 
-Workers declared in [`stream_writers`](#stream_writers) and [`instance_map`](#instance_map)
- will need to include a `replication` listener here, in order to accept internal HTTP 
-requests from other workers.
+Workers declared in [`stream_writers`](#stream_writers) will need to include a
+`replication` listener here, in order to accept internal HTTP requests from
+other workers.
 
 Example configuration:
 ```yaml

docs/workers.md

@@ -87,18 +87,12 @@ shared configuration file.
 
 ### Shared configuration
 
-Normally, only a few changes are needed to make an existing configuration
-file suitable for use with workers:
-* First, you need to enable an
+Normally, only a couple of changes are needed to make an existing configuration
+file suitable for use with workers. First, you need to enable an
 ["HTTP replication listener"](usage/configuration/config_documentation.md#listeners)
-for the main process
-* Secondly, you need to enable
-[redis-based replication](usage/configuration/config_documentation.md#redis)
-* You will need to add an [`instance_map`](usage/configuration/config_documentation.md#instance_map) 
-with the `main` process defined, as well as the relevant connection information from 
-it's HTTP `replication` listener (defined in step 1 above). Note that the `host` defined 
-is the address the worker needs to look for the `main` process at, not necessarily the same address that is bound to.
-* Optionally, a [shared secret](usage/configuration/config_documentation.md#worker_replication_secret)
+for the main process; and secondly, you need to enable
+[redis-based replication](usage/configuration/config_documentation.md#redis).
+Optionally, a [shared secret](usage/configuration/config_documentation.md#worker_replication_secret)
 can be used to authenticate HTTP traffic between workers. For example:
 
 ```yaml
@@ -117,11 +111,6 @@ worker_replication_secret: ""
 
 redis:
     enabled: true
-
-instance_map:
-    main:
-        host: 'localhost'
-        port: 9093
 ```
 
 See the [configuration manual](usage/configuration/config_documentation.md)
@@ -141,13 +130,13 @@ In the config file for each worker, you must specify:
  * The type of worker ([`worker_app`](usage/configuration/config_documentation.md#worker_app)).
    The currently available worker applications are listed [below](#available-worker-applications).
  * A unique name for the worker ([`worker_name`](usage/configuration/config_documentation.md#worker_name)).
+ * The HTTP replication endpoint that it should talk to on the main synapse process
+   ([`worker_replication_host`](usage/configuration/config_documentation.md#worker_replication_host) and
+   [`worker_replication_http_port`](usage/configuration/config_documentation.md#worker_replication_http_port)).
  * If handling HTTP requests, a [`worker_listeners`](usage/configuration/config_documentation.md#worker_listeners) option
    with an `http` listener.
  * **Synapse 1.72 and older:** if handling the `^/_matrix/client/v3/keys/upload` endpoint, the HTTP URI for
    the main process (`worker_main_http_uri`). This config option is no longer required and is ignored when running Synapse 1.73 and newer.
- * **Synapse 1.83 and older:** The HTTP replication endpoint that the worker should talk to on the main synapse process
-   ([`worker_replication_host`](usage/configuration/config_documentation.md#worker_replication_host) and
-   [`worker_replication_http_port`](usage/configuration/config_documentation.md#worker_replication_http_port)). If using Synapse 1.84 and newer, these are not needed if `main` is defined on the [shared configuration](#shared-configuration) `instance_map`
 
 For example:
 
@@ -428,14 +417,11 @@ effects of bursts of events from that bridge on events sent by normal users.
 Additionally, the writing of specific streams (such as events) can be moved off
 of the main process to a particular worker.
 
-To enable this, the worker must have:
-* An [HTTP `replication` listener](usage/configuration/config_documentation.md#listeners) configured,
-* Have a [`worker_name`](usage/configuration/config_documentation.md#worker_name)
+To enable this, the worker must have a
+[HTTP `replication` listener](usage/configuration/config_documentation.md#listeners) configured,
+have a [`worker_name`](usage/configuration/config_documentation.md#worker_name)
 and be listed in the [`instance_map`](usage/configuration/config_documentation.md#instance_map)
-config. 
-* Have the main process declared on the [`instance_map`](usage/configuration/config_documentation.md#instance_map) as well.
-
-Note: The same worker can handle multiple streams, but unless otherwise documented,
+config. The same worker can handle multiple streams, but unless otherwise documented,
 each stream can only have a single writer.
 
 For example, to move event persistence off to a dedicated worker, the shared
@@ -443,9 +429,6 @@ configuration would include:
 
 ```yaml
 instance_map:
-    main:
-        host: localhost
-        port: 8030
     event_persister1:
         host: localhost
         port: 8034

flake.nix

@@ -1,30 +1,35 @@
-# A Nix flake that sets up a complete Synapse development environment. Dependencies
+# A nix flake that sets up a complete Synapse development environment. Dependencies
 # for the SyTest (https://github.com/matrix-org/sytest) and Complement
 # (https://github.com/matrix-org/complement) Matrix homeserver test suites are also
 # installed automatically.
 #
-# You must have already installed Nix (https://nixos.org) on your system to use this.
-# Nix can be installed on Linux or MacOS; NixOS is not required. Windows is not
-# directly supported, but Nix can be installed inside of WSL2 or even Docker
+# You must have already installed nix (https://nixos.org) on your system to use this.
+# nix can be installed on Linux or MacOS; NixOS is not required. Windows is not
+# directly supported, but nix can be installed inside of WSL2 or even Docker
 # containers. Please refer to https://nixos.org/download for details.
 #
 # You must also enable support for flakes in Nix. See the following for how to
 # do so permanently: https://nixos.wiki/wiki/Flakes#Enable_flakes
 #
-# Be warned: you'll need over 3.75 GB of free space to download all the dependencies.
-#
 # Usage:
 #
-# With Nix installed, navigate to the directory containing this flake and run
+# With nix installed, navigate to the directory containing this flake and run
 # `nix develop --impure`. The `--impure` is necessary in order to store state
 # locally from "services", such as PostgreSQL and Redis.
 #
 # You should now be dropped into a new shell with all programs and dependencies
 # availabile to you!
 #
-# You can start up pre-configured local Synapse, PostgreSQL and Redis instances by
+# You can start up pre-configured, local PostgreSQL and Redis instances by
 # running: `devenv up`. To stop them, use Ctrl-C.
 #
+# A PostgreSQL database called 'synapse' will be set up for you, along with
+# a PostgreSQL user named 'synapse_user'.
+# The 'host' can be found by running `echo $PGHOST` with the development
+# shell activated. Use these values to configure your Synapse to connect
+# to the local PostgreSQL database. You do not need to specify a password.
+# https://matrix-org.github.io/synapse/latest/postgres
+#
 # All state (the venv, postgres and redis data and config) are stored in
 # .devenv/state. Deleting a file from here and then re-entering the shell
 # will recreate these files from scratch.
@@ -61,7 +66,7 @@
         let
           pkgs = nixpkgs.legacyPackages.${system};
         in {
-          # Everything is configured via devenv - a Nix module for creating declarative
+          # Everything is configured via devenv - a nix module for creating declarative
           # developer environments. See https://devenv.sh/reference/options/ for a list
           # of all possible options.
           default = devenv.lib.mkShell {
@@ -148,39 +153,11 @@
                 # Redis is needed in order to run Synapse in worker mode.
                 services.redis.enable = true;
 
-                # Configure and start Synapse. Before starting Synapse, this shell code:
-                #  * generates a default homeserver.yaml config file if one does not exist, and
-                #  * ensures a directory containing two additional homeserver config files exists;
-                #    one to configure using the development environment's PostgreSQL as the
-                #    database backend and another for enabling Redis support.
-                process.before = ''
-                  python -m synapse.app.homeserver -c homeserver.yaml --generate-config --server-name=synapse.dev --report-stats=no
-                  mkdir -p homeserver-config-overrides.d
-                  cat > homeserver-config-overrides.d/database.yaml << EOF
-                  ## Do not edit this file. This file is generated by flake.nix
-                  database:
-                    name: psycopg2
-                    args:
-                      user: synapse_user
-                      database: synapse
-                      host: $PGHOST
-                      cp_min: 5
-                      cp_max: 10
-                  EOF
-                  cat > homeserver-config-overrides.d/redis.yaml << EOF
-                  ## Do not edit this file. This file is generated by flake.nix
-                  redis:
-                    enabled: true
-                  EOF
-                '';
-                # Start synapse when `devenv up` is run.
-                processes.synapse.exec = "poetry run python -m synapse.app.homeserver -c homeserver.yaml --config-directory homeserver-config-overrides.d";
-
                 # Define the perl modules we require to run SyTest.
                 #
                 # This list was compiled by cross-referencing https://metacpan.org/
                 # with the modules defined in './cpanfile' and then finding the
-                # corresponding Nix packages on https://search.nixos.org/packages.
+                # corresponding nix packages on https://search.nixos.org/packages.
                 #
                 # This was done until `./install-deps.pl --dryrun` produced no output.
                 env.PERL5LIB = "${with pkgs.perl536Packages; makePerlPath [

mypy.ini

@@ -13,9 +13,6 @@ no_implicit_optional = True
 disallow_untyped_defs = True
 strict_equality = True
 warn_redundant_casts = True
-# Run mypy type checking with the minimum supported Python version to catch new usage
-# that isn't backwards-compatible (types, overloads, etc).
-python_version = 3.8
 
 files =
   docker/,

poetry.lock

@@ -580,20 +580,20 @@ dev = ["Sphinx", "coverage", "flake8", "lxml", "lxml-stubs", "memory-profiler",
 
 [[package]]
 name = "furo"
-version = "2023.5.20"
+version = "2023.3.27"
 description = "A clean customisable Sphinx documentation theme."
 category = "dev"
 optional = false
 python-versions = ">=3.7"
 files = [
-    {file = "furo-2023.5.20-py3-none-any.whl", hash = "sha256:594a8436ddfe0c071f3a9e9a209c314a219d8341f3f1af33fdf7c69544fab9e6"},
-    {file = "furo-2023.5.20.tar.gz", hash = "sha256:40e09fa17c6f4b22419d122e933089226dcdb59747b5b6c79363089827dea16f"},
+    {file = "furo-2023.3.27-py3-none-any.whl", hash = "sha256:4ab2be254a2d5e52792d0ca793a12c35582dd09897228a6dd47885dabd5c9521"},
+    {file = "furo-2023.3.27.tar.gz", hash = "sha256:b99e7867a5cc833b2b34d7230631dd6558c7a29f93071fdbb5709634bb33c5a5"},
 ]
 
 [package.dependencies]
 beautifulsoup4 = "*"
 pygments = ">=2.7"
-sphinx = ">=6.0,<8.0"
+sphinx = ">=5.0,<7.0"
 sphinx-basic-ng = "*"
 
 [[package]]
@@ -1632,14 +1632,14 @@ files = [
 
 [[package]]
 name = "phonenumbers"
-version = "8.13.11"
+version = "8.13.7"
 description = "Python version of Google's common library for parsing, formatting, storing and validating international phone numbers."
 category = "main"
 optional = false
 python-versions = "*"
 files = [
-    {file = "phonenumbers-8.13.11-py2.py3-none-any.whl", hash = "sha256:107469114fd297258a485bdf8238d0522cb392db1257faf2bf23384ecbdb0e8a"},
-    {file = "phonenumbers-8.13.11.tar.gz", hash = "sha256:3e3274d88cab3609b55ff5b93417075dbca2d13064f103fbf562e0ea1dda0f9a"},
+    {file = "phonenumbers-8.13.7-py2.py3-none-any.whl", hash = "sha256:d3e3555b38c89b121f5b2e917847003bdd07027569d758d5f40156c01aeac089"},
+    {file = "phonenumbers-8.13.7.tar.gz", hash = "sha256:253bb0e01250d21a11f2b42b3e6e161b7f6cb2ac440e2e2a95c1da71d221ee1a"},
 ]
 
 [[package]]
@@ -1940,14 +1940,14 @@ email = ["email-validator (>=1.0.3)"]
 
 [[package]]
 name = "pygithub"
-version = "1.58.2"
+version = "1.58.1"
 description = "Use the full Github API v3"
 category = "dev"
 optional = false
 python-versions = ">=3.7"
 files = [
-    {file = "PyGithub-1.58.2-py3-none-any.whl", hash = "sha256:f435884af617c6debaa76cbc355372d1027445a56fbc39972a3b9ed4968badc8"},
-    {file = "PyGithub-1.58.2.tar.gz", hash = "sha256:1e6b1b7afe31f75151fb81f7ab6b984a7188a852bdb123dbb9ae90023c3ce60f"},
+    {file = "PyGithub-1.58.1-py3-none-any.whl", hash = "sha256:4e7fe9c3ec30d5fde5b4fbb97f18821c9dbf372bf6df337fe66f6689a65e0a83"},
+    {file = "PyGithub-1.58.1.tar.gz", hash = "sha256:7d528b4ad92bc13122129fafd444ce3d04c47d2d801f6446b6e6ee2d410235b3"},
 ]
 
 [package.dependencies]
@@ -2251,21 +2251,21 @@ md = ["cmarkgfm (>=0.8.0)"]
 
 [[package]]
 name = "requests"
-version = "2.31.0"
+version = "2.28.2"
 description = "Python HTTP for Humans."
 category = "main"
 optional = false
-python-versions = ">=3.7"
+python-versions = ">=3.7, <4"
 files = [
-    {file = "requests-2.31.0-py3-none-any.whl", hash = "sha256:58cd2187c01e70e6e26505bca751777aa9f2ee0b7f4300988b709f44e013003f"},
-    {file = "requests-2.31.0.tar.gz", hash = "sha256:942c5a758f98d790eaed1a29cb6eefc7ffb0d1cf7af05c3d2791656dbd6ad1e1"},
+    {file = "requests-2.28.2-py3-none-any.whl", hash = "sha256:64299f4909223da747622c030b781c0d7811e359c37124b4bd368fb8c6518baa"},
+    {file = "requests-2.28.2.tar.gz", hash = "sha256:98b1b2782e3c6c4904938b84c0eb932721069dfdb9134313beff7c83c2df24bf"},
 ]
 
 [package.dependencies]
 certifi = ">=2017.4.17"
 charset-normalizer = ">=2,<4"
 idna = ">=2.5,<4"
-urllib3 = ">=1.21.1,<3"
+urllib3 = ">=1.21.1,<1.27"
 
 [package.extras]
 socks = ["PySocks (>=1.5.6,!=1.5.7)"]
@@ -2565,21 +2565,21 @@ files = [
 
 [[package]]
 name = "sphinx"
-version = "6.2.1"
+version = "6.1.3"
 description = "Python documentation generator"
 category = "dev"
 optional = false
 python-versions = ">=3.8"
 files = [
-    {file = "Sphinx-6.2.1.tar.gz", hash = "sha256:6d56a34697bb749ffa0152feafc4b19836c755d90a7c59b72bc7dfd371b9cc6b"},
-    {file = "sphinx-6.2.1-py3-none-any.whl", hash = "sha256:97787ff1fa3256a3eef9eda523a63dbf299f7b47e053cfcf684a1c2a8380c912"},
+    {file = "Sphinx-6.1.3.tar.gz", hash = "sha256:0dac3b698538ffef41716cf97ba26c1c7788dba73ce6f150c1ff5b4720786dd2"},
+    {file = "sphinx-6.1.3-py3-none-any.whl", hash = "sha256:807d1cb3d6be87eb78a381c3e70ebd8d346b9a25f3753e9947e866b2786865fc"},
 ]
 
 [package.dependencies]
 alabaster = ">=0.7,<0.8"
 babel = ">=2.9"
 colorama = {version = ">=0.4.5", markers = "sys_platform == \"win32\""}
-docutils = ">=0.18.1,<0.20"
+docutils = ">=0.18,<0.20"
 imagesize = ">=1.3"
 importlib-metadata = {version = ">=4.8", markers = "python_version < \"3.10\""}
 Jinja2 = ">=3.0"
@@ -2597,7 +2597,7 @@ sphinxcontrib-serializinghtml = ">=1.1.5"
 [package.extras]
 docs = ["sphinxcontrib-websupport"]
 lint = ["docutils-stubs", "flake8 (>=3.5.0)", "flake8-simplify", "isort", "mypy (>=0.990)", "ruff", "sphinx-lint", "types-requests"]
-test = ["cython", "filelock", "html5lib", "pytest (>=4.6)"]
+test = ["cython", "html5lib", "pytest (>=4.6)"]
 
 [[package]]
 name = "sphinx-autodoc2"
@@ -3010,14 +3010,14 @@ files = [
 
 [[package]]
 name = "types-commonmark"
-version = "0.9.2.3"
+version = "0.9.2.2"
 description = "Typing stubs for commonmark"
 category = "dev"
 optional = false
 python-versions = "*"
 files = [
-    {file = "types-commonmark-0.9.2.3.tar.gz", hash = "sha256:42769a2c194fd5b49fd9eedfd4a83cd1d2514c6d0a36f00f5c5ffe0b6a2d2fcf"},
-    {file = "types_commonmark-0.9.2.3-py3-none-any.whl", hash = "sha256:b575156e1b8a292d43acb36f861110b85c4bc7aa53bbfb5ac64addec15d18cfa"},
+    {file = "types-commonmark-0.9.2.2.tar.gz", hash = "sha256:f3259350634c2ce68ae503398430482f7cf44e5cae3d344995e916fbf453b4be"},
+    {file = "types_commonmark-0.9.2.2-py3-none-any.whl", hash = "sha256:d3d878692615e7fbe47bf19ba67497837b135812d665012a3d42219c1f2c3a61"},
 ]
 
 [[package]]
@@ -3058,26 +3058,26 @@ files = [
 
 [[package]]
 name = "types-pillow"
-version = "9.5.0.4"
+version = "9.5.0.2"
 description = "Typing stubs for Pillow"
 category = "dev"
 optional = false
 python-versions = "*"
 files = [
-    {file = "types-Pillow-9.5.0.4.tar.gz", hash = "sha256:f1b6af47abd151847ee25911ffeba784899bc7dc7f9eba8ca6a5aac522b012ef"},
-    {file = "types_Pillow-9.5.0.4-py3-none-any.whl", hash = "sha256:69427d9fa4320ff6e30f00fb9c0dd71185dc0a16de4757774220104759483466"},
+    {file = "types-Pillow-9.5.0.2.tar.gz", hash = "sha256:b3f9f621f259566c19c1deca21901017c8b1e3e200ed2e49e0a2d83c0a5175db"},
+    {file = "types_Pillow-9.5.0.2-py3-none-any.whl", hash = "sha256:58fdebd0ffa2353ecccdd622adde23bce89da5c0c8b96c34f2d1eca7b7e42d0e"},
 ]
 
 [[package]]
 name = "types-psycopg2"
-version = "2.9.21.10"
+version = "2.9.21.9"
 description = "Typing stubs for psycopg2"
 category = "dev"
 optional = false
 python-versions = "*"
 files = [
-    {file = "types-psycopg2-2.9.21.10.tar.gz", hash = "sha256:c2600892312ae1c34e12f145749795d93dc4eac3ef7dbf8a9c1bfd45385e80d7"},
-    {file = "types_psycopg2-2.9.21.10-py3-none-any.whl", hash = "sha256:918224a0731a3650832e46633e720703b5beef7693a064e777d9748654fcf5e5"},
+    {file = "types-psycopg2-2.9.21.9.tar.gz", hash = "sha256:388dc36a04551632289c4aaf1fc5b91e147654b165db896d094844e216f22bf5"},
+    {file = "types_psycopg2-2.9.21.9-py3-none-any.whl", hash = "sha256:0332525fb9d3031d3da46f091e7d40b2c4d4958e9c00d2b4c1eaaa9f8ef9de4e"},
 ]
 
 [[package]]
@@ -3124,14 +3124,14 @@ types-urllib3 = "*"
 
 [[package]]
 name = "types-setuptools"
-version = "67.8.0.0"
+version = "67.7.0.1"
 description = "Typing stubs for setuptools"
 category = "dev"
 optional = false
 python-versions = "*"
 files = [
-    {file = "types-setuptools-67.8.0.0.tar.gz", hash = "sha256:95c9ed61871d6c0e258433373a4e1753c0a7c3627a46f4d4058c7b5a08ab844f"},
-    {file = "types_setuptools-67.8.0.0-py3-none-any.whl", hash = "sha256:6df73340d96b238a4188b7b7668814b37e8018168aef1eef94a3b1872e3f60ff"},
+    {file = "types-setuptools-67.7.0.1.tar.gz", hash = "sha256:980a2651b2b019809817e1585071596b87fbafcb54433ff3b12445461db23790"},
+    {file = "types_setuptools-67.7.0.1-py3-none-any.whl", hash = "sha256:471a4ecf6984ffada63ffcfa884bfcb62718bd2d1a1acf8ee5513ec99789ed5e"},
 ]
 
 [[package]]

pyproject.toml

@@ -89,7 +89,7 @@ manifest-path = "rust/Cargo.toml"
 
 [tool.poetry]
 name = "matrix-synapse"
-version = "1.84.0"
+version = "1.83.0"
 description = "Homeserver for the Matrix decentralised comms protocol"
 authors = ["Matrix.org Team and Contributors <packages@matrix.org>"]
 license = "Apache-2.0"
@@ -368,7 +368,7 @@ furo = ">=2022.12.7,<2024.0.0"
 # system changes.
 # We are happy to raise these upper bounds upon request,
 # provided we check that it's safe to do so (i.e. that CI passes).
-requires = ["poetry-core>=1.1.0,<=1.6.0", "setuptools_rust>=1.3,<=1.6.0"]
+requires = ["poetry-core>=1.0.0,<=1.5.0", "setuptools_rust>=1.3,<=1.5.2"]
 build-backend = "poetry.core.masonry.api"
 
 

scripts-dev/mypy_synapse_plugin.py

@@ -18,11 +18,10 @@ can crop up, e.g the cache descriptors.
 
 from typing import Callable, Optional, Type
 
-from mypy.erasetype import remove_instance_last_known_values
 from mypy.nodes import ARG_NAMED_OPT
 from mypy.plugin import MethodSigContext, Plugin
 from mypy.typeops import bind_self
-from mypy.types import CallableType, Instance, NoneType, UnionType
+from mypy.types import CallableType, NoneType, UnionType
 
 
 class SynapsePlugin(Plugin):
@@ -93,41 +92,10 @@ def cached_function_method_signature(ctx: MethodSigContext) -> CallableType:
     arg_names.append("on_invalidate")
     arg_kinds.append(ARG_NAMED_OPT)  # Arg is an optional kwarg.
 
-    # Finally we ensure the return type is a Deferred.
-    if (
-        isinstance(signature.ret_type, Instance)
-        and signature.ret_type.type.fullname == "twisted.internet.defer.Deferred"
-    ):
-        # If it is already a Deferred, nothing to do.
-        ret_type = signature.ret_type
-    else:
-        ret_arg = None
-        if isinstance(signature.ret_type, Instance):
-            # If a coroutine, wrap the coroutine's return type in a Deferred.
-            if signature.ret_type.type.fullname == "typing.Coroutine":
-                ret_arg = signature.ret_type.args[2]
-
-            # If an awaitable, wrap the awaitable's final value in a Deferred.
-            elif signature.ret_type.type.fullname == "typing.Awaitable":
-                ret_arg = signature.ret_type.args[0]
-
-        # Otherwise, wrap the return value in a Deferred.
-        if ret_arg is None:
-            ret_arg = signature.ret_type
-
-        # This should be able to use ctx.api.named_generic_type, but that doesn't seem
-        # to find the correct symbol for anything more than 1 module deep.
-        #
-        # modules is not part of CheckerPluginInterface. The following is a combination
-        # of TypeChecker.named_generic_type and TypeChecker.lookup_typeinfo.
-        sym = ctx.api.modules["twisted.internet.defer"].names.get("Deferred")  # type: ignore[attr-defined]
-        ret_type = Instance(sym.node, [remove_instance_last_known_values(ret_arg)])
-
     signature = signature.copy_modified(
         arg_types=arg_types,
         arg_names=arg_names,
         arg_kinds=arg_kinds,
-        ret_type=ret_type,
     )
 
     return signature

synapse/api/filtering.py

@@ -128,7 +128,20 @@ USER_FILTER_SCHEMA = {
         "account_data": {"$ref": "#/definitions/filter"},
         "room": {"$ref": "#/definitions/room_filter"},
         "event_format": {"type": "string", "enum": ["client", "federation"]},
-        "event_fields": {"type": "array", "items": {"type": "string"}},
+        "event_fields": {
+            "type": "array",
+            "items": {
+                "type": "string",
+                # Don't allow '\\' in event field filters. This makes matching
+                # events a lot easier as we can then use a negative lookbehind
+                # assertion to split '\.' If we allowed \\ then it would
+                # incorrectly split '\\.' See synapse.events.utils.serialize_event
+                #
+                # Note that because this is a regular expression, we have to escape
+                # each backslash in the pattern.
+                "pattern": r"^((?!\\\\).)*$",
+            },
+        },
     },
     "additionalProperties": True,  # Allow new fields for forward compatibility
 }

synapse/api/room_versions.py

@@ -96,15 +96,11 @@ class RoomVersion:
     msc2716_historical: bool
     # MSC2716: Adds support for redacting "insertion", "chunk", and "marker" events
     msc2716_redactions: bool
-    # MSC3389: Protect relation information from redaction.
-    msc3389_relation_redactions: bool
     # MSC3787: Adds support for a `knock_restricted` join rule, mixing concepts of
     # knocks and restricted join rules into the same join condition.
     msc3787_knock_restricted_join_rule: bool
     # MSC3667: Enforce integer power levels
     msc3667_int_only_power_levels: bool
-    # MSC3821: Do not redact the third_party_invite content field for membership events.
-    msc3821_redaction_rules: bool
     # MSC3931: Adds a push rule condition for "room version feature flags", making
     # some push rules room version dependent. Note that adding a flag to this list
     # is not enough to mark it "supported": the push rule evaluator also needs to
@@ -132,10 +128,8 @@ class RoomVersions:
         msc2403_knocking=False,
         msc2716_historical=False,
         msc2716_redactions=False,
-        msc3389_relation_redactions=False,
         msc3787_knock_restricted_join_rule=False,
         msc3667_int_only_power_levels=False,
-        msc3821_redaction_rules=False,
         msc3931_push_features=(),
         msc3989_redaction_rules=False,
     )
@@ -155,10 +149,8 @@ class RoomVersions:
         msc2403_knocking=False,
         msc2716_historical=False,
         msc2716_redactions=False,
-        msc3389_relation_redactions=False,
         msc3787_knock_restricted_join_rule=False,
         msc3667_int_only_power_levels=False,
-        msc3821_redaction_rules=False,
         msc3931_push_features=(),
         msc3989_redaction_rules=False,
     )
@@ -178,10 +170,8 @@ class RoomVersions:
         msc2403_knocking=False,
         msc2716_historical=False,
         msc2716_redactions=False,
-        msc3389_relation_redactions=False,
         msc3787_knock_restricted_join_rule=False,
         msc3667_int_only_power_levels=False,
-        msc3821_redaction_rules=False,
         msc3931_push_features=(),
         msc3989_redaction_rules=False,
     )
@@ -201,10 +191,8 @@ class RoomVersions:
         msc2403_knocking=False,
         msc2716_historical=False,
         msc2716_redactions=False,
-        msc3389_relation_redactions=False,
         msc3787_knock_restricted_join_rule=False,
         msc3667_int_only_power_levels=False,
-        msc3821_redaction_rules=False,
         msc3931_push_features=(),
         msc3989_redaction_rules=False,
     )
@@ -224,10 +212,8 @@ class RoomVersions:
         msc2403_knocking=False,
         msc2716_historical=False,
         msc2716_redactions=False,
-        msc3389_relation_redactions=False,
         msc3787_knock_restricted_join_rule=False,
         msc3667_int_only_power_levels=False,
-        msc3821_redaction_rules=False,
         msc3931_push_features=(),
         msc3989_redaction_rules=False,
     )
@@ -247,10 +233,8 @@ class RoomVersions:
         msc2403_knocking=False,
         msc2716_historical=False,
         msc2716_redactions=False,
-        msc3389_relation_redactions=False,
         msc3787_knock_restricted_join_rule=False,
         msc3667_int_only_power_levels=False,
-        msc3821_redaction_rules=False,
         msc3931_push_features=(),
         msc3989_redaction_rules=False,
     )
@@ -270,10 +254,8 @@ class RoomVersions:
         msc2403_knocking=False,
         msc2716_historical=False,
         msc2716_redactions=False,
-        msc3389_relation_redactions=False,
         msc3787_knock_restricted_join_rule=False,
         msc3667_int_only_power_levels=False,
-        msc3821_redaction_rules=False,
         msc3931_push_features=(),
         msc3989_redaction_rules=False,
     )
@@ -293,10 +275,8 @@ class RoomVersions:
         msc2403_knocking=True,
         msc2716_historical=False,
         msc2716_redactions=False,
-        msc3389_relation_redactions=False,
         msc3787_knock_restricted_join_rule=False,
         msc3667_int_only_power_levels=False,
-        msc3821_redaction_rules=False,
         msc3931_push_features=(),
         msc3989_redaction_rules=False,
     )
@@ -316,10 +296,8 @@ class RoomVersions:
         msc2403_knocking=True,
         msc2716_historical=False,
         msc2716_redactions=False,
-        msc3389_relation_redactions=False,
         msc3787_knock_restricted_join_rule=False,
         msc3667_int_only_power_levels=False,
-        msc3821_redaction_rules=False,
         msc3931_push_features=(),
         msc3989_redaction_rules=False,
     )
@@ -339,10 +317,8 @@ class RoomVersions:
         msc2403_knocking=True,
         msc2716_historical=False,
         msc2716_redactions=False,
-        msc3389_relation_redactions=False,
         msc3787_knock_restricted_join_rule=False,
         msc3667_int_only_power_levels=False,
-        msc3821_redaction_rules=False,
         msc3931_push_features=(),
         msc3989_redaction_rules=False,
     )
@@ -362,33 +338,8 @@ class RoomVersions:
         msc2403_knocking=True,
         msc2716_historical=False,
         msc2716_redactions=False,
-        msc3389_relation_redactions=False,
         msc3787_knock_restricted_join_rule=True,
         msc3667_int_only_power_levels=False,
-        msc3821_redaction_rules=False,
-        msc3931_push_features=(),
-        msc3989_redaction_rules=False,
-    )
-    MSC3821 = RoomVersion(
-        "org.matrix.msc3821.opt1",
-        RoomDisposition.UNSTABLE,
-        EventFormatVersions.ROOM_V4_PLUS,
-        StateResolutionVersions.V2,
-        enforce_key_validity=True,
-        special_case_aliases_auth=False,
-        strict_canonicaljson=True,
-        limit_notifications_power_levels=True,
-        msc2175_implicit_room_creator=False,
-        msc2176_redaction_rules=False,
-        msc3083_join_rules=True,
-        msc3375_redaction_rules=True,
-        msc2403_knocking=True,
-        msc2716_historical=False,
-        msc2716_redactions=False,
-        msc3389_relation_redactions=False,
-        msc3787_knock_restricted_join_rule=False,
-        msc3667_int_only_power_levels=False,
-        msc3821_redaction_rules=True,
         msc3931_push_features=(),
         msc3989_redaction_rules=False,
     )
@@ -408,10 +359,8 @@ class RoomVersions:
         msc2403_knocking=True,
         msc2716_historical=False,
         msc2716_redactions=False,
-        msc3389_relation_redactions=False,
         msc3787_knock_restricted_join_rule=True,
         msc3667_int_only_power_levels=True,
-        msc3821_redaction_rules=False,
         msc3931_push_features=(),
         msc3989_redaction_rules=False,
     )
@@ -431,10 +380,8 @@ class RoomVersions:
         msc2403_knocking=True,
         msc2716_historical=True,
         msc2716_redactions=True,
-        msc3389_relation_redactions=False,
         msc3787_knock_restricted_join_rule=False,
         msc3667_int_only_power_levels=False,
-        msc3821_redaction_rules=False,
         msc3931_push_features=(),
         msc3989_redaction_rules=False,
     )
@@ -455,10 +402,8 @@ class RoomVersions:
         msc2403_knocking=True,
         msc2716_historical=False,
         msc2716_redactions=False,
-        msc3389_relation_redactions=False,
         msc3787_knock_restricted_join_rule=True,
         msc3667_int_only_power_levels=True,
-        msc3821_redaction_rules=False,
         msc3931_push_features=(PushRuleRoomFlag.EXTENSIBLE_EVENTS,),
         msc3989_redaction_rules=False,
     )
@@ -478,10 +423,8 @@ class RoomVersions:
         msc2403_knocking=True,
         msc2716_historical=False,
         msc2716_redactions=False,
-        msc3389_relation_redactions=False,
         msc3787_knock_restricted_join_rule=True,
         msc3667_int_only_power_levels=True,
-        msc3821_redaction_rules=False,
         msc3931_push_features=(),
         msc3989_redaction_rules=True,
     )

synapse/app/_base.py

@@ -21,7 +21,6 @@ import socket
 import sys
 import traceback
 import warnings
-from textwrap import indent
 from typing import (
     TYPE_CHECKING,
     Any,
@@ -213,12 +212,8 @@ def handle_startup_exception(e: Exception) -> NoReturn:
     # Exceptions that occur between setting up the logging and forking or starting
     # the reactor are written to the logs, followed by a summary to stderr.
     logger.exception("Exception during startup")
-
-    error_string = "".join(traceback.format_exception(type(e), e, e.__traceback__))
-    indented_error_string = indent(error_string, "    ")
-
     quit_with_error(
-        f"Error during initialisation:\n{indented_error_string}\nThere may be more information in the logs."
+        f"Error during initialisation:\n   {e}\nThere may be more information in the logs."
     )
 
 
@@ -386,6 +381,7 @@ def listen_unix(
 
 
 def listen_http(
+    hs: "HomeServer",
     listener_config: ListenerConfig,
     root_resource: Resource,
     version_string: str,
@@ -406,6 +402,7 @@ def listen_http(
         version_string,
         max_request_body_size=max_request_body_size,
         reactor=reactor,
+        federation_agent=hs.get_federation_http_client().agent,
     )
 
     if isinstance(listener_config, TCPListenerConfig):

synapse/app/admin_cmd.py

@@ -64,7 +64,7 @@ from synapse.util.logcontext import LoggingContext
 logger = logging.getLogger("synapse.app.admin_cmd")
 
 
-class AdminCmdStore(
+class AdminCmdSlavedStore(
     FilteringWorkerStore,
     ClientIpWorkerStore,
     DeviceWorkerStore,
@@ -103,7 +103,7 @@ class AdminCmdStore(
 
 
 class AdminCmdServer(HomeServer):
-    DATASTORE_CLASS = AdminCmdStore  # type: ignore
+    DATASTORE_CLASS = AdminCmdSlavedStore  # type: ignore
 
 
 async def export_data_command(hs: HomeServer, args: argparse.Namespace) -> None:

synapse/app/generic_worker.py

@@ -102,7 +102,7 @@ from synapse.util.httpresourcetree import create_resource_tree
 logger = logging.getLogger("synapse.app.generic_worker")
 
 
-class GenericWorkerStore(
+class GenericWorkerSlavedStore(
     # FIXME(#3714): We need to add UserDirectoryStore as we write directly
     # rather than going via the correct worker.
     UserDirectoryStore,
@@ -154,7 +154,7 @@ class GenericWorkerStore(
 
 
 class GenericWorkerServer(HomeServer):
-    DATASTORE_CLASS = GenericWorkerStore  # type: ignore
+    DATASTORE_CLASS = GenericWorkerSlavedStore  # type: ignore
 
     def _listen_http(self, listener_config: ListenerConfig) -> None:
         assert listener_config.http_options is not None
@@ -223,6 +223,7 @@ class GenericWorkerServer(HomeServer):
         root_resource = create_resource_tree(resources, OptionsResource())
 
         _base.listen_http(
+            self,
             listener_config,
             root_resource,
             self.version_string,

synapse/app/homeserver.py

@@ -139,6 +139,7 @@ class SynapseHomeServer(HomeServer):
             root_resource = OptionsResource()
 
         ports = listen_http(
+            self,
             listener_config,
             create_resource_tree(resources, root_resource),
             self.version_string,

synapse/app/phone_stats_home.py

@@ -127,6 +127,10 @@ async def phone_stats_home(
     daily_sent_messages = await store.count_daily_sent_messages()
     stats["daily_sent_messages"] = daily_sent_messages
 
+    r30_results = await store.count_r30_users()
+    for name, count in r30_results.items():
+        stats["r30_users_" + name] = count
+
     r30v2_results = await store.count_r30v2_users()
     for name, count in r30v2_results.items():
         stats["r30v2_users_" + name] = count