properly puncuate newsfragment

.ci/scripts/calculate_jobs.py

@@ -29,12 +29,11 @@ IS_PR = os.environ["GITHUB_REF"].startswith("refs/pull/")
 
 # First calculate the various trial jobs.
 #
-# For PRs, we only run each type of test with the oldest Python version supported (which
-# is Python 3.8 right now)
+# For each type of test we only run on Py3.7 on PRs
 
 trial_sqlite_tests = [
     {
-        "python-version": "3.8",
+        "python-version": "3.7",
         "database": "sqlite",
         "extras": "all",
     }
@@ -47,13 +46,13 @@ if not IS_PR:
             "database": "sqlite",
             "extras": "all",
         }
-        for version in ("3.9", "3.10", "3.11")
+        for version in ("3.8", "3.9", "3.10", "3.11")
     )
 
 
 trial_postgres_tests = [
     {
-        "python-version": "3.8",
+        "python-version": "3.7",
         "database": "postgres",
         "postgres-version": "11",
         "extras": "all",
@@ -72,7 +71,7 @@ if not IS_PR:
 
 trial_no_extra_tests = [
     {
-        "python-version": "3.8",
+        "python-version": "3.7",
         "database": "sqlite",
         "extras": "",
     }
@@ -134,6 +133,11 @@ if not IS_PR:
                 "sytest-tag": "testing",
                 "postgres": "postgres",
             },
+            {
+                "sytest-tag": "buster",
+                "postgres": "multi-postgres",
+                "workers": "workers",
+            },
         ]
     )
 

.github/workflows/docker.yml

@@ -29,16 +29,6 @@ jobs:
       - name: Inspect builder
         run: docker buildx inspect
 
-      - name: Checkout repository
-        uses: actions/checkout@v3
-
-      - name: Extract version from pyproject.toml
-        # Note: explicitly requesting bash will mean bash is invoked with `-eo pipefail`, see
-        # https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsshell
-        shell: bash
-        run: |
-          echo "SYNAPSE_VERSION=$(grep "^version" pyproject.toml | sed -E 's/version\s*=\s*["]([^"]*)["]/\1/')" >> $GITHUB_ENV
-
       - name: Log in to DockerHub
         uses: docker/login-action@v2
         with:
@@ -71,9 +61,7 @@ jobs:
         uses: docker/build-push-action@v4
         with:
           push: true
-          labels: |
-            gitsha1=${{ github.sha }}
-            org.opencontainers.image.version=${{ env.SYNAPSE_VERSION }}
+          labels: "gitsha1=${{ github.sha }}"
           tags: "${{ steps.set-tag.outputs.tags }}"
           file: "docker/Dockerfile"
           platforms: linux/amd64,linux/arm64

.github/workflows/release-artifacts.yml

@@ -144,7 +144,7 @@ jobs:
 
       - name: Only build a single wheel on PR
         if: startsWith(github.ref, 'refs/pull/')
-        run: echo "CIBW_BUILD="cp38-manylinux_${{ matrix.arch }}"" >> $GITHUB_ENV
+        run: echo "CIBW_BUILD="cp37-manylinux_${{ matrix.arch }}"" >> $GITHUB_ENV
 
       - name: Build wheels
         run: python -m cibuildwheel --output-dir wheelhouse

.github/workflows/tests.yml

@@ -320,7 +320,7 @@ jobs:
 
       - uses: actions/setup-python@v4
         with:
-          python-version: '3.8'
+          python-version: '3.7'
 
       - name: Prepare old deps
         if: steps.cache-poetry-old-deps.outputs.cache-hit != 'true'
@@ -362,7 +362,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: ["pypy-3.8"]
+        python-version: ["pypy-3.7"]
         extras: ["all"]
 
     steps:
@@ -477,7 +477,7 @@ jobs:
     strategy:
       matrix:
         include:
-          - python-version: "3.8"
+          - python-version: "3.7"
             postgres-version: "11"
 
           - python-version: "3.11"

.github/workflows/twisted_trunk.yml

@@ -5,9 +5,6 @@ on:
     - cron: 0 8 * * *
 
   workflow_dispatch:
-    # NB: inputs are only present when this workflow is dispatched manually.
-    # (The default below is the default field value in the form to trigger
-    # a manual dispatch). Otherwise the inputs will evaluate to null.
     inputs:
       twisted_ref:
         description: Commit, branch or tag to checkout from upstream Twisted.
@@ -52,7 +49,7 @@ jobs:
           extras: "all"
       - run: |
           poetry remove twisted
-          poetry add --extras tls git+https://github.com/twisted/twisted.git#${{ inputs.twisted_ref || 'trunk' }}
+          poetry add --extras tls git+https://github.com/twisted/twisted.git#${{ inputs.twisted_ref }}
           poetry install --no-interaction --extras "all test"
       - name: Remove warn_unused_ignores from mypy config
         run: sed '/warn_unused_ignores = True/d' -i mypy.ini
@@ -99,11 +96,7 @@ jobs:
     if: needs.check_repo.outputs.should_run_workflow == 'true'
     runs-on: ubuntu-latest
     container:
-      # We're using ubuntu:focal because it uses Python 3.8 which is our minimum supported Python version.
-      # This job is a canary to warn us about unreleased twisted changes that would cause problems for us if
-      # they were to be released immediately. For simplicity's sake (and to save CI runners) we use the oldest
-      # version, assuming that any incompatibilities on newer versions would also be present on the oldest.
-      image: matrixdotorg/sytest-synapse:focal
+      image: matrixdotorg/sytest-synapse:buster
       volumes:
         - ${{ github.workspace }}:/src
 

.gitignore

@@ -34,7 +34,6 @@ __pycache__/
 /logs
 /media_store/
 /uploads
-/homeserver-config-overrides.d
 
 # For direnv users
 /.envrc

Cargo.lock

@@ -13,9 +13,9 @@ dependencies = [
 
 [[package]]
 name = "anyhow"
-version = "1.0.72"
+version = "1.0.71"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3b13c32d80ecc7ab747b80c3784bce54ee8a7a0cc4fbda9bf4cda2cf6fe90854"
+checksum = "9c7d0618f0e0b7e8ff11427422b64564d5fb0be1940354bfe2e0529b18a9d9b8"
 
 [[package]]
 name = "arc-swap"
@@ -132,9 +132,9 @@ dependencies = [
 
 [[package]]
 name = "log"
-version = "0.4.20"
+version = "0.4.19"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b5e6163cb8c49088c2c36f57875e58ccd8c87c7427f7fbd50ea6710b2f3f2e8f"
+checksum = "b06a4cde4c0f271a446782e3eff8de789548ce57dbc8eca9292c27f4a42004b4"
 
 [[package]]
 name = "memchr"
@@ -182,9 +182,9 @@ dependencies = [
 
 [[package]]
 name = "proc-macro2"
-version = "1.0.64"
+version = "1.0.52"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "78803b62cbf1f46fde80d7c0e803111524b9877184cfe7c3033659490ac7a7da"
+checksum = "1d0e1ae9e836cc3beddd63db0df682593d7e2d3d891ae8c9083d2113e1744224"
 dependencies = [
  "unicode-ident",
 ]
@@ -229,9 +229,9 @@ dependencies = [
 
 [[package]]
 name = "pyo3-log"
-version = "0.8.3"
+version = "0.8.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f47b0777feb17f61eea78667d61103758b243a871edc09a7786500a50467b605"
+checksum = "c94ff6535a6bae58d7d0b85e60d4c53f7f84d0d0aa35d6a28c3f3e70bfe51444"
 dependencies = [
  "arc-swap",
  "log",
@@ -273,9 +273,9 @@ dependencies = [
 
 [[package]]
 name = "quote"
-version = "1.0.29"
+version = "1.0.26"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "573015e8ab27661678357f27dc26460738fd2b6c86e46f386fde94cb5d913105"
+checksum = "4424af4bf778aae2051a77b60283332f386554255d722233d09fbfc7e30da2fc"
 dependencies = [
  "proc-macro2",
 ]
@@ -291,21 +291,9 @@ dependencies = [
 
 [[package]]
 name = "regex"
-version = "1.9.3"
+version = "1.8.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "81bc1d4caf89fac26a70747fe603c130093b53c773888797a6329091246d651a"
-dependencies = [
- "aho-corasick",
- "memchr",
- "regex-automata",
- "regex-syntax",
-]
-
-[[package]]
-name = "regex-automata"
-version = "0.3.6"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fed1ceff11a1dddaee50c9dc8e4938bd106e9d89ae372f192311e7da498e3b69"
+checksum = "d0ab3ca65655bb1e41f2a8c8cd662eb4fb035e67c3f78da1d61dffe89d07300f"
 dependencies = [
  "aho-corasick",
  "memchr",
@@ -314,9 +302,9 @@ dependencies = [
 
 [[package]]
 name = "regex-syntax"
-version = "0.7.4"
+version = "0.7.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e5ea92a5b6195c6ef2a0295ea818b312502c6fc94dde986c5553242e18fd4ce2"
+checksum = "436b050e76ed2903236f032a59761c1eb99e1b0aead2c257922771dab1fc8c78"
 
 [[package]]
 name = "ryu"
@@ -332,29 +320,29 @@ checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd"
 
 [[package]]
 name = "serde"
-version = "1.0.184"
+version = "1.0.164"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2c911f4b04d7385c9035407a4eff5903bf4fe270fa046fda448b69e797f4fff0"
+checksum = "9e8c8cf938e98f769bc164923b06dce91cea1751522f46f8466461af04c9027d"
 dependencies = [
  "serde_derive",
 ]
 
 [[package]]
 name = "serde_derive"
-version = "1.0.184"
+version = "1.0.164"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c1df27f5b29406ada06609b2e2f77fb34f6dbb104a457a671cc31dbed237e09e"
+checksum = "d9735b638ccc51c28bf6914d90a2e9725b377144fc612c49a611fddd1b631d68"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.28",
+ "syn 2.0.10",
 ]
 
 [[package]]
 name = "serde_json"
-version = "1.0.104"
+version = "1.0.99"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "076066c5f1078eac5b722a31827a8832fe108bed65dfa75e233c89f8206e976c"
+checksum = "46266871c240a00b8f503b877622fe33430b3c7d963bdc0f2adc511e54a1eae3"
 dependencies = [
  "itoa",
  "ryu",
@@ -386,9 +374,9 @@ dependencies = [
 
 [[package]]
 name = "syn"
-version = "2.0.28"
+version = "2.0.10"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "04361975b3f5e348b2189d8dc55bc942f278b2d482a6a0365de5bdd62d351567"
+checksum = "5aad1363ed6d37b84299588d62d3a7d95b5a5c2d9aad5c85609fda12afaa1f40"
 dependencies = [
  "proc-macro2",
  "quote",

Cargo.toml

@@ -3,4 +3,3 @@
 
 [workspace]
 members = ["rust"]
-resolver = "2"

changelog.d/15233.misc

@@ -0,0 +1 @@
+Replace `EventContext` fields `prev_group` and `delta_ids` with field `state_group_deltas`.

changelog.d/15680.bugfix

@@ -0,0 +1 @@
+Fix a long-standing bug where media files were served in an unsafe manner. Contributed by @joshqou.

changelog.d/15737.feature

@@ -0,0 +1 @@
+Improve `/messages` response time by avoiding backfill when we already have messages to return.

changelog.d/15743.misc

@@ -0,0 +1 @@
+Regularly try to send transactions to other servers after they failed instead of waiting for a new event to be available before trying.

changelog.d/15748.removal

@@ -0,0 +1 @@
+Remove experimental [MSC2716](https://github.com/matrix-org/matrix-spec-proposals/pull/2716) implementation to incrementally import history into existing rooms.

changelog.d/15755.misc

@@ -0,0 +1 @@
+Fix requesting multiple keys at once over federation, related to [MSC3983](https://github.com/matrix-org/matrix-spec-proposals/pull/3983).

changelog.d/15758.bugfix

@@ -0,0 +1 @@
+Avoid invalidating a cache that was just prefilled.

changelog.d/15770.bugfix

@@ -0,0 +1 @@
+Fix requesting multiple keys at once over federation, related to [MSC3983](https://github.com/matrix-org/matrix-spec-proposals/pull/3983).

changelog.d/15772.doc

@@ -0,0 +1 @@
+Document `looping_call()` functionality that will wait for the given function to finish before scheduling another.

changelog.d/15776.bugfix

@@ -0,0 +1 @@
+Fix joining rooms through aliases where the alias server isn't a real homeserver. Contributed by @tulir @ Beeper.

changelog.d/15781.bugfix

@@ -0,0 +1 @@
+Fix a bug in push rules handling leading to an invalid (per spec) `is_user_mention` rule sent to clients. Also fix wrong rule names for `is_user_mention` and `is_room_mention`.

changelog.d/15783.misc

@@ -0,0 +1 @@
+Allow for the configuration of max request retries and min/max retry delays in the matrix federation client.

changelog.d/15788.bugfix

@@ -0,0 +1 @@
+Fix a bug introduced in 1.57.0 where the wrong table would be locked on updating database rows when using SQLite as the database backend.

changelog.d/15804.bugfix

@@ -0,0 +1 @@
+Fix Sytest environmental variable evaluation in CI.

changelog.d/15805.doc

@@ -0,0 +1 @@
+Fix a typo in the [Admin API](https://matrix-org.github.io/synapse/latest/usage/administration/admin_api/index.html).

changelog.d/15806.misc

@@ -0,0 +1 @@
+Switch from `matrix://` to `matrix-federation://` scheme for internal Synapse routing of outbound federation traffic.

changelog.d/15812.doc

@@ -0,0 +1 @@
+Fix typo in MSC number in faster remote room join architecture doc.

changelog.d/15814.misc

@@ -0,0 +1 @@
+Fix harmless exceptions being printed when running the port DB script.

changelog.d/15815.bugfix

@@ -0,0 +1 @@
+Fix forgotten rooms missing from initial sync after rejoining them. Contributed by Nico from Famedly.

changelog.d/15817.bugfix

@@ -0,0 +1 @@
+Fix sqlite `user_filters` upgrade introduced in v1.86.0.

changelog.d/15838.feature

@@ -0,0 +1 @@
+Add spam checker module API for logins.

changelog.d/15849.misc

@@ -0,0 +1 @@
+Add check constraint to current_state_delta_stream (#15849).

changelog.d/15870.feature

@@ -1 +0,0 @@
-Implements an admin API to lock an user without deactivating them. Based on [MSC3939](https://github.com/matrix-org/matrix-spec-proposals/pull/3939).

changelog.d/15891.feature

@@ -1 +0,0 @@
-Implements a task scheduler for	resumable potentially long running tasks.

changelog.d/16010.misc

@@ -1 +0,0 @@
-Update dehydrated devices implementation.

changelog.d/16030.feature

@@ -1 +0,0 @@
-Allow specifying `client_secret_path` as alternative to `client_secret` for OIDC providers. This avoids leaking the client secret in the homeserver config. Contributed by @Ma27.

changelog.d/16052.bugfix

@@ -1 +0,0 @@
-Fix long-standing bug where concurrent requests to change a user's push rules could cause a deadlock. Contributed by Nick @ Beeper (@fizzadar).

changelog.d/16061.misc

@@ -1 +0,0 @@
-Fix database performance of read/write worker locks.

changelog.d/16080.bugfix

@@ -1 +0,0 @@
-Fix a long-standing bu in `/sync` where timeout=0 does not skip caching, resulting in slow calls in cases where there are no new changes. Contributed by @PlasmaIntec.

changelog.d/16085.misc

@@ -1 +0,0 @@
-Override global statement timeout when creating indexes in Postgres.

changelog.d/16089.misc

@@ -1 +0,0 @@
-Fix the type annotation on `run_db_interaction` in the Module API.

changelog.d/16091.doc

@@ -1 +0,0 @@
-Structured logging docs: add a link to explain the ELK stack

changelog.d/16092.misc

@@ -1 +0,0 @@
-Clean-up the presence code.

changelog.d/16094.feature

@@ -1 +0,0 @@
-Allow customising the IdP display name, icon, and brand for SAML and CAS providers (in addition to OIDC provider).

changelog.d/16110.misc

@@ -1 +0,0 @@
-Run `pyupgrade` for Python 3.8+.

changelog.d/16112.misc

@@ -1 +0,0 @@
-Rename pagination and purge locks and add comments to explain why they exist and how they work.

changelog.d/16114.feature

@@ -1 +0,0 @@
-Add an `admins` query parameter to the [List Accounts](https://matrix-org.github.io/synapse/v1.91/admin_api/user_admin_api.html#list-accounts) [admin API](https://matrix-org.github.io/synapse/v1.91/usage/administration/admin_api/index.html), to include only admins or to exclude admins in user queries.

changelog.d/16115.misc

@@ -1 +0,0 @@
-Attempt to fix the twisted trunk job.

changelog.d/16116.bugfix

@@ -1 +0,0 @@
-Fix performance of state resolutions for large, old rooms that did not have the full auth chain persisted.

changelog.d/16117.misc

@@ -1 +0,0 @@
-Cache token introspection response from OIDC provider.

changelog.d/16123.misc

@@ -1 +0,0 @@
-Add cache to `get_server_keys_json_for_remote`.

changelog.d/16124.bugfix

@@ -1 +0,0 @@
-Filter out user agent references to the sliding sync proxy and rust-sdk from the user_daily_visits table to ensure that Element X can be represented fully.

changelog.d/16125.misc

@@ -1 +0,0 @@
-Add an admin endpoint to allow authorizing server to signal token revocations.

changelog.d/16127.bugfix

@@ -1 +0,0 @@
-User constent and 3-PID changes capability cannot be enabled when using experimental [MSC3861](https://github.com/matrix-org/matrix-spec-proposals/pull/3861) support.

changelog.d/16131.misc

@@ -1 +0,0 @@
-Add response time metrics for introspection requests for delegated auth.

changelog.d/16132.misc

@@ -1 +0,0 @@
-MSC3861: allow impersonation by an admin user using `_oidc_admin_impersonate_user_id` query parameter.

changelog.d/16133.bugfix

@@ -1 +0,0 @@
-Fix a rare race that could block new events from being sent for up to two minutes. Introduced in v1.90.0.

changelog.d/16134.bugfix

@@ -1 +0,0 @@
-User constent and 3-PID changes capability cannot be enabled when using experimental [MSC3861](https://github.com/matrix-org/matrix-spec-proposals/pull/3861) support.

changelog.d/16148.bugfix

@@ -1 +0,0 @@
-Fix performance degredation when there are a lot of in-flight replication requests.

changelog.d/16149.misc

@@ -1 +0,0 @@
-Increase performance of read/write locks.

changelog.d/16150.misc

@@ -1 +0,0 @@
-Improve presence tests.

changelog.d/16151.misc

@@ -1 +0,0 @@
-Improve presence tests.

changelog.d/16152.misc

@@ -1 +0,0 @@
-Raised the poetry-core version cap to 1.7.0.

changelog.d/16156.bugfix

@@ -1 +0,0 @@
-Fix a bug introduced in 1.87 where synapse would send an excessive amount of federation requests to servers which have been offline for a long time. Contributed by Nico.

changelog.d/16157.misc

@@ -1 +0,0 @@
-Fix assertion in user directory unit tests.

changelog.d/16158.misc

@@ -1 +0,0 @@
-Improve presence tests.

changelog.d/16159.misc

@@ -1 +0,0 @@
-Reduce scope of locks when paginating to alleviate DB contention.

changelog.d/16160.misc

@@ -1 +0,0 @@
-Reduce DB contention on worker locks.

changelog.d/16164.bugfix

@@ -1 +0,0 @@
-Fix a bug introduced in 1.87 where synapse would send an excessive amount of federation requests to servers which have been offline for a long time. Contributed by Nico.

changelog.d/16165.misc

@@ -1 +0,0 @@
-Task scheduler: mark task as active if we are scheduling as soon as possible.

contrib/cmdclient/console.py

@@ -769,7 +769,7 @@ def main(server_url, identity_server_url, username, token, config_path):
     global CONFIG_JSON
     CONFIG_JSON = config_path  # bit cheeky, but just overwrite the global
     try:
-        with open(config_path) as config:
+        with open(config_path, "r") as config:
             syn_cmd.config = json.load(config)
             try:
                 http_client.verbose = "on" == syn_cmd.config["verbose"]

contrib/grafana/synapse.json

@@ -63,7 +63,7 @@
           "uid": "${DS_PROMETHEUS}"
         },
         "enable": true,
-        "expr": "changes(process_start_time_seconds{instance=\"$instance\",job=~\"synapse\"}[$bucket_size]) * on (instance, job) group_left(version) synapse_build_info{instance=\"$instance\",job=\"synapse\"}",
+        "expr": "changes(process_start_time_seconds{instance=\"matrix.org\",job=~\"synapse\"}[$bucket_size]) * on (instance, job) group_left(version) synapse_build_info{instance=\"matrix.org\",job=\"synapse\"}",
         "iconColor": "purple",
         "name": "deploys",
         "titleFormat": "Deployed {{version}}"

debian/changelog

@@ -1,51 +1,3 @@
-matrix-synapse-py3 (1.90.0) stable; urgency=medium
-
-  * New Synapse release 1.90.0.
-
- -- Synapse Packaging team <packages@matrix.org>  Tue, 15 Aug 2023 11:17:34 +0100
-
-matrix-synapse-py3 (1.90.0~rc1) stable; urgency=medium
-
-  * New Synapse release 1.90.0rc1.
-
- -- Synapse Packaging team <packages@matrix.org>  Tue, 08 Aug 2023 15:29:34 +0100
-
-matrix-synapse-py3 (1.89.0) stable; urgency=medium
-
-  * New Synapse release 1.89.0.
-
- -- Synapse Packaging team <packages@matrix.org>  Tue, 01 Aug 2023 11:07:15 +0100
-
-matrix-synapse-py3 (1.89.0~rc1) stable; urgency=medium
-
-  * New Synapse release 1.89.0rc1.
-
- -- Synapse Packaging team <packages@matrix.org>  Tue, 25 Jul 2023 14:31:07 +0200
-
-matrix-synapse-py3 (1.88.0) stable; urgency=medium
-
-  * New Synapse release 1.88.0.
-
- -- Synapse Packaging team <packages@matrix.org>  Tue, 18 Jul 2023 13:59:28 +0100
-
-matrix-synapse-py3 (1.88.0~rc1) stable; urgency=medium
-
-  * New Synapse release 1.88.0rc1.
-
- -- Synapse Packaging team <packages@matrix.org>  Tue, 11 Jul 2023 10:20:19 +0100
-
-matrix-synapse-py3 (1.87.0) stable; urgency=medium
-
-  * New Synapse release 1.87.0.
-
- -- Synapse Packaging team <packages@matrix.org>  Tue, 04 Jul 2023 16:24:00 +0100
-
-matrix-synapse-py3 (1.87.0~rc1) stable; urgency=medium
-
-  * New synapse release 1.87.0rc1.
-
- -- Synapse Packaging team <packages@matrix.org>  Tue, 27 Jun 2023 15:27:04 +0000
-
 matrix-synapse-py3 (1.86.0) stable; urgency=medium
 
   * New Synapse release 1.86.0.

docker/Dockerfile-dhvirtualenv

@@ -28,12 +28,12 @@ FROM docker.io/library/${distro} as builder
 
 RUN apt-get update -qq -o Acquire::Languages=none
 RUN env DEBIAN_FRONTEND=noninteractive apt-get install \
-    -yqq --no-install-recommends \
-    build-essential \
-    ca-certificates \
-    devscripts \
-    equivs \
-    wget
+        -yqq --no-install-recommends \
+        build-essential \
+        ca-certificates \
+        devscripts \
+        equivs \
+        wget
 
 # fetch and unpack the package
 # We are temporarily using a fork of dh-virtualenv due to an incompatibility with Python 3.11, which ships with
@@ -62,29 +62,33 @@ FROM docker.io/library/${distro}
 ARG distro=""
 ENV distro ${distro}
 
+# Python < 3.7 assumes LANG="C" means ASCII-only and throws on printing unicode
+# http://bugs.python.org/issue19846
+ENV LANG C.UTF-8
+
 # Install the build dependencies
 #
 # NB: keep this list in sync with the list of build-deps in debian/control
 # TODO: it would be nice to do that automatically.
 RUN apt-get update -qq -o Acquire::Languages=none \
     && env DEBIAN_FRONTEND=noninteractive apt-get install \
-    -yqq --no-install-recommends -o Dpkg::Options::=--force-unsafe-io \
-    build-essential \
-    curl \
-    debhelper \
-    devscripts \
-    libsystemd-dev \
-    lsb-release \
-    pkg-config \
-    python3-dev \
-    python3-pip \
-    python3-setuptools \
-    python3-venv \
-    sqlite3 \
-    libpq-dev \
-    libicu-dev \
-    pkg-config \
-    xmlsec1
+        -yqq --no-install-recommends -o Dpkg::Options::=--force-unsafe-io \
+        build-essential \
+        curl \
+        debhelper \
+        devscripts \
+        libsystemd-dev \
+        lsb-release \
+        pkg-config \
+        python3-dev \
+        python3-pip \
+        python3-setuptools \
+        python3-venv \
+        sqlite3 \
+        libpq-dev \
+        libicu-dev \
+        pkg-config \
+        xmlsec1
 
 # Install rust and ensure it's in the PATH
 ENV RUSTUP_HOME=/rust

docker/conf-workers/nginx.conf.j2

@@ -35,11 +35,7 @@ server {
 
     # Send all other traffic to the main process
     location ~* ^(\\/_matrix|\\/_synapse) {
-{% if using_unix_sockets %}
-        proxy_pass http://unix:/run/main_public.sock;
-{% else %}
         proxy_pass http://localhost:8080;
-{% endif %}
         proxy_set_header X-Forwarded-For $remote_addr;
         proxy_set_header X-Forwarded-Proto $scheme;
         proxy_set_header Host $host;

docker/conf-workers/shared.yaml.j2

@@ -6,9 +6,6 @@
 {% if enable_redis %}
 redis:
     enabled: true
-    {% if using_unix_sockets %}
-    path: /tmp/redis.sock
-    {% endif %}
 {% endif %}
 
 {% if appservice_registrations is not none %}

docker/conf-workers/supervisord.conf.j2

@@ -19,11 +19,7 @@ username=www-data
 autorestart=true
 
 [program:redis]
-{% if using_unix_sockets %}
-command=/usr/local/bin/prefix-log /usr/local/bin/redis-server --unixsocket /tmp/redis.sock
-{% else %}
 command=/usr/local/bin/prefix-log /usr/local/bin/redis-server
-{% endif %}
 priority=1
 stdout_logfile=/dev/stdout
 stdout_logfile_maxbytes=0

docker/conf-workers/worker.yaml.j2

@@ -8,11 +8,7 @@ worker_name: "{{ name }}"
 
 worker_listeners:
   - type: http
-{% if using_unix_sockets %}
-    path: "/run/worker.{{ port }}"
-{% else %}
     port: {{ port }}
-{% endif %}
 {% if listener_resources %}
     resources:
       - names:

docker/conf/homeserver.yaml

@@ -36,17 +36,12 @@ listeners:
 
   # Allow configuring in case we want to reverse proxy 8008
   # using another process in the same container
-{% if SYNAPSE_USE_UNIX_SOCKET %}
-  # Unix sockets don't care about TLS or IP addresses or ports
-  - path: '/run/main_public.sock'
-    type: http
-{% else %}
   - port: {{ SYNAPSE_HTTP_PORT or 8008 }}
     tls: false
     bind_addresses: ['::']
     type: http
     x_forwarded: false
-{% endif %}
+
     resources:
       - names: [client]
         compress: true
@@ -62,11 +57,8 @@ database:
     user: "{{ POSTGRES_USER or "synapse" }}"
     password: "{{ POSTGRES_PASSWORD }}"
     database: "{{ POSTGRES_DB or "synapse" }}"
-{% if not SYNAPSE_USE_UNIX_SOCKET %}
-{# Synapse will use a default unix socket for Postgres when host/port is not specified (behavior from `psycopg2`). #}
     host: "{{ POSTGRES_HOST or "db" }}"
     port: "{{ POSTGRES_PORT or "5432" }}"
-{% endif %}
     cp_min: 5
     cp_max: 10
 {% else %}

docker/configure_workers_and_start.py

@@ -74,9 +74,6 @@ MAIN_PROCESS_HTTP_LISTENER_PORT = 8080
 MAIN_PROCESS_INSTANCE_NAME = "main"
 MAIN_PROCESS_LOCALHOST_ADDRESS = "127.0.0.1"
 MAIN_PROCESS_REPLICATION_PORT = 9093
-# Obviously, these would only be used with the UNIX socket option
-MAIN_PROCESS_UNIX_SOCKET_PUBLIC_PATH = "/run/main_public.sock"
-MAIN_PROCESS_UNIX_SOCKET_PRIVATE_PATH = "/run/main_private.sock"
 
 # A simple name used as a placeholder in the WORKERS_CONFIG below. This will be replaced
 # during processing with the name of the worker.
@@ -410,15 +407,11 @@ def add_worker_roles_to_shared_config(
         )
 
         # Map of stream writer instance names to host/ports combos
-        if os.environ.get("SYNAPSE_USE_UNIX_SOCKET", False):
-            instance_map[worker_name] = {
-                "path": f"/run/worker.{worker_port}",
-            }
-        else:
-            instance_map[worker_name] = {
-                "host": "localhost",
-                "port": worker_port,
-            }
+        instance_map[worker_name] = {
+            "host": "localhost",
+            "port": worker_port,
+        }
+
     # Update the list of stream writers. It's convenient that the name of the worker
     # type is the same as the stream to write. Iterate over the whole list in case there
     # is more than one.
@@ -430,15 +423,10 @@ def add_worker_roles_to_shared_config(
 
             # Map of stream writer instance names to host/ports combos
             # For now, all stream writers need http replication ports
-            if os.environ.get("SYNAPSE_USE_UNIX_SOCKET", False):
-                instance_map[worker_name] = {
-                    "path": f"/run/worker.{worker_port}",
-                }
-            else:
-                instance_map[worker_name] = {
-                    "host": "localhost",
-                    "port": worker_port,
-                }
+            instance_map[worker_name] = {
+                "host": "localhost",
+                "port": worker_port,
+            }
 
 
 def merge_worker_template_configs(
@@ -730,29 +718,17 @@ def generate_worker_files(
     # Note that yaml cares about indentation, so care should be taken to insert lines
     # into files at the correct indentation below.
 
-    # Convenience helper for if using unix sockets instead of host:port
-    using_unix_sockets = environ.get("SYNAPSE_USE_UNIX_SOCKET", False)
     # First read the original config file and extract the listeners block. Then we'll
     # add another listener for replication. Later we'll write out the result to the
     # shared config file.
-    listeners: List[Any]
-    if using_unix_sockets:
-        listeners = [
-            {
-                "path": MAIN_PROCESS_UNIX_SOCKET_PRIVATE_PATH,
-                "type": "http",
-                "resources": [{"names": ["replication"]}],
-            }
-        ]
-    else:
-        listeners = [
-            {
-                "port": MAIN_PROCESS_REPLICATION_PORT,
-                "bind_address": MAIN_PROCESS_LOCALHOST_ADDRESS,
-                "type": "http",
-                "resources": [{"names": ["replication"]}],
-            }
-        ]
+    listeners = [
+        {
+            "port": MAIN_PROCESS_REPLICATION_PORT,
+            "bind_address": MAIN_PROCESS_LOCALHOST_ADDRESS,
+            "type": "http",
+            "resources": [{"names": ["replication"]}],
+        }
+    ]
     with open(config_path) as file_stream:
         original_config = yaml.safe_load(file_stream)
         original_listeners = original_config.get("listeners")
@@ -793,17 +769,7 @@ def generate_worker_files(
 
     # A list of internal endpoints to healthcheck, starting with the main process
     # which exists even if no workers do.
-    # This list ends up being part of the command line to curl, (curl added support for
-    # Unix sockets in version 7.40).
-    if using_unix_sockets:
-        healthcheck_urls = [
-            f"--unix-socket {MAIN_PROCESS_UNIX_SOCKET_PUBLIC_PATH} "
-            # The scheme and hostname from the following URL are ignored.
-            # The only thing that matters is the path `/health`
-            "http://localhost/health"
-        ]
-    else:
-        healthcheck_urls = ["http://localhost:8080/health"]
+    healthcheck_urls = ["http://localhost:8080/health"]
 
     # Get the set of all worker types that we have configured
     all_worker_types_in_use = set(chain(*requested_worker_types.values()))
@@ -840,12 +806,8 @@ def generate_worker_files(
         # given worker_type needs to stay assigned and not be replaced.
         worker_config["shared_extra_conf"].update(shared_config)
         shared_config = worker_config["shared_extra_conf"]
-        if using_unix_sockets:
-            healthcheck_urls.append(
-                f"--unix-socket /run/worker.{worker_port} http://localhost/health"
-            )
-        else:
-            healthcheck_urls.append("http://localhost:%d/health" % (worker_port,))
+
+        healthcheck_urls.append("http://localhost:%d/health" % (worker_port,))
 
         # Update the shared config with sharding-related options if necessary
         add_worker_roles_to_shared_config(
@@ -861,10 +823,9 @@ def generate_worker_files(
         # Then a worker config file
         convert(
             "/conf/worker.yaml.j2",
-            f"/conf/workers/{worker_name}.yaml",
+            "/conf/workers/{name}.yaml".format(name=worker_name),
             **worker_config,
             worker_log_config_filepath=log_config_filepath,
-            using_unix_sockets=using_unix_sockets,
         )
 
         # Save this worker's port number to the correct nginx upstreams
@@ -885,13 +846,8 @@ def generate_worker_files(
     nginx_upstream_config = ""
     for upstream_worker_base_name, upstream_worker_ports in nginx_upstreams.items():
         body = ""
-        if using_unix_sockets:
-            for port in upstream_worker_ports:
-                body += f"    server unix:/run/worker.{port};\n"
-
-        else:
-            for port in upstream_worker_ports:
-                body += f"    server localhost:{port};\n"
+        for port in upstream_worker_ports:
+            body += f"    server localhost:{port};\n"
 
         # Add to the list of configured upstreams
         nginx_upstream_config += NGINX_UPSTREAM_CONFIG_BLOCK.format(
@@ -921,15 +877,10 @@ def generate_worker_files(
     # If there are workers, add the main process to the instance_map too.
     if workers_in_use:
         instance_map = shared_config.setdefault("instance_map", {})
-        if using_unix_sockets:
-            instance_map[MAIN_PROCESS_INSTANCE_NAME] = {
-                "path": MAIN_PROCESS_UNIX_SOCKET_PRIVATE_PATH,
-            }
-        else:
-            instance_map[MAIN_PROCESS_INSTANCE_NAME] = {
-                "host": MAIN_PROCESS_LOCALHOST_ADDRESS,
-                "port": MAIN_PROCESS_REPLICATION_PORT,
-            }
+        instance_map[MAIN_PROCESS_INSTANCE_NAME] = {
+            "host": MAIN_PROCESS_LOCALHOST_ADDRESS,
+            "port": MAIN_PROCESS_REPLICATION_PORT,
+        }
 
     # Shared homeserver config
     convert(
@@ -939,7 +890,6 @@ def generate_worker_files(
         appservice_registrations=appservice_registrations,
         enable_redis=workers_in_use,
         workers_in_use=workers_in_use,
-        using_unix_sockets=using_unix_sockets,
     )
 
     # Nginx config
@@ -950,7 +900,6 @@ def generate_worker_files(
         upstream_directives=nginx_upstream_config,
         tls_cert_path=os.environ.get("SYNAPSE_TLS_CERT"),
         tls_key_path=os.environ.get("SYNAPSE_TLS_KEY"),
-        using_unix_sockets=using_unix_sockets,
     )
 
     # Supervisord config
@@ -960,7 +909,6 @@ def generate_worker_files(
         "/etc/supervisor/supervisord.conf",
         main_config_path=config_path,
         enable_redis=workers_in_use,
-        using_unix_sockets=using_unix_sockets,
     )
 
     convert(

docker/start.py

@@ -82,7 +82,7 @@ def generate_config_from_template(
                 with open(filename) as handle:
                     value = handle.read()
             else:
-                log(f"Generating a random secret for {secret}")
+                log("Generating a random secret for {}".format(secret))
                 value = codecs.encode(os.urandom(32), "hex").decode()
                 with open(filename, "w") as handle:
                     handle.write(value)

docs/SUMMARY.md

@@ -97,7 +97,6 @@
     - [Cancellation](development/synapse_architecture/cancellation.md)
     - [Log Contexts](log_contexts.md)
     - [Replication](replication.md)
-    - [Streams](development/synapse_architecture/streams.md)
     - [TCP Replication](tcp_replication.md)
     - [Faster remote joins](development/synapse_architecture/faster_joins.md)
   - [Internal Documentation](development/internal_documentation/README.md)

docs/admin_api/user_admin_api.md

@@ -146,7 +146,6 @@ Body parameters:
 - `admin` - **bool**, optional, defaults to `false`. Whether the user is a homeserver administrator,
   granting them access to the Admin API, among other things.
 - `deactivated` - **bool**, optional. If unspecified, deactivation state will be left unchanged.
-- `locked` - **bool**, optional. If unspecified, locked state will be left unchanged.
 
   Note: the `password` field must also be set if both of the following are true:
   - `deactivated` is set to `false` and the user was previously deactivated (you are reactivating this user)
@@ -219,8 +218,6 @@ The following parameters should be set in the URL:
   **or** displaynames that contain this value.
 - `guests` - string representing a bool - Is optional and if `false` will **exclude** guest users.
   Defaults to `true` to include guest users.
-- `admins` - Optional flag to filter admins. If `true`, only admins are queried. If `false`, admins are excluded from 
-  the query. When the flag is absent (the default), **both** admins and non-admins are included in the search results.
 - `deactivated` - string representing a bool - Is optional and if `true` will **include** deactivated users.
   Defaults to `false` to exclude deactivated users.
 - `limit` - string representing a positive integer - Is optional but is used for pagination,
@@ -245,9 +242,6 @@ The following parameters should be set in the URL:
 
 - `dir` - Direction of media order. Either `f` for forwards or `b` for backwards.
   Setting this value to `b` will reverse the above sort order. Defaults to `f`.
-- `not_user_type` - Exclude certain user types, such as bot users, from the request.
-   Can be provided multiple times. Possible values are `bot`, `support` or "empty string".
-   "empty string" here means to exclude users without a type.
 
 Caution. The database only has indexes on the columns `name` and `creation_ts`.
 This means that if a different sort order is used (`is_guest`, `admin`,
@@ -735,8 +729,7 @@ POST /_synapse/admin/v1/users/<user_id>/login
 
 An optional `valid_until_ms` field can be specified in the request body as an
 integer timestamp that specifies when the token should expire. By default tokens
-do not expire. Note that this API does not allow a user to login as themselves
-(to create more tokens).
+do not expire.
 
 A response body like the following is returned:
 
@@ -1187,7 +1180,7 @@ The following parameters should be set in the URL:
 - `user_id` - The fully qualified MXID: for example, `@user:server.com`. The user must
   be local.
 
-## Check username availability
+### Check username availability
 
 Checks to see if a username is available, and valid, for the server. See [the client-server 
 API](https://matrix.org/docs/spec/client_server/r0.6.0#get-matrix-client-r0-register-available)
@@ -1205,7 +1198,7 @@ GET /_synapse/admin/v1/username_available?username=$localpart
 The request and response format is the same as the
 [/_matrix/client/r0/register/available](https://matrix.org/docs/spec/client_server/r0.6.0#get-matrix-client-r0-register-available) API.
 
-## Find a user based on their ID in an auth provider
+### Find a user based on their ID in an auth provider
 
 The API is:
 
@@ -1244,7 +1237,7 @@ Returns a `404` HTTP status code if no user was found, with a response body like
 _Added in Synapse 1.68.0._
 
 
-## Find a user based on their Third Party ID (ThreePID or 3PID)
+### Find a user based on their Third Party ID (ThreePID or 3PID)
 
 The API is:
 

docs/deprecation_policy.md

@@ -23,7 +23,7 @@ people building from source should ensure they can fetch recent versions of Rust
 (e.g. by using [rustup](https://rustup.rs/)).
 
 The oldest supported version of SQLite is the version
-[provided](https://packages.debian.org/bullseye/libsqlite3-0) by
+[provided](https://packages.debian.org/buster/libsqlite3-0) by
 [Debian oldstable](https://wiki.debian.org/DebianOldStable).
 
 Context

docs/development/contributing_guide.md

@@ -322,7 +322,7 @@ The following command will let you run the integration test with the most common
 configuration:
 
 ```sh
-$ docker run --rm -it -v /path/where/you/have/cloned/the/repository\:/src:ro -v /path/to/where/you/want/logs\:/logs matrixdotorg/sytest-synapse:focal
+$ docker run --rm -it -v /path/where/you/have/cloned/the/repository\:/src:ro -v /path/to/where/you/want/logs\:/logs matrixdotorg/sytest-synapse:buster
 ```
 (Note that the paths must be full paths! You could also write `$(realpath relative/path)` if needed.)
 
@@ -370,7 +370,6 @@ The above will run a monolithic (single-process) Synapse with SQLite as the data
     See the [worker documentation](../workers.md) for additional information on workers.
 - Passing `ASYNCIO_REACTOR=1` as an environment variable to use the Twisted asyncio reactor instead of the default one.
 - Passing `PODMAN=1` will use the [podman](https://podman.io/) container runtime, instead of docker.
-- Passing `UNIX_SOCKETS=1` will utilise Unix socket functionality for Synapse, Redis, and Postgres(when applicable).
 
 To increase the log level for the tests, set `SYNAPSE_TEST_LOG_LEVEL`, e.g:
 ```sh

docs/development/synapse_architecture/streams.md

@@ -1,157 +0,0 @@
-## Streams
-
-Synapse has a concept of "streams", which are roughly described in [`id_generators.py`](
-    https://github.com/matrix-org/synapse/blob/develop/synapse/storage/util/id_generators.py
-).
-Generally speaking, streams are a series of notifications that something in Synapse's database has changed that the application might need to respond to.
-For example:
-
-- The events stream reports new events (PDUs) that Synapse creates, or that Synapse accepts from another homeserver.
-- The account data stream reports changes to users' [account data](https://spec.matrix.org/v1.7/client-server-api/#client-config).
-- The to-device stream reports when a device has a new [to-device message](https://spec.matrix.org/v1.7/client-server-api/#send-to-device-messaging).
-
-See [`synapse.replication.tcp.streams`](
-    https://github.com/matrix-org/synapse/blob/develop/synapse/replication/tcp/streams/__init__.py
-) for the full list of streams.
-
-It is very helpful to understand the streams mechanism when working on any part of Synapse that needs to respond to changes—especially if those changes are made by different workers.
-To that end, let's describe streams formally, paraphrasing from the docstring of [`AbstractStreamIdGenerator`](
-    https://github.com/matrix-org/synapse/blob/a719b703d9bd0dade2565ddcad0e2f3a7a9d4c37/synapse/storage/util/id_generators.py#L96
-).
-
-### Definition
-
-A stream is an append-only log `T1, T2, ..., Tn, ...` of facts[^1] which grows over time.
-Only "writers" can add facts to a stream, and there may be multiple writers.
-
-Each fact has an ID, called its "stream ID".
-Readers should only process facts in ascending stream ID order.
-
-Roughly speaking, each stream is backed by a database table.
-It should have a `stream_id` (or similar) bigint column holding stream IDs, plus additional columns as necessary to describe the fact.
-Typically, a fact is expressed with a single row in its backing table.[^2]
-Within a stream, no two facts may have the same stream_id.
-
-> _Aside_. Some additional notes on streams' backing tables.
->
-> 1. Rich would like to [ditch the backing tables](https://github.com/matrix-org/synapse/issues/13456).
-> 2. The backing tables may have other uses.
-     >    For example, the events table serves backs the events stream, and is read when processing new events.
-     >    But old rows are read from the table all the time, whenever Synapse needs to lookup some facts about an event.
-> 3. Rich suspects that sometimes the stream is backed by multiple tables, so the stream proper is the union of those tables.
-
-Stream writers can "reserve" a stream ID, and then later mark it as having being completed.
-Stream writers need to track the completion of each stream fact.
-In the happy case, completion means a fact has been written to the stream table.
-But unhappy cases (e.g. transaction rollback due to an error) also count as completion.
-Once completed, the rows written with that stream ID are fixed, and no new rows
-will be inserted with that ID.
-
-### Current stream ID
-
-For any given stream reader (including writers themselves), we may define a per-writer current stream ID:
-
-> The current stream ID _for a writer W_ is the largest stream ID such that
-> all transactions added by W with equal or smaller ID have completed.
-
-Similarly, there is a "linear" notion of current stream ID:
-
-> The "linear" current stream ID is the largest stream ID such that
-> all facts (added by any writer) with equal or smaller ID have completed.
-
-Because different stream readers A and B learn about new facts at different times, A and B may disagree about current stream IDs.
-Put differently: we should think of stream readers as being independent of each other, proceeding through a stream of facts at different rates.
-
-**NB.** For both senses of "current", that if a writer opens a transaction that never completes, the current stream ID will never advance beyond that writer's last written stream ID.
-
-For single-writer streams, the per-writer current ID and the linear current ID are the same.
-Both senses of current ID are monotonic, but they may "skip" or jump over IDs because facts complete out of order.
-
-
-_Example_.
-Consider a single-writer stream which is initially at ID 1.
-
-| Action     | Current stream ID | Notes                                           |
-|------------|-------------------|-------------------------------------------------|
-|            | 1                 |                                                 |
-| Reserve 2  | 1                 |                                                 |
-| Reserve 3  | 1                 |                                                 |
-| Complete 3 | 1                 | current ID unchanged, waiting for 2 to complete |
-| Complete 2 | 3                 | current ID jumps from 1 -> 3                    |
-| Reserve 4  | 3                 |                                                 |
-| Reserve 5  | 3                 |                                                 |
-| Reserve 6  | 3                 |                                                 |
-| Complete 5 | 3                 |                                                 |
-| Complete 4 | 5                 | current ID jumps 3->5, even though 6 is pending |
-| Complete 6 | 6                 |                                                 |
-
-
-### Multi-writer streams
-
-There are two ways to view a multi-writer stream.
-
-1. Treat it as a collection of distinct single-writer streams, one
-   for each writer.
-2. Treat it as a single stream.
-
-The single stream (option 2) is conceptually simpler, and easier to represent (a single stream id).
-However, it requires each reader to know about the entire set of writers, to ensures that readers don't erroneously advance their current stream position too early and miss a fact from an unknown writer.
-In contrast, multiple parallel streams (option 1) are more complex, requiring more state to represent (map from writer to stream id).
-The payoff for doing so is that readers can "peek" ahead to facts that completed on one writer no matter the state of the others, reducing latency.
-
-Note that a multi-writer stream can be viewed in both ways.
-For example, the events stream is treated as multiple single-writer streams (option 1) by the sync handler, so that events are sent to clients as soon as possible.
-But the background process that works through events treats them as a single linear stream.
-
-Another useful example is the cache invalidation stream.
-The facts this stream holds are instructions to "you should now invalidate these cache entries".
-We only ever treat this as a multiple single-writer streams as there is no important ordering between cache invalidations.
-(Invalidations are self-contained facts; and the invalidations commute/are idempotent).
-
-### Writing to streams
-
-Writers need to track:
- - track their current position (i.e. its own per-writer stream ID).
- - their facts currently awaiting completion.
-
-At startup, 
- - the current position of that writer can be found by querying the database (which suggests that facts need to be written to the database atomically, in a transaction); and
- - there are no facts awaiting completion.
-
-To reserve a stream ID, call [`nextval`](https://www.postgresql.org/docs/current/functions-sequence.html) on the appropriate postgres sequence.
-
-To write a fact to the stream: insert the appropriate rows to the appropriate backing table.
-
-To complete a fact, first remove it from your map of facts currently awaiting completion.
-Then, if no earlier fact is awaiting completion, the writer can advance its current position in that stream.
-Upon doing so it should emit an `RDATA` message[^3], once for every fact between the old and the new stream ID.
-
-### Subscribing to streams
-
-Readers need to track the current position of every writer.
-
-At startup, they can find this by contacting each writer with a `REPLICATE` message,
-requesting that all writers reply describing their current position in their streams.
-Writers reply with a `POSITION` message.
-
-To learn about new facts, readers should listen for `RDATA` messages and process them to respond to the new fact.
-The `RDATA` itself is not a self-contained representation of the fact;
-readers will have to query the stream tables for the full details.
-Readers must also advance their record of the writer's current position for that stream.
-
-# Summary
-
-In a nutshell: we have an append-only log with a "buffer/scratchpad" at the end where we have to wait for the sequence to be linear and contiguous.
-
-
----
-
-[^1]: we use the word _fact_ here for two reasons.
-Firstly, the word "event" is already heavily overloaded (PDUs, EDUs, account data, ...) and we don't need to make that worse.
-Secondly, "fact" emphasises that the things we append to a stream cannot change after the fact.
-
-[^2]: A fact might be expressed with 0 rows, e.g. if we opened a transaction to persist an event, but failed and rolled the transaction back before marking the fact as completed.
-In principle a fact might be expressed with 2 or more rows; if so, each of those rows should share the fact's stream ID.
-
-[^3]: This communication used to happen directly with the writers [over TCP](../../tcp_replication.md);
-nowadays it's done via Redis's Pubsub.

docs/reverse_proxy.md

@@ -95,7 +95,7 @@ matrix.example.com {
 }
 
 example.com:8448 {
-  reverse_proxy /_matrix/* localhost:8008
+  reverse_proxy localhost:8008
 }
 ```
 

docs/setup/installation.md

@@ -135,8 +135,8 @@ Unofficial package are built for SLES 15 in the openSUSE:Backports:SLE-15 reposi
 
 #### ArchLinux
 
-The quickest way to get up and running with ArchLinux is probably with the package provided by ArchLinux
-<https://archlinux.org/packages/extra/x86_64/matrix-synapse/>, which should pull in most of
+The quickest way to get up and running with ArchLinux is probably with the community package
+<https://archlinux.org/packages/community/x86_64/matrix-synapse/>, which should pull in most of
 the necessary dependencies.
 
 pip may be outdated (6.0.7-1 and needs to be upgraded to 6.0.8-1 ):
@@ -200,7 +200,7 @@ When following this route please make sure that the [Platform-specific prerequis
 System requirements:
 
 - POSIX-compliant system (tested on Linux & OS X)
-- Python 3.8 or later, up to Python 3.11.
+- Python 3.7 or later, up to Python 3.11.
 - At least 1GB of free RAM if you want to join large public rooms like #matrix:matrix.org
 
 If building on an uncommon architecture for which pre-built wheels are

docs/structured_logging.md

@@ -3,7 +3,7 @@
 A structured logging system can be useful when your logs are destined for a
 machine to parse and process. By maintaining its machine-readable characteristics,
 it enables more efficient searching and aggregations when consumed by software
-such as the [ELK stack](https://opensource.com/article/18/9/open-source-log-aggregation-tools).
+such as the "ELK stack".
 
 Synapse's structured logging system is configured via the file that Synapse's
 `log_config` config option points to. The file should include a formatter which

docs/systemd-with-workers/workers/background_worker.yaml

@@ -1,4 +1,8 @@
 worker_app: synapse.app.generic_worker
 worker_name: background_worker
 
+# The replication listener on the main synapse process.
+worker_replication_host: 127.0.0.1
+worker_replication_http_port: 9093
+
 worker_log_config: /etc/matrix-synapse/background-worker-log.yaml

docs/systemd-with-workers/workers/event_persister.yaml

@@ -1,5 +1,9 @@
 worker_app: synapse.app.generic_worker
-worker_name: event_persister1
+worker_name: event_persister1 
+
+# The replication listener on the main synapse process.
+worker_replication_host: 127.0.0.1
+worker_replication_http_port: 9093
 
 worker_listeners:
   - type: http

docs/systemd-with-workers/workers/federation_sender.yaml

@@ -1,4 +1,8 @@
 worker_app: synapse.app.federation_sender
 worker_name: federation_sender1
 
+# The replication listener on the main synapse process.
+worker_replication_host: 127.0.0.1
+worker_replication_http_port: 9093
+
 worker_log_config: /etc/matrix-synapse/federation-sender-log.yaml

docs/systemd-with-workers/workers/media_worker.yaml

@@ -1,6 +1,10 @@
 worker_app: synapse.app.media_repository
 worker_name: media_worker
 
+# The replication listener on the main synapse process.
+worker_replication_host: 127.0.0.1
+worker_replication_http_port: 9093
+
 worker_listeners:
   - type: http
     port: 8085

docs/systemd-with-workers/workers/pusher_worker.yaml

@@ -1,4 +1,8 @@
 worker_app: synapse.app.pusher
 worker_name: pusher_worker1
 
+# The replication listener on the main synapse process.
+worker_replication_host: 127.0.0.1
+worker_replication_http_port: 9093
+
 worker_log_config: /etc/matrix-synapse/pusher-worker-log.yaml

docs/upgrade.md

@@ -87,57 +87,6 @@ process, for example:
     wget https://packages.matrix.org/debian/pool/main/m/matrix-synapse-py3/matrix-synapse-py3_1.3.0+stretch1_amd64.deb
     dpkg -i matrix-synapse-py3_1.3.0+stretch1_amd64.deb
     ```
-
-# Upgrading to v1.90.0
-
-## App service query parameter authorization is now a configuration option
-
-Synapse v1.81.0 deprecated application service authorization via query parameters as this is
-considered insecure - and from Synapse v1.71.0 forwards the application service token has also been sent via 
-[the `Authorization` header](https://spec.matrix.org/v1.6/application-service-api/#authorization)], making the insecure
-query parameter authorization redundant. Since removing the ability to continue to use query parameters could break 
-backwards compatibility it has now been put behind a configuration option, `use_appservice_legacy_authorization`.  
-This option defaults to false, but can be activated by adding 
-```yaml
-use_appservice_legacy_authorization: true 
-```
-to your configuration.
-
-# Upgrading to v1.89.0
-
-## Removal of unspecced `user` property for `/register`
-
-Application services can no longer call `/register` with a `user` property to create new users.
-The standard `username` property should be used instead. See the
-[Application Service specification](https://spec.matrix.org/v1.7/application-service-api/#server-admin-style-permissions)
-for more information.
-
-# Upgrading to v1.88.0
-
-## Minimum supported Python version
-
-The minimum supported Python version has been increased from v3.7 to v3.8.
-You will need Python 3.8 to run Synapse v1.88.0 (due out July 18th, 2023).
-
-If you use current versions of the Matrix.org-distributed Debian
-packages or Docker images, no action is required.
-
-## Removal of `worker_replication_*` settings
-
-As mentioned previously in [Upgrading to v1.84.0](#upgrading-to-v1840), the following deprecated settings
-are being removed in this release of Synapse:
-
-* [`worker_replication_host`](https://matrix-org.github.io/synapse/v1.86/usage/configuration/config_documentation.html#worker_replication_host)
-* [`worker_replication_http_port`](https://matrix-org.github.io/synapse/v1.86/usage/configuration/config_documentation.html#worker_replication_http_port)
-* [`worker_replication_http_tls`](https://matrix-org.github.io/synapse/v1.86/usage/configuration/config_documentation.html#worker_replication_http_tls)
-
-Please ensure that you have migrated to using `main` on your shared configuration's `instance_map`
-(or create one if necessary). This is required if you have ***any*** workers at all;
-administrators of single-process (monolith) installations don't need to do anything.
-
-For an illustrative example, please see [Upgrading to v1.84.0](#upgrading-to-v1840) below.
-
-
 # Upgrading to v1.86.0
 
 ## Minimum supported Rust version

docs/usage/configuration/config_documentation.md

@@ -462,20 +462,6 @@ See the docs [request log format](../administration/request_log.md).
 * `additional_resources`: Only valid for an 'http' listener. A map of
    additional endpoints which should be loaded via dynamic modules.
 
-Unix socket support (_Added in Synapse 1.89.0_):
-* `path`: A path and filename for a Unix socket. Make sure it is located in a
-  directory with read and write permissions, and that it already exists (the directory
-  will not be created). Defaults to `None`.
-  * **Note**: The use of both `path` and `port` options for the same `listener` is not
-    compatible.
-  * The `x_forwarded` option defaults to true  when using Unix sockets and can be omitted.
-  * Other options that would not make sense to use with a UNIX socket, such as 
-    `bind_addresses` and `tls` will be ignored and can be removed.
-* `mode`: The file permissions to set on the UNIX socket. Defaults to `666`
-* **Note:** Must be set as `type: http` (does not support `metrics` and `manhole`). 
-  Also make sure that `metrics` is not included in `resources` -> `names`
-
-
 Valid resource names are:
 
 * `client`: the client-server API (/_matrix/client), and the synapse admin API (/_synapse/admin). Also implies `media` and `static`.
@@ -488,7 +474,7 @@ Valid resource names are:
 
 * `media`: the media API (/_matrix/media).
 
-* `metrics`: the metrics interface. See [here](../../metrics-howto.md). (Not compatible with Unix sockets)
+* `metrics`: the metrics interface. See [here](../../metrics-howto.md).
 
 * `openid`: OpenID authentication. See [here](../../openid.md).
 
@@ -547,22 +533,6 @@ listeners:
     bind_addresses: ['::1', '127.0.0.1']
     type: manhole
 ```
-Example configuration #3:
-```yaml
-listeners:
-  # Unix socket listener: Ideal for Synapse deployments behind a reverse proxy, offering
-  # lightweight interprocess communication without TCP/IP overhead, avoid port
-  # conflicts, and providing enhanced security through system file permissions.
-  #
-  # Note that x_forwarded will default to true, when using a UNIX socket. Please see
-  # https://matrix-org.github.io/synapse/latest/reverse_proxy.html.
-  #
-  - path: /var/run/synapse/main_public.sock
-    type: http
-    resources:
-      - names: [client, federation]
-```
-
 ---
 ### `manhole_settings`
 
@@ -1242,14 +1212,6 @@ like sending a federation transaction.
 * `max_short_retries`: maximum number of retries for the short retry algo. Default to 3 attempts.
 * `max_long_retries`: maximum number of retries for the long retry algo. Default to 10 attempts.
 
-The following options control the retry logic when communicating with a specific homeserver destination.
-Unlike the previous configuration options, these values apply across all requests
-for a given destination and the state of the backoff is stored in the database.
-
-* `destination_min_retry_interval`: the initial backoff, after the first request fails. Defaults to 10m.
-* `destination_retry_multiplier`: how much we multiply the backoff by after each subsequent fail. Defaults to 2.
-* `destination_max_retry_interval`: a cap on the backoff. Defaults to a week.
-
 Example configuration:
 ```yaml
 federation:
@@ -1258,9 +1220,6 @@ federation:
   max_long_retry_delay: 100s
   max_short_retries: 5
   max_long_retries: 20
-  destination_min_retry_interval: 30s
-  destination_retry_multiplier: 5
-  destination_max_retry_interval: 12h
 ```
 ---
 ## Caching
@@ -2848,20 +2807,6 @@ Example configuration:
 ```yaml
 track_appservice_user_ips: true
 ```
----
-### `use_appservice_legacy_authorization`
-
-Whether to send the application service access tokens via the `access_token` query parameter
-per older versions of the Matrix specification. Defaults to false. Set to true to enable sending
-access tokens via a query parameter.
-
-**Enabling this option is considered insecure and is not recommended. **
-
-Example configuration:
-```yaml
-use_appservice_legacy_authorization: true 
-```
-
 ---
 ### `macaroon_secret_key`
 
@@ -3025,16 +2970,6 @@ enable SAML login. You can either put your entire pysaml config inline using the
 option, or you can specify a path to a psyaml config file with the sub-option `config_path`.
 This setting has the following sub-options:
 
-* `idp_name`: A user-facing name for this identity provider, which is used to
-   offer the user a choice of login mechanisms.
-* `idp_icon`: An optional icon for this identity provider, which is presented
-   by clients and Synapse's own IdP picker page. If given, must be an
-   MXC URI of the format `mxc://<server-name>/<media-id>`. (An easy way to
-   obtain such an MXC URI is to upload an image to an (unencrypted) room
-   and then copy the "url" from the source of the event.)
-* `idp_brand`: An optional brand for this identity provider, allowing clients
-   to style the login flow according to the identity provider in question.
-   See the [spec](https://spec.matrix.org/latest/) for possible options here.
 * `sp_config`: the configuration for the pysaml2 Service Provider. See pysaml2 docs for format of config.
    Default values will be used for the `entityid` and `service` settings,
    so it is not normally necessary to specify them unless you need to
@@ -3186,7 +3121,7 @@ Options for each entry include:
 
 * `idp_icon`: An optional icon for this identity provider, which is presented
    by clients and Synapse's own IdP picker page. If given, must be an
-   MXC URI of the format `mxc://<server-name>/<media-id>`. (An easy way to
+   MXC URI of the format mxc://<server-name>/<media-id>. (An easy way to
    obtain such an MXC URI is to upload an image to an (unencrypted) room
    and then copy the "url" from the source of the event.)
 
@@ -3204,14 +3139,6 @@ Options for each entry include:
 
 * `client_secret`: oauth2 client secret to use. May be omitted if
   `client_secret_jwt_key` is given, or if `client_auth_method` is 'none'.
-  Must be omitted if `client_secret_path` is specified.
-
-* `client_secret_path`: path to the oauth2 client secret to use. With that
-   it's not necessary to leak secrets into the config file itself.
-   Mutually exclusive with `client_secret`. Can be omitted if
-   `client_secret_jwt_key` is specified.
-
-   *Added in Synapse 1.91.0.*
 
 * `client_secret_jwt_key`: Alternative to client_secret: details of a key used
    to create a JSON Web Token to be used as an OAuth2 client secret. If
@@ -3409,16 +3336,6 @@ Enable Central Authentication Service (CAS) for registration and login.
 Has the following sub-options:
 * `enabled`: Set this to true to enable authorization against a CAS server.
    Defaults to false.
-* `idp_name`: A user-facing name for this identity provider, which is used to
-   offer the user a choice of login mechanisms.
-* `idp_icon`: An optional icon for this identity provider, which is presented
-   by clients and Synapse's own IdP picker page. If given, must be an
-   MXC URI of the format `mxc://<server-name>/<media-id>`. (An easy way to
-   obtain such an MXC URI is to upload an image to an (unencrypted) room
-   and then copy the "url" from the source of the event.)
-* `idp_brand`: An optional brand for this identity provider, allowing clients
-   to style the login flow according to the identity provider in question.
-   See the [spec](https://spec.matrix.org/latest/) for possible options here.
 * `server_url`: The URL of the CAS authorization endpoint.
 * `displayname_attribute`: The attribute of the CAS response to use as the display name.
    If no name is given here, no displayname will be set.
@@ -3659,7 +3576,6 @@ This option has the following sub-options:
 * `prefer_local_users`: Defines whether to prefer local users in search query results.
    If set to true, local users are more likely to appear above remote users when searching the
    user directory. Defaults to false.
-* `show_locked_users`: Defines whether to show locked users in search query results. Defaults to false.
 
 Example configuration:
 ```yaml
@@ -3667,7 +3583,6 @@ user_directory:
     enabled: false
     search_all_users: true
     prefer_local_users: true
-    show_locked_users: true
 ```
 ---
 ### `user_consent`
@@ -4015,14 +3930,13 @@ federation_sender_instances:
 ---
 ### `instance_map`
 
-When using workers this should be a map from [`worker_name`](#worker_name) to the HTTP
-replication listener of the worker, if configured, and to the main process. Each worker
-declared under [`stream_writers`](../../workers.md#stream-writers) and
-[`outbound_federation_restricted_to`](#outbound_federation_restricted_to) needs a HTTP
-replication listener, and that listener should be included in the `instance_map`. The
-main process also needs an entry on the `instance_map`, and it should be listed under
-`main` **if even one other worker exists**. Ensure the port matches with what is
-declared inside the `listener` block for a `replication` listener.
+When using workers this should be a map from [`worker_name`](#worker_name) to the
+HTTP replication listener of the worker, if configured, and to the main process.
+Each worker declared under [`stream_writers`](../../workers.md#stream-writers) needs
+a HTTP replication listener, and that listener should be included in the `instance_map`.
+The main process also needs an entry on the `instance_map`, and it should be listed under
+`main` **if even one other worker exists**. Ensure the port matches with what is declared 
+inside the `listener` block for a `replication` listener.
 
 
 Example configuration:
@@ -4035,14 +3949,6 @@ instance_map:
     host: localhost
     port: 8034
 ```
-Example configuration(#2, for UNIX sockets):
-```yaml
-instance_map:
-  main:
-    path: /var/run/synapse/main_replication.sock
-  worker1:
-    path: /var/run/synapse/worker1_replication.sock
-```
 ---
 ### `stream_writers`
 
@@ -4060,24 +3966,6 @@ stream_writers:
   typing: worker1
 ```
 ---
-### `outbound_federation_restricted_to`
-
-When using workers, you can restrict outbound federation traffic to only go through a
-specific subset of workers. Any worker specified here must also be in the
-[`instance_map`](#instance_map).
-[`worker_replication_secret`](#worker_replication_secret) must also be configured to
-authorize inter-worker communication.
-
-```yaml
-outbound_federation_restricted_to:
-  - federation_sender1
-  - federation_sender2
-```
-
-Also see the [worker
-documentation](../../workers.md#restrict-outbound-federation-traffic-to-a-specific-set-of-workers)
-for more info.
----
 ### `run_background_tasks_on`
 
 The [worker](../../workers.md#background-tasks) that is used to run
@@ -4202,6 +4090,51 @@ Example configuration:
 worker_name: generic_worker1
 ```
 ---
+### `worker_replication_host`
+*Deprecated as of version 1.84.0. Place `host` under `main` entry on the [`instance_map`](#instance_map) in your shared yaml configuration instead.*
+
+The HTTP replication endpoint that it should talk to on the main Synapse process.
+The main Synapse process defines this with a `replication` resource in
+[`listeners` option](#listeners).
+
+Example configuration:
+```yaml
+worker_replication_host: 127.0.0.1
+```
+---
+### `worker_replication_http_port`
+*Deprecated as of version 1.84.0. Place `port` under `main` entry on the [`instance_map`](#instance_map) in your shared yaml configuration instead.*
+
+The HTTP replication port that it should talk to on the main Synapse process.
+The main Synapse process defines this with a `replication` resource in
+[`listeners` option](#listeners).
+
+Example configuration:
+```yaml
+worker_replication_http_port: 9093
+```
+---
+### `worker_replication_http_tls`
+*Deprecated as of version 1.84.0. Place `tls` under `main` entry on the [`instance_map`](#instance_map) in your shared yaml configuration instead.*
+
+Whether TLS should be used for talking to the HTTP replication port on the main
+Synapse process.
+The main Synapse process defines this with the `tls` option on its [listener](#listeners) that
+has the `replication` resource enabled.
+
+**Please note:** by default, it is not safe to expose replication ports to the
+public Internet, even with TLS enabled.
+See [`worker_replication_secret`](#worker_replication_secret).
+
+Defaults to `false`.
+
+*Added in Synapse 1.72.0.*
+
+Example configuration:
+```yaml
+worker_replication_http_tls: true
+```
+---
 ### `worker_listeners`
 
 A worker can handle HTTP requests. To do so, a `worker_listeners` option
@@ -4220,18 +4153,6 @@ worker_listeners:
     resources:
       - names: [client, federation]
 ```
-Example configuration(#2, using UNIX sockets with a `replication` listener):
-```yaml
-worker_listeners:
-  - type: http
-    path: /var/run/synapse/worker_public.sock
-    resources:
-      - names: [client, federation]
-  - type: http
-    path: /var/run/synapse/worker_replication.sock
-    resources:
-      - names: [replication]
-```
 ---
 ### `worker_manhole`
 

docs/workers.md

@@ -95,12 +95,9 @@ for the main process
 * Secondly, you need to enable
 [redis-based replication](usage/configuration/config_documentation.md#redis)
 * You will need to add an [`instance_map`](usage/configuration/config_documentation.md#instance_map) 
-with the `main` process defined, as well as the relevant connection information from
-it's HTTP `replication` listener (defined in step 1 above).
-  * Note that the `host` defined is the address the worker needs to look for the `main`
-  process at, not necessarily the same address that is bound to.
-  * If you are using Unix sockets for the `replication` resource, make sure to
-  use a `path` to the socket file instead of a `port`.
+with the `main` process defined, as well as the relevant connection information from 
+it's HTTP `replication` listener (defined in step 1 above). Note that the `host` defined 
+is the address the worker needs to look for the `main` process at, not necessarily the same address that is bound to.
 * Optionally, a [shared secret](usage/configuration/config_documentation.md#worker_replication_secret)
 can be used to authenticate HTTP traffic between workers. For example:
 
@@ -148,6 +145,9 @@ In the config file for each worker, you must specify:
    with an `http` listener.
  * **Synapse 1.72 and older:** if handling the `^/_matrix/client/v3/keys/upload` endpoint, the HTTP URI for
    the main process (`worker_main_http_uri`). This config option is no longer required and is ignored when running Synapse 1.73 and newer.
+ * **Synapse 1.83 and older:** The HTTP replication endpoint that the worker should talk to on the main synapse process
+   ([`worker_replication_host`](usage/configuration/config_documentation.md#worker_replication_host) and
+   [`worker_replication_http_port`](usage/configuration/config_documentation.md#worker_replication_http_port)). If using Synapse 1.84 and newer, these are not needed if `main` is defined on the [shared configuration](#shared-configuration) `instance_map`
 
 For example:
 
@@ -177,11 +177,11 @@ The following applies to Synapse installations that have been installed from sou
 
 You can start the main Synapse process with Poetry by running the following command:
 ```console
-poetry run synapse_homeserver --config-file [your homeserver.yaml]
+poetry run synapse_homeserver -c [your homeserver.yaml]
 ```
 For worker setups, you can run the following command
 ```console
-poetry run synapse_worker --config-file [your homeserver.yaml] --config-file [your worker.yaml]
+poetry run synapse_worker -c [your worker.yaml]
 ```
 ## Available worker applications
 
@@ -531,30 +531,6 @@ the stream writer for the `presence` stream:
 
     ^/_matrix/client/(api/v1|r0|v3|unstable)/presence/
 
-#### Restrict outbound federation traffic to a specific set of workers
-
-The
-[`outbound_federation_restricted_to`](usage/configuration/config_documentation.md#outbound_federation_restricted_to)
-configuration is useful to make sure outbound federation traffic only goes through a
-specified subset of workers. This allows you to set more strict access controls (like a
-firewall) for all workers and only allow the `federation_sender`'s to contact the
-outside world.
-
-```yaml
-instance_map:
-    main:
-        host: localhost
-        port: 8030
-    federation_sender1:
-        host: localhost
-        port: 8034
-
-outbound_federation_restricted_to:
-  - federation_sender1
-
-worker_replication_secret: "secret_secret"
-```
-
 #### Background tasks
 
 There is also support for moving background tasks to a separate

flake.lock

@@ -8,11 +8,11 @@
         "pre-commit-hooks": "pre-commit-hooks"
       },
       "locked": {
-        "lastModified": 1690534632,
-        "narHash": "sha256-kOXS9x5y17VKliC7wZxyszAYrWdRl1JzggbQl0gyo94=",
+        "lastModified": 1683102061,
+        "narHash": "sha256-kOphT6V0uQUlFNBP3GBjs7DAU7fyZGGqCs9ue1gNY6E=",
         "owner": "cachix",
         "repo": "devenv",
-        "rev": "6568e7e485a46bbf32051e4d6347fa1fed8b2f25",
+        "rev": "ff1f29e41756553174d596cafe3a9fa77595100b",
         "type": "github"
       },
       "original": {
@@ -22,6 +22,27 @@
         "type": "github"
       }
     },
+    "fenix": {
+      "inputs": {
+        "nixpkgs": [
+          "nixpkgs"
+        ],
+        "rust-analyzer-src": "rust-analyzer-src"
+      },
+      "locked": {
+        "lastModified": 1682490133,
+        "narHash": "sha256-tR2Qx0uuk97WySpSSk4rGS/oH7xb5LykbjATcw1vw1I=",
+        "owner": "nix-community",
+        "repo": "fenix",
+        "rev": "4e9412753ab75ef0e038a5fe54a062fb44c27c6a",
+        "type": "github"
+      },
+      "original": {
+        "owner": "nix-community",
+        "repo": "fenix",
+        "type": "github"
+      }
+    },
     "flake-compat": {
       "flake": false,
       "locked": {
@@ -39,33 +60,12 @@
       }
     },
     "flake-utils": {
-      "inputs": {
-        "systems": "systems"
-      },
       "locked": {
-        "lastModified": 1685518550,
-        "narHash": "sha256-o2d0KcvaXzTrPRIo0kOLV0/QXHhDQ5DTi+OxcjO8xqY=",
+        "lastModified": 1667395993,
+        "narHash": "sha256-nuEHfE/LcWyuSWnS8t12N1wc105Qtau+/OdUAjtQ0rA=",
         "owner": "numtide",
         "repo": "flake-utils",
-        "rev": "a1720a10a6cfe8234c0e93907ffe81be440f4cef",
-        "type": "github"
-      },
-      "original": {
-        "owner": "numtide",
-        "repo": "flake-utils",
-        "type": "github"
-      }
-    },
-    "flake-utils_2": {
-      "inputs": {
-        "systems": "systems_2"
-      },
-      "locked": {
-        "lastModified": 1681202837,
-        "narHash": "sha256-H+Rh19JDwRtpVPAWp64F+rlEtxUWBAQW28eAi3SRSzg=",
-        "owner": "numtide",
-        "repo": "flake-utils",
-        "rev": "cfacdce06f30d2b68473a46042957675eebb3401",
+        "rev": "5aed5285a952e0b949eb3ba02c12fa4fcfef535f",
         "type": "github"
       },
       "original": {
@@ -170,27 +170,27 @@
     },
     "nixpkgs-stable": {
       "locked": {
-        "lastModified": 1685801374,
-        "narHash": "sha256-otaSUoFEMM+LjBI1XL/xGB5ao6IwnZOXc47qhIgJe8U=",
+        "lastModified": 1673800717,
+        "narHash": "sha256-SFHraUqLSu5cC6IxTprex/nTsI81ZQAtDvlBvGDWfnA=",
         "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "c37ca420157f4abc31e26f436c1145f8951ff373",
+        "rev": "2f9fd351ec37f5d479556cd48be4ca340da59b8f",
         "type": "github"
       },
       "original": {
         "owner": "NixOS",
-        "ref": "nixos-23.05",
+        "ref": "nixos-22.11",
         "repo": "nixpkgs",
         "type": "github"
       }
     },
     "nixpkgs_2": {
       "locked": {
-        "lastModified": 1690535733,
-        "narHash": "sha256-WgjUPscQOw3cB8yySDGlyzo6cZNihnRzUwE9kadv/5I=",
+        "lastModified": 1682519441,
+        "narHash": "sha256-Vsq/8NOtvW1AoC6shCBxRxZyMQ+LhvPuJT6ltbzuv+Y=",
         "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "8cacc05fbfffeaab910e8c2c9e2a7c6b32ce881a",
+        "rev": "7a32a141db568abde9bc389845949dc2a454dfd3",
         "type": "github"
       },
       "original": {
@@ -200,22 +200,6 @@
         "type": "github"
       }
     },
-    "nixpkgs_3": {
-      "locked": {
-        "lastModified": 1681358109,
-        "narHash": "sha256-eKyxW4OohHQx9Urxi7TQlFBTDWII+F+x2hklDOQPB50=",
-        "owner": "NixOS",
-        "repo": "nixpkgs",
-        "rev": "96ba1c52e54e74c3197f4d43026b3f3d92e83ff9",
-        "type": "github"
-      },
-      "original": {
-        "owner": "NixOS",
-        "ref": "nixpkgs-unstable",
-        "repo": "nixpkgs",
-        "type": "github"
-      }
-    },
     "pre-commit-hooks": {
       "inputs": {
         "flake-compat": [
@@ -231,11 +215,11 @@
         "nixpkgs-stable": "nixpkgs-stable"
       },
       "locked": {
-        "lastModified": 1688056373,
-        "narHash": "sha256-2+SDlNRTKsgo3LBRiMUcoEUb6sDViRNQhzJquZ4koOI=",
+        "lastModified": 1678376203,
+        "narHash": "sha256-3tyYGyC8h7fBwncLZy5nCUjTJPrHbmNwp47LlNLOHSM=",
         "owner": "cachix",
         "repo": "pre-commit-hooks.nix",
-        "rev": "5843cf069272d92b60c3ed9e55b7a8989c01d4c7",
+        "rev": "1a20b9708962096ec2481eeb2ddca29ed747770a",
         "type": "github"
       },
       "original": {
@@ -247,27 +231,25 @@
     "root": {
       "inputs": {
         "devenv": "devenv",
+        "fenix": "fenix",
         "nixpkgs": "nixpkgs_2",
-        "rust-overlay": "rust-overlay",
-        "systems": "systems_3"
+        "systems": "systems"
       }
     },
-    "rust-overlay": {
-      "inputs": {
-        "flake-utils": "flake-utils_2",
-        "nixpkgs": "nixpkgs_3"
-      },
+    "rust-analyzer-src": {
+      "flake": false,
       "locked": {
-        "lastModified": 1690510705,
-        "narHash": "sha256-6mjs3Gl9/xrseFh9iNcNq1u5yJ/MIoAmjoaG7SXZDIE=",
-        "owner": "oxalica",
-        "repo": "rust-overlay",
-        "rev": "851ae4c128905a62834d53ce7704ebc1ba481bea",
+        "lastModified": 1682426789,
+        "narHash": "sha256-UqnLmJESRZE0tTEaGbRAw05Hm19TWIPA+R3meqi5I4w=",
+        "owner": "rust-lang",
+        "repo": "rust-analyzer",
+        "rev": "943d2a8a1ca15e8b28a1f51f5a5c135e3728da04",
         "type": "github"
       },
       "original": {
-        "owner": "oxalica",
-        "repo": "rust-overlay",
+        "owner": "rust-lang",
+        "ref": "nightly",
+        "repo": "rust-analyzer",
         "type": "github"
       }
     },
@@ -285,36 +267,6 @@
         "repo": "default",
         "type": "github"
       }
-    },
-    "systems_2": {
-      "locked": {
-        "lastModified": 1681028828,
-        "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=",
-        "owner": "nix-systems",
-        "repo": "default",
-        "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e",
-        "type": "github"
-      },
-      "original": {
-        "owner": "nix-systems",
-        "repo": "default",
-        "type": "github"
-      }
-    },
-    "systems_3": {
-      "locked": {
-        "lastModified": 1681028828,
-        "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=",
-        "owner": "nix-systems",
-        "repo": "default",
-        "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e",
-        "type": "github"
-      },
-      "original": {
-        "owner": "nix-systems",
-        "repo": "default",
-        "type": "github"
-      }
     }
   },
   "root": "root",

flake.nix

@@ -39,27 +39,27 @@
 
 {
   inputs = {
-    # Use the master/unstable branch of nixpkgs. Used to fetch the latest
-    # available versions of packages.
+    # Use the master/unstable branch of nixpkgs. The latest stable, 22.11,
+    # does not contain 'perl536Packages.NetAsyncHTTP', needed by Sytest.
     nixpkgs.url = "github:NixOS/nixpkgs/master";
     # Output a development shell for x86_64/aarch64 Linux/Darwin (MacOS).
     systems.url = "github:nix-systems/default";
     # A development environment manager built on Nix. See https://devenv.sh.
     devenv.url = "github:cachix/devenv/main";
-    # Rust toolchain.
-    rust-overlay.url = "github:oxalica/rust-overlay";
+    # Rust toolchains and rust-analyzer nightly.
+    fenix = {
+      url = "github:nix-community/fenix";
+      inputs.nixpkgs.follows = "nixpkgs";
+    };
   };
 
-  outputs = { self, nixpkgs, devenv, systems, rust-overlay, ... } @ inputs:
+  outputs = { self, nixpkgs, devenv, systems, ... } @ inputs:
     let
       forEachSystem = nixpkgs.lib.genAttrs (import systems);
     in {
       devShells = forEachSystem (system:
         let
-          overlays = [ (import rust-overlay) ];
-          pkgs = import nixpkgs {
-            inherit system overlays;
-          };
+          pkgs = nixpkgs.legacyPackages.${system};
         in {
           # Everything is configured via devenv - a Nix module for creating declarative
           # developer environments. See https://devenv.sh/reference/options/ for a list
@@ -76,20 +76,6 @@
                 # Configure packages to install.
                 # Search for package names at https://search.nixos.org/packages?channel=unstable
                 packages = with pkgs; [
-                  # The rust toolchain and related tools.
-                  # This will install the "default" profile of rust components.
-                  # https://rust-lang.github.io/rustup/concepts/profiles.html
-                  #
-                  # NOTE: We currently need to set the Rust version unnecessarily high
-                  # in order to work around https://github.com/matrix-org/synapse/issues/15939
-                  (rust-bin.stable."1.70.0".default.override {
-                    # Additionally install the "rust-src" extension to allow diving into the
-                    # Rust source code in an IDE (rust-analyzer will also make use of it).
-                    extensions = [ "rust-src" ];
-                  })
-                  # The rust-analyzer language server implementation.
-                  rust-analyzer
-
                   # Native dependencies for running Synapse.
                   icu
                   libffi
@@ -138,11 +124,12 @@
                 # Install dependencies for the additional programming languages
                 # involved with Synapse development.
                 #
+                # * Rust is used for developing and running Synapse.
                 # * Golang is needed to run the Complement test suite.
                 # * Perl is needed to run the SyTest test suite.
-                # * Rust is used for developing and running Synapse.
-                #   It is installed manually with `packages` above.
                 languages.go.enable = true;
+                languages.rust.enable = true;
+                languages.rust.version = "stable";
                 languages.perl.enable = true;
 
                 # Postgres is needed to run Synapse with postgres support and
@@ -191,7 +178,7 @@
                   EOF
                 '';
                 # Start synapse when `devenv up` is run.
-                processes.synapse.exec = "poetry run python -m synapse.app.homeserver -c homeserver.yaml -c homeserver-config-overrides.d";
+                processes.synapse.exec = "poetry run python -m synapse.app.homeserver -c homeserver.yaml --config-directory homeserver-config-overrides.d";
 
                 # Define the perl modules we require to run SyTest.
                 #

mypy.ini

@@ -45,13 +45,6 @@ warn_unused_ignores = False
 disallow_untyped_defs = False
 disallow_incomplete_defs = False
 
-[mypy-synapse.util.manhole]
-# This module imports something from Twisted which has a bad annotation in Twisted trunk,
-# but is unannotated in Twisted's latest release. We want to type-ignore the problem 
-# in the twisted trunk job, even though it has no effect on normal mypy runs.
-warn_unused_ignores = False
-
-
 ;; Dependencies without annotations
 ;; Before ignoring a module, check to see if type stubs are available.
 ;; The `typeshed` project maintains stubs here: