Ensure persistent connections are always used.

Reduce the size of the HTTP connection pool for non-pushers.
2023-05-30 15:04:26 -04:00 · 2023-05-30 15:04:26 -04:00
276 changed files with 7197 additions and 10925 deletions
@@ -29,12 +29,11 @@ IS_PR = os.environ["GITHUB_REF"].startswith("refs/pull/")

 # First calculate the various trial jobs.
 #
-# For PRs, we only run each type of test with the oldest Python version supported (which
-# is Python 3.8 right now)
+# For each type of test we only run on Py3.7 on PRs

 trial_sqlite_tests = [
    {
-        "python-version": "3.8",
+        "python-version": "3.7",
        "database": "sqlite",
        "extras": "all",
    }
@@ -47,13 +46,13 @@ if not IS_PR:
            "database": "sqlite",
            "extras": "all",
        }
-        for version in ("3.9", "3.10", "3.11")
+        for version in ("3.8", "3.9", "3.10", "3.11")
    )


 trial_postgres_tests = [
    {
-        "python-version": "3.8",
+        "python-version": "3.7",
        "database": "postgres",
        "postgres-version": "11",
        "extras": "all",
@@ -72,7 +71,7 @@ if not IS_PR:

 trial_no_extra_tests = [
    {
-        "python-version": "3.8",
+        "python-version": "3.7",
        "database": "sqlite",
        "extras": "",
    }
@@ -134,6 +133,11 @@ if not IS_PR:
                "sytest-tag": "testing",
                "postgres": "postgres",
            },
+            {
+                "sytest-tag": "buster",
+                "postgres": "multi-postgres",
+                "workers": "workers",
+            },
        ]
    )

@@ -22,21 +22,7 @@ concurrency:
  cancel-in-progress: true

 jobs:
-  check_repo:
-    # Prevent this workflow from running on any fork of Synapse other than matrix-org/synapse, as it is
-    # only useful to the Synapse core team.
-    # All other workflow steps depend on this one, thus if 'should_run_workflow' is not 'true', the rest
-    # of the workflow will be skipped as well.
-    runs-on: ubuntu-latest
-    outputs:
-      should_run_workflow: ${{ steps.check_condition.outputs.should_run_workflow }}
-    steps:
-      - id: check_condition
-        run: echo "should_run_workflow=${{ github.repository == 'matrix-org/synapse' }}" >> "$GITHUB_OUTPUT"
-
  mypy:
-    needs: check_repo
-    if: needs.check_repo.outputs.should_run_workflow == 'true'
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v3
@@ -61,8 +47,6 @@ jobs:
        run: sed '/warn_unused_ignores = True/d' -i mypy.ini
      - run: poetry run mypy
  trial:
-    needs: check_repo
-    if: needs.check_repo.outputs.should_run_workflow == 'true'
    runs-on: ubuntu-latest
    strategy:
      matrix:
@@ -121,8 +105,6 @@ jobs:


  sytest:
-    needs: check_repo
-    if: needs.check_repo.outputs.should_run_workflow == 'true'
    runs-on: ubuntu-latest
    container:
      image: matrixdotorg/sytest-synapse:testing
@@ -174,8 +156,7 @@ jobs:


  complement:
-    needs: check_repo
-    if: "!failure() && !cancelled() && needs.check_repo.outputs.should_run_workflow == 'true'"
+    if: "${{ !failure() && !cancelled() }}"
    runs-on: ubuntu-latest

    strategy:
@@ -211,7 +192,7 @@ jobs:
  # Open an issue if the build fails, so we know about it.
  # Only do this if we're not experimenting with this action in a PR.
  open-issue:
-    if: "failure() && github.event_name != 'push' && github.event_name != 'pull_request' && needs.check_repo.outputs.should_run_workflow == 'true'"
+    if: "failure() && github.event_name != 'push' && github.event_name != 'pull_request'"
    needs:
      # TODO: should mypy be included here? It feels more brittle than the others.
      - mypy
@@ -34,7 +34,6 @@ jobs:
      - id: set-distros
        run: |
          # if we're running from a tag, get the full list of distros; otherwise just use debian:sid
-          # NOTE: inside the actual Dockerfile-dhvirtualenv, the image name is expanded into its full image path
          dists='["debian:sid"]'
          if [[ $GITHUB_REF == refs/tags/* ]]; then
              dists=$(scripts-dev/build_debian_packages.py --show-dists-json)
@@ -144,7 +143,7 @@ jobs:

      - name: Only build a single wheel on PR
        if: startsWith(github.ref, 'refs/pull/')
-        run: echo "CIBW_BUILD="cp38-manylinux_${{ matrix.arch }}"" >> $GITHUB_ENV
+        run: echo "CIBW_BUILD="cp37-manylinux_${{ matrix.arch }}"" >> $GITHUB_ENV

      - name: Build wheels
        run: python -m cibuildwheel --output-dir wheelhouse
@@ -35,7 +35,7 @@ jobs:
    steps:
      - uses: actions/checkout@v3
      - name: Install Rust
-        uses: dtolnay/rust-toolchain@1.60.0
+        uses: dtolnay/rust-toolchain@1.58.1
      - uses: Swatinem/rust-cache@v2
      - uses: matrix-org/setup-python-poetry@v1
        with:
@@ -92,10 +92,6 @@ jobs:
      - name: Checkout repository
        uses: actions/checkout@v3

-      - name: Install Rust
-        uses: dtolnay/rust-toolchain@1.60.0
-      - uses: Swatinem/rust-cache@v2
-
      - name: Setup Poetry
        uses: matrix-org/setup-python-poetry@v1
        with:
@@ -107,6 +103,10 @@ jobs:
          # To make CI green, err towards caution and install the project.
          install-project: "true"

+      - name: Install Rust
+        uses: dtolnay/rust-toolchain@1.58.1
+      - uses: Swatinem/rust-cache@v2
+
      # Cribbed from
      # https://github.com/AustinScola/mypy-cache-github-action/blob/85ea4f2972abed39b33bd02c36e341b28ca59213/src/restore.ts#L10-L17
      - name: Restore/persist mypy's cache
@@ -150,7 +150,7 @@ jobs:
        with:
          ref: ${{ github.event.pull_request.head.sha }}
      - name: Install Rust
-        uses: dtolnay/rust-toolchain@1.60.0
+        uses: dtolnay/rust-toolchain@1.58.1
      - uses: Swatinem/rust-cache@v2
      - uses: matrix-org/setup-python-poetry@v1
        with:
@@ -167,7 +167,7 @@ jobs:
      - uses: actions/checkout@v3

      - name: Install Rust
-        uses: dtolnay/rust-toolchain@1.60.0
+        uses: dtolnay/rust-toolchain@1.58.1
        with:
            components: clippy
      - uses: Swatinem/rust-cache@v2
@@ -268,7 +268,7 @@ jobs:
            postgres:${{ matrix.job.postgres-version }}

      - name: Install Rust
-        uses: dtolnay/rust-toolchain@1.60.0
+        uses: dtolnay/rust-toolchain@1.58.1
      - uses: Swatinem/rust-cache@v2

      - uses: matrix-org/setup-python-poetry@v1
@@ -308,7 +308,7 @@ jobs:
      - uses: actions/checkout@v3

      - name: Install Rust
-        uses: dtolnay/rust-toolchain@1.60.0
+        uses: dtolnay/rust-toolchain@1.58.1
      - uses: Swatinem/rust-cache@v2

      # There aren't wheels for some of the older deps, so we need to install
@@ -320,7 +320,7 @@ jobs:

      - uses: actions/setup-python@v4
        with:
-          python-version: '3.8'
+          python-version: '3.7'

      - name: Prepare old deps
        if: steps.cache-poetry-old-deps.outputs.cache-hit != 'true'
@@ -362,7 +362,7 @@ jobs:
    runs-on: ubuntu-latest
    strategy:
      matrix:
-        python-version: ["pypy-3.8"]
+        python-version: ["pypy-3.7"]
        extras: ["all"]

    steps:
@@ -399,8 +399,8 @@ jobs:
      env:
        SYTEST_BRANCH: ${{ github.head_ref }}
        POSTGRES: ${{ matrix.job.postgres && 1}}
-        MULTI_POSTGRES: ${{ (matrix.job.postgres == 'multi-postgres') || '' }}
-        ASYNCIO_REACTOR: ${{ (matrix.job.reactor == 'asyncio') || '' }}
+        MULTI_POSTGRES: ${{ (matrix.job.postgres == 'multi-postgres') && 1}}
+        ASYNCIO_REACTOR: ${{ (matrix.job.reactor == 'asyncio') && 1 }}
        WORKERS: ${{ matrix.job.workers && 1 }}
        BLACKLIST: ${{ matrix.job.workers && 'synapse-blacklist-with-workers' }}
        TOP: ${{ github.workspace }}
@@ -416,7 +416,7 @@ jobs:
        run: cat sytest-blacklist .ci/worker-blacklist > synapse-blacklist-with-workers

      - name: Install Rust
-        uses: dtolnay/rust-toolchain@1.60.0
+        uses: dtolnay/rust-toolchain@1.58.1
      - uses: Swatinem/rust-cache@v2

      - name: Run SyTest
@@ -477,7 +477,7 @@ jobs:
    strategy:
      matrix:
        include:
-          - python-version: "3.8"
+          - python-version: "3.7"
            postgres-version: "11"

          - python-version: "3.11"
@@ -556,7 +556,7 @@ jobs:
          path: synapse

      - name: Install Rust
-        uses: dtolnay/rust-toolchain@1.60.0
+        uses: dtolnay/rust-toolchain@1.58.1
      - uses: Swatinem/rust-cache@v2

      - uses: actions/setup-go@v4
@@ -584,7 +584,7 @@ jobs:
      - uses: actions/checkout@v3

      - name: Install Rust
-        uses: dtolnay/rust-toolchain@1.60.0
+        uses: dtolnay/rust-toolchain@1.58.1
      - uses: Swatinem/rust-cache@v2

      - run: cargo test
@@ -18,22 +18,7 @@ concurrency:
  cancel-in-progress: true

 jobs:
-  check_repo:
-    # Prevent this workflow from running on any fork of Synapse other than matrix-org/synapse, as it is
-    # only useful to the Synapse core team.
-    # All other workflow steps depend on this one, thus if 'should_run_workflow' is not 'true', the rest
-    # of the workflow will be skipped as well.
-    if: github.repository == 'matrix-org/synapse'
-    runs-on: ubuntu-latest
-    outputs:
-      should_run_workflow: ${{ steps.check_condition.outputs.should_run_workflow }}
-    steps:
-      - id: check_condition
-        run: echo "should_run_workflow=${{ github.repository == 'matrix-org/synapse' }}" >> "$GITHUB_OUTPUT"
-
  mypy:
-    needs: check_repo
-    if: needs.check_repo.outputs.should_run_workflow == 'true'
    runs-on: ubuntu-latest

    steps:
@@ -56,8 +41,6 @@ jobs:
      - run: poetry run mypy

  trial:
-    needs: check_repo
-    if: needs.check_repo.outputs.should_run_workflow == 'true'
    runs-on: ubuntu-latest

    steps:
@@ -92,15 +75,9 @@ jobs:
          || true

  sytest:
-    needs: check_repo
-    if: needs.check_repo.outputs.should_run_workflow == 'true'
    runs-on: ubuntu-latest
    container:
-      # We're using ubuntu:focal because it uses Python 3.8 which is our minimum supported Python version.
-      # This job is a canary to warn us about unreleased twisted changes that would cause problems for us if
-      # they were to be released immediately. For simplicity's sake (and to save CI runners) we use the oldest
-      # version, assuming that any incompatibilities on newer versions would also be present on the oldest.
-      image: matrixdotorg/sytest-synapse:focal
+      image: matrixdotorg/sytest-synapse:buster
      volumes:
        - ${{ github.workspace }}:/src

@@ -142,8 +119,7 @@ jobs:
            /logs/**/*.log*

  complement:
-    needs: check_repo
-    if: "!failure() && !cancelled() && needs.check_repo.outputs.should_run_workflow == 'true'"
+    if: "${{ !failure() && !cancelled() }}"
    runs-on: ubuntu-latest

    strategy:
@@ -190,7 +166,7 @@ jobs:

  # open an issue if the build fails, so we know about it.
  open-issue:
-    if: failure() && needs.check_repo.outputs.should_run_workflow == 'true'
+    if: failure()
    needs:
      - mypy
      - trial
@@ -34,7 +34,6 @@ __pycache__/
 /logs
 /media_store/
 /uploads
-/homeserver-config-overrides.d

 # For direnv users
 /.envrc
@@ -4,18 +4,18 @@ version = 3

 [[package]]
 name = "aho-corasick"
-version = "1.0.2"
+version = "0.7.19"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "43f6cb1bf222025340178f382c426f13757b2960e89779dfcb319c32542a5a41"
+checksum = "b4f55bd91a0978cbfd91c457a164bab8b4001c833b7f323132c0a4e1922dd44e"
 dependencies = [
 "memchr",
 ]

 [[package]]
 name = "anyhow"
-version = "1.0.72"
+version = "1.0.71"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3b13c32d80ecc7ab747b80c3784bce54ee8a7a0cc4fbda9bf4cda2cf6fe90854"
+checksum = "9c7d0618f0e0b7e8ff11427422b64564d5fb0be1940354bfe2e0529b18a9d9b8"

 [[package]]
 name = "arc-swap"
@@ -132,9 +132,9 @@ dependencies = [

 [[package]]
 name = "log"
-version = "0.4.19"
+version = "0.4.18"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b06a4cde4c0f271a446782e3eff8de789548ce57dbc8eca9292c27f4a42004b4"
+checksum = "518ef76f2f87365916b142844c16d8fefd85039bc5699050210a7778ee1cd1de"

 [[package]]
 name = "memchr"
@@ -182,9 +182,9 @@ dependencies = [

 [[package]]
 name = "proc-macro2"
-version = "1.0.64"
+version = "1.0.52"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "78803b62cbf1f46fde80d7c0e803111524b9877184cfe7c3033659490ac7a7da"
+checksum = "1d0e1ae9e836cc3beddd63db0df682593d7e2d3d891ae8c9083d2113e1744224"
 dependencies = [
 "unicode-ident",
 ]
@@ -229,9 +229,9 @@ dependencies = [

 [[package]]
 name = "pyo3-log"
-version = "0.8.3"
+version = "0.8.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f47b0777feb17f61eea78667d61103758b243a871edc09a7786500a50467b605"
+checksum = "f9c8b57fe71fb5dcf38970ebedc2b1531cf1c14b1b9b4c560a182a57e115575c"
 dependencies = [
 "arc-swap",
 "log",
@@ -273,9 +273,9 @@ dependencies = [

 [[package]]
 name = "quote"
-version = "1.0.29"
+version = "1.0.26"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "573015e8ab27661678357f27dc26460738fd2b6c86e46f386fde94cb5d913105"
+checksum = "4424af4bf778aae2051a77b60283332f386554255d722233d09fbfc7e30da2fc"
 dependencies = [
 "proc-macro2",
 ]
@@ -291,21 +291,9 @@ dependencies = [

 [[package]]
 name = "regex"
-version = "1.9.1"
+version = "1.7.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b2eae68fc220f7cf2532e4494aded17545fce192d59cd996e0fe7887f4ceb575"
-dependencies = [
- "aho-corasick",
- "memchr",
- "regex-automata",
- "regex-syntax",
-]
-
-[[package]]
-name = "regex-automata"
-version = "0.3.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "83d3daa6976cffb758ec878f108ba0e062a45b2d6ca3a2cca965338855476caf"
+checksum = "8b1f693b24f6ac912f4893ef08244d70b6067480d2f1a46e950c9691e6749d1d"
 dependencies = [
 "aho-corasick",
 "memchr",
@@ -314,9 +302,9 @@ dependencies = [

 [[package]]
 name = "regex-syntax"
-version = "0.7.3"
+version = "0.6.29"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2ab07dc67230e4a4718e70fd5c20055a4334b121f1f9db8fe63ef39ce9b8c846"
+checksum = "f162c6dd7b008981e4d40210aca20b4bd0f9b60ca9271061b07f78537722f2e1"

 [[package]]
 name = "ryu"
@@ -332,29 +320,29 @@ checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd"

 [[package]]
 name = "serde"
-version = "1.0.171"
+version = "1.0.163"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "30e27d1e4fd7659406c492fd6cfaf2066ba8773de45ca75e855590f856dc34a9"
+checksum = "2113ab51b87a539ae008b5c6c02dc020ffa39afd2d83cffcb3f4eb2722cebec2"
 dependencies = [
 "serde_derive",
 ]

 [[package]]
 name = "serde_derive"
-version = "1.0.171"
+version = "1.0.163"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "389894603bd18c46fa56231694f8d827779c0951a667087194cf9de94ed24682"
+checksum = "8c805777e3930c8883389c602315a24224bcc738b63905ef87cd1420353ea93e"
 dependencies = [
 "proc-macro2",
 "quote",
- "syn 2.0.25",
+ "syn 2.0.10",
 ]

 [[package]]
 name = "serde_json"
-version = "1.0.103"
+version = "1.0.96"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d03b412469450d4404fe8499a268edd7f8b79fecb074b0d812ad64ca21f4031b"
+checksum = "057d394a50403bcac12672b2b18fb387ab6d289d957dab67dd201875391e52f1"
 dependencies = [
 "itoa",
 "ryu",
@@ -386,9 +374,9 @@ dependencies = [

 [[package]]
 name = "syn"
-version = "2.0.25"
+version = "2.0.10"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "15e3fc8c0c74267e2df136e5e5fb656a464158aa57624053375eb9c8c6e25ae2"
+checksum = "5aad1363ed6d37b84299588d62d3a7d95b5a5c2d9aad5c85609fda12afaa1f40"
 dependencies = [
 "proc-macro2",
 "quote",
@@ -3,4 +3,3 @@

 [workspace]
 members = ["rust"]
-resolver = "2"
@@ -0,0 +1 @@
+Experimental [MSC3861](https://github.com/matrix-org/matrix-spec-proposals/pull/3861) support: delegate auth to an OIDC provider.
@@ -0,0 +1 @@
+Remove some unused code.
@@ -1 +0,0 @@
-Add Unix Socket support for HTTP Replication Listeners. Document and provide usage instructions for utilizing Unix sockets in Synapse. Contributed by Jason Little.
@@ -1 +0,0 @@
-Fix long-standing bug where remote invites weren't correctly pushed.
@@ -1 +0,0 @@
-Mark `get_user_in_directory` private since it is only used in tests. Also remove the cache from it.
@@ -1 +0,0 @@
-Fix background schema updates failing over a large upgrade gap.
@@ -1 +0,0 @@
-Document which Python version runs on a given Linux distribution so we can more easily clean up later.
@@ -1 +0,0 @@
-Allow `+` in Matrix IDs, per [MSC4009](https://github.com/matrix-org/matrix-spec-proposals/pull/4009).
@@ -1 +0,0 @@
-Support room version 11 from [MSC3820](https://github.com/matrix-org/matrix-spec-proposals/pull/3820).
@@ -1 +0,0 @@
-Allow configuring the set of workers to proxy outbound federation traffic through via `outbound_federation_restricted_to`.
@@ -1 +0,0 @@
-Better clarify how to run a worker instance (pass both configs).
@@ -1 +0,0 @@
-Add details to warning in log when we fail to fetch an alias.
@@ -1 +0,0 @@
-Add Unix Socket support for HTTP Replication Listeners. Document and provide usage instructions for utilizing Unix sockets in Synapse. Contributed by Jason Little.
@@ -1 +0,0 @@
-Fix a bug introduced in 1.86.0 where Synapse starting with an empty `experimental_features` configuration setting.
@@ -1 +0,0 @@
-Remove unneeded `__init__`.
@@ -1 +0,0 @@
-Remove support for calling the `/register` endpoint with an unspecced `user` property for application services.
@@ -1 +0,0 @@
-Fix bug with read/write lock implementation. This is currently unused so has no observable effects.
@@ -1 +0,0 @@
-Improve the documentation for the login as a user admin API.
@@ -1 +0,0 @@
-Unbreak the nix development environment by pinning the Rust version to 1.70.0.
@@ -1 +0,0 @@
-Update presence metrics to differentiate remote vs local users.
@@ -1 +0,0 @@
-Fix bug with read/write lock implementation. This is currently unused so has no observable effects.
@@ -1 +0,0 @@
-Ensure a long state res does not starve CPU by occasionally yielding to the reactor.
@@ -1 +0,0 @@
-Reduce the amount of state we pull out.
@@ -1 +0,0 @@
-Allow configuring the set of workers to proxy outbound federation traffic through via `outbound_federation_restricted_to`.
@@ -1 +0,0 @@
-Speed up calculation of remote hosts.
@@ -29,7 +29,7 @@
        "level": "error"
      },
      {
-        "line": "my-matrix-server-federation-sender-1 | 2023-01-25 20:56:20,995 - synapse.http.matrixfederationclient - 709 - WARNING - federation_transaction_transmission_loop-3 - {PUT-O-3} [example.com] Request failed: PUT matrix-federation://example.com/_matrix/federation/v1/send/1674680155797: HttpResponseException('403: Forbidden')",
+        "line": "my-matrix-server-federation-sender-1 | 2023-01-25 20:56:20,995 - synapse.http.matrixfederationclient - 709 - WARNING - federation_transaction_transmission_loop-3 - {PUT-O-3} [example.com] Request failed: PUT matrix://example.com/_matrix/federation/v1/send/1674680155797: HttpResponseException('403: Forbidden')",
        "level": "warning"
      },
      {
@@ -1,69 +1,3 @@
-matrix-synapse-py3 (1.88.0) stable; urgency=medium
-
-  * New Synapse release 1.88.0.
-
- -- Synapse Packaging team <packages@matrix.org>  Tue, 18 Jul 2023 13:59:28 +0100
-
-matrix-synapse-py3 (1.88.0~rc1) stable; urgency=medium
-
-  * New Synapse release 1.88.0rc1.
-
- -- Synapse Packaging team <packages@matrix.org>  Tue, 11 Jul 2023 10:20:19 +0100
-
-matrix-synapse-py3 (1.87.0) stable; urgency=medium
-
-  * New Synapse release 1.87.0.
-
- -- Synapse Packaging team <packages@matrix.org>  Tue, 04 Jul 2023 16:24:00 +0100
-
-matrix-synapse-py3 (1.87.0~rc1) stable; urgency=medium
-
-  * New synapse release 1.87.0rc1.
-
- -- Synapse Packaging team <packages@matrix.org>  Tue, 27 Jun 2023 15:27:04 +0000
-
-matrix-synapse-py3 (1.86.0) stable; urgency=medium
-
-  * New Synapse release 1.86.0.
-
- -- Synapse Packaging team <packages@matrix.org>  Tue, 20 Jun 2023 17:22:46 +0200
-
-matrix-synapse-py3 (1.86.0~rc2) stable; urgency=medium
-
-  * New Synapse release 1.86.0rc2.
-
- -- Synapse Packaging team <packages@matrix.org>  Wed, 14 Jun 2023 12:16:27 +0200
-
-matrix-synapse-py3 (1.86.0~rc1) stable; urgency=medium
-
-  * New Synapse release 1.86.0rc1.
-
- -- Synapse Packaging team <packages@matrix.org>  Tue, 13 Jun 2023 14:30:45 +0200
-
-matrix-synapse-py3 (1.85.2) stable; urgency=medium
-
-  * New Synapse release 1.85.2.
-
- -- Synapse Packaging team <packages@matrix.org>  Thu, 08 Jun 2023 13:04:18 +0100
-
-matrix-synapse-py3 (1.85.1) stable; urgency=medium
-
-  * New Synapse release 1.85.1.
-
- -- Synapse Packaging team <packages@matrix.org>  Wed, 07 Jun 2023 10:51:12 +0100
-
-matrix-synapse-py3 (1.85.0) stable; urgency=medium
-
-  * New Synapse release 1.85.0.
-
- -- Synapse Packaging team <packages@matrix.org>  Tue, 06 Jun 2023 09:39:29 +0100
-
-matrix-synapse-py3 (1.85.0~rc2) stable; urgency=medium
-
-  * New Synapse release 1.85.0rc2.
-
- -- Synapse Packaging team <packages@matrix.org>  Thu, 01 Jun 2023 09:16:18 -0700
-
 matrix-synapse-py3 (1.85.0~rc1) stable; urgency=medium

  * New Synapse release 1.85.0rc1.
@@ -27,7 +27,7 @@ ARG PYTHON_VERSION=3.11
 ###
 # We hardcode the use of Debian bullseye here because this could change upstream
 # and other Dockerfiles used for testing are expecting bullseye.
-FROM docker.io/library/python:${PYTHON_VERSION}-slim-bullseye as requirements
+FROM docker.io/python:${PYTHON_VERSION}-slim-bullseye as requirements

 # RUN --mount is specific to buildkit and is documented at
 # https://github.com/moby/buildkit/blob/master/frontend/dockerfile/docs/syntax.md#build-mounts-run---mount.
@@ -87,7 +87,7 @@ RUN if [ -z "$TEST_ONLY_IGNORE_POETRY_LOCKFILE" ]; then \
 ###
 ### Stage 1: builder
 ###
-FROM docker.io/library/python:${PYTHON_VERSION}-slim-bullseye as builder
+FROM docker.io/python:${PYTHON_VERSION}-slim-bullseye as builder

 # install the OS build deps
 RUN \
@@ -158,7 +158,7 @@ RUN --mount=type=cache,target=/synapse/target,sharing=locked \
 ### Stage 2: runtime
 ###

-FROM docker.io/library/python:${PYTHON_VERSION}-slim-bullseye
+FROM docker.io/python:${PYTHON_VERSION}-slim-bullseye

 LABEL org.opencontainers.image.url='https://matrix.org/docs/projects/server/synapse'
 LABEL org.opencontainers.image.documentation='https://github.com/matrix-org/synapse/blob/master/docker/README.md'
@@ -24,16 +24,16 @@ ARG distro=""
 # https://launchpad.net/~jyrki-pulliainen/+archive/ubuntu/dh-virtualenv, but
 # it's not obviously easier to use that than to build our own.)

-FROM docker.io/library/${distro} as builder
+FROM ${distro} as builder

 RUN apt-get update -qq -o Acquire::Languages=none
 RUN env DEBIAN_FRONTEND=noninteractive apt-get install \
-    -yqq --no-install-recommends \
-    build-essential \
-    ca-certificates \
-    devscripts \
-    equivs \
-    wget
+        -yqq --no-install-recommends \
+        build-essential \
+        ca-certificates \
+        devscripts \
+        equivs \
+        wget

 # fetch and unpack the package
 # We are temporarily using a fork of dh-virtualenv due to an incompatibility with Python 3.11, which ships with
@@ -55,36 +55,40 @@ RUN cd /dh-virtualenv && DEB_BUILD_OPTIONS=nodoc dpkg-buildpackage -us -uc -b
 ###
 ### Stage 1
 ###
-FROM docker.io/library/${distro}
+FROM ${distro}

 # Get the distro we want to pull from as a dynamic build variable
 # (We need to define it in each build stage)
 ARG distro=""
 ENV distro ${distro}

+# Python < 3.7 assumes LANG="C" means ASCII-only and throws on printing unicode
+# http://bugs.python.org/issue19846
+ENV LANG C.UTF-8
+
 # Install the build dependencies
 #
 # NB: keep this list in sync with the list of build-deps in debian/control
 # TODO: it would be nice to do that automatically.
 RUN apt-get update -qq -o Acquire::Languages=none \
    && env DEBIAN_FRONTEND=noninteractive apt-get install \
-    -yqq --no-install-recommends -o Dpkg::Options::=--force-unsafe-io \
-    build-essential \
-    curl \
-    debhelper \
-    devscripts \
-    libsystemd-dev \
-    lsb-release \
-    pkg-config \
-    python3-dev \
-    python3-pip \
-    python3-setuptools \
-    python3-venv \
-    sqlite3 \
-    libpq-dev \
-    libicu-dev \
-    pkg-config \
-    xmlsec1
+        -yqq --no-install-recommends -o Dpkg::Options::=--force-unsafe-io \
+        build-essential \
+        curl \
+        debhelper \
+        devscripts \
+        libsystemd-dev \
+        lsb-release \
+        pkg-config \
+        python3-dev \
+        python3-pip \
+        python3-setuptools \
+        python3-venv \
+        sqlite3 \
+        libpq-dev \
+        libicu-dev \
+        pkg-config \
+        xmlsec1

 # Install rust and ensure it's in the PATH
 ENV RUSTUP_HOME=/rust
@@ -7,7 +7,7 @@ ARG FROM=matrixdotorg/synapse:$SYNAPSE_VERSION
 # target image. For repeated rebuilds, this is much faster than apt installing
 # each time.

-FROM docker.io/library/debian:bullseye-slim AS deps_base
+FROM debian:bullseye-slim AS deps_base
    RUN \
       --mount=type=cache,target=/var/cache/apt,sharing=locked \
       --mount=type=cache,target=/var/lib/apt,sharing=locked \
@@ -21,7 +21,7 @@ FROM docker.io/library/debian:bullseye-slim AS deps_base
 # which makes it much easier to copy (but we need to make sure we use an image
 # based on the same debian version as the synapse image, to make sure we get
 # the expected version of libc.
-FROM docker.io/library/redis:7-bullseye AS redis_base
+FROM redis:6-bullseye AS redis_base

 # now build the final image, based on the the regular Synapse docker image
 FROM $FROM
@@ -73,8 +73,7 @@ The following environment variables are supported in `generate` mode:
  will log sensitive information such as access tokens.
  This should not be needed unless you are a developer attempting to debug something
  particularly tricky.
-* `SYNAPSE_LOG_TESTING`: if set, Synapse will log additional information useful
-  for testing.
+

 ## Postgres

@@ -7,7 +7,6 @@
 # https://github.com/matrix-org/synapse/blob/develop/docker/README-testing.md#testing-with-postgresql-and-single-or-multi-process-synapse

 ARG SYNAPSE_VERSION=latest
-# This is an intermediate image, to be built locally (not pulled from a registry).
 ARG FROM=matrixdotorg/synapse-workers:$SYNAPSE_VERSION

 FROM $FROM
@@ -20,8 +19,8 @@ FROM $FROM
    # the same debian version as Synapse's docker image (so the versions of the
    # shared libraries match).
    RUN adduser --system --uid 999 postgres --home /var/lib/postgresql
-    COPY --from=docker.io/library/postgres:13-bullseye /usr/lib/postgresql /usr/lib/postgresql
-    COPY --from=docker.io/library/postgres:13-bullseye /usr/share/postgresql /usr/share/postgresql
+    COPY --from=postgres:13-bullseye /usr/lib/postgresql /usr/lib/postgresql
+    COPY --from=postgres:13-bullseye /usr/share/postgresql /usr/share/postgresql
    RUN mkdir /var/run/postgresql && chown postgres /var/run/postgresql
    ENV PATH="${PATH}:/usr/lib/postgresql/13/bin"
    ENV PGDATA=/var/lib/postgresql/data
@@ -92,6 +92,8 @@ allow_device_name_lookup_over_federation: true
 ## Experimental Features ##

 experimental_features:
+  # Enable history backfilling support
+  msc2716_enabled: true
  # client-side support for partial state in /send_join responses
  faster_joins: true
  # Enable support for polls
@@ -35,11 +35,7 @@ server {

    # Send all other traffic to the main process
    location ~* ^(\\/_matrix|\\/_synapse) {
-{% if using_unix_sockets %}
-        proxy_pass http://unix:/run/main_public.sock;
-{% else %}
        proxy_pass http://localhost:8080;
-{% endif %}
        proxy_set_header X-Forwarded-For $remote_addr;
        proxy_set_header X-Forwarded-Proto $scheme;
        proxy_set_header Host $host;
@@ -6,9 +6,6 @@
 {% if enable_redis %}
 redis:
    enabled: true
-    {% if using_unix_sockets %}
-    path: /tmp/redis.sock
-    {% endif %}
 {% endif %}

 {% if appservice_registrations is not none %}
@@ -19,11 +19,7 @@ username=www-data
 autorestart=true

 [program:redis]
-{% if using_unix_sockets %}
-command=/usr/local/bin/prefix-log /usr/local/bin/redis-server --unixsocket /tmp/redis.sock
-{% else %}
 command=/usr/local/bin/prefix-log /usr/local/bin/redis-server
-{% endif %}
 priority=1
 stdout_logfile=/dev/stdout
 stdout_logfile_maxbytes=0
@@ -8,11 +8,7 @@ worker_name: "{{ name }}"

 worker_listeners:
  - type: http
-{% if using_unix_sockets %}
-    path: "/run/worker.{{ port }}"
-{% else %}
    port: {{ port }}
-{% endif %}
 {% if listener_resources %}
    resources:
      - names:
@@ -36,17 +36,12 @@ listeners:

  # Allow configuring in case we want to reverse proxy 8008
  # using another process in the same container
-{% if SYNAPSE_USE_UNIX_SOCKET %}
-  # Unix sockets don't care about TLS or IP addresses or ports
-  - path: '/run/main_public.sock'
-    type: http
-{% else %}
  - port: {{ SYNAPSE_HTTP_PORT or 8008 }}
    tls: false
    bind_addresses: ['::']
    type: http
    x_forwarded: false
-{% endif %}
+
    resources:
      - names: [client]
        compress: true
@@ -62,11 +57,8 @@ database:
    user: "{{ POSTGRES_USER or "synapse" }}"
    password: "{{ POSTGRES_PASSWORD }}"
    database: "{{ POSTGRES_DB or "synapse" }}"
-{% if not SYNAPSE_USE_UNIX_SOCKET %}
-{# Synapse will use a default unix socket for Postgres when host/port is not specified (behavior from `psycopg2`). #}
    host: "{{ POSTGRES_HOST or "db" }}"
    port: "{{ POSTGRES_PORT or "5432" }}"
-{% endif %}
    cp_min: 5
    cp_max: 10
 {% else %}
@@ -49,35 +49,17 @@ handlers:
    class: logging.StreamHandler
    formatter: precise

+{% if not SYNAPSE_LOG_SENSITIVE %}
+{#
+  If SYNAPSE_LOG_SENSITIVE is unset, then override synapse.storage.SQL to INFO
+  so that DEBUG entries (containing sensitive information) are not emitted.
+#}
 loggers:
-    # This is just here so we can leave `loggers` in the config regardless of whether
-    # we configure other loggers below (avoid empty yaml dict error).
-    _placeholder:
-        level: "INFO"
-
-    {% if not SYNAPSE_LOG_SENSITIVE %}
-    {#
-      If SYNAPSE_LOG_SENSITIVE is unset, then override synapse.storage.SQL to INFO
-      so that DEBUG entries (containing sensitive information) are not emitted.
-    #}
    synapse.storage.SQL:
        # beware: increasing this to DEBUG will make synapse log sensitive
        # information such as access tokens.
        level: INFO
-    {% endif %}
-
-    {% if SYNAPSE_LOG_TESTING %}
-    {#
-      If Synapse is under test, log a few more useful things for a developer
-      attempting to debug something particularly tricky.
-
-      With `synapse.visibility.filtered_event_debug`, it logs when events are (maybe
-      unexpectedly) filtered out of responses in tests. It's just nice to be able to
-      look at the CI log and figure out why an event isn't being returned.
-    #}
-    synapse.visibility.filtered_event_debug:
-        level: DEBUG
-    {% endif %}
+{% endif %}

 root:
    level: {{ SYNAPSE_LOG_LEVEL or "INFO" }}
@@ -40,8 +40,6 @@
 #         log level. INFO is the default.
 #   * SYNAPSE_LOG_SENSITIVE: If unset, SQL and SQL values won't be logged,
 #         regardless of the SYNAPSE_LOG_LEVEL setting.
-#   * SYNAPSE_LOG_TESTING: if set, Synapse will log additional information useful
-#     for testing.
 #
 # NOTE: According to Complement's ENTRYPOINT expectations for a homeserver image (as defined
 # in the project's README), this script may be run multiple times, and functionality should
@@ -74,9 +72,6 @@ MAIN_PROCESS_HTTP_LISTENER_PORT = 8080
 MAIN_PROCESS_INSTANCE_NAME = "main"
 MAIN_PROCESS_LOCALHOST_ADDRESS = "127.0.0.1"
 MAIN_PROCESS_REPLICATION_PORT = 9093
-# Obviously, these would only be used with the UNIX socket option
-MAIN_PROCESS_UNIX_SOCKET_PUBLIC_PATH = "/run/main_public.sock"
-MAIN_PROCESS_UNIX_SOCKET_PRIVATE_PATH = "/run/main_private.sock"

 # A simple name used as a placeholder in the WORKERS_CONFIG below. This will be replaced
 # during processing with the name of the worker.
@@ -247,6 +242,7 @@ WORKERS_CONFIG: Dict[str, Dict[str, Any]] = {
            "^/_matrix/client/(api/v1|r0|v3|unstable)/join/",
            "^/_matrix/client/(api/v1|r0|v3|unstable)/knock/",
            "^/_matrix/client/(api/v1|r0|v3|unstable)/profile/",
+            "^/_matrix/client/(v1|unstable/org.matrix.msc2716)/rooms/.*/batch_send",
        ],
        "shared_extra_conf": {},
        "worker_extra_conf": "",
@@ -410,15 +406,11 @@ def add_worker_roles_to_shared_config(
        )

        # Map of stream writer instance names to host/ports combos
-        if os.environ.get("SYNAPSE_USE_UNIX_SOCKET", False):
-            instance_map[worker_name] = {
-                "path": f"/run/worker.{worker_port}",
-            }
-        else:
-            instance_map[worker_name] = {
-                "host": "localhost",
-                "port": worker_port,
-            }
+        instance_map[worker_name] = {
+            "host": "localhost",
+            "port": worker_port,
+        }
+
    # Update the list of stream writers. It's convenient that the name of the worker
    # type is the same as the stream to write. Iterate over the whole list in case there
    # is more than one.
@@ -430,15 +422,10 @@ def add_worker_roles_to_shared_config(

            # Map of stream writer instance names to host/ports combos
            # For now, all stream writers need http replication ports
-            if os.environ.get("SYNAPSE_USE_UNIX_SOCKET", False):
-                instance_map[worker_name] = {
-                    "path": f"/run/worker.{worker_port}",
-                }
-            else:
-                instance_map[worker_name] = {
-                    "host": "localhost",
-                    "port": worker_port,
-                }
+            instance_map[worker_name] = {
+                "host": "localhost",
+                "port": worker_port,
+            }


 def merge_worker_template_configs(
@@ -730,29 +717,17 @@ def generate_worker_files(
    # Note that yaml cares about indentation, so care should be taken to insert lines
    # into files at the correct indentation below.

-    # Convenience helper for if using unix sockets instead of host:port
-    using_unix_sockets = environ.get("SYNAPSE_USE_UNIX_SOCKET", False)
    # First read the original config file and extract the listeners block. Then we'll
    # add another listener for replication. Later we'll write out the result to the
    # shared config file.
-    listeners: List[Any]
-    if using_unix_sockets:
-        listeners = [
-            {
-                "path": MAIN_PROCESS_UNIX_SOCKET_PRIVATE_PATH,
-                "type": "http",
-                "resources": [{"names": ["replication"]}],
-            }
-        ]
-    else:
-        listeners = [
-            {
-                "port": MAIN_PROCESS_REPLICATION_PORT,
-                "bind_address": MAIN_PROCESS_LOCALHOST_ADDRESS,
-                "type": "http",
-                "resources": [{"names": ["replication"]}],
-            }
-        ]
+    listeners = [
+        {
+            "port": MAIN_PROCESS_REPLICATION_PORT,
+            "bind_address": MAIN_PROCESS_LOCALHOST_ADDRESS,
+            "type": "http",
+            "resources": [{"names": ["replication"]}],
+        }
+    ]
    with open(config_path) as file_stream:
        original_config = yaml.safe_load(file_stream)
        original_listeners = original_config.get("listeners")
@@ -793,17 +768,7 @@ def generate_worker_files(

    # A list of internal endpoints to healthcheck, starting with the main process
    # which exists even if no workers do.
-    # This list ends up being part of the command line to curl, (curl added support for
-    # Unix sockets in version 7.40).
-    if using_unix_sockets:
-        healthcheck_urls = [
-            f"--unix-socket {MAIN_PROCESS_UNIX_SOCKET_PUBLIC_PATH} "
-            # The scheme and hostname from the following URL are ignored.
-            # The only thing that matters is the path `/health`
-            "http://localhost/health"
-        ]
-    else:
-        healthcheck_urls = ["http://localhost:8080/health"]
+    healthcheck_urls = ["http://localhost:8080/health"]

    # Get the set of all worker types that we have configured
    all_worker_types_in_use = set(chain(*requested_worker_types.values()))
@@ -840,12 +805,8 @@ def generate_worker_files(
        # given worker_type needs to stay assigned and not be replaced.
        worker_config["shared_extra_conf"].update(shared_config)
        shared_config = worker_config["shared_extra_conf"]
-        if using_unix_sockets:
-            healthcheck_urls.append(
-                f"--unix-socket /run/worker.{worker_port} http://localhost/health"
-            )
-        else:
-            healthcheck_urls.append("http://localhost:%d/health" % (worker_port,))
+
+        healthcheck_urls.append("http://localhost:%d/health" % (worker_port,))

        # Update the shared config with sharding-related options if necessary
        add_worker_roles_to_shared_config(
@@ -864,7 +825,6 @@ def generate_worker_files(
            "/conf/workers/{name}.yaml".format(name=worker_name),
            **worker_config,
            worker_log_config_filepath=log_config_filepath,
-            using_unix_sockets=using_unix_sockets,
        )

        # Save this worker's port number to the correct nginx upstreams
@@ -885,13 +845,8 @@ def generate_worker_files(
    nginx_upstream_config = ""
    for upstream_worker_base_name, upstream_worker_ports in nginx_upstreams.items():
        body = ""
-        if using_unix_sockets:
-            for port in upstream_worker_ports:
-                body += f"    server unix:/run/worker.{port};\n"
-
-        else:
-            for port in upstream_worker_ports:
-                body += f"    server localhost:{port};\n"
+        for port in upstream_worker_ports:
+            body += f"    server localhost:{port};\n"

        # Add to the list of configured upstreams
        nginx_upstream_config += NGINX_UPSTREAM_CONFIG_BLOCK.format(
@@ -921,15 +876,10 @@ def generate_worker_files(
    # If there are workers, add the main process to the instance_map too.
    if workers_in_use:
        instance_map = shared_config.setdefault("instance_map", {})
-        if using_unix_sockets:
-            instance_map[MAIN_PROCESS_INSTANCE_NAME] = {
-                "path": MAIN_PROCESS_UNIX_SOCKET_PRIVATE_PATH,
-            }
-        else:
-            instance_map[MAIN_PROCESS_INSTANCE_NAME] = {
-                "host": MAIN_PROCESS_LOCALHOST_ADDRESS,
-                "port": MAIN_PROCESS_REPLICATION_PORT,
-            }
+        instance_map[MAIN_PROCESS_INSTANCE_NAME] = {
+            "host": MAIN_PROCESS_LOCALHOST_ADDRESS,
+            "port": MAIN_PROCESS_REPLICATION_PORT,
+        }

    # Shared homeserver config
    convert(
@@ -939,7 +889,6 @@ def generate_worker_files(
        appservice_registrations=appservice_registrations,
        enable_redis=workers_in_use,
        workers_in_use=workers_in_use,
-        using_unix_sockets=using_unix_sockets,
    )

    # Nginx config
@@ -950,7 +899,6 @@ def generate_worker_files(
        upstream_directives=nginx_upstream_config,
        tls_cert_path=os.environ.get("SYNAPSE_TLS_CERT"),
        tls_key_path=os.environ.get("SYNAPSE_TLS_KEY"),
-        using_unix_sockets=using_unix_sockets,
    )

    # Supervisord config
@@ -960,7 +908,6 @@ def generate_worker_files(
        "/etc/supervisor/supervisord.conf",
        main_config_path=config_path,
        enable_redis=workers_in_use,
-        using_unix_sockets=using_unix_sockets,
    )

    convert(
@@ -1000,7 +947,6 @@ def generate_worker_log_config(
    extra_log_template_args["SYNAPSE_LOG_SENSITIVE"] = environ.get(
        "SYNAPSE_LOG_SENSITIVE"
    )
-    extra_log_template_args["SYNAPSE_LOG_TESTING"] = environ.get("SYNAPSE_LOG_TESTING")

    # Render and write the file
    log_config_filepath = f"/conf/workers/{worker_name}.log.config"
@@ -10,7 +10,7 @@ ARG PYTHON_VERSION=3.9
 ###
 # We hardcode the use of Debian bullseye here because this could change upstream
 # and other Dockerfiles used for testing are expecting bullseye.
-FROM docker.io/library/python:${PYTHON_VERSION}-slim-bullseye
+FROM docker.io/python:${PYTHON_VERSION}-slim-bullseye

 # Install Rust and other dependencies (stolen from normal Dockerfile)
 # install the OS build deps
@@ -419,7 +419,7 @@ The following query parameters are available:

 * `from` (required) - The token to start returning events from. This token can be obtained from a prev_batch
  or next_batch token returned by the /sync endpoint, or from an end token returned by a previous request to this endpoint.
-* `to` - The token to stop returning events at.
+* `to` - The token to spot returning events at.
 * `limit` - The maximum number of events to return. Defaults to `10`.
 * `filter` - A JSON RoomEventFilter to filter returned events with.
 * `dir` - The direction to return events from. Either `f` for forwards or `b` for backwards. Setting
@@ -242,9 +242,6 @@ The following parameters should be set in the URL:

 - `dir` - Direction of media order. Either `f` for forwards or `b` for backwards.
  Setting this value to `b` will reverse the above sort order. Defaults to `f`.
- `not_user_type` - Exclude certain user types, such as bot users, from the request.
-   Can be provided multiple times. Possible values are `bot`, `support` or "empty string".
-   "empty string" here means to exclude users without a type.

 Caution. The database only has indexes on the columns `name` and `creation_ts`.
 This means that if a different sort order is used (`is_guest`, `admin`,
@@ -732,8 +729,7 @@ POST /_synapse/admin/v1/users/<user_id>/login

 An optional `valid_until_ms` field can be specified in the request body as an
 integer timestamp that specifies when the token should expire. By default tokens
-do not expire. Note that this API does not allow a user to login as themselves
-(to create more tokens).
+do not expire.

 A response body like the following is returned:

@@ -1184,7 +1180,7 @@ The following parameters should be set in the URL:
 - `user_id` - The fully qualified MXID: for example, `@user:server.com`. The user must
  be local.

-## Check username availability
+### Check username availability

 Checks to see if a username is available, and valid, for the server. See [the client-server 
 API](https://matrix.org/docs/spec/client_server/r0.6.0#get-matrix-client-r0-register-available)
@@ -1202,7 +1198,7 @@ GET /_synapse/admin/v1/username_available?username=$localpart
 The request and response format is the same as the
 [/_matrix/client/r0/register/available](https://matrix.org/docs/spec/client_server/r0.6.0#get-matrix-client-r0-register-available) API.

-## Find a user based on their ID in an auth provider
+### Find a user based on their ID in an auth provider

 The API is:

@@ -1241,7 +1237,7 @@ Returns a `404` HTTP status code if no user was found, with a response body like
 _Added in Synapse 1.68.0._


-## Find a user based on their Third Party ID (ThreePID or 3PID)
+### Find a user based on their Third Party ID (ThreePID or 3PID)

 The API is:

@@ -23,7 +23,7 @@ people building from source should ensure they can fetch recent versions of Rust
 (e.g. by using [rustup](https://rustup.rs/)).

 The oldest supported version of SQLite is the version
-[provided](https://packages.debian.org/bullseye/libsqlite3-0) by
+[provided](https://packages.debian.org/buster/libsqlite3-0) by
 [Debian oldstable](https://wiki.debian.org/DebianOldStable).

 Context
@@ -322,7 +322,7 @@ The following command will let you run the integration test with the most common
 configuration:

 ```sh
-$ docker run --rm -it -v /path/where/you/have/cloned/the/repository\:/src:ro -v /path/to/where/you/want/logs\:/logs matrixdotorg/sytest-synapse:focal
+$ docker run --rm -it -v /path/where/you/have/cloned/the/repository\:/src:ro -v /path/to/where/you/want/logs\:/logs matrixdotorg/sytest-synapse:buster
 ```
 (Note that the paths must be full paths! You could also write `$(realpath relative/path)` if needed.)

@@ -370,7 +370,6 @@ The above will run a monolithic (single-process) Synapse with SQLite as the data
    See the [worker documentation](../workers.md) for additional information on workers.
 - Passing `ASYNCIO_REACTOR=1` as an environment variable to use the Twisted asyncio reactor instead of the default one.
 - Passing `PODMAN=1` will use the [podman](https://podman.io/) container runtime, instead of docker.
- Passing `UNIX_SOCKETS=1` will utilise Unix socket functionality for Synapse, Redis, and Postgres(when applicable).

 To increase the log level for the tests, set `SYNAPSE_TEST_LOG_LEVEL`, e.g:
 ```sh
@@ -6,7 +6,7 @@ This is a work-in-progress set of notes with two goals:

 See also [MSC3902](https://github.com/matrix-org/matrix-spec-proposals/pull/3902).

-The key idea is described by [MSC3706](https://github.com/matrix-org/matrix-spec-proposals/pull/3706). This allows servers to
+The key idea is described by [MSC706](https://github.com/matrix-org/matrix-spec-proposals/pull/3902). This allows servers to
 request a lightweight response to the federation `/send_join` endpoint.
 This is called a **faster join**, also known as a **partial join**. In these
 notes we'll usually use the word "partial" as it matches the database schema.
@@ -348,42 +348,6 @@ callback returns `False`, Synapse falls through to the next one. The value of th
 callback that does not return `False` will be used. If this happens, Synapse will not call
 any of the subsequent implementations of this callback.

-
-### `check_login_for_spam`
-
-_First introduced in Synapse v1.87.0_
-
-```python
-async def check_login_for_spam(
-    user_id: str,
-    device_id: Optional[str],
-    initial_display_name: Optional[str],
-    request_info: Collection[Tuple[Optional[str], str]],
-    auth_provider_id: Optional[str] = None,
-) -> Union["synapse.module_api.NOT_SPAM", "synapse.module_api.errors.Codes"]
-```
-
-Called when a user logs in.
-
-The arguments passed to this callback are:
-
-* `user_id`: The user ID the user is logging in with
-* `device_id`: The device ID the user is re-logging into.
-* `initial_display_name`: The device display name, if any.
-* `request_info`: A collection of tuples, which first item is a user agent, and which
-  second item is an IP address. These user agents and IP addresses are the ones that were
-  used during the login process.
-* `auth_provider_id`: The identifier of the SSO authentication provider, if any.
-
-If multiple modules implement this callback, they will be considered in order. If a
-callback returns `synapse.module_api.NOT_SPAM`, Synapse falls through to the next one.
-The value of the first callback that does not return `synapse.module_api.NOT_SPAM` will
-be used. If this happens, Synapse will not call any of the subsequent implementations of
-this callback.
-
-*Note:* This will not be called when a user registers.
-
-
 ## Example

 The example below is a module that implements the spam checker callback
@@ -200,7 +200,7 @@ When following this route please make sure that the [Platform-specific prerequis
 System requirements:

 - POSIX-compliant system (tested on Linux & OS X)
- Python 3.8 or later, up to Python 3.11.
+- Python 3.7 or later, up to Python 3.11.
 - At least 1GB of free RAM if you want to join large public rooms like #matrix:matrix.org

 If building on an uncommon architecture for which pre-built wheels are
@@ -1,4 +1,8 @@
 worker_app: synapse.app.generic_worker
 worker_name: background_worker

+# The replication listener on the main synapse process.
+worker_replication_host: 127.0.0.1
+worker_replication_http_port: 9093
+
 worker_log_config: /etc/matrix-synapse/background-worker-log.yaml
@@ -1,5 +1,9 @@
 worker_app: synapse.app.generic_worker
-worker_name: event_persister1
+worker_name: event_persister1 
+
+# The replication listener on the main synapse process.
+worker_replication_host: 127.0.0.1
+worker_replication_http_port: 9093

 worker_listeners:
  - type: http
@@ -1,4 +1,8 @@
 worker_app: synapse.app.federation_sender
 worker_name: federation_sender1

+# The replication listener on the main synapse process.
+worker_replication_host: 127.0.0.1
+worker_replication_http_port: 9093
+
 worker_log_config: /etc/matrix-synapse/federation-sender-log.yaml
@@ -1,6 +1,10 @@
 worker_app: synapse.app.media_repository
 worker_name: media_worker

+# The replication listener on the main synapse process.
+worker_replication_host: 127.0.0.1
+worker_replication_http_port: 9093
+
 worker_listeners:
  - type: http
    port: 8085
@@ -1,4 +1,8 @@
 worker_app: synapse.app.pusher
 worker_name: pusher_worker1

+# The replication listener on the main synapse process.
+worker_replication_host: 127.0.0.1
+worker_replication_http_port: 9093
+
 worker_log_config: /etc/matrix-synapse/pusher-worker-log.yaml
@@ -88,62 +88,6 @@ process, for example:
    dpkg -i matrix-synapse-py3_1.3.0+stretch1_amd64.deb
    ```

-# Upgrading to v1.89.0
-
-## Removal of unspecced `user` property for `/register`
-
-Application services can no longer call `/register` with a `user` property to create new users.
-The standard `username` property should be used instead. See the
-[Application Service specification](https://spec.matrix.org/v1.7/application-service-api/#server-admin-style-permissions)
-for more information.
-
-
-# Upgrading to v1.88.0
-
-## Minimum supported Python version
-
-The minimum supported Python version has been increased from v3.7 to v3.8.
-You will need Python 3.8 to run Synapse v1.88.0 (due out July 18th, 2023).
-
-If you use current versions of the Matrix.org-distributed Debian
-packages or Docker images, no action is required.
-
-## Removal of `worker_replication_*` settings
-
-As mentioned previously in [Upgrading to v1.84.0](#upgrading-to-v1840), the following deprecated settings
-are being removed in this release of Synapse:
-
-* [`worker_replication_host`](https://matrix-org.github.io/synapse/v1.86/usage/configuration/config_documentation.html#worker_replication_host)
-* [`worker_replication_http_port`](https://matrix-org.github.io/synapse/v1.86/usage/configuration/config_documentation.html#worker_replication_http_port)
-* [`worker_replication_http_tls`](https://matrix-org.github.io/synapse/v1.86/usage/configuration/config_documentation.html#worker_replication_http_tls)
-
-Please ensure that you have migrated to using `main` on your shared configuration's `instance_map`
-(or create one if necessary). This is required if you have ***any*** workers at all;
-administrators of single-process (monolith) installations don't need to do anything.
-
-For an illustrative example, please see [Upgrading to v1.84.0](#upgrading-to-v1840) below.
-
-
-# Upgrading to v1.86.0
-
-## Minimum supported Rust version
-
-The minimum supported Rust version has been increased from v1.58.1 to v1.60.0.
-Users building from source will need to ensure their `rustc` version is up to
-date.
-
-
-# Upgrading to v1.85.0
-
-## Application service registration with "user" property deprecation
-
-Application services should ensure they call the `/register` endpoint with a
-`username` property. The legacy `user` property is considered deprecated and
-should no longer be included.
-
-A future version of Synapse (v1.88.0 or later) will remove support for legacy
-application service login.
-
 # Upgrading to v1.84.0

 ## Deprecation of `worker_replication_*` configuration settings
@@ -27,8 +27,9 @@ What servers are currently participating in this room?
 Run this sql query on your db:
 ```sql
 SELECT DISTINCT split_part(state_key, ':', 2)
-FROM current_state_events
-WHERE room_id = '!cURbafjkfsMDVwdRDQ:matrix.org' AND membership = 'join';
+    FROM current_state_events AS c
+    INNER JOIN room_memberships AS m USING (room_id, event_id)
+    WHERE room_id = '!cURbafjkfsMDVwdRDQ:matrix.org' AND membership = 'join';
 ```

 What users are registered on my server?
@@ -462,20 +462,6 @@ See the docs [request log format](../administration/request_log.md).
 * `additional_resources`: Only valid for an 'http' listener. A map of
   additional endpoints which should be loaded via dynamic modules.

-Unix socket support (_Added in Synapse 1.89.0_):
-* `path`: A path and filename for a Unix socket. Make sure it is located in a
-  directory with read and write permissions, and that it already exists (the directory
-  will not be created). Defaults to `None`.
-  * **Note**: The use of both `path` and `port` options for the same `listener` is not
-    compatible.
-  * The `x_forwarded` option defaults to true  when using Unix sockets and can be omitted.
-  * Other options that would not make sense to use with a UNIX socket, such as 
-    `bind_addresses` and `tls` will be ignored and can be removed.
-* `mode`: The file permissions to set on the UNIX socket. Defaults to `666`
-* **Note:** Must be set as `type: http` (does not support `metrics` and `manhole`). 
-  Also make sure that `metrics` is not included in `resources` -> `names`
-
-
 Valid resource names are:

 * `client`: the client-server API (/_matrix/client), and the synapse admin API (/_synapse/admin). Also implies `media` and `static`.
@@ -488,7 +474,7 @@ Valid resource names are:

 * `media`: the media API (/_matrix/media).

-* `metrics`: the metrics interface. See [here](../../metrics-howto.md). (Not compatible with Unix sockets)
+* `metrics`: the metrics interface. See [here](../../metrics-howto.md).

 * `openid`: OpenID authentication. See [here](../../openid.md).

@@ -547,22 +533,6 @@ listeners:
    bind_addresses: ['::1', '127.0.0.1']
    type: manhole
 ```
-Example configuration #3:
-```yaml
-listeners:
-  # Unix socket listener: Ideal for Synapse deployments behind a reverse proxy, offering
-  # lightweight interprocess communication without TCP/IP overhead, avoid port
-  # conflicts, and providing enhanced security through system file permissions.
-  #
-  # Note that x_forwarded will default to true, when using a UNIX socket. Please see
-  # https://matrix-org.github.io/synapse/latest/reverse_proxy.html.
-  #
-  - path: /var/run/synapse/main_public.sock
-    type: http
-    resources:
-      - names: [client, federation]
-```
-
 ---
 ### `manhole_settings`

@@ -1226,32 +1196,6 @@ Example configuration:
 allow_device_name_lookup_over_federation: true
 ```
 ---
-### `federation`
-
-The federation section defines some sub-options related to federation.
-
-The following options are related to configuring timeout and retry logic for one request,
-independently of the others.
-Short retry algorithm is used when something or someone will wait for the request to have an
-answer, while long retry is used for requests that happen in the background,
-like sending a federation transaction.
-
-* `client_timeout`: timeout for the federation requests. Default to 60s.
-* `max_short_retry_delay`: maximum delay to be used for the short retry algo. Default to 2s.
-* `max_long_retry_delay`: maximum delay to be used for the short retry algo. Default to 60s.
-* `max_short_retries`: maximum number of retries for the short retry algo. Default to 3 attempts.
-* `max_long_retries`: maximum number of retries for the long retry algo. Default to 10 attempts.
-
-Example configuration:
-```yaml
-federation:
-  client_timeout: 180s
-  max_short_retry_delay: 7s
-  max_long_retry_delay: 100s
-  max_short_retries: 5
-  max_long_retries: 20
-```
---
 ## Caching

 Options related to caching.
@@ -2626,50 +2570,7 @@ Example configuration:
 ```yaml
 nonrefreshable_access_token_lifetime: 24h
 ```
---
-### `ui_auth`

-The amount of time to allow a user-interactive authentication session to be active.
-
-This defaults to 0, meaning the user is queried for their credentials
-before every action, but this can be overridden to allow a single
-validation to be re-used.  This weakens the protections afforded by
-the user-interactive authentication process, by allowing for multiple
-(and potentially different) operations to use the same validation session.
-
-This is ignored for potentially "dangerous" operations (including
-deactivating an account, modifying an account password, adding a 3PID,
-and minting additional login tokens).
-
-Use the `session_timeout` sub-option here to change the time allowed for credential validation.
-
-Example configuration:
-```yaml
-ui_auth:
-    session_timeout: "15s"
-```
---
-### `login_via_existing_session`
-
-Matrix supports the ability of an existing session to mint a login token for
-another client.
-
-Synapse disables this by default as it has security ramifications -- a malicious
-client could use the mechanism to spawn more than one session.
-
-The duration of time the generated token is valid for can be configured with the
-`token_timeout` sub-option.
-
-User-interactive authentication is required when this is enabled unless the
-`require_ui_auth` sub-option is set to `False`.
-
-Example configuration:
-```yaml
-login_via_existing_session:
-    enabled: true
-    require_ui_auth: false
-    token_timeout: "5m"
-```
 ---
 ## Metrics
 Config options related to metrics.
@@ -3514,6 +3415,28 @@ password_config:
      require_uppercase: true
 ```
 ---
+### `ui_auth`
+
+The amount of time to allow a user-interactive authentication session to be active.
+
+This defaults to 0, meaning the user is queried for their credentials
+before every action, but this can be overridden to allow a single
+validation to be re-used.  This weakens the protections afforded by
+the user-interactive authentication process, by allowing for multiple
+(and potentially different) operations to use the same validation session.
+
+This is ignored for potentially "dangerous" operations (including
+deactivating an account, modifying an account password, and
+adding a 3PID).
+
+Use the `session_timeout` sub-option here to change the time allowed for credential validation.
+
+Example configuration:
+```yaml
+ui_auth:
+    session_timeout: "15s"
+```
+---
 ## Push
 Configuration settings related to push notifications

@@ -3960,14 +3883,13 @@ federation_sender_instances:
 ---
 ### `instance_map`

-When using workers this should be a map from [`worker_name`](#worker_name) to the HTTP
-replication listener of the worker, if configured, and to the main process. Each worker
-declared under [`stream_writers`](../../workers.md#stream-writers) and
-[`outbound_federation_restricted_to`](#outbound_federation_restricted_to) needs a HTTP
-replication listener, and that listener should be included in the `instance_map`. The
-main process also needs an entry on the `instance_map`, and it should be listed under
-`main` **if even one other worker exists**. Ensure the port matches with what is
-declared inside the `listener` block for a `replication` listener.
+When using workers this should be a map from [`worker_name`](#worker_name) to the
+HTTP replication listener of the worker, if configured, and to the main process.
+Each worker declared under [`stream_writers`](../../workers.md#stream-writers) needs
+a HTTP replication listener, and that listener should be included in the `instance_map`.
+The main process also needs an entry on the `instance_map`, and it should be listed under
+`main` **if even one other worker exists**. Ensure the port matches with what is declared 
+inside the `listener` block for a `replication` listener.


 Example configuration:
@@ -3980,14 +3902,6 @@ instance_map:
    host: localhost
    port: 8034
 ```
-Example configuration(#2, for UNIX sockets):
-```yaml
-instance_map:
-  main:
-    path: /var/run/synapse/main_replication.sock
-  worker1:
-    path: /var/run/synapse/worker1_replication.sock
-```
 ---
 ### `stream_writers`

@@ -4005,24 +3919,6 @@ stream_writers:
  typing: worker1
 ```
 ---
-### `outbound_federation_restricted_to`
-
-When using workers, you can restrict outbound federation traffic to only go through a
-specific subset of workers. Any worker specified here must also be in the
-[`instance_map`](#instance_map).
-[`worker_replication_secret`](#worker_replication_secret) must also be configured to
-authorize inter-worker communication.
-
-```yaml
-outbound_federation_restricted_to:
-  - federation_sender1
-  - federation_sender2
-```
-
-Also see the [worker
-documentation](../../workers.md#restrict-outbound-federation-traffic-to-a-specific-set-of-workers)
-for more info.
---
 ### `run_background_tasks_on`

 The [worker](../../workers.md#background-tasks) that is used to run
@@ -4147,6 +4043,51 @@ Example configuration:
 worker_name: generic_worker1
 ```
 ---
+### `worker_replication_host`
+*Deprecated as of version 1.84.0. Place `host` under `main` entry on the [`instance_map`](#instance_map) in your shared yaml configuration instead.*
+
+The HTTP replication endpoint that it should talk to on the main Synapse process.
+The main Synapse process defines this with a `replication` resource in
+[`listeners` option](#listeners).
+
+Example configuration:
+```yaml
+worker_replication_host: 127.0.0.1
+```
+---
+### `worker_replication_http_port`
+*Deprecated as of version 1.84.0. Place `port` under `main` entry on the [`instance_map`](#instance_map) in your shared yaml configuration instead.*
+
+The HTTP replication port that it should talk to on the main Synapse process.
+The main Synapse process defines this with a `replication` resource in
+[`listeners` option](#listeners).
+
+Example configuration:
+```yaml
+worker_replication_http_port: 9093
+```
+---
+### `worker_replication_http_tls`
+*Deprecated as of version 1.84.0. Place `tls` under `main` entry on the [`instance_map`](#instance_map) in your shared yaml configuration instead.*
+
+Whether TLS should be used for talking to the HTTP replication port on the main
+Synapse process.
+The main Synapse process defines this with the `tls` option on its [listener](#listeners) that
+has the `replication` resource enabled.
+
+**Please note:** by default, it is not safe to expose replication ports to the
+public Internet, even with TLS enabled.
+See [`worker_replication_secret`](#worker_replication_secret).
+
+Defaults to `false`.
+
+*Added in Synapse 1.72.0.*
+
+Example configuration:
+```yaml
+worker_replication_http_tls: true
+```
+---
 ### `worker_listeners`

 A worker can handle HTTP requests. To do so, a `worker_listeners` option
@@ -4165,18 +4106,6 @@ worker_listeners:
    resources:
      - names: [client, federation]
 ```
-Example configuration(#2, using UNIX sockets with a `replication` listener):
-```yaml
-worker_listeners:
-  - type: http
-    path: /var/run/synapse/worker_public.sock
-    resources:
-      - names: [client, federation]
-  - type: http
-    path: /var/run/synapse/worker_replication.sock
-    resources:
-      - names: [replication]
-```
 ---
 ### `worker_manhole`

@@ -95,12 +95,9 @@ for the main process
 * Secondly, you need to enable
 [redis-based replication](usage/configuration/config_documentation.md#redis)
 * You will need to add an [`instance_map`](usage/configuration/config_documentation.md#instance_map) 
-with the `main` process defined, as well as the relevant connection information from
-it's HTTP `replication` listener (defined in step 1 above).
-  * Note that the `host` defined is the address the worker needs to look for the `main`
-  process at, not necessarily the same address that is bound to.
-  * If you are using Unix sockets for the `replication` resource, make sure to
-  use a `path` to the socket file instead of a `port`.
+with the `main` process defined, as well as the relevant connection information from 
+it's HTTP `replication` listener (defined in step 1 above). Note that the `host` defined 
+is the address the worker needs to look for the `main` process at, not necessarily the same address that is bound to.
 * Optionally, a [shared secret](usage/configuration/config_documentation.md#worker_replication_secret)
 can be used to authenticate HTTP traffic between workers. For example:

@@ -148,6 +145,9 @@ In the config file for each worker, you must specify:
   with an `http` listener.
 * **Synapse 1.72 and older:** if handling the `^/_matrix/client/v3/keys/upload` endpoint, the HTTP URI for
   the main process (`worker_main_http_uri`). This config option is no longer required and is ignored when running Synapse 1.73 and newer.
+ * **Synapse 1.83 and older:** The HTTP replication endpoint that the worker should talk to on the main synapse process
+   ([`worker_replication_host`](usage/configuration/config_documentation.md#worker_replication_host) and
+   [`worker_replication_http_port`](usage/configuration/config_documentation.md#worker_replication_http_port)). If using Synapse 1.84 and newer, these are not needed if `main` is defined on the [shared configuration](#shared-configuration) `instance_map`

 For example:

@@ -177,11 +177,11 @@ The following applies to Synapse installations that have been installed from sou

 You can start the main Synapse process with Poetry by running the following command:
 ```console
-poetry run synapse_homeserver --config-file [your homeserver.yaml]
+poetry run synapse_homeserver -c [your homeserver.yaml]
 ```
 For worker setups, you can run the following command
 ```console
-poetry run synapse_worker --config-file [your homeserver.yaml] --config-file [your worker.yaml]
+poetry run synapse_worker -c [your worker.yaml]
 ```
 ## Available worker applications

@@ -232,6 +232,7 @@ information.
    ^/_matrix/client/v1/rooms/.*/hierarchy$
    ^/_matrix/client/(v1|unstable)/rooms/.*/relations/
    ^/_matrix/client/v1/rooms/.*/threads$
+    ^/_matrix/client/unstable/org.matrix.msc2716/rooms/.*/batch_send$
    ^/_matrix/client/unstable/im.nheko.summary/rooms/.*/summary$
    ^/_matrix/client/(r0|v3|unstable)/account/3pid$
    ^/_matrix/client/(r0|v3|unstable)/account/whoami$
@@ -531,30 +532,6 @@ the stream writer for the `presence` stream:

    ^/_matrix/client/(api/v1|r0|v3|unstable)/presence/

-#### Restrict outbound federation traffic to a specific set of workers
-
-The
-[`outbound_federation_restricted_to`](usage/configuration/config_documentation.md#outbound_federation_restricted_to)
-configuration is useful to make sure outbound federation traffic only goes through a
-specified subset of workers. This allows you to set more strict access controls (like a
-firewall) for all workers and only allow the `federation_sender`'s to contact the
-outside world.
-
-```yaml
-instance_map:
-    main:
-        host: localhost
-        port: 8030
-    federation_sender1:
-        host: localhost
-        port: 8034
-
-outbound_federation_restricted_to:
-  - federation_sender1
-
-worker_replication_secret: "secret_secret"
-```
-
 #### Background tasks

 There is also support for moving background tasks to a separate
@@ -22,6 +22,27 @@
        "type": "github"
      }
    },
+    "fenix": {
+      "inputs": {
+        "nixpkgs": [
+          "nixpkgs"
+        ],
+        "rust-analyzer-src": "rust-analyzer-src"
+      },
+      "locked": {
+        "lastModified": 1682490133,
+        "narHash": "sha256-tR2Qx0uuk97WySpSSk4rGS/oH7xb5LykbjATcw1vw1I=",
+        "owner": "nix-community",
+        "repo": "fenix",
+        "rev": "4e9412753ab75ef0e038a5fe54a062fb44c27c6a",
+        "type": "github"
+      },
+      "original": {
+        "owner": "nix-community",
+        "repo": "fenix",
+        "type": "github"
+      }
+    },
    "flake-compat": {
      "flake": false,
      "locked": {
@@ -53,24 +74,6 @@
        "type": "github"
      }
    },
-    "flake-utils_2": {
-      "inputs": {
-        "systems": "systems"
-      },
-      "locked": {
-        "lastModified": 1681202837,
-        "narHash": "sha256-H+Rh19JDwRtpVPAWp64F+rlEtxUWBAQW28eAi3SRSzg=",
-        "owner": "numtide",
-        "repo": "flake-utils",
-        "rev": "cfacdce06f30d2b68473a46042957675eebb3401",
-        "type": "github"
-      },
-      "original": {
-        "owner": "numtide",
-        "repo": "flake-utils",
-        "type": "github"
-      }
-    },
    "gitignore": {
      "inputs": {
        "nixpkgs": [
@@ -197,22 +200,6 @@
        "type": "github"
      }
    },
-    "nixpkgs_3": {
-      "locked": {
-        "lastModified": 1681358109,
-        "narHash": "sha256-eKyxW4OohHQx9Urxi7TQlFBTDWII+F+x2hklDOQPB50=",
-        "owner": "NixOS",
-        "repo": "nixpkgs",
-        "rev": "96ba1c52e54e74c3197f4d43026b3f3d92e83ff9",
-        "type": "github"
-      },
-      "original": {
-        "owner": "NixOS",
-        "ref": "nixpkgs-unstable",
-        "repo": "nixpkgs",
-        "type": "github"
-      }
-    },
    "pre-commit-hooks": {
      "inputs": {
        "flake-compat": [
@@ -244,27 +231,25 @@
    "root": {
      "inputs": {
        "devenv": "devenv",
+        "fenix": "fenix",
        "nixpkgs": "nixpkgs_2",
-        "rust-overlay": "rust-overlay",
-        "systems": "systems_2"
+        "systems": "systems"
      }
    },
-    "rust-overlay": {
-      "inputs": {
-        "flake-utils": "flake-utils_2",
-        "nixpkgs": "nixpkgs_3"
-      },
+    "rust-analyzer-src": {
+      "flake": false,
      "locked": {
-        "lastModified": 1689302058,
-        "narHash": "sha256-yD74lcHTrw4niXcE9goJLbzsgyce48rQQoy5jK5ZK40=",
-        "owner": "oxalica",
-        "repo": "rust-overlay",
-        "rev": "7b8dbbf4c67ed05a9bf3d9e658c12d4108bc24c8",
+        "lastModified": 1682426789,
+        "narHash": "sha256-UqnLmJESRZE0tTEaGbRAw05Hm19TWIPA+R3meqi5I4w=",
+        "owner": "rust-lang",
+        "repo": "rust-analyzer",
+        "rev": "943d2a8a1ca15e8b28a1f51f5a5c135e3728da04",
        "type": "github"
      },
      "original": {
-        "owner": "oxalica",
-        "repo": "rust-overlay",
+        "owner": "rust-lang",
+        "ref": "nightly",
+        "repo": "rust-analyzer",
        "type": "github"
      }
    },
@@ -282,21 +267,6 @@
        "repo": "default",
        "type": "github"
      }
-    },
-    "systems_2": {
-      "locked": {
-        "lastModified": 1681028828,
-        "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=",
-        "owner": "nix-systems",
-        "repo": "default",
-        "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e",
-        "type": "github"
-      },
-      "original": {
-        "owner": "nix-systems",
-        "repo": "default",
-        "type": "github"
-      }
    }
  },
  "root": "root",
@@ -46,20 +46,20 @@
    systems.url = "github:nix-systems/default";
    # A development environment manager built on Nix. See https://devenv.sh.
    devenv.url = "github:cachix/devenv/main";
-    # Rust toolchain.
-    rust-overlay.url = "github:oxalica/rust-overlay";
+    # Rust toolchains and rust-analyzer nightly.
+    fenix = {
+      url = "github:nix-community/fenix";
+      inputs.nixpkgs.follows = "nixpkgs";
+    };
  };

-  outputs = { self, nixpkgs, devenv, systems, rust-overlay, ... } @ inputs:
+  outputs = { self, nixpkgs, devenv, systems, ... } @ inputs:
    let
      forEachSystem = nixpkgs.lib.genAttrs (import systems);
    in {
      devShells = forEachSystem (system:
        let
-          overlays = [ (import rust-overlay) ];
-          pkgs = import nixpkgs {
-            inherit system overlays;
-          };
+          pkgs = nixpkgs.legacyPackages.${system};
        in {
          # Everything is configured via devenv - a Nix module for creating declarative
          # developer environments. See https://devenv.sh/reference/options/ for a list
@@ -76,20 +76,6 @@
                # Configure packages to install.
                # Search for package names at https://search.nixos.org/packages?channel=unstable
                packages = with pkgs; [
-                  # The rust toolchain and related tools.
-                  # This will install the "default" profile of rust components.
-                  # https://rust-lang.github.io/rustup/concepts/profiles.html
-                  #
-                  # NOTE: We currently need to set the Rust version unnecessarily high
-                  # in order to work around https://github.com/matrix-org/synapse/issues/15939
-                  (rust-bin.stable."1.70.0".default.override {
-                    # Additionally install the "rust-src" extension to allow diving into the
-                    # Rust source code in an IDE (rust-analyzer will also make use of it).
-                    extensions = [ "rust-src" ];
-                  })
-                  # The rust-analyzer language server implementation.
-                  rust-analyzer
-
                  # Native dependencies for running Synapse.
                  icu
                  libffi
@@ -138,11 +124,12 @@
                # Install dependencies for the additional programming languages
                # involved with Synapse development.
                #
+                # * Rust is used for developing and running Synapse.
                # * Golang is needed to run the Complement test suite.
                # * Perl is needed to run the SyTest test suite.
-                # * Rust is used for developing and running Synapse.
-                #   It is installed manually with `packages` above.
                languages.go.enable = true;
+                languages.rust.enable = true;
+                languages.rust.version = "stable";
                languages.perl.enable = true;

                # Postgres is needed to run Synapse with postgres support and
@@ -191,7 +178,7 @@
                  EOF
                '';
                # Start synapse when `devenv up` is run.
-                processes.synapse.exec = "poetry run python -m synapse.app.homeserver -c homeserver.yaml -c homeserver-config-overrides.d";
+                processes.synapse.exec = "poetry run python -m synapse.app.homeserver -c homeserver.yaml --config-directory homeserver-config-overrides.d";

                # Define the perl modules we require to run SyTest.
                #
@@ -2,29 +2,17 @@
 namespace_packages = True
 plugins = pydantic.mypy, mypy_zope:plugin, scripts-dev/mypy_synapse_plugin.py
 follow_imports = normal
+check_untyped_defs = True
 show_error_codes = True
 show_traceback = True
 mypy_path = stubs
 warn_unreachable = True
+warn_unused_ignores = True
 local_partial_types = True
 no_implicit_optional = True
-
-# Strict checks, see mypy --help
-warn_unused_configs = True
-# disallow_any_generics = True
-disallow_subclassing_any = True
-# disallow_untyped_calls = True
 disallow_untyped_defs = True
-disallow_incomplete_defs = True
-# check_untyped_defs = True
-# disallow_untyped_decorators = True
-warn_redundant_casts = True
-warn_unused_ignores = True
-# warn_return_any = True
-# no_implicit_reexport = True
 strict_equality = True
-strict_concatenate = True
-
+warn_redundant_casts = True
 # Run mypy type checking with the minimum supported Python version to catch new usage
 # that isn't backwards-compatible (types, overloads, etc).
 python_version = 3.8
@@ -43,7 +31,6 @@ warn_unused_ignores = False

 [mypy-synapse.util.caches.treecache]
 disallow_untyped_defs = False
-disallow_incomplete_defs = False

 ;; Dependencies without annotations
 ;; Before ignoring a module, check to see if type stubs are available.
@@ -53,18 +40,18 @@ disallow_incomplete_defs = False
 ;; which we can pull in as a dev dependency by adding to `pyproject.toml`'s
 ;; `[tool.poetry.dev-dependencies]` list.

-# https://github.com/lepture/authlib/issues/460
 [mypy-authlib.*]
 ignore_missing_imports = True

 [mypy-ijson.*]
 ignore_missing_imports = True

-# https://github.com/msgpack/msgpack-python/issues/448
+[mypy-lxml]
+ignore_missing_imports = True
+
 [mypy-msgpack]
 ignore_missing_imports = True

-# https://github.com/wolever/parameterized/issues/143
 [mypy-parameterized.*]
 ignore_missing_imports = True

@@ -86,7 +73,6 @@ ignore_missing_imports = True
 [mypy-srvlookup.*]
 ignore_missing_imports = True

-# https://github.com/twisted/treq/pull/366
 [mypy-treq.*]
 ignore_missing_imports = True

@@ -89,7 +89,7 @@ manifest-path = "rust/Cargo.toml"

 [tool.poetry]
 name = "matrix-synapse"
-version = "1.88.0"
+version = "1.85.0rc1"
 description = "Homeserver for the Matrix decentralised comms protocol"
 authors = ["Matrix.org Team and Contributors <packages@matrix.org>"]
 license = "Apache-2.0"
@@ -147,7 +147,7 @@ synapse_review_recent_signups = "synapse._scripts.review_recent_signups:main"
 update_synapse_database = "synapse._scripts.update_synapse_database:main"

 [tool.poetry.dependencies]
-python = "^3.8.0"
+python = "^3.7.1"

 # Mandatory Dependencies
 # ----------------------
@@ -203,9 +203,11 @@ ijson = ">=3.1.4"
 matrix-common = "^1.3.0"
 # We need packaging.requirements.Requirement, added in 16.1.
 packaging = ">=16.1"
+# At the time of writing, we only use functions from the version `importlib.metadata`
+# which shipped in Python 3.8. This corresponds to version 1.4 of the backport.
+importlib_metadata = { version = ">=1.4", python = "<3.8" }
 # This is the most recent version of Pydantic with available on common distros.
-# We are currently incompatible with >=2.0.0: (https://github.com/matrix-org/synapse/issues/15858)
-pydantic = "^1.7.4"
+pydantic = ">=1.7.4"

 # This is for building the rust components during "poetry install", which
 # currently ignores the `build-system.requires` directive (c.f.
@@ -309,10 +311,9 @@ all = [
 # We pin black so that our tests don't start failing on new releases.
 isort = ">=5.10.1"
 black = ">=22.3.0"
-ruff = "0.0.277"
+ruff = "0.0.265"

 # Typechecking
-lxml-stubs = ">=0.4.0"
 mypy = "*"
 mypy-zope = "*"
 types-bleach = ">=4.1.0"
@@ -373,15 +374,7 @@ build-backend = "poetry.core.masonry.api"

 [tool.cibuildwheel]
 # Skip unsupported platforms (by us or by Rust).
-# See https://cibuildwheel.readthedocs.io/en/stable/options/#build-skip for the list of build targets.
-# We skip:
-#  - CPython 3.6 and 3.7: EOLed
-#  - PyPy 3.7: we only support Python 3.8+
-#  - musllinux i686: excluded to reduce number of wheels we build.
-#    c.f. https://github.com/matrix-org/synapse/pull/12595#discussion_r963107677
-#  - PyPy on Aarch64 and musllinux on aarch64: too slow to build.
-#    c.f. https://github.com/matrix-org/synapse/pull/14259
-skip = "cp36* cp37* pp37* *-musllinux_i686 pp*aarch64 *-musllinux_aarch64"
+skip = "cp36* *-musllinux_i686 pp*aarch64 *-musllinux_aarch64"

 # We need a rust compiler
 before-all =  "curl https://sh.rustup.rs -sSf | sh -s -- --default-toolchain stable -y --profile minimal"
@@ -7,7 +7,7 @@ name = "synapse"
 version = "0.1.0"

 edition = "2021"
-rust-version = "1.60.0"
+rust-version = "1.58.1"

 [lib]
 name = "synapse"
@@ -13,6 +13,8 @@
 // limitations under the License.

 #![feature(test)]
+use std::collections::BTreeSet;
+
 use synapse::push::{
    evaluator::PushRuleEvaluator, Condition, EventMatchCondition, FilteredPushRules, JsonValue,
    PushRules, SimpleJsonValue,
@@ -195,6 +197,7 @@ fn bench_eval_message(b: &mut Bencher) {
        false,
        false,
        false,
+        false,
    );

    b.iter(|| eval.run(&rules, Some("bob"), Some("person")));
@@ -142,11 +142,11 @@ pub const BASE_APPEND_OVERRIDE_RULES: &[PushRule] = &[
        default_enabled: true,
    },
    PushRule {
-        rule_id: Cow::Borrowed("global/override/.m.rule.is_user_mention"),
+        rule_id: Cow::Borrowed(".org.matrix.msc3952.is_user_mention"),
        priority_class: 5,
        conditions: Cow::Borrowed(&[Condition::Known(
            KnownCondition::ExactEventPropertyContainsType(EventPropertyIsTypeCondition {
-                key: Cow::Borrowed("content.m\\.mentions.user_ids"),
+                key: Cow::Borrowed("content.org\\.matrix\\.msc3952\\.mentions.user_ids"),
                value_type: Cow::Borrowed(&EventMatchPatternType::UserId),
            }),
        )]),
@@ -163,11 +163,11 @@ pub const BASE_APPEND_OVERRIDE_RULES: &[PushRule] = &[
        default_enabled: true,
    },
    PushRule {
-        rule_id: Cow::Borrowed("global/override/.m.rule.is_room_mention"),
+        rule_id: Cow::Borrowed(".org.matrix.msc3952.is_room_mention"),
        priority_class: 5,
        conditions: Cow::Borrowed(&[
            Condition::Known(KnownCondition::EventPropertyIs(EventPropertyIsCondition {
-                key: Cow::Borrowed("content.m\\.mentions.room"),
+                key: Cow::Borrowed("content.org\\.matrix\\.msc3952\\.mentions.room"),
                value: Cow::Borrowed(&SimpleJsonValue::Bool(true)),
            })),
            Condition::Known(KnownCondition::SenderNotificationPermission {
@@ -70,9 +70,7 @@ pub struct PushRuleEvaluator {
    /// The "content.body", if any.
    body: String,

-    /// True if the event has a m.mentions property. (Note that this is a separate
-    /// flag instead of checking flattened_keys since the m.mentions property
-    /// might be an empty map and not appear in flattened_keys.
+    /// True if the event has a mentions property and MSC3952 support is enabled.
    has_mentions: bool,

    /// The number of users in the room.
@@ -157,7 +155,9 @@ impl PushRuleEvaluator {
            let rule_id = &push_rule.rule_id().to_string();

            // For backwards-compatibility the legacy mention rules are disabled
-            // if the event contains the 'm.mentions' property.
+            // if the event contains the 'm.mentions' property (and if the
+            // experimental feature is enabled, both of these are represented
+            // by the has_mentions flag).
            if self.has_mentions
                && (rule_id == "global/override/.m.rule.contains_display_name"
                    || rule_id == "global/content/.m.rule.contains_user_name"
@@ -562,7 +562,7 @@ fn test_requires_room_version_supports_condition() {
    };
    let rules = PushRules::new(vec![custom_rule]);
    result = evaluator.run(
-        &FilteredPushRules::py_new(rules, BTreeMap::new(), true, false, true, false),
+        &FilteredPushRules::py_new(rules, BTreeMap::new(), true, false, true, false, false),
        None,
        None,
    );
@@ -527,6 +527,7 @@ pub struct FilteredPushRules {
    msc1767_enabled: bool,
    msc3381_polls_enabled: bool,
    msc3664_enabled: bool,
+    msc3952_intentional_mentions: bool,
    msc3958_suppress_edits_enabled: bool,
 }

@@ -539,6 +540,7 @@ impl FilteredPushRules {
        msc1767_enabled: bool,
        msc3381_polls_enabled: bool,
        msc3664_enabled: bool,
+        msc3952_intentional_mentions: bool,
        msc3958_suppress_edits_enabled: bool,
    ) -> Self {
        Self {
@@ -547,6 +549,7 @@ impl FilteredPushRules {
            msc1767_enabled,
            msc3381_polls_enabled,
            msc3664_enabled,
+            msc3952_intentional_mentions,
            msc3958_suppress_edits_enabled,
        }
    }
@@ -584,6 +587,10 @@ impl FilteredPushRules {
                    return false;
                }

+                if !self.msc3952_intentional_mentions && rule.rule_id.contains("org.matrix.msc3952")
+                {
+                    return false;
+                }
                if !self.msc3958_suppress_edits_enabled
                    && rule.rule_id == "global/override/.com.beeper.suppress_edits"
                {
@@ -20,20 +20,15 @@ from concurrent.futures import ThreadPoolExecutor
 from types import FrameType
 from typing import Collection, Optional, Sequence, Set

-# These are expanded inside the dockerfile to be a fully qualified image name.
-# e.g. docker.io/library/debian:bullseye
-#
-# If an EOL is forced by a Python version and we're dropping support for it, make sure
-# to remove references to the distibution across Synapse (search for "bullseye" for
-# example)
 DISTS = (
-    "debian:bullseye",  # (EOL ~2024-07) (our EOL forced by Python 3.9 is 2025-10-05)
-    "debian:bookworm",  # (EOL not specified yet) (our EOL forced by Python 3.11 is 2027-10-24)
-    "debian:sid",  # (EOL not specified yet) (our EOL forced by Python 3.11 is 2027-10-24)
-    "ubuntu:focal",  # 20.04 LTS (EOL 2025-04) (our EOL forced by Python 3.8 is 2024-10-14)
-    "ubuntu:jammy",  # 22.04 LTS (EOL 2027-04) (our EOL forced by Python 3.10 is 2026-10-04)
-    "ubuntu:kinetic",  # 22.10 (EOL 2023-07-20) (our EOL forced by Python 3.10 is 2026-10-04)
-    "ubuntu:lunar",  # 23.04 (EOL 2024-01) (our EOL forced by Python 3.11 is 2027-10-24)
+    "debian:buster",  # oldstable: EOL 2022-08
+    "debian:bullseye",
+    "debian:bookworm",
+    "debian:sid",
+    "ubuntu:focal",  # 20.04 LTS (our EOL forced by Py38 on 2024-10-14)
+    "ubuntu:jammy",  # 22.04 LTS (EOL 2027-04)
+    "ubuntu:kinetic",  # 22.10 (EOL 2023-07-20)
+    "ubuntu:lunar",  # 23.04 (EOL 2024-01)
 )

 DESC = """\
@@ -214,7 +214,7 @@ fi

 extra_test_args=()

-test_tags="synapse_blacklist,msc3874,msc3890,msc3391,msc3930,faster_joins"
+test_tags="synapse_blacklist,msc3787,msc3874,msc3890,msc3391,msc3930,faster_joins"

 # All environment variables starting with PASS_ will be shared.
 # (The prefix is stripped off before reaching the container.)
@@ -246,6 +246,10 @@ else
  else
    export PASS_SYNAPSE_COMPLEMENT_DATABASE=sqlite
  fi
+
+  # The tests for importing historical messages (MSC2716)
+  # only pass with monoliths, currently.
+  test_tags="$test_tags,msc2716"
 fi

 if [[ -n "$ASYNCIO_REACTOR" ]]; then
@@ -253,10 +257,6 @@ if [[ -n "$ASYNCIO_REACTOR" ]]; then
  export PASS_SYNAPSE_COMPLEMENT_USE_ASYNCIO_REACTOR=true
 fi

-if [[ -n "$UNIX_SOCKETS" ]]; then
-  # Enable full on Unix socket mode for Synapse, Redis and Postgresql
-  export PASS_SYNAPSE_USE_UNIX_SOCKET=1
-fi

 if [[ -n "$SYNAPSE_TEST_LOG_LEVEL" ]]; then
  # Set the log level to what is desired
@@ -269,10 +269,6 @@ if [[ -n "$SYNAPSE_TEST_LOG_LEVEL" ]]; then
  export PASS_SYNAPSE_LOG_SENSITIVE=1
 fi

-# Log a few more useful things for a developer attempting to debug something
-# particularly tricky.
-export PASS_SYNAPSE_LOG_TESTING=1
-
 # Run the tests!
 echo "Images built; running complement"
 cd "$COMPLEMENT_DIR"
@@ -136,11 +136,11 @@ def request(
        authorization_headers.append(header)
        print("Authorization: %s" % header, file=sys.stderr)

-    dest = "matrix-federation://%s%s" % (destination, path)
+    dest = "matrix://%s%s" % (destination, path)
    print("Requesting %s" % dest, file=sys.stderr)

    s = requests.Session()
-    s.mount("matrix-federation://", MatrixConnectionAdapter())
+    s.mount("matrix://", MatrixConnectionAdapter())

    headers: Dict[str, str] = {
        "Authorization": authorization_headers[0],
@@ -46,6 +46,7 @@ class FilteredPushRules:
        msc1767_enabled: bool,
        msc3381_polls_enabled: bool,
        msc3664_enabled: bool,
+        msc3952_intentional_mentions: bool,
        msc3958_suppress_edits_enabled: bool,
    ): ...
    def rules(self) -> Collection[Tuple[PushRule, bool]]: ...
@@ -25,8 +25,8 @@ from synapse.util.rust import check_rust_lib_up_to_date
 from synapse.util.stringutils import strtobool

 # Check that we're not running on an unsupported Python version.
-if sys.version_info < (3, 8):
-    print("Synapse requires Python 3.8 or above.")
+if sys.version_info < (3, 7):
+    print("Synapse requires Python 3.7 or above.")
    sys.exit(1)

 # Allow using the asyncio reactor via env var.
@@ -61,7 +61,6 @@ from synapse.storage.databases.main.deviceinbox import DeviceInboxBackgroundUpda
 from synapse.storage.databases.main.devices import DeviceBackgroundUpdateStore
 from synapse.storage.databases.main.e2e_room_keys import EndToEndRoomKeyBackgroundStore
 from synapse.storage.databases.main.end_to_end_keys import EndToEndKeyBackgroundStore
-from synapse.storage.databases.main.event_federation import EventFederationWorkerStore
 from synapse.storage.databases.main.event_push_actions import EventPushActionsStore
 from synapse.storage.databases.main.events_bg_updates import (
    EventsBackgroundUpdatesStore,
@@ -197,11 +196,6 @@ IGNORED_TABLES = {
    "ui_auth_sessions",
    "ui_auth_sessions_credentials",
    "ui_auth_sessions_ips",
-    # Ignore the worker locks table, as a) there shouldn't be any acquired locks
-    # after porting, and b) the circular foreign key constraints make it hard to
-    # port.
-    "worker_read_write_locks_mode",
-    "worker_read_write_locks",
 }


@@ -245,7 +239,6 @@ class Store(
    PresenceBackgroundUpdateStore,
    ReceiptsBackgroundUpdateStore,
    RelationsWorkerStore,
-    EventFederationWorkerStore,
 ):
    def execute(self, f: Callable[..., R], *args: Any, **kwargs: Any) -> Awaitable[R]:
        return self.db_pool.runInteraction(f.__name__, f, *args, **kwargs)
@@ -810,9 +803,7 @@ class Porter:
            )
            # Map from table name to args passed to `handle_table`, i.e. a tuple
            # of: `postgres_size`, `table_size`, `forward_chunk`, `backward_chunk`.
-            tables_to_port_info_map = {
-                r[0]: r[1:] for r in setup_res if r[0] not in IGNORED_TABLES
-            }
+            tables_to_port_info_map = {r[0]: r[1:] for r in setup_res}

            # Step 5. Do the copying.
            #
@@ -1378,9 +1369,6 @@ def main() -> None:
        sys.stderr.write("Database must use the 'psycopg2' connector.\n")
        sys.exit(3)

-    # Don't run the background tasks that get started by the data stores.
-    hs_config["run_background_tasks_on"] = "some_other_process"
-
    config = HomeServerConfig()
    config.parse_config_dict(hs_config, "", "")

@@ -59,7 +59,7 @@ def scope_to_list(scope: str) -> List[str]:
    return scope.strip().split(" ")


-class PrivateKeyJWTWithKid(PrivateKeyJWT):  # type: ignore[misc]
+class PrivateKeyJWTWithKid(PrivateKeyJWT):
    """An implementation of the private_key_jwt client auth method that includes a kid header.

    This is needed because some providers (Keycloak) require the kid header to figure
@@ -123,6 +123,10 @@ class EventTypes:
    SpaceChild: Final = "m.space.child"
    SpaceParent: Final = "m.space.parent"

+    MSC2716_INSERTION: Final = "org.matrix.msc2716.insertion"
+    MSC2716_BATCH: Final = "org.matrix.msc2716.batch"
+    MSC2716_MARKER: Final = "org.matrix.msc2716.marker"
+
    Reaction: Final = "m.reaction"


@@ -218,11 +222,21 @@ class EventContentFields:
    # Used in m.room.guest_access events.
    GUEST_ACCESS: Final = "guest_access"

+    # Used on normal messages to indicate they were historically imported after the fact
+    MSC2716_HISTORICAL: Final = "org.matrix.msc2716.historical"
+    # For "insertion" events to indicate what the next batch ID should be in
+    # order to connect to it
+    MSC2716_NEXT_BATCH_ID: Final = "next_batch_id"
+    # Used on "batch" events to indicate which insertion event it connects to
+    MSC2716_BATCH_ID: Final = "batch_id"
+    # For "marker" events
+    MSC2716_INSERTION_EVENT_REFERENCE: Final = "insertion_event_reference"
+
    # The authorising user for joining a restricted room.
    AUTHORISING_USER: Final = "join_authorised_via_users_server"

    # Use for mentioning users.
-    MENTIONS: Final = "m.mentions"
+    MSC3952_MENTIONS: Final = "org.matrix.msc3952.mentions"

    # an unspecced field added to to-device messages to identify them uniquely-ish
    TO_DEVICE_MSGID: Final = "org.matrix.msgid"
@@ -217,13 +217,6 @@ class InvalidAPICallError(SynapseError):
        super().__init__(HTTPStatus.BAD_REQUEST, msg, Codes.BAD_JSON)


-class InvalidProxyCredentialsError(SynapseError):
-    """Error raised when the proxy credentials are invalid."""
-
-    def __init__(self, msg: str, errcode: str = Codes.UNKNOWN):
-        super().__init__(401, msg, errcode)
-
-
 class ProxiedRequestError(SynapseError):
    """An error from a general matrix endpoint, eg. from a proxied Matrix API call.

@@ -152,9 +152,9 @@ class Filtering:
        self.DEFAULT_FILTER_COLLECTION = FilterCollection(hs, {})

    async def get_user_filter(
-        self, user_id: UserID, filter_id: Union[int, str]
+        self, user_localpart: str, filter_id: Union[int, str]
    ) -> "FilterCollection":
-        result = await self.store.get_user_filter(user_id, filter_id)
+        result = await self.store.get_user_filter(user_localpart, filter_id)
        return FilterCollection(self._hs, result)

    def add_user_filter(self, user_id: UserID, user_filter: JsonDict) -> Awaitable[int]:
@@ -78,29 +78,41 @@ class RoomVersion:
    # MSC2209: Check 'notifications' key while verifying
    # m.room.power_levels auth rules.
    limit_notifications_power_levels: bool
-    # No longer include the creator in m.room.create events.
-    implicit_room_creator: bool
-    # Apply updated redaction rules algorithm from room version 11.
-    updated_redaction_rules: bool
-    # Support the 'restricted' join rule.
-    restricted_join_rule: bool
-    # Support for the proper redaction rules for the restricted join rule. This requires
-    # restricted_join_rule to be enabled.
-    restricted_join_rule_fix: bool
-    # Support the 'knock' join rule.
-    knock_join_rule: bool
+    # MSC2175: No longer include the creator in m.room.create events.
+    msc2175_implicit_room_creator: bool
+    # MSC2174/MSC2176: Apply updated redaction rules algorithm, move redacts to
+    # content property.
+    msc2176_redaction_rules: bool
+    # MSC3083: Support the 'restricted' join_rule.
+    msc3083_join_rules: bool
+    # MSC3375: Support for the proper redaction rules for MSC3083. This mustn't
+    #          be enabled if MSC3083 is not.
+    msc3375_redaction_rules: bool
+    # MSC2403: Allows join_rules to be set to 'knock', changes auth rules to allow sending
+    # m.room.membership event with membership 'knock'.
+    msc2403_knocking: bool
+    # MSC2716: Adds m.room.power_levels -> content.historical field to control
+    # whether "insertion", "chunk", "marker" events can be sent
+    msc2716_historical: bool
+    # MSC2716: Adds support for redacting "insertion", "chunk", and "marker" events
+    msc2716_redactions: bool
    # MSC3389: Protect relation information from redaction.
    msc3389_relation_redactions: bool
-    # Support the 'knock_restricted' join rule.
-    knock_restricted_join_rule: bool
-    # Enforce integer power levels
-    enforce_int_power_levels: bool
+    # MSC3787: Adds support for a `knock_restricted` join rule, mixing concepts of
+    # knocks and restricted join rules into the same join condition.
+    msc3787_knock_restricted_join_rule: bool
+    # MSC3667: Enforce integer power levels
+    msc3667_int_only_power_levels: bool
+    # MSC3821: Do not redact the third_party_invite content field for membership events.
+    msc3821_redaction_rules: bool
    # MSC3931: Adds a push rule condition for "room version feature flags", making
    # some push rules room version dependent. Note that adding a flag to this list
    # is not enough to mark it "supported": the push rule evaluator also needs to
    # support the flag. Unknown flags are ignored by the evaluator, making conditions
    # fail if used.
    msc3931_push_features: Tuple[str, ...]  # values from PushRuleRoomFlag
+    # MSC3989: Redact the origin field.
+    msc3989_redaction_rules: bool


 class RoomVersions:
@@ -113,15 +125,19 @@ class RoomVersions:
        special_case_aliases_auth=True,
        strict_canonicaljson=False,
        limit_notifications_power_levels=False,
-        implicit_room_creator=False,
-        updated_redaction_rules=False,
-        restricted_join_rule=False,
-        restricted_join_rule_fix=False,
-        knock_join_rule=False,
+        msc2175_implicit_room_creator=False,
+        msc2176_redaction_rules=False,
+        msc3083_join_rules=False,
+        msc3375_redaction_rules=False,
+        msc2403_knocking=False,
+        msc2716_historical=False,
+        msc2716_redactions=False,
        msc3389_relation_redactions=False,
-        knock_restricted_join_rule=False,
-        enforce_int_power_levels=False,
+        msc3787_knock_restricted_join_rule=False,
+        msc3667_int_only_power_levels=False,
+        msc3821_redaction_rules=False,
        msc3931_push_features=(),
+        msc3989_redaction_rules=False,
    )
    V2 = RoomVersion(
        "2",
@@ -132,15 +148,19 @@ class RoomVersions:
        special_case_aliases_auth=True,
        strict_canonicaljson=False,
        limit_notifications_power_levels=False,
-        implicit_room_creator=False,
-        updated_redaction_rules=False,
-        restricted_join_rule=False,
-        restricted_join_rule_fix=False,
-        knock_join_rule=False,
+        msc2175_implicit_room_creator=False,
+        msc2176_redaction_rules=False,
+        msc3083_join_rules=False,
+        msc3375_redaction_rules=False,
+        msc2403_knocking=False,
+        msc2716_historical=False,
+        msc2716_redactions=False,
        msc3389_relation_redactions=False,
-        knock_restricted_join_rule=False,
-        enforce_int_power_levels=False,
+        msc3787_knock_restricted_join_rule=False,
+        msc3667_int_only_power_levels=False,
+        msc3821_redaction_rules=False,
        msc3931_push_features=(),
+        msc3989_redaction_rules=False,
    )
    V3 = RoomVersion(
        "3",
@@ -151,15 +171,19 @@ class RoomVersions:
        special_case_aliases_auth=True,
        strict_canonicaljson=False,
        limit_notifications_power_levels=False,
-        implicit_room_creator=False,
-        updated_redaction_rules=False,
-        restricted_join_rule=False,
-        restricted_join_rule_fix=False,
-        knock_join_rule=False,
+        msc2175_implicit_room_creator=False,
+        msc2176_redaction_rules=False,
+        msc3083_join_rules=False,
+        msc3375_redaction_rules=False,
+        msc2403_knocking=False,
+        msc2716_historical=False,
+        msc2716_redactions=False,
        msc3389_relation_redactions=False,
-        knock_restricted_join_rule=False,
-        enforce_int_power_levels=False,
+        msc3787_knock_restricted_join_rule=False,
+        msc3667_int_only_power_levels=False,
+        msc3821_redaction_rules=False,
        msc3931_push_features=(),
+        msc3989_redaction_rules=False,
    )
    V4 = RoomVersion(
        "4",
@@ -170,15 +194,19 @@ class RoomVersions:
        special_case_aliases_auth=True,
        strict_canonicaljson=False,
        limit_notifications_power_levels=False,
-        implicit_room_creator=False,
-        updated_redaction_rules=False,
-        restricted_join_rule=False,
-        restricted_join_rule_fix=False,
-        knock_join_rule=False,
+        msc2175_implicit_room_creator=False,
+        msc2176_redaction_rules=False,
+        msc3083_join_rules=False,
+        msc3375_redaction_rules=False,
+        msc2403_knocking=False,
+        msc2716_historical=False,
+        msc2716_redactions=False,
        msc3389_relation_redactions=False,
-        knock_restricted_join_rule=False,
-        enforce_int_power_levels=False,
+        msc3787_knock_restricted_join_rule=False,
+        msc3667_int_only_power_levels=False,
+        msc3821_redaction_rules=False,
        msc3931_push_features=(),
+        msc3989_redaction_rules=False,
    )
    V5 = RoomVersion(
        "5",
@@ -189,15 +217,19 @@ class RoomVersions:
        special_case_aliases_auth=True,
        strict_canonicaljson=False,
        limit_notifications_power_levels=False,
-        implicit_room_creator=False,
-        updated_redaction_rules=False,
-        restricted_join_rule=False,
-        restricted_join_rule_fix=False,
-        knock_join_rule=False,
+        msc2175_implicit_room_creator=False,
+        msc2176_redaction_rules=False,
+        msc3083_join_rules=False,
+        msc3375_redaction_rules=False,
+        msc2403_knocking=False,
+        msc2716_historical=False,
+        msc2716_redactions=False,
        msc3389_relation_redactions=False,
-        knock_restricted_join_rule=False,
-        enforce_int_power_levels=False,
+        msc3787_knock_restricted_join_rule=False,
+        msc3667_int_only_power_levels=False,
+        msc3821_redaction_rules=False,
        msc3931_push_features=(),
+        msc3989_redaction_rules=False,
    )
    V6 = RoomVersion(
        "6",
@@ -208,15 +240,42 @@ class RoomVersions:
        special_case_aliases_auth=False,
        strict_canonicaljson=True,
        limit_notifications_power_levels=True,
-        implicit_room_creator=False,
-        updated_redaction_rules=False,
-        restricted_join_rule=False,
-        restricted_join_rule_fix=False,
-        knock_join_rule=False,
+        msc2175_implicit_room_creator=False,
+        msc2176_redaction_rules=False,
+        msc3083_join_rules=False,
+        msc3375_redaction_rules=False,
+        msc2403_knocking=False,
+        msc2716_historical=False,
+        msc2716_redactions=False,
        msc3389_relation_redactions=False,
-        knock_restricted_join_rule=False,
-        enforce_int_power_levels=False,
+        msc3787_knock_restricted_join_rule=False,
+        msc3667_int_only_power_levels=False,
+        msc3821_redaction_rules=False,
        msc3931_push_features=(),
+        msc3989_redaction_rules=False,
+    )
+    MSC2176 = RoomVersion(
+        "org.matrix.msc2176",
+        RoomDisposition.UNSTABLE,
+        EventFormatVersions.ROOM_V4_PLUS,
+        StateResolutionVersions.V2,
+        enforce_key_validity=True,
+        special_case_aliases_auth=False,
+        strict_canonicaljson=True,
+        limit_notifications_power_levels=True,
+        msc2175_implicit_room_creator=False,
+        msc2176_redaction_rules=True,
+        msc3083_join_rules=False,
+        msc3375_redaction_rules=False,
+        msc2403_knocking=False,
+        msc2716_historical=False,
+        msc2716_redactions=False,
+        msc3389_relation_redactions=False,
+        msc3787_knock_restricted_join_rule=False,
+        msc3667_int_only_power_levels=False,
+        msc3821_redaction_rules=False,
+        msc3931_push_features=(),
+        msc3989_redaction_rules=False,
    )
    V7 = RoomVersion(
        "7",
@@ -227,15 +286,19 @@ class RoomVersions:
        special_case_aliases_auth=False,
        strict_canonicaljson=True,
        limit_notifications_power_levels=True,
-        implicit_room_creator=False,
-        updated_redaction_rules=False,
-        restricted_join_rule=False,
-        restricted_join_rule_fix=False,
-        knock_join_rule=True,
+        msc2175_implicit_room_creator=False,
+        msc2176_redaction_rules=False,
+        msc3083_join_rules=False,
+        msc3375_redaction_rules=False,
+        msc2403_knocking=True,
+        msc2716_historical=False,
+        msc2716_redactions=False,
        msc3389_relation_redactions=False,
-        knock_restricted_join_rule=False,
-        enforce_int_power_levels=False,
+        msc3787_knock_restricted_join_rule=False,
+        msc3667_int_only_power_levels=False,
+        msc3821_redaction_rules=False,
        msc3931_push_features=(),
+        msc3989_redaction_rules=False,
    )
    V8 = RoomVersion(
        "8",
@@ -246,15 +309,19 @@ class RoomVersions:
        special_case_aliases_auth=False,
        strict_canonicaljson=True,
        limit_notifications_power_levels=True,
-        implicit_room_creator=False,
-        updated_redaction_rules=False,
-        restricted_join_rule=True,
-        restricted_join_rule_fix=False,
-        knock_join_rule=True,
+        msc2175_implicit_room_creator=False,
+        msc2176_redaction_rules=False,
+        msc3083_join_rules=True,
+        msc3375_redaction_rules=False,
+        msc2403_knocking=True,
+        msc2716_historical=False,
+        msc2716_redactions=False,
        msc3389_relation_redactions=False,
-        knock_restricted_join_rule=False,
-        enforce_int_power_levels=False,
+        msc3787_knock_restricted_join_rule=False,
+        msc3667_int_only_power_levels=False,
+        msc3821_redaction_rules=False,
        msc3931_push_features=(),
+        msc3989_redaction_rules=False,
    )
    V9 = RoomVersion(
        "9",
@@ -265,15 +332,65 @@ class RoomVersions:
        special_case_aliases_auth=False,
        strict_canonicaljson=True,
        limit_notifications_power_levels=True,
-        implicit_room_creator=False,
-        updated_redaction_rules=False,
-        restricted_join_rule=True,
-        restricted_join_rule_fix=True,
-        knock_join_rule=True,
+        msc2175_implicit_room_creator=False,
+        msc2176_redaction_rules=False,
+        msc3083_join_rules=True,
+        msc3375_redaction_rules=True,
+        msc2403_knocking=True,
+        msc2716_historical=False,
+        msc2716_redactions=False,
        msc3389_relation_redactions=False,
-        knock_restricted_join_rule=False,
-        enforce_int_power_levels=False,
+        msc3787_knock_restricted_join_rule=False,
+        msc3667_int_only_power_levels=False,
+        msc3821_redaction_rules=False,
        msc3931_push_features=(),
+        msc3989_redaction_rules=False,
+    )
+    MSC3787 = RoomVersion(
+        "org.matrix.msc3787",
+        RoomDisposition.UNSTABLE,
+        EventFormatVersions.ROOM_V4_PLUS,
+        StateResolutionVersions.V2,
+        enforce_key_validity=True,
+        special_case_aliases_auth=False,
+        strict_canonicaljson=True,
+        limit_notifications_power_levels=True,
+        msc2175_implicit_room_creator=False,
+        msc2176_redaction_rules=False,
+        msc3083_join_rules=True,
+        msc3375_redaction_rules=True,
+        msc2403_knocking=True,
+        msc2716_historical=False,
+        msc2716_redactions=False,
+        msc3389_relation_redactions=False,
+        msc3787_knock_restricted_join_rule=True,
+        msc3667_int_only_power_levels=False,
+        msc3821_redaction_rules=False,
+        msc3931_push_features=(),
+        msc3989_redaction_rules=False,
+    )
+    MSC3821 = RoomVersion(
+        "org.matrix.msc3821.opt1",
+        RoomDisposition.UNSTABLE,
+        EventFormatVersions.ROOM_V4_PLUS,
+        StateResolutionVersions.V2,
+        enforce_key_validity=True,
+        special_case_aliases_auth=False,
+        strict_canonicaljson=True,
+        limit_notifications_power_levels=True,
+        msc2175_implicit_room_creator=False,
+        msc2176_redaction_rules=False,
+        msc3083_join_rules=True,
+        msc3375_redaction_rules=True,
+        msc2403_knocking=True,
+        msc2716_historical=False,
+        msc2716_redactions=False,
+        msc3389_relation_redactions=False,
+        msc3787_knock_restricted_join_rule=False,
+        msc3667_int_only_power_levels=False,
+        msc3821_redaction_rules=True,
+        msc3931_push_features=(),
+        msc3989_redaction_rules=False,
    )
    V10 = RoomVersion(
        "10",
@@ -284,15 +401,42 @@ class RoomVersions:
        special_case_aliases_auth=False,
        strict_canonicaljson=True,
        limit_notifications_power_levels=True,
-        implicit_room_creator=False,
-        updated_redaction_rules=False,
-        restricted_join_rule=True,
-        restricted_join_rule_fix=True,
-        knock_join_rule=True,
+        msc2175_implicit_room_creator=False,
+        msc2176_redaction_rules=False,
+        msc3083_join_rules=True,
+        msc3375_redaction_rules=True,
+        msc2403_knocking=True,
+        msc2716_historical=False,
+        msc2716_redactions=False,
        msc3389_relation_redactions=False,
-        knock_restricted_join_rule=True,
-        enforce_int_power_levels=True,
+        msc3787_knock_restricted_join_rule=True,
+        msc3667_int_only_power_levels=True,
+        msc3821_redaction_rules=False,
        msc3931_push_features=(),
+        msc3989_redaction_rules=False,
+    )
+    MSC2716v4 = RoomVersion(
+        "org.matrix.msc2716v4",
+        RoomDisposition.UNSTABLE,
+        EventFormatVersions.ROOM_V4_PLUS,
+        StateResolutionVersions.V2,
+        enforce_key_validity=True,
+        special_case_aliases_auth=False,
+        strict_canonicaljson=True,
+        limit_notifications_power_levels=True,
+        msc2175_implicit_room_creator=False,
+        msc2176_redaction_rules=False,
+        msc3083_join_rules=False,
+        msc3375_redaction_rules=False,
+        msc2403_knocking=True,
+        msc2716_historical=True,
+        msc2716_redactions=True,
+        msc3389_relation_redactions=False,
+        msc3787_knock_restricted_join_rule=False,
+        msc3667_int_only_power_levels=False,
+        msc3821_redaction_rules=False,
+        msc3931_push_features=(),
+        msc3989_redaction_rules=False,
    )
    MSC1767v10 = RoomVersion(
        # MSC1767 (Extensible Events) based on room version "10"
@@ -304,34 +448,66 @@ class RoomVersions:
        special_case_aliases_auth=False,
        strict_canonicaljson=True,
        limit_notifications_power_levels=True,
-        implicit_room_creator=False,
-        updated_redaction_rules=False,
-        restricted_join_rule=True,
-        restricted_join_rule_fix=True,
-        knock_join_rule=True,
+        msc2175_implicit_room_creator=False,
+        msc2176_redaction_rules=False,
+        msc3083_join_rules=True,
+        msc3375_redaction_rules=True,
+        msc2403_knocking=True,
+        msc2716_historical=False,
+        msc2716_redactions=False,
        msc3389_relation_redactions=False,
-        knock_restricted_join_rule=True,
-        enforce_int_power_levels=True,
+        msc3787_knock_restricted_join_rule=True,
+        msc3667_int_only_power_levels=True,
+        msc3821_redaction_rules=False,
        msc3931_push_features=(PushRuleRoomFlag.EXTENSIBLE_EVENTS,),
+        msc3989_redaction_rules=False,
    )
-    V11 = RoomVersion(
-        "11",
-        RoomDisposition.STABLE,
+    MSC3989 = RoomVersion(
+        "org.matrix.msc3989",
+        RoomDisposition.UNSTABLE,
        EventFormatVersions.ROOM_V4_PLUS,
        StateResolutionVersions.V2,
        enforce_key_validity=True,
        special_case_aliases_auth=False,
        strict_canonicaljson=True,
        limit_notifications_power_levels=True,
-        implicit_room_creator=True,  # Used by MSC3820
-        updated_redaction_rules=True,  # Used by MSC3820
-        restricted_join_rule=True,
-        restricted_join_rule_fix=True,
-        knock_join_rule=True,
+        msc2175_implicit_room_creator=False,
+        msc2176_redaction_rules=False,
+        msc3083_join_rules=True,
+        msc3375_redaction_rules=True,
+        msc2403_knocking=True,
+        msc2716_historical=False,
+        msc2716_redactions=False,
        msc3389_relation_redactions=False,
-        knock_restricted_join_rule=True,
-        enforce_int_power_levels=True,
+        msc3787_knock_restricted_join_rule=True,
+        msc3667_int_only_power_levels=True,
+        msc3821_redaction_rules=False,
        msc3931_push_features=(),
+        msc3989_redaction_rules=True,
+    )
+    MSC3820opt2 = RoomVersion(
+        # Based upon v10
+        "org.matrix.msc3820.opt2",
+        RoomDisposition.UNSTABLE,
+        EventFormatVersions.ROOM_V4_PLUS,
+        StateResolutionVersions.V2,
+        enforce_key_validity=True,
+        special_case_aliases_auth=False,
+        strict_canonicaljson=True,
+        limit_notifications_power_levels=True,
+        msc2175_implicit_room_creator=True,  # Used by MSC3820
+        msc2176_redaction_rules=True,  # Used by MSC3820
+        msc3083_join_rules=True,
+        msc3375_redaction_rules=True,
+        msc2403_knocking=True,
+        msc2716_historical=False,
+        msc2716_redactions=False,
+        msc3389_relation_redactions=False,
+        msc3787_knock_restricted_join_rule=True,
+        msc3667_int_only_power_levels=True,
+        msc3821_redaction_rules=True,  # Used by MSC3820
+        msc3931_push_features=(),
+        msc3989_redaction_rules=True,  # Used by MSC3820
    )


@@ -344,11 +520,15 @@ KNOWN_ROOM_VERSIONS: Dict[str, RoomVersion] = {
        RoomVersions.V4,
        RoomVersions.V5,
        RoomVersions.V6,
+        RoomVersions.MSC2176,
        RoomVersions.V7,
        RoomVersions.V8,
        RoomVersions.V9,
+        RoomVersions.MSC3787,
        RoomVersions.V10,
-        RoomVersions.V11,
+        RoomVersions.MSC2716v4,
+        RoomVersions.MSC3989,
+        RoomVersions.MSC3820opt2,
    )
 }

@@ -377,12 +557,12 @@ MSC3244_CAPABILITIES = {
        RoomVersionCapability(
            "knock",
            RoomVersions.V7,
-            lambda room_version: room_version.knock_join_rule,
+            lambda room_version: room_version.msc2403_knocking,
        ),
        RoomVersionCapability(
            "restricted",
            RoomVersions.V9,
-            lambda room_version: room_version.restricted_join_rule,
+            lambda room_version: room_version.msc3083_join_rules,
        ),
    )
 }
@@ -386,7 +386,6 @@ def listen_unix(


 def listen_http(
-    hs: "HomeServer",
    listener_config: ListenerConfig,
    root_resource: Resource,
    version_string: str,
@@ -407,7 +406,6 @@ def listen_http(
        version_string,
        max_request_body_size=max_request_body_size,
        reactor=reactor,
-        hs=hs,
    )

    if isinstance(listener_config, TCPListenerConfig):
@@ -83,6 +83,7 @@ from synapse.storage.databases.main.receipts import ReceiptsWorkerStore
 from synapse.storage.databases.main.registration import RegistrationWorkerStore
 from synapse.storage.databases.main.relations import RelationsWorkerStore
 from synapse.storage.databases.main.room import RoomWorkerStore
+from synapse.storage.databases.main.room_batch import RoomBatchStore
 from synapse.storage.databases.main.roommember import RoomMemberWorkerStore
 from synapse.storage.databases.main.search import SearchStore
 from synapse.storage.databases.main.session import SessionStore
@@ -119,6 +120,7 @@ class GenericWorkerStore(
    # the races it creates aren't too bad.
    KeyStore,
    RoomWorkerStore,
+    RoomBatchStore,
    DirectoryWorkerStore,
    PushRulesWorkerStore,
    ApplicationServiceTransactionWorkerStore,
@@ -221,7 +223,6 @@ class GenericWorkerServer(HomeServer):
        root_resource = create_resource_tree(resources, OptionsResource())

        _base.listen_http(
-            self,
            listener_config,
            root_resource,
            self.version_string,
@@ -139,7 +139,6 @@ class SynapseHomeServer(HomeServer):
            root_resource = OptionsResource()

        ports = listen_http(
-            self,
            listener_config,
            create_resource_tree(resources, root_resource),
            self.version_string,
@@ -31,7 +31,7 @@ class AuthConfig(Config):

        # The default value of password_config.enabled is True, unless msc3861 is enabled.
        msc3861_enabled = (
-            (config.get("experimental_features") or {})
+            config.get("experimental_features", {})
            .get("msc3861", {})
            .get("enabled", False)
        )
@@ -60,13 +60,3 @@ class AuthConfig(Config):
        self.ui_auth_session_timeout = self.parse_duration(
            ui_auth.get("session_timeout", 0)
        )
-
-        # Logging in with an existing session.
-        login_via_existing = config.get("login_via_existing_session", {})
-        self.login_via_existing_enabled = login_via_existing.get("enabled", False)
-        self.login_via_existing_require_ui_auth = login_via_existing.get(
-            "require_ui_auth", True
-        )
-        self.login_via_existing_token_timeout = self.parse_duration(
-            login_via_existing.get("token_timeout", "5m")
-        )
@@ -192,10 +192,10 @@ class MSC3861:
                ("captcha", "enable_registration_captcha"),
            )

-        if root.auth.login_via_existing_enabled:
+        if root.experimental.msc3882_enabled:
            raise ConfigError(
-                "Login via existing session cannot be enabled when OAuth delegation is enabled",
-                ("login_via_existing_session", "enabled"),
+                "MSC3882 cannot be enabled when OAuth delegation is enabled",
+                ("experimental_features", "msc3882_enabled"),
            )

        if root.registration.refresh_token_lifetime:
@@ -247,6 +247,9 @@ class ExperimentalConfig(Config):
        # MSC3026 (busy presence state)
        self.msc3026_enabled: bool = experimental.get("msc3026_enabled", False)

+        # MSC2716 (importing historical messages)
+        self.msc2716_enabled: bool = experimental.get("msc2716_enabled", False)
+
        # MSC3244 (room version capabilities)
        self.msc3244_enabled: bool = experimental.get("msc3244_enabled", True)

@@ -316,6 +319,13 @@ class ExperimentalConfig(Config):
        # MSC3881: Remotely toggle push notifications for another client
        self.msc3881_enabled: bool = experimental.get("msc3881_enabled", False)

+        # MSC3882: Allow an existing session to sign in a new session
+        self.msc3882_enabled: bool = experimental.get("msc3882_enabled", False)
+        self.msc3882_ui_auth: bool = experimental.get("msc3882_ui_auth", True)
+        self.msc3882_token_timeout = self.parse_duration(
+            experimental.get("msc3882_token_timeout", "5m")
+        )
+
        # MSC3874: Filtering /messages with rel_types / not_rel_types.
        self.msc3874_enabled: bool = experimental.get("msc3874_enabled", False)

@@ -355,6 +365,11 @@ class ExperimentalConfig(Config):
        # MSC3391: Removing account data.
        self.msc3391_enabled = experimental.get("msc3391_enabled", False)

+        # MSC3952: Intentional mentions, this depends on MSC3966.
+        self.msc3952_intentional_mentions = experimental.get(
+            "msc3952_intentional_mentions", False
+        )
+
        # MSC3959: Do not generate notifications for edits.
        self.msc3958_supress_edit_notifs = experimental.get(
            "msc3958_supress_edit_notifs", False
@@ -382,6 +397,9 @@ class ExperimentalConfig(Config):
        # Check that none of the other config options conflict with MSC3861 when enabled
        self.msc3861.check_config_conflicts(self.root)

+        # MSC4009: E.164 Matrix IDs
+        self.msc4009_e164_mxids = experimental.get("msc4009_e164_mxids", False)
+
        # MSC4010: Do not allow setting m.push_rules account data.
        self.msc4010_push_rules_account_data = experimental.get(
            "msc4010_push_rules_account_data", False
@@ -22,8 +22,6 @@ class FederationConfig(Config):
    section = "federation"

    def read_config(self, config: JsonDict, **kwargs: Any) -> None:
-        federation_config = config.setdefault("federation", {})
-
        # FIXME: federation_domain_whitelist needs sytests
        self.federation_domain_whitelist: Optional[dict] = None
        federation_domain_whitelist = config.get("federation_domain_whitelist", None)
@@ -51,19 +49,5 @@ class FederationConfig(Config):
            "allow_device_name_lookup_over_federation", False
        )

-        # Allow for the configuration of timeout, max request retries
-        # and min/max retry delays in the matrix federation client.
-        self.client_timeout_ms = Config.parse_duration(
-            federation_config.get("client_timeout", "60s")
-        )
-        self.max_long_retry_delay_ms = Config.parse_duration(
-            federation_config.get("max_long_retry_delay", "60s")
-        )
-        self.max_short_retry_delay_ms = Config.parse_duration(
-            federation_config.get("max_short_retry_delay", "2s")
-        )
-        self.max_long_retries = federation_config.get("max_long_retries", 10)
-        self.max_short_retries = federation_config.get("max_short_retries", 3)
-

 _METRICS_FOR_DOMAINS_SCHEMA = {"type": "array", "items": {"type": "string"}}
@@ -15,7 +15,7 @@

 import argparse
 import logging
-from typing import Any, Dict, List, Optional, Union
+from typing import Any, Dict, List, Union

 import attr
 from pydantic import BaseModel, Extra, StrictBool, StrictInt, StrictStr
@@ -41,17 +41,11 @@ Synapse version. Please use ``%s: name_of_worker`` instead.

 _MISSING_MAIN_PROCESS_INSTANCE_MAP_DATA = """
 Missing data for a worker to connect to main process. Please include '%s' in the
-`instance_map` declared in your shared yaml configuration as defined in configuration
-documentation here:
-`https://matrix-org.github.io/synapse/latest/usage/configuration/config_documentation.html#instance_map`
-"""
-
-WORKER_REPLICATION_SETTING_DEPRECATED_MESSAGE = """
-'%s' is no longer a supported worker setting, please place '%s' onto your shared
-configuration under `main` inside the `instance_map`. See workers documentation here:
+`instance_map` declared in your shared yaml configuration, or optionally(as a deprecated
+solution) in every worker's yaml as various `worker_replication_*` settings as defined
+in workers documentation here:
 `https://matrix-org.github.io/synapse/latest/workers.html#worker-configuration`
 """
-
 # This allows for a handy knob when it's time to change from 'master' to
 # something with less 'history'
 MAIN_PROCESS_INSTANCE_NAME = "master"
@@ -94,7 +88,7 @@ class ConfigModel(BaseModel):
        allow_mutation = False


-class InstanceTcpLocationConfig(ConfigModel):
+class InstanceLocationConfig(ConfigModel):
    """The host and port to talk to an instance via HTTP replication."""

    host: StrictStr
@@ -110,23 +104,6 @@ class InstanceTcpLocationConfig(ConfigModel):
        return f"{self.host}:{self.port}"


-class InstanceUnixLocationConfig(ConfigModel):
-    """The socket file to talk to an instance via HTTP replication."""
-
-    path: StrictStr
-
-    def scheme(self) -> str:
-        """Hardcode a retrievable scheme"""
-        return "unix"
-
-    def netloc(self) -> str:
-        """Nicely format the address location data"""
-        return f"{self.path}"
-
-
-InstanceLocationConfig = Union[InstanceTcpLocationConfig, InstanceUnixLocationConfig]
-
-
@attr.s
 class WriterLocations:
    """Specifies the instances that write various streams.
@@ -171,27 +148,6 @@ class WriterLocations:
    )


-@attr.s(auto_attribs=True)
-class OutboundFederationRestrictedTo:
-    """Whether we limit outbound federation to a certain set of instances.
-
-    Attributes:
-        instances: optional list of instances that can make outbound federation
-            requests. If None then all instances can make federation requests.
-        locations: list of instance locations to connect to proxy via.
-    """
-
-    instances: Optional[List[str]]
-    locations: List[InstanceLocationConfig] = attr.Factory(list)
-
-    def __contains__(self, instance: str) -> bool:
-        # It feels a bit dirty to return `True` if `instances` is `None`, but it makes
-        # sense in downstream usage in the sense that if
-        # `outbound_federation_restricted_to` is not configured, then any instance can
-        # talk to federation (no restrictions so always return `True`).
-        return self.instances is None or instance in self.instances
-
-
 class WorkerConfig(Config):
    """The workers are processes run separately to the main synapse process.
    They have their own pid_file and listener configuration. They use the
@@ -260,37 +216,22 @@ class WorkerConfig(Config):
        )

        # A map from instance name to host/port of their HTTP replication endpoint.
-        # Check if the main process is declared. The main process itself doesn't need
-        # this data as it would never have to talk to itself.
+        # Check if the main process is declared. Inject it into the map if it's not,
+        # based first on if a 'main' block is declared then on 'worker_replication_*'
+        # data. If both are available, default to instance_map. The main process
+        # itself doesn't need this data as it would never have to talk to itself.
        instance_map: Dict[str, Any] = config.get("instance_map", {})

        if self.instance_name is not MAIN_PROCESS_INSTANCE_NAME:
-            # TODO: The next 3 condition blocks can be deleted after some time has
-            #  passed and we're ready to stop checking for these settings.
            # The host used to connect to the main synapse
            main_host = config.get("worker_replication_host", None)
-            if main_host:
-                raise ConfigError(
-                    WORKER_REPLICATION_SETTING_DEPRECATED_MESSAGE
-                    % ("worker_replication_host", main_host)
-                )

            # The port on the main synapse for HTTP replication endpoint
            main_port = config.get("worker_replication_http_port")
-            if main_port:
-                raise ConfigError(
-                    WORKER_REPLICATION_SETTING_DEPRECATED_MESSAGE
-                    % ("worker_replication_http_port", main_port)
-                )

            # The tls mode on the main synapse for HTTP replication endpoint.
            # For backward compatibility this defaults to False.
            main_tls = config.get("worker_replication_http_tls", False)
-            if main_tls:
-                raise ConfigError(
-                    WORKER_REPLICATION_SETTING_DEPRECATED_MESSAGE
-                    % ("worker_replication_http_tls", main_tls)
-                )

            # For now, accept 'main' in the instance_map, but the replication system
            # expects 'master', force that into being until it's changed later.
@@ -300,20 +241,30 @@ class WorkerConfig(Config):
                ]
                del instance_map[MAIN_PROCESS_INSTANCE_MAP_NAME]

+            # This is the backwards compatibility bit that handles the
+            # worker_replication_* bits using setdefault() to not overwrite anything.
+            elif main_host is not None and main_port is not None:
+                instance_map.setdefault(
+                    MAIN_PROCESS_INSTANCE_NAME,
+                    {
+                        "host": main_host,
+                        "port": main_port,
+                        "tls": main_tls,
+                    },
+                )
+
            else:
                # If we've gotten here, it means that the main process is not on the
-                # instance_map.
+                # instance_map and that not enough worker_replication_* variables
+                # were declared in the worker's yaml.
                raise ConfigError(
                    _MISSING_MAIN_PROCESS_INSTANCE_MAP_DATA
                    % MAIN_PROCESS_INSTANCE_MAP_NAME
                )

-        # type-ignore: the expression `Union[A, B]` is not a Type[Union[A, B]] currently
        self.instance_map: Dict[
            str, InstanceLocationConfig
-        ] = parse_and_validate_mapping(
-            instance_map, InstanceLocationConfig  # type: ignore[arg-type]
-        )
+        ] = parse_and_validate_mapping(instance_map, InstanceLocationConfig)

        # Map from type of streams to source, c.f. WriterLocations.
        writers = config.get("stream_writers") or {}
@@ -406,28 +357,6 @@ class WorkerConfig(Config):
            new_option_name="update_user_directory_from_worker",
        )

-        outbound_federation_restricted_to = config.get(
-            "outbound_federation_restricted_to", None
-        )
-        self.outbound_federation_restricted_to = OutboundFederationRestrictedTo(
-            outbound_federation_restricted_to
-        )
-        if outbound_federation_restricted_to:
-            if not self.worker_replication_secret:
-                raise ConfigError(
-                    "`worker_replication_secret` must be configured when using `outbound_federation_restricted_to`."
-                )
-
-            for instance in outbound_federation_restricted_to:
-                if instance not in self.instance_map:
-                    raise ConfigError(
-                        "Instance %r is configured in 'outbound_federation_restricted_to' but does not appear in `instance_map` config."
-                        % (instance,)
-                    )
-                self.outbound_federation_restricted_to.locations.append(
-                    self.instance_map[instance]
-                )
-
    def _should_this_worker_perform_duty(
        self,
        config: Dict[str, Any],
@@ -126,7 +126,7 @@ def validate_event_for_room_version(event: "EventBase") -> None:
            raise AuthError(403, "Event not signed by sending server")

    is_invite_via_allow_rule = (
-        event.room_version.restricted_join_rule
+        event.room_version.msc3083_join_rules
        and event.type == EventTypes.Member
        and event.membership == Membership.JOIN
        and EventContentFields.AUTHORISING_USER in event.content
@@ -339,6 +339,13 @@ def check_state_dependent_auth_rules(
    if event.type == EventTypes.Redaction:
        check_redaction(event.room_version, event, auth_dict)

+    if (
+        event.type == EventTypes.MSC2716_INSERTION
+        or event.type == EventTypes.MSC2716_BATCH
+        or event.type == EventTypes.MSC2716_MARKER
+    ):
+        check_historical(event.room_version, event, auth_dict)
+
    logger.debug("Allowing! %s", event)


@@ -352,10 +359,13 @@ LENIENT_EVENT_BYTE_LIMITS_ROOM_VERSIONS = {
    RoomVersions.V4,
    RoomVersions.V5,
    RoomVersions.V6,
+    RoomVersions.MSC2176,
    RoomVersions.V7,
    RoomVersions.V8,
    RoomVersions.V9,
+    RoomVersions.MSC3787,
    RoomVersions.V10,
+    RoomVersions.MSC2716v4,
    RoomVersions.MSC1767v10,
 }

@@ -447,7 +457,7 @@ def _check_create(event: "EventBase") -> None:

    # 1.4 If content has no creator field, reject if the room version requires it.
    if (
-        not event.room_version.implicit_room_creator
+        not event.room_version.msc2175_implicit_room_creator
        and EventContentFields.ROOM_CREATOR not in event.content
    ):
        raise AuthError(403, "Create event lacks a 'creator' property")
@@ -484,7 +494,7 @@ def _is_membership_change_allowed(
        key = (EventTypes.Create, "")
        create = auth_events.get(key)
        if create and event.prev_event_ids()[0] == create.event_id:
-            if room_version.implicit_room_creator:
+            if room_version.msc2175_implicit_room_creator:
                creator = create.sender
            else:
                creator = create.content[EventContentFields.ROOM_CREATOR]
@@ -507,7 +517,7 @@ def _is_membership_change_allowed(
    caller_invited = caller and caller.membership == Membership.INVITE
    caller_knocked = (
        caller
-        and room_version.knock_join_rule
+        and room_version.msc2403_knocking
        and caller.membership == Membership.KNOCK
    )

@@ -607,9 +617,9 @@ def _is_membership_change_allowed(
        elif join_rule == JoinRules.PUBLIC:
            pass
        elif (
-            room_version.restricted_join_rule and join_rule == JoinRules.RESTRICTED
+            room_version.msc3083_join_rules and join_rule == JoinRules.RESTRICTED
        ) or (
-            room_version.knock_restricted_join_rule
+            room_version.msc3787_knock_restricted_join_rule
            and join_rule == JoinRules.KNOCK_RESTRICTED
        ):
            # This is the same as public, but the event must contain a reference
@@ -639,9 +649,9 @@ def _is_membership_change_allowed(

        elif (
            join_rule == JoinRules.INVITE
-            or (room_version.knock_join_rule and join_rule == JoinRules.KNOCK)
+            or (room_version.msc2403_knocking and join_rule == JoinRules.KNOCK)
            or (
-                room_version.knock_restricted_join_rule
+                room_version.msc3787_knock_restricted_join_rule
                and join_rule == JoinRules.KNOCK_RESTRICTED
            )
        ):
@@ -675,9 +685,9 @@ def _is_membership_change_allowed(
                "You don't have permission to ban",
                errcode=Codes.INSUFFICIENT_POWER,
            )
-    elif room_version.knock_join_rule and Membership.KNOCK == membership:
+    elif room_version.msc2403_knocking and Membership.KNOCK == membership:
        if join_rule != JoinRules.KNOCK and (
-            not room_version.knock_restricted_join_rule
+            not room_version.msc3787_knock_restricted_join_rule
            or join_rule != JoinRules.KNOCK_RESTRICTED
        ):
            raise AuthError(403, "You don't have permission to knock")
@@ -813,6 +823,38 @@ def check_redaction(
    raise AuthError(403, "You don't have permission to redact events")


+def check_historical(
+    room_version_obj: RoomVersion,
+    event: "EventBase",
+    auth_events: StateMap["EventBase"],
+) -> None:
+    """Check whether the event sender is allowed to send historical related
+    events like "insertion", "batch", and "marker".
+
+    Returns:
+        None
+
+    Raises:
+        AuthError if the event sender is not allowed to send historical related events
+        ("insertion", "batch", and "marker").
+    """
+    # Ignore the auth checks in room versions that do not support historical
+    # events
+    if not room_version_obj.msc2716_historical:
+        return
+
+    user_level = get_user_power_level(event.user_id, auth_events)
+
+    historical_level = get_named_level(auth_events, "historical", 100)
+
+    if user_level < historical_level:
+        raise UnstableSpecAuthError(
+            403,
+            'You don\'t have permission to send send historical related events ("insertion", "batch", and "marker")',
+            errcode=Codes.INSUFFICIENT_POWER,
+        )
+
+
 def _check_power_levels(
    room_version_obj: RoomVersion,
    event: "EventBase",
@@ -834,7 +876,7 @@ def _check_power_levels(
    # Reject events with stringy power levels if required by room version
    if (
        event.type == EventTypes.PowerLevels
-        and room_version_obj.enforce_int_power_levels
+        and room_version_obj.msc3667_int_only_power_levels
    ):
        for k, v in event.content.items():
            if k in {
@@ -970,7 +1012,7 @@ def get_user_power_level(user_id: str, auth_events: StateMap["EventBase"]) -> in
        key = (EventTypes.Create, "")
        create_event = auth_events.get(key)
        if create_event is not None:
-            if create_event.room_version.implicit_room_creator:
+            if create_event.room_version.msc2175_implicit_room_creator:
                creator = create_event.sender
            else:
                creator = create_event.content[EventContentFields.ROOM_CREATOR]
@@ -1108,7 +1150,7 @@ def auth_types_for_event(
                )
                auth_types.add(key)

-        if room_version.restricted_join_rule and membership == Membership.JOIN:
+        if room_version.msc3083_join_rules and membership == Membership.JOIN:
            if EventContentFields.AUTHORISING_USER in event.content:
                key = (
                    EventTypes.Member,
@@ -198,6 +198,7 @@ class _EventInternalMetadata:
    soft_failed: DictProperty[bool] = DictProperty("soft_failed")
    proactively_send: DictProperty[bool] = DictProperty("proactively_send")
    redacted: DictProperty[bool] = DictProperty("redacted")
+    historical: DictProperty[bool] = DictProperty("historical")

    txn_id: DictProperty[str] = DictProperty("txn_id")
    """The transaction ID, if it was set when the event was created."""
@@ -287,6 +288,14 @@ class _EventInternalMetadata:
        """
        return self._dict.get("redacted", False)

+    def is_historical(self) -> bool:
+        """Whether this is a historical message.
+        This is used by the batchsend historical message endpoint and
+        is needed to and mark the event as backfilled and skip some checks
+        like push notifications.
+        """
+        return self._dict.get("historical", False)
+
    def is_notifiable(self) -> bool:
        """Whether this event can trigger a push notification"""
        return not self.is_outlier() or self.is_out_of_band_membership()
@@ -346,7 +355,7 @@ class EventBase(metaclass=abc.ABCMeta):
    @property
    def redacts(self) -> Optional[str]:
        """MSC2176 moved the redacts field into the content."""
-        if self.room_version.updated_redaction_rules:
+        if self.room_version.msc2176_redaction_rules:
            return self.content.get("redacts")
        return self.get("redacts")

@@ -175,7 +175,7 @@ class EventBuilder:

        # MSC2174 moves the redacts property to the content, it is invalid to
        # provide it as a top-level property.
-        if self._redacts is not None and not self.room_version.updated_redaction_rules:
+        if self._redacts is not None and not self.room_version.msc2176_redaction_rules:
            event_dict["redacts"] = self._redacts

        if self._origin_server_ts is not None:
@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 from abc import ABC, abstractmethod
-from typing import TYPE_CHECKING, Dict, List, Optional, Tuple
+from typing import TYPE_CHECKING, List, Optional, Tuple

 import attr
 from immutabledict import immutabledict
@@ -107,32 +107,33 @@ class EventContext(UnpersistedEventContextBase):
        state_delta_due_to_event: If `state_group` and `state_group_before_event` are not None
            then this is the delta of the state between the two groups.

-        state_group_deltas: If not empty, this is a dict collecting a mapping of the state
-            difference between state groups.
+        prev_group: If it is known, ``state_group``'s prev_group. Note that this being
+            None does not necessarily mean that ``state_group`` does not have
+            a prev_group!

-            The keys are a tuple of two integers: the initial group and final state group.
-            The corresponding value is a state map representing the state delta between
-            these state groups.
+            If the event is a state event, this is normally the same as
+            ``state_group_before_event``.

-            The dictionary is expected to have at most two entries with state groups of:
+            If ``state_group`` is None (ie, the event is an outlier), ``prev_group``
+            will always also be ``None``.

-            1. The state group before the event and after the event.
-            2. The state group preceding the state group before the event and the
-               state group before the event.
+            Note that this *not* (necessarily) the state group associated with
+            ``_prev_state_ids``.

-            This information is collected and stored as part of an optimization for persisting
-            events.
+        delta_ids: If ``prev_group`` is not None, the state delta between ``prev_group``
+            and ``state_group``.

        partial_state: if True, we may be storing this event with a temporary,
            incomplete state.
    """

    _storage: "StorageControllers"
-    state_group_deltas: Dict[Tuple[int, int], StateMap[str]]
    rejected: Optional[str] = None
    _state_group: Optional[int] = None
    state_group_before_event: Optional[int] = None
    _state_delta_due_to_event: Optional[StateMap[str]] = None
+    prev_group: Optional[int] = None
+    delta_ids: Optional[StateMap[str]] = None
    app_service: Optional[ApplicationService] = None

    partial_state: bool = False
@@ -144,14 +145,16 @@ class EventContext(UnpersistedEventContextBase):
        state_group_before_event: Optional[int],
        state_delta_due_to_event: Optional[StateMap[str]],
        partial_state: bool,
-        state_group_deltas: Dict[Tuple[int, int], StateMap[str]],
+        prev_group: Optional[int] = None,
+        delta_ids: Optional[StateMap[str]] = None,
    ) -> "EventContext":
        return EventContext(
            storage=storage,
            state_group=state_group,
            state_group_before_event=state_group_before_event,
            state_delta_due_to_event=state_delta_due_to_event,
-            state_group_deltas=state_group_deltas,
+            prev_group=prev_group,
+            delta_ids=delta_ids,
            partial_state=partial_state,
        )

@@ -160,7 +163,7 @@ class EventContext(UnpersistedEventContextBase):
        storage: "StorageControllers",
    ) -> "EventContext":
        """Return an EventContext instance suitable for persisting an outlier event"""
-        return EventContext(storage=storage, state_group_deltas={})
+        return EventContext(storage=storage)

    async def persist(self, event: EventBase) -> "EventContext":
        return self
@@ -180,15 +183,13 @@ class EventContext(UnpersistedEventContextBase):
            "state_group": self._state_group,
            "state_group_before_event": self.state_group_before_event,
            "rejected": self.rejected,
-            "state_group_deltas": _encode_state_group_delta(self.state_group_deltas),
+            "prev_group": self.prev_group,
            "state_delta_due_to_event": _encode_state_dict(
                self._state_delta_due_to_event
            ),
+            "delta_ids": _encode_state_dict(self.delta_ids),
            "app_service_id": self.app_service.id if self.app_service else None,
            "partial_state": self.partial_state,
-            # add dummy delta_ids and prev_group for backwards compatibility
-            "delta_ids": None,
-            "prev_group": None,
        }

    @staticmethod
@@ -203,24 +204,17 @@ class EventContext(UnpersistedEventContextBase):
        Returns:
            The event context.
        """
-        # workaround for backwards/forwards compatibility: if the input doesn't have a value
-        # for "state_group_deltas" just assign an empty dict
-        state_group_deltas = input.get("state_group_deltas", None)
-        if state_group_deltas:
-            state_group_deltas = _decode_state_group_delta(state_group_deltas)
-        else:
-            state_group_deltas = {}
-
        context = EventContext(
            # We use the state_group and prev_state_id stuff to pull the
            # current_state_ids out of the DB and construct prev_state_ids.
            storage=storage,
            state_group=input["state_group"],
            state_group_before_event=input["state_group_before_event"],
-            state_group_deltas=state_group_deltas,
+            prev_group=input["prev_group"],
            state_delta_due_to_event=_decode_state_dict(
                input["state_delta_due_to_event"]
            ),
+            delta_ids=_decode_state_dict(input["delta_ids"]),
            rejected=input["rejected"],
            partial_state=input.get("partial_state", False),
        )
@@ -355,7 +349,7 @@ class UnpersistedEventContext(UnpersistedEventContextBase):
    _storage: "StorageControllers"
    state_group_before_event: Optional[int]
    state_group_after_event: Optional[int]
-    state_delta_due_to_event: Optional[StateMap[str]]
+    state_delta_due_to_event: Optional[dict]
    prev_group_for_state_group_before_event: Optional[int]
    delta_ids_to_state_group_before_event: Optional[StateMap[str]]
    partial_state: bool
@@ -386,16 +380,26 @@ class UnpersistedEventContext(UnpersistedEventContextBase):

        events_and_persisted_context = []
        for event, unpersisted_context in amended_events_and_context:
-            state_group_deltas = unpersisted_context._build_state_group_deltas()
-
-            context = EventContext(
-                storage=unpersisted_context._storage,
-                state_group=unpersisted_context.state_group_after_event,
-                state_group_before_event=unpersisted_context.state_group_before_event,
-                state_delta_due_to_event=unpersisted_context.state_delta_due_to_event,
-                partial_state=unpersisted_context.partial_state,
-                state_group_deltas=state_group_deltas,
-            )
+            if event.is_state():
+                context = EventContext(
+                    storage=unpersisted_context._storage,
+                    state_group=unpersisted_context.state_group_after_event,
+                    state_group_before_event=unpersisted_context.state_group_before_event,
+                    state_delta_due_to_event=unpersisted_context.state_delta_due_to_event,
+                    partial_state=unpersisted_context.partial_state,
+                    prev_group=unpersisted_context.state_group_before_event,
+                    delta_ids=unpersisted_context.state_delta_due_to_event,
+                )
+            else:
+                context = EventContext(
+                    storage=unpersisted_context._storage,
+                    state_group=unpersisted_context.state_group_after_event,
+                    state_group_before_event=unpersisted_context.state_group_before_event,
+                    state_delta_due_to_event=unpersisted_context.state_delta_due_to_event,
+                    partial_state=unpersisted_context.partial_state,
+                    prev_group=unpersisted_context.prev_group_for_state_group_before_event,
+                    delta_ids=unpersisted_context.delta_ids_to_state_group_before_event,
+                )
            events_and_persisted_context.append((event, context))
        return events_and_persisted_context

@@ -448,11 +452,11 @@ class UnpersistedEventContext(UnpersistedEventContextBase):

        # if the event isn't a state event the state group doesn't change
        if not self.state_delta_due_to_event:
-            self.state_group_after_event = self.state_group_before_event
+            state_group_after_event = self.state_group_before_event

        # otherwise if it is a state event we need to get a state group for it
        else:
-            self.state_group_after_event = await self._storage.state.store_state_group(
+            state_group_after_event = await self._storage.state.store_state_group(
                event.event_id,
                event.room_id,
                prev_group=self.state_group_before_event,
@@ -460,81 +464,16 @@ class UnpersistedEventContext(UnpersistedEventContextBase):
                current_state_ids=None,
            )

-        state_group_deltas = self._build_state_group_deltas()
-
        return EventContext.with_state(
            storage=self._storage,
-            state_group=self.state_group_after_event,
+            state_group=state_group_after_event,
            state_group_before_event=self.state_group_before_event,
            state_delta_due_to_event=self.state_delta_due_to_event,
-            state_group_deltas=state_group_deltas,
            partial_state=self.partial_state,
+            prev_group=self.state_group_before_event,
+            delta_ids=self.state_delta_due_to_event,
        )

-    def _build_state_group_deltas(self) -> Dict[Tuple[int, int], StateMap]:
-        """
-        Collect deltas between the state groups associated with this context
-        """
-        state_group_deltas = {}
-
-        # if we know the state group before the event and after the event, add them and the
-        # state delta between them to state_group_deltas
-        if self.state_group_before_event and self.state_group_after_event:
-            # if we have the state groups we should have the delta
-            assert self.state_delta_due_to_event is not None
-            state_group_deltas[
-                (
-                    self.state_group_before_event,
-                    self.state_group_after_event,
-                )
-            ] = self.state_delta_due_to_event
-
-        # the state group before the event may also have a state group which precedes it, if
-        # we have that and the state group before the event, add them and the state
-        # delta between them to state_group_deltas
-        if (
-            self.prev_group_for_state_group_before_event
-            and self.state_group_before_event
-        ):
-            # if we have both state groups we should have the delta between them
-            assert self.delta_ids_to_state_group_before_event is not None
-            state_group_deltas[
-                (
-                    self.prev_group_for_state_group_before_event,
-                    self.state_group_before_event,
-                )
-            ] = self.delta_ids_to_state_group_before_event
-
-        return state_group_deltas
-
-
-def _encode_state_group_delta(
-    state_group_delta: Dict[Tuple[int, int], StateMap[str]]
-) -> List[Tuple[int, int, Optional[List[Tuple[str, str, str]]]]]:
-    if not state_group_delta:
-        return []
-
-    state_group_delta_encoded = []
-    for key, value in state_group_delta.items():
-        state_group_delta_encoded.append((key[0], key[1], _encode_state_dict(value)))
-
-    return state_group_delta_encoded
-
-
-def _decode_state_group_delta(
-    input: List[Tuple[int, int, List[Tuple[str, str, str]]]]
-) -> Dict[Tuple[int, int], StateMap[str]]:
-    if not input:
-        return {}
-
-    state_group_deltas = {}
-    for state_group_1, state_group_2, state_dict in input:
-        state_map = _decode_state_dict(state_dict)
-        assert state_map is not None
-        state_group_deltas[(state_group_1, state_group_2)] = state_map
-
-    return state_group_deltas
-

 def _encode_state_dict(
    state_dict: Optional[StateMap[str]],
--- a/Show More
+++ b/Show More
Author	SHA1	Message	Date
Patrick Cloke	63723b10d7	Ensure persistent connections are always used.	2023-05-30 15:04:26 -04:00
Patrick Cloke	b14f5f1fb5	Reduce the size of the HTTP connection pool for non-pushers.	2023-05-30 15:04:26 -04:00
				`@@ -0,0 +1 @@`
				`Experimental [MSC3861](https://github.com/matrix-org/matrix-spec-proposals/pull/3861) support: delegate auth to an OIDC provider.`
				`@@ -1 +0,0 @@`
				`Add Unix Socket support for HTTP Replication Listeners. Document and provide usage instructions for utilizing Unix sockets in Synapse. Contributed by Jason Little.`
				`@@ -1 +0,0 @@`
				`Fix long-standing bug where remote invites weren't correctly pushed.`
				`@@ -1 +0,0 @@`
				Mark `get_user_in_directory` private since it is only used in tests. Also remove the cache from it.
				`@@ -1 +0,0 @@`
				`Fix background schema updates failing over a large upgrade gap.`
				`@@ -1 +0,0 @@`
				`Document which Python version runs on a given Linux distribution so we can more easily clean up later.`
				`@@ -1 +0,0 @@`
				Allow `+` in Matrix IDs, per [MSC4009](https://github.com/matrix-org/matrix-spec-proposals/pull/4009).
				`@@ -1 +0,0 @@`
				`Support room version 11 from [MSC3820](https://github.com/matrix-org/matrix-spec-proposals/pull/3820).`
				`@@ -1 +0,0 @@`
				Allow configuring the set of workers to proxy outbound federation traffic through via `outbound_federation_restricted_to`.