Add changelog

Use parse_duration for newly introduced options
2023-06-14 00:50:53 +02:00 · 2023-06-13 23:55:15 +02:00
225 changed files with 6782 additions and 10155 deletions
@@ -29,12 +29,11 @@ IS_PR = os.environ["GITHUB_REF"].startswith("refs/pull/")

 # First calculate the various trial jobs.
 #
-# For PRs, we only run each type of test with the oldest Python version supported (which
-# is Python 3.8 right now)
+# For each type of test we only run on Py3.7 on PRs

 trial_sqlite_tests = [
    {
-        "python-version": "3.8",
+        "python-version": "3.7",
        "database": "sqlite",
        "extras": "all",
    }
@@ -47,13 +46,13 @@ if not IS_PR:
            "database": "sqlite",
            "extras": "all",
        }
-        for version in ("3.9", "3.10", "3.11")
+        for version in ("3.8", "3.9", "3.10", "3.11")
    )


 trial_postgres_tests = [
    {
-        "python-version": "3.8",
+        "python-version": "3.7",
        "database": "postgres",
        "postgres-version": "11",
        "extras": "all",
@@ -72,7 +71,7 @@ if not IS_PR:

 trial_no_extra_tests = [
    {
-        "python-version": "3.8",
+        "python-version": "3.7",
        "database": "sqlite",
        "extras": "",
    }
@@ -134,6 +133,11 @@ if not IS_PR:
                "sytest-tag": "testing",
                "postgres": "postgres",
            },
+            {
+                "sytest-tag": "buster",
+                "postgres": "multi-postgres",
+                "workers": "workers",
+            },
        ]
    )

@@ -144,7 +144,7 @@ jobs:

      - name: Only build a single wheel on PR
        if: startsWith(github.ref, 'refs/pull/')
-        run: echo "CIBW_BUILD="cp38-manylinux_${{ matrix.arch }}"" >> $GITHUB_ENV
+        run: echo "CIBW_BUILD="cp37-manylinux_${{ matrix.arch }}"" >> $GITHUB_ENV

      - name: Build wheels
        run: python -m cibuildwheel --output-dir wheelhouse
@@ -320,7 +320,7 @@ jobs:

      - uses: actions/setup-python@v4
        with:
-          python-version: '3.8'
+          python-version: '3.7'

      - name: Prepare old deps
        if: steps.cache-poetry-old-deps.outputs.cache-hit != 'true'
@@ -362,7 +362,7 @@ jobs:
    runs-on: ubuntu-latest
    strategy:
      matrix:
-        python-version: ["pypy-3.8"]
+        python-version: ["pypy-3.7"]
        extras: ["all"]

    steps:
@@ -399,8 +399,8 @@ jobs:
      env:
        SYTEST_BRANCH: ${{ github.head_ref }}
        POSTGRES: ${{ matrix.job.postgres && 1}}
-        MULTI_POSTGRES: ${{ (matrix.job.postgres == 'multi-postgres') || '' }}
-        ASYNCIO_REACTOR: ${{ (matrix.job.reactor == 'asyncio') || '' }}
+        MULTI_POSTGRES: ${{ (matrix.job.postgres == 'multi-postgres') && 1}}
+        ASYNCIO_REACTOR: ${{ (matrix.job.reactor == 'asyncio') && 1 }}
        WORKERS: ${{ matrix.job.workers && 1 }}
        BLACKLIST: ${{ matrix.job.workers && 'synapse-blacklist-with-workers' }}
        TOP: ${{ github.workspace }}
@@ -477,7 +477,7 @@ jobs:
    strategy:
      matrix:
        include:
-          - python-version: "3.8"
+          - python-version: "3.7"
            postgres-version: "11"

          - python-version: "3.11"
@@ -96,11 +96,7 @@ jobs:
    if: needs.check_repo.outputs.should_run_workflow == 'true'
    runs-on: ubuntu-latest
    container:
-      # We're using ubuntu:focal because it uses Python 3.8 which is our minimum supported Python version.
-      # This job is a canary to warn us about unreleased twisted changes that would cause problems for us if
-      # they were to be released immediately. For simplicity's sake (and to save CI runners) we use the oldest
-      # version, assuming that any incompatibilities on newer versions would also be present on the oldest.
-      image: matrixdotorg/sytest-synapse:focal
+      image: matrixdotorg/sytest-synapse:buster
      volumes:
        - ${{ github.workspace }}:/src

@@ -34,7 +34,6 @@ __pycache__/
 /logs
 /media_store/
 /uploads
-/homeserver-config-overrides.d

 # For direnv users
 /.envrc
@@ -13,9 +13,9 @@ dependencies = [

 [[package]]
 name = "anyhow"
-version = "1.0.72"
+version = "1.0.71"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3b13c32d80ecc7ab747b80c3784bce54ee8a7a0cc4fbda9bf4cda2cf6fe90854"
+checksum = "9c7d0618f0e0b7e8ff11427422b64564d5fb0be1940354bfe2e0529b18a9d9b8"

 [[package]]
 name = "arc-swap"
@@ -182,9 +182,9 @@ dependencies = [

 [[package]]
 name = "proc-macro2"
-version = "1.0.64"
+version = "1.0.52"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "78803b62cbf1f46fde80d7c0e803111524b9877184cfe7c3033659490ac7a7da"
+checksum = "1d0e1ae9e836cc3beddd63db0df682593d7e2d3d891ae8c9083d2113e1744224"
 dependencies = [
 "unicode-ident",
 ]
@@ -229,9 +229,9 @@ dependencies = [

 [[package]]
 name = "pyo3-log"
-version = "0.8.3"
+version = "0.8.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f47b0777feb17f61eea78667d61103758b243a871edc09a7786500a50467b605"
+checksum = "c94ff6535a6bae58d7d0b85e60d4c53f7f84d0d0aa35d6a28c3f3e70bfe51444"
 dependencies = [
 "arc-swap",
 "log",
@@ -273,9 +273,9 @@ dependencies = [

 [[package]]
 name = "quote"
-version = "1.0.29"
+version = "1.0.26"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "573015e8ab27661678357f27dc26460738fd2b6c86e46f386fde94cb5d913105"
+checksum = "4424af4bf778aae2051a77b60283332f386554255d722233d09fbfc7e30da2fc"
 dependencies = [
 "proc-macro2",
 ]
@@ -291,21 +291,9 @@ dependencies = [

 [[package]]
 name = "regex"
-version = "1.9.1"
+version = "1.8.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b2eae68fc220f7cf2532e4494aded17545fce192d59cd996e0fe7887f4ceb575"
-dependencies = [
- "aho-corasick",
- "memchr",
- "regex-automata",
- "regex-syntax",
-]
-
-[[package]]
-name = "regex-automata"
-version = "0.3.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "83d3daa6976cffb758ec878f108ba0e062a45b2d6ca3a2cca965338855476caf"
+checksum = "d0ab3ca65655bb1e41f2a8c8cd662eb4fb035e67c3f78da1d61dffe89d07300f"
 dependencies = [
 "aho-corasick",
 "memchr",
@@ -314,9 +302,9 @@ dependencies = [

 [[package]]
 name = "regex-syntax"
-version = "0.7.3"
+version = "0.7.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2ab07dc67230e4a4718e70fd5c20055a4334b121f1f9db8fe63ef39ce9b8c846"
+checksum = "436b050e76ed2903236f032a59761c1eb99e1b0aead2c257922771dab1fc8c78"

 [[package]]
 name = "ryu"
@@ -332,29 +320,29 @@ checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd"

 [[package]]
 name = "serde"
-version = "1.0.175"
+version = "1.0.164"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5d25439cd7397d044e2748a6fe2432b5e85db703d6d097bd014b3c0ad1ebff0b"
+checksum = "9e8c8cf938e98f769bc164923b06dce91cea1751522f46f8466461af04c9027d"
 dependencies = [
 "serde_derive",
 ]

 [[package]]
 name = "serde_derive"
-version = "1.0.175"
+version = "1.0.164"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b23f7ade6f110613c0d63858ddb8b94c1041f550eab58a16b371bdf2c9c80ab4"
+checksum = "d9735b638ccc51c28bf6914d90a2e9725b377144fc612c49a611fddd1b631d68"
 dependencies = [
 "proc-macro2",
 "quote",
- "syn 2.0.25",
+ "syn 2.0.10",
 ]

 [[package]]
 name = "serde_json"
-version = "1.0.103"
+version = "1.0.96"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d03b412469450d4404fe8499a268edd7f8b79fecb074b0d812ad64ca21f4031b"
+checksum = "057d394a50403bcac12672b2b18fb387ab6d289d957dab67dd201875391e52f1"
 dependencies = [
 "itoa",
 "ryu",
@@ -386,9 +374,9 @@ dependencies = [

 [[package]]
 name = "syn"
-version = "2.0.25"
+version = "2.0.10"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "15e3fc8c0c74267e2df136e5e5fb656a464158aa57624053375eb9c8c6e25ae2"
+checksum = "5aad1363ed6d37b84299588d62d3a7d95b5a5c2d9aad5c85609fda12afaa1f40"
 dependencies = [
 "proc-macro2",
 "quote",
@@ -3,4 +3,3 @@

 [workspace]
 members = ["rust"]
-resolver = "2"
@@ -0,0 +1 @@
+Allow for the configuration of max request retries and min/max retry delays in the matrix federation client.
@@ -0,0 +1 @@
+Log when events are (maybe unexpectedly) filtered out of responses in tests.
@@ -0,0 +1 @@
+Replace `EventContext` fields `prev_group` and `delta_ids` with field `state_group_deltas`.
@@ -0,0 +1 @@
+Stable support for [MSC3882](https://github.com/matrix-org/matrix-spec-proposals/pull/3882) to allow an existing device/session to generate a login token for use on a new device/session.
@@ -0,0 +1 @@
+Support resolving a room's [canonical alias](https://spec.matrix.org/v1.7/client-server-api/#mroomcanonical_alias) via the module API.
@@ -0,0 +1 @@
+Enable support for [MSC3952](https://github.com/matrix-org/matrix-spec-proposals/pull/3952): intentional mentions.
@@ -0,0 +1 @@
+Experimental [MSC3861](https://github.com/matrix-org/matrix-spec-proposals/pull/3861) support: delegate auth to an OIDC provider.
@@ -0,0 +1 @@
+Correctly clear caches when we delete a room.
@@ -0,0 +1 @@
+Read from column `full_user_id` rather than `user_id` of tables `profiles` and `user_filters`.
@@ -0,0 +1 @@
+Add support for tracing functions which return `Awaitable`s.
@@ -0,0 +1 @@
+Add Syanpse version deploy annotations to Grafana dashboard which enables easy correlation between behavior changes witnessed in a graph to a certain Synapse version and nail down regressions.
@@ -0,0 +1 @@
+Cache requests for user's devices over federation.
@@ -0,0 +1 @@
+Add fully qualified docker image names to Dockerfiles.
@@ -0,0 +1 @@
+Remove some unused code.
@@ -0,0 +1 @@
+Improve type hints.
@@ -0,0 +1 @@
+Check permissions for enabling encryption earlier during room creation to avoid creating broken rooms.
@@ -0,0 +1 @@
+Improve type hints.
@@ -0,0 +1 @@
+Add a catch-all * to the supported relation types when redacting an event and its related events. This is an update to [MSC3912](https://github.com/matrix-org/matrix-spec-proposals/pull/3861) implementation.
@@ -0,0 +1 @@
+Update docstring and traces on `maybe_backfill()` functions.
@@ -0,0 +1 @@
+Speed up `/messages` by backfilling in the background when there are no backward extremities where we are directly paginating.
@@ -0,0 +1 @@
+Add context for when/why to use the `long_retries` option when sending Federation requests.
@@ -0,0 +1 @@
+Removed some unused fields.
@@ -0,0 +1 @@
+Update federation error to more plainly explain we can only authorize our own membership events.
@@ -0,0 +1 @@
+Prevent the `latest_deps` and `twisted_trunk` daily GitHub Actions workflows from running on forks of the codebase.
@@ -0,0 +1 @@
+Improve performance of user directory search.
@@ -0,0 +1 @@
+Remove redundant table join with `room_memberships` when doing a `is_host_joined()`/`is_host_invited()` call (`membership` is already part of the `current_state_events`).
@@ -0,0 +1 @@
+Simplify query to find participating servers in a room.
@@ -0,0 +1 @@
+Remove superfluous `room_memberships` join from background update.
@@ -0,0 +1 @@
+Improve `/messages` response time by avoiding backfill when we already have messages to return.
@@ -0,0 +1 @@
+Expose a metric reporting the database background update status.
@@ -0,0 +1 @@
+Speed up typechecking CI.
@@ -0,0 +1 @@
+Fix requesting multiple keys at once over federation, related to [MSC3983](https://github.com/matrix-org/matrix-spec-proposals/pull/3983).
@@ -0,0 +1 @@
+Bump minimum supported Rust version to 1.60.0.
@@ -0,0 +1 @@
+Fix requesting multiple keys at once over federation, related to [MSC3983](https://github.com/matrix-org/matrix-spec-proposals/pull/3983).
@@ -0,0 +1 @@
+Use parse_duration for federation client timeout and retry options.
@@ -63,7 +63,7 @@
          "uid": "${DS_PROMETHEUS}"
        },
        "enable": true,
-        "expr": "changes(process_start_time_seconds{instance=\"$instance\",job=~\"synapse\"}[$bucket_size]) * on (instance, job) group_left(version) synapse_build_info{instance=\"$instance\",job=\"synapse\"}",
+        "expr": "changes(process_start_time_seconds{instance=\"matrix.org\",job=~\"synapse\"}[$bucket_size]) * on (instance, job) group_left(version) synapse_build_info{instance=\"matrix.org\",job=\"synapse\"}",
        "iconColor": "purple",
        "name": "deploys",
        "titleFormat": "Deployed {{version}}"
@@ -29,7 +29,7 @@
        "level": "error"
      },
      {
-        "line": "my-matrix-server-federation-sender-1 | 2023-01-25 20:56:20,995 - synapse.http.matrixfederationclient - 709 - WARNING - federation_transaction_transmission_loop-3 - {PUT-O-3} [example.com] Request failed: PUT matrix-federation://example.com/_matrix/federation/v1/send/1674680155797: HttpResponseException('403: Forbidden')",
+        "line": "my-matrix-server-federation-sender-1 | 2023-01-25 20:56:20,995 - synapse.http.matrixfederationclient - 709 - WARNING - federation_transaction_transmission_loop-3 - {PUT-O-3} [example.com] Request failed: PUT matrix://example.com/_matrix/federation/v1/send/1674680155797: HttpResponseException('403: Forbidden')",
        "level": "warning"
      },
      {
@@ -1,51 +1,3 @@
-matrix-synapse-py3 (1.89.0~rc1) stable; urgency=medium
-
-  * New Synapse release 1.89.0rc1.
-
- -- Synapse Packaging team <packages@matrix.org>  Tue, 25 Jul 2023 14:31:07 +0200
-
-matrix-synapse-py3 (1.88.0) stable; urgency=medium
-
-  * New Synapse release 1.88.0.
-
- -- Synapse Packaging team <packages@matrix.org>  Tue, 18 Jul 2023 13:59:28 +0100
-
-matrix-synapse-py3 (1.88.0~rc1) stable; urgency=medium
-
-  * New Synapse release 1.88.0rc1.
-
- -- Synapse Packaging team <packages@matrix.org>  Tue, 11 Jul 2023 10:20:19 +0100
-
-matrix-synapse-py3 (1.87.0) stable; urgency=medium
-
-  * New Synapse release 1.87.0.
-
- -- Synapse Packaging team <packages@matrix.org>  Tue, 04 Jul 2023 16:24:00 +0100
-
-matrix-synapse-py3 (1.87.0~rc1) stable; urgency=medium
-
-  * New synapse release 1.87.0rc1.
-
- -- Synapse Packaging team <packages@matrix.org>  Tue, 27 Jun 2023 15:27:04 +0000
-
-matrix-synapse-py3 (1.86.0) stable; urgency=medium
-
-  * New Synapse release 1.86.0.
-
- -- Synapse Packaging team <packages@matrix.org>  Tue, 20 Jun 2023 17:22:46 +0200
-
-matrix-synapse-py3 (1.86.0~rc2) stable; urgency=medium
-
-  * New Synapse release 1.86.0rc2.
-
- -- Synapse Packaging team <packages@matrix.org>  Wed, 14 Jun 2023 12:16:27 +0200
-
-matrix-synapse-py3 (1.86.0~rc1) stable; urgency=medium
-
-  * New Synapse release 1.86.0rc1.
-
- -- Synapse Packaging team <packages@matrix.org>  Tue, 13 Jun 2023 14:30:45 +0200
-
 matrix-synapse-py3 (1.85.2) stable; urgency=medium

  * New Synapse release 1.85.2.
@@ -28,12 +28,12 @@ FROM docker.io/library/${distro} as builder

 RUN apt-get update -qq -o Acquire::Languages=none
 RUN env DEBIAN_FRONTEND=noninteractive apt-get install \
-    -yqq --no-install-recommends \
-    build-essential \
-    ca-certificates \
-    devscripts \
-    equivs \
-    wget
+        -yqq --no-install-recommends \
+        build-essential \
+        ca-certificates \
+        devscripts \
+        equivs \
+        wget

 # fetch and unpack the package
 # We are temporarily using a fork of dh-virtualenv due to an incompatibility with Python 3.11, which ships with
@@ -62,29 +62,33 @@ FROM docker.io/library/${distro}
 ARG distro=""
 ENV distro ${distro}

+# Python < 3.7 assumes LANG="C" means ASCII-only and throws on printing unicode
+# http://bugs.python.org/issue19846
+ENV LANG C.UTF-8
+
 # Install the build dependencies
 #
 # NB: keep this list in sync with the list of build-deps in debian/control
 # TODO: it would be nice to do that automatically.
 RUN apt-get update -qq -o Acquire::Languages=none \
    && env DEBIAN_FRONTEND=noninteractive apt-get install \
-    -yqq --no-install-recommends -o Dpkg::Options::=--force-unsafe-io \
-    build-essential \
-    curl \
-    debhelper \
-    devscripts \
-    libsystemd-dev \
-    lsb-release \
-    pkg-config \
-    python3-dev \
-    python3-pip \
-    python3-setuptools \
-    python3-venv \
-    sqlite3 \
-    libpq-dev \
-    libicu-dev \
-    pkg-config \
-    xmlsec1
+        -yqq --no-install-recommends -o Dpkg::Options::=--force-unsafe-io \
+        build-essential \
+        curl \
+        debhelper \
+        devscripts \
+        libsystemd-dev \
+        lsb-release \
+        pkg-config \
+        python3-dev \
+        python3-pip \
+        python3-setuptools \
+        python3-venv \
+        sqlite3 \
+        libpq-dev \
+        libicu-dev \
+        pkg-config \
+        xmlsec1

 # Install rust and ensure it's in the PATH
 ENV RUSTUP_HOME=/rust
@@ -92,6 +92,8 @@ allow_device_name_lookup_over_federation: true
 ## Experimental Features ##

 experimental_features:
+  # Enable history backfilling support
+  msc2716_enabled: true
  # client-side support for partial state in /send_join responses
  faster_joins: true
  # Enable support for polls
@@ -35,11 +35,7 @@ server {

    # Send all other traffic to the main process
    location ~* ^(\\/_matrix|\\/_synapse) {
-{% if using_unix_sockets %}
-        proxy_pass http://unix:/run/main_public.sock;
-{% else %}
        proxy_pass http://localhost:8080;
-{% endif %}
        proxy_set_header X-Forwarded-For $remote_addr;
        proxy_set_header X-Forwarded-Proto $scheme;
        proxy_set_header Host $host;
@@ -6,9 +6,6 @@
 {% if enable_redis %}
 redis:
    enabled: true
-    {% if using_unix_sockets %}
-    path: /tmp/redis.sock
-    {% endif %}
 {% endif %}

 {% if appservice_registrations is not none %}
@@ -19,11 +19,7 @@ username=www-data
 autorestart=true

 [program:redis]
-{% if using_unix_sockets %}
-command=/usr/local/bin/prefix-log /usr/local/bin/redis-server --unixsocket /tmp/redis.sock
-{% else %}
 command=/usr/local/bin/prefix-log /usr/local/bin/redis-server
-{% endif %}
 priority=1
 stdout_logfile=/dev/stdout
 stdout_logfile_maxbytes=0
@@ -8,11 +8,7 @@ worker_name: "{{ name }}"

 worker_listeners:
  - type: http
-{% if using_unix_sockets %}
-    path: "/run/worker.{{ port }}"
-{% else %}
    port: {{ port }}
-{% endif %}
 {% if listener_resources %}
    resources:
      - names:
@@ -36,17 +36,12 @@ listeners:

  # Allow configuring in case we want to reverse proxy 8008
  # using another process in the same container
-{% if SYNAPSE_USE_UNIX_SOCKET %}
-  # Unix sockets don't care about TLS or IP addresses or ports
-  - path: '/run/main_public.sock'
-    type: http
-{% else %}
  - port: {{ SYNAPSE_HTTP_PORT or 8008 }}
    tls: false
    bind_addresses: ['::']
    type: http
    x_forwarded: false
-{% endif %}
+
    resources:
      - names: [client]
        compress: true
@@ -62,11 +57,8 @@ database:
    user: "{{ POSTGRES_USER or "synapse" }}"
    password: "{{ POSTGRES_PASSWORD }}"
    database: "{{ POSTGRES_DB or "synapse" }}"
-{% if not SYNAPSE_USE_UNIX_SOCKET %}
-{# Synapse will use a default unix socket for Postgres when host/port is not specified (behavior from `psycopg2`). #}
    host: "{{ POSTGRES_HOST or "db" }}"
    port: "{{ POSTGRES_PORT or "5432" }}"
-{% endif %}
    cp_min: 5
    cp_max: 10
 {% else %}
@@ -74,9 +74,6 @@ MAIN_PROCESS_HTTP_LISTENER_PORT = 8080
 MAIN_PROCESS_INSTANCE_NAME = "main"
 MAIN_PROCESS_LOCALHOST_ADDRESS = "127.0.0.1"
 MAIN_PROCESS_REPLICATION_PORT = 9093
-# Obviously, these would only be used with the UNIX socket option
-MAIN_PROCESS_UNIX_SOCKET_PUBLIC_PATH = "/run/main_public.sock"
-MAIN_PROCESS_UNIX_SOCKET_PRIVATE_PATH = "/run/main_private.sock"

 # A simple name used as a placeholder in the WORKERS_CONFIG below. This will be replaced
 # during processing with the name of the worker.
@@ -247,6 +244,7 @@ WORKERS_CONFIG: Dict[str, Dict[str, Any]] = {
            "^/_matrix/client/(api/v1|r0|v3|unstable)/join/",
            "^/_matrix/client/(api/v1|r0|v3|unstable)/knock/",
            "^/_matrix/client/(api/v1|r0|v3|unstable)/profile/",
+            "^/_matrix/client/(v1|unstable/org.matrix.msc2716)/rooms/.*/batch_send",
        ],
        "shared_extra_conf": {},
        "worker_extra_conf": "",
@@ -410,15 +408,11 @@ def add_worker_roles_to_shared_config(
        )

        # Map of stream writer instance names to host/ports combos
-        if os.environ.get("SYNAPSE_USE_UNIX_SOCKET", False):
-            instance_map[worker_name] = {
-                "path": f"/run/worker.{worker_port}",
-            }
-        else:
-            instance_map[worker_name] = {
-                "host": "localhost",
-                "port": worker_port,
-            }
+        instance_map[worker_name] = {
+            "host": "localhost",
+            "port": worker_port,
+        }
+
    # Update the list of stream writers. It's convenient that the name of the worker
    # type is the same as the stream to write. Iterate over the whole list in case there
    # is more than one.
@@ -430,15 +424,10 @@ def add_worker_roles_to_shared_config(

            # Map of stream writer instance names to host/ports combos
            # For now, all stream writers need http replication ports
-            if os.environ.get("SYNAPSE_USE_UNIX_SOCKET", False):
-                instance_map[worker_name] = {
-                    "path": f"/run/worker.{worker_port}",
-                }
-            else:
-                instance_map[worker_name] = {
-                    "host": "localhost",
-                    "port": worker_port,
-                }
+            instance_map[worker_name] = {
+                "host": "localhost",
+                "port": worker_port,
+            }


 def merge_worker_template_configs(
@@ -730,29 +719,17 @@ def generate_worker_files(
    # Note that yaml cares about indentation, so care should be taken to insert lines
    # into files at the correct indentation below.

-    # Convenience helper for if using unix sockets instead of host:port
-    using_unix_sockets = environ.get("SYNAPSE_USE_UNIX_SOCKET", False)
    # First read the original config file and extract the listeners block. Then we'll
    # add another listener for replication. Later we'll write out the result to the
    # shared config file.
-    listeners: List[Any]
-    if using_unix_sockets:
-        listeners = [
-            {
-                "path": MAIN_PROCESS_UNIX_SOCKET_PRIVATE_PATH,
-                "type": "http",
-                "resources": [{"names": ["replication"]}],
-            }
-        ]
-    else:
-        listeners = [
-            {
-                "port": MAIN_PROCESS_REPLICATION_PORT,
-                "bind_address": MAIN_PROCESS_LOCALHOST_ADDRESS,
-                "type": "http",
-                "resources": [{"names": ["replication"]}],
-            }
-        ]
+    listeners = [
+        {
+            "port": MAIN_PROCESS_REPLICATION_PORT,
+            "bind_address": MAIN_PROCESS_LOCALHOST_ADDRESS,
+            "type": "http",
+            "resources": [{"names": ["replication"]}],
+        }
+    ]
    with open(config_path) as file_stream:
        original_config = yaml.safe_load(file_stream)
        original_listeners = original_config.get("listeners")
@@ -793,17 +770,7 @@ def generate_worker_files(

    # A list of internal endpoints to healthcheck, starting with the main process
    # which exists even if no workers do.
-    # This list ends up being part of the command line to curl, (curl added support for
-    # Unix sockets in version 7.40).
-    if using_unix_sockets:
-        healthcheck_urls = [
-            f"--unix-socket {MAIN_PROCESS_UNIX_SOCKET_PUBLIC_PATH} "
-            # The scheme and hostname from the following URL are ignored.
-            # The only thing that matters is the path `/health`
-            "http://localhost/health"
-        ]
-    else:
-        healthcheck_urls = ["http://localhost:8080/health"]
+    healthcheck_urls = ["http://localhost:8080/health"]

    # Get the set of all worker types that we have configured
    all_worker_types_in_use = set(chain(*requested_worker_types.values()))
@@ -840,12 +807,8 @@ def generate_worker_files(
        # given worker_type needs to stay assigned and not be replaced.
        worker_config["shared_extra_conf"].update(shared_config)
        shared_config = worker_config["shared_extra_conf"]
-        if using_unix_sockets:
-            healthcheck_urls.append(
-                f"--unix-socket /run/worker.{worker_port} http://localhost/health"
-            )
-        else:
-            healthcheck_urls.append("http://localhost:%d/health" % (worker_port,))
+
+        healthcheck_urls.append("http://localhost:%d/health" % (worker_port,))

        # Update the shared config with sharding-related options if necessary
        add_worker_roles_to_shared_config(
@@ -864,7 +827,6 @@ def generate_worker_files(
            "/conf/workers/{name}.yaml".format(name=worker_name),
            **worker_config,
            worker_log_config_filepath=log_config_filepath,
-            using_unix_sockets=using_unix_sockets,
        )

        # Save this worker's port number to the correct nginx upstreams
@@ -885,13 +847,8 @@ def generate_worker_files(
    nginx_upstream_config = ""
    for upstream_worker_base_name, upstream_worker_ports in nginx_upstreams.items():
        body = ""
-        if using_unix_sockets:
-            for port in upstream_worker_ports:
-                body += f"    server unix:/run/worker.{port};\n"
-
-        else:
-            for port in upstream_worker_ports:
-                body += f"    server localhost:{port};\n"
+        for port in upstream_worker_ports:
+            body += f"    server localhost:{port};\n"

        # Add to the list of configured upstreams
        nginx_upstream_config += NGINX_UPSTREAM_CONFIG_BLOCK.format(
@@ -921,15 +878,10 @@ def generate_worker_files(
    # If there are workers, add the main process to the instance_map too.
    if workers_in_use:
        instance_map = shared_config.setdefault("instance_map", {})
-        if using_unix_sockets:
-            instance_map[MAIN_PROCESS_INSTANCE_NAME] = {
-                "path": MAIN_PROCESS_UNIX_SOCKET_PRIVATE_PATH,
-            }
-        else:
-            instance_map[MAIN_PROCESS_INSTANCE_NAME] = {
-                "host": MAIN_PROCESS_LOCALHOST_ADDRESS,
-                "port": MAIN_PROCESS_REPLICATION_PORT,
-            }
+        instance_map[MAIN_PROCESS_INSTANCE_NAME] = {
+            "host": MAIN_PROCESS_LOCALHOST_ADDRESS,
+            "port": MAIN_PROCESS_REPLICATION_PORT,
+        }

    # Shared homeserver config
    convert(
@@ -939,7 +891,6 @@ def generate_worker_files(
        appservice_registrations=appservice_registrations,
        enable_redis=workers_in_use,
        workers_in_use=workers_in_use,
-        using_unix_sockets=using_unix_sockets,
    )

    # Nginx config
@@ -950,7 +901,6 @@ def generate_worker_files(
        upstream_directives=nginx_upstream_config,
        tls_cert_path=os.environ.get("SYNAPSE_TLS_CERT"),
        tls_key_path=os.environ.get("SYNAPSE_TLS_KEY"),
-        using_unix_sockets=using_unix_sockets,
    )

    # Supervisord config
@@ -960,7 +910,6 @@ def generate_worker_files(
        "/etc/supervisor/supervisord.conf",
        main_config_path=config_path,
        enable_redis=workers_in_use,
-        using_unix_sockets=using_unix_sockets,
    )

    convert(
@@ -419,7 +419,7 @@ The following query parameters are available:

 * `from` (required) - The token to start returning events from. This token can be obtained from a prev_batch
  or next_batch token returned by the /sync endpoint, or from an end token returned by a previous request to this endpoint.
-* `to` - The token to stop returning events at.
+* `to` - The token to spot returning events at.
 * `limit` - The maximum number of events to return. Defaults to `10`.
 * `filter` - A JSON RoomEventFilter to filter returned events with.
 * `dir` - The direction to return events from. Either `f` for forwards or `b` for backwards. Setting
@@ -242,9 +242,6 @@ The following parameters should be set in the URL:

 - `dir` - Direction of media order. Either `f` for forwards or `b` for backwards.
  Setting this value to `b` will reverse the above sort order. Defaults to `f`.
- `not_user_type` - Exclude certain user types, such as bot users, from the request.
-   Can be provided multiple times. Possible values are `bot`, `support` or "empty string".
-   "empty string" here means to exclude users without a type.

 Caution. The database only has indexes on the columns `name` and `creation_ts`.
 This means that if a different sort order is used (`is_guest`, `admin`,
@@ -732,8 +729,7 @@ POST /_synapse/admin/v1/users/<user_id>/login

 An optional `valid_until_ms` field can be specified in the request body as an
 integer timestamp that specifies when the token should expire. By default tokens
-do not expire. Note that this API does not allow a user to login as themselves
-(to create more tokens).
+do not expire.

 A response body like the following is returned:

@@ -1184,7 +1180,7 @@ The following parameters should be set in the URL:
 - `user_id` - The fully qualified MXID: for example, `@user:server.com`. The user must
  be local.

-## Check username availability
+### Check username availability

 Checks to see if a username is available, and valid, for the server. See [the client-server 
 API](https://matrix.org/docs/spec/client_server/r0.6.0#get-matrix-client-r0-register-available)
@@ -1202,7 +1198,7 @@ GET /_synapse/admin/v1/username_available?username=$localpart
 The request and response format is the same as the
 [/_matrix/client/r0/register/available](https://matrix.org/docs/spec/client_server/r0.6.0#get-matrix-client-r0-register-available) API.

-## Find a user based on their ID in an auth provider
+### Find a user based on their ID in an auth provider

 The API is:

@@ -1241,7 +1237,7 @@ Returns a `404` HTTP status code if no user was found, with a response body like
 _Added in Synapse 1.68.0._


-## Find a user based on their Third Party ID (ThreePID or 3PID)
+### Find a user based on their Third Party ID (ThreePID or 3PID)

 The API is:

@@ -23,7 +23,7 @@ people building from source should ensure they can fetch recent versions of Rust
 (e.g. by using [rustup](https://rustup.rs/)).

 The oldest supported version of SQLite is the version
-[provided](https://packages.debian.org/bullseye/libsqlite3-0) by
+[provided](https://packages.debian.org/buster/libsqlite3-0) by
 [Debian oldstable](https://wiki.debian.org/DebianOldStable).

 Context
@@ -322,7 +322,7 @@ The following command will let you run the integration test with the most common
 configuration:

 ```sh
-$ docker run --rm -it -v /path/where/you/have/cloned/the/repository\:/src:ro -v /path/to/where/you/want/logs\:/logs matrixdotorg/sytest-synapse:focal
+$ docker run --rm -it -v /path/where/you/have/cloned/the/repository\:/src:ro -v /path/to/where/you/want/logs\:/logs matrixdotorg/sytest-synapse:buster
 ```
 (Note that the paths must be full paths! You could also write `$(realpath relative/path)` if needed.)

@@ -370,7 +370,6 @@ The above will run a monolithic (single-process) Synapse with SQLite as the data
    See the [worker documentation](../workers.md) for additional information on workers.
 - Passing `ASYNCIO_REACTOR=1` as an environment variable to use the Twisted asyncio reactor instead of the default one.
 - Passing `PODMAN=1` will use the [podman](https://podman.io/) container runtime, instead of docker.
- Passing `UNIX_SOCKETS=1` will utilise Unix socket functionality for Synapse, Redis, and Postgres(when applicable).

 To increase the log level for the tests, set `SYNAPSE_TEST_LOG_LEVEL`, e.g:
 ```sh
@@ -6,7 +6,7 @@ This is a work-in-progress set of notes with two goals:

 See also [MSC3902](https://github.com/matrix-org/matrix-spec-proposals/pull/3902).

-The key idea is described by [MSC3706](https://github.com/matrix-org/matrix-spec-proposals/pull/3706). This allows servers to
+The key idea is described by [MSC706](https://github.com/matrix-org/matrix-spec-proposals/pull/3902). This allows servers to
 request a lightweight response to the federation `/send_join` endpoint.
 This is called a **faster join**, also known as a **partial join**. In these
 notes we'll usually use the word "partial" as it matches the database schema.
@@ -348,42 +348,6 @@ callback returns `False`, Synapse falls through to the next one. The value of th
 callback that does not return `False` will be used. If this happens, Synapse will not call
 any of the subsequent implementations of this callback.

-
-### `check_login_for_spam`
-
-_First introduced in Synapse v1.87.0_
-
-```python
-async def check_login_for_spam(
-    user_id: str,
-    device_id: Optional[str],
-    initial_display_name: Optional[str],
-    request_info: Collection[Tuple[Optional[str], str]],
-    auth_provider_id: Optional[str] = None,
-) -> Union["synapse.module_api.NOT_SPAM", "synapse.module_api.errors.Codes"]
-```
-
-Called when a user logs in.
-
-The arguments passed to this callback are:
-
-* `user_id`: The user ID the user is logging in with
-* `device_id`: The device ID the user is re-logging into.
-* `initial_display_name`: The device display name, if any.
-* `request_info`: A collection of tuples, which first item is a user agent, and which
-  second item is an IP address. These user agents and IP addresses are the ones that were
-  used during the login process.
-* `auth_provider_id`: The identifier of the SSO authentication provider, if any.
-
-If multiple modules implement this callback, they will be considered in order. If a
-callback returns `synapse.module_api.NOT_SPAM`, Synapse falls through to the next one.
-The value of the first callback that does not return `synapse.module_api.NOT_SPAM` will
-be used. If this happens, Synapse will not call any of the subsequent implementations of
-this callback.
-
-*Note:* This will not be called when a user registers.
-
-
 ## Example

 The example below is a module that implements the spam checker callback
@@ -135,8 +135,8 @@ Unofficial package are built for SLES 15 in the openSUSE:Backports:SLE-15 reposi

 #### ArchLinux

-The quickest way to get up and running with ArchLinux is probably with the package provided by ArchLinux
-<https://archlinux.org/packages/extra/x86_64/matrix-synapse/>, which should pull in most of
+The quickest way to get up and running with ArchLinux is probably with the community package
+<https://archlinux.org/packages/community/x86_64/matrix-synapse/>, which should pull in most of
 the necessary dependencies.

 pip may be outdated (6.0.7-1 and needs to be upgraded to 6.0.8-1 ):
@@ -200,7 +200,7 @@ When following this route please make sure that the [Platform-specific prerequis
 System requirements:

 - POSIX-compliant system (tested on Linux & OS X)
- Python 3.8 or later, up to Python 3.11.
+- Python 3.7 or later, up to Python 3.11.
 - At least 1GB of free RAM if you want to join large public rooms like #matrix:matrix.org

 If building on an uncommon architecture for which pre-built wheels are
@@ -1,4 +1,8 @@
 worker_app: synapse.app.generic_worker
 worker_name: background_worker

+# The replication listener on the main synapse process.
+worker_replication_host: 127.0.0.1
+worker_replication_http_port: 9093
+
 worker_log_config: /etc/matrix-synapse/background-worker-log.yaml
@@ -1,5 +1,9 @@
 worker_app: synapse.app.generic_worker
-worker_name: event_persister1
+worker_name: event_persister1 
+
+# The replication listener on the main synapse process.
+worker_replication_host: 127.0.0.1
+worker_replication_http_port: 9093

 worker_listeners:
  - type: http
@@ -1,4 +1,8 @@
 worker_app: synapse.app.federation_sender
 worker_name: federation_sender1

+# The replication listener on the main synapse process.
+worker_replication_host: 127.0.0.1
+worker_replication_http_port: 9093
+
 worker_log_config: /etc/matrix-synapse/federation-sender-log.yaml
@@ -1,6 +1,10 @@
 worker_app: synapse.app.media_repository
 worker_name: media_worker

+# The replication listener on the main synapse process.
+worker_replication_host: 127.0.0.1
+worker_replication_http_port: 9093
+
 worker_listeners:
  - type: http
    port: 8085
@@ -1,4 +1,8 @@
 worker_app: synapse.app.pusher
 worker_name: pusher_worker1

+# The replication listener on the main synapse process.
+worker_replication_host: 127.0.0.1
+worker_replication_http_port: 9093
+
 worker_log_config: /etc/matrix-synapse/pusher-worker-log.yaml
@@ -87,43 +87,6 @@ process, for example:
    wget https://packages.matrix.org/debian/pool/main/m/matrix-synapse-py3/matrix-synapse-py3_1.3.0+stretch1_amd64.deb
    dpkg -i matrix-synapse-py3_1.3.0+stretch1_amd64.deb
    ```
-
-# Upgrading to v1.89.0
-
-## Removal of unspecced `user` property for `/register`
-
-Application services can no longer call `/register` with a `user` property to create new users.
-The standard `username` property should be used instead. See the
-[Application Service specification](https://spec.matrix.org/v1.7/application-service-api/#server-admin-style-permissions)
-for more information.
-
-
-# Upgrading to v1.88.0
-
-## Minimum supported Python version
-
-The minimum supported Python version has been increased from v3.7 to v3.8.
-You will need Python 3.8 to run Synapse v1.88.0 (due out July 18th, 2023).
-
-If you use current versions of the Matrix.org-distributed Debian
-packages or Docker images, no action is required.
-
-## Removal of `worker_replication_*` settings
-
-As mentioned previously in [Upgrading to v1.84.0](#upgrading-to-v1840), the following deprecated settings
-are being removed in this release of Synapse:
-
-* [`worker_replication_host`](https://matrix-org.github.io/synapse/v1.86/usage/configuration/config_documentation.html#worker_replication_host)
-* [`worker_replication_http_port`](https://matrix-org.github.io/synapse/v1.86/usage/configuration/config_documentation.html#worker_replication_http_port)
-* [`worker_replication_http_tls`](https://matrix-org.github.io/synapse/v1.86/usage/configuration/config_documentation.html#worker_replication_http_tls)
-
-Please ensure that you have migrated to using `main` on your shared configuration's `instance_map`
-(or create one if necessary). This is required if you have ***any*** workers at all;
-administrators of single-process (monolith) installations don't need to do anything.
-
-For an illustrative example, please see [Upgrading to v1.84.0](#upgrading-to-v1840) below.
-
-
 # Upgrading to v1.86.0

 ## Minimum supported Rust version
@@ -462,20 +462,6 @@ See the docs [request log format](../administration/request_log.md).
 * `additional_resources`: Only valid for an 'http' listener. A map of
   additional endpoints which should be loaded via dynamic modules.

-Unix socket support (_Added in Synapse 1.89.0_):
-* `path`: A path and filename for a Unix socket. Make sure it is located in a
-  directory with read and write permissions, and that it already exists (the directory
-  will not be created). Defaults to `None`.
-  * **Note**: The use of both `path` and `port` options for the same `listener` is not
-    compatible.
-  * The `x_forwarded` option defaults to true  when using Unix sockets and can be omitted.
-  * Other options that would not make sense to use with a UNIX socket, such as 
-    `bind_addresses` and `tls` will be ignored and can be removed.
-* `mode`: The file permissions to set on the UNIX socket. Defaults to `666`
-* **Note:** Must be set as `type: http` (does not support `metrics` and `manhole`). 
-  Also make sure that `metrics` is not included in `resources` -> `names`
-
-
 Valid resource names are:

 * `client`: the client-server API (/_matrix/client), and the synapse admin API (/_synapse/admin). Also implies `media` and `static`.
@@ -488,7 +474,7 @@ Valid resource names are:

 * `media`: the media API (/_matrix/media).

-* `metrics`: the metrics interface. See [here](../../metrics-howto.md). (Not compatible with Unix sockets)
+* `metrics`: the metrics interface. See [here](../../metrics-howto.md).

 * `openid`: OpenID authentication. See [here](../../openid.md).

@@ -547,22 +533,6 @@ listeners:
    bind_addresses: ['::1', '127.0.0.1']
    type: manhole
 ```
-Example configuration #3:
-```yaml
-listeners:
-  # Unix socket listener: Ideal for Synapse deployments behind a reverse proxy, offering
-  # lightweight interprocess communication without TCP/IP overhead, avoid port
-  # conflicts, and providing enhanced security through system file permissions.
-  #
-  # Note that x_forwarded will default to true, when using a UNIX socket. Please see
-  # https://matrix-org.github.io/synapse/latest/reverse_proxy.html.
-  #
-  - path: /var/run/synapse/main_public.sock
-    type: http
-    resources:
-      - names: [client, federation]
-```
-
 ---
 ### `manhole_settings`

@@ -1236,18 +1206,18 @@ Short retry algorithm is used when something or someone will wait for the reques
 answer, while long retry is used for requests that happen in the background,
 like sending a federation transaction.

-* `client_timeout`: timeout for the federation requests. Default to 60s.
-* `max_short_retry_delay`: maximum delay to be used for the short retry algo. Default to 2s.
-* `max_long_retry_delay`: maximum delay to be used for the short retry algo. Default to 60s.
+* `client_timeout`: timeout for the federation requests in seconds. Default to 60s.
+* `max_short_retry_delay`: maximum delay to be used for the short retry algo in seconds. Default to 2s.
+* `max_long_retry_delay`: maximum delay to be used for the short retry algo in seconds. Default to 60s.
 * `max_short_retries`: maximum number of retries for the short retry algo. Default to 3 attempts.
 * `max_long_retries`: maximum number of retries for the long retry algo. Default to 10 attempts.

 Example configuration:
 ```yaml
 federation:
-  client_timeout: 180s
-  max_short_retry_delay: 7s
-  max_long_retry_delay: 100s
+  client_timeout: 180
+  max_short_retry_delay: 7
+  max_long_retry_delay: 100
  max_short_retries: 5
  max_long_retries: 20
 ```
@@ -3960,14 +3930,13 @@ federation_sender_instances:
 ---
 ### `instance_map`

-When using workers this should be a map from [`worker_name`](#worker_name) to the HTTP
-replication listener of the worker, if configured, and to the main process. Each worker
-declared under [`stream_writers`](../../workers.md#stream-writers) and
-[`outbound_federation_restricted_to`](#outbound_federation_restricted_to) needs a HTTP
-replication listener, and that listener should be included in the `instance_map`. The
-main process also needs an entry on the `instance_map`, and it should be listed under
-`main` **if even one other worker exists**. Ensure the port matches with what is
-declared inside the `listener` block for a `replication` listener.
+When using workers this should be a map from [`worker_name`](#worker_name) to the
+HTTP replication listener of the worker, if configured, and to the main process.
+Each worker declared under [`stream_writers`](../../workers.md#stream-writers) needs
+a HTTP replication listener, and that listener should be included in the `instance_map`.
+The main process also needs an entry on the `instance_map`, and it should be listed under
+`main` **if even one other worker exists**. Ensure the port matches with what is declared 
+inside the `listener` block for a `replication` listener.


 Example configuration:
@@ -3980,14 +3949,6 @@ instance_map:
    host: localhost
    port: 8034
 ```
-Example configuration(#2, for UNIX sockets):
-```yaml
-instance_map:
-  main:
-    path: /var/run/synapse/main_replication.sock
-  worker1:
-    path: /var/run/synapse/worker1_replication.sock
-```
 ---
 ### `stream_writers`

@@ -4005,24 +3966,6 @@ stream_writers:
  typing: worker1
 ```
 ---
-### `outbound_federation_restricted_to`
-
-When using workers, you can restrict outbound federation traffic to only go through a
-specific subset of workers. Any worker specified here must also be in the
-[`instance_map`](#instance_map).
-[`worker_replication_secret`](#worker_replication_secret) must also be configured to
-authorize inter-worker communication.
-
-```yaml
-outbound_federation_restricted_to:
-  - federation_sender1
-  - federation_sender2
-```
-
-Also see the [worker
-documentation](../../workers.md#restrict-outbound-federation-traffic-to-a-specific-set-of-workers)
-for more info.
---
 ### `run_background_tasks_on`

 The [worker](../../workers.md#background-tasks) that is used to run
@@ -4147,6 +4090,51 @@ Example configuration:
 worker_name: generic_worker1
 ```
 ---
+### `worker_replication_host`
+*Deprecated as of version 1.84.0. Place `host` under `main` entry on the [`instance_map`](#instance_map) in your shared yaml configuration instead.*
+
+The HTTP replication endpoint that it should talk to on the main Synapse process.
+The main Synapse process defines this with a `replication` resource in
+[`listeners` option](#listeners).
+
+Example configuration:
+```yaml
+worker_replication_host: 127.0.0.1
+```
+---
+### `worker_replication_http_port`
+*Deprecated as of version 1.84.0. Place `port` under `main` entry on the [`instance_map`](#instance_map) in your shared yaml configuration instead.*
+
+The HTTP replication port that it should talk to on the main Synapse process.
+The main Synapse process defines this with a `replication` resource in
+[`listeners` option](#listeners).
+
+Example configuration:
+```yaml
+worker_replication_http_port: 9093
+```
+---
+### `worker_replication_http_tls`
+*Deprecated as of version 1.84.0. Place `tls` under `main` entry on the [`instance_map`](#instance_map) in your shared yaml configuration instead.*
+
+Whether TLS should be used for talking to the HTTP replication port on the main
+Synapse process.
+The main Synapse process defines this with the `tls` option on its [listener](#listeners) that
+has the `replication` resource enabled.
+
+**Please note:** by default, it is not safe to expose replication ports to the
+public Internet, even with TLS enabled.
+See [`worker_replication_secret`](#worker_replication_secret).
+
+Defaults to `false`.
+
+*Added in Synapse 1.72.0.*
+
+Example configuration:
+```yaml
+worker_replication_http_tls: true
+```
+---
 ### `worker_listeners`

 A worker can handle HTTP requests. To do so, a `worker_listeners` option
@@ -4165,18 +4153,6 @@ worker_listeners:
    resources:
      - names: [client, federation]
 ```
-Example configuration(#2, using UNIX sockets with a `replication` listener):
-```yaml
-worker_listeners:
-  - type: http
-    path: /var/run/synapse/worker_public.sock
-    resources:
-      - names: [client, federation]
-  - type: http
-    path: /var/run/synapse/worker_replication.sock
-    resources:
-      - names: [replication]
-```
 ---
 ### `worker_manhole`

@@ -95,12 +95,9 @@ for the main process
 * Secondly, you need to enable
 [redis-based replication](usage/configuration/config_documentation.md#redis)
 * You will need to add an [`instance_map`](usage/configuration/config_documentation.md#instance_map) 
-with the `main` process defined, as well as the relevant connection information from
-it's HTTP `replication` listener (defined in step 1 above).
-  * Note that the `host` defined is the address the worker needs to look for the `main`
-  process at, not necessarily the same address that is bound to.
-  * If you are using Unix sockets for the `replication` resource, make sure to
-  use a `path` to the socket file instead of a `port`.
+with the `main` process defined, as well as the relevant connection information from 
+it's HTTP `replication` listener (defined in step 1 above). Note that the `host` defined 
+is the address the worker needs to look for the `main` process at, not necessarily the same address that is bound to.
 * Optionally, a [shared secret](usage/configuration/config_documentation.md#worker_replication_secret)
 can be used to authenticate HTTP traffic between workers. For example:

@@ -148,6 +145,9 @@ In the config file for each worker, you must specify:
   with an `http` listener.
 * **Synapse 1.72 and older:** if handling the `^/_matrix/client/v3/keys/upload` endpoint, the HTTP URI for
   the main process (`worker_main_http_uri`). This config option is no longer required and is ignored when running Synapse 1.73 and newer.
+ * **Synapse 1.83 and older:** The HTTP replication endpoint that the worker should talk to on the main synapse process
+   ([`worker_replication_host`](usage/configuration/config_documentation.md#worker_replication_host) and
+   [`worker_replication_http_port`](usage/configuration/config_documentation.md#worker_replication_http_port)). If using Synapse 1.84 and newer, these are not needed if `main` is defined on the [shared configuration](#shared-configuration) `instance_map`

 For example:

@@ -177,11 +177,11 @@ The following applies to Synapse installations that have been installed from sou

 You can start the main Synapse process with Poetry by running the following command:
 ```console
-poetry run synapse_homeserver --config-file [your homeserver.yaml]
+poetry run synapse_homeserver -c [your homeserver.yaml]
 ```
 For worker setups, you can run the following command
 ```console
-poetry run synapse_worker --config-file [your homeserver.yaml] --config-file [your worker.yaml]
+poetry run synapse_worker -c [your worker.yaml]
 ```
 ## Available worker applications

@@ -232,6 +232,7 @@ information.
    ^/_matrix/client/v1/rooms/.*/hierarchy$
    ^/_matrix/client/(v1|unstable)/rooms/.*/relations/
    ^/_matrix/client/v1/rooms/.*/threads$
+    ^/_matrix/client/unstable/org.matrix.msc2716/rooms/.*/batch_send$
    ^/_matrix/client/unstable/im.nheko.summary/rooms/.*/summary$
    ^/_matrix/client/(r0|v3|unstable)/account/3pid$
    ^/_matrix/client/(r0|v3|unstable)/account/whoami$
@@ -531,30 +532,6 @@ the stream writer for the `presence` stream:

    ^/_matrix/client/(api/v1|r0|v3|unstable)/presence/

-#### Restrict outbound federation traffic to a specific set of workers
-
-The
-[`outbound_federation_restricted_to`](usage/configuration/config_documentation.md#outbound_federation_restricted_to)
-configuration is useful to make sure outbound federation traffic only goes through a
-specified subset of workers. This allows you to set more strict access controls (like a
-firewall) for all workers and only allow the `federation_sender`'s to contact the
-outside world.
-
-```yaml
-instance_map:
-    main:
-        host: localhost
-        port: 8030
-    federation_sender1:
-        host: localhost
-        port: 8034
-
-outbound_federation_restricted_to:
-  - federation_sender1
-
-worker_replication_secret: "secret_secret"
-```
-
 #### Background tasks

 There is also support for moving background tasks to a separate
@@ -22,6 +22,27 @@
        "type": "github"
      }
    },
+    "fenix": {
+      "inputs": {
+        "nixpkgs": [
+          "nixpkgs"
+        ],
+        "rust-analyzer-src": "rust-analyzer-src"
+      },
+      "locked": {
+        "lastModified": 1682490133,
+        "narHash": "sha256-tR2Qx0uuk97WySpSSk4rGS/oH7xb5LykbjATcw1vw1I=",
+        "owner": "nix-community",
+        "repo": "fenix",
+        "rev": "4e9412753ab75ef0e038a5fe54a062fb44c27c6a",
+        "type": "github"
+      },
+      "original": {
+        "owner": "nix-community",
+        "repo": "fenix",
+        "type": "github"
+      }
+    },
    "flake-compat": {
      "flake": false,
      "locked": {
@@ -53,24 +74,6 @@
        "type": "github"
      }
    },
-    "flake-utils_2": {
-      "inputs": {
-        "systems": "systems"
-      },
-      "locked": {
-        "lastModified": 1681202837,
-        "narHash": "sha256-H+Rh19JDwRtpVPAWp64F+rlEtxUWBAQW28eAi3SRSzg=",
-        "owner": "numtide",
-        "repo": "flake-utils",
-        "rev": "cfacdce06f30d2b68473a46042957675eebb3401",
-        "type": "github"
-      },
-      "original": {
-        "owner": "numtide",
-        "repo": "flake-utils",
-        "type": "github"
-      }
-    },
    "gitignore": {
      "inputs": {
        "nixpkgs": [
@@ -197,22 +200,6 @@
        "type": "github"
      }
    },
-    "nixpkgs_3": {
-      "locked": {
-        "lastModified": 1681358109,
-        "narHash": "sha256-eKyxW4OohHQx9Urxi7TQlFBTDWII+F+x2hklDOQPB50=",
-        "owner": "NixOS",
-        "repo": "nixpkgs",
-        "rev": "96ba1c52e54e74c3197f4d43026b3f3d92e83ff9",
-        "type": "github"
-      },
-      "original": {
-        "owner": "NixOS",
-        "ref": "nixpkgs-unstable",
-        "repo": "nixpkgs",
-        "type": "github"
-      }
-    },
    "pre-commit-hooks": {
      "inputs": {
        "flake-compat": [
@@ -244,27 +231,25 @@
    "root": {
      "inputs": {
        "devenv": "devenv",
+        "fenix": "fenix",
        "nixpkgs": "nixpkgs_2",
-        "rust-overlay": "rust-overlay",
-        "systems": "systems_2"
+        "systems": "systems"
      }
    },
-    "rust-overlay": {
-      "inputs": {
-        "flake-utils": "flake-utils_2",
-        "nixpkgs": "nixpkgs_3"
-      },
+    "rust-analyzer-src": {
+      "flake": false,
      "locked": {
-        "lastModified": 1689302058,
-        "narHash": "sha256-yD74lcHTrw4niXcE9goJLbzsgyce48rQQoy5jK5ZK40=",
-        "owner": "oxalica",
-        "repo": "rust-overlay",
-        "rev": "7b8dbbf4c67ed05a9bf3d9e658c12d4108bc24c8",
+        "lastModified": 1682426789,
+        "narHash": "sha256-UqnLmJESRZE0tTEaGbRAw05Hm19TWIPA+R3meqi5I4w=",
+        "owner": "rust-lang",
+        "repo": "rust-analyzer",
+        "rev": "943d2a8a1ca15e8b28a1f51f5a5c135e3728da04",
        "type": "github"
      },
      "original": {
-        "owner": "oxalica",
-        "repo": "rust-overlay",
+        "owner": "rust-lang",
+        "ref": "nightly",
+        "repo": "rust-analyzer",
        "type": "github"
      }
    },
@@ -282,21 +267,6 @@
        "repo": "default",
        "type": "github"
      }
-    },
-    "systems_2": {
-      "locked": {
-        "lastModified": 1681028828,
-        "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=",
-        "owner": "nix-systems",
-        "repo": "default",
-        "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e",
-        "type": "github"
-      },
-      "original": {
-        "owner": "nix-systems",
-        "repo": "default",
-        "type": "github"
-      }
    }
  },
  "root": "root",
@@ -46,20 +46,20 @@
    systems.url = "github:nix-systems/default";
    # A development environment manager built on Nix. See https://devenv.sh.
    devenv.url = "github:cachix/devenv/main";
-    # Rust toolchain.
-    rust-overlay.url = "github:oxalica/rust-overlay";
+    # Rust toolchains and rust-analyzer nightly.
+    fenix = {
+      url = "github:nix-community/fenix";
+      inputs.nixpkgs.follows = "nixpkgs";
+    };
  };

-  outputs = { self, nixpkgs, devenv, systems, rust-overlay, ... } @ inputs:
+  outputs = { self, nixpkgs, devenv, systems, ... } @ inputs:
    let
      forEachSystem = nixpkgs.lib.genAttrs (import systems);
    in {
      devShells = forEachSystem (system:
        let
-          overlays = [ (import rust-overlay) ];
-          pkgs = import nixpkgs {
-            inherit system overlays;
-          };
+          pkgs = nixpkgs.legacyPackages.${system};
        in {
          # Everything is configured via devenv - a Nix module for creating declarative
          # developer environments. See https://devenv.sh/reference/options/ for a list
@@ -76,20 +76,6 @@
                # Configure packages to install.
                # Search for package names at https://search.nixos.org/packages?channel=unstable
                packages = with pkgs; [
-                  # The rust toolchain and related tools.
-                  # This will install the "default" profile of rust components.
-                  # https://rust-lang.github.io/rustup/concepts/profiles.html
-                  #
-                  # NOTE: We currently need to set the Rust version unnecessarily high
-                  # in order to work around https://github.com/matrix-org/synapse/issues/15939
-                  (rust-bin.stable."1.70.0".default.override {
-                    # Additionally install the "rust-src" extension to allow diving into the
-                    # Rust source code in an IDE (rust-analyzer will also make use of it).
-                    extensions = [ "rust-src" ];
-                  })
-                  # The rust-analyzer language server implementation.
-                  rust-analyzer
-
                  # Native dependencies for running Synapse.
                  icu
                  libffi
@@ -138,11 +124,12 @@
                # Install dependencies for the additional programming languages
                # involved with Synapse development.
                #
+                # * Rust is used for developing and running Synapse.
                # * Golang is needed to run the Complement test suite.
                # * Perl is needed to run the SyTest test suite.
-                # * Rust is used for developing and running Synapse.
-                #   It is installed manually with `packages` above.
                languages.go.enable = true;
+                languages.rust.enable = true;
+                languages.rust.version = "stable";
                languages.perl.enable = true;

                # Postgres is needed to run Synapse with postgres support and
@@ -191,7 +178,7 @@
                  EOF
                '';
                # Start synapse when `devenv up` is run.
-                processes.synapse.exec = "poetry run python -m synapse.app.homeserver -c homeserver.yaml -c homeserver-config-overrides.d";
+                processes.synapse.exec = "poetry run python -m synapse.app.homeserver -c homeserver.yaml --config-directory homeserver-config-overrides.d";

                # Define the perl modules we require to run SyTest.
                #
@@ -89,7 +89,7 @@ manifest-path = "rust/Cargo.toml"

 [tool.poetry]
 name = "matrix-synapse"
-version = "1.89.0rc1"
+version = "1.85.2"
 description = "Homeserver for the Matrix decentralised comms protocol"
 authors = ["Matrix.org Team and Contributors <packages@matrix.org>"]
 license = "Apache-2.0"
@@ -147,7 +147,7 @@ synapse_review_recent_signups = "synapse._scripts.review_recent_signups:main"
 update_synapse_database = "synapse._scripts.update_synapse_database:main"

 [tool.poetry.dependencies]
-python = "^3.8.0"
+python = "^3.7.1"

 # Mandatory Dependencies
 # ----------------------
@@ -203,9 +203,11 @@ ijson = ">=3.1.4"
 matrix-common = "^1.3.0"
 # We need packaging.requirements.Requirement, added in 16.1.
 packaging = ">=16.1"
+# At the time of writing, we only use functions from the version `importlib.metadata`
+# which shipped in Python 3.8. This corresponds to version 1.4 of the backport.
+importlib_metadata = { version = ">=1.4", python = "<3.8" }
 # This is the most recent version of Pydantic with available on common distros.
-# We are currently incompatible with >=2.0.0: (https://github.com/matrix-org/synapse/issues/15858)
-pydantic = "^1.7.4"
+pydantic = ">=1.7.4"

 # This is for building the rust components during "poetry install", which
 # currently ignores the `build-system.requires` directive (c.f.
@@ -309,7 +311,7 @@ all = [
 # We pin black so that our tests don't start failing on new releases.
 isort = ">=5.10.1"
 black = ">=22.3.0"
-ruff = "0.0.277"
+ruff = "0.0.265"

 # Typechecking
 lxml-stubs = ">=0.4.0"
@@ -373,15 +375,7 @@ build-backend = "poetry.core.masonry.api"

 [tool.cibuildwheel]
 # Skip unsupported platforms (by us or by Rust).
-# See https://cibuildwheel.readthedocs.io/en/stable/options/#build-skip for the list of build targets.
-# We skip:
-#  - CPython 3.6 and 3.7: EOLed
-#  - PyPy 3.7: we only support Python 3.8+
-#  - musllinux i686: excluded to reduce number of wheels we build.
-#    c.f. https://github.com/matrix-org/synapse/pull/12595#discussion_r963107677
-#  - PyPy on Aarch64 and musllinux on aarch64: too slow to build.
-#    c.f. https://github.com/matrix-org/synapse/pull/14259
-skip = "cp36* cp37* pp37* *-musllinux_i686 pp*aarch64 *-musllinux_aarch64"
+skip = "cp36* *-musllinux_i686 pp*aarch64 *-musllinux_aarch64"

 # We need a rust compiler
 before-all =  "curl https://sh.rustup.rs -sSf | sh -s -- --default-toolchain stable -y --profile minimal"
@@ -142,7 +142,7 @@ pub const BASE_APPEND_OVERRIDE_RULES: &[PushRule] = &[
        default_enabled: true,
    },
    PushRule {
-        rule_id: Cow::Borrowed("global/override/.m.rule.is_user_mention"),
+        rule_id: Cow::Borrowed("global/override/.m.is_user_mention"),
        priority_class: 5,
        conditions: Cow::Borrowed(&[Condition::Known(
            KnownCondition::ExactEventPropertyContainsType(EventPropertyIsTypeCondition {
@@ -163,7 +163,7 @@ pub const BASE_APPEND_OVERRIDE_RULES: &[PushRule] = &[
        default_enabled: true,
    },
    PushRule {
-        rule_id: Cow::Borrowed("global/override/.m.rule.is_room_mention"),
+        rule_id: Cow::Borrowed("global/override/.m.is_room_mention"),
        priority_class: 5,
        conditions: Cow::Borrowed(&[
            Condition::Known(KnownCondition::EventPropertyIs(EventPropertyIsCondition {
@@ -22,19 +22,15 @@ from typing import Collection, Optional, Sequence, Set

 # These are expanded inside the dockerfile to be a fully qualified image name.
 # e.g. docker.io/library/debian:bullseye
-#
-# If an EOL is forced by a Python version and we're dropping support for it, make sure
-# to remove references to the distibution across Synapse (search for "bullseye" for
-# example)
 DISTS = (
-    "debian:bullseye",  # (EOL ~2024-07) (our EOL forced by Python 3.9 is 2025-10-05)
-    "debian:bookworm",  # (EOL not specified yet) (our EOL forced by Python 3.11 is 2027-10-24)
-    "debian:sid",  # (EOL not specified yet) (our EOL forced by Python 3.11 is 2027-10-24)
-    "ubuntu:focal",  # 20.04 LTS (EOL 2025-04) (our EOL forced by Python 3.8 is 2024-10-14)
-    "ubuntu:jammy",  # 22.04 LTS (EOL 2027-04) (our EOL forced by Python 3.10 is 2026-10-04)
-    "ubuntu:kinetic",  # 22.10 (EOL 2023-07-20) (our EOL forced by Python 3.10 is 2026-10-04)
-    "ubuntu:lunar",  # 23.04 (EOL 2024-01) (our EOL forced by Python 3.11 is 2027-10-24)
-    "debian:trixie",  # (EOL not specified yet)
+    "debian:buster",  # oldstable: EOL 2022-08
+    "debian:bullseye",
+    "debian:bookworm",
+    "debian:sid",
+    "ubuntu:focal",  # 20.04 LTS (our EOL forced by Py38 on 2024-10-14)
+    "ubuntu:jammy",  # 22.04 LTS (EOL 2027-04)
+    "ubuntu:kinetic",  # 22.10 (EOL 2023-07-20)
+    "ubuntu:lunar",  # 23.04 (EOL 2024-01)
 )

 DESC = """\
@@ -214,7 +214,7 @@ fi

 extra_test_args=()

-test_tags="synapse_blacklist,msc3874,msc3890,msc3391,msc3930,faster_joins"
+test_tags="synapse_blacklist,msc3787,msc3874,msc3890,msc3391,msc3930,faster_joins"

 # All environment variables starting with PASS_ will be shared.
 # (The prefix is stripped off before reaching the container.)
@@ -246,6 +246,10 @@ else
  else
    export PASS_SYNAPSE_COMPLEMENT_DATABASE=sqlite
  fi
+
+  # The tests for importing historical messages (MSC2716)
+  # only pass with monoliths, currently.
+  test_tags="$test_tags,msc2716"
 fi

 if [[ -n "$ASYNCIO_REACTOR" ]]; then
@@ -253,10 +257,6 @@ if [[ -n "$ASYNCIO_REACTOR" ]]; then
  export PASS_SYNAPSE_COMPLEMENT_USE_ASYNCIO_REACTOR=true
 fi

-if [[ -n "$UNIX_SOCKETS" ]]; then
-  # Enable full on Unix socket mode for Synapse, Redis and Postgresql
-  export PASS_SYNAPSE_USE_UNIX_SOCKET=1
-fi

 if [[ -n "$SYNAPSE_TEST_LOG_LEVEL" ]]; then
  # Set the log level to what is desired
@@ -136,11 +136,11 @@ def request(
        authorization_headers.append(header)
        print("Authorization: %s" % header, file=sys.stderr)

-    dest = "matrix-federation://%s%s" % (destination, path)
+    dest = "matrix://%s%s" % (destination, path)
    print("Requesting %s" % dest, file=sys.stderr)

    s = requests.Session()
-    s.mount("matrix-federation://", MatrixConnectionAdapter())
+    s.mount("matrix://", MatrixConnectionAdapter())

    headers: Dict[str, str] = {
        "Authorization": authorization_headers[0],
@@ -25,8 +25,8 @@ from synapse.util.rust import check_rust_lib_up_to_date
 from synapse.util.stringutils import strtobool

 # Check that we're not running on an unsupported Python version.
-if sys.version_info < (3, 8):
-    print("Synapse requires Python 3.8 or above.")
+if sys.version_info < (3, 7):
+    print("Synapse requires Python 3.7 or above.")
    sys.exit(1)

 # Allow using the asyncio reactor via env var.
@@ -61,7 +61,6 @@ from synapse.storage.databases.main.deviceinbox import DeviceInboxBackgroundUpda
 from synapse.storage.databases.main.devices import DeviceBackgroundUpdateStore
 from synapse.storage.databases.main.e2e_room_keys import EndToEndRoomKeyBackgroundStore
 from synapse.storage.databases.main.end_to_end_keys import EndToEndKeyBackgroundStore
-from synapse.storage.databases.main.event_federation import EventFederationWorkerStore
 from synapse.storage.databases.main.event_push_actions import EventPushActionsStore
 from synapse.storage.databases.main.events_bg_updates import (
    EventsBackgroundUpdatesStore,
@@ -197,11 +196,6 @@ IGNORED_TABLES = {
    "ui_auth_sessions",
    "ui_auth_sessions_credentials",
    "ui_auth_sessions_ips",
-    # Ignore the worker locks table, as a) there shouldn't be any acquired locks
-    # after porting, and b) the circular foreign key constraints make it hard to
-    # port.
-    "worker_read_write_locks_mode",
-    "worker_read_write_locks",
 }


@@ -245,7 +239,6 @@ class Store(
    PresenceBackgroundUpdateStore,
    ReceiptsBackgroundUpdateStore,
    RelationsWorkerStore,
-    EventFederationWorkerStore,
 ):
    def execute(self, f: Callable[..., R], *args: Any, **kwargs: Any) -> Awaitable[R]:
        return self.db_pool.runInteraction(f.__name__, f, *args, **kwargs)
@@ -810,9 +803,7 @@ class Porter:
            )
            # Map from table name to args passed to `handle_table`, i.e. a tuple
            # of: `postgres_size`, `table_size`, `forward_chunk`, `backward_chunk`.
-            tables_to_port_info_map = {
-                r[0]: r[1:] for r in setup_res if r[0] not in IGNORED_TABLES
-            }
+            tables_to_port_info_map = {r[0]: r[1:] for r in setup_res}

            # Step 5. Do the copying.
            #
@@ -1378,9 +1369,6 @@ def main() -> None:
        sys.stderr.write("Database must use the 'psycopg2' connector.\n")
        sys.exit(3)

-    # Don't run the background tasks that get started by the data stores.
-    hs_config["run_background_tasks_on"] = "some_other_process"
-
    config = HomeServerConfig()
    config.parse_config_dict(hs_config, "", "")

@@ -123,6 +123,10 @@ class EventTypes:
    SpaceChild: Final = "m.space.child"
    SpaceParent: Final = "m.space.parent"

+    MSC2716_INSERTION: Final = "org.matrix.msc2716.insertion"
+    MSC2716_BATCH: Final = "org.matrix.msc2716.batch"
+    MSC2716_MARKER: Final = "org.matrix.msc2716.marker"
+
    Reaction: Final = "m.reaction"


@@ -218,6 +222,16 @@ class EventContentFields:
    # Used in m.room.guest_access events.
    GUEST_ACCESS: Final = "guest_access"

+    # Used on normal messages to indicate they were historically imported after the fact
+    MSC2716_HISTORICAL: Final = "org.matrix.msc2716.historical"
+    # For "insertion" events to indicate what the next batch ID should be in
+    # order to connect to it
+    MSC2716_NEXT_BATCH_ID: Final = "next_batch_id"
+    # Used on "batch" events to indicate which insertion event it connects to
+    MSC2716_BATCH_ID: Final = "batch_id"
+    # For "marker" events
+    MSC2716_INSERTION_EVENT_REFERENCE: Final = "insertion_event_reference"
+
    # The authorising user for joining a restricted room.
    AUTHORISING_USER: Final = "join_authorised_via_users_server"

@@ -217,13 +217,6 @@ class InvalidAPICallError(SynapseError):
        super().__init__(HTTPStatus.BAD_REQUEST, msg, Codes.BAD_JSON)


-class InvalidProxyCredentialsError(SynapseError):
-    """Error raised when the proxy credentials are invalid."""
-
-    def __init__(self, msg: str, errcode: str = Codes.UNKNOWN):
-        super().__init__(401, msg, errcode)
-
-
 class ProxiedRequestError(SynapseError):
    """An error from a general matrix endpoint, eg. from a proxied Matrix API call.

@@ -30,14 +30,12 @@ class EventFormatVersions:
    ROOM_V1_V2 = 1  # $id:server event id format: used for room v1 and v2
    ROOM_V3 = 2  # MSC1659-style $hash event id format: used for room v3
    ROOM_V4_PLUS = 3  # MSC1884-style $hash format: introduced for room v4
-    LINEARIZED = 4  # Delegated Linearized event


 KNOWN_EVENT_FORMAT_VERSIONS = {
    EventFormatVersions.ROOM_V1_V2,
    EventFormatVersions.ROOM_V3,
    EventFormatVersions.ROOM_V4_PLUS,
-    EventFormatVersions.LINEARIZED,
 }


@@ -80,30 +78,41 @@ class RoomVersion:
    # MSC2209: Check 'notifications' key while verifying
    # m.room.power_levels auth rules.
    limit_notifications_power_levels: bool
-    # No longer include the creator in m.room.create events.
-    implicit_room_creator: bool
-    # Apply updated redaction rules algorithm from room version 11.
-    updated_redaction_rules: bool
-    # Support the 'restricted' join rule.
-    restricted_join_rule: bool
-    # Support for the proper redaction rules for the restricted join rule. This requires
-    # restricted_join_rule to be enabled.
-    restricted_join_rule_fix: bool
-    # Support the 'knock' join rule.
-    knock_join_rule: bool
+    # MSC2175: No longer include the creator in m.room.create events.
+    msc2175_implicit_room_creator: bool
+    # MSC2174/MSC2176: Apply updated redaction rules algorithm, move redacts to
+    # content property.
+    msc2176_redaction_rules: bool
+    # MSC3083: Support the 'restricted' join_rule.
+    msc3083_join_rules: bool
+    # MSC3375: Support for the proper redaction rules for MSC3083. This mustn't
+    #          be enabled if MSC3083 is not.
+    msc3375_redaction_rules: bool
+    # MSC2403: Allows join_rules to be set to 'knock', changes auth rules to allow sending
+    # m.room.membership event with membership 'knock'.
+    msc2403_knocking: bool
+    # MSC2716: Adds m.room.power_levels -> content.historical field to control
+    # whether "insertion", "chunk", "marker" events can be sent
+    msc2716_historical: bool
+    # MSC2716: Adds support for redacting "insertion", "chunk", and "marker" events
+    msc2716_redactions: bool
    # MSC3389: Protect relation information from redaction.
    msc3389_relation_redactions: bool
-    # Support the 'knock_restricted' join rule.
-    knock_restricted_join_rule: bool
-    # Enforce integer power levels
-    enforce_int_power_levels: bool
+    # MSC3787: Adds support for a `knock_restricted` join rule, mixing concepts of
+    # knocks and restricted join rules into the same join condition.
+    msc3787_knock_restricted_join_rule: bool
+    # MSC3667: Enforce integer power levels
+    msc3667_int_only_power_levels: bool
+    # MSC3821: Do not redact the third_party_invite content field for membership events.
+    msc3821_redaction_rules: bool
    # MSC3931: Adds a push rule condition for "room version feature flags", making
    # some push rules room version dependent. Note that adding a flag to this list
    # is not enough to mark it "supported": the push rule evaluator also needs to
    # support the flag. Unknown flags are ignored by the evaluator, making conditions
    # fail if used.
    msc3931_push_features: Tuple[str, ...]  # values from PushRuleRoomFlag
-    linearized_matrix: bool
+    # MSC3989: Redact the origin field.
+    msc3989_redaction_rules: bool


 class RoomVersions:
@@ -116,16 +125,19 @@ class RoomVersions:
        special_case_aliases_auth=True,
        strict_canonicaljson=False,
        limit_notifications_power_levels=False,
-        implicit_room_creator=False,
-        updated_redaction_rules=False,
-        restricted_join_rule=False,
-        restricted_join_rule_fix=False,
-        knock_join_rule=False,
+        msc2175_implicit_room_creator=False,
+        msc2176_redaction_rules=False,
+        msc3083_join_rules=False,
+        msc3375_redaction_rules=False,
+        msc2403_knocking=False,
+        msc2716_historical=False,
+        msc2716_redactions=False,
        msc3389_relation_redactions=False,
-        knock_restricted_join_rule=False,
-        enforce_int_power_levels=False,
+        msc3787_knock_restricted_join_rule=False,
+        msc3667_int_only_power_levels=False,
+        msc3821_redaction_rules=False,
        msc3931_push_features=(),
-        linearized_matrix=False,
+        msc3989_redaction_rules=False,
    )
    V2 = RoomVersion(
        "2",
@@ -136,16 +148,19 @@ class RoomVersions:
        special_case_aliases_auth=True,
        strict_canonicaljson=False,
        limit_notifications_power_levels=False,
-        implicit_room_creator=False,
-        updated_redaction_rules=False,
-        restricted_join_rule=False,
-        restricted_join_rule_fix=False,
-        knock_join_rule=False,
+        msc2175_implicit_room_creator=False,
+        msc2176_redaction_rules=False,
+        msc3083_join_rules=False,
+        msc3375_redaction_rules=False,
+        msc2403_knocking=False,
+        msc2716_historical=False,
+        msc2716_redactions=False,
        msc3389_relation_redactions=False,
-        knock_restricted_join_rule=False,
-        enforce_int_power_levels=False,
+        msc3787_knock_restricted_join_rule=False,
+        msc3667_int_only_power_levels=False,
+        msc3821_redaction_rules=False,
        msc3931_push_features=(),
-        linearized_matrix=False,
+        msc3989_redaction_rules=False,
    )
    V3 = RoomVersion(
        "3",
@@ -156,16 +171,19 @@ class RoomVersions:
        special_case_aliases_auth=True,
        strict_canonicaljson=False,
        limit_notifications_power_levels=False,
-        implicit_room_creator=False,
-        updated_redaction_rules=False,
-        restricted_join_rule=False,
-        restricted_join_rule_fix=False,
-        knock_join_rule=False,
+        msc2175_implicit_room_creator=False,
+        msc2176_redaction_rules=False,
+        msc3083_join_rules=False,
+        msc3375_redaction_rules=False,
+        msc2403_knocking=False,
+        msc2716_historical=False,
+        msc2716_redactions=False,
        msc3389_relation_redactions=False,
-        knock_restricted_join_rule=False,
-        enforce_int_power_levels=False,
+        msc3787_knock_restricted_join_rule=False,
+        msc3667_int_only_power_levels=False,
+        msc3821_redaction_rules=False,
        msc3931_push_features=(),
-        linearized_matrix=False,
+        msc3989_redaction_rules=False,
    )
    V4 = RoomVersion(
        "4",
@@ -176,16 +194,19 @@ class RoomVersions:
        special_case_aliases_auth=True,
        strict_canonicaljson=False,
        limit_notifications_power_levels=False,
-        implicit_room_creator=False,
-        updated_redaction_rules=False,
-        restricted_join_rule=False,
-        restricted_join_rule_fix=False,
-        knock_join_rule=False,
+        msc2175_implicit_room_creator=False,
+        msc2176_redaction_rules=False,
+        msc3083_join_rules=False,
+        msc3375_redaction_rules=False,
+        msc2403_knocking=False,
+        msc2716_historical=False,
+        msc2716_redactions=False,
        msc3389_relation_redactions=False,
-        knock_restricted_join_rule=False,
-        enforce_int_power_levels=False,
+        msc3787_knock_restricted_join_rule=False,
+        msc3667_int_only_power_levels=False,
+        msc3821_redaction_rules=False,
        msc3931_push_features=(),
-        linearized_matrix=False,
+        msc3989_redaction_rules=False,
    )
    V5 = RoomVersion(
        "5",
@@ -196,16 +217,19 @@ class RoomVersions:
        special_case_aliases_auth=True,
        strict_canonicaljson=False,
        limit_notifications_power_levels=False,
-        implicit_room_creator=False,
-        updated_redaction_rules=False,
-        restricted_join_rule=False,
-        restricted_join_rule_fix=False,
-        knock_join_rule=False,
+        msc2175_implicit_room_creator=False,
+        msc2176_redaction_rules=False,
+        msc3083_join_rules=False,
+        msc3375_redaction_rules=False,
+        msc2403_knocking=False,
+        msc2716_historical=False,
+        msc2716_redactions=False,
        msc3389_relation_redactions=False,
-        knock_restricted_join_rule=False,
-        enforce_int_power_levels=False,
+        msc3787_knock_restricted_join_rule=False,
+        msc3667_int_only_power_levels=False,
+        msc3821_redaction_rules=False,
        msc3931_push_features=(),
-        linearized_matrix=False,
+        msc3989_redaction_rules=False,
    )
    V6 = RoomVersion(
        "6",
@@ -216,16 +240,42 @@ class RoomVersions:
        special_case_aliases_auth=False,
        strict_canonicaljson=True,
        limit_notifications_power_levels=True,
-        implicit_room_creator=False,
-        updated_redaction_rules=False,
-        restricted_join_rule=False,
-        restricted_join_rule_fix=False,
-        knock_join_rule=False,
+        msc2175_implicit_room_creator=False,
+        msc2176_redaction_rules=False,
+        msc3083_join_rules=False,
+        msc3375_redaction_rules=False,
+        msc2403_knocking=False,
+        msc2716_historical=False,
+        msc2716_redactions=False,
        msc3389_relation_redactions=False,
-        knock_restricted_join_rule=False,
-        enforce_int_power_levels=False,
+        msc3787_knock_restricted_join_rule=False,
+        msc3667_int_only_power_levels=False,
+        msc3821_redaction_rules=False,
        msc3931_push_features=(),
-        linearized_matrix=False,
+        msc3989_redaction_rules=False,
+    )
+    MSC2176 = RoomVersion(
+        "org.matrix.msc2176",
+        RoomDisposition.UNSTABLE,
+        EventFormatVersions.ROOM_V4_PLUS,
+        StateResolutionVersions.V2,
+        enforce_key_validity=True,
+        special_case_aliases_auth=False,
+        strict_canonicaljson=True,
+        limit_notifications_power_levels=True,
+        msc2175_implicit_room_creator=False,
+        msc2176_redaction_rules=True,
+        msc3083_join_rules=False,
+        msc3375_redaction_rules=False,
+        msc2403_knocking=False,
+        msc2716_historical=False,
+        msc2716_redactions=False,
+        msc3389_relation_redactions=False,
+        msc3787_knock_restricted_join_rule=False,
+        msc3667_int_only_power_levels=False,
+        msc3821_redaction_rules=False,
+        msc3931_push_features=(),
+        msc3989_redaction_rules=False,
    )
    V7 = RoomVersion(
        "7",
@@ -236,16 +286,19 @@ class RoomVersions:
        special_case_aliases_auth=False,
        strict_canonicaljson=True,
        limit_notifications_power_levels=True,
-        implicit_room_creator=False,
-        updated_redaction_rules=False,
-        restricted_join_rule=False,
-        restricted_join_rule_fix=False,
-        knock_join_rule=True,
+        msc2175_implicit_room_creator=False,
+        msc2176_redaction_rules=False,
+        msc3083_join_rules=False,
+        msc3375_redaction_rules=False,
+        msc2403_knocking=True,
+        msc2716_historical=False,
+        msc2716_redactions=False,
        msc3389_relation_redactions=False,
-        knock_restricted_join_rule=False,
-        enforce_int_power_levels=False,
+        msc3787_knock_restricted_join_rule=False,
+        msc3667_int_only_power_levels=False,
+        msc3821_redaction_rules=False,
        msc3931_push_features=(),
-        linearized_matrix=False,
+        msc3989_redaction_rules=False,
    )
    V8 = RoomVersion(
        "8",
@@ -256,16 +309,19 @@ class RoomVersions:
        special_case_aliases_auth=False,
        strict_canonicaljson=True,
        limit_notifications_power_levels=True,
-        implicit_room_creator=False,
-        updated_redaction_rules=False,
-        restricted_join_rule=True,
-        restricted_join_rule_fix=False,
-        knock_join_rule=True,
+        msc2175_implicit_room_creator=False,
+        msc2176_redaction_rules=False,
+        msc3083_join_rules=True,
+        msc3375_redaction_rules=False,
+        msc2403_knocking=True,
+        msc2716_historical=False,
+        msc2716_redactions=False,
        msc3389_relation_redactions=False,
-        knock_restricted_join_rule=False,
-        enforce_int_power_levels=False,
+        msc3787_knock_restricted_join_rule=False,
+        msc3667_int_only_power_levels=False,
+        msc3821_redaction_rules=False,
        msc3931_push_features=(),
-        linearized_matrix=False,
+        msc3989_redaction_rules=False,
    )
    V9 = RoomVersion(
        "9",
@@ -276,16 +332,65 @@ class RoomVersions:
        special_case_aliases_auth=False,
        strict_canonicaljson=True,
        limit_notifications_power_levels=True,
-        implicit_room_creator=False,
-        updated_redaction_rules=False,
-        restricted_join_rule=True,
-        restricted_join_rule_fix=True,
-        knock_join_rule=True,
+        msc2175_implicit_room_creator=False,
+        msc2176_redaction_rules=False,
+        msc3083_join_rules=True,
+        msc3375_redaction_rules=True,
+        msc2403_knocking=True,
+        msc2716_historical=False,
+        msc2716_redactions=False,
        msc3389_relation_redactions=False,
-        knock_restricted_join_rule=False,
-        enforce_int_power_levels=False,
+        msc3787_knock_restricted_join_rule=False,
+        msc3667_int_only_power_levels=False,
+        msc3821_redaction_rules=False,
        msc3931_push_features=(),
-        linearized_matrix=False,
+        msc3989_redaction_rules=False,
+    )
+    MSC3787 = RoomVersion(
+        "org.matrix.msc3787",
+        RoomDisposition.UNSTABLE,
+        EventFormatVersions.ROOM_V4_PLUS,
+        StateResolutionVersions.V2,
+        enforce_key_validity=True,
+        special_case_aliases_auth=False,
+        strict_canonicaljson=True,
+        limit_notifications_power_levels=True,
+        msc2175_implicit_room_creator=False,
+        msc2176_redaction_rules=False,
+        msc3083_join_rules=True,
+        msc3375_redaction_rules=True,
+        msc2403_knocking=True,
+        msc2716_historical=False,
+        msc2716_redactions=False,
+        msc3389_relation_redactions=False,
+        msc3787_knock_restricted_join_rule=True,
+        msc3667_int_only_power_levels=False,
+        msc3821_redaction_rules=False,
+        msc3931_push_features=(),
+        msc3989_redaction_rules=False,
+    )
+    MSC3821 = RoomVersion(
+        "org.matrix.msc3821.opt1",
+        RoomDisposition.UNSTABLE,
+        EventFormatVersions.ROOM_V4_PLUS,
+        StateResolutionVersions.V2,
+        enforce_key_validity=True,
+        special_case_aliases_auth=False,
+        strict_canonicaljson=True,
+        limit_notifications_power_levels=True,
+        msc2175_implicit_room_creator=False,
+        msc2176_redaction_rules=False,
+        msc3083_join_rules=True,
+        msc3375_redaction_rules=True,
+        msc2403_knocking=True,
+        msc2716_historical=False,
+        msc2716_redactions=False,
+        msc3389_relation_redactions=False,
+        msc3787_knock_restricted_join_rule=False,
+        msc3667_int_only_power_levels=False,
+        msc3821_redaction_rules=True,
+        msc3931_push_features=(),
+        msc3989_redaction_rules=False,
    )
    V10 = RoomVersion(
        "10",
@@ -296,16 +401,42 @@ class RoomVersions:
        special_case_aliases_auth=False,
        strict_canonicaljson=True,
        limit_notifications_power_levels=True,
-        implicit_room_creator=False,
-        updated_redaction_rules=False,
-        restricted_join_rule=True,
-        restricted_join_rule_fix=True,
-        knock_join_rule=True,
+        msc2175_implicit_room_creator=False,
+        msc2176_redaction_rules=False,
+        msc3083_join_rules=True,
+        msc3375_redaction_rules=True,
+        msc2403_knocking=True,
+        msc2716_historical=False,
+        msc2716_redactions=False,
        msc3389_relation_redactions=False,
-        knock_restricted_join_rule=True,
-        enforce_int_power_levels=True,
+        msc3787_knock_restricted_join_rule=True,
+        msc3667_int_only_power_levels=True,
+        msc3821_redaction_rules=False,
        msc3931_push_features=(),
-        linearized_matrix=False,
+        msc3989_redaction_rules=False,
+    )
+    MSC2716v4 = RoomVersion(
+        "org.matrix.msc2716v4",
+        RoomDisposition.UNSTABLE,
+        EventFormatVersions.ROOM_V4_PLUS,
+        StateResolutionVersions.V2,
+        enforce_key_validity=True,
+        special_case_aliases_auth=False,
+        strict_canonicaljson=True,
+        limit_notifications_power_levels=True,
+        msc2175_implicit_room_creator=False,
+        msc2176_redaction_rules=False,
+        msc3083_join_rules=False,
+        msc3375_redaction_rules=False,
+        msc2403_knocking=True,
+        msc2716_historical=True,
+        msc2716_redactions=True,
+        msc3389_relation_redactions=False,
+        msc3787_knock_restricted_join_rule=False,
+        msc3667_int_only_power_levels=False,
+        msc3821_redaction_rules=False,
+        msc3931_push_features=(),
+        msc3989_redaction_rules=False,
    )
    MSC1767v10 = RoomVersion(
        # MSC1767 (Extensible Events) based on room version "10"
@@ -317,61 +448,66 @@ class RoomVersions:
        special_case_aliases_auth=False,
        strict_canonicaljson=True,
        limit_notifications_power_levels=True,
-        implicit_room_creator=False,
-        updated_redaction_rules=False,
-        restricted_join_rule=True,
-        restricted_join_rule_fix=True,
-        knock_join_rule=True,
+        msc2175_implicit_room_creator=False,
+        msc2176_redaction_rules=False,
+        msc3083_join_rules=True,
+        msc3375_redaction_rules=True,
+        msc2403_knocking=True,
+        msc2716_historical=False,
+        msc2716_redactions=False,
        msc3389_relation_redactions=False,
-        knock_restricted_join_rule=True,
-        enforce_int_power_levels=True,
+        msc3787_knock_restricted_join_rule=True,
+        msc3667_int_only_power_levels=True,
+        msc3821_redaction_rules=False,
        msc3931_push_features=(PushRuleRoomFlag.EXTENSIBLE_EVENTS,),
-        linearized_matrix=False,
+        msc3989_redaction_rules=False,
    )
-    V11 = RoomVersion(
-        "11",
-        RoomDisposition.STABLE,
+    MSC3989 = RoomVersion(
+        "org.matrix.msc3989",
+        RoomDisposition.UNSTABLE,
        EventFormatVersions.ROOM_V4_PLUS,
        StateResolutionVersions.V2,
        enforce_key_validity=True,
        special_case_aliases_auth=False,
        strict_canonicaljson=True,
        limit_notifications_power_levels=True,
-        implicit_room_creator=True,  # Used by MSC3820
-        updated_redaction_rules=True,  # Used by MSC3820
-        restricted_join_rule=True,
-        restricted_join_rule_fix=True,
-        knock_join_rule=True,
+        msc2175_implicit_room_creator=False,
+        msc2176_redaction_rules=False,
+        msc3083_join_rules=True,
+        msc3375_redaction_rules=True,
+        msc2403_knocking=True,
+        msc2716_historical=False,
+        msc2716_redactions=False,
        msc3389_relation_redactions=False,
-        knock_restricted_join_rule=True,
-        enforce_int_power_levels=True,
+        msc3787_knock_restricted_join_rule=True,
+        msc3667_int_only_power_levels=True,
+        msc3821_redaction_rules=False,
        msc3931_push_features=(),
-        linearized_matrix=False,
+        msc3989_redaction_rules=True,
    )
-    # Based on room version 11:
-    #
-    # - Enable MSC2176, MSC2175, MSC3989, MSC2174, MSC1767, MSC3821.
-    # - Disable 'restricted' and 'knock_restricted' join rules.
-    # - Mark as linearized.
-    LINEARIZED = RoomVersion(
-        "org.matrix.i-d.ralston-mimi-linearized-matrix.02",
+    MSC3820opt2 = RoomVersion(
+        # Based upon v10
+        "org.matrix.msc3820.opt2",
        RoomDisposition.UNSTABLE,
-        EventFormatVersions.LINEARIZED,
+        EventFormatVersions.ROOM_V4_PLUS,
        StateResolutionVersions.V2,
        enforce_key_validity=True,
        special_case_aliases_auth=False,
        strict_canonicaljson=True,
        limit_notifications_power_levels=True,
-        implicit_room_creator=True,
-        updated_redaction_rules=True,
-        restricted_join_rule=True,
-        restricted_join_rule_fix=True,
-        knock_join_rule=True,
+        msc2175_implicit_room_creator=True,  # Used by MSC3820
+        msc2176_redaction_rules=True,  # Used by MSC3820
+        msc3083_join_rules=True,
+        msc3375_redaction_rules=True,
+        msc2403_knocking=True,
+        msc2716_historical=False,
+        msc2716_redactions=False,
        msc3389_relation_redactions=False,
-        knock_restricted_join_rule=True,
-        enforce_int_power_levels=True,
+        msc3787_knock_restricted_join_rule=True,
+        msc3667_int_only_power_levels=True,
+        msc3821_redaction_rules=True,  # Used by MSC3820
        msc3931_push_features=(),
-        linearized_matrix=True,
+        msc3989_redaction_rules=True,  # Used by MSC3820
    )


@@ -384,12 +520,15 @@ KNOWN_ROOM_VERSIONS: Dict[str, RoomVersion] = {
        RoomVersions.V4,
        RoomVersions.V5,
        RoomVersions.V6,
+        RoomVersions.MSC2176,
        RoomVersions.V7,
        RoomVersions.V8,
        RoomVersions.V9,
+        RoomVersions.MSC3787,
        RoomVersions.V10,
-        RoomVersions.V11,
-        RoomVersions.LINEARIZED,
+        RoomVersions.MSC2716v4,
+        RoomVersions.MSC3989,
+        RoomVersions.MSC3820opt2,
    )
 }

@@ -418,12 +557,12 @@ MSC3244_CAPABILITIES = {
        RoomVersionCapability(
            "knock",
            RoomVersions.V7,
-            lambda room_version: room_version.knock_join_rule,
+            lambda room_version: room_version.msc2403_knocking,
        ),
        RoomVersionCapability(
            "restricted",
            RoomVersions.V9,
-            lambda room_version: room_version.restricted_join_rule,
+            lambda room_version: room_version.msc3083_join_rules,
        ),
    )
 }
@@ -386,7 +386,6 @@ def listen_unix(


 def listen_http(
-    hs: "HomeServer",
    listener_config: ListenerConfig,
    root_resource: Resource,
    version_string: str,
@@ -407,7 +406,6 @@ def listen_http(
        version_string,
        max_request_body_size=max_request_body_size,
        reactor=reactor,
-        hs=hs,
    )

    if isinstance(listener_config, TCPListenerConfig):
@@ -83,6 +83,7 @@ from synapse.storage.databases.main.receipts import ReceiptsWorkerStore
 from synapse.storage.databases.main.registration import RegistrationWorkerStore
 from synapse.storage.databases.main.relations import RelationsWorkerStore
 from synapse.storage.databases.main.room import RoomWorkerStore
+from synapse.storage.databases.main.room_batch import RoomBatchStore
 from synapse.storage.databases.main.roommember import RoomMemberWorkerStore
 from synapse.storage.databases.main.search import SearchStore
 from synapse.storage.databases.main.session import SessionStore
@@ -119,6 +120,7 @@ class GenericWorkerStore(
    # the races it creates aren't too bad.
    KeyStore,
    RoomWorkerStore,
+    RoomBatchStore,
    DirectoryWorkerStore,
    PushRulesWorkerStore,
    ApplicationServiceTransactionWorkerStore,
@@ -221,7 +223,6 @@ class GenericWorkerServer(HomeServer):
        root_resource = create_resource_tree(resources, OptionsResource())

        _base.listen_http(
-            self,
            listener_config,
            root_resource,
            self.version_string,
@@ -139,7 +139,6 @@ class SynapseHomeServer(HomeServer):
            root_resource = OptionsResource()

        ports = listen_http(
-            self,
            listener_config,
            create_resource_tree(resources, root_resource),
            self.version_string,
@@ -31,7 +31,7 @@ class AuthConfig(Config):

        # The default value of password_config.enabled is True, unless msc3861 is enabled.
        msc3861_enabled = (
-            (config.get("experimental_features") or {})
+            config.get("experimental_features", {})
            .get("msc3861", {})
            .get("enabled", False)
        )
@@ -247,26 +247,8 @@ class ExperimentalConfig(Config):
        # MSC3026 (busy presence state)
        self.msc3026_enabled: bool = experimental.get("msc3026_enabled", False)

-        # MSC2697 (device dehydration)
-        # Enabled by default since this option was added after adding the feature.
-        # It is not recommended that both MSC2697 and MSC3814 both be enabled at
-        # once.
-        self.msc2697_enabled: bool = experimental.get("msc2697_enabled", True)
-
-        # MSC3814 (dehydrated devices with SSSS)
-        # This is an alternative method to achieve the same goals as MSC2697.
-        # It is not recommended that both MSC2697 and MSC3814 both be enabled at
-        # once.
-        self.msc3814_enabled: bool = experimental.get("msc3814_enabled", False)
-
-        if self.msc2697_enabled and self.msc3814_enabled:
-            raise ConfigError(
-                "MSC2697 and MSC3814 should not both be enabled.",
-                (
-                    "experimental_features",
-                    "msc3814_enabled",
-                ),
-            )
+        # MSC2716 (importing historical messages)
+        self.msc2716_enabled: bool = experimental.get("msc2716_enabled", False)

        # MSC3244 (room version capabilities)
        self.msc3244_enabled: bool = experimental.get("msc3244_enabled", True)
@@ -403,6 +385,9 @@ class ExperimentalConfig(Config):
        # Check that none of the other config options conflict with MSC3861 when enabled
        self.msc3861.check_config_conflicts(self.root)

+        # MSC4009: E.164 Matrix IDs
+        self.msc4009_e164_mxids = experimental.get("msc4009_e164_mxids", False)
+
        # MSC4010: Do not allow setting m.push_rules account data.
        self.msc4010_push_rules_account_data = experimental.get(
            "msc4010_push_rules_account_data", False
@@ -53,13 +53,13 @@ class FederationConfig(Config):

        # Allow for the configuration of timeout, max request retries
        # and min/max retry delays in the matrix federation client.
-        self.client_timeout_ms = Config.parse_duration(
+        self.client_timeout = Config.parse_duration(
            federation_config.get("client_timeout", "60s")
        )
-        self.max_long_retry_delay_ms = Config.parse_duration(
+        self.max_long_retry_delay = Config.parse_duration(
            federation_config.get("max_long_retry_delay", "60s")
        )
-        self.max_short_retry_delay_ms = Config.parse_duration(
+        self.max_short_retry_delay = Config.parse_duration(
            federation_config.get("max_short_retry_delay", "2s")
        )
        self.max_long_retries = federation_config.get("max_long_retries", 10)
@@ -15,7 +15,7 @@

 import argparse
 import logging
-from typing import Any, Dict, List, Optional, Union
+from typing import Any, Dict, List, Union

 import attr
 from pydantic import BaseModel, Extra, StrictBool, StrictInt, StrictStr
@@ -41,17 +41,11 @@ Synapse version. Please use ``%s: name_of_worker`` instead.

 _MISSING_MAIN_PROCESS_INSTANCE_MAP_DATA = """
 Missing data for a worker to connect to main process. Please include '%s' in the
-`instance_map` declared in your shared yaml configuration as defined in configuration
-documentation here:
-`https://matrix-org.github.io/synapse/latest/usage/configuration/config_documentation.html#instance_map`
-"""
-
-WORKER_REPLICATION_SETTING_DEPRECATED_MESSAGE = """
-'%s' is no longer a supported worker setting, please place '%s' onto your shared
-configuration under `main` inside the `instance_map`. See workers documentation here:
+`instance_map` declared in your shared yaml configuration, or optionally(as a deprecated
+solution) in every worker's yaml as various `worker_replication_*` settings as defined
+in workers documentation here:
 `https://matrix-org.github.io/synapse/latest/workers.html#worker-configuration`
 """
-
 # This allows for a handy knob when it's time to change from 'master' to
 # something with less 'history'
 MAIN_PROCESS_INSTANCE_NAME = "master"
@@ -94,7 +88,7 @@ class ConfigModel(BaseModel):
        allow_mutation = False


-class InstanceTcpLocationConfig(ConfigModel):
+class InstanceLocationConfig(ConfigModel):
    """The host and port to talk to an instance via HTTP replication."""

    host: StrictStr
@@ -110,23 +104,6 @@ class InstanceTcpLocationConfig(ConfigModel):
        return f"{self.host}:{self.port}"


-class InstanceUnixLocationConfig(ConfigModel):
-    """The socket file to talk to an instance via HTTP replication."""
-
-    path: StrictStr
-
-    def scheme(self) -> str:
-        """Hardcode a retrievable scheme"""
-        return "unix"
-
-    def netloc(self) -> str:
-        """Nicely format the address location data"""
-        return f"{self.path}"
-
-
-InstanceLocationConfig = Union[InstanceTcpLocationConfig, InstanceUnixLocationConfig]
-
-
@attr.s
 class WriterLocations:
    """Specifies the instances that write various streams.
@@ -171,27 +148,6 @@ class WriterLocations:
    )


-@attr.s(auto_attribs=True)
-class OutboundFederationRestrictedTo:
-    """Whether we limit outbound federation to a certain set of instances.
-
-    Attributes:
-        instances: optional list of instances that can make outbound federation
-            requests. If None then all instances can make federation requests.
-        locations: list of instance locations to connect to proxy via.
-    """
-
-    instances: Optional[List[str]]
-    locations: List[InstanceLocationConfig] = attr.Factory(list)
-
-    def __contains__(self, instance: str) -> bool:
-        # It feels a bit dirty to return `True` if `instances` is `None`, but it makes
-        # sense in downstream usage in the sense that if
-        # `outbound_federation_restricted_to` is not configured, then any instance can
-        # talk to federation (no restrictions so always return `True`).
-        return self.instances is None or instance in self.instances
-
-
 class WorkerConfig(Config):
    """The workers are processes run separately to the main synapse process.
    They have their own pid_file and listener configuration. They use the
@@ -260,37 +216,22 @@ class WorkerConfig(Config):
        )

        # A map from instance name to host/port of their HTTP replication endpoint.
-        # Check if the main process is declared. The main process itself doesn't need
-        # this data as it would never have to talk to itself.
+        # Check if the main process is declared. Inject it into the map if it's not,
+        # based first on if a 'main' block is declared then on 'worker_replication_*'
+        # data. If both are available, default to instance_map. The main process
+        # itself doesn't need this data as it would never have to talk to itself.
        instance_map: Dict[str, Any] = config.get("instance_map", {})

        if self.instance_name is not MAIN_PROCESS_INSTANCE_NAME:
-            # TODO: The next 3 condition blocks can be deleted after some time has
-            #  passed and we're ready to stop checking for these settings.
            # The host used to connect to the main synapse
            main_host = config.get("worker_replication_host", None)
-            if main_host:
-                raise ConfigError(
-                    WORKER_REPLICATION_SETTING_DEPRECATED_MESSAGE
-                    % ("worker_replication_host", main_host)
-                )

            # The port on the main synapse for HTTP replication endpoint
            main_port = config.get("worker_replication_http_port")
-            if main_port:
-                raise ConfigError(
-                    WORKER_REPLICATION_SETTING_DEPRECATED_MESSAGE
-                    % ("worker_replication_http_port", main_port)
-                )

            # The tls mode on the main synapse for HTTP replication endpoint.
            # For backward compatibility this defaults to False.
            main_tls = config.get("worker_replication_http_tls", False)
-            if main_tls:
-                raise ConfigError(
-                    WORKER_REPLICATION_SETTING_DEPRECATED_MESSAGE
-                    % ("worker_replication_http_tls", main_tls)
-                )

            # For now, accept 'main' in the instance_map, but the replication system
            # expects 'master', force that into being until it's changed later.
@@ -300,20 +241,30 @@ class WorkerConfig(Config):
                ]
                del instance_map[MAIN_PROCESS_INSTANCE_MAP_NAME]

+            # This is the backwards compatibility bit that handles the
+            # worker_replication_* bits using setdefault() to not overwrite anything.
+            elif main_host is not None and main_port is not None:
+                instance_map.setdefault(
+                    MAIN_PROCESS_INSTANCE_NAME,
+                    {
+                        "host": main_host,
+                        "port": main_port,
+                        "tls": main_tls,
+                    },
+                )
+
            else:
                # If we've gotten here, it means that the main process is not on the
-                # instance_map.
+                # instance_map and that not enough worker_replication_* variables
+                # were declared in the worker's yaml.
                raise ConfigError(
                    _MISSING_MAIN_PROCESS_INSTANCE_MAP_DATA
                    % MAIN_PROCESS_INSTANCE_MAP_NAME
                )

-        # type-ignore: the expression `Union[A, B]` is not a Type[Union[A, B]] currently
        self.instance_map: Dict[
            str, InstanceLocationConfig
-        ] = parse_and_validate_mapping(
-            instance_map, InstanceLocationConfig  # type: ignore[arg-type]
-        )
+        ] = parse_and_validate_mapping(instance_map, InstanceLocationConfig)

        # Map from type of streams to source, c.f. WriterLocations.
        writers = config.get("stream_writers") or {}
@@ -406,28 +357,6 @@ class WorkerConfig(Config):
            new_option_name="update_user_directory_from_worker",
        )

-        outbound_federation_restricted_to = config.get(
-            "outbound_federation_restricted_to", None
-        )
-        self.outbound_federation_restricted_to = OutboundFederationRestrictedTo(
-            outbound_federation_restricted_to
-        )
-        if outbound_federation_restricted_to:
-            if not self.worker_replication_secret:
-                raise ConfigError(
-                    "`worker_replication_secret` must be configured when using `outbound_federation_restricted_to`."
-                )
-
-            for instance in outbound_federation_restricted_to:
-                if instance not in self.instance_map:
-                    raise ConfigError(
-                        "Instance %r is configured in 'outbound_federation_restricted_to' but does not appear in `instance_map` config."
-                        % (instance,)
-                    )
-                self.outbound_federation_restricted_to.locations.append(
-                    self.instance_map[instance]
-                )
-
    def _should_this_worker_perform_duty(
        self,
        config: Dict[str, Any],
@@ -36,38 +36,31 @@ logger = logging.getLogger(__name__)
 Hasher = Callable[[bytes], "hashlib._Hash"]


-def _check_dict_hash(
-    event_id: str,
-    hash_log: str,
-    hashes: Any,
-    d: JsonDict,
-    hash_algorithm: Hasher,
-    lpdu_hash: bool,
+@trace
+def check_event_content_hash(
+    event: EventBase, hash_algorithm: Hasher = hashlib.sha256
 ) -> bool:
-    name, expected_hash = compute_content_hash(d, hash_algorithm, lpdu_hash)
+    """Check whether the hash for this PDU matches the contents"""
+    name, expected_hash = compute_content_hash(event.get_pdu_json(), hash_algorithm)
    logger.debug(
-        "Verifying %s hash on %s (expecting: %s)",
-        hash_log,
-        event_id,
+        "Verifying content hash on %s (expecting: %s)",
+        event.event_id,
        encode_base64(expected_hash),
    )

+    # some malformed events lack a 'hashes'. Protect against it being missing
+    # or a weird type by basically treating it the same as an unhashed event.
+    hashes = event.get("hashes")
    # nb it might be a immutabledict or a dict
    if not isinstance(hashes, collections.abc.Mapping):
        raise SynapseError(
-            400,
-            "Malformed %s hashes: %s"
-            % (
-                hash_log,
-                type(hashes),
-            ),
-            Codes.UNAUTHORIZED,
+            400, "Malformed 'hashes': %s" % (type(hashes),), Codes.UNAUTHORIZED
        )

    if name not in hashes:
        raise SynapseError(
            400,
-            "Algorithm %s not in %s hashes %s" % (name, hash_log, list(hashes)),
+            "Algorithm %s not in hashes %s" % (name, list(hashes)),
            Codes.UNAUTHORIZED,
        )
    message_hash_base64 = hashes[name]
@@ -80,45 +73,8 @@ def _check_dict_hash(
    return message_hash_bytes == expected_hash


-@trace
-def check_event_content_hash(
-    event: EventBase, hash_algorithm: Hasher = hashlib.sha256
-) -> bool:
-    """Check whether the hash for this PDU matches the contents"""
-
-    # some malformed events lack a 'hashes'. Protect against it being missing
-    # or a weird type by basically treating it the same as an unhashed event.
-    hashes = event.get("hashes")
-
-    if not _check_dict_hash(
-        event.event_id,
-        "content",
-        hashes,
-        event.get_pdu_json(),
-        hash_algorithm,
-        lpdu_hash=False,
-    ):
-        return False
-
-    # Check the content hash of the LPDU, if this was sent via a hub.
-    if event.room_version.linearized_matrix and event.hub_server:
-        # hashes must be a dictionary to have passed _check_dict_hash above.
-        lpdu_hashes = hashes.get("lpdu")
-        return _check_dict_hash(
-            event.event_id,
-            "linearized content",
-            lpdu_hashes,
-            event.get_linearized_pdu_json(),
-            hash_algorithm,
-            lpdu_hash=True,
-        )
-
-    # Non-linearized matrix doesn't care about other checks.
-    return True
-
-
 def compute_content_hash(
-    event_dict: Dict[str, Any], hash_algorithm: Hasher, lpdu_hash: False
+    event_dict: Dict[str, Any], hash_algorithm: Hasher
 ) -> Tuple[str, bytes]:
    """Compute the content hash of an event, which is the hash of the
    unredacted event.
@@ -127,7 +83,6 @@ def compute_content_hash(
        event_dict: The unredacted event as a dict
        hash_algorithm: A hasher from `hashlib`, e.g. hashlib.sha256, to use
            to hash the event
-        lpdu_hash: True if the LPDU hash is being calculated.

    Returns:
        A tuple of the name of hash and the hash as raw bytes.
@@ -136,17 +91,7 @@ def compute_content_hash(
    event_dict.pop("age_ts", None)
    event_dict.pop("unsigned", None)
    event_dict.pop("signatures", None)
-
-    hashes = event_dict.pop("hashes", {})
-
-    # If we are calculating the content hash of a PDU sent on behalf of another
-    # server, hashes only`contains the lpdu hash.
-    #
-    # TODO(LM) This is wrong in that an incorrectly formatted event could have
-    # bad content hashes from this.
-    if not lpdu_hash and "lpdu" in hashes:
-        event_dict["hashes"] = {"lpdu": hashes["lpdu"]}
-
+    event_dict.pop("hashes", None)
    event_dict.pop("outlier", None)
    event_dict.pop("destinations", None)

@@ -231,10 +176,7 @@ def add_hashes_and_signatures(
        signing_key: The key to sign with
    """

-    # Synapse only ever generates PDUs, not LPDUs.
-    name, digest = compute_content_hash(
-        event_dict, hash_algorithm=hashlib.sha256, lpdu_hash=False
-    )
+    name, digest = compute_content_hash(event_dict, hash_algorithm=hashlib.sha256)

    event_dict.setdefault("hashes", {})[name] = encode_base64(digest)

@@ -49,7 +49,6 @@ from synapse.types import JsonDict
 from synapse.util import unwrapFirstError
 from synapse.util.async_helpers import yieldable_gather_results
 from synapse.util.batching_queue import BatchingQueue
-from synapse.util.caches.descriptors import cached
 from synapse.util.retryutils import NotRetryingDestination

 if TYPE_CHECKING:
@@ -190,39 +189,6 @@ class Keyring:
            valid_until_ts=2**63,  # fake future timestamp
        )

-        self._client = hs.get_federation_http_client()
-
-    @cached()
-    async def is_server_linearized(self, server_name: str) -> bool:
-        # TODO(LM) Cache this in the database.
-        # TODO(LM) Support perspectives server.
-        try:
-            response = await self._client.get_json(
-                destination=server_name,
-                path="/_matrix/key/v2/server",
-                ignore_backoff=True,
-                # we only give the remote server 10s to respond. It should be an
-                # easy request to handle, so if it doesn't reply within 10s, it's
-                # probably not going to.
-                #
-                # Furthermore, when we are acting as a notary server, we cannot
-                # wait all day for all of the origin servers, as the requesting
-                # server will otherwise time out before we can respond.
-                #
-                # (Note that get_json may make 4 attempts, so this can still take
-                # almost 45 seconds to fetch the headers, plus up to another 60s to
-                # read the response).
-                timeout=10000,
-            )
-        except (NotRetryingDestination, RequestSendFailed) as e:
-            # these both have str() representations which we can't really improve
-            # upon
-            raise KeyLookupError(str(e))
-        except HttpResponseException as e:
-            raise KeyLookupError("Remote server returned an error: %s" % (e,))
-
-        return response.get("m.linearized", False)
-
    async def verify_json_for_server(
        self,
        server_name: str,
@@ -117,7 +117,7 @@ def validate_event_for_room_version(event: "EventBase") -> None:
        if not is_invite_via_3pid:
            raise AuthError(403, "Event not signed by sender's server")

-    if event.format_version == EventFormatVersions.ROOM_V1_V2:
+    if event.format_version in (EventFormatVersions.ROOM_V1_V2,):
        # Only older room versions have event IDs to check.
        event_id_domain = get_domain_from_id(event.event_id)

@@ -125,17 +125,8 @@ def validate_event_for_room_version(event: "EventBase") -> None:
        if not event.signatures.get(event_id_domain):
            raise AuthError(403, "Event not signed by sending server")

-    if event.format_version == EventFormatVersions.LINEARIZED:
-        # TODO Are these handling DAG-native events properly? Is the null-checks
-        # a bypass?
-
-        # Check that the hub server signed it.
-        hub_server = event.hub_server
-        if hub_server and not event.signatures.get(hub_server):
-            raise AuthError(403, "Event not signed by hub server")
-
    is_invite_via_allow_rule = (
-        event.room_version.restricted_join_rule
+        event.room_version.msc3083_join_rules
        and event.type == EventTypes.Member
        and event.membership == Membership.JOIN
        and EventContentFields.AUTHORISING_USER in event.content
@@ -286,13 +277,6 @@ def check_state_dependent_auth_rules(

    auth_dict = {(e.type, e.state_key): e for e in auth_events}

-    # If the event was sent from a hub server it must be the current hub server.
-    if event.room_version.linearized_matrix:
-        if event.hub_server:
-            current_hub_server = get_hub_server(auth_dict)
-            if current_hub_server != event.hub_server:
-                raise AuthError(403, "Event sent from incorrect hub server.")
-
    # additional check for m.federate
    creating_domain = get_domain_from_id(event.room_id)
    originating_domain = get_domain_from_id(event.sender)
@@ -355,6 +339,13 @@ def check_state_dependent_auth_rules(
    if event.type == EventTypes.Redaction:
        check_redaction(event.room_version, event, auth_dict)

+    if (
+        event.type == EventTypes.MSC2716_INSERTION
+        or event.type == EventTypes.MSC2716_BATCH
+        or event.type == EventTypes.MSC2716_MARKER
+    ):
+        check_historical(event.room_version, event, auth_dict)
+
    logger.debug("Allowing! %s", event)


@@ -368,10 +359,13 @@ LENIENT_EVENT_BYTE_LIMITS_ROOM_VERSIONS = {
    RoomVersions.V4,
    RoomVersions.V5,
    RoomVersions.V6,
+    RoomVersions.MSC2176,
    RoomVersions.V7,
    RoomVersions.V8,
    RoomVersions.V9,
+    RoomVersions.MSC3787,
    RoomVersions.V10,
+    RoomVersions.MSC2716v4,
    RoomVersions.MSC1767v10,
 }

@@ -463,7 +457,7 @@ def _check_create(event: "EventBase") -> None:

    # 1.4 If content has no creator field, reject if the room version requires it.
    if (
-        not event.room_version.implicit_room_creator
+        not event.room_version.msc2175_implicit_room_creator
        and EventContentFields.ROOM_CREATOR not in event.content
    ):
        raise AuthError(403, "Create event lacks a 'creator' property")
@@ -500,7 +494,7 @@ def _is_membership_change_allowed(
        key = (EventTypes.Create, "")
        create = auth_events.get(key)
        if create and event.prev_event_ids()[0] == create.event_id:
-            if room_version.implicit_room_creator:
+            if room_version.msc2175_implicit_room_creator:
                creator = create.sender
            else:
                creator = create.content[EventContentFields.ROOM_CREATOR]
@@ -523,7 +517,7 @@ def _is_membership_change_allowed(
    caller_invited = caller and caller.membership == Membership.INVITE
    caller_knocked = (
        caller
-        and room_version.knock_join_rule
+        and room_version.msc2403_knocking
        and caller.membership == Membership.KNOCK
    )

@@ -623,9 +617,9 @@ def _is_membership_change_allowed(
        elif join_rule == JoinRules.PUBLIC:
            pass
        elif (
-            room_version.restricted_join_rule and join_rule == JoinRules.RESTRICTED
+            room_version.msc3083_join_rules and join_rule == JoinRules.RESTRICTED
        ) or (
-            room_version.knock_restricted_join_rule
+            room_version.msc3787_knock_restricted_join_rule
            and join_rule == JoinRules.KNOCK_RESTRICTED
        ):
            # This is the same as public, but the event must contain a reference
@@ -655,9 +649,9 @@ def _is_membership_change_allowed(

        elif (
            join_rule == JoinRules.INVITE
-            or (room_version.knock_join_rule and join_rule == JoinRules.KNOCK)
+            or (room_version.msc2403_knocking and join_rule == JoinRules.KNOCK)
            or (
-                room_version.knock_restricted_join_rule
+                room_version.msc3787_knock_restricted_join_rule
                and join_rule == JoinRules.KNOCK_RESTRICTED
            )
        ):
@@ -691,9 +685,9 @@ def _is_membership_change_allowed(
                "You don't have permission to ban",
                errcode=Codes.INSUFFICIENT_POWER,
            )
-    elif room_version.knock_join_rule and Membership.KNOCK == membership:
+    elif room_version.msc2403_knocking and Membership.KNOCK == membership:
        if join_rule != JoinRules.KNOCK and (
-            not room_version.knock_restricted_join_rule
+            not room_version.msc3787_knock_restricted_join_rule
            or join_rule != JoinRules.KNOCK_RESTRICTED
        ):
            raise AuthError(403, "You don't have permission to knock")
@@ -829,6 +823,38 @@ def check_redaction(
    raise AuthError(403, "You don't have permission to redact events")


+def check_historical(
+    room_version_obj: RoomVersion,
+    event: "EventBase",
+    auth_events: StateMap["EventBase"],
+) -> None:
+    """Check whether the event sender is allowed to send historical related
+    events like "insertion", "batch", and "marker".
+
+    Returns:
+        None
+
+    Raises:
+        AuthError if the event sender is not allowed to send historical related events
+        ("insertion", "batch", and "marker").
+    """
+    # Ignore the auth checks in room versions that do not support historical
+    # events
+    if not room_version_obj.msc2716_historical:
+        return
+
+    user_level = get_user_power_level(event.user_id, auth_events)
+
+    historical_level = get_named_level(auth_events, "historical", 100)
+
+    if user_level < historical_level:
+        raise UnstableSpecAuthError(
+            403,
+            'You don\'t have permission to send send historical related events ("insertion", "batch", and "marker")',
+            errcode=Codes.INSUFFICIENT_POWER,
+        )
+
+
 def _check_power_levels(
    room_version_obj: RoomVersion,
    event: "EventBase",
@@ -850,7 +876,7 @@ def _check_power_levels(
    # Reject events with stringy power levels if required by room version
    if (
        event.type == EventTypes.PowerLevels
-        and room_version_obj.enforce_int_power_levels
+        and room_version_obj.msc3667_int_only_power_levels
    ):
        for k, v in event.content.items():
            if k in {
@@ -986,7 +1012,7 @@ def get_user_power_level(user_id: str, auth_events: StateMap["EventBase"]) -> in
        key = (EventTypes.Create, "")
        create_event = auth_events.get(key)
        if create_event is not None:
-            if create_event.room_version.implicit_room_creator:
+            if create_event.room_version.msc2175_implicit_room_creator:
                creator = create_event.sender
            else:
                creator = create_event.content[EventContentFields.ROOM_CREATOR]
@@ -1079,12 +1105,6 @@ def _verify_third_party_invite(
    return False


-def get_hub_server(auth_events: StateMap["EventBase"]) -> str:
-    # The current hub is the sender of the create event.
-    create_event = auth_events[(EventTypes.Create, "")]
-    return get_domain_from_id(create_event.sender)
-
-
 def get_public_keys(invite_event: "EventBase") -> List[Dict[str, Any]]:
    public_keys = []
    if "public_key" in invite_event.content:
@@ -1130,7 +1150,7 @@ def auth_types_for_event(
                )
                auth_types.add(key)

-        if room_version.restricted_join_rule and membership == Membership.JOIN:
+        if room_version.msc3083_join_rules and membership == Membership.JOIN:
            if EventContentFields.AUTHORISING_USER in event.content:
                key = (
                    EventTypes.Member,
@@ -39,7 +39,7 @@ from unpaddedbase64 import encode_base64

 from synapse.api.constants import RelationTypes
 from synapse.api.room_versions import EventFormatVersions, RoomVersion, RoomVersions
-from synapse.types import JsonDict, RoomStreamToken, StrCollection, get_domain_from_id
+from synapse.types import JsonDict, RoomStreamToken, StrCollection
 from synapse.util.caches import intern_dict
 from synapse.util.frozenutils import freeze
 from synapse.util.stringutils import strtobool
@@ -198,6 +198,7 @@ class _EventInternalMetadata:
    soft_failed: DictProperty[bool] = DictProperty("soft_failed")
    proactively_send: DictProperty[bool] = DictProperty("proactively_send")
    redacted: DictProperty[bool] = DictProperty("redacted")
+    historical: DictProperty[bool] = DictProperty("historical")

    txn_id: DictProperty[str] = DictProperty("txn_id")
    """The transaction ID, if it was set when the event was created."""
@@ -287,6 +288,14 @@ class _EventInternalMetadata:
        """
        return self._dict.get("redacted", False)

+    def is_historical(self) -> bool:
+        """Whether this is a historical message.
+        This is used by the batchsend historical message endpoint and
+        is needed to and mark the event as backfilled and skip some checks
+        like push notifications.
+        """
+        return self._dict.get("historical", False)
+
    def is_notifiable(self) -> bool:
        """Whether this event can trigger a push notification"""
        return not self.is_outlier() or self.is_out_of_band_membership()
@@ -334,18 +343,11 @@ class EventBase(metaclass=abc.ABCMeta):
    state_key: DictProperty[str] = DictProperty("state_key")
    type: DictProperty[str] = DictProperty("type")
    user_id: DictProperty[str] = DictProperty("sender")
-    # Should only matter for Linearized Matrix.
-    hub_server: DictProperty[Optional[str]] = DefaultDictProperty("hub_server", None)

    @property
    def event_id(self) -> str:
        raise NotImplementedError()

-    @property
-    def pdu_domain(self) -> str:
-        """The domain which added this event to the DAG."""
-        return get_domain_from_id(self.sender)
-
    @property
    def membership(self) -> str:
        return self.content["membership"]
@@ -353,7 +355,7 @@ class EventBase(metaclass=abc.ABCMeta):
    @property
    def redacts(self) -> Optional[str]:
        """MSC2176 moved the redacts field into the content."""
-        if self.room_version.updated_redaction_rules:
+        if self.room_version.msc2176_redaction_rules:
            return self.content.get("redacts")
        return self.get("redacts")

@@ -389,9 +391,6 @@ class EventBase(metaclass=abc.ABCMeta):

        return pdu_json

-    def get_linearized_pdu_json(self) -> JsonDict:
-        raise NotImplementedError()
-
    def get_templated_pdu_json(self) -> JsonDict:
        """
        Return a JSON object suitable for a templated event, as used in the
@@ -406,9 +405,6 @@ class EventBase(metaclass=abc.ABCMeta):

        return template_json

-    def get_templated_linearized_pdu_json(self) -> JsonDict:
-        raise NotImplementedError()
-
    def __getitem__(self, field: str) -> Optional[Any]:
        return self._dict[field]

@@ -604,68 +600,6 @@ class FrozenEventV3(FrozenEventV2):
        return self._event_id


-class FrozenLinearizedEvent(FrozenEventV3):
-    """
-    Represents a Delegated Linearized PDU.
-    """
-
-    format_version = EventFormatVersions.LINEARIZED
-
-    # TODO(LM): Do we re-calculate depth at some point?
-    depth = 0  # type: ignore[assignment]
-
-    @property
-    def origin(self) -> str:
-        return get_domain_from_id(self.sender)
-
-    @property
-    def pdu_domain(self) -> str:
-        """The domain which added this event to the DAG.
-
-        It could be the authorized server or the sender."""
-        if self.hub_server is not None:
-            return self.hub_server
-        return super().pdu_domain
-
-    def get_pdu_json(self, time_now: Optional[int] = None) -> JsonDict:
-        pdu = super().get_pdu_json()
-
-        # TODO(LM) Why does this sometimes exist?
-        pdu.pop("depth", None)
-
-        # Internally Synapse uses depth & unsigned, but this isn't part of LM.
-        pdu.pop("unsigned")
-
-        return pdu
-
-    def get_linearized_pdu_json(self) -> JsonDict:
-        # Get the full PDU and then remove fields from it.
-        pdu = self.get_pdu_json()
-        # Strip everything except for the lpdu property from the hashes.
-        pdu["hashes"] = {"lpdu": pdu["hashes"]["lpdu"]}
-        pdu.pop("auth_events")
-        pdu.pop("prev_events")
-        return pdu
-
-    def get_templated_linearized_pdu_json(self) -> JsonDict:
-        """
-        Return a JSON object suitable for a templated event, as used in the
-        make_{join,leave,knock} workflow.
-        """
-        # By using _dict directly we don't pull in signatures/unsigned.
-        template_json = dict(self._dict)
-        # The hashes (similar to the signature) need to be recalculated by the
-        # joining/leaving/knocking server after (potentially) modifying the
-        # event.
-        template_json.pop("hashes")
-
-        # Linearized Matrix servers don't know about auth/prev events.
-        template_json.pop("auth_events")
-        template_json.pop("prev_events")
-
-        return template_json
-
-
 def _event_type_from_format_version(
    format_version: int,
 ) -> Type[Union[FrozenEvent, FrozenEventV2, FrozenEventV3]]:
@@ -685,8 +619,6 @@ def _event_type_from_format_version(
        return FrozenEventV2
    elif format_version == EventFormatVersions.ROOM_V4_PLUS:
        return FrozenEventV3
-    elif format_version == EventFormatVersions.LINEARIZED:
-        return FrozenLinearizedEvent
    else:
        raise Exception("No event format %r" % (format_version,))

@@ -87,11 +87,6 @@ class EventBuilder:
    _redacts: Optional[str] = None
    _origin_server_ts: Optional[int] = None

-    # TODO(LM) If Synapse is acting as a hub-server this should be itself.
-    hub_server: Optional[str] = None
-    _lpdu_hashes: Optional[Dict[str, str]] = None
-    _lpdu_signatures: Optional[Dict[str, Dict[str, str]]] = None
-
    internal_metadata: _EventInternalMetadata = attr.Factory(
        lambda: _EventInternalMetadata({})
    )
@@ -150,6 +145,20 @@ class EventBuilder:
            auth_events = auth_event_ids
            prev_events = prev_event_ids

+        # Otherwise, progress the depth as normal
+        if depth is None:
+            (
+                _,
+                most_recent_prev_event_depth,
+            ) = await self._store.get_max_depth_of(prev_event_ids)
+
+            depth = most_recent_prev_event_depth + 1
+
+        # we cap depth of generated events, to ensure that they are not
+        # rejected by other servers (and so that they can be persisted in
+        # the db)
+        depth = min(depth, MAX_DEPTH)
+
        event_dict: Dict[str, Any] = {
            "auth_events": auth_events,
            "prev_events": prev_events,
@@ -158,43 +167,21 @@ class EventBuilder:
            "sender": self.sender,
            "content": self.content,
            "unsigned": self.unsigned,
+            "depth": depth,
        }
-        if not self.room_version.linearized_matrix:
-            # Otherwise, progress the depth as normal
-            if depth is None:
-                (
-                    _,
-                    most_recent_prev_event_depth,
-                ) = await self._store.get_max_depth_of(prev_event_ids)
-
-                depth = most_recent_prev_event_depth + 1
-
-            # we cap depth of generated events, to ensure that they are not
-            # rejected by other servers (and so that they can be persisted in
-            # the db)
-            depth = min(depth, MAX_DEPTH)
-
-            event_dict["depth"] = depth
-        elif self.hub_server is not None:
-            event_dict["hub_server"] = self.hub_server

        if self.is_state():
            event_dict["state_key"] = self._state_key

        # MSC2174 moves the redacts property to the content, it is invalid to
        # provide it as a top-level property.
-        if self._redacts is not None and not self.room_version.updated_redaction_rules:
+        if self._redacts is not None and not self.room_version.msc2176_redaction_rules:
            event_dict["redacts"] = self._redacts

        if self._origin_server_ts is not None:
            event_dict["origin_server_ts"] = self._origin_server_ts

-        if self.room_version.linearized_matrix and self._lpdu_hashes:
-            event_dict.setdefault("hashes", {})["lpdu"] = self._lpdu_hashes
-        if self.room_version.linearized_matrix and self._lpdu_signatures:
-            event_dict.setdefault("signatures", {}).update(self._lpdu_signatures)
-
-        event = create_local_event_from_event_dict(
+        return create_local_event_from_event_dict(
            clock=self._clock,
            hostname=self._hostname,
            signing_key=self._signing_key,
@@ -203,8 +190,6 @@ class EventBuilder:
            internal_metadata_dict=self.internal_metadata.get_dict(),
        )

-        return event
-

 class EventBuilderFactory:
    def __init__(self, hs: "HomeServer"):
@@ -245,9 +230,6 @@ class EventBuilderFactory:
            unsigned=key_values.get("unsigned", {}),
            redacts=key_values.get("redacts", None),
            origin_server_ts=key_values.get("origin_server_ts", None),
-            hub_server=key_values.get("hub_server", None),
-            lpdu_hashes=key_values.get("lpdu_hashes", None),
-            lpdu_signatures=key_values.get("lpdu_signatures", None),
        )


@@ -276,11 +258,7 @@ def create_local_event_from_event_dict(
    if format_version == EventFormatVersions.ROOM_V1_V2:
        event_dict["event_id"] = _create_event_id(clock, hostname)

-    if not room_version.linearized_matrix:
-        # Do not add "origin" field to events (LPDUs and PDUs) sent in
-        # rooms that are meant to be compatible with linearized matrix.
-        event_dict["origin"] = hostname
-
+    event_dict["origin"] = hostname
    event_dict.setdefault("origin_server_ts", time_now)

    event_dict.setdefault("unsigned", {})
@@ -64,7 +64,6 @@ def prune_event(event: EventBase) -> EventBase:
    the user has specified, but we do want to keep necessary information like
    type, state_key etc.
    """
-
    pruned_event_dict = prune_event_dict(event.room_version, event.get_dict())

    from . import make_event_from_dict
@@ -103,20 +102,19 @@ def prune_event_dict(room_version: RoomVersion, event_dict: JsonDict) -> JsonDic
        "content",
        "type",
        "state_key",
+        "depth",
        "prev_events",
        "auth_events",
        "origin_server_ts",
    ]

-    # Earlier room versions from had additional allowed keys.
-    if not room_version.updated_redaction_rules:
-        allowed_keys.extend(["prev_state", "membership", "origin"])
+    # Room versions from before MSC2176 had additional allowed keys.
+    if not room_version.msc2176_redaction_rules:
+        allowed_keys.extend(["prev_state", "membership"])

-    # The hub server should not be redacted for linear matrix.
-    if room_version.linearized_matrix:
-        allowed_keys.append("hub_server")
-    else:
-        allowed_keys.append("depth")
+    # Room versions before MSC3989 kept the origin field.
+    if not room_version.msc3989_redaction_rules:
+        allowed_keys.append("origin")

    event_type = event_dict["type"]

@@ -129,9 +127,9 @@ def prune_event_dict(room_version: RoomVersion, event_dict: JsonDict) -> JsonDic

    if event_type == EventTypes.Member:
        add_fields("membership")
-        if room_version.restricted_join_rule_fix:
+        if room_version.msc3375_redaction_rules:
            add_fields(EventContentFields.AUTHORISING_USER)
-        if room_version.updated_redaction_rules:
+        if room_version.msc3821_redaction_rules:
            # Preserve the signed field under third_party_invite.
            third_party_invite = event_dict["content"].get("third_party_invite")
            if isinstance(third_party_invite, collections.abc.Mapping):
@@ -142,16 +140,14 @@ def prune_event_dict(room_version: RoomVersion, event_dict: JsonDict) -> JsonDic
                    ]

    elif event_type == EventTypes.Create:
-        if room_version.updated_redaction_rules:
-            # MSC2176 rules state that create events cannot have their `content` redacted.
-            new_content = event_dict["content"]
-        elif not room_version.implicit_room_creator:
-            # Some room versions give meaning to `creator`
-            add_fields("creator")
+        # MSC2176 rules state that create events cannot be redacted.
+        if room_version.msc2176_redaction_rules:
+            return event_dict

+        add_fields("creator")
    elif event_type == EventTypes.JoinRules:
        add_fields("join_rule")
-        if room_version.restricted_join_rule:
+        if room_version.msc3083_join_rules:
            add_fields("allow")
    elif event_type == EventTypes.PowerLevels:
        add_fields(
@@ -165,15 +161,24 @@ def prune_event_dict(room_version: RoomVersion, event_dict: JsonDict) -> JsonDic
            "redact",
        )

-        if room_version.updated_redaction_rules:
+        if room_version.msc2176_redaction_rules:
            add_fields("invite")

+        if room_version.msc2716_historical:
+            add_fields("historical")
+
    elif event_type == EventTypes.Aliases and room_version.special_case_aliases_auth:
        add_fields("aliases")
    elif event_type == EventTypes.RoomHistoryVisibility:
        add_fields("history_visibility")
-    elif event_type == EventTypes.Redaction and room_version.updated_redaction_rules:
+    elif event_type == EventTypes.Redaction and room_version.msc2176_redaction_rules:
        add_fields("redacts")
+    elif room_version.msc2716_redactions and event_type == EventTypes.MSC2716_INSERTION:
+        add_fields(EventContentFields.MSC2716_NEXT_BATCH_ID)
+    elif room_version.msc2716_redactions and event_type == EventTypes.MSC2716_BATCH:
+        add_fields(EventContentFields.MSC2716_BATCH_ID)
+    elif room_version.msc2716_redactions and event_type == EventTypes.MSC2716_MARKER:
+        add_fields(EventContentFields.MSC2716_INSERTION_EVENT_REFERENCE)

    # Protect the rel_type and event_id fields under the m.relates_to field.
    if room_version.msc3389_relation_redactions:
@@ -481,15 +486,6 @@ def serialize_event(
    if config.as_client_event:
        d = config.event_format(d)

-    # If the event is a redaction, copy the redacts field from the content to
-    # top-level for backwards compatibility.
-    if (
-        e.type == EventTypes.Redaction
-        and e.room_version.updated_redaction_rules
-        and e.redacts is not None
-    ):
-        d["redacts"] = e.redacts
-
    only_event_fields = config.only_event_fields
    if only_event_fields:
        if not isinstance(only_event_fields, list) or not all(
@@ -66,11 +66,6 @@ class EventValidator:
            "type",
        ]

-        if event.room_version.linearized_matrix:
-            # Do not add "origin" field to events (LPDUs and PDUs) sent in
-            # rooms that are meant to be compatible with linearized matrix.
-            required.remove("origin")
-
        for k in required:
            if k not in event:
                raise SynapseError(400, "Event does not have key %s" % (k,))
@@ -21,7 +21,7 @@ from synapse.api.room_versions import EventFormatVersions, RoomVersion
 from synapse.crypto.event_signing import check_event_content_hash
 from synapse.crypto.keyring import Keyring
 from synapse.events import EventBase, make_event_from_dict
-from synapse.events.utils import prune_event, prune_event_dict, validate_canonicaljson
+from synapse.events.utils import prune_event, validate_canonicaljson
 from synapse.http.servlet import assert_params_in_dict
 from synapse.logging.opentracing import log_kv, trace
 from synapse.types import JsonDict, get_domain_from_id
@@ -174,7 +174,7 @@ async def _check_sigs_on_pdu(

    # we want to check that the event is signed by:
    #
-    # (a) the sender's server or the hub
+    # (a) the sender's server
    #
    #     - except in the case of invites created from a 3pid invite, which are exempt
    #     from this check, because the sender has to match that of the original 3pid
@@ -193,22 +193,19 @@ async def _check_sigs_on_pdu(
    # let's start by getting the domain for each pdu, and flattening the event back
    # to JSON.

-    # First we check that the sender event is signed by the domain of the server
-    # which added it to the DAG.
+    # First we check that the sender event is signed by the sender's domain
    # (except if its a 3pid invite, in which case it may be sent by any server)
-    pdu_domain = pdu.pdu_domain
    sender_domain = get_domain_from_id(pdu.sender)
-    origin_server_ts_for_signing = (
-        pdu.origin_server_ts if room_version.enforce_key_validity else 0
-    )
    if not _is_invite_via_3pid(pdu):
        try:
            await keyring.verify_event_for_server(
-                pdu_domain, pdu, origin_server_ts_for_signing
+                sender_domain,
+                pdu,
+                pdu.origin_server_ts if room_version.enforce_key_validity else 0,
            )
        except Exception as e:
            raise InvalidEventSignatureError(
-                f"unable to verify signature for PDU domain {pdu_domain}: {e}",
+                f"unable to verify signature for sender domain {sender_domain}: {e}",
                pdu.event_id,
            ) from None

@@ -221,7 +218,9 @@ async def _check_sigs_on_pdu(
        if event_domain != sender_domain:
            try:
                await keyring.verify_event_for_server(
-                    event_domain, pdu, origin_server_ts_for_signing
+                    event_domain,
+                    pdu,
+                    pdu.origin_server_ts if room_version.enforce_key_validity else 0,
                )
            except Exception as e:
                raise InvalidEventSignatureError(
@@ -232,7 +231,7 @@ async def _check_sigs_on_pdu(
    # If this is a join event for a restricted room it may have been authorised
    # via a different server from the sending server. Check those signatures.
    if (
-        room_version.restricted_join_rule
+        room_version.msc3083_join_rules
        and pdu.type == EventTypes.Member
        and pdu.membership == Membership.JOIN
        and EventContentFields.AUTHORISING_USER in pdu.content
@@ -242,35 +241,16 @@ async def _check_sigs_on_pdu(
        )
        try:
            await keyring.verify_event_for_server(
-                authorising_server, pdu, origin_server_ts_for_signing
+                authorising_server,
+                pdu,
+                pdu.origin_server_ts if room_version.enforce_key_validity else 0,
            )
        except Exception as e:
            raise InvalidEventSignatureError(
-                f"unable to verify signature for authorising server {authorising_server}: {e}",
+                f"unable to verify signature for authorising serve {authorising_server}: {e}",
                pdu.event_id,
            ) from None

-    # If this is a linearized PDU we may need to check signatures of the hub
-    # and sender.
-    if room_version.event_format == EventFormatVersions.LINEARIZED:
-        # If the event was sent via a hub server, check the signature of the
-        # sender against the Linear PDU. (But only if the sender isn't the hub.)
-        #
-        # Note that the signature of the hub server, if one exists, is checked
-        # against the full PDU above.
-        if pdu.hub_server and pdu.hub_server != sender_domain:
-            try:
-                await keyring.verify_json_for_server(
-                    sender_domain,
-                    prune_event_dict(pdu.room_version, pdu.get_linearized_pdu_json()),
-                    origin_server_ts_for_signing,
-                )
-            except Exception as e:
-                raise InvalidEventSignatureError(
-                    f"unable to verify signature for sender {sender_domain}: {e}",
-                    pdu.event_id,
-                ) from None
-

 def _is_invite_via_3pid(event: EventBase) -> bool:
    return (
@@ -293,26 +273,21 @@ def event_from_pdu_json(pdu_json: JsonDict, room_version: RoomVersion) -> EventB
    """
    # we could probably enforce a bunch of other fields here (room_id, sender,
    # origin, etc etc)
-    if room_version.event_format == EventFormatVersions.LINEARIZED:
-        assert_params_in_dict(pdu_json, ("type",))
-    else:
-        assert_params_in_dict(pdu_json, ("type", "depth"))
-
-        depth = pdu_json["depth"]
-        if type(depth) is not int:
-            raise SynapseError(
-                400, "Depth %r not an integer" % (depth,), Codes.BAD_JSON
-            )
-
-        if depth < 0:
-            raise SynapseError(400, "Depth too small", Codes.BAD_JSON)
-        elif depth > MAX_DEPTH:
-            raise SynapseError(400, "Depth too large", Codes.BAD_JSON)
+    assert_params_in_dict(pdu_json, ("type", "depth"))

    # Strip any unauthorized values from "unsigned" if they exist
    if "unsigned" in pdu_json:
        _strip_unsigned_values(pdu_json)

+    depth = pdu_json["depth"]
+    if type(depth) is not int:
+        raise SynapseError(400, "Depth %r not an intger" % (depth,), Codes.BAD_JSON)
+
+    if depth < 0:
+        raise SynapseError(400, "Depth too small", Codes.BAD_JSON)
+    elif depth > MAX_DEPTH:
+        raise SynapseError(400, "Depth too large", Codes.BAD_JSON)
+
    # Validate that the JSON conforms to the specification.
    if room_version.strict_canonicaljson:
        validate_canonicaljson(pdu_json)
@@ -325,21 +325,9 @@ class FederationClient(FederationBase):
        if not extremities:
            return None

-        try:
-            # Note that this only returns pdus now, but this is close enough to a transaction.
-            transaction_data = await self.transport_layer.backfill_unstable(
-                dest, room_id, extremities, limit
-            )
-        except HttpResponseException as e:
-            # If an error is received that is due to an unrecognised endpoint,
-            # fallback to the v1 endpoint. Otherwise, consider it a legitimate error
-            # and raise.
-            if not is_unknown_endpoint(e):
-                raise
-
-            transaction_data = await self.transport_layer.backfill(
-                dest, room_id, extremities, limit
-            )
+        transaction_data = await self.transport_layer.backfill(
+            dest, room_id, extremities, limit
+        )

        logger.debug("backfill transaction_data=%r", transaction_data)

@@ -385,58 +373,45 @@ class FederationClient(FederationBase):
        Raises:
            SynapseError, NotRetryingDestination, FederationDeniedError
        """
-        try:
-            # Note that this only returns pdus now, but this is close enough to a transaction.
-            pdu_json = await self.transport_layer.get_event_unstable(
-                destination, event_id, timeout=timeout
-            )
-
-            pdu = event_from_pdu_json(pdu_json, room_version)
-
-        except HttpResponseException as e:
-            # If an error is received that is due to an unrecognised endpoint,
-            # fallback to the v1 endpoint. Otherwise, consider it a legitimate error
-            # and raise.
-            if not is_unknown_endpoint(e):
-                raise
-
-            transaction_data = await self.transport_layer.get_event(
-                destination, event_id, timeout=timeout
-            )
-
-            pdu_list: List[EventBase] = [
-                event_from_pdu_json(p, room_version) for p in transaction_data["pdus"]
-            ]
-
-            if pdu_list and pdu_list[0]:
-                pdu = pdu_list[0]
-            else:
-                return None
+        transaction_data = await self.transport_layer.get_event(
+            destination, event_id, timeout=timeout
+        )

        logger.debug(
            "get_pdu_from_destination_raw: retrieved event id %s from %s: %r",
            event_id,
            destination,
-            pdu,
+            transaction_data,
        )

-        # Check signatures are correct.
-        try:
+        pdu_list: List[EventBase] = [
+            event_from_pdu_json(p, room_version) for p in transaction_data["pdus"]
+        ]

-            async def _record_failure_callback(event: EventBase, cause: str) -> None:
-                await self.store.record_event_failed_pull_attempt(
-                    event.room_id, event.event_id, cause
+        if pdu_list and pdu_list[0]:
+            pdu = pdu_list[0]
+
+            # Check signatures are correct.
+            try:
+
+                async def _record_failure_callback(
+                    event: EventBase, cause: str
+                ) -> None:
+                    await self.store.record_event_failed_pull_attempt(
+                        event.room_id, event.event_id, cause
+                    )
+
+                signed_pdu = await self._check_sigs_and_hash(
+                    room_version, pdu, _record_failure_callback
                )
+            except InvalidEventSignatureError as e:
+                errmsg = f"event id {pdu.event_id}: {e}"
+                logger.warning("%s", errmsg)
+                raise SynapseError(403, errmsg, Codes.FORBIDDEN)

-            signed_pdu = await self._check_sigs_and_hash(
-                room_version, pdu, _record_failure_callback
-            )
-        except InvalidEventSignatureError as e:
-            errmsg = f"event id {pdu.event_id}: {e}"
-            logger.warning("%s", errmsg)
-            raise SynapseError(403, errmsg, Codes.FORBIDDEN)
+            return signed_pdu

-        return signed_pdu
+        return None

    @trace
    @tag_args
@@ -1008,7 +983,7 @@ class FederationClient(FederationBase):
            if not room_version:
                raise UnsupportedRoomVersionError()

-            if not room_version.knock_join_rule and membership == Membership.KNOCK:
+            if not room_version.msc2403_knocking and membership == Membership.KNOCK:
                raise SynapseError(
                    400,
                    "This room version does not support knocking",
@@ -1094,7 +1069,7 @@ class FederationClient(FederationBase):
            # * Ensure the signatures are good.
            #
            # Otherwise, fallback to the provided event.
-            if room_version.restricted_join_rule and response.event:
+            if room_version.msc3083_join_rules and response.event:
                event = response.event

                valid_pdu = await self._check_sigs_and_hash_and_fetch_one(
@@ -1220,7 +1195,7 @@ class FederationClient(FederationBase):

        # MSC3083 defines additional error codes for room joins.
        failover_errcodes = None
-        if room_version.restricted_join_rule:
+        if room_version.msc3083_join_rules:
            failover_errcodes = (
                Codes.UNABLE_AUTHORISE_JOIN,
                Codes.UNABLE_TO_GRANT_JOIN,
@@ -1247,19 +1222,6 @@ class FederationClient(FederationBase):
    ) -> SendJoinResponse:
        time_now = self._clock.time_msec()

-        try:
-            return await self.transport_layer.send_join_unstable(
-                room_version=room_version,
-                destination=destination,
-                txn_id=pdu.event_id,
-                content=pdu.get_pdu_json(time_now),
-            )
-        except HttpResponseException as e:
-            # If an error is received that is due to an unrecognised endpoint,
-            # fallback to the v2 endpoint.
-            if not is_unknown_endpoint(e):
-                raise
-
        try:
            return await self.transport_layer.send_join_v2(
                room_version=room_version,
@@ -1331,22 +1293,6 @@ class FederationClient(FederationBase):
        """
        time_now = self._clock.time_msec()

-        try:
-            return await self.transport_layer.send_invite_unstable(
-                destination=destination,
-                txn_id=pdu.event_id,
-                content={
-                    "event": pdu.get_pdu_json(time_now),
-                    "room_version": room_version.identifier,
-                    "invite_room_state": pdu.unsigned.get("invite_room_state", []),
-                },
-            )
-        except HttpResponseException as e:
-            # If an error is received that is due to an unrecognised endpoint,
-            # fallback to the v2.
-            if not is_unknown_endpoint(e):
-                raise
-
        try:
            return await self.transport_layer.send_invite_v2(
                destination=destination,
@@ -1413,18 +1359,6 @@ class FederationClient(FederationBase):
    async def _do_send_leave(self, destination: str, pdu: EventBase) -> JsonDict:
        time_now = self._clock.time_msec()

-        try:
-            return await self.transport_layer.send_leave_unstable(
-                destination=destination,
-                txn_id=pdu.event_id,
-                content=pdu.get_pdu_json(time_now),
-            )
-        except HttpResponseException as e:
-            # If an error is received that is due to an unrecognised endpoint,
-            # fallback to the v2.
-            if not is_unknown_endpoint(e):
-                raise
-
        try:
            return await self.transport_layer.send_leave_v2(
                destination=destination,
@@ -1501,18 +1435,6 @@ class FederationClient(FederationBase):
        """
        time_now = self._clock.time_msec()

-        try:
-            return await self.transport_layer.send_knock_unstable(
-                destination=destination,
-                txn_id=pdu.event_id,
-                content=pdu.get_pdu_json(time_now),
-            )
-        except HttpResponseException as e:
-            # If an error is received that is due to an unrecognised endpoint,
-            # fallback to the v2.
-            if not is_unknown_endpoint(e):
-                raise
-
        return await self.transport_layer.send_knock_v1(
            destination=destination,
            room_id=pdu.room_id,
@@ -13,7 +13,6 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-import collections.abc
 import logging
 import random
 from typing import (
@@ -27,6 +26,7 @@ from typing import (
    Mapping,
    Optional,
    Tuple,
+    Union,
 )

 from matrix_common.regex import glob_to_regex
@@ -56,14 +56,13 @@ from synapse.api.room_versions import KNOWN_ROOM_VERSIONS, RoomVersion
 from synapse.crypto.event_signing import compute_event_signature
 from synapse.events import EventBase
 from synapse.events.snapshot import EventContext
-from synapse.events.utils import validate_canonicaljson
 from synapse.federation.federation_base import (
    FederationBase,
    InvalidEventSignatureError,
    event_from_pdu_json,
 )
 from synapse.federation.persistence import TransactionActions
-from synapse.federation.units import Transaction
+from synapse.federation.units import Edu, Transaction
 from synapse.http.servlet import assert_params_in_dict
 from synapse.logging.context import (
    make_deferred_yieldable,
@@ -214,15 +213,19 @@ class FederationServer(FederationBase):

    async def on_backfill_request(
        self, origin: str, room_id: str, versions: List[str], limit: int
-    ) -> List[EventBase]:
+    ) -> Tuple[int, Dict[str, Any]]:
        async with self._server_linearizer.queue((origin, room_id)):
            origin_host, _ = parse_server_name(origin)
            await self.check_server_matches_acl(origin_host, room_id)

-            return await self.handler.on_backfill_request(
+            pdus = await self.handler.on_backfill_request(
                origin, room_id, versions, limit
            )

+            res = self._transaction_dict_from_pdus(pdus)
+
+        return 200, res
+
    async def on_timestamp_to_event_request(
        self, origin: str, room_id: str, timestamp: int, direction: Direction
    ) -> Tuple[int, Dict[str, Any]]:
@@ -460,21 +463,7 @@ class FederationServer(FederationBase):
                logger.info("Ignoring PDU: %s", e)
                continue

-            # If the transaction is from a linearized matrix server, create PDUs
-            # from the LPDUs.
-            if (
-                room_version.linearized_matrix
-                and await self.keyring.is_server_linearized(origin)
-            ):
-                event = await self._on_lpdu_event(p, room_version)
-
-                # Send this event on behalf of the other server.
-                #
-                # We are acting as the hub for this transaction and need to send
-                # this event out to other participants.
-                event.internal_metadata.send_on_behalf_of = origin
-            else:
-                event = event_from_pdu_json(p, room_version)
+            event = event_from_pdu_json(p, room_version)
            pdus_by_room.setdefault(room_id, []).append(event)

            if event.origin_server_ts > newest_pdu_ts:
@@ -545,45 +534,13 @@ class FederationServer(FederationBase):
        async def _process_edu(edu_dict: JsonDict) -> None:
            received_edus_counter.inc()

-            # TODO(LM) Handle this more natively instead of munging to the current form.
-            if "type" in edu_dict:
-                edu_type = edu_dict["type"]
-                content = edu_dict["content"]
-                sender = edu_dict["sender"]
-
-                if edu_type == EduTypes.DEVICE_LIST_UPDATE:
-                    for device_info in content.get("changed", []):
-                        device_info["stream_id"] = 0  # XXX Will this work?
-                        await self.registry.on_edu(edu_type, origin, device_info)
-
-                    for device_id in content.get("removed", []):
-                        new_content = {
-                            "device_id": device_id,
-                            "deleted": True,
-                            "stream_id": 0,  # XXX Will this work?
-                            "user_id": sender,
-                        }
-                        await self.registry.on_edu(edu_type, origin, new_content)
-                elif edu_type == EduTypes.DIRECT_TO_DEVICE:
-                    new_content = {
-                        "message_id": 0,  # XXX Will this work?
-                        "messages": {
-                            content["target"]: {
-                                content["target_device"]: content["message"]
-                            }
-                        },
-                        "sender": sender,
-                        "type": content["message_type"],
-                    }
-                    await self.registry.on_edu(edu_type, origin, new_content)
-                else:
-                    raise ValueError()
-
-            else:
-                edu_type = edu_dict["edu_type"]
-                content = edu_dict["content"]
-
-                await self.registry.on_edu(edu_type, origin, content)
+            edu = Edu(
+                origin=origin,
+                destination=self.server_name,
+                edu_type=edu_dict["edu_type"],
+                content=edu_dict["content"],
+            )
+            await self.registry.on_edu(edu.edu_type, origin, edu.content)

        await concurrently_execute(
            _process_edu,
@@ -659,8 +616,15 @@ class FederationServer(FederationBase):
            "auth_chain": [pdu.get_pdu_json() for pdu in auth_chain],
        }

-    async def on_pdu_request(self, origin: str, event_id: str) -> Optional[EventBase]:
-        return await self.handler.get_persisted_pdu(origin, event_id)
+    async def on_pdu_request(
+        self, origin: str, event_id: str
+    ) -> Tuple[int, Union[JsonDict, str]]:
+        pdu = await self.handler.get_persisted_pdu(origin, event_id)
+
+        if pdu:
+            return 200, self._transaction_dict_from_pdus([pdu])
+        else:
+            return 404, ""

    async def on_query_request(
        self, query_type: str, args: Dict[str, str]
@@ -675,14 +639,12 @@ class FederationServer(FederationBase):
        origin_host, _ = parse_server_name(origin)
        await self.check_server_matches_acl(origin_host, room_id)

-        # checking the room version will check that we've actually heard of the room
-        # (and return a 404 otherwise)
-        room_version_id = await self.store.get_room_version_id(room_id)
-        if room_version_id not in supported_versions:
+        room_version = await self.store.get_room_version_id(room_id)
+        if room_version not in supported_versions:
            logger.warning(
-                "Room version %s not in %s", room_version_id, supported_versions
+                "Room version %s not in %s", room_version, supported_versions
            )
-            raise IncompatibleRoomVersionError(room_version=room_version_id)
+            raise IncompatibleRoomVersionError(room_version=room_version)

        # Refuse the request if that room has seen too many joins recently.
        # This is in addition to the HS-level rate limiting applied by
@@ -693,21 +655,8 @@ class FederationServer(FederationBase):
            key=room_id,
            update=False,
        )
-        pdu = await self.handler.on_make_join_request(
-            origin, room_id, KNOWN_ROOM_VERSIONS[room_version_id], user_id
-        )
-
-        # For a linearized matrix server return a templated LPDU, otherwise a templated PDU.
-        # TODO(LM) Is it an error for a LM server to not be on a LM room version?
-        if (
-            pdu.room_version.linearized_matrix
-            and await self.keyring.is_server_linearized(origin)
-        ):
-            event_json = pdu.get_templated_linearized_pdu_json()
-        else:
-            event_json = pdu.get_templated_pdu_json()
-
-        return {"event": event_json, "room_version": room_version_id}
+        pdu = await self.handler.on_make_join_request(origin, room_id, user_id)
+        return {"event": pdu.get_templated_pdu_json(), "room_version": room_version}

    async def on_invite_request(
        self, origin: str, content: JsonDict, room_version_id: str
@@ -737,14 +686,13 @@ class FederationServer(FederationBase):
        self,
        origin: str,
        content: JsonDict,
-        expected_room_id: Optional[str] = None,
+        room_id: str,
        caller_supports_partial_state: bool = False,
    ) -> Dict[str, Any]:
        set_tag(
            SynapseTags.SEND_JOIN_RESPONSE_IS_PARTIAL_STATE,
            caller_supports_partial_state,
        )
-        room_id = content["room_id"]
        await self._room_member_handler._join_rate_per_room_limiter.ratelimit(  # type: ignore[has-type]
            requester=None,
            key=room_id,
@@ -752,7 +700,7 @@ class FederationServer(FederationBase):
        )

        event, context = await self._on_send_membership_event(
-            origin, content, Membership.JOIN, expected_room_id
+            origin, content, Membership.JOIN, room_id
        )

        prev_state_ids = await context.get_prev_state_ids()
@@ -783,14 +731,6 @@ class FederationServer(FederationBase):
        auth_chain_events = await self.store.get_events_as_list(auth_chain_event_ids)
        state_events = await self.store.get_events_as_list(state_event_ids)

-        # TODO(LM) eigen-server wants events in order.
-        auth_chain_events = sorted(
-            auth_chain_events, key=lambda e: e.internal_metadata.stream_ordering
-        )
-        state_events = sorted(
-            state_events, key=lambda e: e.internal_metadata.stream_ordering
-        )
-
        # we try to do all the async stuff before this point, so that time_now is as
        # accurate as possible.
        time_now = self._clock.time_msec()
@@ -816,25 +756,13 @@ class FederationServer(FederationBase):

        room_version = await self.store.get_room_version_id(room_id)

-        # For a linearized matrix server return a templated LPDU, otherwise a templated PDU.
-        # TODO(LM) Is it an error for a LM server to not be on a LM room version?
-        if (
-            pdu.room_version.linearized_matrix
-            and await self.keyring.is_server_linearized(origin)
-        ):
-            event_json = pdu.get_templated_linearized_pdu_json()
-        else:
-            event_json = pdu.get_templated_pdu_json()
-
-        return {"event": event_json, "room_version": room_version}
+        return {"event": pdu.get_templated_pdu_json(), "room_version": room_version}

    async def on_send_leave_request(
-        self, origin: str, content: JsonDict, expected_room_id: Optional[str] = None
+        self, origin: str, content: JsonDict, room_id: str
    ) -> dict:
        logger.debug("on_send_leave_request: content: %s", content)
-        await self._on_send_membership_event(
-            origin, content, Membership.LEAVE, expected_room_id
-        )
+        await self._on_send_membership_event(origin, content, Membership.LEAVE, room_id)
        return {}

    async def on_make_knock_request(
@@ -878,7 +806,7 @@ class FederationServer(FederationBase):
            raise IncompatibleRoomVersionError(room_version=room_version.identifier)

        # Check that this room supports knocking as defined by its room version
-        if not room_version.knock_join_rule:
+        if not room_version.msc2403_knocking:
            raise SynapseError(
                403,
                "This room version does not support knocking",
@@ -886,24 +814,16 @@ class FederationServer(FederationBase):
            )

        pdu = await self.handler.on_make_knock_request(origin, room_id, user_id)
-
-        # For a linearized matrix server return a templated LPDU, otherwise a templated PDU.
-        # TODO(LM) Is it an error for a LM server to not be on a LM room version?
-        if (
-            pdu.room_version.linearized_matrix
-            and await self.keyring.is_server_linearized(origin)
-        ):
-            event_json = pdu.get_templated_linearized_pdu_json()
-        else:
-            event_json = pdu.get_templated_pdu_json()
-
        return {
-            "event": event_json,
+            "event": pdu.get_templated_pdu_json(),
            "room_version": room_version.identifier,
        }

    async def on_send_knock_request(
-        self, origin: str, content: JsonDict, expected_room_id: Optional[str] = None
+        self,
+        origin: str,
+        content: JsonDict,
+        room_id: str,
    ) -> Dict[str, List[JsonDict]]:
        """
        We have received a knock event for a room. Verify and send the event into the room
@@ -913,13 +833,13 @@ class FederationServer(FederationBase):
        Args:
            origin: The remote homeserver of the knocking user.
            content: The content of the request.
-            expected_room_id: The room ID included in the request.
+            room_id: The ID of the room to knock on.

        Returns:
            The stripped room state.
        """
        _, context = await self._on_send_membership_event(
-            origin, content, Membership.KNOCK, expected_room_id
+            origin, content, Membership.KNOCK, room_id
        )

        # Retrieve stripped state events from the room and send them back to the remote
@@ -940,11 +860,7 @@ class FederationServer(FederationBase):
        }

    async def _on_send_membership_event(
-        self,
-        origin: str,
-        content: JsonDict,
-        membership_type: str,
-        expected_room_id: Optional[str],
+        self, origin: str, content: JsonDict, membership_type: str, room_id: str
    ) -> Tuple[EventBase, EventContext]:
        """Handle an on_send_{join,leave,knock} request

@@ -956,8 +872,8 @@ class FederationServer(FederationBase):
            content: The body of the send_* request - a complete membership event
            membership_type: The expected membership type (join or leave, depending
                on the endpoint)
-            expected_room_id: The room_id from the request, to be validated against
-                the room_id in the event. None if the request did not include a room ID.
+            room_id: The room_id from the request, to be validated against the room_id
+                in the event

        Returns:
            The event and context of the event after inserting it into the room graph.
@@ -967,16 +883,12 @@ class FederationServer(FederationBase):
               the room_id not matching or the event not being authorized.
        """
        assert_params_in_dict(content, ["room_id"])
-        if expected_room_id is None:
-            room_id = content["room_id"]
-        elif content["room_id"] != expected_room_id:
+        if content["room_id"] != room_id:
            raise SynapseError(
                400,
                "Room ID in body does not match that in request path",
                Codes.BAD_JSON,
            )
-        else:
-            room_id = expected_room_id

        # Note that get_room_version throws if the room does not exist here.
        room_version = await self.store.get_room_version(room_id)
@@ -997,21 +909,14 @@ class FederationServer(FederationBase):
                errcode=Codes.NOT_FOUND,
            )

-        if membership_type == Membership.KNOCK and not room_version.knock_join_rule:
+        if membership_type == Membership.KNOCK and not room_version.msc2403_knocking:
            raise SynapseError(
                403,
                "This room version does not support knocking",
                errcode=Codes.FORBIDDEN,
            )

-        # Linearized Matrix requires building the event (i.e. adding auth/prev
-        # events). The input content is an LPDU.
-        if room_version.linearized_matrix and await self.keyring.is_server_linearized(
-            origin
-        ):
-            event = await self._on_lpdu_event(content, room_version)
-        else:
-            event = event_from_pdu_json(content, room_version)
+        event = event_from_pdu_json(content, room_version)

        if event.type != EventTypes.Member or not event.is_state():
            raise SynapseError(400, "Not an m.room.member event", Codes.BAD_JSON)
@@ -1028,7 +933,7 @@ class FederationServer(FederationBase):
        # the event is valid to be sent into the room. Currently this is only done
        # if the user is being joined via restricted join rules.
        if (
-            room_version.restricted_join_rule
+            room_version.msc3083_join_rules
            and event.membership == Membership.JOIN
            and EventContentFields.AUTHORISING_USER in event.content
        ):
@@ -1073,120 +978,6 @@ class FederationServer(FederationBase):
                origin, event
            )

-    async def _on_lpdu_event(
-        self, lpdu_json: JsonDict, room_version: RoomVersion
-    ) -> EventBase:
-        """Construct an EventBase from a linearized event json received over federation
-
-        See event_from_pdu_json.
-
-        Args:
-            lpdu_json: lpdu as received over federation
-            room_version: The version of the room this event belongs to
-
-        Raises:
-            SynapseError: if the lpdu is missing required fields or is otherwise
-                not a valid matrix event
-        """
-        if not room_version.linearized_matrix:
-            raise ValueError("Cannot be used on non-linearized matrix")
-
-        # The minimum fields to create an LPDU.
-        assert_params_in_dict(
-            lpdu_json,
-            (
-                "type",
-                "room_id",
-                "sender",
-                "hub_server",
-                "origin_server_ts",
-                "content",
-                "hashes",
-                "signatures",
-            ),
-        )
-
-        # The participant server should *not* provide auth/prev events.
-        for field in ("auth_events", "prev_events"):
-            if field in lpdu_json:
-                raise SynapseError(400, f"LPDU contained {field}", Codes.BAD_JSON)
-
-        # Hashes must contain (only) "lpdu".
-        hashes = lpdu_json.pop("hashes")
-        if not isinstance(hashes, collections.abc.Mapping):
-            raise SynapseError(400, "Invalid hashes", Codes.BAD_JSON)
-        if hashes.keys() != {"lpdu"}:
-            raise SynapseError(
-                400, "hashes must contain exactly one key: 'lpdu'", Codes.BAD_JSON
-            )
-        lpdu_json["lpdu_hashes"] = hashes["lpdu"]
-        lpdu_json["lpdu_signatures"] = lpdu_json.pop("signatures")
-
-        # Validate that the JSON conforms to the specification.
-        if room_version.strict_canonicaljson:
-            validate_canonicaljson(lpdu_json)
-
-        # An LPDU doesn't have enough information to just create an event, it needs
-        # to be built and signed, etc.
-        builder = self.hs.get_event_builder_factory().for_room_version(
-            room_version, lpdu_json
-        )
-
-        try:
-            (
-                event,
-                unpersisted_context,
-            ) = await self.hs.get_event_creation_handler().create_new_client_event(
-                builder=builder
-            )
-        except SynapseError as e:
-            logger.warning(
-                "Failed to create PDU from template for room %s because %s",
-                lpdu_json["room_id"],
-                e,
-            )
-            raise
-
-        if not event.hub_server:
-            raise SynapseError(400, "Cannot send PDU via hub server", Codes.BAD_JSON)
-
-        # If an LPDU is trying to be sent through us, but we're not the hub
-        # then deny.
-        if not self._is_mine_server_name(event.hub_server):
-            raise SynapseError(
-                400,
-                f"Cannot authorise event for hub server: {event.hub_server}",
-                Codes.FORBIDDEN,
-            )
-
-        # Double check that we *are* the hub.
-        state_ids = await self._state_storage_controller.get_current_state_ids(
-            event.room_id
-        )
-        # Get the current hub server from the sender of the create event.
-        create_event = await self.store.get_event(state_ids[(EventTypes.Create, "")])
-        hub_server = get_domain_from_id(create_event.sender)
-
-        if not self._is_mine_server_name(hub_server):
-            raise SynapseError(400, "Not the hub server", Codes.FORBIDDEN)
-
-        # Sign the event as the hub.
-        event.signatures.update(
-            compute_event_signature(
-                room_version,
-                event.get_pdu_json(),
-                self.hs.hostname,
-                self.hs.signing_key,
-            )
-        )
-
-        # Note that the signatures, etc. get checked later on in _handle_received_pdu.
-        # Server ACLs are checked in the caller: _handle_pdus_in_txn.
-
-        # TODO(LM) Do we need to check that the event will be accepted here?
-
-        return event
-
    async def on_event_auth(
        self, origin: str, room_id: str, event_id: str
    ) -> Tuple[int, Dict[str, Any]]:
@@ -1282,6 +1073,21 @@ class FederationServer(FederationBase):
        ts_now_ms = self._clock.time_msec()
        return await self.store.get_user_id_for_open_id_token(token, ts_now_ms)

+    def _transaction_dict_from_pdus(self, pdu_list: List[EventBase]) -> JsonDict:
+        """Returns a new Transaction containing the given PDUs suitable for
+        transmission.
+        """
+        time_now = self._clock.time_msec()
+        pdus = [p.get_pdu_json(time_now) for p in pdu_list]
+        return Transaction(
+            # Just need a dummy transaction ID and destination since it won't be used.
+            transaction_id="",
+            origin=self.server_name,
+            pdus=pdus,
+            origin_server_ts=int(time_now),
+            destination="",
+        ).get_dict()
+
    async def _handle_received_pdu(self, origin: str, pdu: EventBase) -> None:
        """Process a PDU received in a federation /send/ transaction.

--- a/Show More
+++ b/Show More
Author	SHA1	Message	Date
Mathieu Velten	cec52ea9b0	Add changelog	2023-06-14 00:50:53 +02:00
Mathieu Velten	0cb8502bbb	Use parse_duration for newly introduced options	2023-06-13 23:55:15 +02:00
				`@@ -0,0 +1 @@`
				`Allow for the configuration of max request retries and min/max retry delays in the matrix federation client.`
				`@@ -0,0 +1 @@`
				`Log when events are (maybe unexpectedly) filtered out of responses in tests.`
				`@@ -0,0 +1 @@`
				Replace `EventContext` fields `prev_group` and `delta_ids` with field `state_group_deltas`.
				`@@ -0,0 +1 @@`
				`Stable support for [MSC3882](https://github.com/matrix-org/matrix-spec-proposals/pull/3882) to allow an existing device/session to generate a login token for use on a new device/session.`
				`@@ -0,0 +1 @@`
				`Support resolving a room's [canonical alias](https://spec.matrix.org/v1.7/client-server-api/#mroomcanonical_alias) via the module API.`
				`@@ -0,0 +1 @@`
				`Enable support for [MSC3952](https://github.com/matrix-org/matrix-spec-proposals/pull/3952): intentional mentions.`
				`@@ -0,0 +1 @@`
				`Experimental [MSC3861](https://github.com/matrix-org/matrix-spec-proposals/pull/3861) support: delegate auth to an OIDC provider.`
				`@@ -0,0 +1 @@`
				`Correctly clear caches when we delete a room.`
				`@@ -0,0 +1 @@`
				Read from column `full_user_id` rather than `user_id` of tables `profiles` and `user_filters`.