Fix other test cases

Changelog
Enable login_via_existing_session by default
2023-06-05 18:02:20 +01:00 · 2023-06-05 14:45:29 +01:00 · 2023-06-05 14:42:31 +01:00
150 changed files with 2912 additions and 2140 deletions
@@ -22,21 +22,7 @@ concurrency:
  cancel-in-progress: true

 jobs:
-  check_repo:
-    # Prevent this workflow from running on any fork of Synapse other than matrix-org/synapse, as it is
-    # only useful to the Synapse core team.
-    # All other workflow steps depend on this one, thus if 'should_run_workflow' is not 'true', the rest
-    # of the workflow will be skipped as well.
-    runs-on: ubuntu-latest
-    outputs:
-      should_run_workflow: ${{ steps.check_condition.outputs.should_run_workflow }}
-    steps:
-      - id: check_condition
-        run: echo "should_run_workflow=${{ github.repository == 'matrix-org/synapse' }}" >> "$GITHUB_OUTPUT"
-
  mypy:
-    needs: check_repo
-    if: needs.check_repo.outputs.should_run_workflow == 'true'
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v3
@@ -61,8 +47,6 @@ jobs:
        run: sed '/warn_unused_ignores = True/d' -i mypy.ini
      - run: poetry run mypy
  trial:
-    needs: check_repo
-    if: needs.check_repo.outputs.should_run_workflow == 'true'
    runs-on: ubuntu-latest
    strategy:
      matrix:
@@ -121,8 +105,6 @@ jobs:


  sytest:
-    needs: check_repo
-    if: needs.check_repo.outputs.should_run_workflow == 'true'
    runs-on: ubuntu-latest
    container:
      image: matrixdotorg/sytest-synapse:testing
@@ -174,8 +156,7 @@ jobs:


  complement:
-    needs: check_repo
-    if: "!failure() && !cancelled() && needs.check_repo.outputs.should_run_workflow == 'true'"
+    if: "${{ !failure() && !cancelled() }}"
    runs-on: ubuntu-latest

    strategy:
@@ -211,7 +192,7 @@ jobs:
  # Open an issue if the build fails, so we know about it.
  # Only do this if we're not experimenting with this action in a PR.
  open-issue:
-    if: "failure() && github.event_name != 'push' && github.event_name != 'pull_request' && needs.check_repo.outputs.should_run_workflow == 'true'"
+    if: "failure() && github.event_name != 'push' && github.event_name != 'pull_request'"
    needs:
      # TODO: should mypy be included here? It feels more brittle than the others.
      - mypy
@@ -35,7 +35,7 @@ jobs:
    steps:
      - uses: actions/checkout@v3
      - name: Install Rust
-        uses: dtolnay/rust-toolchain@1.60.0
+        uses: dtolnay/rust-toolchain@1.58.1
      - uses: Swatinem/rust-cache@v2
      - uses: matrix-org/setup-python-poetry@v1
        with:
@@ -92,10 +92,6 @@ jobs:
      - name: Checkout repository
        uses: actions/checkout@v3

-      - name: Install Rust
-        uses: dtolnay/rust-toolchain@1.60.0
-      - uses: Swatinem/rust-cache@v2
-
      - name: Setup Poetry
        uses: matrix-org/setup-python-poetry@v1
        with:
@@ -107,6 +103,10 @@ jobs:
          # To make CI green, err towards caution and install the project.
          install-project: "true"

+      - name: Install Rust
+        uses: dtolnay/rust-toolchain@1.58.1
+      - uses: Swatinem/rust-cache@v2
+
      # Cribbed from
      # https://github.com/AustinScola/mypy-cache-github-action/blob/85ea4f2972abed39b33bd02c36e341b28ca59213/src/restore.ts#L10-L17
      - name: Restore/persist mypy's cache
@@ -150,7 +150,7 @@ jobs:
        with:
          ref: ${{ github.event.pull_request.head.sha }}
      - name: Install Rust
-        uses: dtolnay/rust-toolchain@1.60.0
+        uses: dtolnay/rust-toolchain@1.58.1
      - uses: Swatinem/rust-cache@v2
      - uses: matrix-org/setup-python-poetry@v1
        with:
@@ -167,7 +167,7 @@ jobs:
      - uses: actions/checkout@v3

      - name: Install Rust
-        uses: dtolnay/rust-toolchain@1.60.0
+        uses: dtolnay/rust-toolchain@1.58.1
        with:
            components: clippy
      - uses: Swatinem/rust-cache@v2
@@ -268,7 +268,7 @@ jobs:
            postgres:${{ matrix.job.postgres-version }}

      - name: Install Rust
-        uses: dtolnay/rust-toolchain@1.60.0
+        uses: dtolnay/rust-toolchain@1.58.1
      - uses: Swatinem/rust-cache@v2

      - uses: matrix-org/setup-python-poetry@v1
@@ -308,7 +308,7 @@ jobs:
      - uses: actions/checkout@v3

      - name: Install Rust
-        uses: dtolnay/rust-toolchain@1.60.0
+        uses: dtolnay/rust-toolchain@1.58.1
      - uses: Swatinem/rust-cache@v2

      # There aren't wheels for some of the older deps, so we need to install
@@ -399,8 +399,8 @@ jobs:
      env:
        SYTEST_BRANCH: ${{ github.head_ref }}
        POSTGRES: ${{ matrix.job.postgres && 1}}
-        MULTI_POSTGRES: ${{ (matrix.job.postgres == 'multi-postgres') || '' }}
-        ASYNCIO_REACTOR: ${{ (matrix.job.reactor == 'asyncio') || '' }}
+        MULTI_POSTGRES: ${{ (matrix.job.postgres == 'multi-postgres') && 1}}
+        ASYNCIO_REACTOR: ${{ (matrix.job.reactor == 'asyncio') && 1 }}
        WORKERS: ${{ matrix.job.workers && 1 }}
        BLACKLIST: ${{ matrix.job.workers && 'synapse-blacklist-with-workers' }}
        TOP: ${{ github.workspace }}
@@ -416,7 +416,7 @@ jobs:
        run: cat sytest-blacklist .ci/worker-blacklist > synapse-blacklist-with-workers

      - name: Install Rust
-        uses: dtolnay/rust-toolchain@1.60.0
+        uses: dtolnay/rust-toolchain@1.58.1
      - uses: Swatinem/rust-cache@v2

      - name: Run SyTest
@@ -556,7 +556,7 @@ jobs:
          path: synapse

      - name: Install Rust
-        uses: dtolnay/rust-toolchain@1.60.0
+        uses: dtolnay/rust-toolchain@1.58.1
      - uses: Swatinem/rust-cache@v2

      - uses: actions/setup-go@v4
@@ -584,7 +584,7 @@ jobs:
      - uses: actions/checkout@v3

      - name: Install Rust
-        uses: dtolnay/rust-toolchain@1.60.0
+        uses: dtolnay/rust-toolchain@1.58.1
      - uses: Swatinem/rust-cache@v2

      - run: cargo test
@@ -18,22 +18,7 @@ concurrency:
  cancel-in-progress: true

 jobs:
-  check_repo:
-    # Prevent this workflow from running on any fork of Synapse other than matrix-org/synapse, as it is
-    # only useful to the Synapse core team.
-    # All other workflow steps depend on this one, thus if 'should_run_workflow' is not 'true', the rest
-    # of the workflow will be skipped as well.
-    if: github.repository == 'matrix-org/synapse'
-    runs-on: ubuntu-latest
-    outputs:
-      should_run_workflow: ${{ steps.check_condition.outputs.should_run_workflow }}
-    steps:
-      - id: check_condition
-        run: echo "should_run_workflow=${{ github.repository == 'matrix-org/synapse' }}" >> "$GITHUB_OUTPUT"
-
  mypy:
-    needs: check_repo
-    if: needs.check_repo.outputs.should_run_workflow == 'true'
    runs-on: ubuntu-latest

    steps:
@@ -56,8 +41,6 @@ jobs:
      - run: poetry run mypy

  trial:
-    needs: check_repo
-    if: needs.check_repo.outputs.should_run_workflow == 'true'
    runs-on: ubuntu-latest

    steps:
@@ -92,8 +75,6 @@ jobs:
          || true

  sytest:
-    needs: check_repo
-    if: needs.check_repo.outputs.should_run_workflow == 'true'
    runs-on: ubuntu-latest
    container:
      image: matrixdotorg/sytest-synapse:buster
@@ -138,8 +119,7 @@ jobs:
            /logs/**/*.log*

  complement:
-    needs: check_repo
-    if: "!failure() && !cancelled() && needs.check_repo.outputs.should_run_workflow == 'true'"
+    if: "${{ !failure() && !cancelled() }}"
    runs-on: ubuntu-latest

    strategy:
@@ -186,7 +166,7 @@ jobs:

  # open an issue if the build fails, so we know about it.
  open-issue:
-    if: failure() && needs.check_repo.outputs.should_run_workflow == 'true'
+    if: failure()
    needs:
      - mypy
      - trial
@@ -1,129 +1,3 @@
-Synapse 1.86.0 (2023-06-20)
-===========================
-
-No significant changes since 1.86.0rc2.
-
-
-Synapse 1.86.0rc2 (2023-06-14)
-==============================
-
-Bugfixes
--------
-
- Fix an error when having workers of different versions running. ([\#15774](https://github.com/matrix-org/synapse/issues/15774))
-
-
-Synapse 1.86.0rc1 (2023-06-13)
-==============================
-
-This version was tagged but never released.
-
-Features
--------
-
- Stable support for [MSC3882](https://github.com/matrix-org/matrix-spec-proposals/pull/3882) to allow an existing device/session to generate a login token for use on a new device/session. ([\#15388](https://github.com/matrix-org/synapse/issues/15388))
- Support resolving a room's [canonical alias](https://spec.matrix.org/v1.7/client-server-api/#mroomcanonical_alias) via the module API. ([\#15450](https://github.com/matrix-org/synapse/issues/15450))
- Enable support for [MSC3952](https://github.com/matrix-org/matrix-spec-proposals/pull/3952): intentional mentions. ([\#15520](https://github.com/matrix-org/synapse/issues/15520))
- Experimental [MSC3861](https://github.com/matrix-org/matrix-spec-proposals/pull/3861) support: delegate auth to an OIDC provider. ([\#15582](https://github.com/matrix-org/synapse/issues/15582))
- Add Synapse version deploy annotations to Grafana dashboard which enables easy correlation between behavior changes witnessed in a graph to a certain Synapse version and nail down regressions. ([\#15674](https://github.com/matrix-org/synapse/issues/15674))
- Add a catch-all * to the supported relation types when redacting an event and its related events. This is an update to [MSC3912](https://github.com/matrix-org/matrix-spec-proposals/pull/3861) implementation. ([\#15705](https://github.com/matrix-org/synapse/issues/15705))
- Speed up `/messages` by backfilling in the background when there are no backward extremities where we are directly paginating. ([\#15710](https://github.com/matrix-org/synapse/issues/15710))
- Expose a metric reporting the database background update status. ([\#15740](https://github.com/matrix-org/synapse/issues/15740))
-
-
-Bugfixes
--------
-
- Correctly clear caches when we delete a room. ([\#15609](https://github.com/matrix-org/synapse/issues/15609))
- Check permissions for enabling encryption earlier during room creation to avoid creating broken rooms. ([\#15695](https://github.com/matrix-org/synapse/issues/15695))
-
-
-Improved Documentation
----------------------
-
- Simplify query to find participating servers in a room. ([\#15732](https://github.com/matrix-org/synapse/issues/15732))
-
-
-Internal Changes
----------------
-
- Log when events are (maybe unexpectedly) filtered out of responses in tests. ([\#14213](https://github.com/matrix-org/synapse/issues/14213))
- Read from column `full_user_id` rather than `user_id` of tables `profiles` and `user_filters`. ([\#15649](https://github.com/matrix-org/synapse/issues/15649))
- Add support for tracing functions which return `Awaitable`s. ([\#15650](https://github.com/matrix-org/synapse/issues/15650))
- Cache requests for user's devices over federation. ([\#15675](https://github.com/matrix-org/synapse/issues/15675))
- Add fully qualified docker image names to Dockerfiles. ([\#15689](https://github.com/matrix-org/synapse/issues/15689))
- Remove some unused code. ([\#15690](https://github.com/matrix-org/synapse/issues/15690))
- Improve type hints. ([\#15694](https://github.com/matrix-org/synapse/issues/15694), [\#15697](https://github.com/matrix-org/synapse/issues/15697))
- Update docstring and traces on `maybe_backfill()` functions. ([\#15709](https://github.com/matrix-org/synapse/issues/15709))
- Add context for when/why to use the `long_retries` option when sending Federation requests. ([\#15721](https://github.com/matrix-org/synapse/issues/15721))
- Removed some unused fields. ([\#15723](https://github.com/matrix-org/synapse/issues/15723))
- Update federation error to more plainly explain we can only authorize our own membership events. ([\#15725](https://github.com/matrix-org/synapse/issues/15725))
- Prevent the `latest_deps` and `twisted_trunk` daily GitHub Actions workflows from running on forks of the codebase. ([\#15726](https://github.com/matrix-org/synapse/issues/15726))
- Improve performance of user directory search. ([\#15729](https://github.com/matrix-org/synapse/issues/15729))
- Remove redundant table join with `room_memberships` when doing a `is_host_joined()`/`is_host_invited()` call (`membership` is already part of the `current_state_events`). ([\#15731](https://github.com/matrix-org/synapse/issues/15731))
- Remove superfluous `room_memberships` join from background update. ([\#15733](https://github.com/matrix-org/synapse/issues/15733))
- Speed up typechecking CI. ([\#15752](https://github.com/matrix-org/synapse/issues/15752))
- Bump minimum supported Rust version to 1.60.0. ([\#15768](https://github.com/matrix-org/synapse/issues/15768))
-
-### Updates to locked dependencies
-
-* Bump importlib-metadata from 6.1.0 to 6.6.0. ([\#15711](https://github.com/matrix-org/synapse/issues/15711))
-* Bump library/redis from 6-bullseye to 7-bullseye in /docker. ([\#15712](https://github.com/matrix-org/synapse/issues/15712))
-* Bump log from 0.4.18 to 0.4.19. ([\#15761](https://github.com/matrix-org/synapse/issues/15761))
-* Bump phonenumbers from 8.13.11 to 8.13.13. ([\#15763](https://github.com/matrix-org/synapse/issues/15763))
-* Bump pyasn1 from 0.4.8 to 0.5.0. ([\#15713](https://github.com/matrix-org/synapse/issues/15713))
-* Bump pydantic from 1.10.8 to 1.10.9. ([\#15762](https://github.com/matrix-org/synapse/issues/15762))
-* Bump pyo3-log from 0.8.1 to 0.8.2. ([\#15759](https://github.com/matrix-org/synapse/issues/15759))
-* Bump pyopenssl from 23.1.1 to 23.2.0. ([\#15765](https://github.com/matrix-org/synapse/issues/15765))
-* Bump regex from 1.7.3 to 1.8.4. ([\#15769](https://github.com/matrix-org/synapse/issues/15769))
-* Bump sentry-sdk from 1.22.1 to 1.25.0. ([\#15714](https://github.com/matrix-org/synapse/issues/15714))
-* Bump sentry-sdk from 1.25.0 to 1.25.1. ([\#15764](https://github.com/matrix-org/synapse/issues/15764))
-* Bump serde from 1.0.163 to 1.0.164. ([\#15760](https://github.com/matrix-org/synapse/issues/15760))
-* Bump types-jsonschema from 4.17.0.7 to 4.17.0.8. ([\#15716](https://github.com/matrix-org/synapse/issues/15716))
-* Bump types-pyopenssl from 23.1.0.2 to 23.2.0.0. ([\#15766](https://github.com/matrix-org/synapse/issues/15766))
-* Bump types-requests from 2.31.0.0 to 2.31.0.1. ([\#15715](https://github.com/matrix-org/synapse/issues/15715))
-
-Synapse 1.85.2 (2023-06-08)
-===========================
-
-Bugfixes
--------
-
- Fix regression where using TLS for HTTP replication between workers did not work. Introduced in v1.85.0. ([\#15746](https://github.com/matrix-org/synapse/issues/15746))
-
-
-Synapse 1.85.1 (2023-06-07)
-===========================
-
-Note: this release only fixes a bug that stopped some deployments from upgrading to v1.85.0. There is no need to upgrade to v1.85.1 if successfully running v1.85.0.
-
-Bugfixes
--------
-
- Fix bug in schema delta that broke upgrades for some deployments. Introduced in v1.85.0. ([\#15738](https://github.com/matrix-org/synapse/issues/15738), [\#15739](https://github.com/matrix-org/synapse/issues/15739))
-
-
-Synapse 1.85.0 (2023-06-06)
-===========================
-
-No significant changes since 1.85.0rc2.
-
-
-## Security advisory
-
-The following issues are fixed in 1.85.0 (and RCs).
-
- [GHSA-26c5-ppr8-f33p](https://github.com/matrix-org/synapse/security/advisories/GHSA-26c5-ppr8-f33p) / [CVE-2023-32682](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2023-32682) — Low Severity
-
-  It may be possible for a deactivated user to login when using uncommon configurations.
-
- [GHSA-98px-6486-j7qc](https://github.com/matrix-org/synapse/security/advisories/GHSA-98px-6486-j7qc) / [CVE-2023-32683](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2023-32683) — Low Severity
-
-  A discovered oEmbed or image URL can bypass the `url_preview_url_blacklist` setting potentially allowing server side request forgery or bypassing network policies. Impact is limited to IP addresses allowed by the `url_preview_ip_range_blacklist` setting (by default this only allows public IPs).
-
-See the advisories for more details. If you have any questions, email security@matrix.org.
-
-
 Synapse 1.85.0rc2 (2023-06-01)
 ==============================

@@ -4,9 +4,9 @@ version = 3

 [[package]]
 name = "aho-corasick"
-version = "1.0.2"
+version = "0.7.19"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "43f6cb1bf222025340178f382c426f13757b2960e89779dfcb319c32542a5a41"
+checksum = "b4f55bd91a0978cbfd91c457a164bab8b4001c833b7f323132c0a4e1922dd44e"
 dependencies = [
 "memchr",
 ]
@@ -132,9 +132,9 @@ dependencies = [

 [[package]]
 name = "log"
-version = "0.4.19"
+version = "0.4.18"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b06a4cde4c0f271a446782e3eff8de789548ce57dbc8eca9292c27f4a42004b4"
+checksum = "518ef76f2f87365916b142844c16d8fefd85039bc5699050210a7778ee1cd1de"

 [[package]]
 name = "memchr"
@@ -229,9 +229,9 @@ dependencies = [

 [[package]]
 name = "pyo3-log"
-version = "0.8.2"
+version = "0.8.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c94ff6535a6bae58d7d0b85e60d4c53f7f84d0d0aa35d6a28c3f3e70bfe51444"
+checksum = "f9c8b57fe71fb5dcf38970ebedc2b1531cf1c14b1b9b4c560a182a57e115575c"
 dependencies = [
 "arc-swap",
 "log",
@@ -291,9 +291,9 @@ dependencies = [

 [[package]]
 name = "regex"
-version = "1.8.4"
+version = "1.7.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d0ab3ca65655bb1e41f2a8c8cd662eb4fb035e67c3f78da1d61dffe89d07300f"
+checksum = "8b1f693b24f6ac912f4893ef08244d70b6067480d2f1a46e950c9691e6749d1d"
 dependencies = [
 "aho-corasick",
 "memchr",
@@ -302,9 +302,9 @@ dependencies = [

 [[package]]
 name = "regex-syntax"
-version = "0.7.2"
+version = "0.6.29"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "436b050e76ed2903236f032a59761c1eb99e1b0aead2c257922771dab1fc8c78"
+checksum = "f162c6dd7b008981e4d40210aca20b4bd0f9b60ca9271061b07f78537722f2e1"

 [[package]]
 name = "ryu"
@@ -320,18 +320,18 @@ checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd"

 [[package]]
 name = "serde"
-version = "1.0.164"
+version = "1.0.163"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9e8c8cf938e98f769bc164923b06dce91cea1751522f46f8466461af04c9027d"
+checksum = "2113ab51b87a539ae008b5c6c02dc020ffa39afd2d83cffcb3f4eb2722cebec2"
 dependencies = [
 "serde_derive",
 ]

 [[package]]
 name = "serde_derive"
-version = "1.0.164"
+version = "1.0.163"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d9735b638ccc51c28bf6914d90a2e9725b377144fc612c49a611fddd1b631d68"
+checksum = "8c805777e3930c8883389c602315a24224bcc738b63905ef87cd1420353ea93e"
 dependencies = [
 "proc-macro2",
 "quote",
@@ -340,9 +340,9 @@ dependencies = [

 [[package]]
 name = "serde_json"
-version = "1.0.99"
+version = "1.0.96"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "46266871c240a00b8f503b877622fe33430b3c7d963bdc0f2adc511e54a1eae3"
+checksum = "057d394a50403bcac12672b2b18fb387ab6d289d957dab67dd201875391e52f1"
 dependencies = [
 "itoa",
 "ryu",
@@ -0,0 +1 @@
+Log when events are (maybe unexpectedly) filtered out of responses in tests.
@@ -1 +0,0 @@
-Replace `EventContext` fields `prev_group` and `delta_ids` with field `state_group_deltas`.
@@ -0,0 +1 @@
+Stable support for [MSC3882](https://github.com/matrix-org/matrix-spec-proposals/pull/3882) to allow an existing device/session to generate a login token for use on a new device/session.
@@ -0,0 +1 @@
+Support resolving a room's [canonical alias](https://spec.matrix.org/v1.7/client-server-api/#mroomcanonical_alias) via the module API.
@@ -0,0 +1 @@
+Experimental [MSC3861](https://github.com/matrix-org/matrix-spec-proposals/pull/3861) support: delegate auth to an OIDC provider.
@@ -0,0 +1 @@
+Read from column `full_user_id` rather than `user_id` of tables `profiles` and `user_filters`.
@@ -0,0 +1 @@
+Add Syanpse version deploy annotations to Grafana dashboard which enables easy correlation between behavior changes witnessed in a graph to a certain Synapse version and nail down regressions.
@@ -0,0 +1 @@
+Cache requests for user's devices over federation.
@@ -1 +0,0 @@
-Fix a long-standing bug where media files were served in an unsafe manner. Contributed by @joshqou.
@@ -0,0 +1 @@
+Add fully qualified docker image names to Dockerfiles.
@@ -0,0 +1 @@
+Remove some unused code.
@@ -0,0 +1 @@
+Improve type hints.
@@ -0,0 +1 @@
+Improve type hints.
@@ -0,0 +1 @@
+Add a catch-all * to the supported relation types when redacting an event and its related events. This is an update to [MSC3912](https://github.com/matrix-org/matrix-spec-proposals/pull/3861) implementation.
@@ -0,0 +1 @@
+Enabled login_via_existing_session by default.
@@ -1 +0,0 @@
-Improve `/messages` response time by avoiding backfill when we already have messages to return.
@@ -1 +0,0 @@
-Regularly try to send transactions to other servers after they failed instead of waiting for a new event to be available before trying.
@@ -1 +0,0 @@
-Remove experimental [MSC2716](https://github.com/matrix-org/matrix-spec-proposals/pull/2716) implementation to incrementally import history into existing rooms.
@@ -1 +0,0 @@
-Fix requesting multiple keys at once over federation, related to [MSC3983](https://github.com/matrix-org/matrix-spec-proposals/pull/3983).
@@ -1 +0,0 @@
-Avoid invalidating a cache that was just prefilled.
@@ -1 +0,0 @@
-Fix requesting multiple keys at once over federation, related to [MSC3983](https://github.com/matrix-org/matrix-spec-proposals/pull/3983).
@@ -1 +0,0 @@
-Document `looping_call()` functionality that will wait for the given function to finish before scheduling another.
@@ -1 +0,0 @@
-Fix joining rooms through aliases where the alias server isn't a real homeserver. Contributed by @tulir @ Beeper.
@@ -1 +0,0 @@
-Fix a bug in push rules handling leading to an invalid (per spec) `is_user_mention` rule sent to clients. Also fix wrong rule names for `is_user_mention` and `is_room_mention`.
@@ -1 +0,0 @@
-Allow for the configuration of max request retries and min/max retry delays in the matrix federation client.
@@ -1 +0,0 @@
-Fix a bug introduced in 1.57.0 where the wrong table would be locked on updating database rows when using SQLite as the database backend.
@@ -1 +0,0 @@
-Fix Sytest environmental variable evaluation in CI.
@@ -1 +0,0 @@
-Fix a typo in the [Admin API](https://matrix-org.github.io/synapse/latest/usage/administration/admin_api/index.html).
@@ -1 +0,0 @@
-Switch from `matrix://` to `matrix-federation://` scheme for internal Synapse routing of outbound federation traffic.
@@ -1 +0,0 @@
-Fix typo in MSC number in faster remote room join architecture doc.
@@ -1 +0,0 @@
-Fix harmless exceptions being printed when running the port DB script.
@@ -1 +0,0 @@
-Fix forgotten rooms missing from initial sync after rejoining them. Contributed by Nico from Famedly.
@@ -1 +0,0 @@
-Add spam checker module API for logins.
@@ -1 +0,0 @@
-Implements bullets 1 and 2 of [MSC 3944](https://github.com/matrix-org/matrix-spec-proposals/pull/3944) related to dropping cancelled and duplicated `m.room_key_request`.
@@ -29,7 +29,7 @@
        "level": "error"
      },
      {
-        "line": "my-matrix-server-federation-sender-1 | 2023-01-25 20:56:20,995 - synapse.http.matrixfederationclient - 709 - WARNING - federation_transaction_transmission_loop-3 - {PUT-O-3} [example.com] Request failed: PUT matrix-federation://example.com/_matrix/federation/v1/send/1674680155797: HttpResponseException('403: Forbidden')",
+        "line": "my-matrix-server-federation-sender-1 | 2023-01-25 20:56:20,995 - synapse.http.matrixfederationclient - 709 - WARNING - federation_transaction_transmission_loop-3 - {PUT-O-3} [example.com] Request failed: PUT matrix://example.com/_matrix/federation/v1/send/1674680155797: HttpResponseException('403: Forbidden')",
        "level": "warning"
      },
      {
@@ -1,39 +1,3 @@
-matrix-synapse-py3 (1.86.0) stable; urgency=medium
-
-  * New Synapse release 1.86.0.
-
- -- Synapse Packaging team <packages@matrix.org>  Tue, 20 Jun 2023 17:22:46 +0200
-
-matrix-synapse-py3 (1.86.0~rc2) stable; urgency=medium
-
-  * New Synapse release 1.86.0rc2.
-
- -- Synapse Packaging team <packages@matrix.org>  Wed, 14 Jun 2023 12:16:27 +0200
-
-matrix-synapse-py3 (1.86.0~rc1) stable; urgency=medium
-
-  * New Synapse release 1.86.0rc1.
-
- -- Synapse Packaging team <packages@matrix.org>  Tue, 13 Jun 2023 14:30:45 +0200
-
-matrix-synapse-py3 (1.85.2) stable; urgency=medium
-
-  * New Synapse release 1.85.2.
-
- -- Synapse Packaging team <packages@matrix.org>  Thu, 08 Jun 2023 13:04:18 +0100
-
-matrix-synapse-py3 (1.85.1) stable; urgency=medium
-
-  * New Synapse release 1.85.1.
-
- -- Synapse Packaging team <packages@matrix.org>  Wed, 07 Jun 2023 10:51:12 +0100
-
-matrix-synapse-py3 (1.85.0) stable; urgency=medium
-
-  * New Synapse release 1.85.0.
-
- -- Synapse Packaging team <packages@matrix.org>  Tue, 06 Jun 2023 09:39:29 +0100
-
 matrix-synapse-py3 (1.85.0~rc2) stable; urgency=medium

  * New Synapse release 1.85.0rc2.
@@ -92,6 +92,8 @@ allow_device_name_lookup_over_federation: true
 ## Experimental Features ##

 experimental_features:
+  # Enable history backfilling support
+  msc2716_enabled: true
  # client-side support for partial state in /send_join responses
  faster_joins: true
  # Enable support for polls
@@ -244,6 +244,7 @@ WORKERS_CONFIG: Dict[str, Dict[str, Any]] = {
            "^/_matrix/client/(api/v1|r0|v3|unstable)/join/",
            "^/_matrix/client/(api/v1|r0|v3|unstable)/knock/",
            "^/_matrix/client/(api/v1|r0|v3|unstable)/profile/",
+            "^/_matrix/client/(v1|unstable/org.matrix.msc2716)/rooms/.*/batch_send",
        ],
        "shared_extra_conf": {},
        "worker_extra_conf": "",
@@ -419,7 +419,7 @@ The following query parameters are available:

 * `from` (required) - The token to start returning events from. This token can be obtained from a prev_batch
  or next_batch token returned by the /sync endpoint, or from an end token returned by a previous request to this endpoint.
-* `to` - The token to stop returning events at.
+* `to` - The token to spot returning events at.
 * `limit` - The maximum number of events to return. Defaults to `10`.
 * `filter` - A JSON RoomEventFilter to filter returned events with.
 * `dir` - The direction to return events from. Either `f` for forwards or `b` for backwards. Setting
@@ -6,7 +6,7 @@ This is a work-in-progress set of notes with two goals:

 See also [MSC3902](https://github.com/matrix-org/matrix-spec-proposals/pull/3902).

-The key idea is described by [MSC3706](https://github.com/matrix-org/matrix-spec-proposals/pull/3706). This allows servers to
+The key idea is described by [MSC706](https://github.com/matrix-org/matrix-spec-proposals/pull/3902). This allows servers to
 request a lightweight response to the federation `/send_join` endpoint.
 This is called a **faster join**, also known as a **partial join**. In these
 notes we'll usually use the word "partial" as it matches the database schema.
@@ -348,42 +348,6 @@ callback returns `False`, Synapse falls through to the next one. The value of th
 callback that does not return `False` will be used. If this happens, Synapse will not call
 any of the subsequent implementations of this callback.

-
-### `check_login_for_spam`
-
-_First introduced in Synapse v1.87.0_
-
-```python
-async def check_login_for_spam(
-    user_id: str,
-    device_id: Optional[str],
-    initial_display_name: Optional[str],
-    request_info: Collection[Tuple[Optional[str], str]],
-    auth_provider_id: Optional[str] = None,
-) -> Union["synapse.module_api.NOT_SPAM", "synapse.module_api.errors.Codes"]
-```
-
-Called when a user logs in.
-
-The arguments passed to this callback are:
-
-* `user_id`: The user ID the user is logging in with
-* `device_id`: The device ID the user is re-logging into.
-* `initial_display_name`: The device display name, if any.
-* `request_info`: A collection of tuples, which first item is a user agent, and which
-  second item is an IP address. These user agents and IP addresses are the ones that were
-  used during the login process.
-* `auth_provider_id`: The identifier of the SSO authentication provider, if any.
-
-If multiple modules implement this callback, they will be considered in order. If a
-callback returns `synapse.module_api.NOT_SPAM`, Synapse falls through to the next one.
-The value of the first callback that does not return `synapse.module_api.NOT_SPAM` will
-be used. If this happens, Synapse will not call any of the subsequent implementations of
-this callback.
-
-*Note:* This will not be called when a user registers.
-
-
 ## Example

 The example below is a module that implements the spam checker callback
@@ -87,14 +87,6 @@ process, for example:
    wget https://packages.matrix.org/debian/pool/main/m/matrix-synapse-py3/matrix-synapse-py3_1.3.0+stretch1_amd64.deb
    dpkg -i matrix-synapse-py3_1.3.0+stretch1_amd64.deb
    ```
-# Upgrading to v1.86.0
-
-## Minimum supported Rust version
-
-The minimum supported Rust version has been increased from v1.58.1 to v1.60.0.
-Users building from source will need to ensure their `rustc` version is up to
-date.
-

 # Upgrading to v1.85.0

@@ -27,8 +27,9 @@ What servers are currently participating in this room?
 Run this sql query on your db:
 ```sql
 SELECT DISTINCT split_part(state_key, ':', 2)
-FROM current_state_events
-WHERE room_id = '!cURbafjkfsMDVwdRDQ:matrix.org' AND membership = 'join';
+    FROM current_state_events AS c
+    INNER JOIN room_memberships AS m USING (room_id, event_id)
+    WHERE room_id = '!cURbafjkfsMDVwdRDQ:matrix.org' AND membership = 'join';
 ```

 What users are registered on my server?
@@ -1196,32 +1196,6 @@ Example configuration:
 allow_device_name_lookup_over_federation: true
 ```
 ---
-### `federation`
-
-The federation section defines some sub-options related to federation.
-
-The following options are related to configuring timeout and retry logic for one request,
-independently of the others.
-Short retry algorithm is used when something or someone will wait for the request to have an
-answer, while long retry is used for requests that happen in the background,
-like sending a federation transaction.
-
-* `client_timeout`: timeout for the federation requests. Default to 60s.
-* `max_short_retry_delay`: maximum delay to be used for the short retry algo. Default to 2s.
-* `max_long_retry_delay`: maximum delay to be used for the short retry algo. Default to 60s.
-* `max_short_retries`: maximum number of retries for the short retry algo. Default to 3 attempts.
-* `max_long_retries`: maximum number of retries for the long retry algo. Default to 10 attempts.
-
-Example configuration:
-```yaml
-federation:
-  client_timeout: 180s
-  max_short_retry_delay: 7s
-  max_long_retry_delay: 100s
-  max_short_retries: 5
-  max_long_retries: 20
-```
---
 ## Caching

 Options related to caching.
@@ -2624,14 +2598,11 @@ ui_auth:
 Matrix supports the ability of an existing session to mint a login token for
 another client.

-Synapse disables this by default as it has security ramifications -- a malicious
-client could use the mechanism to spawn more than one session.
-
 The duration of time the generated token is valid for can be configured with the
 `token_timeout` sub-option.

-User-interactive authentication is required when this is enabled unless the
-`require_ui_auth` sub-option is set to `False`.
+To protect against malicious clients abusing this capability, user-interactive authentication
+is required unless the `require_ui_auth` sub-option is set to `False`.

 Example configuration:
 ```yaml
@@ -232,6 +232,7 @@ information.
    ^/_matrix/client/v1/rooms/.*/hierarchy$
    ^/_matrix/client/(v1|unstable)/rooms/.*/relations/
    ^/_matrix/client/v1/rooms/.*/threads$
+    ^/_matrix/client/unstable/org.matrix.msc2716/rooms/.*/batch_send$
    ^/_matrix/client/unstable/im.nheko.summary/rooms/.*/summary$
    ^/_matrix/client/(r0|v3|unstable)/account/3pid$
    ^/_matrix/client/(r0|v3|unstable)/account/whoami$
@@ -89,7 +89,7 @@ manifest-path = "rust/Cargo.toml"

 [tool.poetry]
 name = "matrix-synapse"
-version = "1.86.0"
+version = "1.85.0rc2"
 description = "Homeserver for the Matrix decentralised comms protocol"
 authors = ["Matrix.org Team and Contributors <packages@matrix.org>"]
 license = "Apache-2.0"
@@ -311,7 +311,7 @@ all = [
 # We pin black so that our tests don't start failing on new releases.
 isort = ">=5.10.1"
 black = ">=22.3.0"
-ruff = "0.0.275"
+ruff = "0.0.265"

 # Typechecking
 lxml-stubs = ">=0.4.0"
@@ -7,7 +7,7 @@ name = "synapse"
 version = "0.1.0"

 edition = "2021"
-rust-version = "1.60.0"
+rust-version = "1.58.1"

 [lib]
 name = "synapse"
@@ -13,6 +13,8 @@
 // limitations under the License.

 #![feature(test)]
+use std::collections::BTreeSet;
+
 use synapse::push::{
    evaluator::PushRuleEvaluator, Condition, EventMatchCondition, FilteredPushRules, JsonValue,
    PushRules, SimpleJsonValue,
@@ -195,6 +197,7 @@ fn bench_eval_message(b: &mut Bencher) {
        false,
        false,
        false,
+        false,
    );

    b.iter(|| eval.run(&rules, Some("bob"), Some("person")));
@@ -142,11 +142,11 @@ pub const BASE_APPEND_OVERRIDE_RULES: &[PushRule] = &[
        default_enabled: true,
    },
    PushRule {
-        rule_id: Cow::Borrowed("global/override/.m.rule.is_user_mention"),
+        rule_id: Cow::Borrowed(".org.matrix.msc3952.is_user_mention"),
        priority_class: 5,
        conditions: Cow::Borrowed(&[Condition::Known(
            KnownCondition::ExactEventPropertyContainsType(EventPropertyIsTypeCondition {
-                key: Cow::Borrowed("content.m\\.mentions.user_ids"),
+                key: Cow::Borrowed("content.org\\.matrix\\.msc3952\\.mentions.user_ids"),
                value_type: Cow::Borrowed(&EventMatchPatternType::UserId),
            }),
        )]),
@@ -163,11 +163,11 @@ pub const BASE_APPEND_OVERRIDE_RULES: &[PushRule] = &[
        default_enabled: true,
    },
    PushRule {
-        rule_id: Cow::Borrowed("global/override/.m.rule.is_room_mention"),
+        rule_id: Cow::Borrowed(".org.matrix.msc3952.is_room_mention"),
        priority_class: 5,
        conditions: Cow::Borrowed(&[
            Condition::Known(KnownCondition::EventPropertyIs(EventPropertyIsCondition {
-                key: Cow::Borrowed("content.m\\.mentions.room"),
+                key: Cow::Borrowed("content.org\\.matrix\\.msc3952\\.mentions.room"),
                value: Cow::Borrowed(&SimpleJsonValue::Bool(true)),
            })),
            Condition::Known(KnownCondition::SenderNotificationPermission {
@@ -70,9 +70,7 @@ pub struct PushRuleEvaluator {
    /// The "content.body", if any.
    body: String,

-    /// True if the event has a m.mentions property. (Note that this is a separate
-    /// flag instead of checking flattened_keys since the m.mentions property
-    /// might be an empty map and not appear in flattened_keys.
+    /// True if the event has a mentions property and MSC3952 support is enabled.
    has_mentions: bool,

    /// The number of users in the room.
@@ -157,7 +155,9 @@ impl PushRuleEvaluator {
            let rule_id = &push_rule.rule_id().to_string();

            // For backwards-compatibility the legacy mention rules are disabled
-            // if the event contains the 'm.mentions' property.
+            // if the event contains the 'm.mentions' property (and if the
+            // experimental feature is enabled, both of these are represented
+            // by the has_mentions flag).
            if self.has_mentions
                && (rule_id == "global/override/.m.rule.contains_display_name"
                    || rule_id == "global/content/.m.rule.contains_user_name"
@@ -562,7 +562,7 @@ fn test_requires_room_version_supports_condition() {
    };
    let rules = PushRules::new(vec![custom_rule]);
    result = evaluator.run(
-        &FilteredPushRules::py_new(rules, BTreeMap::new(), true, false, true, false),
+        &FilteredPushRules::py_new(rules, BTreeMap::new(), true, false, true, false, false),
        None,
        None,
    );
@@ -527,6 +527,7 @@ pub struct FilteredPushRules {
    msc1767_enabled: bool,
    msc3381_polls_enabled: bool,
    msc3664_enabled: bool,
+    msc3952_intentional_mentions: bool,
    msc3958_suppress_edits_enabled: bool,
 }

@@ -539,6 +540,7 @@ impl FilteredPushRules {
        msc1767_enabled: bool,
        msc3381_polls_enabled: bool,
        msc3664_enabled: bool,
+        msc3952_intentional_mentions: bool,
        msc3958_suppress_edits_enabled: bool,
    ) -> Self {
        Self {
@@ -547,6 +549,7 @@ impl FilteredPushRules {
            msc1767_enabled,
            msc3381_polls_enabled,
            msc3664_enabled,
+            msc3952_intentional_mentions,
            msc3958_suppress_edits_enabled,
        }
    }
@@ -584,6 +587,10 @@ impl FilteredPushRules {
                    return false;
                }

+                if !self.msc3952_intentional_mentions && rule.rule_id.contains("org.matrix.msc3952")
+                {
+                    return false;
+                }
                if !self.msc3958_suppress_edits_enabled
                    && rule.rule_id == "global/override/.com.beeper.suppress_edits"
                {
@@ -246,6 +246,10 @@ else
  else
    export PASS_SYNAPSE_COMPLEMENT_DATABASE=sqlite
  fi
+
+  # The tests for importing historical messages (MSC2716)
+  # only pass with monoliths, currently.
+  test_tags="$test_tags,msc2716"
 fi

 if [[ -n "$ASYNCIO_REACTOR" ]]; then
@@ -136,11 +136,11 @@ def request(
        authorization_headers.append(header)
        print("Authorization: %s" % header, file=sys.stderr)

-    dest = "matrix-federation://%s%s" % (destination, path)
+    dest = "matrix://%s%s" % (destination, path)
    print("Requesting %s" % dest, file=sys.stderr)

    s = requests.Session()
-    s.mount("matrix-federation://", MatrixConnectionAdapter())
+    s.mount("matrix://", MatrixConnectionAdapter())

    headers: Dict[str, str] = {
        "Authorization": authorization_headers[0],
@@ -46,6 +46,7 @@ class FilteredPushRules:
        msc1767_enabled: bool,
        msc3381_polls_enabled: bool,
        msc3664_enabled: bool,
+        msc3952_intentional_mentions: bool,
        msc3958_suppress_edits_enabled: bool,
    ): ...
    def rules(self) -> Collection[Tuple[PushRule, bool]]: ...
@@ -1369,9 +1369,6 @@ def main() -> None:
        sys.stderr.write("Database must use the 'psycopg2' connector.\n")
        sys.exit(3)

-    # Don't run the background tasks that get started by the data stores.
-    hs_config["run_background_tasks_on"] = "some_other_process"
-
    config = HomeServerConfig()
    config.parse_config_dict(hs_config, "", "")

@@ -123,6 +123,10 @@ class EventTypes:
    SpaceChild: Final = "m.space.child"
    SpaceParent: Final = "m.space.parent"

+    MSC2716_INSERTION: Final = "org.matrix.msc2716.insertion"
+    MSC2716_BATCH: Final = "org.matrix.msc2716.batch"
+    MSC2716_MARKER: Final = "org.matrix.msc2716.marker"
+
    Reaction: Final = "m.reaction"


@@ -218,11 +222,21 @@ class EventContentFields:
    # Used in m.room.guest_access events.
    GUEST_ACCESS: Final = "guest_access"

+    # Used on normal messages to indicate they were historically imported after the fact
+    MSC2716_HISTORICAL: Final = "org.matrix.msc2716.historical"
+    # For "insertion" events to indicate what the next batch ID should be in
+    # order to connect to it
+    MSC2716_NEXT_BATCH_ID: Final = "next_batch_id"
+    # Used on "batch" events to indicate which insertion event it connects to
+    MSC2716_BATCH_ID: Final = "batch_id"
+    # For "marker" events
+    MSC2716_INSERTION_EVENT_REFERENCE: Final = "insertion_event_reference"
+
    # The authorising user for joining a restricted room.
    AUTHORISING_USER: Final = "join_authorised_via_users_server"

    # Use for mentioning users.
-    MENTIONS: Final = "m.mentions"
+    MSC3952_MENTIONS: Final = "org.matrix.msc3952.mentions"

    # an unspecced field added to to-device messages to identify them uniquely-ish
    TO_DEVICE_MSGID: Final = "org.matrix.msgid"
@@ -91,6 +91,11 @@ class RoomVersion:
    # MSC2403: Allows join_rules to be set to 'knock', changes auth rules to allow sending
    # m.room.membership event with membership 'knock'.
    msc2403_knocking: bool
+    # MSC2716: Adds m.room.power_levels -> content.historical field to control
+    # whether "insertion", "chunk", "marker" events can be sent
+    msc2716_historical: bool
+    # MSC2716: Adds support for redacting "insertion", "chunk", and "marker" events
+    msc2716_redactions: bool
    # MSC3389: Protect relation information from redaction.
    msc3389_relation_redactions: bool
    # MSC3787: Adds support for a `knock_restricted` join rule, mixing concepts of
@@ -125,6 +130,8 @@ class RoomVersions:
        msc3083_join_rules=False,
        msc3375_redaction_rules=False,
        msc2403_knocking=False,
+        msc2716_historical=False,
+        msc2716_redactions=False,
        msc3389_relation_redactions=False,
        msc3787_knock_restricted_join_rule=False,
        msc3667_int_only_power_levels=False,
@@ -146,6 +153,8 @@ class RoomVersions:
        msc3083_join_rules=False,
        msc3375_redaction_rules=False,
        msc2403_knocking=False,
+        msc2716_historical=False,
+        msc2716_redactions=False,
        msc3389_relation_redactions=False,
        msc3787_knock_restricted_join_rule=False,
        msc3667_int_only_power_levels=False,
@@ -167,6 +176,8 @@ class RoomVersions:
        msc3083_join_rules=False,
        msc3375_redaction_rules=False,
        msc2403_knocking=False,
+        msc2716_historical=False,
+        msc2716_redactions=False,
        msc3389_relation_redactions=False,
        msc3787_knock_restricted_join_rule=False,
        msc3667_int_only_power_levels=False,
@@ -188,6 +199,8 @@ class RoomVersions:
        msc3083_join_rules=False,
        msc3375_redaction_rules=False,
        msc2403_knocking=False,
+        msc2716_historical=False,
+        msc2716_redactions=False,
        msc3389_relation_redactions=False,
        msc3787_knock_restricted_join_rule=False,
        msc3667_int_only_power_levels=False,
@@ -209,6 +222,8 @@ class RoomVersions:
        msc3083_join_rules=False,
        msc3375_redaction_rules=False,
        msc2403_knocking=False,
+        msc2716_historical=False,
+        msc2716_redactions=False,
        msc3389_relation_redactions=False,
        msc3787_knock_restricted_join_rule=False,
        msc3667_int_only_power_levels=False,
@@ -230,6 +245,8 @@ class RoomVersions:
        msc3083_join_rules=False,
        msc3375_redaction_rules=False,
        msc2403_knocking=False,
+        msc2716_historical=False,
+        msc2716_redactions=False,
        msc3389_relation_redactions=False,
        msc3787_knock_restricted_join_rule=False,
        msc3667_int_only_power_levels=False,
@@ -251,6 +268,8 @@ class RoomVersions:
        msc3083_join_rules=False,
        msc3375_redaction_rules=False,
        msc2403_knocking=False,
+        msc2716_historical=False,
+        msc2716_redactions=False,
        msc3389_relation_redactions=False,
        msc3787_knock_restricted_join_rule=False,
        msc3667_int_only_power_levels=False,
@@ -272,6 +291,8 @@ class RoomVersions:
        msc3083_join_rules=False,
        msc3375_redaction_rules=False,
        msc2403_knocking=True,
+        msc2716_historical=False,
+        msc2716_redactions=False,
        msc3389_relation_redactions=False,
        msc3787_knock_restricted_join_rule=False,
        msc3667_int_only_power_levels=False,
@@ -293,6 +314,8 @@ class RoomVersions:
        msc3083_join_rules=True,
        msc3375_redaction_rules=False,
        msc2403_knocking=True,
+        msc2716_historical=False,
+        msc2716_redactions=False,
        msc3389_relation_redactions=False,
        msc3787_knock_restricted_join_rule=False,
        msc3667_int_only_power_levels=False,
@@ -314,6 +337,8 @@ class RoomVersions:
        msc3083_join_rules=True,
        msc3375_redaction_rules=True,
        msc2403_knocking=True,
+        msc2716_historical=False,
+        msc2716_redactions=False,
        msc3389_relation_redactions=False,
        msc3787_knock_restricted_join_rule=False,
        msc3667_int_only_power_levels=False,
@@ -335,6 +360,8 @@ class RoomVersions:
        msc3083_join_rules=True,
        msc3375_redaction_rules=True,
        msc2403_knocking=True,
+        msc2716_historical=False,
+        msc2716_redactions=False,
        msc3389_relation_redactions=False,
        msc3787_knock_restricted_join_rule=True,
        msc3667_int_only_power_levels=False,
@@ -356,6 +383,8 @@ class RoomVersions:
        msc3083_join_rules=True,
        msc3375_redaction_rules=True,
        msc2403_knocking=True,
+        msc2716_historical=False,
+        msc2716_redactions=False,
        msc3389_relation_redactions=False,
        msc3787_knock_restricted_join_rule=False,
        msc3667_int_only_power_levels=False,
@@ -377,6 +406,8 @@ class RoomVersions:
        msc3083_join_rules=True,
        msc3375_redaction_rules=True,
        msc2403_knocking=True,
+        msc2716_historical=False,
+        msc2716_redactions=False,
        msc3389_relation_redactions=False,
        msc3787_knock_restricted_join_rule=True,
        msc3667_int_only_power_levels=True,
@@ -384,6 +415,29 @@ class RoomVersions:
        msc3931_push_features=(),
        msc3989_redaction_rules=False,
    )
+    MSC2716v4 = RoomVersion(
+        "org.matrix.msc2716v4",
+        RoomDisposition.UNSTABLE,
+        EventFormatVersions.ROOM_V4_PLUS,
+        StateResolutionVersions.V2,
+        enforce_key_validity=True,
+        special_case_aliases_auth=False,
+        strict_canonicaljson=True,
+        limit_notifications_power_levels=True,
+        msc2175_implicit_room_creator=False,
+        msc2176_redaction_rules=False,
+        msc3083_join_rules=False,
+        msc3375_redaction_rules=False,
+        msc2403_knocking=True,
+        msc2716_historical=True,
+        msc2716_redactions=True,
+        msc3389_relation_redactions=False,
+        msc3787_knock_restricted_join_rule=False,
+        msc3667_int_only_power_levels=False,
+        msc3821_redaction_rules=False,
+        msc3931_push_features=(),
+        msc3989_redaction_rules=False,
+    )
    MSC1767v10 = RoomVersion(
        # MSC1767 (Extensible Events) based on room version "10"
        "org.matrix.msc1767.10",
@@ -399,6 +453,8 @@ class RoomVersions:
        msc3083_join_rules=True,
        msc3375_redaction_rules=True,
        msc2403_knocking=True,
+        msc2716_historical=False,
+        msc2716_redactions=False,
        msc3389_relation_redactions=False,
        msc3787_knock_restricted_join_rule=True,
        msc3667_int_only_power_levels=True,
@@ -420,6 +476,8 @@ class RoomVersions:
        msc3083_join_rules=True,
        msc3375_redaction_rules=True,
        msc2403_knocking=True,
+        msc2716_historical=False,
+        msc2716_redactions=False,
        msc3389_relation_redactions=False,
        msc3787_knock_restricted_join_rule=True,
        msc3667_int_only_power_levels=True,
@@ -442,6 +500,8 @@ class RoomVersions:
        msc3083_join_rules=True,
        msc3375_redaction_rules=True,
        msc2403_knocking=True,
+        msc2716_historical=False,
+        msc2716_redactions=False,
        msc3389_relation_redactions=False,
        msc3787_knock_restricted_join_rule=True,
        msc3667_int_only_power_levels=True,
@@ -466,6 +526,7 @@ KNOWN_ROOM_VERSIONS: Dict[str, RoomVersion] = {
        RoomVersions.V9,
        RoomVersions.MSC3787,
        RoomVersions.V10,
+        RoomVersions.MSC2716v4,
        RoomVersions.MSC3989,
        RoomVersions.MSC3820opt2,
    )
@@ -83,6 +83,7 @@ from synapse.storage.databases.main.receipts import ReceiptsWorkerStore
 from synapse.storage.databases.main.registration import RegistrationWorkerStore
 from synapse.storage.databases.main.relations import RelationsWorkerStore
 from synapse.storage.databases.main.room import RoomWorkerStore
+from synapse.storage.databases.main.room_batch import RoomBatchStore
 from synapse.storage.databases.main.roommember import RoomMemberWorkerStore
 from synapse.storage.databases.main.search import SearchStore
 from synapse.storage.databases.main.session import SessionStore
@@ -119,6 +120,7 @@ class GenericWorkerStore(
    # the races it creates aren't too bad.
    KeyStore,
    RoomWorkerStore,
+    RoomBatchStore,
    DirectoryWorkerStore,
    PushRulesWorkerStore,
    ApplicationServiceTransactionWorkerStore,
@@ -63,7 +63,7 @@ class AuthConfig(Config):

        # Logging in with an existing session.
        login_via_existing = config.get("login_via_existing_session", {})
-        self.login_via_existing_enabled = login_via_existing.get("enabled", False)
+        self.login_via_existing_enabled = login_via_existing.get("enabled", True)
        self.login_via_existing_require_ui_auth = login_via_existing.get(
            "require_ui_auth", True
        )
@@ -247,6 +247,9 @@ class ExperimentalConfig(Config):
        # MSC3026 (busy presence state)
        self.msc3026_enabled: bool = experimental.get("msc3026_enabled", False)

+        # MSC2716 (importing historical messages)
+        self.msc2716_enabled: bool = experimental.get("msc2716_enabled", False)
+
        # MSC3244 (room version capabilities)
        self.msc3244_enabled: bool = experimental.get("msc3244_enabled", True)

@@ -355,6 +358,11 @@ class ExperimentalConfig(Config):
        # MSC3391: Removing account data.
        self.msc3391_enabled = experimental.get("msc3391_enabled", False)

+        # MSC3952: Intentional mentions, this depends on MSC3966.
+        self.msc3952_intentional_mentions = experimental.get(
+            "msc3952_intentional_mentions", False
+        )
+
        # MSC3959: Do not generate notifications for edits.
        self.msc3958_supress_edit_notifs = experimental.get(
            "msc3958_supress_edit_notifs", False
@@ -389,6 +397,3 @@ class ExperimentalConfig(Config):
        self.msc4010_push_rules_account_data = experimental.get(
            "msc4010_push_rules_account_data", False
        )
-
-        # MSC3944: Dropping stale send-to-device messages
-        self.msc3944_enabled: bool = experimental.get("msc3944_enabled", False)
@@ -22,8 +22,6 @@ class FederationConfig(Config):
    section = "federation"

    def read_config(self, config: JsonDict, **kwargs: Any) -> None:
-        federation_config = config.setdefault("federation", {})
-
        # FIXME: federation_domain_whitelist needs sytests
        self.federation_domain_whitelist: Optional[dict] = None
        federation_domain_whitelist = config.get("federation_domain_whitelist", None)
@@ -51,19 +49,5 @@ class FederationConfig(Config):
            "allow_device_name_lookup_over_federation", False
        )

-        # Allow for the configuration of timeout, max request retries
-        # and min/max retry delays in the matrix federation client.
-        self.client_timeout_ms = Config.parse_duration(
-            federation_config.get("client_timeout", "60s")
-        )
-        self.max_long_retry_delay_ms = Config.parse_duration(
-            federation_config.get("max_long_retry_delay", "60s")
-        )
-        self.max_short_retry_delay_ms = Config.parse_duration(
-            federation_config.get("max_short_retry_delay", "2s")
-        )
-        self.max_long_retries = federation_config.get("max_long_retries", 10)
-        self.max_short_retries = federation_config.get("max_short_retries", 3)
-

 _METRICS_FOR_DOMAINS_SCHEMA = {"type": "array", "items": {"type": "string"}}
@@ -339,6 +339,13 @@ def check_state_dependent_auth_rules(
    if event.type == EventTypes.Redaction:
        check_redaction(event.room_version, event, auth_dict)

+    if (
+        event.type == EventTypes.MSC2716_INSERTION
+        or event.type == EventTypes.MSC2716_BATCH
+        or event.type == EventTypes.MSC2716_MARKER
+    ):
+        check_historical(event.room_version, event, auth_dict)
+
    logger.debug("Allowing! %s", event)


@@ -358,6 +365,7 @@ LENIENT_EVENT_BYTE_LIMITS_ROOM_VERSIONS = {
    RoomVersions.V9,
    RoomVersions.MSC3787,
    RoomVersions.V10,
+    RoomVersions.MSC2716v4,
    RoomVersions.MSC1767v10,
 }

@@ -815,6 +823,38 @@ def check_redaction(
    raise AuthError(403, "You don't have permission to redact events")


+def check_historical(
+    room_version_obj: RoomVersion,
+    event: "EventBase",
+    auth_events: StateMap["EventBase"],
+) -> None:
+    """Check whether the event sender is allowed to send historical related
+    events like "insertion", "batch", and "marker".
+
+    Returns:
+        None
+
+    Raises:
+        AuthError if the event sender is not allowed to send historical related events
+        ("insertion", "batch", and "marker").
+    """
+    # Ignore the auth checks in room versions that do not support historical
+    # events
+    if not room_version_obj.msc2716_historical:
+        return
+
+    user_level = get_user_power_level(event.user_id, auth_events)
+
+    historical_level = get_named_level(auth_events, "historical", 100)
+
+    if user_level < historical_level:
+        raise UnstableSpecAuthError(
+            403,
+            'You don\'t have permission to send send historical related events ("insertion", "batch", and "marker")',
+            errcode=Codes.INSUFFICIENT_POWER,
+        )
+
+
 def _check_power_levels(
    room_version_obj: RoomVersion,
    event: "EventBase",
@@ -198,6 +198,7 @@ class _EventInternalMetadata:
    soft_failed: DictProperty[bool] = DictProperty("soft_failed")
    proactively_send: DictProperty[bool] = DictProperty("proactively_send")
    redacted: DictProperty[bool] = DictProperty("redacted")
+    historical: DictProperty[bool] = DictProperty("historical")

    txn_id: DictProperty[str] = DictProperty("txn_id")
    """The transaction ID, if it was set when the event was created."""
@@ -287,6 +288,14 @@ class _EventInternalMetadata:
        """
        return self._dict.get("redacted", False)

+    def is_historical(self) -> bool:
+        """Whether this is a historical message.
+        This is used by the batchsend historical message endpoint and
+        is needed to and mark the event as backfilled and skip some checks
+        like push notifications.
+        """
+        return self._dict.get("historical", False)
+
    def is_notifiable(self) -> bool:
        """Whether this event can trigger a push notification"""
        return not self.is_outlier() or self.is_out_of_band_membership()
@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 from abc import ABC, abstractmethod
-from typing import TYPE_CHECKING, Dict, List, Optional, Tuple
+from typing import TYPE_CHECKING, List, Optional, Tuple

 import attr
 from immutabledict import immutabledict
@@ -107,32 +107,33 @@ class EventContext(UnpersistedEventContextBase):
        state_delta_due_to_event: If `state_group` and `state_group_before_event` are not None
            then this is the delta of the state between the two groups.

-        state_group_deltas: If not empty, this is a dict collecting a mapping of the state
-            difference between state groups.
+        prev_group: If it is known, ``state_group``'s prev_group. Note that this being
+            None does not necessarily mean that ``state_group`` does not have
+            a prev_group!

-            The keys are a tuple of two integers: the initial group and final state group.
-            The corresponding value is a state map representing the state delta between
-            these state groups.
+            If the event is a state event, this is normally the same as
+            ``state_group_before_event``.

-            The dictionary is expected to have at most two entries with state groups of:
+            If ``state_group`` is None (ie, the event is an outlier), ``prev_group``
+            will always also be ``None``.

-            1. The state group before the event and after the event.
-            2. The state group preceding the state group before the event and the
-               state group before the event.
+            Note that this *not* (necessarily) the state group associated with
+            ``_prev_state_ids``.

-            This information is collected and stored as part of an optimization for persisting
-            events.
+        delta_ids: If ``prev_group`` is not None, the state delta between ``prev_group``
+            and ``state_group``.

        partial_state: if True, we may be storing this event with a temporary,
            incomplete state.
    """

    _storage: "StorageControllers"
-    state_group_deltas: Dict[Tuple[int, int], StateMap[str]]
    rejected: Optional[str] = None
    _state_group: Optional[int] = None
    state_group_before_event: Optional[int] = None
    _state_delta_due_to_event: Optional[StateMap[str]] = None
+    prev_group: Optional[int] = None
+    delta_ids: Optional[StateMap[str]] = None
    app_service: Optional[ApplicationService] = None

    partial_state: bool = False
@@ -144,14 +145,16 @@ class EventContext(UnpersistedEventContextBase):
        state_group_before_event: Optional[int],
        state_delta_due_to_event: Optional[StateMap[str]],
        partial_state: bool,
-        state_group_deltas: Dict[Tuple[int, int], StateMap[str]],
+        prev_group: Optional[int] = None,
+        delta_ids: Optional[StateMap[str]] = None,
    ) -> "EventContext":
        return EventContext(
            storage=storage,
            state_group=state_group,
            state_group_before_event=state_group_before_event,
            state_delta_due_to_event=state_delta_due_to_event,
-            state_group_deltas=state_group_deltas,
+            prev_group=prev_group,
+            delta_ids=delta_ids,
            partial_state=partial_state,
        )

@@ -160,7 +163,7 @@ class EventContext(UnpersistedEventContextBase):
        storage: "StorageControllers",
    ) -> "EventContext":
        """Return an EventContext instance suitable for persisting an outlier event"""
-        return EventContext(storage=storage, state_group_deltas={})
+        return EventContext(storage=storage)

    async def persist(self, event: EventBase) -> "EventContext":
        return self
@@ -180,15 +183,13 @@ class EventContext(UnpersistedEventContextBase):
            "state_group": self._state_group,
            "state_group_before_event": self.state_group_before_event,
            "rejected": self.rejected,
-            "state_group_deltas": _encode_state_group_delta(self.state_group_deltas),
+            "prev_group": self.prev_group,
            "state_delta_due_to_event": _encode_state_dict(
                self._state_delta_due_to_event
            ),
+            "delta_ids": _encode_state_dict(self.delta_ids),
            "app_service_id": self.app_service.id if self.app_service else None,
            "partial_state": self.partial_state,
-            # add dummy delta_ids and prev_group for backwards compatibility
-            "delta_ids": None,
-            "prev_group": None,
        }

    @staticmethod
@@ -203,24 +204,17 @@ class EventContext(UnpersistedEventContextBase):
        Returns:
            The event context.
        """
-        # workaround for backwards/forwards compatibility: if the input doesn't have a value
-        # for "state_group_deltas" just assign an empty dict
-        state_group_deltas = input.get("state_group_deltas", None)
-        if state_group_deltas:
-            state_group_deltas = _decode_state_group_delta(state_group_deltas)
-        else:
-            state_group_deltas = {}
-
        context = EventContext(
            # We use the state_group and prev_state_id stuff to pull the
            # current_state_ids out of the DB and construct prev_state_ids.
            storage=storage,
            state_group=input["state_group"],
            state_group_before_event=input["state_group_before_event"],
-            state_group_deltas=state_group_deltas,
+            prev_group=input["prev_group"],
            state_delta_due_to_event=_decode_state_dict(
                input["state_delta_due_to_event"]
            ),
+            delta_ids=_decode_state_dict(input["delta_ids"]),
            rejected=input["rejected"],
            partial_state=input.get("partial_state", False),
        )
@@ -355,7 +349,7 @@ class UnpersistedEventContext(UnpersistedEventContextBase):
    _storage: "StorageControllers"
    state_group_before_event: Optional[int]
    state_group_after_event: Optional[int]
-    state_delta_due_to_event: Optional[StateMap[str]]
+    state_delta_due_to_event: Optional[dict]
    prev_group_for_state_group_before_event: Optional[int]
    delta_ids_to_state_group_before_event: Optional[StateMap[str]]
    partial_state: bool
@@ -386,16 +380,26 @@ class UnpersistedEventContext(UnpersistedEventContextBase):

        events_and_persisted_context = []
        for event, unpersisted_context in amended_events_and_context:
-            state_group_deltas = unpersisted_context._build_state_group_deltas()
-
-            context = EventContext(
-                storage=unpersisted_context._storage,
-                state_group=unpersisted_context.state_group_after_event,
-                state_group_before_event=unpersisted_context.state_group_before_event,
-                state_delta_due_to_event=unpersisted_context.state_delta_due_to_event,
-                partial_state=unpersisted_context.partial_state,
-                state_group_deltas=state_group_deltas,
-            )
+            if event.is_state():
+                context = EventContext(
+                    storage=unpersisted_context._storage,
+                    state_group=unpersisted_context.state_group_after_event,
+                    state_group_before_event=unpersisted_context.state_group_before_event,
+                    state_delta_due_to_event=unpersisted_context.state_delta_due_to_event,
+                    partial_state=unpersisted_context.partial_state,
+                    prev_group=unpersisted_context.state_group_before_event,
+                    delta_ids=unpersisted_context.state_delta_due_to_event,
+                )
+            else:
+                context = EventContext(
+                    storage=unpersisted_context._storage,
+                    state_group=unpersisted_context.state_group_after_event,
+                    state_group_before_event=unpersisted_context.state_group_before_event,
+                    state_delta_due_to_event=unpersisted_context.state_delta_due_to_event,
+                    partial_state=unpersisted_context.partial_state,
+                    prev_group=unpersisted_context.prev_group_for_state_group_before_event,
+                    delta_ids=unpersisted_context.delta_ids_to_state_group_before_event,
+                )
            events_and_persisted_context.append((event, context))
        return events_and_persisted_context

@@ -448,11 +452,11 @@ class UnpersistedEventContext(UnpersistedEventContextBase):

        # if the event isn't a state event the state group doesn't change
        if not self.state_delta_due_to_event:
-            self.state_group_after_event = self.state_group_before_event
+            state_group_after_event = self.state_group_before_event

        # otherwise if it is a state event we need to get a state group for it
        else:
-            self.state_group_after_event = await self._storage.state.store_state_group(
+            state_group_after_event = await self._storage.state.store_state_group(
                event.event_id,
                event.room_id,
                prev_group=self.state_group_before_event,
@@ -460,81 +464,16 @@ class UnpersistedEventContext(UnpersistedEventContextBase):
                current_state_ids=None,
            )

-        state_group_deltas = self._build_state_group_deltas()
-
        return EventContext.with_state(
            storage=self._storage,
-            state_group=self.state_group_after_event,
+            state_group=state_group_after_event,
            state_group_before_event=self.state_group_before_event,
            state_delta_due_to_event=self.state_delta_due_to_event,
-            state_group_deltas=state_group_deltas,
            partial_state=self.partial_state,
+            prev_group=self.state_group_before_event,
+            delta_ids=self.state_delta_due_to_event,
        )

-    def _build_state_group_deltas(self) -> Dict[Tuple[int, int], StateMap]:
-        """
-        Collect deltas between the state groups associated with this context
-        """
-        state_group_deltas = {}
-
-        # if we know the state group before the event and after the event, add them and the
-        # state delta between them to state_group_deltas
-        if self.state_group_before_event and self.state_group_after_event:
-            # if we have the state groups we should have the delta
-            assert self.state_delta_due_to_event is not None
-            state_group_deltas[
-                (
-                    self.state_group_before_event,
-                    self.state_group_after_event,
-                )
-            ] = self.state_delta_due_to_event
-
-        # the state group before the event may also have a state group which precedes it, if
-        # we have that and the state group before the event, add them and the state
-        # delta between them to state_group_deltas
-        if (
-            self.prev_group_for_state_group_before_event
-            and self.state_group_before_event
-        ):
-            # if we have both state groups we should have the delta between them
-            assert self.delta_ids_to_state_group_before_event is not None
-            state_group_deltas[
-                (
-                    self.prev_group_for_state_group_before_event,
-                    self.state_group_before_event,
-                )
-            ] = self.delta_ids_to_state_group_before_event
-
-        return state_group_deltas
-
-
-def _encode_state_group_delta(
-    state_group_delta: Dict[Tuple[int, int], StateMap[str]]
-) -> List[Tuple[int, int, Optional[List[Tuple[str, str, str]]]]]:
-    if not state_group_delta:
-        return []
-
-    state_group_delta_encoded = []
-    for key, value in state_group_delta.items():
-        state_group_delta_encoded.append((key[0], key[1], _encode_state_dict(value)))
-
-    return state_group_delta_encoded
-
-
-def _decode_state_group_delta(
-    input: List[Tuple[int, int, List[Tuple[str, str, str]]]]
-) -> Dict[Tuple[int, int], StateMap[str]]:
-    if not input:
-        return {}
-
-    state_group_deltas = {}
-    for state_group_1, state_group_2, state_dict in input:
-        state_map = _decode_state_dict(state_dict)
-        assert state_map is not None
-        state_group_deltas[(state_group_1, state_group_2)] = state_map
-
-    return state_group_deltas
-

 def _encode_state_dict(
    state_dict: Optional[StateMap[str]],
@@ -164,12 +164,21 @@ def prune_event_dict(room_version: RoomVersion, event_dict: JsonDict) -> JsonDic
        if room_version.msc2176_redaction_rules:
            add_fields("invite")

+        if room_version.msc2716_historical:
+            add_fields("historical")
+
    elif event_type == EventTypes.Aliases and room_version.special_case_aliases_auth:
        add_fields("aliases")
    elif event_type == EventTypes.RoomHistoryVisibility:
        add_fields("history_visibility")
    elif event_type == EventTypes.Redaction and room_version.msc2176_redaction_rules:
        add_fields("redacts")
+    elif room_version.msc2716_redactions and event_type == EventTypes.MSC2716_INSERTION:
+        add_fields(EventContentFields.MSC2716_NEXT_BATCH_ID)
+    elif room_version.msc2716_redactions and event_type == EventTypes.MSC2716_BATCH:
+        add_fields(EventContentFields.MSC2716_BATCH_ID)
+    elif room_version.msc2716_redactions and event_type == EventTypes.MSC2716_MARKER:
+        add_fields(EventContentFields.MSC2716_INSERTION_EVENT_REFERENCE)

    # Protect the rel_type and event_id fields under the m.relates_to field.
    if room_version.msc3389_relation_redactions:
@@ -134,8 +134,13 @@ class EventValidator:
                )

        # If the event contains a mentions key, validate it.
-        if EventContentFields.MENTIONS in event.content:
-            validate_json_object(event.content[EventContentFields.MENTIONS], Mentions)
+        if (
+            EventContentFields.MSC3952_MENTIONS in event.content
+            and config.experimental.msc3952_intentional_mentions
+        ):
+            validate_json_object(
+                event.content[EventContentFields.MSC3952_MENTIONS], Mentions
+            )

    def _validate_retention(self, event: EventBase) -> None:
        """Checks that an event that defines the retention policy for a room respects the
@@ -260,9 +260,7 @@ class FederationClient(FederationBase):
        use_unstable = False
        for user_id, one_time_keys in query.items():
            for device_id, algorithms in one_time_keys.items():
-                # If more than one algorithm is requested, attempt to use the unstable
-                # endpoint.
-                if sum(algorithms.values()) > 1:
+                if any(count > 1 for count in algorithms.values()):
                    use_unstable = True
                if algorithms:
                    # For the stable query, choose only the first algorithm.
@@ -298,7 +296,6 @@ class FederationClient(FederationBase):
        else:
            logger.debug("Skipping unstable claim client keys API")

-        # TODO Potentially attempt multiple queries and combine the results?
        return await self.transport_layer.claim_client_keys(
            user, destination, content, timeout
        )
@@ -944,7 +944,7 @@ class FederationServer(FederationBase):
            if not self._is_mine_server_name(authorising_server):
                raise SynapseError(
                    400,
-                    f"Cannot authorise membership event for {authorising_server}. We can only authorise requests from our own homeserver",
+                    f"Cannot authorise request from resident server: {authorising_server}",
                )

            event.signatures.update(
@@ -1016,9 +1016,7 @@ class FederationServer(FederationBase):
            for user_id, device_keys in result.items():
                for device_id, keys in device_keys.items():
                    for key_id, key in keys.items():
-                        json_result.setdefault(user_id, {}).setdefault(device_id, {})[
-                            key_id
-                        ] = key
+                        json_result.setdefault(user_id, {})[device_id] = {key_id: key}

        logger.info(
            "Claimed one-time-keys: %s",
@@ -109,8 +109,10 @@ was enabled*, Catch-Up Mode is exited and we return to `_transaction_transmissio

 If a remote server is unreachable over federation, we back off from that server,
 with an exponentially-increasing retry interval.
-We automatically retry after the retry interval expires (roughly, the logic to do so
-being triggered every minute).
+Whilst we don't automatically retry after the interval, we prevent making new attempts
+until such time as the back-off has cleared.
+Once the back-off is cleared and a new PDU or EDU arrives for transmission, the transmission
+loop resumes and empties the queue by making federation requests.

 If the backoff grows too large (> 1 hour), the in-memory queue is emptied (to prevent
 unbounded growth) and Catch-Up Mode is entered.
@@ -143,6 +145,7 @@ from prometheus_client import Counter
 from typing_extensions import Literal

 from twisted.internet import defer
+from twisted.internet.interfaces import IDelayedCall

 import synapse.metrics
 from synapse.api.presence import UserPresenceState
@@ -181,18 +184,14 @@ sent_pdus_destination_dist_total = Counter(
    "Total number of PDUs queued for sending across all destinations",
 )

-# Time (in s) to wait before trying to wake up destinations that have
-# catch-up outstanding. This will also be the delay applied at startup
-# before trying the same.
-# Please note that rate limiting still applies, so while the loop is
-# executed every X seconds the destinations may not be wake up because
-# they are being rate limited following previous attempt failures.
-WAKEUP_RETRY_PERIOD_SEC = 60
+# Time (in s) after Synapse's startup that we will begin to wake up destinations
+# that have catch-up outstanding.
+CATCH_UP_STARTUP_DELAY_SEC = 15

 # Time (in s) to wait in between waking up each destination, i.e. one destination
-# will be woken up every <x> seconds until we have woken every destination
-# has outstanding catch-up.
-WAKEUP_INTERVAL_BETWEEN_DESTINATIONS_SEC = 5
+# will be woken up every <x> seconds after Synapse's startup until we have woken
+# every destination has outstanding catch-up.
+CATCH_UP_STARTUP_INTERVAL_SEC = 5


 class AbstractFederationSender(metaclass=abc.ABCMeta):
@@ -416,10 +415,12 @@ class FederationSender(AbstractFederationSender):
            / hs.config.ratelimiting.federation_rr_transactions_per_room_per_second
        )

-        # Regularly wake up destinations that have outstanding PDUs to be caught up
-        self.clock.looping_call(
+        # wake up destinations that have outstanding PDUs to be caught up
+        self._catchup_after_startup_timer: Optional[
+            IDelayedCall
+        ] = self.clock.call_later(
+            CATCH_UP_STARTUP_DELAY_SEC,
            run_as_background_process,
-            WAKEUP_RETRY_PERIOD_SEC * 1000.0,
            "wake_destinations_needing_catchup",
            self._wake_destinations_needing_catchup,
        )
@@ -965,6 +966,7 @@ class FederationSender(AbstractFederationSender):

            if not destinations_to_wake:
                # finished waking all destinations!
+                self._catchup_after_startup_timer = None
                break

            last_processed = destinations_to_wake[-1]
@@ -981,4 +983,4 @@ class FederationSender(AbstractFederationSender):
                    last_processed,
                )
                self.wake_destination(destination)
-                await self.clock.sleep(WAKEUP_INTERVAL_BETWEEN_DESTINATIONS_SEC)
+                await self.clock.sleep(CATCH_UP_STARTUP_INTERVAL_SEC)
@@ -12,7 +12,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.

-import json
 import logging
 from typing import TYPE_CHECKING, Any, Dict

@@ -91,8 +90,6 @@ class DeviceMessageHandler:
            burst_count=hs.config.ratelimiting.rc_key_requests.burst_count,
        )

-        self._msc3944_enabled = hs.config.experimental.msc3944_enabled
-
    async def on_direct_to_device_edu(self, origin: str, content: JsonDict) -> None:
        """
        Handle receiving to-device messages from remote homeservers.
@@ -223,7 +220,7 @@ class DeviceMessageHandler:

        set_tag(SynapseTags.TO_DEVICE_TYPE, message_type)
        set_tag(SynapseTags.TO_DEVICE_SENDER, sender_user_id)
-        local_messages: Dict[str, Dict[str, JsonDict]] = {}
+        local_messages = {}
        remote_messages: Dict[str, Dict[str, Dict[str, JsonDict]]] = {}
        for user_id, by_device in messages.items():
            # add an opentracing log entry for each message
@@ -258,56 +255,16 @@ class DeviceMessageHandler:

            # we use UserID.from_string to catch invalid user ids
            if self.is_mine(UserID.from_string(user_id)):
-                for device_id, message_content in by_device.items():
-                    # Drop any previous identical (same request_id and requesting_device_id)
-                    # room_key_request, ignoring the action property when comparing.
-                    # This handles dropping previous identical and cancelled requests.
-                    if (
-                        self._msc3944_enabled
-                        and message_type == ToDeviceEventTypes.RoomKeyRequest
-                        and user_id == sender_user_id
-                    ):
-                        req_id = message_content.get("request_id")
-                        requesting_device_id = message_content.get(
-                            "requesting_device_id"
-                        )
-                        if req_id and requesting_device_id:
-                            previous_request_deleted = False
-                            for (
-                                stream_id,
-                                message_json,
-                            ) in await self.store.get_all_device_messages(
-                                user_id, device_id
-                            ):
-                                orig_message = json.loads(message_json)
-                                if (
-                                    orig_message["type"]
-                                    == ToDeviceEventTypes.RoomKeyRequest
-                                ):
-                                    content = orig_message.get("content", {})
-                                    if (
-                                        content.get("request_id") == req_id
-                                        and content.get("requesting_device_id")
-                                        == requesting_device_id
-                                    ):
-                                        if await self.store.delete_device_message(
-                                            stream_id
-                                        ):
-                                            previous_request_deleted = True
-
-                            if (
-                                message_content.get("action") == "request_cancellation"
-                                and previous_request_deleted
-                            ):
-                                # Do not store the cancellation since we deleted the matching
-                                # request(s) before it reaches the device.
-                                continue
-                    message = {
+                messages_by_device = {
+                    device_id: {
                        "content": message_content,
                        "type": message_type,
                        "sender": sender_user_id,
                    }
-                    local_messages.setdefault(user_id, {})[device_id] = message
+                    for device_id, message_content in by_device.items()
+                }
+                if messages_by_device:
+                    local_messages[user_id] = messages_by_device
            else:
                destination = get_domain_from_id(user_id)
                remote_messages.setdefault(destination, {})[user_id] = by_device
@@ -105,12 +105,14 @@ backfill_processing_before_timer = Histogram(
 )


-# TODO: We can refactor this away now that there is only one backfill point again
 class _BackfillPointType(Enum):
    # a regular backwards extremity (ie, an event which we don't yet have, but which
    # is referred to by other events in the DAG)
    BACKWARDS_EXTREMITY = enum.auto()

+    # an MSC2716 "insertion event"
+    INSERTION_PONT = enum.auto()
+

@attr.s(slots=True, auto_attribs=True, frozen=True)
 class _BackfillPoint:
@@ -198,7 +200,6 @@ class FederationHandler:
            )

    @trace
-    @tag_args
    async def maybe_backfill(
        self, room_id: str, current_depth: int, limit: int
    ) -> bool:
@@ -213,9 +214,6 @@ class FederationHandler:
            limit: The number of events that the pagination request will
                return. This is used as part of the heuristic to decide if we
                should back paginate.
-
-        Returns:
-            True if we actually tried to backfill something, otherwise False.
        """
        # Starting the processing time here so we can include the room backfill
        # linearizer lock queue in the timing
@@ -229,8 +227,6 @@ class FederationHandler:
                processing_start_time=processing_start_time,
            )

-    @trace
-    @tag_args
    async def _maybe_backfill_inner(
        self,
        room_id: str,
@@ -251,9 +247,6 @@ class FederationHandler:
            limit: The max number of events to request from the remote federated server.
            processing_start_time: The time when `maybe_backfill` started processing.
                Only used for timing. If `None`, no timing observation will be made.
-
-        Returns:
-            True if we actually tried to backfill something, otherwise False.
        """
        backwards_extremities = [
            _BackfillPoint(event_id, depth, _BackfillPointType.BACKWARDS_EXTREMITY)
@@ -271,10 +264,32 @@ class FederationHandler:
            )
        ]

+        insertion_events_to_be_backfilled: List[_BackfillPoint] = []
+        if self.hs.config.experimental.msc2716_enabled:
+            insertion_events_to_be_backfilled = [
+                _BackfillPoint(event_id, depth, _BackfillPointType.INSERTION_PONT)
+                for event_id, depth in await self.store.get_insertion_event_backward_extremities_in_room(
+                    room_id=room_id,
+                    current_depth=current_depth,
+                    # We only need to end up with 5 extremities combined with
+                    # the backfill points to make the `/backfill` request ...
+                    # (see the other comment above for more context).
+                    limit=50,
+                )
+            ]
+        logger.debug(
+            "_maybe_backfill_inner: backwards_extremities=%s insertion_events_to_be_backfilled=%s",
+            backwards_extremities,
+            insertion_events_to_be_backfilled,
+        )
+
        # we now have a list of potential places to backpaginate from. We prefer to
        # start with the most recent (ie, max depth), so let's sort the list.
        sorted_backfill_points: List[_BackfillPoint] = sorted(
-            backwards_extremities,
+            itertools.chain(
+                backwards_extremities,
+                insertion_events_to_be_backfilled,
+            ),
            key=lambda e: -int(e.depth),
        )

@@ -287,30 +302,15 @@ class FederationHandler:
            len(sorted_backfill_points),
            sorted_backfill_points,
        )
-        set_tag(
-            SynapseTags.RESULT_PREFIX + "sorted_backfill_points",
-            str(sorted_backfill_points),
-        )
-        set_tag(
-            SynapseTags.RESULT_PREFIX + "sorted_backfill_points.length",
-            str(len(sorted_backfill_points)),
-        )

-        # If we have no backfill points lower than the `current_depth` then either we
-        # can a) bail or b) still attempt to backfill. We opt to try backfilling anyway
-        # just in case we do get relevant events. This is good for eventual consistency
-        # sake but we don't need to block the client for something that is just as
-        # likely not to return anything relevant so we backfill in the background. The
-        # only way, this could return something relevant is if we discover a new branch
-        # of history that extends all the way back to where we are currently paginating
-        # and it's within the 100 events that are returned from `/backfill`.
+        # If we have no backfill points lower than the `current_depth` then
+        # either we can a) bail or b) still attempt to backfill. We opt to try
+        # backfilling anyway just in case we do get relevant events.
        if not sorted_backfill_points and current_depth != MAX_DEPTH:
            logger.debug(
                "_maybe_backfill_inner: all backfill points are *after* current depth. Trying again with later backfill points."
            )
-            run_as_background_process(
-                "_maybe_backfill_inner_anyway_with_max_depth",
-                self._maybe_backfill_inner,
+            return await self._maybe_backfill_inner(
                room_id=room_id,
                # We use `MAX_DEPTH` so that we find all backfill points next
                # time (all events are below the `MAX_DEPTH`)
@@ -321,9 +321,6 @@ class FederationHandler:
                # overall otherwise the smaller one will throw off the results.
                processing_start_time=None,
            )
-            # We return `False` because we're backfilling in the background and there is
-            # no new events immediately for the caller to know about yet.
-            return False

        # Even after recursing with `MAX_DEPTH`, we didn't find any
        # backward extremities to backfill from.
@@ -387,7 +384,10 @@ class FederationHandler:
            #   event but not anything before it. This would require looking at the
            #   state *before* the event, ignoring the special casing certain event
            #   types have.
-            event_ids_to_check = await self.store.get_successor_events(bp.event_id)
+            if bp.type == _BackfillPointType.INSERTION_PONT:
+                event_ids_to_check = [bp.event_id]
+            else:
+                event_ids_to_check = await self.store.get_successor_events(bp.event_id)

            events_to_check = await self.store.get_events_as_list(
                event_ids_to_check,
@@ -601,6 +601,18 @@ class FederationEventHandler:
                room_id, [(event, context)]
            )

+            # If we're joining the room again, check if there is new marker
+            # state indicating that there is new history imported somewhere in
+            # the DAG. Multiple markers can exist in the current state with
+            # unique state_keys.
+            #
+            # Do this after the state from the remote join was persisted (via
+            # `persist_events_and_notify`). Otherwise we can run into a
+            # situation where the create event doesn't exist yet in the
+            # `current_state_events`
+            for e in state:
+                await self._handle_marker_event(origin, e)
+
            return stream_id_after_persist

    async def update_state_for_partial_state_event(
@@ -903,6 +915,13 @@ class FederationEventHandler:
            )
        )

+        # We construct the event lists in source order from `/backfill` response because
+        # it's a) easiest, but also b) the order in which we process things matters for
+        # MSC2716 historical batches because many historical events are all at the same
+        # `depth` and we rely on the tenuous sort that the other server gave us and hope
+        # they're doing their best. The brittle nature of this ordering for historical
+        # messages over federation is one of the reasons why we don't want to continue
+        # on MSC2716 until we have online topological ordering.
        events_with_failed_pull_attempts, fresh_events = partition(
            new_events, lambda e: e.event_id in event_ids_with_failed_pull_attempts
        )
@@ -1441,6 +1460,8 @@ class FederationEventHandler:

        await self._run_push_actions_and_persist_event(event, context, backfilled)

+        await self._handle_marker_event(origin, event)
+
        if backfilled or context.rejected:
            return

@@ -1538,6 +1559,94 @@ class FederationEventHandler:
        except Exception:
            logger.exception("Failed to resync device for %s", sender)

+    @trace
+    async def _handle_marker_event(self, origin: str, marker_event: EventBase) -> None:
+        """Handles backfilling the insertion event when we receive a marker
+        event that points to one.
+
+        Args:
+            origin: Origin of the event. Will be called to get the insertion event
+            marker_event: The event to process
+        """
+
+        if marker_event.type != EventTypes.MSC2716_MARKER:
+            # Not a marker event
+            return
+
+        if marker_event.rejected_reason is not None:
+            # Rejected event
+            return
+
+        # Skip processing a marker event if the room version doesn't
+        # support it or the event is not from the room creator.
+        room_version = await self._store.get_room_version(marker_event.room_id)
+        create_event = await self._store.get_create_event_for_room(marker_event.room_id)
+        if not room_version.msc2175_implicit_room_creator:
+            room_creator = create_event.content.get(EventContentFields.ROOM_CREATOR)
+        else:
+            room_creator = create_event.sender
+        if not room_version.msc2716_historical and (
+            not self._config.experimental.msc2716_enabled
+            or marker_event.sender != room_creator
+        ):
+            return
+
+        logger.debug("_handle_marker_event: received %s", marker_event)
+
+        insertion_event_id = marker_event.content.get(
+            EventContentFields.MSC2716_INSERTION_EVENT_REFERENCE
+        )
+
+        if insertion_event_id is None:
+            # Nothing to retrieve then (invalid marker)
+            return
+
+        already_seen_insertion_event = await self._store.have_seen_event(
+            marker_event.room_id, insertion_event_id
+        )
+        if already_seen_insertion_event:
+            # No need to process a marker again if we have already seen the
+            # insertion event that it was pointing to
+            return
+
+        logger.debug(
+            "_handle_marker_event: backfilling insertion event %s", insertion_event_id
+        )
+
+        await self._get_events_and_persist(
+            origin,
+            marker_event.room_id,
+            [insertion_event_id],
+        )
+
+        insertion_event = await self._store.get_event(
+            insertion_event_id, allow_none=True
+        )
+        if insertion_event is None:
+            logger.warning(
+                "_handle_marker_event: server %s didn't return insertion event %s for marker %s",
+                origin,
+                insertion_event_id,
+                marker_event.event_id,
+            )
+            return
+
+        logger.debug(
+            "_handle_marker_event: succesfully backfilled insertion event %s from marker event %s",
+            insertion_event,
+            marker_event,
+        )
+
+        await self._store.insert_insertion_extremity(
+            insertion_event_id, marker_event.room_id
+        )
+
+        logger.debug(
+            "_handle_marker_event: insertion extremity added for %s from marker event %s",
+            insertion_event,
+            marker_event,
+        )
+
    async def backfill_event_id(
        self, destinations: List[str], room_id: str, event_id: str
    ) -> PulledPduInfo:
@@ -60,6 +60,7 @@ from synapse.replication.http.send_event import ReplicationSendEventRestServlet
 from synapse.replication.http.send_events import ReplicationSendEventsRestServlet
 from synapse.storage.databases.main.events_worker import EventRedactBehaviour
 from synapse.types import (
+    MutableStateMap,
    PersistedEventPosition,
    Requester,
    RoomAlias,
@@ -572,6 +573,7 @@ class EventCreationHandler:
        state_event_ids: Optional[List[str]] = None,
        require_consent: bool = True,
        outlier: bool = False,
+        historical: bool = False,
        depth: Optional[int] = None,
        state_map: Optional[StateMap[str]] = None,
        for_batch: bool = False,
@@ -597,7 +599,7 @@ class EventCreationHandler:
            allow_no_prev_events: Whether to allow this event to be created an empty
                list of prev_events. Normally this is prohibited just because most
                events should have a prev_event and we should only use this in special
-                cases (previously useful for MSC2716).
+                cases like MSC2716.
            prev_event_ids:
                the forward extremities to use as the prev_events for the
                new event.
@@ -612,10 +614,13 @@ class EventCreationHandler:
                If non-None, prev_event_ids must also be provided.

            state_event_ids:
-                The full state at a given event. This was previously used particularly
-                by the MSC2716 /batch_send endpoint. This should normally be left as
-                None, which will cause the auth_event_ids to be calculated based on the
-                room state at the prev_events.
+                The full state at a given event. This is used particularly by the MSC2716
+                /batch_send endpoint. One use case is with insertion events which float at
+                the beginning of a historical batch and don't have any `prev_events` to
+                derive from; we add all of these state events as the explicit state so the
+                rest of the historical batch can inherit the same state and state_group.
+                This should normally be left as None, which will cause the auth_event_ids
+                to be calculated based on the room state at the prev_events.

            require_consent: Whether to check if the requester has
                consented to the privacy policy.
@@ -624,6 +629,10 @@ class EventCreationHandler:
                it's from an arbitrary point and floating in the DAG as
                opposed to being inline with the current DAG.

+            historical: Indicates whether the message is being inserted
+                back in time around some existing events. This is used to skip
+                a few checks and mark the event as backfilled.
+
            depth: Override the depth used to order the event in the DAG.
                Should normally be set to None, which will cause the depth to be calculated
                based on the prev_events.
@@ -708,6 +717,8 @@ class EventCreationHandler:

        builder.internal_metadata.outlier = outlier

+        builder.internal_metadata.historical = historical
+
        event, unpersisted_context = await self.create_new_client_event(
            builder=builder,
            requester=requester,
@@ -936,6 +947,7 @@ class EventCreationHandler:
        txn_id: Optional[str] = None,
        ignore_shadow_ban: bool = False,
        outlier: bool = False,
+        historical: bool = False,
        depth: Optional[int] = None,
    ) -> Tuple[EventBase, int]:
        """
@@ -949,16 +961,19 @@ class EventCreationHandler:
            allow_no_prev_events: Whether to allow this event to be created an empty
                list of prev_events. Normally this is prohibited just because most
                events should have a prev_event and we should only use this in special
-                cases (previously useful for MSC2716).
+                cases like MSC2716.
            prev_event_ids:
                The event IDs to use as the prev events.
                Should normally be left as None to automatically request them
                from the database.
            state_event_ids:
-                The full state at a given event. This was previously used particularly
-                by the MSC2716 /batch_send endpoint. This should normally be left as
-                None, which will cause the auth_event_ids to be calculated based on the
-                room state at the prev_events.
+                The full state at a given event. This is used particularly by the MSC2716
+                /batch_send endpoint. One use case is with insertion events which float at
+                the beginning of a historical batch and don't have any `prev_events` to
+                derive from; we add all of these state events as the explicit state so the
+                rest of the historical batch can inherit the same state and state_group.
+                This should normally be left as None, which will cause the auth_event_ids
+                to be calculated based on the room state at the prev_events.
            ratelimit: Whether to rate limit this send.
            txn_id: The transaction ID.
            ignore_shadow_ban: True if shadow-banned users should be allowed to
@@ -966,6 +981,9 @@ class EventCreationHandler:
            outlier: Indicates whether the event is an `outlier`, i.e. if
                it's from an arbitrary point and floating in the DAG as
                opposed to being inline with the current DAG.
+            historical: Indicates whether the message is being inserted
+                back in time around some existing events. This is used to skip
+                a few checks and mark the event as backfilled.
            depth: Override the depth used to order the event in the DAG.
                Should normally be set to None, which will cause the depth to be calculated
                based on the prev_events.
@@ -1035,6 +1053,7 @@ class EventCreationHandler:
                    prev_event_ids=prev_event_ids,
                    state_event_ids=state_event_ids,
                    outlier=outlier,
+                    historical=historical,
                    depth=depth,
                )
                context = await unpersisted_context.persist(event)
@@ -1126,7 +1145,7 @@ class EventCreationHandler:
            allow_no_prev_events: Whether to allow this event to be created an empty
                list of prev_events. Normally this is prohibited just because most
                events should have a prev_event and we should only use this in special
-                cases (previously useful for MSC2716).
+                cases like MSC2716.
            prev_event_ids:
                the forward extremities to use as the prev_events for the
                new event.
@@ -1139,10 +1158,13 @@ class EventCreationHandler:
                based on the room state at the prev_events.

            state_event_ids:
-                The full state at a given event. This was previously used particularly
-                by the MSC2716 /batch_send endpoint. This should normally be left as
-                None, which will cause the auth_event_ids to be calculated based on the
-                room state at the prev_events.
+                The full state at a given event. This is used particularly by the MSC2716
+                /batch_send endpoint. One use case is with insertion events which float at
+                the beginning of a historical batch and don't have any `prev_events` to
+                derive from; we add all of these state events as the explicit state so the
+                rest of the historical batch can inherit the same state and state_group.
+                This should normally be left as None, which will cause the auth_event_ids
+                to be calculated based on the room state at the prev_events.

            depth: Override the depth used to order the event in the DAG.
                Should normally be set to None, which will cause the depth to be calculated
@@ -1239,6 +1261,52 @@ class EventCreationHandler:
            if builder.internal_metadata.outlier:
                event.internal_metadata.outlier = True
                context = EventContext.for_outlier(self._storage_controllers)
+            elif (
+                event.type == EventTypes.MSC2716_INSERTION
+                and state_event_ids
+                and builder.internal_metadata.is_historical()
+            ):
+                # Add explicit state to the insertion event so it has state to derive
+                # from even though it's floating with no `prev_events`. The rest of
+                # the batch can derive from this state and state_group.
+                #
+                # TODO(faster_joins): figure out how this works, and make sure that the
+                #   old state is complete.
+                #   https://github.com/matrix-org/synapse/issues/13003
+                metadata = await self.store.get_metadata_for_events(state_event_ids)
+
+                state_map_for_event: MutableStateMap[str] = {}
+                for state_id in state_event_ids:
+                    data = metadata.get(state_id)
+                    if data is None:
+                        # We're trying to persist a new historical batch of events
+                        # with the given state, e.g. via
+                        # `RoomBatchSendEventRestServlet`. The state can be inferred
+                        # by Synapse or set directly by the client.
+                        #
+                        # Either way, we should have persisted all the state before
+                        # getting here.
+                        raise Exception(
+                            f"State event {state_id} not found in DB,"
+                            " Synapse should have persisted it before using it."
+                        )
+
+                    if data.state_key is None:
+                        raise Exception(
+                            f"Trying to set non-state event {state_id} as state"
+                        )
+
+                    state_map_for_event[(data.event_type, data.state_key)] = state_id
+
+                # TODO(faster_joins): check how MSC2716 works and whether we can have
+                #   partial state here
+                #   https://github.com/matrix-org/synapse/issues/13003
+                context = await self.state.calculate_context_info(
+                    event,
+                    state_ids_before_event=state_map_for_event,
+                    partial_state=False,
+                )
+
            else:
                context = await self.state.calculate_context_info(event)

@@ -1808,6 +1876,28 @@ class EventCreationHandler:
                            403, "Redacting server ACL events is not permitted"
                        )

+                    # Add a little safety stop-gap to prevent people from trying to
+                    # redact MSC2716 related events when they're in a room version
+                    # which does not support it yet. We allow people to use MSC2716
+                    # events in existing room versions but only from the room
+                    # creator since it does not require any changes to the auth
+                    # rules and in effect, the redaction algorithm . In the
+                    # supported room version, we add the `historical` power level to
+                    # auth the MSC2716 related events and adjust the redaction
+                    # algorthim to keep the `historical` field around (redacting an
+                    # event should only strip fields which don't affect the
+                    # structural protocol level).
+                    is_msc2716_event = (
+                        original_event.type == EventTypes.MSC2716_INSERTION
+                        or original_event.type == EventTypes.MSC2716_BATCH
+                        or original_event.type == EventTypes.MSC2716_MARKER
+                    )
+                    if not room_version_obj.msc2716_historical and is_msc2716_event:
+                        raise AuthError(
+                            403,
+                            "Redacting MSC2716 events is not supported in this room version",
+                        )
+
                event_types = event_auth.auth_types_for_event(event.room_version, event)
                prev_state_ids = await context.get_prev_state_ids(
                    StateFilter.from_types(event_types)
@@ -1845,12 +1935,58 @@ class EventCreationHandler:
                if prev_state_ids:
                    raise AuthError(403, "Changing the room create event is forbidden")

+            if event.type == EventTypes.MSC2716_INSERTION:
+                room_version = await self.store.get_room_version_id(event.room_id)
+                room_version_obj = KNOWN_ROOM_VERSIONS[room_version]
+
+                create_event = await self.store.get_create_event_for_room(event.room_id)
+                if not room_version_obj.msc2175_implicit_room_creator:
+                    room_creator = create_event.content.get(
+                        EventContentFields.ROOM_CREATOR
+                    )
+                else:
+                    room_creator = create_event.sender
+
+                # Only check an insertion event if the room version
+                # supports it or the event is from the room creator.
+                if room_version_obj.msc2716_historical or (
+                    self.config.experimental.msc2716_enabled
+                    and event.sender == room_creator
+                ):
+                    next_batch_id = event.content.get(
+                        EventContentFields.MSC2716_NEXT_BATCH_ID
+                    )
+                    conflicting_insertion_event_id = None
+                    if next_batch_id:
+                        conflicting_insertion_event_id = (
+                            await self.store.get_insertion_event_id_by_batch_id(
+                                event.room_id, next_batch_id
+                            )
+                        )
+                    if conflicting_insertion_event_id is not None:
+                        # The current insertion event that we're processing is invalid
+                        # because an insertion event already exists in the room with the
+                        # same next_batch_id. We can't allow multiple because the batch
+                        # pointing will get weird, e.g. we can't determine which insertion
+                        # event the batch event is pointing to.
+                        raise SynapseError(
+                            HTTPStatus.BAD_REQUEST,
+                            "Another insertion event already exists with the same next_batch_id",
+                            errcode=Codes.INVALID_PARAM,
+                        )
+
+            # Mark any `m.historical` messages as backfilled so they don't appear
+            # in `/sync` and have the proper decrementing `stream_ordering` as we import
+            backfilled = False
+            if event.internal_metadata.is_historical():
+                backfilled = True
+
        assert self._storage_controllers.persistence is not None
        (
            persisted_events,
            max_stream_token,
        ) = await self._storage_controllers.persistence.persist_events(
-            events_and_context,
+            events_and_context, backfilled=backfilled
        )

        events_and_pos = []
@@ -40,11 +40,6 @@ if TYPE_CHECKING:

 logger = logging.getLogger(__name__)

-# How many single event gaps we tolerate returning in a `/messages` response before we
-# backfill and try to fill in the history. This is an arbitrarily picked number so feel
-# free to tune it in the future.
-BACKFILL_BECAUSE_TOO_MANY_GAPS_THRESHOLD = 3
-

@attr.s(slots=True, auto_attribs=True)
 class PurgeStatus:
@@ -491,35 +486,35 @@ class PaginationHandler:
                        room_id, room_token.stream
                    )

-            # If they have left the room then clamp the token to be before
-            # they left the room, to save the effort of loading from the
-            # database.
-            if (
-                pagin_config.direction == Direction.BACKWARDS
-                and not use_admin_priviledge
-                and membership == Membership.LEAVE
-            ):
-                # This is only None if the room is world_readable, in which case
-                # "Membership.JOIN" would have been returned and we should never hit
-                # this branch.
-                assert member_event_id
+                if not use_admin_priviledge and membership == Membership.LEAVE:
+                    # If they have left the room then clamp the token to be before
+                    # they left the room, to save the effort of loading from the
+                    # database.

-                leave_token = await self.store.get_topological_token_for_event(
-                    member_event_id
-                )
-                assert leave_token.topological is not None
+                    # This is only None if the room is world_readable, in which
+                    # case "JOIN" would have been returned.
+                    assert member_event_id

-                if leave_token.topological < curr_topo:
-                    from_token = from_token.copy_and_replace(
-                        StreamKeyType.ROOM, leave_token
+                    leave_token = await self.store.get_topological_token_for_event(
+                        member_event_id
                    )
+                    assert leave_token.topological is not None
+
+                    if leave_token.topological < curr_topo:
+                        from_token = from_token.copy_and_replace(
+                            StreamKeyType.ROOM, leave_token
+                        )
+
+                await self.hs.get_federation_handler().maybe_backfill(
+                    room_id,
+                    curr_topo,
+                    limit=pagin_config.limit,
+                )

            to_room_key = None
            if pagin_config.to_token:
                to_room_key = pagin_config.to_token.room_key

-            # Initially fetch the events from the database. With any luck, we can return
-            # these without blocking on backfill (handled below).
            events, next_key = await self.store.paginate_room_events(
                room_id=room_id,
                from_key=from_token.room_key,
@@ -529,94 +524,6 @@ class PaginationHandler:
                event_filter=event_filter,
            )

-            if pagin_config.direction == Direction.BACKWARDS:
-                # We use a `Set` because there can be multiple events at a given depth
-                # and we only care about looking at the unique continum of depths to
-                # find gaps.
-                event_depths: Set[int] = {event.depth for event in events}
-                sorted_event_depths = sorted(event_depths)
-
-                # Inspect the depths of the returned events to see if there are any gaps
-                found_big_gap = False
-                number_of_gaps = 0
-                previous_event_depth = (
-                    sorted_event_depths[0] if len(sorted_event_depths) > 0 else 0
-                )
-                for event_depth in sorted_event_depths:
-                    # We don't expect a negative depth but we'll just deal with it in
-                    # any case by taking the absolute value to get the true gap between
-                    # any two integers.
-                    depth_gap = abs(event_depth - previous_event_depth)
-                    # A `depth_gap` of 1 is a normal continuous chain to the next event
-                    # (1 <-- 2 <-- 3) so anything larger indicates a missing event (it's
-                    # also possible there is no event at a given depth but we can't ever
-                    # know that for sure)
-                    if depth_gap > 1:
-                        number_of_gaps += 1
-
-                    # We only tolerate a small number single-event long gaps in the
-                    # returned events because those are most likely just events we've
-                    # failed to pull in the past. Anything longer than that is probably
-                    # a sign that we're missing a decent chunk of history and we should
-                    # try to backfill it.
-                    #
-                    # XXX: It's possible we could tolerate longer gaps if we checked
-                    # that a given events `prev_events` is one that has failed pull
-                    # attempts and we could just treat it like a dead branch of history
-                    # for now or at least something that we don't need the block the
-                    # client on to try pulling.
-                    #
-                    # XXX: If we had something like MSC3871 to indicate gaps in the
-                    # timeline to the client, we could also get away with any sized gap
-                    # and just have the client refetch the holes as they see fit.
-                    if depth_gap > 2:
-                        found_big_gap = True
-                        break
-                    previous_event_depth = event_depth
-
-                # Backfill in the foreground if we found a big gap, have too many holes,
-                # or we don't have enough events to fill the limit that the client asked
-                # for.
-                missing_too_many_events = (
-                    number_of_gaps > BACKFILL_BECAUSE_TOO_MANY_GAPS_THRESHOLD
-                )
-                not_enough_events_to_fill_response = len(events) < pagin_config.limit
-                if (
-                    found_big_gap
-                    or missing_too_many_events
-                    or not_enough_events_to_fill_response
-                ):
-                    did_backfill = (
-                        await self.hs.get_federation_handler().maybe_backfill(
-                            room_id,
-                            curr_topo,
-                            limit=pagin_config.limit,
-                        )
-                    )
-
-                    # If we did backfill something, refetch the events from the database to
-                    # catch anything new that might have been added since we last fetched.
-                    if did_backfill:
-                        events, next_key = await self.store.paginate_room_events(
-                            room_id=room_id,
-                            from_key=from_token.room_key,
-                            to_key=to_room_key,
-                            direction=pagin_config.direction,
-                            limit=pagin_config.limit,
-                            event_filter=event_filter,
-                        )
-                else:
-                    # Otherwise, we can backfill in the background for eventual
-                    # consistency's sake but we don't need to block the client waiting
-                    # for a costly federation call and processing.
-                    run_as_background_process(
-                        "maybe_backfill_in_the_background",
-                        self.hs.get_federation_handler().maybe_backfill,
-                        room_id,
-                        curr_topo,
-                        limit=pagin_config.limit,
-                    )
-
            next_token = from_token.copy_and_replace(StreamKeyType.ROOM, next_key)

        # if no events are returned from pagination, that implies
@@ -648,6 +648,7 @@ class PresenceHandler(BasePresenceHandler):
    def __init__(self, hs: "HomeServer"):
        super().__init__(hs)
        self.hs = hs
+        self.server_name = hs.hostname
        self.wheel_timer: WheelTimer[str] = WheelTimer()
        self.notifier = hs.get_notifier()
        self._presence_enabled = hs.config.server.use_presence
@@ -27,6 +27,7 @@ logger = logging.getLogger(__name__)

 class ReadMarkerHandler:
    def __init__(self, hs: "HomeServer"):
+        self.server_name = hs.config.server.server_name
        self.store = hs.get_datastores().main
        self.account_data_handler = hs.get_account_data_handler()
        self.read_marker_linearizer = Linearizer(name="read_marker")
@@ -872,8 +872,6 @@ class RoomCreationHandler:
        visibility = config.get("visibility", "private")
        is_public = visibility == "public"

-        self._validate_room_config(config, visibility)
-
        room_id = await self._generate_and_create_room_id(
            creator_id=user_id,
            is_public=is_public,
@@ -1113,7 +1111,20 @@ class RoomCreationHandler:

            return new_event, new_unpersisted_context

-        preset_config, config = self._room_preset_config(room_config)
+        visibility = room_config.get("visibility", "private")
+        preset_config = room_config.get(
+            "preset",
+            RoomCreationPreset.PRIVATE_CHAT
+            if visibility == "private"
+            else RoomCreationPreset.PUBLIC_CHAT,
+        )
+
+        try:
+            config = self._presets_dict[preset_config]
+        except KeyError:
+            raise SynapseError(
+                400, f"'{preset_config}' is not a valid preset", errcode=Codes.BAD_JSON
+            )

        # MSC2175 removes the creator field from the create event.
        if not room_version.msc2175_implicit_room_creator:
@@ -1295,65 +1306,6 @@ class RoomCreationHandler:
        assert last_event.internal_metadata.stream_ordering is not None
        return last_event.internal_metadata.stream_ordering, last_event.event_id, depth

-    def _validate_room_config(
-        self,
-        config: JsonDict,
-        visibility: str,
-    ) -> None:
-        """Checks configuration parameters for a /createRoom request.
-
-        If validation detects invalid parameters an exception may be raised to
-        cause room creation to be aborted and an error response to be returned
-        to the client.
-
-        Args:
-            config: A dict of configuration options. Originally from the body of
-                the /createRoom request
-            visibility: One of "public" or "private"
-        """
-
-        # Validate the requested preset, raise a 400 error if not valid
-        preset_name, preset_config = self._room_preset_config(config)
-
-        # If the user is trying to create an encrypted room and this is forbidden
-        # by the configured default_power_level_content_override, then reject the
-        # request before the room is created.
-        raw_initial_state = config.get("initial_state", [])
-        room_encryption_event = any(
-            s.get("type", "") == EventTypes.RoomEncryption for s in raw_initial_state
-        )
-
-        if preset_config["encrypted"] or room_encryption_event:
-            if self._default_power_level_content_override:
-                override = self._default_power_level_content_override.get(preset_name)
-                if override is not None:
-                    event_levels = override.get("events", {})
-                    room_admin_level = event_levels.get(EventTypes.PowerLevels, 100)
-                    encryption_level = event_levels.get(EventTypes.RoomEncryption, 100)
-                    if encryption_level > room_admin_level:
-                        raise SynapseError(
-                            403,
-                            f"You cannot create an encrypted room. user_level ({room_admin_level}) < send_level ({encryption_level})",
-                        )
-
-    def _room_preset_config(self, room_config: JsonDict) -> Tuple[str, dict]:
-        # The spec says rooms should default to private visibility if
-        # `visibility` is not specified.
-        visibility = room_config.get("visibility", "private")
-        preset_name = room_config.get(
-            "preset",
-            RoomCreationPreset.PRIVATE_CHAT
-            if visibility == "private"
-            else RoomCreationPreset.PUBLIC_CHAT,
-        )
-        try:
-            preset_config = self._presets_dict[preset_name]
-        except KeyError:
-            raise SynapseError(
-                400, f"'{preset_name}' is not a valid preset", errcode=Codes.BAD_JSON
-            )
-        return preset_name, preset_config
-
    def _generate_room_id(self) -> str:
        """Generates a random room ID.

@@ -1538,6 +1490,7 @@ class RoomContextHandler:

 class TimestampLookupHandler:
    def __init__(self, hs: "HomeServer"):
+        self.server_name = hs.hostname
        self.store = hs.get_datastores().main
        self.state_handler = hs.get_state_handler()
        self.federation_client = hs.get_federation_client()
@@ -0,0 +1,466 @@
+import logging
+from typing import TYPE_CHECKING, List, Tuple
+
+from synapse.api.constants import EventContentFields, EventTypes
+from synapse.appservice import ApplicationService
+from synapse.http.servlet import assert_params_in_dict
+from synapse.types import JsonDict, Requester, UserID, create_requester
+from synapse.util.stringutils import random_string
+
+if TYPE_CHECKING:
+    from synapse.server import HomeServer
+
+logger = logging.getLogger(__name__)
+
+
+class RoomBatchHandler:
+    def __init__(self, hs: "HomeServer"):
+        self.hs = hs
+        self.store = hs.get_datastores().main
+        self._state_storage_controller = hs.get_storage_controllers().state
+        self.event_creation_handler = hs.get_event_creation_handler()
+        self.room_member_handler = hs.get_room_member_handler()
+        self.auth = hs.get_auth()
+
+    async def inherit_depth_from_prev_ids(self, prev_event_ids: List[str]) -> int:
+        """Finds the depth which would sort it after the most-recent
+        prev_event_id but before the successors of those events. If no
+        successors are found, we assume it's an historical extremity part of the
+        current batch and use the same depth of the prev_event_ids.
+
+        Args:
+            prev_event_ids: List of prev event IDs
+
+        Returns:
+            Inherited depth
+        """
+        (
+            most_recent_prev_event_id,
+            most_recent_prev_event_depth,
+        ) = await self.store.get_max_depth_of(prev_event_ids)
+
+        # We want to insert the historical event after the `prev_event` but before the successor event
+        #
+        # We inherit depth from the successor event instead of the `prev_event`
+        # because events returned from `/messages` are first sorted by `topological_ordering`
+        # which is just the `depth` and then tie-break with `stream_ordering`.
+        #
+        # We mark these inserted historical events as "backfilled" which gives them a
+        # negative `stream_ordering`. If we use the same depth as the `prev_event`,
+        # then our historical event will tie-break and be sorted before the `prev_event`
+        # when it should come after.
+        #
+        # We want to use the successor event depth so they appear after `prev_event` because
+        # it has a larger `depth` but before the successor event because the `stream_ordering`
+        # is negative before the successor event.
+        assert most_recent_prev_event_id is not None
+        successor_event_ids = await self.store.get_successor_events(
+            most_recent_prev_event_id
+        )
+
+        # If we can't find any successor events, then it's a forward extremity of
+        # historical messages and we can just inherit from the previous historical
+        # event which we can already assume has the correct depth where we want
+        # to insert into.
+        if not successor_event_ids:
+            depth = most_recent_prev_event_depth
+        else:
+            (
+                _,
+                oldest_successor_depth,
+            ) = await self.store.get_min_depth_of(successor_event_ids)
+
+            depth = oldest_successor_depth
+
+        return depth
+
+    def create_insertion_event_dict(
+        self, sender: str, room_id: str, origin_server_ts: int
+    ) -> JsonDict:
+        """Creates an event dict for an "insertion" event with the proper fields
+        and a random batch ID.
+
+        Args:
+            sender: The event author MXID
+            room_id: The room ID that the event belongs to
+            origin_server_ts: Timestamp when the event was sent
+
+        Returns:
+            The new event dictionary to insert.
+        """
+
+        next_batch_id = random_string(8)
+        insertion_event = {
+            "type": EventTypes.MSC2716_INSERTION,
+            "sender": sender,
+            "room_id": room_id,
+            "content": {
+                EventContentFields.MSC2716_NEXT_BATCH_ID: next_batch_id,
+                EventContentFields.MSC2716_HISTORICAL: True,
+            },
+            "origin_server_ts": origin_server_ts,
+        }
+
+        return insertion_event
+
+    async def create_requester_for_user_id_from_app_service(
+        self, user_id: str, app_service: ApplicationService
+    ) -> Requester:
+        """Creates a new requester for the given user_id
+        and validates that the app service is allowed to control
+        the given user.
+
+        Args:
+            user_id: The author MXID that the app service is controlling
+            app_service: The app service that controls the user
+
+        Returns:
+            Requester object
+        """
+
+        await self.auth.validate_appservice_can_control_user_id(app_service, user_id)
+
+        return create_requester(user_id, app_service=app_service)
+
+    async def get_most_recent_full_state_ids_from_event_id_list(
+        self, event_ids: List[str]
+    ) -> List[str]:
+        """Find the most recent event_id and grab the full state at that event.
+        We will use this as a base to auth our historical messages against.
+
+        Args:
+            event_ids: List of event ID's to look at
+
+        Returns:
+            List of event ID's
+        """
+
+        (
+            most_recent_event_id,
+            _,
+        ) = await self.store.get_max_depth_of(event_ids)
+        # mapping from (type, state_key) -> state_event_id
+        assert most_recent_event_id is not None
+        prev_state_map = await self._state_storage_controller.get_state_ids_for_event(
+            most_recent_event_id
+        )
+        # List of state event ID's
+        full_state_ids = list(prev_state_map.values())
+
+        return full_state_ids
+
+    async def persist_state_events_at_start(
+        self,
+        state_events_at_start: List[JsonDict],
+        room_id: str,
+        initial_state_event_ids: List[str],
+        app_service_requester: Requester,
+    ) -> List[str]:
+        """Takes all `state_events_at_start` event dictionaries and creates/persists
+        them in a floating state event chain which don't resolve into the current room
+        state. They are floating because they reference no prev_events which disconnects
+        them from the normal DAG.
+
+        Args:
+            state_events_at_start:
+            room_id: Room where you want the events persisted in.
+            initial_state_event_ids:
+                The base set of state for the historical batch which the floating
+                state chain will derive from. This should probably be the state
+                from the `prev_event` defined by `/batch_send?prev_event_id=$abc`.
+            app_service_requester: The requester of an application service.
+
+        Returns:
+            List of state event ID's we just persisted
+        """
+        assert app_service_requester.app_service
+
+        state_event_ids_at_start = []
+        state_event_ids = initial_state_event_ids.copy()
+
+        # Make the state events float off on their own by specifying no
+        # prev_events for the first one in the chain so we don't have a bunch of
+        # `@mxid joined the room` noise between each batch.
+        prev_event_ids_for_state_chain: List[str] = []
+
+        for index, state_event in enumerate(state_events_at_start):
+            assert_params_in_dict(
+                state_event, ["type", "origin_server_ts", "content", "sender"]
+            )
+
+            logger.debug(
+                "RoomBatchSendEventRestServlet inserting state_event=%s", state_event
+            )
+
+            event_dict = {
+                "type": state_event["type"],
+                "origin_server_ts": state_event["origin_server_ts"],
+                "content": state_event["content"],
+                "room_id": room_id,
+                "sender": state_event["sender"],
+                "state_key": state_event["state_key"],
+            }
+
+            # Mark all events as historical
+            event_dict["content"][EventContentFields.MSC2716_HISTORICAL] = True
+
+            # TODO: This is pretty much the same as some other code to handle inserting state in this file
+            if event_dict["type"] == EventTypes.Member:
+                membership = event_dict["content"].get("membership", None)
+                event_id, _ = await self.room_member_handler.update_membership(
+                    await self.create_requester_for_user_id_from_app_service(
+                        state_event["sender"], app_service_requester.app_service
+                    ),
+                    target=UserID.from_string(event_dict["state_key"]),
+                    room_id=room_id,
+                    action=membership,
+                    content=event_dict["content"],
+                    historical=True,
+                    # Only the first event in the state chain should be floating.
+                    # The rest should hang off each other in a chain.
+                    allow_no_prev_events=index == 0,
+                    prev_event_ids=prev_event_ids_for_state_chain,
+                    # The first event in the state chain is floating with no
+                    # `prev_events` which means it can't derive state from
+                    # anywhere automatically. So we need to set some state
+                    # explicitly.
+                    #
+                    # Make sure to use a copy of this list because we modify it
+                    # later in the loop here. Otherwise it will be the same
+                    # reference and also update in the event when we append
+                    # later.
+                    state_event_ids=state_event_ids.copy(),
+                )
+            else:
+                (
+                    event,
+                    _,
+                ) = await self.event_creation_handler.create_and_send_nonmember_event(
+                    await self.create_requester_for_user_id_from_app_service(
+                        state_event["sender"], app_service_requester.app_service
+                    ),
+                    event_dict,
+                    historical=True,
+                    # Only the first event in the state chain should be floating.
+                    # The rest should hang off each other in a chain.
+                    allow_no_prev_events=index == 0,
+                    prev_event_ids=prev_event_ids_for_state_chain,
+                    # The first event in the state chain is floating with no
+                    # `prev_events` which means it can't derive state from
+                    # anywhere automatically. So we need to set some state
+                    # explicitly.
+                    #
+                    # Make sure to use a copy of this list because we modify it
+                    # later in the loop here. Otherwise it will be the same
+                    # reference and also update in the event when we append later.
+                    state_event_ids=state_event_ids.copy(),
+                )
+                event_id = event.event_id
+
+            state_event_ids_at_start.append(event_id)
+            state_event_ids.append(event_id)
+            # Connect all the state in a floating chain
+            prev_event_ids_for_state_chain = [event_id]
+
+        return state_event_ids_at_start
+
+    async def persist_historical_events(
+        self,
+        events_to_create: List[JsonDict],
+        room_id: str,
+        inherited_depth: int,
+        initial_state_event_ids: List[str],
+        app_service_requester: Requester,
+    ) -> List[str]:
+        """Create and persists all events provided sequentially. Handles the
+        complexity of creating events in chronological order so they can
+        reference each other by prev_event but still persists in
+        reverse-chronoloical order so they have the correct
+        (topological_ordering, stream_ordering) and sort correctly from
+        /messages.
+
+        Args:
+            events_to_create: List of historical events to create in JSON
+                dictionary format.
+            room_id: Room where you want the events persisted in.
+            inherited_depth: The depth to create the events at (you will
+                probably by calling inherit_depth_from_prev_ids(...)).
+            initial_state_event_ids:
+                This is used to set explicit state for the insertion event at
+                the start of the historical batch since it's floating with no
+                prev_events to derive state from automatically.
+            app_service_requester: The requester of an application service.
+
+        Returns:
+            List of persisted event IDs
+        """
+        assert app_service_requester.app_service
+
+        # We expect the first event in a historical batch to be an insertion event
+        assert events_to_create[0]["type"] == EventTypes.MSC2716_INSERTION
+        # We expect the last event in a historical batch to be an batch event
+        assert events_to_create[-1]["type"] == EventTypes.MSC2716_BATCH
+
+        # Make the historical event chain float off on its own by specifying no
+        # prev_events for the first event in the chain which causes the HS to
+        # ask for the state at the start of the batch later.
+        prev_event_ids: List[str] = []
+
+        event_ids = []
+        events_to_persist = []
+        for index, ev in enumerate(events_to_create):
+            assert_params_in_dict(ev, ["type", "origin_server_ts", "content", "sender"])
+
+            assert self.hs.is_mine_id(ev["sender"]), "User must be our own: %s" % (
+                ev["sender"],
+            )
+
+            event_dict = {
+                "type": ev["type"],
+                "origin_server_ts": ev["origin_server_ts"],
+                "content": ev["content"],
+                "room_id": room_id,
+                "sender": ev["sender"],  # requester.user.to_string(),
+                "prev_events": prev_event_ids.copy(),
+            }
+
+            # Mark all events as historical
+            event_dict["content"][EventContentFields.MSC2716_HISTORICAL] = True
+
+            event, unpersisted_context = await self.event_creation_handler.create_event(
+                await self.create_requester_for_user_id_from_app_service(
+                    ev["sender"], app_service_requester.app_service
+                ),
+                event_dict,
+                # Only the first event (which is the insertion event) in the
+                # chain should be floating. The rest should hang off each other
+                # in a chain.
+                allow_no_prev_events=index == 0,
+                prev_event_ids=event_dict.get("prev_events"),
+                # Since the first event (which is the insertion event) in the
+                # chain is floating with no `prev_events`, it can't derive state
+                # from anywhere automatically. So we need to set some state
+                # explicitly.
+                state_event_ids=initial_state_event_ids if index == 0 else None,
+                historical=True,
+                depth=inherited_depth,
+            )
+            context = await unpersisted_context.persist(event)
+            assert context._state_group
+
+            # Normally this is done when persisting the event but we have to
+            # pre-emptively do it here because we create all the events first,
+            # then persist them in another pass below. And we want to share
+            # state_groups across the whole batch so this lookup needs to work
+            # for the next event in the batch in this loop.
+            await self.store.store_state_group_id_for_event_id(
+                event_id=event.event_id,
+                state_group_id=context._state_group,
+            )
+
+            logger.debug(
+                "RoomBatchSendEventRestServlet inserting event=%s, prev_event_ids=%s",
+                event,
+                prev_event_ids,
+            )
+
+            events_to_persist.append((event, context))
+            event_id = event.event_id
+
+            event_ids.append(event_id)
+            prev_event_ids = [event_id]
+
+        # Persist events in reverse-chronological order so they have the
+        # correct stream_ordering as they are backfilled (which decrements).
+        # Events are sorted by (topological_ordering, stream_ordering)
+        # where topological_ordering is just depth.
+        for event, context in reversed(events_to_persist):
+            # This call can't raise `PartialStateConflictError` since we forbid
+            # use of the historical batch API during partial state
+            await self.event_creation_handler.handle_new_client_event(
+                await self.create_requester_for_user_id_from_app_service(
+                    event.sender, app_service_requester.app_service
+                ),
+                events_and_context=[(event, context)],
+            )
+
+        return event_ids
+
+    async def handle_batch_of_events(
+        self,
+        events_to_create: List[JsonDict],
+        room_id: str,
+        batch_id_to_connect_to: str,
+        inherited_depth: int,
+        initial_state_event_ids: List[str],
+        app_service_requester: Requester,
+    ) -> Tuple[List[str], str]:
+        """
+        Handles creating and persisting all of the historical events as well as
+        insertion and batch meta events to make the batch navigable in the DAG.
+
+        Args:
+            events_to_create: List of historical events to create in JSON
+                dictionary format.
+            room_id: Room where you want the events created in.
+            batch_id_to_connect_to: The batch_id from the insertion event you
+                want this batch to connect to.
+            inherited_depth: The depth to create the events at (you will
+                probably by calling inherit_depth_from_prev_ids(...)).
+            initial_state_event_ids:
+                This is used to set explicit state for the insertion event at
+                the start of the historical batch since it's floating with no
+                prev_events to derive state from automatically. This should
+                probably be the state from the `prev_event` defined by
+                `/batch_send?prev_event_id=$abc` plus the outcome of
+                `persist_state_events_at_start`
+            app_service_requester: The requester of an application service.
+
+        Returns:
+            Tuple containing a list of created events and the next_batch_id
+        """
+
+        # Connect this current batch to the insertion event from the previous batch
+        last_event_in_batch = events_to_create[-1]
+        batch_event = {
+            "type": EventTypes.MSC2716_BATCH,
+            "sender": app_service_requester.user.to_string(),
+            "room_id": room_id,
+            "content": {
+                EventContentFields.MSC2716_BATCH_ID: batch_id_to_connect_to,
+                EventContentFields.MSC2716_HISTORICAL: True,
+            },
+            # Since the batch event is put at the end of the batch,
+            # where the newest-in-time event is, copy the origin_server_ts from
+            # the last event we're inserting
+            "origin_server_ts": last_event_in_batch["origin_server_ts"],
+        }
+        # Add the batch event to the end of the batch (newest-in-time)
+        events_to_create.append(batch_event)
+
+        # Add an "insertion" event to the start of each batch (next to the oldest-in-time
+        # event in the batch) so the next batch can be connected to this one.
+        insertion_event = self.create_insertion_event_dict(
+            sender=app_service_requester.user.to_string(),
+            room_id=room_id,
+            # Since the insertion event is put at the start of the batch,
+            # where the oldest-in-time event is, copy the origin_server_ts from
+            # the first event we're inserting
+            origin_server_ts=events_to_create[0]["origin_server_ts"],
+        )
+        next_batch_id = insertion_event["content"][
+            EventContentFields.MSC2716_NEXT_BATCH_ID
+        ]
+        # Prepend the insertion event to the start of the batch (oldest-in-time)
+        events_to_create = [insertion_event] + events_to_create
+
+        # Create and persist all of the historical events
+        event_ids = await self.persist_historical_events(
+            events_to_create=events_to_create,
+            room_id=room_id,
+            inherited_depth=inherited_depth,
+            initial_state_event_ids=initial_state_event_ids,
+            app_service_requester=app_service_requester,
+        )
+
+        return event_ids, next_batch_id
@@ -362,6 +362,7 @@ class RoomMemberHandler(metaclass=abc.ABCMeta):
        content: Optional[dict] = None,
        require_consent: bool = True,
        outlier: bool = False,
+        historical: bool = False,
        origin_server_ts: Optional[int] = None,
    ) -> Tuple[str, int]:
        """
@@ -377,13 +378,16 @@ class RoomMemberHandler(metaclass=abc.ABCMeta):
            allow_no_prev_events: Whether to allow this event to be created an empty
                list of prev_events. Normally this is prohibited just because most
                events should have a prev_event and we should only use this in special
-                cases (previously useful for MSC2716).
+                cases like MSC2716.
            prev_event_ids: The event IDs to use as the prev events
            state_event_ids:
-                The full state at a given event. This was previously used particularly
-                by the MSC2716 /batch_send endpoint. This should normally be left as
-                None, which will cause the auth_event_ids to be calculated based on the
-                room state at the prev_events.
+                The full state at a given event. This is used particularly by the MSC2716
+                /batch_send endpoint. One use case is the historical `state_events_at_start`;
+                since each is marked as an `outlier`, the `EventContext.for_outlier()` won't
+                have any `state_ids` set and therefore can't derive any state even though the
+                prev_events are set so we need to set them ourself via this argument.
+                This should normally be left as None, which will cause the auth_event_ids
+                to be calculated based on the room state at the prev_events.
            depth: Override the depth used to order the event in the DAG.
                Should normally be set to None, which will cause the depth to be calculated
                based on the prev_events.
@@ -396,6 +400,9 @@ class RoomMemberHandler(metaclass=abc.ABCMeta):
            outlier: Indicates whether the event is an `outlier`, i.e. if
                it's from an arbitrary point and floating in the DAG as
                opposed to being inline with the current DAG.
+            historical: Indicates whether the message is being inserted
+                back in time around some existing events. This is used to skip
+                a few checks and mark the event as backfilled.
            origin_server_ts: The origin_server_ts to use if a new event is created. Uses
                the current timestamp if set to None.

@@ -470,6 +477,7 @@ class RoomMemberHandler(metaclass=abc.ABCMeta):
                    depth=depth,
                    require_consent=require_consent,
                    outlier=outlier,
+                    historical=historical,
                )
                context = await unpersisted_context.persist(event)
                prev_state_ids = await context.get_prev_state_ids(
@@ -577,6 +585,7 @@ class RoomMemberHandler(metaclass=abc.ABCMeta):
        new_room: bool = False,
        require_consent: bool = True,
        outlier: bool = False,
+        historical: bool = False,
        allow_no_prev_events: bool = False,
        prev_event_ids: Optional[List[str]] = None,
        state_event_ids: Optional[List[str]] = None,
@@ -601,16 +610,22 @@ class RoomMemberHandler(metaclass=abc.ABCMeta):
            outlier: Indicates whether the event is an `outlier`, i.e. if
                it's from an arbitrary point and floating in the DAG as
                opposed to being inline with the current DAG.
+            historical: Indicates whether the message is being inserted
+                back in time around some existing events. This is used to skip
+                a few checks and mark the event as backfilled.
            allow_no_prev_events: Whether to allow this event to be created an empty
                list of prev_events. Normally this is prohibited just because most
                events should have a prev_event and we should only use this in special
-                cases (previously useful for MSC2716).
+                cases like MSC2716.
            prev_event_ids: The event IDs to use as the prev events
            state_event_ids:
-                The full state at a given event. This was previously used particularly
-                by the MSC2716 /batch_send endpoint. This should normally be left as
-                None, which will cause the auth_event_ids to be calculated based on the
-                room state at the prev_events.
+                The full state at a given event. This is used particularly by the MSC2716
+                /batch_send endpoint. One use case is the historical `state_events_at_start`;
+                since each is marked as an `outlier`, the `EventContext.for_outlier()` won't
+                have any `state_ids` set and therefore can't derive any state even though the
+                prev_events are set so we need to set them ourself via this argument.
+                This should normally be left as None, which will cause the auth_event_ids
+                to be calculated based on the room state at the prev_events.
            depth: Override the depth used to order the event in the DAG.
                Should normally be set to None, which will cause the depth to be calculated
                based on the prev_events.
@@ -652,6 +667,7 @@ class RoomMemberHandler(metaclass=abc.ABCMeta):
                        new_room=new_room,
                        require_consent=require_consent,
                        outlier=outlier,
+                        historical=historical,
                        allow_no_prev_events=allow_no_prev_events,
                        prev_event_ids=prev_event_ids,
                        state_event_ids=state_event_ids,
@@ -675,6 +691,7 @@ class RoomMemberHandler(metaclass=abc.ABCMeta):
        new_room: bool = False,
        require_consent: bool = True,
        outlier: bool = False,
+        historical: bool = False,
        allow_no_prev_events: bool = False,
        prev_event_ids: Optional[List[str]] = None,
        state_event_ids: Optional[List[str]] = None,
@@ -701,16 +718,22 @@ class RoomMemberHandler(metaclass=abc.ABCMeta):
            outlier: Indicates whether the event is an `outlier`, i.e. if
                it's from an arbitrary point and floating in the DAG as
                opposed to being inline with the current DAG.
+            historical: Indicates whether the message is being inserted
+                back in time around some existing events. This is used to skip
+                a few checks and mark the event as backfilled.
            allow_no_prev_events: Whether to allow this event to be created an empty
                list of prev_events. Normally this is prohibited just because most
                events should have a prev_event and we should only use this in special
-                cases (previously useful for MSC2716).
+                cases like MSC2716.
            prev_event_ids: The event IDs to use as the prev events
            state_event_ids:
-                The full state at a given event. This was previously used particularly
-                by the MSC2716 /batch_send endpoint. This should normally be left as
-                None, which will cause the auth_event_ids to be calculated based on the
-                room state at the prev_events.
+                The full state at a given event. This is used particularly by the MSC2716
+                /batch_send endpoint. One use case is the historical `state_events_at_start`;
+                since each is marked as an `outlier`, the `EventContext.for_outlier()` won't
+                have any `state_ids` set and therefore can't derive any state even though the
+                prev_events are set so we need to set them ourself via this argument.
+                This should normally be left as None, which will cause the auth_event_ids
+                to be calculated based on the room state at the prev_events.
            depth: Override the depth used to order the event in the DAG.
                Should normally be set to None, which will cause the depth to be calculated
                based on the prev_events.
@@ -854,6 +877,7 @@ class RoomMemberHandler(metaclass=abc.ABCMeta):
                content=content,
                require_consent=require_consent,
                outlier=outlier,
+                historical=historical,
                origin_server_ts=origin_server_ts,
            )

@@ -1474,7 +1498,7 @@ class RoomMemberHandler(metaclass=abc.ABCMeta):
        # put the server which owns the alias at the front of the server list.
        if room_alias.domain in servers:
            servers.remove(room_alias.domain)
-            servers.insert(0, room_alias.domain)
+        servers.insert(0, room_alias.domain)

        return RoomID.from_string(room_id), servers

@@ -42,6 +42,7 @@ class StatsHandler:
        self.store = hs.get_datastores().main
        self._storage_controllers = hs.get_storage_controllers()
        self.state = hs.get_state_handler()
+        self.server_name = hs.hostname
        self.clock = hs.get_clock()
        self.notifier = hs.get_notifier()
        self.is_mine_id = hs.is_mine_id
@@ -51,10 +51,8 @@ logger = logging.getLogger(__name__)
@implementer(IAgent)
 class MatrixFederationAgent:
    """An Agent-like thing which provides a `request` method which correctly
-    handles resolving matrix server names when using `matrix-federation://`. Handles
-    standard https URIs as normal. The `matrix-federation://` scheme is internal to
-    Synapse and we purposely want to avoid colliding with the `matrix://` URL scheme
-    which is now specced.
+    handles resolving matrix server names when using matrix://. Handles standard
+    https URIs as normal.

    Doesn't implement any retries. (Those are done in MatrixFederationHttpClient.)

@@ -169,14 +167,14 @@ class MatrixFederationAgent:
        # There must be a valid hostname.
        assert parsed_uri.hostname

-        # If this is a matrix-federation:// URI check if the server has delegated matrix
+        # If this is a matrix:// URI check if the server has delegated matrix
        # traffic using well-known delegation.
        #
        # We have to do this here and not in the endpoint as we need to rewrite
        # the host header with the delegated server name.
        delegated_server = None
        if (
-            parsed_uri.scheme == b"matrix-federation"
+            parsed_uri.scheme == b"matrix"
            and not _is_ip_literal(parsed_uri.hostname)
            and not parsed_uri.port
        ):
@@ -252,7 +250,7 @@ class MatrixHostnameEndpointFactory:

@implementer(IStreamClientEndpoint)
 class MatrixHostnameEndpoint:
-    """An endpoint that resolves matrix-federation:// URLs using Matrix server name
+    """An endpoint that resolves matrix:// URLs using Matrix server name
    resolution (i.e. via SRV). Does not check for well-known delegation.

    Args:
@@ -381,7 +379,7 @@ class MatrixHostnameEndpoint:
        connect to.
        """

-        if self._parsed_uri.scheme != b"matrix-federation":
+        if self._parsed_uri.scheme != b"matrix":
            return [Server(host=self._parsed_uri.host, port=self._parsed_uri.port)]

        # Note: We don't do well-known lookup as that needs to have happened
@@ -95,6 +95,8 @@ incoming_responses_counter = Counter(
 )


+MAX_LONG_RETRIES = 10
+MAX_SHORT_RETRIES = 3
 MAXINT = sys.maxsize


@@ -172,14 +174,7 @@ class MatrixFederationRequest:

        # The object is frozen so we can pre-compute this.
        uri = urllib.parse.urlunparse(
-            (
-                b"matrix-federation",
-                destination_bytes,
-                path_bytes,
-                None,
-                query_bytes,
-                b"",
-            )
+            (b"matrix", destination_bytes, path_bytes, None, query_bytes, b"")
        )
        object.__setattr__(self, "uri", uri)

@@ -411,16 +406,7 @@ class MatrixFederationHttpClient:
        self.clock = hs.get_clock()
        self._store = hs.get_datastores().main
        self.version_string_bytes = hs.version_string.encode("ascii")
-        self.default_timeout_seconds = hs.config.federation.client_timeout_ms / 1000
-
-        self.max_long_retry_delay_seconds = (
-            hs.config.federation.max_long_retry_delay_ms / 1000
-        )
-        self.max_short_retry_delay_seconds = (
-            hs.config.federation.max_short_retry_delay_ms / 1000
-        )
-        self.max_long_retries = hs.config.federation.max_long_retries
-        self.max_short_retries = hs.config.federation.max_short_retries
+        self.default_timeout = 60

        self._cooperator = Cooperator(scheduler=_make_scheduler(self.reactor))

@@ -513,15 +499,8 @@ class MatrixFederationHttpClient:
                Note that the above intervals are *in addition* to the time spent
                waiting for the request to complete (up to `timeout` ms).

-                NB: the long retry algorithm takes over 20 minutes to complete, with a
-                default timeout of 60s! It's best not to use the `long_retries` option
-                for something that is blocking a client so we don't make them wait for
-                aaaaages, whereas some things like sending transactions (server to
-                server) we can be a lot more lenient but its very fuzzy / hand-wavey.
-
-                In the future, we could be more intelligent about doing this sort of
-                thing by looking at things with the bigger picture in mind,
-                https://github.com/matrix-org/synapse/issues/8917
+                NB: the long retry algorithm takes over 20 minutes to complete, with
+                a default timeout of 60s!

            ignore_backoff: true to ignore the historical backoff data
                and try the request anyway.
@@ -549,10 +528,10 @@ class MatrixFederationHttpClient:
            logger.exception(f"Invalid destination: {request.destination}.")
            raise FederationDeniedError(request.destination)

-        if timeout is not None:
+        if timeout:
            _sec_timeout = timeout / 1000
        else:
-            _sec_timeout = self.default_timeout_seconds
+            _sec_timeout = self.default_timeout

        if (
            self.hs.config.federation.federation_domain_whitelist is not None
@@ -597,9 +576,9 @@ class MatrixFederationHttpClient:
            # XXX: Would be much nicer to retry only at the transaction-layer
            # (once we have reliable transactions in place)
            if long_retries:
-                retries_left = self.max_long_retries
+                retries_left = MAX_LONG_RETRIES
            else:
-                retries_left = self.max_short_retries
+                retries_left = MAX_SHORT_RETRIES

            url_bytes = request.uri
            url_str = url_bytes.decode("ascii")
@@ -744,34 +723,24 @@ class MatrixFederationHttpClient:

                    if retries_left and not timeout:
                        if long_retries:
-                            delay_seconds = 4 ** (
-                                self.max_long_retries + 1 - retries_left
-                            )
-                            delay_seconds = min(
-                                delay_seconds, self.max_long_retry_delay_seconds
-                            )
-                            delay_seconds *= random.uniform(0.8, 1.4)
+                            delay = 4 ** (MAX_LONG_RETRIES + 1 - retries_left)
+                            delay = min(delay, 60)
+                            delay *= random.uniform(0.8, 1.4)
                        else:
-                            delay_seconds = 0.5 * 2 ** (
-                                self.max_short_retries - retries_left
-                            )
-                            delay_seconds = min(
-                                delay_seconds, self.max_short_retry_delay_seconds
-                            )
-                            delay_seconds *= random.uniform(0.8, 1.4)
+                            delay = 0.5 * 2 ** (MAX_SHORT_RETRIES - retries_left)
+                            delay = min(delay, 2)
+                            delay *= random.uniform(0.8, 1.4)

                        logger.debug(
                            "{%s} [%s] Waiting %ss before re-sending...",
                            request.txn_id,
                            request.destination,
-                            delay_seconds,
+                            delay,
                        )

                        # Sleep for the calculated delay, or wake up immediately
                        # if we get notified that the server is back up.
-                        await self._sleeper.sleep(
-                            request.destination, delay_seconds * 1000
-                        )
+                        await self._sleeper.sleep(request.destination, delay * 1000)
                        retries_left -= 1
                    else:
                        raise
@@ -970,7 +939,7 @@ class MatrixFederationHttpClient:
        if timeout is not None:
            _sec_timeout = timeout / 1000
        else:
-            _sec_timeout = self.default_timeout_seconds
+            _sec_timeout = self.default_timeout

        if parser is None:
            parser = cast(ByteParser[T], JsonParser())
@@ -1048,10 +1017,10 @@ class MatrixFederationHttpClient:
            ignore_backoff=ignore_backoff,
        )

-        if timeout is not None:
+        if timeout:
            _sec_timeout = timeout / 1000
        else:
-            _sec_timeout = self.default_timeout_seconds
+            _sec_timeout = self.default_timeout

        body = await _handle_response(
            self.reactor, _sec_timeout, request, response, start_ms, parser=JsonParser()
@@ -1159,7 +1128,7 @@ class MatrixFederationHttpClient:
        if timeout is not None:
            _sec_timeout = timeout / 1000
        else:
-            _sec_timeout = self.default_timeout_seconds
+            _sec_timeout = self.default_timeout

        if parser is None:
            parser = cast(ByteParser[T], JsonParser())
@@ -1235,7 +1204,7 @@ class MatrixFederationHttpClient:
        if timeout is not None:
            _sec_timeout = timeout / 1000
        else:
-            _sec_timeout = self.default_timeout_seconds
+            _sec_timeout = self.default_timeout

        body = await _handle_response(
            self.reactor, _sec_timeout, request, response, start_ms, parser=JsonParser()
@@ -1287,7 +1256,7 @@ class MatrixFederationHttpClient:

        try:
            d = read_body_with_max_size(response, output_stream, max_size)
-            d.addTimeout(self.default_timeout_seconds, self.reactor)
+            d.addTimeout(self.default_timeout, self.reactor)
            length = await make_deferred_yieldable(d)
        except BodyExceededMaxSize:
            msg = "Requested file is too large > %r bytes" % (max_size,)
@@ -76,7 +76,7 @@ class ReplicationEndpointFactory:
                endpoint = wrapClientTLS(
                    # The 'port' argument below isn't actually used by the function
                    self.context_factory.creatorForNetloc(
-                        self.instance_map[worker_name].host.encode("utf-8"),
+                        self.instance_map[worker_name].host,
                        self.instance_map[worker_name].port,
                    ),
                    endpoint,
@@ -521,11 +521,6 @@ class SynapseRequest(Request):
        else:
            return self.getClientAddress().host

-    def request_info(self) -> "RequestInfo":
-        h = self.getHeader(b"User-Agent")
-        user_agent = h.decode("ascii", "replace") if h else None
-        return RequestInfo(user_agent=user_agent, ip=self.get_client_ip_if_available())
-

 class XForwardedForRequest(SynapseRequest):
    """Request object which honours proxy headers
@@ -666,9 +661,3 @@ class SynapseSite(Site):

    def log(self, request: SynapseRequest) -> None:
        pass
-
-
-@attr.s(auto_attribs=True, frozen=True, slots=True)
-class RequestInfo:
-    user_agent: Optional[str]
-    ip: str
@@ -171,7 +171,6 @@ from functools import wraps
 from typing import (
    TYPE_CHECKING,
    Any,
-    Awaitable,
    Callable,
    Collection,
    ContextManager,
@@ -904,7 +903,6 @@ def _custom_sync_async_decorator(
    """

    if inspect.iscoroutinefunction(func):
-        # For this branch, we handle async functions like `async def func() -> RInner`.
        # In this branch, R = Awaitable[RInner], for some other type RInner
        @wraps(func)
        async def _wrapper(
@@ -916,16 +914,15 @@ def _custom_sync_async_decorator(
                return await func(*args, **kwargs)  # type: ignore[misc]

    else:
-        # The other case here handles sync functions including those decorated with
-        # `@defer.inlineCallbacks` or that return a `Deferred` or other `Awaitable`.
+        # The other case here handles both sync functions and those
+        # decorated with inlineDeferred.
        @wraps(func)
-        def _wrapper(*args: P.args, **kwargs: P.kwargs) -> Any:
+        def _wrapper(*args: P.args, **kwargs: P.kwargs) -> R:
            scope = wrapping_logic(func, *args, **kwargs)
            scope.__enter__()

            try:
                result = func(*args, **kwargs)
-
                if isinstance(result, defer.Deferred):

                    def call_back(result: R) -> R:
@@ -933,32 +930,20 @@ def _custom_sync_async_decorator(
                        return result

                    def err_back(result: R) -> R:
-                        # TODO: Pass the error details into `scope.__exit__(...)` for
-                        #       consistency with the other paths.
                        scope.__exit__(None, None, None)
                        return result

                    result.addCallbacks(call_back, err_back)

-                elif inspect.isawaitable(result):
-
-                    async def wrap_awaitable() -> Any:
-                        try:
-                            assert isinstance(result, Awaitable)
-                            awaited_result = await result
-                            scope.__exit__(None, None, None)
-                            return awaited_result
-                        except Exception as e:
-                            scope.__exit__(type(e), None, e.__traceback__)
-                            raise
-
-                    # The original method returned an awaitable, eg. a coroutine, so we
-                    # create another awaitable wrapping it that calls
-                    # `scope.__exit__(...)`.
-                    return wrap_awaitable()
                else:
-                    # Just a simple sync function so we can just exit the scope and
-                    # return the result without any fuss.
+                    if inspect.isawaitable(result):
+                        logger.error(
+                            "@trace may not have wrapped %s correctly! "
+                            "The function is not async but returned a %s.",
+                            func.__qualname__,
+                            type(result).__name__,
+                        )
+
                    scope.__exit__(None, None, None)

                return result
@@ -152,9 +152,6 @@ def add_file_headers(
        content_type = media_type

    request.setHeader(b"Content-Type", content_type.encode("UTF-8"))
-
-    # Use a Content-Disposition of attachment to force download of media.
-    disposition = "attachment"
    if upload_name:
        # RFC6266 section 4.1 [1] defines both `filename` and `filename*`.
        #
@@ -176,17 +173,11 @@ def add_file_headers(
        # correctly interpret those as of 0.99.2 and (b) they are a bit of a pain and we
        # may as well just do the filename* version.
        if _can_encode_filename_as_token(upload_name):
-            disposition = "%s; filename=%s" % (
-                disposition,
-                upload_name,
-            )
+            disposition = "inline; filename=%s" % (upload_name,)
        else:
-            disposition = "%s; filename*=utf-8''%s" % (
-                disposition,
-                _quote(upload_name),
-            )
+            disposition = "inline; filename*=utf-8''%s" % (_quote(upload_name),)

-    request.setHeader(b"Content-Disposition", disposition.encode("ascii"))
+        request.setHeader(b"Content-Disposition", disposition.encode("ascii"))

    # cache for at least a day.
    # XXX: we might want to turn this off for data we don't want to
@@ -77,8 +77,6 @@ RegistryProxy = cast(CollectorRegistry, _RegistryProxy)

@attr.s(slots=True, hash=True, auto_attribs=True)
 class LaterGauge(Collector):
-    """A Gauge which periodically calls a user-provided callback to produce metrics."""
-
    name: str
    desc: str
    labels: Optional[Sequence[str]] = attr.ib(hash=False)
@@ -80,7 +80,6 @@ from synapse.module_api.callbacks.account_validity_callbacks import (
 )
 from synapse.module_api.callbacks.spamchecker_callbacks import (
    CHECK_EVENT_FOR_SPAM_CALLBACK,
-    CHECK_LOGIN_FOR_SPAM_CALLBACK,
    CHECK_MEDIA_FILE_FOR_SPAM_CALLBACK,
    CHECK_REGISTRATION_FOR_SPAM_CALLBACK,
    CHECK_USERNAME_FOR_SPAM_CALLBACK,
@@ -303,7 +302,6 @@ class ModuleApi:
            CHECK_REGISTRATION_FOR_SPAM_CALLBACK
        ] = None,
        check_media_file_for_spam: Optional[CHECK_MEDIA_FILE_FOR_SPAM_CALLBACK] = None,
-        check_login_for_spam: Optional[CHECK_LOGIN_FOR_SPAM_CALLBACK] = None,
    ) -> None:
        """Registers callbacks for spam checking capabilities.

@@ -321,7 +319,6 @@ class ModuleApi:
            check_username_for_spam=check_username_for_spam,
            check_registration_for_spam=check_registration_for_spam,
            check_media_file_for_spam=check_media_file_for_spam,
-            check_login_for_spam=check_login_for_spam,
        )

    def register_account_validity_callbacks(
@@ -196,26 +196,6 @@ CHECK_MEDIA_FILE_FOR_SPAM_CALLBACK = Callable[
        ]
    ],
 ]
-CHECK_LOGIN_FOR_SPAM_CALLBACK = Callable[
-    [
-        str,
-        Optional[str],
-        Optional[str],
-        Collection[Tuple[Optional[str], str]],
-        Optional[str],
-    ],
-    Awaitable[
-        Union[
-            Literal["NOT_SPAM"],
-            Codes,
-            # Highly experimental, not officially part of the spamchecker API, may
-            # disappear without warning depending on the results of ongoing
-            # experiments.
-            # Use this to return additional information as part of an error.
-            Tuple[Codes, JsonDict],
-        ]
-    ],
-]


 def load_legacy_spam_checkers(hs: "synapse.server.HomeServer") -> None:
@@ -335,7 +315,6 @@ class SpamCheckerModuleApiCallbacks:
        self._check_media_file_for_spam_callbacks: List[
            CHECK_MEDIA_FILE_FOR_SPAM_CALLBACK
        ] = []
-        self._check_login_for_spam_callbacks: List[CHECK_LOGIN_FOR_SPAM_CALLBACK] = []

    def register_callbacks(
        self,
@@ -356,7 +335,6 @@ class SpamCheckerModuleApiCallbacks:
            CHECK_REGISTRATION_FOR_SPAM_CALLBACK
        ] = None,
        check_media_file_for_spam: Optional[CHECK_MEDIA_FILE_FOR_SPAM_CALLBACK] = None,
-        check_login_for_spam: Optional[CHECK_LOGIN_FOR_SPAM_CALLBACK] = None,
    ) -> None:
        """Register callbacks from module for each hook."""
        if check_event_for_spam is not None:
@@ -400,9 +378,6 @@ class SpamCheckerModuleApiCallbacks:
        if check_media_file_for_spam is not None:
            self._check_media_file_for_spam_callbacks.append(check_media_file_for_spam)

-        if check_login_for_spam is not None:
-            self._check_login_for_spam_callbacks.append(check_login_for_spam)
-
    @trace
    async def check_event_for_spam(
        self, event: "synapse.events.EventBase"
@@ -844,58 +819,3 @@ class SpamCheckerModuleApiCallbacks:
                    return synapse.api.errors.Codes.FORBIDDEN, {}

        return self.NOT_SPAM
-
-    async def check_login_for_spam(
-        self,
-        user_id: str,
-        device_id: Optional[str],
-        initial_display_name: Optional[str],
-        request_info: Collection[Tuple[Optional[str], str]],
-        auth_provider_id: Optional[str] = None,
-    ) -> Union[Tuple[Codes, dict], Literal["NOT_SPAM"]]:
-        """Checks if we should allow the given registration request.
-
-        Args:
-            user_id: The request user ID
-            request_info: List of tuples of user agent and IP that
-                were used during the registration process.
-            auth_provider_id: The SSO IdP the user used, e.g "oidc", "saml",
-                "cas". If any. Note this does not include users registered
-                via a password provider.
-
-        Returns:
-            Enum for how the request should be handled
-        """
-
-        for callback in self._check_login_for_spam_callbacks:
-            with Measure(
-                self.clock, "{}.{}".format(callback.__module__, callback.__qualname__)
-            ):
-                res = await delay_cancellation(
-                    callback(
-                        user_id,
-                        device_id,
-                        initial_display_name,
-                        request_info,
-                        auth_provider_id,
-                    )
-                )
-                # Normalize return values to `Codes` or `"NOT_SPAM"`.
-                if res is self.NOT_SPAM:
-                    continue
-                elif isinstance(res, synapse.api.errors.Codes):
-                    return res, {}
-                elif (
-                    isinstance(res, tuple)
-                    and len(res) == 2
-                    and isinstance(res[0], synapse.api.errors.Codes)
-                    and isinstance(res[1], dict)
-                ):
-                    return res
-                else:
-                    logger.warning(
-                        "Module returned invalid value, rejecting login as spam"
-                    )
-                    return synapse.api.errors.Codes.FORBIDDEN, {}
-
-        return self.NOT_SPAM
@@ -120,6 +120,9 @@ class BulkPushRuleEvaluator:
        self.should_calculate_push_rules = self.hs.config.push.enable_push

        self._related_event_match_enabled = self.hs.config.experimental.msc3664_enabled
+        self._intentional_mentions_enabled = (
+            self.hs.config.experimental.msc3952_intentional_mentions
+        )

        self.room_push_rule_cache_metrics = register_cache(
            "cache",
@@ -322,6 +325,7 @@ class BulkPushRuleEvaluator:
    ) -> None:
        if (
            not event.internal_metadata.is_notifiable()
+            or event.internal_metadata.is_historical()
            or event.room_id in self.hs.config.server.rooms_to_exclude_from_sync
        ):
            # Push rules for events that aren't notifiable can't be processed by this and
@@ -386,7 +390,10 @@ class BulkPushRuleEvaluator:
                        del notification_levels[key]

        # Pull out any user and room mentions.
-        has_mentions = EventContentFields.MENTIONS in event.content
+        has_mentions = (
+            self._intentional_mentions_enabled
+            and EventContentFields.MSC3952_MENTIONS in event.content
+        )

        evaluator = PushRuleEvaluator(
            _flatten_dict(event),
@@ -41,7 +41,12 @@ def format_push_rules_for_user(

        rulearray.append(template_rule)

-        _convert_type_to_value(template_rule, user)
+        for type_key in ("pattern", "value"):
+            type_value = template_rule.pop(f"{type_key}_type", None)
+            if type_value == "user_id":
+                template_rule[type_key] = user.to_string()
+            elif type_value == "user_localpart":
+                template_rule[type_key] = user.localpart

        template_rule["enabled"] = enabled

@@ -58,20 +63,19 @@ def format_push_rules_for_user(
        for c in template_rule["conditions"]:
            c.pop("_cache_key", None)

-            _convert_type_to_value(c, user)
+            pattern_type = c.pop("pattern_type", None)
+            if pattern_type == "user_id":
+                c["pattern"] = user.to_string()
+            elif pattern_type == "user_localpart":
+                c["pattern"] = user.localpart
+
+            sender_type = c.pop("sender_type", None)
+            if sender_type == "user_id":
+                c["sender"] = user.to_string()

    return rules


-def _convert_type_to_value(rule_or_cond: Dict[str, Any], user: UserID) -> None:
-    for type_key in ("pattern", "value"):
-        type_value = rule_or_cond.pop(f"{type_key}_type", None)
-        if type_value == "user_id":
-            rule_or_cond[type_key] = user.to_string()
-        elif type_value == "user_localpart":
-            rule_or_cond[type_key] = user.localpart
-
-
 def _add_empty_priority_class_arrays(d: Dict[str, list]) -> Dict[str, list]:
    for pc in PRIORITY_CLASS_MAP.keys():
        d[pc] = []
@@ -48,6 +48,7 @@ from synapse.rest.client import (
    rendezvous,
    report_event,
    room,
+    room_batch,
    room_keys,
    room_upgrade_rest_servlet,
    sendtodevice,
@@ -131,6 +132,7 @@ class ClientRestResource(JsonResource):
        user_directory.register_servlets(hs, client_resource)
        if is_main_process:
            room_upgrade_rest_servlet.register_servlets(hs, client_resource)
+        room_batch.register_servlets(hs, client_resource)
        capabilities.register_servlets(hs, client_resource)
        if is_main_process:
            account_validity.register_servlets(hs, client_resource)
@@ -50,7 +50,7 @@ from synapse.http.servlet import (
    parse_json_object_from_request,
    parse_string,
 )
-from synapse.http.site import RequestInfo, SynapseRequest
+from synapse.http.site import SynapseRequest
 from synapse.rest.client._base import client_patterns
 from synapse.rest.well_known import WellKnownBuilder
 from synapse.types import JsonDict, UserID
@@ -114,7 +114,6 @@ class LoginRestServlet(RestServlet):
        self.auth_handler = self.hs.get_auth_handler()
        self.registration_handler = hs.get_registration_handler()
        self._sso_handler = hs.get_sso_handler()
-        self._spam_checker = hs.get_module_api_callbacks().spam_checker

        self._well_known_builder = WellKnownBuilder(hs)
        self._address_ratelimiter = Ratelimiter(
@@ -198,8 +197,6 @@ class LoginRestServlet(RestServlet):
            self._refresh_tokens_enabled and client_requested_refresh_token
        )

-        request_info = request.request_info()
-
        try:
            if login_submission["type"] == LoginRestServlet.APPSERVICE_TYPE:
                requester = await self.auth.get_user_by_req(request)
@@ -219,7 +216,6 @@ class LoginRestServlet(RestServlet):
                    login_submission,
                    appservice,
                    should_issue_refresh_token=should_issue_refresh_token,
-                    request_info=request_info,
                )
            elif (
                self.jwt_enabled
@@ -231,7 +227,6 @@ class LoginRestServlet(RestServlet):
                result = await self._do_jwt_login(
                    login_submission,
                    should_issue_refresh_token=should_issue_refresh_token,
-                    request_info=request_info,
                )
            elif login_submission["type"] == LoginRestServlet.TOKEN_TYPE:
                await self._address_ratelimiter.ratelimit(
@@ -240,7 +235,6 @@ class LoginRestServlet(RestServlet):
                result = await self._do_token_login(
                    login_submission,
                    should_issue_refresh_token=should_issue_refresh_token,
-                    request_info=request_info,
                )
            else:
                await self._address_ratelimiter.ratelimit(
@@ -249,7 +243,6 @@ class LoginRestServlet(RestServlet):
                result = await self._do_other_login(
                    login_submission,
                    should_issue_refresh_token=should_issue_refresh_token,
-                    request_info=request_info,
                )
        except KeyError:
            raise SynapseError(400, "Missing JSON keys.")
@@ -272,8 +265,6 @@ class LoginRestServlet(RestServlet):
        login_submission: JsonDict,
        appservice: ApplicationService,
        should_issue_refresh_token: bool = False,
-        *,
-        request_info: RequestInfo,
    ) -> LoginResponse:
        identifier = login_submission.get("identifier")
        logger.info("Got appservice login request with identifier: %r", identifier)
@@ -309,15 +300,10 @@ class LoginRestServlet(RestServlet):
            # The user represented by an appservice's configured sender_localpart
            # is not actually created in Synapse.
            should_check_deactivated=qualified_user_id != appservice.sender,
-            request_info=request_info,
        )

    async def _do_other_login(
-        self,
-        login_submission: JsonDict,
-        should_issue_refresh_token: bool = False,
-        *,
-        request_info: RequestInfo,
+        self, login_submission: JsonDict, should_issue_refresh_token: bool = False
    ) -> LoginResponse:
        """Handle non-token/saml/jwt logins

@@ -347,7 +333,6 @@ class LoginRestServlet(RestServlet):
            login_submission,
            callback,
            should_issue_refresh_token=should_issue_refresh_token,
-            request_info=request_info,
        )
        return result

@@ -362,8 +347,6 @@ class LoginRestServlet(RestServlet):
        should_issue_refresh_token: bool = False,
        auth_provider_session_id: Optional[str] = None,
        should_check_deactivated: bool = True,
-        *,
-        request_info: RequestInfo,
    ) -> LoginResponse:
        """Called when we've successfully authed the user and now need to
        actually login them in (e.g. create devices). This gets called on
@@ -388,7 +371,6 @@ class LoginRestServlet(RestServlet):

                This exists purely for appservice's configured sender_localpart
                which doesn't have an associated user in the database.
-            request_info: The user agent/IP address of the user.

        Returns:
            Dictionary of account information after successful login.
@@ -435,22 +417,6 @@ class LoginRestServlet(RestServlet):
                )

        initial_display_name = login_submission.get("initial_device_display_name")
-        spam_check = await self._spam_checker.check_login_for_spam(
-            user_id,
-            device_id=device_id,
-            initial_display_name=initial_display_name,
-            request_info=[(request_info.user_agent, request_info.ip)],
-            auth_provider_id=auth_provider_id,
-        )
-        if spam_check != self._spam_checker.NOT_SPAM:
-            logger.info("Blocking login due to spam checker")
-            raise SynapseError(
-                403,
-                msg="Login was blocked by the server",
-                errcode=spam_check[0],
-                additional_fields=spam_check[1],
-            )
-
        (
            device_id,
            access_token,
@@ -485,11 +451,7 @@ class LoginRestServlet(RestServlet):
        return result

    async def _do_token_login(
-        self,
-        login_submission: JsonDict,
-        should_issue_refresh_token: bool = False,
-        *,
-        request_info: RequestInfo,
+        self, login_submission: JsonDict, should_issue_refresh_token: bool = False
    ) -> LoginResponse:
        """
        Handle token login.
@@ -512,15 +474,10 @@ class LoginRestServlet(RestServlet):
            auth_provider_id=res.auth_provider_id,
            should_issue_refresh_token=should_issue_refresh_token,
            auth_provider_session_id=res.auth_provider_session_id,
-            request_info=request_info,
        )

    async def _do_jwt_login(
-        self,
-        login_submission: JsonDict,
-        should_issue_refresh_token: bool = False,
-        *,
-        request_info: RequestInfo,
+        self, login_submission: JsonDict, should_issue_refresh_token: bool = False
    ) -> LoginResponse:
        """
        Handle the custom JWT login.
@@ -539,7 +496,6 @@ class LoginRestServlet(RestServlet):
            login_submission,
            create_non_existent_users=True,
            should_issue_refresh_token=should_issue_refresh_token,
-            request_info=request_info,
        )


@@ -0,0 +1,254 @@
+# Copyright 2016 OpenMarket Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import logging
+import re
+from http import HTTPStatus
+from typing import TYPE_CHECKING, Tuple
+
+from synapse.api.constants import EventContentFields
+from synapse.api.errors import AuthError, Codes, SynapseError
+from synapse.http.server import HttpServer
+from synapse.http.servlet import (
+    RestServlet,
+    assert_params_in_dict,
+    parse_json_object_from_request,
+    parse_string,
+    parse_strings_from_args,
+)
+from synapse.http.site import SynapseRequest
+from synapse.types import JsonDict
+
+if TYPE_CHECKING:
+    from synapse.server import HomeServer
+
+logger = logging.getLogger(__name__)
+
+
+class RoomBatchSendEventRestServlet(RestServlet):
+    """
+    API endpoint which can insert a batch of events historically back in time
+    next to the given `prev_event`.
+
+    `batch_id` comes from `next_batch_id `in the response of the batch send
+    endpoint and is derived from the "insertion" events added to each batch.
+    It's not required for the first batch send.
+
+    `state_events_at_start` is used to define the historical state events
+    needed to auth the events like join events. These events will float
+    outside of the normal DAG as outlier's and won't be visible in the chat
+    history which also allows us to insert multiple batches without having a bunch
+    of `@mxid joined the room` noise between each batch.
+
+    `events` is chronological list of events you want to insert.
+    There is a reverse-chronological constraint on batches so once you insert
+    some messages, you can only insert older ones after that.
+    tldr; Insert batches from your most recent history -> oldest history.
+
+    POST /_matrix/client/unstable/org.matrix.msc2716/rooms/<roomID>/batch_send?prev_event_id=<eventID>&batch_id=<batchID>
+    {
+        "events": [ ... ],
+        "state_events_at_start": [ ... ]
+    }
+    """
+
+    PATTERNS = (
+        re.compile(
+            "^/_matrix/client/unstable/org.matrix.msc2716"
+            "/rooms/(?P<room_id>[^/]*)/batch_send$"
+        ),
+    )
+    CATEGORY = "Client API requests"
+
+    def __init__(self, hs: "HomeServer"):
+        super().__init__()
+        self.store = hs.get_datastores().main
+        self.event_creation_handler = hs.get_event_creation_handler()
+        self.auth = hs.get_auth()
+        self.room_batch_handler = hs.get_room_batch_handler()
+
+    async def on_POST(
+        self, request: SynapseRequest, room_id: str
+    ) -> Tuple[int, JsonDict]:
+        requester = await self.auth.get_user_by_req(request, allow_guest=False)
+
+        if not requester.app_service:
+            raise AuthError(
+                HTTPStatus.FORBIDDEN,
+                "Only application services can use the /batchsend endpoint",
+            )
+
+        body = parse_json_object_from_request(request)
+        assert_params_in_dict(body, ["state_events_at_start", "events"])
+
+        assert request.args is not None
+        prev_event_ids_from_query = parse_strings_from_args(
+            request.args, "prev_event_id"
+        )
+        batch_id_from_query = parse_string(request, "batch_id")
+
+        if prev_event_ids_from_query is None:
+            raise SynapseError(
+                HTTPStatus.BAD_REQUEST,
+                "prev_event query parameter is required when inserting historical messages back in time",
+                errcode=Codes.MISSING_PARAM,
+            )
+
+        if await self.store.is_partial_state_room(room_id):
+            raise SynapseError(
+                HTTPStatus.BAD_REQUEST,
+                "Cannot insert history batches until we have fully joined the room",
+                errcode=Codes.UNABLE_DUE_TO_PARTIAL_STATE,
+            )
+
+        # Verify the batch_id_from_query corresponds to an actual insertion event
+        # and have the batch connected.
+        if batch_id_from_query:
+            corresponding_insertion_event_id = (
+                await self.store.get_insertion_event_id_by_batch_id(
+                    room_id, batch_id_from_query
+                )
+            )
+            if corresponding_insertion_event_id is None:
+                raise SynapseError(
+                    HTTPStatus.BAD_REQUEST,
+                    "No insertion event corresponds to the given ?batch_id",
+                    errcode=Codes.INVALID_PARAM,
+                )
+
+        # Make sure that the prev_event_ids exist and aren't outliers - ie, they are
+        # regular parts of the room DAG where we know the state.
+        non_outlier_prev_events = await self.store.have_events_in_timeline(
+            prev_event_ids_from_query
+        )
+        for prev_event_id in prev_event_ids_from_query:
+            if prev_event_id not in non_outlier_prev_events:
+                raise SynapseError(
+                    HTTPStatus.BAD_REQUEST,
+                    "prev_event %s does not exist, or is an outlier" % (prev_event_id,),
+                    errcode=Codes.INVALID_PARAM,
+                )
+
+        # For the event we are inserting next to (`prev_event_ids_from_query`),
+        # find the most recent state events that allowed that message to be
+        # sent. We will use that as a base to auth our historical messages
+        # against.
+        state_event_ids = await self.room_batch_handler.get_most_recent_full_state_ids_from_event_id_list(
+            prev_event_ids_from_query
+        )
+
+        state_event_ids_at_start = []
+        # Create and persist all of the state events that float off on their own
+        # before the batch. These will most likely be all of the invite/member
+        # state events used to auth the upcoming historical messages.
+        if body["state_events_at_start"]:
+            state_event_ids_at_start = (
+                await self.room_batch_handler.persist_state_events_at_start(
+                    state_events_at_start=body["state_events_at_start"],
+                    room_id=room_id,
+                    initial_state_event_ids=state_event_ids,
+                    app_service_requester=requester,
+                )
+            )
+            # Update our ongoing auth event ID list with all of the new state we
+            # just created
+            state_event_ids.extend(state_event_ids_at_start)
+
+        inherited_depth = await self.room_batch_handler.inherit_depth_from_prev_ids(
+            prev_event_ids_from_query
+        )
+
+        events_to_create = body["events"]
+
+        # Figure out which batch to connect to. If they passed in
+        # batch_id_from_query let's use it. The batch ID passed in comes
+        # from the batch_id in the "insertion" event from the previous batch.
+        last_event_in_batch = events_to_create[-1]
+        base_insertion_event = None
+        if batch_id_from_query:
+            batch_id_to_connect_to = batch_id_from_query
+        # Otherwise, create an insertion event to act as a starting point.
+        #
+        # We don't always have an insertion event to start hanging more history
+        # off of (ideally there would be one in the main DAG, but that's not the
+        # case if we're wanting to add history to e.g. existing rooms without
+        # an insertion event), in which case we just create a new insertion event
+        # that can then get pointed to by a "marker" event later.
+        else:
+            base_insertion_event_dict = (
+                self.room_batch_handler.create_insertion_event_dict(
+                    sender=requester.user.to_string(),
+                    room_id=room_id,
+                    origin_server_ts=last_event_in_batch["origin_server_ts"],
+                )
+            )
+            base_insertion_event_dict["prev_events"] = prev_event_ids_from_query.copy()
+
+            (
+                base_insertion_event,
+                _,
+            ) = await self.event_creation_handler.create_and_send_nonmember_event(
+                await self.room_batch_handler.create_requester_for_user_id_from_app_service(
+                    base_insertion_event_dict["sender"],
+                    requester.app_service,
+                ),
+                base_insertion_event_dict,
+                prev_event_ids=base_insertion_event_dict.get("prev_events"),
+                # Also set the explicit state here because we want to resolve
+                # any `state_events_at_start` here too. It's not strictly
+                # necessary to accomplish anything but if someone asks for the
+                # state at this point, we probably want to show them the
+                # historical state that was part of this batch.
+                state_event_ids=state_event_ids,
+                historical=True,
+                depth=inherited_depth,
+            )
+
+            batch_id_to_connect_to = base_insertion_event.content[
+                EventContentFields.MSC2716_NEXT_BATCH_ID
+            ]
+
+        # Create and persist all of the historical events as well as insertion
+        # and batch meta events to make the batch navigable in the DAG.
+        event_ids, next_batch_id = await self.room_batch_handler.handle_batch_of_events(
+            events_to_create=events_to_create,
+            room_id=room_id,
+            batch_id_to_connect_to=batch_id_to_connect_to,
+            inherited_depth=inherited_depth,
+            initial_state_event_ids=state_event_ids,
+            app_service_requester=requester,
+        )
+
+        insertion_event_id = event_ids[0]
+        batch_event_id = event_ids[-1]
+        historical_event_ids = event_ids[1:-1]
+
+        response_dict = {
+            "state_event_ids": state_event_ids_at_start,
+            "event_ids": historical_event_ids,
+            "next_batch_id": next_batch_id,
+            "insertion_event_id": insertion_event_id,
+            "batch_event_id": batch_event_id,
+        }
+        if base_insertion_event is not None:
+            response_dict["base_insertion_event_id"] = base_insertion_event.event_id
+
+        return HTTPStatus.OK, response_dict
+
+
+def register_servlets(hs: "HomeServer", http_server: HttpServer) -> None:
+    msc2716_enabled = hs.config.experimental.msc2716_enabled
+
+    if msc2716_enabled:
+        RoomBatchSendEventRestServlet(hs).register(http_server)
--- a/Show More
+++ b/Show More
Author	SHA1	Message	Date
Hugh Nimmo-Smith	327555dddf	Fix other test cases	2023-06-05 18:02:20 +01:00
Hugh Nimmo-Smith	fba9000152	Changelog	2023-06-05 14:45:29 +01:00
Hugh Nimmo-Smith	2eb74c6bdb	Enable login_via_existing_session by default	2023-06-05 14:42:31 +01:00
				`@@ -0,0 +1 @@`
				`Log when events are (maybe unexpectedly) filtered out of responses in tests.`
				`@@ -1 +0,0 @@`
				Replace `EventContext` fields `prev_group` and `delta_ids` with field `state_group_deltas`.
				`@@ -0,0 +1 @@`
				`Stable support for [MSC3882](https://github.com/matrix-org/matrix-spec-proposals/pull/3882) to allow an existing device/session to generate a login token for use on a new device/session.`
				`@@ -0,0 +1 @@`
				`Support resolving a room's [canonical alias](https://spec.matrix.org/v1.7/client-server-api/#mroomcanonical_alias) via the module API.`
				`@@ -0,0 +1 @@`
				`Experimental [MSC3861](https://github.com/matrix-org/matrix-spec-proposals/pull/3861) support: delegate auth to an OIDC provider.`
				`@@ -0,0 +1 @@`
				Read from column `full_user_id` rather than `user_id` of tables `profiles` and `user_filters`.
				`@@ -0,0 +1 @@`
				`Add Syanpse version deploy annotations to Grafana dashboard which enables easy correlation between behavior changes witnessed in a graph to a certain Synapse version and nail down regressions.`
				`@@ -0,0 +1 @@`
				`Cache requests for user's devices over federation.`
				`@@ -1 +0,0 @@`
				`Fix a long-standing bug where media files were served in an unsafe manner. Contributed by @joshqou.`