Lint

inductive sync state WIP
Test cases for sync of state from DAG branches
2024-10-25 17:39:55 +01:00 · 2024-10-25 17:31:06 +01:00 · 2024-10-25 16:13:46 +01:00
116 changed files with 863 additions and 2479 deletions
@@ -36,11 +36,11 @@ IS_PR = os.environ["GITHUB_REF"].startswith("refs/pull/")
 # First calculate the various trial jobs.
 #
 # For PRs, we only run each type of test with the oldest Python version supported (which
-# is Python 3.9 right now)
+# is Python 3.8 right now)

 trial_sqlite_tests = [
    {
-        "python-version": "3.9",
+        "python-version": "3.8",
        "database": "sqlite",
        "extras": "all",
    }
@@ -53,12 +53,12 @@ if not IS_PR:
            "database": "sqlite",
            "extras": "all",
        }
-        for version in ("3.10", "3.11", "3.12", "3.13")
+        for version in ("3.9", "3.10", "3.11", "3.12", "3.13")
    )

 trial_postgres_tests = [
    {
-        "python-version": "3.9",
+        "python-version": "3.8",
        "database": "postgres",
        "postgres-version": "11",
        "extras": "all",
@@ -77,7 +77,7 @@ if not IS_PR:

 trial_no_extra_tests = [
    {
-        "python-version": "3.9",
+        "python-version": "3.8",
        "database": "sqlite",
        "extras": "",
    }
@@ -99,24 +99,24 @@ set_output("trial_test_matrix", test_matrix)

 # First calculate the various sytest jobs.
 #
-# For each type of test we only run on bullseye on PRs
+# For each type of test we only run on focal on PRs


 sytest_tests = [
    {
-        "sytest-tag": "bullseye",
+        "sytest-tag": "focal",
    },
    {
-        "sytest-tag": "bullseye",
+        "sytest-tag": "focal",
        "postgres": "postgres",
    },
    {
-        "sytest-tag": "bullseye",
+        "sytest-tag": "focal",
        "postgres": "multi-postgres",
        "workers": "workers",
    },
    {
-        "sytest-tag": "bullseye",
+        "sytest-tag": "focal",
        "postgres": "multi-postgres",
        "workers": "workers",
        "reactor": "asyncio",
@@ -127,11 +127,11 @@ if not IS_PR:
    sytest_tests.extend(
        [
            {
-                "sytest-tag": "bullseye",
+                "sytest-tag": "focal",
                "reactor": "asyncio",
            },
            {
-                "sytest-tag": "bullseye",
+                "sytest-tag": "focal",
                "postgres": "postgres",
                "reactor": "asyncio",
            },
@@ -1,5 +1,5 @@
 #!/usr/bin/env bash
-# this script is run by GitHub Actions in a plain `jammy` container; it
+# this script is run by GitHub Actions in a plain `focal` container; it
 # - installs the minimal system requirements, and poetry;
 # - patches the project definition file to refer to old versions only;
 # - creates a venv with these old versions using poetry; and finally
@@ -132,9 +132,9 @@ jobs:
      fail-fast: false
      matrix:
        include:
-          - sytest-tag: bullseye
+          - sytest-tag: focal

-          - sytest-tag: bullseye
+          - sytest-tag: focal
            postgres: postgres
            workers: workers
            redis: redis
@@ -92,7 +92,7 @@ jobs:
          mv /tmp/.buildx-cache-new /tmp/.buildx-cache

      - name: Upload debs as artifacts
-        uses: actions/upload-artifact@v4
+        uses: actions/upload-artifact@v3 # Don't upgrade to v4; broken: https://github.com/actions/upload-artifact#breaking-changes
        with:
          name: debs
          path: debs/*
@@ -102,7 +102,7 @@ jobs:
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
-        os: [ubuntu-22.04, macos-12]
+        os: [ubuntu-20.04, macos-12]
        arch: [x86_64, aarch64]
        # is_pr is a flag used to exclude certain jobs from the matrix on PRs.
        # It is not read by the rest of the workflow.
@@ -144,7 +144,7 @@ jobs:

      - name: Only build a single wheel on PR
        if: startsWith(github.ref, 'refs/pull/')
-        run: echo "CIBW_BUILD="cp39-manylinux_${{ matrix.arch }}"" >> $GITHUB_ENV
+        run: echo "CIBW_BUILD="cp38-manylinux_${{ matrix.arch }}"" >> $GITHUB_ENV

      - name: Build wheels
        run: python -m cibuildwheel --output-dir wheelhouse
@@ -156,9 +156,9 @@ jobs:
          CARGO_NET_GIT_FETCH_WITH_CLI: true
          CIBW_ENVIRONMENT_PASS_LINUX: CARGO_NET_GIT_FETCH_WITH_CLI

-      - uses: actions/upload-artifact@v4
+      - uses: actions/upload-artifact@v3 # Don't upgrade to v4; broken: https://github.com/actions/upload-artifact#breaking-changes
        with:
-          name: Wheel-${{ matrix.os }}-${{ matrix.arch }}
+          name: Wheel
          path: ./wheelhouse/*.whl

  build-sdist:
@@ -177,7 +177,7 @@ jobs:
      - name: Build sdist
        run: python -m build --sdist

-      - uses: actions/upload-artifact@v4
+      - uses: actions/upload-artifact@v3 # Don't upgrade to v4; broken: https://github.com/actions/upload-artifact#breaking-changes
        with:
          name: Sdist
          path: dist/*.tar.gz
@@ -194,7 +194,7 @@ jobs:
    runs-on: ubuntu-latest
    steps:
      - name: Download all workflow run artifacts
-        uses: actions/download-artifact@v4
+        uses: actions/download-artifact@v3 # Don't upgrade to v4, it should match upload-artifact
      - name: Build a tarball for the debs
        run: tar -cvJf debs.tar.xz debs
      - name: Attach to release
@@ -397,7 +397,7 @@ jobs:
    needs:
      - linting-done
      - changes
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-20.04
    steps:
      - uses: actions/checkout@v4

@@ -409,12 +409,12 @@ jobs:
      # their build dependencies
      - run: |
          sudo apt-get -qq update
-          sudo apt-get -qq install build-essential libffi-dev python3-dev \
+          sudo apt-get -qq install build-essential libffi-dev python-dev \
            libxml2-dev libxslt-dev xmlsec1 zlib1g-dev libjpeg-dev libwebp-dev

      - uses: actions/setup-python@v5
        with:
-          python-version: '3.9'
+          python-version: '3.8'

      - name: Prepare old deps
        if: steps.cache-poetry-old-deps.outputs.cache-hit != 'true'
@@ -458,7 +458,7 @@ jobs:
    runs-on: ubuntu-latest
    strategy:
      matrix:
-        python-version: ["pypy-3.9"]
+        python-version: ["pypy-3.8"]
        extras: ["all"]

    steps:
@@ -580,11 +580,11 @@ jobs:
    strategy:
      matrix:
        include:
-          - python-version: "3.9"
+          - python-version: "3.8"
            postgres-version: "11"

-          - python-version: "3.13"
-            postgres-version: "17"
+          - python-version: "3.11"
+            postgres-version: "15"

    services:
      postgres:
@@ -99,11 +99,11 @@ jobs:
    if: needs.check_repo.outputs.should_run_workflow == 'true'
    runs-on: ubuntu-latest
    container:
-      # We're using debian:bullseye because it uses Python 3.9 which is our minimum supported Python version.
+      # We're using ubuntu:focal because it uses Python 3.8 which is our minimum supported Python version.
      # This job is a canary to warn us about unreleased twisted changes that would cause problems for us if
      # they were to be released immediately. For simplicity's sake (and to save CI runners) we use the oldest
      # version, assuming that any incompatibilities on newer versions would also be present on the oldest.
-      image: matrixdotorg/sytest-synapse:bullseye
+      image: matrixdotorg/sytest-synapse:focal
      volumes:
        - ${{ github.workspace }}:/src

@@ -1,63 +1,3 @@
-# Synapse 1.118.0 (2024-10-29)
-
-No significant changes since 1.118.0rc1.
-
-### Python 3.8 support will be dropped in the next release
-
-Python 3.8 is now [end-of-life](https://devguide.python.org/versions/). As per our [Deprecation Policy for Platform Dependencies](https://element-hq.github.io/synapse/latest/deprecation_policy.html#policy), Synapse will be dropping support for Python 3.8 in the next release; Synapse 1.119.0.
-
-Synapse 1.118.x will be the final release to support Python 3.8. If you are running Synapse with Python 3.8, please upgrade before the 1.119.0 release, due in less than one month.
-
-### Python 3.13 and PostgreSQL 17 support
-
-On the other end of the spectrum, Synapse 1.118.0 is the first release to support [Python 3.13](https://www.python.org/downloads/release/python-3130/)! [PostgreSQL 17](https://www.postgresql.org/about/news/postgresql-17-released-2936/) is also supported as of this release.
-
-
-# Synapse 1.118.0rc1 (2024-10-22)
-
-### Features
-
- Added the `display_name_claim` option to the JWT configuration. This option allows specifying the claim key that contains the user's display name in the JWT payload. ([\#17708](https://github.com/element-hq/synapse/issues/17708))
- Implement [MSC4210](https://github.com/matrix-org/matrix-spec-proposals/pull/4210): Remove legacy mentions. Contributed by @tulir @ Beeper. ([\#17783](https://github.com/element-hq/synapse/issues/17783))
-
-### Bugfixes
-
- Fix saving of PNG thumbnails, when the original image is in the CMYK color space. ([\#17736](https://github.com/element-hq/synapse/issues/17736))
- Fix bug with sliding sync where the server would not return state that was added to the `required_state` config. ([\#17785](https://github.com/element-hq/synapse/issues/17785), [\#17805](https://github.com/element-hq/synapse/issues/17805))
- Fix a bug in [MSC4186](https://github.com/matrix-org/matrix-spec-proposals/pull/4186) Sliding Sync that would cause rooms to stay forgotten and hidden even after rejoining. ([\#17835](https://github.com/element-hq/synapse/issues/17835))
-
-### Improved Documentation
-
- Clarify when the `user_may_invite` and `user_may_send_3pid_invite` module callbacks are called. ([\#17627](https://github.com/element-hq/synapse/issues/17627))
- Correct documentation to refer to the `--config-path` argument instead of `--config-file`. ([\#17802](https://github.com/element-hq/synapse/issues/17802))
- Fix typo in `target_cache_memory_usage` docs. ([\#17825](https://github.com/element-hq/synapse/issues/17825))
-
-### Internal Changes
-
- Slight optimization when fetching state/events for Sliding Sync. ([\#17718](https://github.com/element-hq/synapse/issues/17718))
- Add Python 3.13 and Postgres 17 to the test matrix. ([\#17752](https://github.com/element-hq/synapse/issues/17752))
- Test github token before running release script steps. ([\#17803](https://github.com/element-hq/synapse/issues/17803))
- Build debian packages for new Ubuntu versions, and stop building for no longer supported versions. ([\#17824](https://github.com/element-hq/synapse/issues/17824))
- Enable the `.org.matrix.msc4028.encrypted_event` push rule by default in accordance with [MSC4028](https://github.com/matrix-org/matrix-spec-proposals/pull/4028). Note that the corresponding experimental feature must still be switched on for this push rule to have any effect. ([\#17826](https://github.com/element-hq/synapse/issues/17826))
- Fix some typing issues uncovered by upgrading mypy to 1.11.x. ([\#17842](https://github.com/element-hq/synapse/issues/17842))
-
-
-
-### Updates to locked dependencies
-
-* Bump mypy from 1.10.1 to 1.11.2. ([\#17842](https://github.com/element-hq/synapse/issues/17842))
-* Bump mypy-zope from 1.0.5 to 1.0.7. ([\#17827](https://github.com/element-hq/synapse/issues/17827))
-* Bump phonenumbers from 8.13.46 to 8.13.47. ([\#17797](https://github.com/element-hq/synapse/issues/17797))
-* Bump psycopg2 from 2.9.9 to 2.9.10. ([\#17843](https://github.com/element-hq/synapse/issues/17843))
-* Bump ruff from 0.6.8 to 0.6.9. ([\#17794](https://github.com/element-hq/synapse/issues/17794))
-* Bump sentry-sdk from 2.14.0 to 2.15.0. ([\#17795](https://github.com/element-hq/synapse/issues/17795))
-* Bump sentry-sdk from 2.15.0 to 2.16.0. ([\#17829](https://github.com/element-hq/synapse/issues/17829))
-* Bump sentry-sdk from 2.16.0 to 2.17.0. ([\#17844](https://github.com/element-hq/synapse/issues/17844))
-* Bump sigstore/cosign-installer from 3.6.0 to 3.7.0. ([\#17798](https://github.com/element-hq/synapse/issues/17798))
-* Bump tomli from 2.0.1 to 2.0.2. ([\#17796](https://github.com/element-hq/synapse/issues/17796))
-* Bump types-requests from 2.32.0.20240914 to 2.32.0.20241016. ([\#17841](https://github.com/element-hq/synapse/issues/17841))
-* Bump types-setuptools from 75.1.0.20240917 to 75.1.0.20241014. ([\#17828](https://github.com/element-hq/synapse/issues/17828))
-
 # Synapse 1.117.0 (2024-10-15)

 No significant changes since 1.117.0rc1.
@@ -13,9 +13,9 @@ dependencies = [

 [[package]]
 name = "anyhow"
-version = "1.0.92"
+version = "1.0.90"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "74f37166d7d48a0284b99dd824694c26119c700b53bf0d1540cdb147dbdaaf13"
+checksum = "37bf3594c4c988a53154954629820791dde498571819ae4ca50ca811e060cc95"

 [[package]]
 name = "arc-swap"
@@ -67,9 +67,9 @@ checksum = "79296716171880943b8470b5f8d03aa55eb2e645a4874bdbb28adb49162e012c"

 [[package]]
 name = "bytes"
-version = "1.8.0"
+version = "1.7.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9ac0150caa2ae65ca5bd83f25c7de183dea78d4d366469f148435e2acfbad0da"
+checksum = "428d9aa8fbc0670b7b8d6030a7fadd0f86151cae55e4dbbece15f3780a3dfaf3"

 [[package]]
 name = "cfg-if"
@@ -302,9 +302,9 @@ checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de"

 [[package]]
 name = "proc-macro2"
-version = "1.0.89"
+version = "1.0.82"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f139b0662de085916d1fb67d2b4169d1addddda1919e696f3252b740b629986e"
+checksum = "8ad3d49ab951a01fbaafe34f2ec74122942fe18a3f9814c3268f1bb72042131b"
 dependencies = [
 "unicode-ident",
 ]
@@ -444,9 +444,9 @@ dependencies = [

 [[package]]
 name = "regex"
-version = "1.11.1"
+version = "1.11.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b544ef1b4eac5dc2db33ea63606ae9ffcfac26c1416a2806ae0bf5f56b201191"
+checksum = "38200e5ee88914975b69f657f0801b6f6dccafd44fd9326302a4aaeecfacb1d8"
 dependencies = [
 "aho-corasick",
 "memchr",
@@ -485,18 +485,18 @@ checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49"

 [[package]]
 name = "serde"
-version = "1.0.214"
+version = "1.0.210"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f55c3193aca71c12ad7890f1785d2b73e1b9f63a0bbc353c08ef26fe03fc56b5"
+checksum = "c8e3592472072e6e22e0a54d5904d9febf8508f65fb8552499a1abc7d1078c3a"
 dependencies = [
 "serde_derive",
 ]

 [[package]]
 name = "serde_derive"
-version = "1.0.214"
+version = "1.0.210"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "de523f781f095e28fa605cdce0f8307e451cc0fd14e2eb4cd2e98a355b147766"
+checksum = "243902eda00fad750862fc144cea25caca5e20d615af0a81bee94ca738f1df1f"
 dependencies = [
 "proc-macro2",
 "quote",
@@ -551,9 +551,9 @@ checksum = "81cdd64d312baedb58e21336b31bc043b77e01cc99033ce76ef539f78e965ebc"

 [[package]]
 name = "syn"
-version = "2.0.85"
+version = "2.0.61"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5023162dfcd14ef8f32034d8bcd4cc5ddc61ef7a247c024a33e24e1f24d21b56"
+checksum = "c993ed8ccba56ae856363b1845da7266a7cb78e1d146c8a32d54b45a8b831fc9"
 dependencies = [
 "proc-macro2",
 "quote",
@@ -1 +0,0 @@
-Support [MSC4151](https://github.com/matrix-org/matrix-spec-proposals/pull/4151)'s stable report room API.
@@ -0,0 +1 @@
+Clarify when the `user_may_invite` and `user_may_send_3pid_invite` module callbacks are called.
@@ -1 +0,0 @@
-Bump actions/download-artifact and actions/upload-artifact from v3 -> v4.
@@ -0,0 +1 @@
+Added the `display_name_claim` option to the JWT configuration. This option allows specifying the claim key that contains the user's display name in the JWT payload.
@@ -0,0 +1 @@
+Slight optimization when fetching state/events for Sliding Sync.
@@ -0,0 +1 @@
+Fix saving of PNG thumbnails, when the original image is in the CMYK color space.
@@ -0,0 +1 @@
+Add Python 3.13 and Postgres 17 to the test matrix.
@@ -0,0 +1 @@
+Implement [MSC4210](https://github.com/matrix-org/matrix-spec-proposals/pull/4210): Remove legacy mentions. Contributed by @tulir @ Beeper.
@@ -0,0 +1 @@
+Fix bug with sliding sync where the server would not return state that was added to the `required_state` config.
@@ -0,0 +1 @@
+Correct documentation to refer to the `--config-path` argument instead of `--config-file`.
@@ -0,0 +1 @@
+Test github token before running release script steps.
@@ -0,0 +1 @@
+Fix bug with sliding sync where the server would not return state that was added to the `required_state` config.
@@ -1 +0,0 @@
-Fix bug with sliding sync where `$LAZY`-loading room members would not return `required_state` membership in incremental syncs.
@@ -1 +0,0 @@
-Refactor database calls to remove `Generator` usage.
@@ -1 +0,0 @@
-Refactor database calls to remove `Generator` usage.
@@ -1 +0,0 @@
-Refactor database calls to remove `Generator` usage.
@@ -1 +0,0 @@
-Refactor database calls to remove `Generator` usage.
@@ -1 +0,0 @@
-Refactor database calls to remove `Generator` usage.
@@ -1 +0,0 @@
-Refactor database calls to remove `Generator` usage.
@@ -0,0 +1 @@
+Build debian packages for new Ubuntu versions, and stop building for no longer supported versions.
@@ -0,0 +1 @@
+Fix typo in `target_cache_memory_usage` docs.
@@ -0,0 +1 @@
+Enable the `.org.matrix.msc4028.encrypted_event` push rule by default in accordance with [MSC4028](https://github.com/matrix-org/matrix-spec-proposals/pull/4028). Note that the corresponding experimental feature must still be switched on for this push rule to have any effect.
@@ -1 +0,0 @@
-Include the destination in the error of 'Destination mismatch' on federation requests.
@@ -0,0 +1 @@
+Fix a bug in [MSC4186](https://github.com/matrix-org/matrix-spec-proposals/pull/4186) Sliding Sync that would cause rooms to stay forgotten and hidden even after rejoining.
@@ -1 +0,0 @@
-Check if user has membership in a room before tagging it. Contributed by Lama Alosaimi.
@@ -0,0 +1 @@
+Fix some typing issues uncovered by upgrading mypy to 1.11.x.
@@ -1,2 +0,0 @@
-Fix a bug in the admin redact endpoint where the background task would not run if a worker was specified in
-the config option `run_background_tasks_on`.
@@ -1 +0,0 @@
-Fix bug when some presence and typing timeouts can expire early.
@@ -1 +0,0 @@
-The nix flake inside the repository no longer tracks nixpkgs/master to not catch the latest bugs from a PR merged 5 minutes ago.
@@ -1 +0,0 @@
-Fix detection when the built Rust library was outdated when using source installations.
@@ -1 +0,0 @@
-Minor speed-up of sliding sync by computing extensions results in parallel.
@@ -1 +0,0 @@
-Bump the default Python version in the Synapse Dockerfile from 3.11 -> 3.12.
@@ -1 +0,0 @@
-Add experimental support for [MSC4222](https://github.com/matrix-org/matrix-spec-proposals/pull/4222).
@@ -1 +0,0 @@
-Refactor database calls to remove `Generator` usage.
@@ -1 +0,0 @@
-Remove usage of internal header encoding API.
@@ -1 +0,0 @@
-Update version constraint to allow the latest poetry-core 1.9.1.
@@ -1 +0,0 @@
-Fix a long-standing bug in Synapse which could cause one-time keys to be issued in the incorrect order, causing message decryption failures.
@@ -1 +0,0 @@
-Use unique name for each os.arch variant when uploading Wheel artifacts.
@@ -1 +0,0 @@
-Fix tests to run with latest Twisted.
@@ -1 +0,0 @@
-Fix tests to run with latest Twisted.
@@ -1 +0,0 @@
-Remove support for python 3.8.
@@ -1 +0,0 @@
-Update the portdb CI to use Python 3.13 and Postgres 17 as latest dependencies.
@@ -1 +0,0 @@
-Fix tests to run with latest Twisted.
@@ -1 +0,0 @@
-Limit event fetches in test to prevent stack size from growing too large.
@@ -1,15 +1,3 @@
-matrix-synapse-py3 (1.118.0) stable; urgency=medium
-
-  * New Synapse release 1.118.0.
-
- -- Synapse Packaging team <packages@matrix.org>  Tue, 29 Oct 2024 15:29:53 +0100
-
-matrix-synapse-py3 (1.118.0~rc1) stable; urgency=medium
-
-  * New Synapse release 1.118.0rc1.
-
- -- Synapse Packaging team <packages@matrix.org>  Tue, 22 Oct 2024 11:48:14 +0100
-
 matrix-synapse-py3 (1.117.0) stable; urgency=medium

  * New Synapse release 1.117.0.
@@ -20,7 +20,7 @@
 # `poetry export | pip install -r /dev/stdin`, but beware: we have experienced bugs in
 # in `poetry export` in the past.

-ARG PYTHON_VERSION=3.12
+ARG PYTHON_VERSION=3.11

 ###
 ### Stage 0: generate requirements.txt
@@ -5,7 +5,6 @@ basis. The currently supported features are:
 - [MSC3881](https://github.com/matrix-org/matrix-spec-proposals/pull/3881): enable remotely toggling push notifications
 for another client
 - [MSC3575](https://github.com/matrix-org/matrix-spec-proposals/pull/3575): enable experimental sliding sync support
- [MSC4222](https://github.com/matrix-org/matrix-spec-proposals/pull/4222): adding `state_after` to sync v2

 To use it, you will need to authenticate by providing an `access_token`
 for a server admin: see [Admin API](../usage/administration/admin_api/).
@@ -1365,9 +1365,6 @@ _Added in Synapse 1.72.0._

 ## Redact all the events of a user

-This endpoint allows an admin to redact the events of a given user. There are no restrictions on redactions for a 
-local user. By default, we puppet the user who sent the message to redact it themselves. Redactions for non-local users are issued using the admin user, and will fail in rooms where the admin user is not admin/does not have the specified power level to issue redactions. 
-
 The API is 
 ```
 POST /_synapse/admin/v1/user/$user_id/redact
@@ -322,7 +322,7 @@ The following command will let you run the integration test with the most common
 configuration:

 ```sh
-$ docker run --rm -it -v /path/where/you/have/cloned/the/repository\:/src:ro -v /path/to/where/you/want/logs\:/logs matrixdotorg/sytest-synapse:bullseye
+$ docker run --rm -it -v /path/where/you/have/cloned/the/repository\:/src:ro -v /path/to/where/you/want/logs\:/logs matrixdotorg/sytest-synapse:focal
 ```
 (Note that the paths must be full paths! You could also write `$(realpath relative/path)` if needed.)

@@ -208,7 +208,7 @@ When following this route please make sure that the [Platform-specific prerequis
 System requirements:

 - POSIX-compliant system (tested on Linux & OS X)
- Python 3.9 or later, up to Python 3.13.
+- Python 3.8 or later, up to Python 3.11.
 - At least 1GB of free RAM if you want to join large public rooms like #matrix:matrix.org

 If building on an uncommon architecture for which pre-built wheels are
@@ -117,17 +117,6 @@ each upgrade are complete before moving on to the next upgrade, to avoid
 stacking them up. You can monitor the currently running background updates with
 [the Admin API](usage/administration/admin_api/background_updates.html#status).

-# Upgrading to v1.119.0
-
-## Minimum supported Python version
-
-The minimum supported Python version has been increased from v3.8 to v3.9.
-You will need Python 3.9+ to run Synapse v1.119.0 (due out Nov 7th, 2024).
-
-If you use current versions of the Matrix.org-distributed Docker images, no action is required.
-Please note that support for Ubuntu `focal` was dropped as well since it uses Python 3.8.
-
-
 # Upgrading to v1.111.0

 ## New worker endpoints for authenticated client and federation media
@@ -186,16 +186,16 @@
    },
    "nixpkgs_2": {
      "locked": {
-        "lastModified": 1729265718,
-        "narHash": "sha256-4HQI+6LsO3kpWTYuVGIzhJs1cetFcwT7quWCk/6rqeo=",
+        "lastModified": 1690535733,
+        "narHash": "sha256-WgjUPscQOw3cB8yySDGlyzo6cZNihnRzUwE9kadv/5I=",
        "owner": "NixOS",
        "repo": "nixpkgs",
-        "rev": "ccc0c2126893dd20963580b6478d1a10a4512185",
+        "rev": "8cacc05fbfffeaab910e8c2c9e2a7c6b32ce881a",
        "type": "github"
      },
      "original": {
        "owner": "NixOS",
-        "ref": "nixpkgs-unstable",
+        "ref": "master",
        "repo": "nixpkgs",
        "type": "github"
      }
@@ -3,13 +3,13 @@
 # (https://github.com/matrix-org/complement) Matrix homeserver test suites are also
 # installed automatically.
 #
-# You must have already installed Nix (https://nixos.org/download/) on your system to use this.
-# Nix can be installed on any Linux distribiution or MacOS; NixOS is not required.
-# Windows is not directly supported, but Nix can be installed inside of WSL2 or even Docker
+# You must have already installed Nix (https://nixos.org) on your system to use this.
+# Nix can be installed on Linux or MacOS; NixOS is not required. Windows is not
+# directly supported, but Nix can be installed inside of WSL2 or even Docker
 # containers. Please refer to https://nixos.org/download for details.
 #
 # You must also enable support for flakes in Nix. See the following for how to
-# do so permanently: https://wiki.nixos.org/wiki/Flakes#Other_Distros,_without_Home-Manager
+# do so permanently: https://nixos.wiki/wiki/Flakes#Enable_flakes
 #
 # Be warned: you'll need over 3.75 GB of free space to download all the dependencies.
 #
@@ -20,7 +20,7 @@
 # locally from "services", such as PostgreSQL and Redis.
 #
 # You should now be dropped into a new shell with all programs and dependencies
-# available to you!
+# availabile to you!
 #
 # You can start up pre-configured local Synapse, PostgreSQL and Redis instances by
 # running: `devenv up`. To stop them, use Ctrl-C.
@@ -39,9 +39,9 @@

 {
  inputs = {
-    # Use the rolling/unstable branch of nixpkgs. Used to fetch the latest
+    # Use the master/unstable branch of nixpkgs. Used to fetch the latest
    # available versions of packages.
-    nixpkgs.url = "github:NixOS/nixpkgs/nixpkgs-unstable";
+    nixpkgs.url = "github:NixOS/nixpkgs/master";
    # Output a development shell for x86_64/aarch64 Linux/Darwin (MacOS).
    systems.url = "github:nix-systems/default";
    # A development environment manager built on Nix. See https://devenv.sh.
@@ -50,7 +50,7 @@
    rust-overlay.url = "github:oxalica/rust-overlay";
  };

-  outputs = { nixpkgs, devenv, systems, rust-overlay, ... } @ inputs:
+  outputs = { self, nixpkgs, devenv, systems, rust-overlay, ... } @ inputs:
    let
      forEachSystem = nixpkgs.lib.genAttrs (import systems);
    in {
@@ -126,7 +126,7 @@
                # Automatically activate the poetry virtualenv upon entering the shell.
                languages.python.poetry.activate.enable = true;
                # Install all extra Python dependencies; this is needed to run the unit
-                # tests and utilise all Synapse features.
+                # tests and utilitise all Synapse features.
                languages.python.poetry.install.arguments = ["--extras all"];
                # Install the 'matrix-synapse' package from the local checkout.
                languages.python.poetry.install.installRootPackage = true;
@@ -163,8 +163,8 @@
                # Create a postgres user called 'synapse_user' which has ownership
                # over the 'synapse' database.
                services.postgres.initialScript = ''
-                  CREATE USER synapse_user;
-                  ALTER DATABASE synapse OWNER TO synapse_user;
+                CREATE USER synapse_user;
+                ALTER DATABASE synapse OWNER TO synapse_user;
                '';

                # Redis is needed in order to run Synapse in worker mode.
@@ -26,7 +26,7 @@ strict_equality = True

 # Run mypy type checking with the minimum supported Python version to catch new usage
 # that isn't backwards-compatible (types, overloads, etc).
-python_version = 3.9
+python_version = 3.8

 files =
  docker/,
@@ -1,4 +1,4 @@
-# This file is automatically @generated by Poetry 1.8.4 and should not be changed by hand.
+# This file is automatically @generated by Poetry 1.8.3 and should not be changed by hand.

 [[package]]
 name = "annotated-types"
@@ -11,6 +11,9 @@ files = [
    {file = "annotated_types-0.7.0.tar.gz", hash = "sha256:aff07c09a53a08bc8cfccb9c85b05f1aa9a2a6f23728d790723543408344ce89"},
 ]

+[package.dependencies]
+typing-extensions = {version = ">=4.0.0", markers = "python_version < \"3.9\""}
+
 [[package]]
 name = "attrs"
 version = "24.2.0"
@@ -871,7 +874,9 @@ files = [

 [package.dependencies]
 attrs = ">=22.2.0"
+importlib-resources = {version = ">=1.4.0", markers = "python_version < \"3.9\""}
 jsonschema-specifications = ">=2023.03.6"
+pkgutil-resolve-name = {version = ">=1.3.10", markers = "python_version < \"3.9\""}
 referencing = ">=0.28.4"
 rpds-py = ">=0.7.1"

@@ -891,6 +896,7 @@ files = [
 ]

 [package.dependencies]
+importlib-resources = {version = ">=1.4.0", markers = "python_version < \"3.9\""}
 referencing = ">=0.28.0"

 [[package]]
@@ -906,6 +912,7 @@ files = [

 [package.dependencies]
 importlib-metadata = {version = ">=4.11.4", markers = "python_version < \"3.12\""}
+importlib-resources = {version = "*", markers = "python_version < \"3.9\""}
 "jaraco.classes" = "*"
 jeepney = {version = ">=0.4.2", markers = "sys_platform == \"linux\""}
 pywin32-ctypes = {version = ">=0.2.0", markers = "sys_platform == \"win32\""}
@@ -1373,17 +1380,17 @@ files = [

 [[package]]
 name = "mypy-zope"
-version = "1.0.8"
+version = "1.0.7"
 description = "Plugin for mypy to support zope interfaces"
 optional = false
 python-versions = "*"
 files = [
-    {file = "mypy_zope-1.0.8-py3-none-any.whl", hash = "sha256:8794a77dae0c7e2f28b8ac48569091310b3ee45bb9d6cd4797dcb837c40f9976"},
-    {file = "mypy_zope-1.0.8.tar.gz", hash = "sha256:854303a95aefc4289e8a0796808e002c2c7ecde0a10a8f7b8f48092f94ef9b9f"},
+    {file = "mypy_zope-1.0.7-py3-none-any.whl", hash = "sha256:f19de249574319d81083b15f8a022c6b15583582f23340a860922141f1b651ca"},
+    {file = "mypy_zope-1.0.7.tar.gz", hash = "sha256:32a79ce78647c0bea61e7e0c0eb1233fcb97bb94e8950cca73f17d3419c602f7"},
 ]

 [package.dependencies]
-mypy = ">=1.0.0,<1.13.0"
+mypy = ">=1.0.0,<1.12.0"
 "zope.interface" = "*"
 "zope.schema" = "*"

@@ -1444,13 +1451,13 @@ dev = ["jinja2"]

 [[package]]
 name = "phonenumbers"
-version = "8.13.49"
+version = "8.13.47"
 description = "Python version of Google's common library for parsing, formatting, storing and validating international phone numbers."
 optional = false
 python-versions = "*"
 files = [
-    {file = "phonenumbers-8.13.49-py2.py3-none-any.whl", hash = "sha256:e17140955ab3d8f9580727372ea64c5ada5327932d6021ef6fd203c3db8c8139"},
-    {file = "phonenumbers-8.13.49.tar.gz", hash = "sha256:e608ccb61f0bd42e6db1d2c421f7c22186b88f494870bf40aa31d1a2718ab0ae"},
+    {file = "phonenumbers-8.13.47-py2.py3-none-any.whl", hash = "sha256:5d3c0142ef7055ca5551884352e3b6b93bfe002a0bc95b8eaba39b0e2184541b"},
+    {file = "phonenumbers-8.13.47.tar.gz", hash = "sha256:53c5e7c6d431cafe4efdd44956078404ae9bc8b0eacc47be3105d3ccc88aaffa"},
 ]

 [[package]]
@@ -1564,6 +1571,17 @@ files = [
 [package.extras]
 testing = ["pytest", "pytest-cov"]

+[[package]]
+name = "pkgutil-resolve-name"
+version = "1.3.10"
+description = "Resolve a name to an object."
+optional = false
+python-versions = ">=3.6"
+files = [
+    {file = "pkgutil_resolve_name-1.3.10-py3-none-any.whl", hash = "sha256:ca27cc078d25c5ad71a9de0a7a330146c4e014c2462d9af19c6b828280649c5e"},
+    {file = "pkgutil_resolve_name-1.3.10.tar.gz", hash = "sha256:357d6c9e6a755653cfd78893817c0853af365dd51ec97f3d358a819373bbd174"},
+]
+
 [[package]]
 name = "prometheus-client"
 version = "0.21.0"
@@ -1930,6 +1948,7 @@ files = [
 [package.dependencies]
 cryptography = ">=3.1"
 defusedxml = "*"
+importlib-resources = {version = "*", markers = "python_version < \"3.9\""}
 pyopenssl = "*"
 python-dateutil = "*"
 pytz = "*"
@@ -1955,13 +1974,13 @@ six = ">=1.5"

 [[package]]
 name = "python-multipart"
-version = "0.0.16"
+version = "0.0.12"
 description = "A streaming multipart parser for Python"
 optional = false
 python-versions = ">=3.8"
 files = [
-    {file = "python_multipart-0.0.16-py3-none-any.whl", hash = "sha256:c2759b7b976ef3937214dfb592446b59dfaa5f04682a076f78b117c94776d87a"},
-    {file = "python_multipart-0.0.16.tar.gz", hash = "sha256:8dee37b88dab9b59922ca173c35acb627cc12ec74019f5cd4578369c6df36554"},
+    {file = "python_multipart-0.0.12-py3-none-any.whl", hash = "sha256:43dcf96cf65888a9cd3423544dd0d75ac10f7aa0c3c28a175bbcd00c9ce1aebf"},
+    {file = "python_multipart-0.0.12.tar.gz", hash = "sha256:045e1f98d719c1ce085ed7f7e1ef9d8ccc8c02ba02b5566d5f7521410ced58cb"},
 ]

 [[package]]
@@ -2145,6 +2164,7 @@ files = [
 [package.dependencies]
 markdown-it-py = ">=2.2.0,<3.0.0"
 pygments = ">=2.13.0,<3.0.0"
+typing-extensions = {version = ">=4.0.0,<5.0", markers = "python_version < \"3.9\""}

 [package.extras]
 jupyter = ["ipywidgets (>=7.5.1,<9)"]
@@ -2257,29 +2277,29 @@ files = [

 [[package]]
 name = "ruff"
-version = "0.7.2"
+version = "0.6.9"
 description = "An extremely fast Python linter and code formatter, written in Rust."
 optional = false
 python-versions = ">=3.7"
 files = [
-    {file = "ruff-0.7.2-py3-none-linux_armv6l.whl", hash = "sha256:b73f873b5f52092e63ed540adefc3c36f1f803790ecf2590e1df8bf0a9f72cb8"},
-    {file = "ruff-0.7.2-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:5b813ef26db1015953daf476202585512afd6a6862a02cde63f3bafb53d0b2d4"},
-    {file = "ruff-0.7.2-py3-none-macosx_11_0_arm64.whl", hash = "sha256:853277dbd9675810c6826dad7a428d52a11760744508340e66bf46f8be9701d9"},
-    {file = "ruff-0.7.2-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:21aae53ab1490a52bf4e3bf520c10ce120987b047c494cacf4edad0ba0888da2"},
-    {file = "ruff-0.7.2-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:ccc7e0fc6e0cb3168443eeadb6445285abaae75142ee22b2b72c27d790ab60ba"},
-    {file = "ruff-0.7.2-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:fd77877a4e43b3a98e5ef4715ba3862105e299af0c48942cc6d51ba3d97dc859"},
-    {file = "ruff-0.7.2-py3-none-manylinux_2_17_ppc64.manylinux2014_ppc64.whl", hash = "sha256:e00163fb897d35523c70d71a46fbaa43bf7bf9af0f4534c53ea5b96b2e03397b"},
-    {file = "ruff-0.7.2-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:f3c54b538633482dc342e9b634d91168fe8cc56b30a4b4f99287f4e339103e88"},
-    {file = "ruff-0.7.2-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:7b792468e9804a204be221b14257566669d1db5c00d6bb335996e5cd7004ba80"},
-    {file = "ruff-0.7.2-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:dba53ed84ac19ae4bfb4ea4bf0172550a2285fa27fbb13e3746f04c80f7fa088"},
-    {file = "ruff-0.7.2-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:b19fafe261bf741bca2764c14cbb4ee1819b67adb63ebc2db6401dcd652e3748"},
-    {file = "ruff-0.7.2-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:28bd8220f4d8f79d590db9e2f6a0674f75ddbc3847277dd44ac1f8d30684b828"},
-    {file = "ruff-0.7.2-py3-none-musllinux_1_2_i686.whl", hash = "sha256:9fd67094e77efbea932e62b5d2483006154794040abb3a5072e659096415ae1e"},
-    {file = "ruff-0.7.2-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:576305393998b7bd6c46018f8104ea3a9cb3fa7908c21d8580e3274a3b04b691"},
-    {file = "ruff-0.7.2-py3-none-win32.whl", hash = "sha256:fa993cfc9f0ff11187e82de874dfc3611df80852540331bc85c75809c93253a8"},
-    {file = "ruff-0.7.2-py3-none-win_amd64.whl", hash = "sha256:dd8800cbe0254e06b8fec585e97554047fb82c894973f7ff18558eee33d1cb88"},
-    {file = "ruff-0.7.2-py3-none-win_arm64.whl", hash = "sha256:bb8368cd45bba3f57bb29cbb8d64b4a33f8415d0149d2655c5c8539452ce7760"},
-    {file = "ruff-0.7.2.tar.gz", hash = "sha256:2b14e77293380e475b4e3a7a368e14549288ed2931fce259a6f99978669e844f"},
+    {file = "ruff-0.6.9-py3-none-linux_armv6l.whl", hash = "sha256:064df58d84ccc0ac0fcd63bc3090b251d90e2a372558c0f057c3f75ed73e1ccd"},
+    {file = "ruff-0.6.9-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:140d4b5c9f5fc7a7b074908a78ab8d384dd7f6510402267bc76c37195c02a7ec"},
+    {file = "ruff-0.6.9-py3-none-macosx_11_0_arm64.whl", hash = "sha256:53fd8ca5e82bdee8da7f506d7b03a261f24cd43d090ea9db9a1dc59d9313914c"},
+    {file = "ruff-0.6.9-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:645d7d8761f915e48a00d4ecc3686969761df69fb561dd914a773c1a8266e14e"},
+    {file = "ruff-0.6.9-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:eae02b700763e3847595b9d2891488989cac00214da7f845f4bcf2989007d577"},
+    {file = "ruff-0.6.9-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7d5ccc9e58112441de8ad4b29dcb7a86dc25c5f770e3c06a9d57e0e5eba48829"},
+    {file = "ruff-0.6.9-py3-none-manylinux_2_17_ppc64.manylinux2014_ppc64.whl", hash = "sha256:417b81aa1c9b60b2f8edc463c58363075412866ae4e2b9ab0f690dc1e87ac1b5"},
+    {file = "ruff-0.6.9-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3c866b631f5fbce896a74a6e4383407ba7507b815ccc52bcedabb6810fdb3ef7"},
+    {file = "ruff-0.6.9-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:7b118afbb3202f5911486ad52da86d1d52305b59e7ef2031cea3425142b97d6f"},
+    {file = "ruff-0.6.9-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a67267654edc23c97335586774790cde402fb6bbdb3c2314f1fc087dee320bfa"},
+    {file = "ruff-0.6.9-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:3ef0cc774b00fec123f635ce5c547dac263f6ee9fb9cc83437c5904183b55ceb"},
+    {file = "ruff-0.6.9-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:12edd2af0c60fa61ff31cefb90aef4288ac4d372b4962c2864aeea3a1a2460c0"},
+    {file = "ruff-0.6.9-py3-none-musllinux_1_2_i686.whl", hash = "sha256:55bb01caeaf3a60b2b2bba07308a02fca6ab56233302406ed5245180a05c5625"},
+    {file = "ruff-0.6.9-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:925d26471fa24b0ce5a6cdfab1bb526fb4159952385f386bdcc643813d472039"},
+    {file = "ruff-0.6.9-py3-none-win32.whl", hash = "sha256:eb61ec9bdb2506cffd492e05ac40e5bc6284873aceb605503d8494180d6fc84d"},
+    {file = "ruff-0.6.9-py3-none-win_amd64.whl", hash = "sha256:785d31851c1ae91f45b3d8fe23b8ae4b5170089021fbb42402d811135f0b7117"},
+    {file = "ruff-0.6.9-py3-none-win_arm64.whl", hash = "sha256:a9641e31476d601f83cd602608739a0840e348bda93fec9f1ee816f8b6798b93"},
+    {file = "ruff-0.6.9.tar.gz", hash = "sha256:b076ef717a8e5bc819514ee1d602bbdca5b4420ae13a9cf61a0c0a4f53a2baa2"},
 ]

 [[package]]
@@ -3101,5 +3121,5 @@ user-search = ["pyicu"]

 [metadata]
 lock-version = "2.0"
-python-versions = "^3.9.0"
-content-hash = "0cd942a5193d01cbcef135a0bebd3fa0f12f7dbc63899d6f1c301e0649e9d902"
+python-versions = "^3.8.0"
+content-hash = "c8a22f901970b2f851151e731532757fd3acf7ba02930952636d2e6c5c9c0c90"
@@ -36,7 +36,7 @@

 [tool.ruff]
 line-length = 88
-target-version = "py39"
+target-version = "py38"

 [tool.ruff.lint]
 # See https://beta.ruff.rs/docs/rules/#error-e
@@ -97,7 +97,7 @@ module-name = "synapse.synapse_rust"

 [tool.poetry]
 name = "matrix-synapse"
-version = "1.118.0"
+version = "1.117.0"
 description = "Homeserver for the Matrix decentralised comms protocol"
 authors = ["Matrix.org Team and Contributors <packages@matrix.org>"]
 license = "AGPL-3.0-or-later"
@@ -155,7 +155,7 @@ synapse_review_recent_signups = "synapse._scripts.review_recent_signups:main"
 update_synapse_database = "synapse._scripts.update_synapse_database:main"

 [tool.poetry.dependencies]
-python = "^3.9.0"
+python = "^3.8.0"

 # Mandatory Dependencies
 # ----------------------
@@ -178,7 +178,7 @@ Twisted = {extras = ["tls"], version = ">=18.9.0"}
 treq = ">=15.1"
 # Twisted has required pyopenssl 16.0 since about Twisted 16.6.
 pyOpenSSL = ">=16.0.0"
-PyYAML = ">=5.3"
+PyYAML = ">=3.13"
 pyasn1 = ">=0.1.9"
 pyasn1-modules = ">=0.0.7"
 bcrypt = ">=3.1.7"
@@ -241,7 +241,7 @@ authlib = { version = ">=0.15.1", optional = true }
 # `contrib/systemd/log_config.yaml`.
 # Note: systemd-python 231 appears to have been yanked from pypi
 systemd-python = { version = ">=231", optional = true }
-lxml = { version = ">=4.5.2", optional = true }
+lxml = { version = ">=4.2.0", optional = true }
 sentry-sdk = { version = ">=0.7.2", optional = true }
 opentracing = { version = ">=2.2.0", optional = true }
 jaeger-client = { version = ">=4.0.0", optional = true }
@@ -320,7 +320,7 @@ all = [
 # failing on new releases. Keeping lower bounds loose here means that dependabot
 # can bump versions without having to update the content-hash in the lockfile.
 # This helps prevents merge conflicts when running a batch of dependabot updates.
-ruff = "0.7.2"
+ruff = "0.6.9"
 # Type checking only works with the pydantic.v1 compat module from pydantic v2
 pydantic = "^2"

@@ -370,7 +370,7 @@ tomli = ">=1.2.3"
 # runtime errors caused by build system changes.
 # We are happy to raise these upper bounds upon request,
 # provided we check that it's safe to do so (i.e. that CI passes).
-requires = ["poetry-core>=1.1.0,<=1.9.1", "setuptools_rust>=1.3,<=1.8.1"]
+requires = ["poetry-core>=1.1.0,<=1.9.0", "setuptools_rust>=1.3,<=1.8.1"]
 build-backend = "poetry.core.masonry.api"


@@ -378,13 +378,13 @@ build-backend = "poetry.core.masonry.api"
 # Skip unsupported platforms (by us or by Rust).
 # See https://cibuildwheel.readthedocs.io/en/stable/options/#build-skip for the list of build targets.
 # We skip:
-#  - CPython 3.6, 3.7 and 3.8: EOLed
-#  - PyPy 3.7 and 3.8: we only support Python 3.9+
+#  - CPython 3.6 and 3.7: EOLed
+#  - PyPy 3.7: we only support Python 3.8+
 #  - musllinux i686: excluded to reduce number of wheels we build.
 #    c.f. https://github.com/matrix-org/synapse/pull/12595#discussion_r963107677
 #  - PyPy on Aarch64 and musllinux on aarch64: too slow to build.
 #    c.f. https://github.com/matrix-org/synapse/pull/14259
-skip = "cp36* cp37* cp38* pp37* pp38* *-musllinux_i686 pp*aarch64 *-musllinux_aarch64"
+skip = "cp36* cp37* pp37* *-musllinux_i686 pp*aarch64 *-musllinux_aarch64"

 # We need a rust compiler
 before-all =  "curl https://sh.rustup.rs -sSf | sh -s -- --default-toolchain stable -y --profile minimal"
@@ -28,8 +28,9 @@ from typing import Collection, Optional, Sequence, Set
 # example)
 DISTS = (
    "debian:bullseye",  # (EOL ~2024-07) (our EOL forced by Python 3.9 is 2025-10-05)
-    "debian:bookworm",  # (EOL 2026-06) (our EOL forced by Python 3.11 is 2027-10-24)
-    "debian:sid",  # (rolling distro, no EOL)
+    "debian:bookworm",  # (EOL not specified yet) (our EOL forced by Python 3.11 is 2027-10-24)
+    "debian:sid",  # (EOL not specified yet) (our EOL forced by Python 3.11 is 2027-10-24)
+    "ubuntu:focal",  # 20.04 LTS (EOL 2025-04) (our EOL forced by Python 3.8 is 2024-10-14)
    "ubuntu:jammy",  # 22.04 LTS (EOL 2027-04) (our EOL forced by Python 3.10 is 2026-10-04)
    "ubuntu:noble",  # 24.04 LTS (EOL 2029-06)
    "ubuntu:oracular",  # 24.10 (EOL 2025-07)
@@ -39,8 +39,8 @@ ImageFile.LOAD_TRUNCATED_IMAGES = True
 # Note that we use an (unneeded) variable here so that pyupgrade doesn't nuke the
 # if-statement completely.
 py_version = sys.version_info
-if py_version < (3, 9):
-    print("Synapse requires Python 3.9 or above.")
+if py_version < (3, 8):
+    print("Synapse requires Python 3.8 or above.")
    sys.exit(1)

 # Allow using the asyncio reactor via env var.
@@ -450,6 +450,3 @@ class ExperimentalConfig(Config):

        # MSC4210: Remove legacy mentions
        self.msc4210_enabled: bool = experimental.get("msc4210_enabled", False)
-
-        # MSC4222: Adding `state_after` to sync v2
-        self.msc4222_enabled: bool = experimental.get("msc4222_enabled", False)
@@ -113,7 +113,7 @@ class Authenticator:
                ):
                    raise AuthenticationError(
                        HTTPStatus.UNAUTHORIZED,
-                        f"Destination mismatch in auth header, received: {destination!r}",
+                        "Destination mismatch in auth header",
                        Codes.UNAUTHORIZED,
                    )
        if (
@@ -73,8 +73,6 @@ class AdminHandler:
            self._redact_all_events, REDACT_ALL_EVENTS_ACTION_NAME
        )

-        self.hs = hs
-
    async def get_redact_task(self, redact_id: str) -> Optional[ScheduledTask]:
        """Get the current status of an active redaction process

@@ -425,10 +423,8 @@ class AdminHandler:
        user_id = task.params.get("user_id")
        assert user_id is not None

-        # puppet the user if they're ours, otherwise use admin to redact
        requester = create_requester(
-            user_id if self.hs.is_mine_id(user_id) else admin.user.to_string(),
-            authenticated_entity=admin.user.to_string(),
+            user_id, authenticated_entity=admin.user.to_string()
        )

        reason = task.params.get("reason")
@@ -615,7 +615,7 @@ class E2eKeysHandler:
        3. Attempt to fetch fallback keys from the database.

        Args:
-            local_query: An iterable of tuples of (user ID, device ID, algorithm, number of keys).
+            local_query: An iterable of tuples of (user ID, device ID, algorithm).
            always_include_fallback_keys: True to always include fallback keys.

        Returns:
@@ -1190,26 +1190,6 @@ class RoomMemberHandler(metaclass=abc.ABCMeta):
            origin_server_ts=origin_server_ts,
        )

-    async def check_for_any_membership_in_room(
-        self, *, user_id: str, room_id: str
-    ) -> None:
-        """
-        Check if the user has any membership in the room and raise error if not.
-
-        Args:
-            user_id: The user to check.
-            room_id: The room to check.
-
-        Raises:
-            AuthError if the user doesn't have any membership in the room.
-        """
-        result = await self.store.get_local_current_membership_for_user_in_room(
-            user_id=user_id, room_id=room_id
-        )
-
-        if result is None or result == (None, None):
-            raise AuthError(403, f"User {user_id} has no membership in room {room_id}")
-
    async def _should_perform_remote_join(
        self,
        user_id: str,
@@ -12,7 +12,6 @@
 # <https://www.gnu.org/licenses/agpl-3.0.html>.
 #

-import itertools
 import logging
 from itertools import chain
 from typing import TYPE_CHECKING, AbstractSet, Dict, List, Mapping, Optional, Set, Tuple
@@ -80,15 +79,6 @@ sync_processing_time = Histogram(
    ["initial"],
 )

-# Limit the number of state_keys we should remember sending down the connection for each
-# (room_id, user_id). We don't want to store and pull out too much data in the database.
-#
-# 100 is an arbitrary but small-ish number. The idea is that we probably won't send down
-# too many redundant member state events (that the client already knows about) for a
-# given ongoing conversation if we keep 100 around. Most rooms don't have 100 members
-# anyway and it takes a while to cycle through 100 members.
-MAX_NUMBER_PREVIOUS_STATE_KEYS_TO_REMEMBER = 100
-

 class SlidingSyncHandler:
    def __init__(self, hs: "HomeServer"):
@@ -883,14 +873,6 @@ class SlidingSyncHandler:
        #
        # Calculate the `StateFilter` based on the `required_state` for the room
        required_state_filter = StateFilter.none()
-        # The requested `required_state_map` with the lazy membership expanded and
-        # `$ME` replaced with the user's ID. This allows us to see what membership we've
-        # sent down to the client in the next request.
-        #
-        # Make a copy so we can modify it. Still need to be careful to make a copy of
-        # the state key sets if we want to add/remove from them. We could make a deep
-        # copy but this saves us some work.
-        expanded_required_state_map = dict(room_sync_config.required_state_map)
        if room_membership_for_user_at_to_token.membership not in (
            Membership.INVITE,
            Membership.KNOCK,
@@ -956,48 +938,21 @@ class SlidingSyncHandler:
                        ):
                            lazy_load_room_members = True
                            # Everyone in the timeline is relevant
-                            #
-                            # FIXME: We probably also care about invite, ban, kick, targets, etc
-                            # but the spec only mentions "senders".
                            timeline_membership: Set[str] = set()
                            if timeline_events is not None:
                                for timeline_event in timeline_events:
                                    timeline_membership.add(timeline_event.sender)

-                            # Update the required state filter so we pick up the new
-                            # membership
                            for user_id in timeline_membership:
                                required_state_types.append(
                                    (EventTypes.Member, user_id)
                                )

-                            # Add an explicit entry for each user in the timeline
-                            #
-                            # Make a new set or copy of the state key set so we can
-                            # modify it without affecting the original
-                            # `required_state_map`
-                            expanded_required_state_map[EventTypes.Member] = (
-                                expanded_required_state_map.get(
-                                    EventTypes.Member, set()
-                                )
-                                | timeline_membership
-                            )
+                            # FIXME: We probably also care about invite, ban, kick, targets, etc
+                            # but the spec only mentions "senders".
                        elif state_key == StateValues.ME:
                            num_others += 1
                            required_state_types.append((state_type, user.to_string()))
-                            # Replace `$ME` with the user's ID so we can deduplicate
-                            # when someone requests the same state with `$ME` or with
-                            # their user ID.
-                            #
-                            # Make a new set or copy of the state key set so we can
-                            # modify it without affecting the original
-                            # `required_state_map`
-                            expanded_required_state_map[EventTypes.Member] = (
-                                expanded_required_state_map.get(
-                                    EventTypes.Member, set()
-                                )
-                                | {user.to_string()}
-                            )
                        else:
                            num_others += 1
                            required_state_types.append((state_type, state_key))
@@ -1061,8 +1016,8 @@ class SlidingSyncHandler:
                changed_required_state_map, added_state_filter = (
                    _required_state_changes(
                        user.to_string(),
-                        prev_required_state_map=prev_room_sync_config.required_state_map,
-                        request_required_state_map=expanded_required_state_map,
+                        previous_room_config=prev_room_sync_config,
+                        room_sync_config=room_sync_config,
                        state_deltas=room_state_delta_id_map,
                    )
                )
@@ -1176,9 +1131,7 @@ class SlidingSyncHandler:
            # sensible order again.
            bump_stamp = 0

-        room_sync_required_state_map_to_persist: Mapping[str, AbstractSet[str]] = (
-            expanded_required_state_map
-        )
+        room_sync_required_state_map_to_persist = room_sync_config.required_state_map
        if changed_required_state_map:
            room_sync_required_state_map_to_persist = changed_required_state_map

@@ -1232,10 +1185,7 @@ class SlidingSyncHandler:
                )

        else:
-            new_connection_state.room_configs[room_id] = RoomSyncConfig(
-                timeline_limit=room_sync_config.timeline_limit,
-                required_state_map=room_sync_required_state_map_to_persist,
-            )
+            new_connection_state.room_configs[room_id] = room_sync_config

        set_tag(SynapseTags.RESULT_PREFIX + "initial", initial)

@@ -1370,8 +1320,8 @@ class SlidingSyncHandler:
 def _required_state_changes(
    user_id: str,
    *,
-    prev_required_state_map: Mapping[str, AbstractSet[str]],
-    request_required_state_map: Mapping[str, AbstractSet[str]],
+    previous_room_config: "RoomSyncConfig",
+    room_sync_config: RoomSyncConfig,
    state_deltas: StateMap[str],
 ) -> Tuple[Optional[Mapping[str, AbstractSet[str]]], StateFilter]:
    """Calculates the changes between the required state room config from the
@@ -1392,6 +1342,10 @@ def _required_state_changes(
        and the state filter to use to fetch extra current state that we need to
        return.
    """
+
+    prev_required_state_map = previous_room_config.required_state_map
+    request_required_state_map = room_sync_config.required_state_map
+
    if prev_required_state_map == request_required_state_map:
        # There has been no change. Return immediately.
        return None, StateFilter.none()
@@ -1424,19 +1378,12 @@ def _required_state_changes(
    # client. Passed to `StateFilter.from_types(...)`
    added: List[Tuple[str, Optional[str]]] = []

-    # Convert the list of state deltas to map from type to state_keys that have
-    # changed.
-    changed_types_to_state_keys: Dict[str, Set[str]] = {}
-    for event_type, state_key in state_deltas:
-        changed_types_to_state_keys.setdefault(event_type, set()).add(state_key)
-
    # First we calculate what, if anything, has been *added*.
    for event_type in (
        prev_required_state_map.keys() | request_required_state_map.keys()
    ):
        old_state_keys = prev_required_state_map.get(event_type, set())
        request_state_keys = request_required_state_map.get(event_type, set())
-        changed_state_keys = changed_types_to_state_keys.get(event_type, set())

        if old_state_keys == request_state_keys:
            # No change to this type
@@ -1446,55 +1393,8 @@ def _required_state_changes(
            # Nothing *added*, so we skip. Removals happen below.
            continue

-        # We only remove state keys from the effective state if they've been
-        # removed from the request *and* the state has changed. This ensures
-        # that if a client removes and then re-adds a state key, we only send
-        # down the associated current state event if its changed (rather than
-        # sending down the same event twice).
-        invalidated_state_keys = (
-            old_state_keys - request_state_keys
-        ) & changed_state_keys
-
-        # Figure out which state keys we should remember sending down the connection
-        inheritable_previous_state_keys = (
-            # Retain the previous state_keys that we've sent down before.
-            # Wildcard and lazy state keys are not sticky from previous requests.
-            (old_state_keys - {StateValues.WILDCARD, StateValues.LAZY})
-            - invalidated_state_keys
-        )
-
-        # Always update changes to include the newly added keys (we've expanded the set
-        # of state keys), use the new requested set with whatever hasn't been
-        # invalidated from the previous set.
-        changes[event_type] = request_state_keys | inheritable_previous_state_keys
-        # Limit the number of state_keys we should remember sending down the connection
-        # for each (room_id, user_id). We don't want to store and pull out too much data
-        # in the database. This is a happy-medium between remembering nothing and
-        # everything. We can avoid sending redundant state down the connection most of
-        # the time given that most rooms don't have 100 members anyway and it takes a
-        # while to cycle through 100 members.
-        #
-        # Only remember up to (MAX_NUMBER_PREVIOUS_STATE_KEYS_TO_REMEMBER)
-        if len(changes[event_type]) > MAX_NUMBER_PREVIOUS_STATE_KEYS_TO_REMEMBER:
-            # Reset back to only the requested state keys
-            changes[event_type] = request_state_keys
-
-            # Skip if there isn't any room to fill in the rest with previous state keys
-            if len(request_state_keys) < MAX_NUMBER_PREVIOUS_STATE_KEYS_TO_REMEMBER:
-                # Fill the rest with previous state_keys. Ideally, we could sort
-                # these by recency but it's just a set so just pick an arbitrary
-                # subset (good enough).
-                changes[event_type] = changes[event_type] | set(
-                    itertools.islice(
-                        inheritable_previous_state_keys,
-                        # Just taking the difference isn't perfect as there could be
-                        # overlap in the keys between the requested and previous but we
-                        # will decide to just take the easy route for now and avoid
-                        # additional set operations to figure it out.
-                        MAX_NUMBER_PREVIOUS_STATE_KEYS_TO_REMEMBER
-                        - len(request_state_keys),
-                    )
-                )
+        # Always update changes to include the newly added keys
+        changes[event_type] = request_state_keys

        if StateValues.WILDCARD in old_state_keys:
            # We were previously fetching everything for this type, so we don't need to
@@ -1521,6 +1421,12 @@ def _required_state_changes(

    added_state_filter = StateFilter.from_types(added)

+    # Convert the list of state deltas to map from type to state_keys that have
+    # changed.
+    changed_types_to_state_keys: Dict[str, Set[str]] = {}
+    for event_type, state_key in state_deltas:
+        changed_types_to_state_keys.setdefault(event_type, set()).add(state_key)
+
    # Figure out what changes we need to apply to the effective required state
    # config.
    for event_type, changed_state_keys in changed_types_to_state_keys.items():
@@ -1531,23 +1437,15 @@ def _required_state_changes(
            # No change.
            continue

-        # If we see the `user_id` as a state_key, also add "$ME" to the list of state
-        # that has changed to account for people requesting `required_state` with `$ME`
-        # or their user ID.
-        if user_id in changed_state_keys:
-            changed_state_keys.add(StateValues.ME)
-
-        # We only remove state keys from the effective state if they've been
-        # removed from the request *and* the state has changed. This ensures
-        # that if a client removes and then re-adds a state key, we only send
-        # down the associated current state event if its changed (rather than
-        # sending down the same event twice).
-        invalidated_state_keys = (
-            old_state_keys - request_state_keys
-        ) & changed_state_keys
-
-        # We've expanded the set of state keys, ... (already handled above)
        if request_state_keys - old_state_keys:
+            # We've expanded the set of state keys, so we just clobber the
+            # current set with the new set.
+            #
+            # We could also ensure that we keep entries where the state hasn't
+            # changed, but are no longer in the requested required state, but
+            # that's a sufficient edge case that we can ignore (as its only a
+            # performance optimization).
+            changes[event_type] = request_state_keys
            continue

        old_state_key_wildcard = StateValues.WILDCARD in old_state_keys
@@ -1569,6 +1467,11 @@ def _required_state_changes(
                changes[event_type] = request_state_keys
                continue

+        # Handle "$ME" values by adding "$ME" if the state key matches the user
+        # ID.
+        if user_id in changed_state_keys:
+            changed_state_keys.add(StateValues.ME)
+
        # At this point there are no wildcards and no additions to the set of
        # state keys requested, only deletions.
        #
@@ -1577,8 +1480,9 @@ def _required_state_changes(
        # that if a client removes and then re-adds a state key, we only send
        # down the associated current state event if its changed (rather than
        # sending down the same event twice).
-        if invalidated_state_keys:
-            changes[event_type] = old_state_keys - invalidated_state_keys
+        invalidated = (old_state_keys - request_state_keys) & changed_state_keys
+        if invalidated:
+            changes[event_type] = old_state_keys - invalidated

    if changes:
        # Update the required state config based on the changes.
@@ -49,10 +49,7 @@ from synapse.types.handlers.sliding_sync import (
    SlidingSyncConfig,
    SlidingSyncResult,
 )
-from synapse.util.async_helpers import (
-    concurrently_execute,
-    gather_optional_coroutines,
-)
+from synapse.util.async_helpers import concurrently_execute

 if TYPE_CHECKING:
    from synapse.server import HomeServer
@@ -100,26 +97,26 @@ class SlidingSyncExtensionHandler:
        if sync_config.extensions is None:
            return SlidingSyncResult.Extensions()

-        to_device_coro = None
+        to_device_response = None
        if sync_config.extensions.to_device is not None:
-            to_device_coro = self.get_to_device_extension_response(
+            to_device_response = await self.get_to_device_extension_response(
                sync_config=sync_config,
                to_device_request=sync_config.extensions.to_device,
                to_token=to_token,
            )

-        e2ee_coro = None
+        e2ee_response = None
        if sync_config.extensions.e2ee is not None:
-            e2ee_coro = self.get_e2ee_extension_response(
+            e2ee_response = await self.get_e2ee_extension_response(
                sync_config=sync_config,
                e2ee_request=sync_config.extensions.e2ee,
                to_token=to_token,
                from_token=from_token,
            )

-        account_data_coro = None
+        account_data_response = None
        if sync_config.extensions.account_data is not None:
-            account_data_coro = self.get_account_data_extension_response(
+            account_data_response = await self.get_account_data_extension_response(
                sync_config=sync_config,
                previous_connection_state=previous_connection_state,
                new_connection_state=new_connection_state,
@@ -130,9 +127,9 @@ class SlidingSyncExtensionHandler:
                from_token=from_token,
            )

-        receipts_coro = None
+        receipts_response = None
        if sync_config.extensions.receipts is not None:
-            receipts_coro = self.get_receipts_extension_response(
+            receipts_response = await self.get_receipts_extension_response(
                sync_config=sync_config,
                previous_connection_state=previous_connection_state,
                new_connection_state=new_connection_state,
@@ -144,9 +141,9 @@ class SlidingSyncExtensionHandler:
                from_token=from_token,
            )

-        typing_coro = None
+        typing_response = None
        if sync_config.extensions.typing is not None:
-            typing_coro = self.get_typing_extension_response(
+            typing_response = await self.get_typing_extension_response(
                sync_config=sync_config,
                actual_lists=actual_lists,
                actual_room_ids=actual_room_ids,
@@ -156,20 +153,6 @@ class SlidingSyncExtensionHandler:
                from_token=from_token,
            )

-        (
-            to_device_response,
-            e2ee_response,
-            account_data_response,
-            receipts_response,
-            typing_response,
-        ) = await gather_optional_coroutines(
-            to_device_coro,
-            e2ee_coro,
-            account_data_coro,
-            receipts_coro,
-            typing_coro,
-        )
-
        return SlidingSyncResult.Extensions(
            to_device=to_device_response,
            e2ee=e2ee_response,
@@ -143,7 +143,6 @@ class SyncConfig:
    filter_collection: FilterCollection
    is_guest: bool
    device_id: Optional[str]
-    use_state_after: bool


@attr.s(slots=True, frozen=True, auto_attribs=True)
@@ -1142,7 +1141,6 @@ class SyncHandler:
        since_token: Optional[StreamToken],
        end_token: StreamToken,
        full_state: bool,
-        joined: bool,
    ) -> MutableStateMap[EventBase]:
        """Works out the difference in state between the end of the previous sync and
        the start of the timeline.
@@ -1157,7 +1155,6 @@ class SyncHandler:
                the point just after their leave event.
            full_state: Whether to force returning the full state.
                `lazy_load_members` still applies when `full_state` is `True`.
-            joined: whether the user is currently joined to the room

        Returns:
            The state to return in the sync response for the room.
@@ -1233,12 +1230,11 @@ class SyncHandler:
            if full_state:
                state_ids = await self._compute_state_delta_for_full_sync(
                    room_id,
-                    sync_config,
+                    sync_config.user,
                    batch,
                    end_token,
                    members_to_fetch,
                    timeline_state,
-                    joined,
                )
            else:
                # If this is an initial sync then full_state should be set, and
@@ -1248,7 +1244,6 @@ class SyncHandler:

                state_ids = await self._compute_state_delta_for_incremental_sync(
                    room_id,
-                    sync_config,
                    batch,
                    since_token,
                    end_token,
@@ -1321,24 +1316,20 @@ class SyncHandler:
    async def _compute_state_delta_for_full_sync(
        self,
        room_id: str,
-        sync_config: SyncConfig,
+        syncing_user: UserID,
        batch: TimelineBatch,
        end_token: StreamToken,
        members_to_fetch: Optional[Set[str]],
        timeline_state: StateMap[str],
-        joined: bool,
    ) -> StateMap[str]:
        """Calculate the state events to be included in a full sync response.

        As with `_compute_state_delta_for_incremental_sync`, the result will include
        the membership events for the senders of each event in `members_to_fetch`.

-        Note that whether this returns the state at the start or the end of the
-        batch depends on `sync_config.use_state_after` (c.f. MSC4222).
-
        Args:
            room_id: The room we are calculating for.
-            sync_confg: The user that is calling `/sync`.
+            syncing_user: The user that is calling `/sync`.
            batch: The timeline batch for the room that will be sent to the user.
            end_token: Token of the end of the current batch. Normally this will be
                the same as the global "now_token", but if the user has left the room,
@@ -1347,11 +1338,10 @@ class SyncHandler:
                events in the timeline.
            timeline_state: The contribution to the room state from state events in
                `batch`. Only contains the last event for any given state key.
-            joined: whether the user is currently joined to the room

        Returns:
            A map from (type, state_key) to event_id, for each event that we believe
-            should be included in the `state` or `state_after` part of the sync response.
+            should be included in the `state` part of the sync response.
        """
        if members_to_fetch is not None:
            # Lazy-loading of membership events is enabled.
@@ -1369,7 +1359,7 @@ class SyncHandler:
            # is no guarantee that our membership will be in the auth events of
            # timeline events when the room is partial stated.
            state_filter = StateFilter.from_lazy_load_member_list(
-                members_to_fetch.union((sync_config.user.to_string(),))
+                members_to_fetch.union((syncing_user.to_string(),))
            )

            # We are happy to use partial state to compute the `/sync` response.
@@ -1383,61 +1373,6 @@ class SyncHandler:
            await_full_state = True
            lazy_load_members = False

-        # Check if we are wanting to return the state at the start or end of the
-        # timeline. If at the end we can just use the current state.
-        if sync_config.use_state_after:
-            # If we're getting the state at the end of the timeline, we can just
-            # use the current state of the room (and roll back any changes
-            # between when we fetched the current state and `end_token`).
-            #
-            # For rooms we're not joined to, there might be a very large number
-            # of deltas between `end_token` and "now", and so instead we fetch
-            # the state at the end of the timeline.
-            if joined:
-                state_ids = await self._state_storage_controller.get_current_state_ids(
-                    room_id,
-                    state_filter=state_filter,
-                    await_full_state=await_full_state,
-                )
-
-                # Now roll back the state by looking at the state deltas between
-                # end_token and now.
-                deltas = await self.store.get_current_state_deltas_for_room(
-                    room_id,
-                    from_token=end_token.room_key,
-                    to_token=self.store.get_room_max_token(),
-                )
-                if deltas:
-                    mutable_state_ids = dict(state_ids)
-
-                    # We iterate over the deltas backwards so that if there are
-                    # multiple changes of the same type/state_key we'll
-                    # correctly pick the earliest delta.
-                    for delta in reversed(deltas):
-                        if delta.prev_event_id:
-                            mutable_state_ids[(delta.event_type, delta.state_key)] = (
-                                delta.prev_event_id
-                            )
-                        elif (delta.event_type, delta.state_key) in mutable_state_ids:
-                            mutable_state_ids.pop((delta.event_type, delta.state_key))
-
-                    state_ids = mutable_state_ids
-
-                return state_ids
-
-            else:
-                # Just use state groups to get the state at the end of the
-                # timeline, i.e. the state at the leave/etc event.
-                state_at_timeline_end = (
-                    await self._state_storage_controller.get_state_ids_at(
-                        room_id,
-                        stream_position=end_token,
-                        state_filter=state_filter,
-                        await_full_state=await_full_state,
-                    )
-                )
-                return state_at_timeline_end
-
        state_at_timeline_end = await self._state_storage_controller.get_state_ids_at(
            room_id,
            stream_position=end_token,
@@ -1462,6 +1397,7 @@ class SyncHandler:
            timeline_contains=timeline_state,
            timeline_start=state_at_timeline_start,
            timeline_end=state_at_timeline_end,
+            previous_timeline_start={},
            previous_timeline_end={},
            lazy_load_members=lazy_load_members,
        )
@@ -1470,7 +1406,6 @@ class SyncHandler:
    async def _compute_state_delta_for_incremental_sync(
        self,
        room_id: str,
-        sync_config: SyncConfig,
        batch: TimelineBatch,
        since_token: StreamToken,
        end_token: StreamToken,
@@ -1485,12 +1420,8 @@ class SyncHandler:
        (`compute_state_delta`) is responsible for keeping track of which membership
        events we have already sent to the client, and hence ripping them out.

-        Note that whether this returns the state at the start or the end of the
-        batch depends on `sync_config.use_state_after` (c.f. MSC4222).
-
        Args:
            room_id: The room we are calculating for.
-            sync_config
            batch: The timeline batch for the room that will be sent to the user.
            since_token: Token of the end of the previous batch.
            end_token: Token of the end of the current batch. Normally this will be
@@ -1503,7 +1434,7 @@ class SyncHandler:

        Returns:
            A map from (type, state_key) to event_id, for each event that we believe
-            should be included in the `state` or `state_after` part of the sync response.
+            should be included in the `state` part of the sync response.
        """
        if members_to_fetch is not None:
            # Lazy-loading is enabled. Only return the state that is needed.
@@ -1515,51 +1446,6 @@ class SyncHandler:
            await_full_state = True
            lazy_load_members = False

-        # Check if we are wanting to return the state at the start or end of the
-        # timeline. If at the end we can just use the current state delta stream.
-        if sync_config.use_state_after:
-            delta_state_ids: MutableStateMap[str] = {}
-
-            if members_to_fetch is not None:
-                # We're lazy-loading, so the client might need some more member
-                # events to understand the events in this timeline. So we always
-                # fish out all the member events corresponding to the timeline
-                # here. The caller will then dedupe any redundant ones.
-                member_ids = await self._state_storage_controller.get_current_state_ids(
-                    room_id=room_id,
-                    state_filter=StateFilter.from_types(
-                        (EventTypes.Member, member) for member in members_to_fetch
-                    ),
-                    await_full_state=await_full_state,
-                )
-                delta_state_ids.update(member_ids)
-
-            # We don't do LL filtering for incremental syncs - see
-            # https://github.com/vector-im/riot-web/issues/7211#issuecomment-419976346
-            # N.B. this slows down incr syncs as we are now processing way more
-            # state in the server than if we were LLing.
-            #
-            # i.e. we return all state deltas, including membership changes that
-            # we'd normally exclude due to LL.
-            deltas = await self.store.get_current_state_deltas_for_room(
-                room_id=room_id,
-                from_token=since_token.room_key,
-                to_token=end_token.room_key,
-            )
-            for delta in deltas:
-                if delta.event_id is None:
-                    # There was a state reset and this state entry is no longer
-                    # present, but we have no way of informing the client about
-                    # this, so we just skip it for now.
-                    continue
-
-                # Note that deltas are in stream ordering, so if there are
-                # multiple deltas for a given type/state_key we'll always pick
-                # the latest one.
-                delta_state_ids[(delta.event_type, delta.state_key)] = delta.event_id
-
-            return delta_state_ids
-
        # For a non-gappy sync if the events in the timeline are simply a linear
        # chain (i.e. no merging/branching of the graph), then we know the state
        # delta between the end of the previous sync and start of the new one is
@@ -1650,6 +1536,17 @@ class SyncHandler:
            await_full_state=await_full_state,
        )

+        state_at_previous_sync_start = (
+            {}
+            if since_token.prev_batch is None
+            else await self._state_storage_controller.get_state_ids_at(
+                room_id,
+                stream_position=since_token.prev_batch,
+                state_filter=state_filter,
+                await_full_state=await_full_state,
+            )
+        )
+
        state_at_timeline_end = await self._state_storage_controller.get_state_ids_at(
            room_id,
            stream_position=end_token,
@@ -1661,6 +1558,7 @@ class SyncHandler:
            timeline_contains=timeline_state,
            timeline_start=state_at_timeline_start,
            timeline_end=state_at_timeline_end,
+            previous_timeline_start=state_at_previous_sync_start,
            previous_timeline_end=state_at_previous_sync,
            lazy_load_members=lazy_load_members,
        )
@@ -2080,7 +1978,7 @@ class SyncHandler:
        # this is due to some of the underlying streams not supporting the ability
        # to query up to a given point.
        # Always use the `now_token` in `SyncResultBuilder`
-        now_token = self.event_sources.get_current_token()
+        now_token = self.event_sources.get_current_token(prev_batch=since_token)
        log_kv({"now_token": now_token})

        # Since we fetched the users room list before calculating the `now_token` (see
@@ -2982,7 +2880,6 @@ class SyncHandler:
                    since_token,
                    room_builder.end_token,
                    full_state=full_state,
-                    joined=room_builder.rtype == "joined",
                )
            else:
                # An out of band room won't have any state changes.
@@ -3096,6 +2993,7 @@ def _calculate_state(
    timeline_contains: StateMap[str],
    timeline_start: StateMap[str],
    timeline_end: StateMap[str],
+    previous_timeline_start: StateMap[str],
    previous_timeline_end: StateMap[str],
    lazy_load_members: bool,
 ) -> StateMap[str]:
@@ -3123,6 +3021,7 @@ def _calculate_state(

    timeline_end_ids = set(timeline_end.values())
    timeline_start_ids = set(timeline_start.values())
+    previous_timeline_start_ids = set(previous_timeline_start.values())
    previous_timeline_end_ids = set(previous_timeline_end.values())
    timeline_contains_ids = set(timeline_contains.values())

@@ -3198,7 +3097,7 @@ def _calculate_state(

    state_ids = (
        (timeline_end_ids | timeline_start_ids)
-        - previous_timeline_end_ids
+        - (previous_timeline_end_ids | previous_timeline_start_ids)
        - timeline_contains_ids
    )

@@ -51,17 +51,25 @@ logger = logging.getLogger(__name__)
 # "Hop-by-hop" headers (as opposed to "end-to-end" headers) as defined by RFC2616
 # section 13.5.1 and referenced in RFC9110 section 7.6.1. These are meant to only be
 # consumed by the immediate recipient and not be forwarded on.
-HOP_BY_HOP_HEADERS_LOWERCASE = {
-    "connection",
-    "keep-alive",
-    "proxy-authenticate",
-    "proxy-authorization",
-    "te",
-    "trailers",
-    "transfer-encoding",
-    "upgrade",
+HOP_BY_HOP_HEADERS = {
+    "Connection",
+    "Keep-Alive",
+    "Proxy-Authenticate",
+    "Proxy-Authorization",
+    "TE",
+    "Trailers",
+    "Transfer-Encoding",
+    "Upgrade",
 }
-assert all(header.lower() == header for header in HOP_BY_HOP_HEADERS_LOWERCASE)
+
+if hasattr(Headers, "_canonicalNameCaps"):
+    # Twisted < 24.7.0rc1
+    _canonicalHeaderName = Headers()._canonicalNameCaps  # type: ignore[attr-defined]
+else:
+    # Twisted >= 24.7.0rc1
+    # But note that `_encodeName` still exists on prior versions,
+    # it just encodes differently
+    _canonicalHeaderName = Headers()._encodeName


 def parse_connection_header_value(
@@ -84,12 +92,12 @@ def parse_connection_header_value(

    Returns:
        The set of header names that should not be copied over from the remote response.
-        The keys are lowercased.
+        The keys are capitalized in canonical capitalization.
    """
    extra_headers_to_remove: Set[str] = set()
    if connection_header_value:
        extra_headers_to_remove = {
-            connection_option.decode("ascii").strip().lower()
+            _canonicalHeaderName(connection_option.strip()).decode("ascii")
            for connection_option in connection_header_value.split(b",")
        }

@@ -186,7 +194,7 @@ class ProxyResource(_AsyncResource):

        # The `Connection` header also defines which headers should not be copied over.
        connection_header = response_headers.getRawHeaders(b"connection")
-        extra_headers_to_remove_lowercase = parse_connection_header_value(
+        extra_headers_to_remove = parse_connection_header_value(
            connection_header[0] if connection_header else None
        )

@@ -194,10 +202,10 @@ class ProxyResource(_AsyncResource):
        for k, v in response_headers.getAllRawHeaders():
            # Do not copy over any hop-by-hop headers. These are meant to only be
            # consumed by the immediate recipient and not be forwarded on.
-            header_key_lowercase = k.decode("ascii").lower()
+            header_key = k.decode("ascii")
            if (
-                header_key_lowercase in HOP_BY_HOP_HEADERS_LOWERCASE
-                or header_key_lowercase in extra_headers_to_remove_lowercase
+                header_key in HOP_BY_HOP_HEADERS
+                or header_key in extra_headers_to_remove
            ):
                continue

@@ -39,7 +39,7 @@ from twisted.internet.endpoints import (
 )
 from twisted.internet.interfaces import (
    IPushProducer,
-    IReactorTime,
+    IReactorTCP,
    IStreamClientEndpoint,
 )
 from twisted.internet.protocol import Factory, Protocol
@@ -113,7 +113,7 @@ class RemoteHandler(logging.Handler):
        port: int,
        maximum_buffer: int = 1000,
        level: int = logging.NOTSET,
-        _reactor: Optional[IReactorTime] = None,
+        _reactor: Optional[IReactorTCP] = None,
    ):
        super().__init__(level=level)
        self.host = host
@@ -37,7 +37,6 @@ import warnings
 from types import TracebackType
 from typing import (
    TYPE_CHECKING,
-    Any,
    Awaitable,
    Callable,
    Optional,
@@ -851,45 +850,6 @@ def run_in_background(
    return d


-def run_coroutine_in_background(
-    coroutine: typing.Coroutine[Any, Any, R],
-) -> "defer.Deferred[R]":
-    """Run the coroutine, ensuring that the current context is restored after
-    return from the function, and that the sentinel context is set once the
-    deferred returned by the function completes.
-
-    Useful for wrapping coroutines that you don't yield or await on (for
-    instance because you want to pass it to deferred.gatherResults()).
-
-    This is a special case of `run_in_background` where we can accept a
-    coroutine directly rather than a function. We can do this because coroutines
-    do not run until called, and so calling an async function without awaiting
-    cannot change the log contexts.
-    """
-
-    current = current_context()
-    d = defer.ensureDeferred(coroutine)
-
-    # The function may have reset the context before returning, so
-    # we need to restore it now.
-    ctx = set_current_context(current)
-
-    # The original context will be restored when the deferred
-    # completes, but there is nothing waiting for it, so it will
-    # get leaked into the reactor or some other function which
-    # wasn't expecting it. We therefore need to reset the context
-    # here.
-    #
-    # (If this feels asymmetric, consider it this way: we are
-    # effectively forking a new thread of execution. We are
-    # probably currently within a ``with LoggingContext()`` block,
-    # which is supposed to have a single entry and exit point. But
-    # by spawning off another deferred, we are effectively
-    # adding a new exit point.)
-    d.addBoth(_set_context_cb, ctx)
-    return d
-
-
 T = TypeVar("T")


@@ -43,15 +43,12 @@ class ExperimentalFeature(str, Enum):

    MSC3881 = "msc3881"
    MSC3575 = "msc3575"
-    MSC4222 = "msc4222"

    def is_globally_enabled(self, config: "HomeServerConfig") -> bool:
        if self is ExperimentalFeature.MSC3881:
            return config.experimental.msc3881_enabled
        if self is ExperimentalFeature.MSC3575:
            return config.experimental.msc3575_enabled
-        if self is ExperimentalFeature.MSC4222:
-            return config.experimental.msc4222_enabled

        assert_never(self)

@@ -20,13 +20,11 @@
 #

 import logging
-import re
 from http import HTTPStatus
 from typing import TYPE_CHECKING, Tuple

 from synapse._pydantic_compat import StrictStr
 from synapse.api.errors import AuthError, Codes, NotFoundError, SynapseError
-from synapse.api.urls import CLIENT_API_PREFIX
 from synapse.http.server import HttpServer
 from synapse.http.servlet import (
    RestServlet,
@@ -107,17 +105,18 @@ class ReportEventRestServlet(RestServlet):
 class ReportRoomRestServlet(RestServlet):
    """This endpoint lets clients report a room for abuse.

-    Introduced by MSC4151: https://github.com/matrix-org/matrix-spec-proposals/pull/4151
+    Whilst MSC4151 is not yet merged, this unstable endpoint is enabled on matrix.org
+    for content moderation purposes, and therefore backwards compatibility should be
+    carefully considered when changing anything on this endpoint.
+
+    More details on the MSC: https://github.com/matrix-org/matrix-spec-proposals/pull/4151
    """

-    # Cast the Iterable to a list so that we can `append` below.
-    PATTERNS = list(
-        client_patterns(
-            "/rooms/(?P<room_id>[^/]*)/report$",
-            releases=("v3",),
-            unstable=False,
-            v1=False,
-        )
+    PATTERNS = client_patterns(
+        "/org.matrix.msc4151/rooms/(?P<room_id>[^/]*)/report$",
+        releases=[],
+        v1=False,
+        unstable=True,
    )

    def __init__(self, hs: "HomeServer"):
@@ -127,16 +126,6 @@ class ReportRoomRestServlet(RestServlet):
        self.clock = hs.get_clock()
        self.store = hs.get_datastores().main

-        # TODO: Remove the unstable variant after 2-3 releases
-        # https://github.com/element-hq/synapse/issues/17373
-        if hs.config.experimental.msc4151_enabled:
-            self.PATTERNS.append(
-                re.compile(
-                    f"^{CLIENT_API_PREFIX}/unstable/org.matrix.msc4151"
-                    "/rooms/(?P<room_id>[^/]*)/report$"
-                )
-            )
-
    class PostBody(RequestBodyModel):
        reason: StrictStr

@@ -164,4 +153,6 @@ class ReportRoomRestServlet(RestServlet):

 def register_servlets(hs: "HomeServer", http_server: HttpServer) -> None:
    ReportEventRestServlet(hs).register(http_server)
-    ReportRoomRestServlet(hs).register(http_server)
+
+    if hs.config.experimental.msc4151_enabled:
+        ReportRoomRestServlet(hs).register(http_server)
@@ -152,14 +152,6 @@ class SyncRestServlet(RestServlet):
        filter_id = parse_string(request, "filter")
        full_state = parse_boolean(request, "full_state", default=False)

-        use_state_after = False
-        if await self.store.is_feature_enabled(
-            user.to_string(), ExperimentalFeature.MSC4222
-        ):
-            use_state_after = parse_boolean(
-                request, "org.matrix.msc4222.use_state_after", default=False
-            )
-
        logger.debug(
            "/sync: user=%r, timeout=%r, since=%r, "
            "set_presence=%r, filter_id=%r, device_id=%r",
@@ -192,7 +184,6 @@ class SyncRestServlet(RestServlet):
            full_state,
            device_id,
            last_ignore_accdata_streampos,
-            use_state_after,
        )

        if filter_id is None:
@@ -229,7 +220,6 @@ class SyncRestServlet(RestServlet):
            filter_collection=filter_collection,
            is_guest=requester.is_guest,
            device_id=device_id,
-            use_state_after=use_state_after,
        )

        since_token = None
@@ -268,7 +258,7 @@ class SyncRestServlet(RestServlet):
        # We know that the the requester has an access token since appservices
        # cannot use sync.
        response_content = await self.encode_response(
-            time_now, sync_config, sync_result, requester, filter_collection
+            time_now, sync_result, requester, filter_collection
        )

        logger.debug("Event formatting complete")
@@ -278,7 +268,6 @@ class SyncRestServlet(RestServlet):
    async def encode_response(
        self,
        time_now: int,
-        sync_config: SyncConfig,
        sync_result: SyncResult,
        requester: Requester,
        filter: FilterCollection,
@@ -303,7 +292,7 @@ class SyncRestServlet(RestServlet):
        )

        joined = await self.encode_joined(
-            sync_config, sync_result.joined, time_now, serialize_options
+            sync_result.joined, time_now, serialize_options
        )

        invited = await self.encode_invited(
@@ -315,7 +304,7 @@ class SyncRestServlet(RestServlet):
        )

        archived = await self.encode_archived(
-            sync_config, sync_result.archived, time_now, serialize_options
+            sync_result.archived, time_now, serialize_options
        )

        logger.debug("building sync response dict")
@@ -383,7 +372,6 @@ class SyncRestServlet(RestServlet):
    @trace_with_opname("sync.encode_joined")
    async def encode_joined(
        self,
-        sync_config: SyncConfig,
        rooms: List[JoinedSyncResult],
        time_now: int,
        serialize_options: SerializeEventConfig,
@@ -392,7 +380,6 @@ class SyncRestServlet(RestServlet):
        Encode the joined rooms in a sync result

        Args:
-            sync_config
            rooms: list of sync results for rooms this user is joined to
            time_now: current time - used as a baseline for age calculations
            serialize_options: Event serializer options
@@ -402,11 +389,7 @@ class SyncRestServlet(RestServlet):
        joined = {}
        for room in rooms:
            joined[room.room_id] = await self.encode_room(
-                sync_config,
-                room,
-                time_now,
-                joined=True,
-                serialize_options=serialize_options,
+                room, time_now, joined=True, serialize_options=serialize_options
            )

        return joined
@@ -494,7 +477,6 @@ class SyncRestServlet(RestServlet):
    @trace_with_opname("sync.encode_archived")
    async def encode_archived(
        self,
-        sync_config: SyncConfig,
        rooms: List[ArchivedSyncResult],
        time_now: int,
        serialize_options: SerializeEventConfig,
@@ -503,7 +485,6 @@ class SyncRestServlet(RestServlet):
        Encode the archived rooms in a sync result

        Args:
-            sync_config
            rooms: list of sync results for rooms this user is joined to
            time_now: current time - used as a baseline for age calculations
            serialize_options: Event serializer options
@@ -513,18 +494,13 @@ class SyncRestServlet(RestServlet):
        joined = {}
        for room in rooms:
            joined[room.room_id] = await self.encode_room(
-                sync_config,
-                room,
-                time_now,
-                joined=False,
-                serialize_options=serialize_options,
+                room, time_now, joined=False, serialize_options=serialize_options
            )

        return joined

    async def encode_room(
        self,
-        sync_config: SyncConfig,
        room: Union[JoinedSyncResult, ArchivedSyncResult],
        time_now: int,
        joined: bool,
@@ -532,7 +508,6 @@ class SyncRestServlet(RestServlet):
    ) -> JsonDict:
        """
        Args:
-            sync_config
            room: sync result for a single room
            time_now: current time - used as a baseline for age calculations
            token_id: ID of the user's auth token - used for namespacing
@@ -573,20 +548,13 @@ class SyncRestServlet(RestServlet):

        account_data = room.account_data

-        # We either include a `state` or `state_after` field depending on
-        # whether the client has opted in to the newer `state_after` behavior.
-        if sync_config.use_state_after:
-            state_key_name = "org.matrix.msc4222.state_after"
-        else:
-            state_key_name = "state"
-
        result: JsonDict = {
            "timeline": {
                "events": serialized_timeline,
                "prev_batch": await room.timeline.prev_batch.to_string(self.store),
                "limited": room.timeline.limited,
            },
-            state_key_name: {"events": serialized_state},
+            "state": {"events": serialized_state},
            "account_data": {"events": account_data},
        }

@@ -720,7 +688,6 @@ class SlidingSyncE2eeRestServlet(RestServlet):
            filter_collection=self.only_member_events_filter_collection,
            is_guest=requester.is_guest,
            device_id=device_id,
-            use_state_after=False,  # We don't return any rooms so this flag is a no-op
        )

        since_token = None
@@ -78,7 +78,6 @@ class TagServlet(RestServlet):
        super().__init__()
        self.auth = hs.get_auth()
        self.handler = hs.get_account_data_handler()
-        self.room_member_handler = hs.get_room_member_handler()

    async def on_PUT(
        self, request: SynapseRequest, user_id: str, room_id: str, tag: str
@@ -86,12 +85,6 @@ class TagServlet(RestServlet):
        requester = await self.auth.get_user_by_req(request)
        if user_id != requester.user.to_string():
            raise AuthError(403, "Cannot add tags for other users.")
-        # Check if the user has any membership in the room and raise error if not.
-        # Although it's not harmful for users to tag random rooms, it's just superfluous
-        # data we don't need to track or allow.
-        await self.room_member_handler.check_for_any_membership_in_room(
-            user_id=user_id, room_id=room_id
-        )

        body = parse_json_object_from_request(request)

@@ -249,7 +249,6 @@ class HomeServer(metaclass=abc.ABCMeta):
    """

    REQUIRED_ON_BACKGROUND_TASK_STARTUP = [
-        "admin",
        "account_validity",
        "auth",
        "deactivate_account",
@@ -1422,7 +1422,7 @@ class DeviceWorkerStore(RoomMemberWorkerStore, EndToEndKeyWorkerStore):
                DELETE FROM device_lists_outbound_last_success
                WHERE destination = ? AND user_id = ?
            """
-            txn.execute_batch(sql, [(row[0], row[1]) for row in rows])
+            txn.execute_batch(sql, ((row[0], row[1]) for row in rows))

            logger.info("Pruned %d device list outbound pokes", count)

@@ -99,13 +99,6 @@ class EndToEndKeyBackgroundStore(SQLBaseStore):
            unique=True,
        )

-        self.db_pool.updates.register_background_index_update(
-            update_name="add_otk_ts_added_index",
-            index_name="e2e_one_time_keys_json_user_id_device_id_algorithm_ts_added_idx",
-            table="e2e_one_time_keys_json",
-            columns=("user_id", "device_id", "algorithm", "ts_added_ms"),
-        )
-

 class EndToEndKeyWorkerStore(EndToEndKeyBackgroundStore, CacheInvalidationWorkerStore):
    def __init__(
@@ -1129,7 +1122,7 @@ class EndToEndKeyWorkerStore(EndToEndKeyBackgroundStore, CacheInvalidationWorker
        """Take a list of one time keys out of the database.

        Args:
-            query_list: An iterable of tuples of (user ID, device ID, algorithm, number of keys).
+            query_list: An iterable of tuples of (user ID, device ID, algorithm).

        Returns:
            A tuple (results, missing) of:
@@ -1317,14 +1310,9 @@ class EndToEndKeyWorkerStore(EndToEndKeyBackgroundStore, CacheInvalidationWorker
            OTK was found.
        """

-        # Return the oldest keys from this device (based on `ts_added_ms`).
-        # Doing so means that keys are issued in the same order they were uploaded,
-        # which reduces the chances of a client expiring its copy of a (private)
-        # key while the public key is still on the server, waiting to be issued.
        sql = """
            SELECT key_id, key_json FROM e2e_one_time_keys_json
            WHERE user_id = ? AND device_id = ? AND algorithm = ?
-            ORDER BY ts_added_ms
            LIMIT ?
        """

@@ -1366,22 +1354,13 @@ class EndToEndKeyWorkerStore(EndToEndKeyBackgroundStore, CacheInvalidationWorker
            A list of tuples (user_id, device_id, algorithm, key_id, key_json)
            for each OTK claimed.
        """
-        # Find, delete, and return the oldest keys from each device (based on
-        # `ts_added_ms`).
-        #
-        # Doing so means that keys are issued in the same order they were uploaded,
-        # which reduces the chances of a client expiring its copy of a (private)
-        # key while the public key is still on the server, waiting to be issued.
        sql = """
            WITH claims(user_id, device_id, algorithm, claim_count) AS (
                VALUES ?
            ), ranked_keys AS (
                SELECT
                    user_id, device_id, algorithm, key_id, claim_count,
-                    ROW_NUMBER() OVER (
-                        PARTITION BY (user_id, device_id, algorithm)
-                        ORDER BY ts_added_ms
-                    ) AS r
+                    ROW_NUMBER() OVER (PARTITION BY (user_id, device_id, algorithm)) AS r
                FROM e2e_one_time_keys_json
                    JOIN claims USING (user_id, device_id, algorithm)
            )
@@ -1686,7 +1686,7 @@ class PersistEventsStore:
                """
            txn.execute_batch(
                sql,
-                [
+                (
                    (
                        stream_id,
                        self._instance_name,
@@ -1699,17 +1699,17 @@ class PersistEventsStore:
                        state_key,
                    )
                    for etype, state_key in itertools.chain(to_delete, to_insert)
-                ],
+                ),
            )
            # Now we actually update the current_state_events table

            txn.execute_batch(
                "DELETE FROM current_state_events"
                " WHERE room_id = ? AND type = ? AND state_key = ?",
-                [
+                (
                    (room_id, etype, state_key)
                    for etype, state_key in itertools.chain(to_delete, to_insert)
-                ],
+                ),
            )

            # We include the membership in the current state table, hence we do
@@ -1799,11 +1799,11 @@ class PersistEventsStore:
            txn.execute_batch(
                "DELETE FROM local_current_membership"
                " WHERE room_id = ? AND user_id = ?",
-                [
+                (
                    (room_id, state_key)
                    for etype, state_key in itertools.chain(to_delete, to_insert)
                    if etype == EventTypes.Member and self.is_mine_id(state_key)
-                ],
+                ),
            )

        if to_insert:
@@ -3208,7 +3208,7 @@ class PersistEventsStore:
        if notifiable_events:
            txn.execute_batch(
                sql,
-                [
+                (
                    (
                        event.room_id,
                        event.internal_metadata.stream_ordering,
@@ -3216,18 +3216,18 @@ class PersistEventsStore:
                        event.event_id,
                    )
                    for event in notifiable_events
-                ],
+                ),
            )

        # Now we delete the staging area for *all* events that were being
        # persisted.
        txn.execute_batch(
            "DELETE FROM event_push_actions_staging WHERE event_id = ?",
-            [
+            (
                (event.event_id,)
                for event, _ in all_events_and_contexts
                if event.internal_metadata.is_notifiable()
-            ],
+            ),
        )

    def _remove_push_actions_for_event_id_txn(
@@ -729,10 +729,10 @@ class MediaRepositoryStore(MediaRepositoryBackgroundUpdateStore):

            txn.execute_batch(
                sql,
-                [
+                (
                    (time_ms, media_origin, media_id)
                    for media_origin, media_id in remote_media
-                ],
+                ),
            )

            sql = (
@@ -740,7 +740,7 @@ class MediaRepositoryStore(MediaRepositoryBackgroundUpdateStore):
                " WHERE media_id = ?"
            )

-            txn.execute_batch(sql, [(time_ms, media_id) for media_id in local_media])
+            txn.execute_batch(sql, ((time_ms, media_id) for media_id in local_media))

        await self.db_pool.runInteraction(
            "update_cached_last_access_time", update_cache_txn
@@ -1175,7 +1175,7 @@ class RoomWorkerStore(CacheInvalidationWorkerStore):
                SET quarantined_by = ?
                WHERE media_origin = ? AND media_id = ?
            """,
-            [(quarantined_by, origin, media_id) for origin, media_id in remote_mxcs],
+            ((quarantined_by, origin, media_id) for origin, media_id in remote_mxcs),
        )
        total_media_quarantined += txn.rowcount if txn.rowcount > 0 else 0

@@ -2550,9 +2550,7 @@ class RoomStore(RoomBackgroundUpdateStore, RoomWorkerStore):
            still contains events with partial state.
        """
        try:
-            async with (
-                self._un_partial_stated_rooms_stream_id_gen.get_next() as un_partial_state_room_stream_id
-            ):
+            async with self._un_partial_stated_rooms_stream_id_gen.get_next() as un_partial_state_room_stream_id:
                await self.db_pool.runInteraction(
                    "clear_partial_state_room",
                    self._clear_partial_state_room_txn,
@@ -94,7 +94,7 @@ class SearchWorkerStore(SQLBaseStore):
            VALUES (?,?,?,to_tsvector('english', ?),?,?)
            """

-            args1 = [
+            args1 = (
                (
                    entry.event_id,
                    entry.room_id,
@@ -104,7 +104,7 @@ class SearchWorkerStore(SQLBaseStore):
                    entry.origin_server_ts,
                )
                for entry in entries
-            ]
+            )

            txn.execute_batch(sql, args1)

@@ -681,9 +681,7 @@ class StateGroupWorkerStore(EventsWorkerStore, SQLBaseStore):
        context: EventContext,
    ) -> None:
        """Update the state group for a partial state event"""
-        async with (
-            self._un_partial_stated_events_stream_id_gen.get_next() as un_partial_state_event_stream_id
-        ):
+        async with self._un_partial_stated_events_stream_id_gen.get_next() as un_partial_state_event_stream_id:
            await self.db_pool.runInteraction(
                "update_state_for_partial_state_event",
                self._update_state_for_partial_state_event_txn,
@@ -804,11 +804,11 @@ class StateGroupDataStore(StateBackgroundUpdateStore, SQLBaseStore):
        logger.info("[purge] removing redundant state groups")
        txn.execute_batch(
            "DELETE FROM state_groups_state WHERE state_group = ?",
-            [(sg,) for sg in state_groups_to_delete],
+            ((sg,) for sg in state_groups_to_delete),
        )
        txn.execute_batch(
            "DELETE FROM state_groups WHERE id = ?",
-            [(sg,) for sg in state_groups_to_delete],
+            ((sg,) for sg in state_groups_to_delete),
        )

    @trace
@@ -1,18 +0,0 @@
--
-- This file is licensed under the Affero General Public License (AGPL) version 3.
--
-- Copyright (C) 2024 New Vector, Ltd
--
-- This program is free software: you can redistribute it and/or modify
-- it under the terms of the GNU Affero General Public License as
-- published by the Free Software Foundation, either version 3 of the
-- License, or (at your option) any later version.
--
-- See the GNU Affero General Public License for more details:
-- <https://www.gnu.org/licenses/agpl-3.0.html>.
-
-
-- Add an index on (user_id, device_id, algorithm, ts_added_ms) on e2e_one_time_keys_json, so that OTKs can
-- efficiently be issued in the same order they were uploaded.
-INSERT INTO background_updates (ordering, update_name, progress_json) VALUES
-    (8803, 'add_otk_ts_added_index', '{}');
@@ -77,7 +77,7 @@ class EventSources:
        self.store = hs.get_datastores().main
        self._instance_name = hs.get_instance_name()

-    def get_current_token(self) -> StreamToken:
+    def get_current_token(self, prev_batch: StreamToken = None) -> StreamToken:
        push_rules_key = self.store.get_max_push_rules_stream_id()
        to_device_key = self.store.get_to_device_stream_token()
        device_list_key = self.store.get_device_stream_token()
@@ -97,6 +97,7 @@ class EventSources:
            # Groups key is unused.
            groups_key=0,
            un_partial_stated_rooms_key=un_partial_stated_rooms_key,
+            prev_batch=prev_batch,
        )
        return token

@@ -980,16 +980,30 @@ class StreamToken:
    groups_key: int
    un_partial_stated_rooms_key: int

+    prev_batch: Optional["StreamToken"] = None
+    _BATCH_SEPARATOR = "~"
+
    _SEPARATOR = "_"
    START: ClassVar["StreamToken"]

    @classmethod
    @cancellable
-    async def from_string(cls, store: "DataStore", string: str) -> "StreamToken":
+    async def from_string(
+        cls, store: "DataStore", string: str, prev_batch: Optional["StreamToken"] = None
+    ) -> "StreamToken":
        """
        Creates a RoomStreamToken from its textual representation.
        """
        try:
+            if string.count(cls._BATCH_SEPARATOR) == 1:
+                # We have a prev_token
+                batches = string.split(cls._BATCH_SEPARATOR)
+                prev_batch = await StreamToken.from_string(store, batches[1])
+                batch = await StreamToken.from_string(
+                    store, batches[0], prev_batch=prev_batch
+                )
+                return batch
+
            keys = string.split(cls._SEPARATOR)
            while len(keys) < len(attr.fields(cls)):
                # i.e. old token from before receipt_key
@@ -1006,6 +1020,7 @@ class StreamToken:
                device_list_key,
                groups_key,
                un_partial_stated_rooms_key,
+                prev_batch,
            ) = keys

            return cls(
@@ -1025,24 +1040,34 @@ class StreamToken:
        except Exception:
            raise SynapseError(400, "Invalid stream token")

-    async def to_string(self, store: "DataStore") -> str:
-        return self._SEPARATOR.join(
-            [
-                await self.room_key.to_string(store),
-                str(self.presence_key),
-                str(self.typing_key),
-                await self.receipt_key.to_string(store),
-                str(self.account_data_key),
-                str(self.push_rules_key),
-                str(self.to_device_key),
-                str(self.device_list_key),
-                # Note that the groups key is no longer used, but it is still
-                # serialized so that there will not be confusion in the future
-                # if additional tokens are added.
-                str(self.groups_key),
-                str(self.un_partial_stated_rooms_key),
-            ]
-        )
+    async def to_string(
+        self, store: "DataStore", include_prev_batch: bool = True
+    ) -> str:
+        if include_prev_batch and self.prev_batch:
+            return self._BATCH_SEPARATOR.join(
+                [
+                    await self.to_string(store, include_prev_batch=False),
+                    await self.prev_batch.to_string(store, include_prev_batch=False),
+                ]
+            )
+        else:
+            return self._SEPARATOR.join(
+                [
+                    await self.room_key.to_string(store),
+                    str(self.presence_key),
+                    str(self.typing_key),
+                    await self.receipt_key.to_string(store),
+                    str(self.account_data_key),
+                    str(self.push_rules_key),
+                    str(self.to_device_key),
+                    str(self.device_list_key),
+                    # Note that the groups key is no longer used, but it is still
+                    # serialized so that there will not be confusion in the future
+                    # if additional tokens are added.
+                    str(self.groups_key),
+                    str(self.un_partial_stated_rooms_key),
+                ]
+            )

    @property
    def room_stream_id(self) -> int:
@@ -51,7 +51,7 @@ from typing import (
 )

 import attr
-from typing_extensions import Concatenate, Literal, ParamSpec, Unpack
+from typing_extensions import Concatenate, Literal, ParamSpec

 from twisted.internet import defer
 from twisted.internet.defer import CancelledError
@@ -61,7 +61,6 @@ from twisted.python.failure import Failure
 from synapse.logging.context import (
    PreserveLoggingContext,
    make_deferred_yieldable,
-    run_coroutine_in_background,
    run_in_background,
 )
 from synapse.util import Clock
@@ -345,7 +344,6 @@ T1 = TypeVar("T1")
 T2 = TypeVar("T2")
 T3 = TypeVar("T3")
 T4 = TypeVar("T4")
-T5 = TypeVar("T5")


@overload
@@ -404,112 +402,6 @@ def gather_results(  # type: ignore[misc]
    return deferred.addCallback(tuple)


-@overload
-async def gather_optional_coroutines(
-    *coroutines: Unpack[Tuple[Optional[Coroutine[Any, Any, T1]]]],
-) -> Tuple[Optional[T1]]: ...
-
-
-@overload
-async def gather_optional_coroutines(
-    *coroutines: Unpack[
-        Tuple[
-            Optional[Coroutine[Any, Any, T1]],
-            Optional[Coroutine[Any, Any, T2]],
-        ]
-    ],
-) -> Tuple[Optional[T1], Optional[T2]]: ...
-
-
-@overload
-async def gather_optional_coroutines(
-    *coroutines: Unpack[
-        Tuple[
-            Optional[Coroutine[Any, Any, T1]],
-            Optional[Coroutine[Any, Any, T2]],
-            Optional[Coroutine[Any, Any, T3]],
-        ]
-    ],
-) -> Tuple[Optional[T1], Optional[T2], Optional[T3]]: ...
-
-
-@overload
-async def gather_optional_coroutines(
-    *coroutines: Unpack[
-        Tuple[
-            Optional[Coroutine[Any, Any, T1]],
-            Optional[Coroutine[Any, Any, T2]],
-            Optional[Coroutine[Any, Any, T3]],
-            Optional[Coroutine[Any, Any, T4]],
-        ]
-    ],
-) -> Tuple[Optional[T1], Optional[T2], Optional[T3], Optional[T4]]: ...
-
-
-@overload
-async def gather_optional_coroutines(
-    *coroutines: Unpack[
-        Tuple[
-            Optional[Coroutine[Any, Any, T1]],
-            Optional[Coroutine[Any, Any, T2]],
-            Optional[Coroutine[Any, Any, T3]],
-            Optional[Coroutine[Any, Any, T4]],
-            Optional[Coroutine[Any, Any, T5]],
-        ]
-    ],
-) -> Tuple[Optional[T1], Optional[T2], Optional[T3], Optional[T4], Optional[T5]]: ...
-
-
-async def gather_optional_coroutines(
-    *coroutines: Unpack[Tuple[Optional[Coroutine[Any, Any, T1]], ...]],
-) -> Tuple[Optional[T1], ...]:
-    """Helper function that allows waiting on multiple coroutines at once.
-
-    The return value is a tuple of the return values of the coroutines in order.
-
-    If a `None` is passed instead of a coroutine, it will be ignored and a None
-    is returned in the tuple.
-
-    Note: For typechecking we need to have an explicit overload for each
-    distinct number of coroutines passed in. If you see type problems, it's
-    likely because you're using many arguments and you need to add a new
-    overload above.
-    """
-
-    try:
-        results = await make_deferred_yieldable(
-            defer.gatherResults(
-                [
-                    run_coroutine_in_background(coroutine)
-                    for coroutine in coroutines
-                    if coroutine is not None
-                ],
-                consumeErrors=True,
-            )
-        )
-
-        results_iter = iter(results)
-        return tuple(
-            next(results_iter) if coroutine is not None else None
-            for coroutine in coroutines
-        )
-    except defer.FirstError as dfe:
-        # unwrap the error from defer.gatherResults.
-
-        # The raised exception's traceback only includes func() etc if
-        # the 'await' happens before the exception is thrown - ie if the failure
-        # happens *asynchronously* - otherwise Twisted throws away the traceback as it
-        # could be large.
-        #
-        # We could maybe reconstruct a fake traceback from Failure.frames. Or maybe
-        # we could throw Twisted into the fires of Mordor.
-
-        # suppress exception chaining, because the FirstError doesn't tell us anything
-        # very interesting.
-        assert isinstance(dfe.subFailure.value, BaseException)
-        raise dfe.subFailure.value from None
-
-
@attr.s(slots=True, auto_attribs=True)
 class _LinearizerEntry:
    # The number of things executing.
@@ -19,12 +19,9 @@
 #
 #

-import json
 import os
-import urllib.parse
+import sys
 from hashlib import blake2b
-from importlib.metadata import Distribution, PackageNotFoundError
-from typing import Optional

 import synapse
 from synapse.synapse_rust import get_rust_file_digest
@@ -35,17 +32,22 @@ def check_rust_lib_up_to_date() -> None:
    be rebuilt.
    """

-    # Get the location of the editable install.
-    synapse_root = get_synapse_source_directory()
-    if synapse_root is None:
-        return None
+    if not _dist_is_editable():
+        return
+
+    synapse_dir = os.path.dirname(synapse.__file__)
+    synapse_root = os.path.abspath(os.path.join(synapse_dir, ".."))
+
+    # Double check we've not gone into site-packages...
+    if os.path.basename(synapse_root) == "site-packages":
+        return
+
+    # ... and it looks like the root of a python project.
+    if not os.path.exists("pyproject.toml"):
+        return

    # Get the hash of all Rust source files
-    rust_path = os.path.join(synapse_root, "rust", "src")
-    if not os.path.exists(rust_path):
-        return None
-
-    hash = _hash_rust_files_in_directory(rust_path)
+    hash = _hash_rust_files_in_directory(os.path.join(synapse_root, "rust", "src"))

    if hash != get_rust_file_digest():
        raise Exception("Rust module outdated. Please rebuild using `poetry install`")
@@ -80,55 +82,10 @@ def _hash_rust_files_in_directory(directory: str) -> str:
    return hasher.hexdigest()


-def get_synapse_source_directory() -> Optional[str]:
-    """Try and find the source directory of synapse for editable installs (like
-    those used in development).
-
-    Returns None if not an editable install (or otherwise can't find the source
-    directory).
-    """
-
-    # Try and find the installed matrix-synapse package.
-    try:
-        package = Distribution.from_name("matrix-synapse")
-    except PackageNotFoundError:
-        # The package is not found, so it's not installed and so must be being
-        # pulled out from a local directory (usually the current one).
-        synapse_dir = os.path.dirname(synapse.__file__)
-        synapse_root = os.path.abspath(os.path.join(synapse_dir, ".."))
-
-        # Double check we've not gone into site-packages...
-        if os.path.basename(synapse_root) == "site-packages":
-            return None
-
-        # ... and it looks like the root of a python project.
-        if not os.path.exists("pyproject.toml"):
-            return None
-
-        return synapse_root
-
-    # Read the `direct_url.json` metadata for the package. This won't exist for
-    # packages installed via a repository/etc.
-    # c.f. https://packaging.python.org/en/latest/specifications/direct-url/
-    direct_url_json = package.read_text("direct_url.json")
-    if direct_url_json is None:
-        return None
-
-    # c.f. https://packaging.python.org/en/latest/specifications/direct-url/ for
-    # the format
-    direct_url_dict: dict = json.loads(direct_url_json)
-
-    # `url` must exist as a key, and point to where we fetched the repo from.
-    project_url = urllib.parse.urlparse(direct_url_dict["url"])
-
-    # If its not a local file then we must have built the rust libs either a)
-    # after we downloaded the package, or b) we built the download wheel.
-    if project_url.scheme != "file":
-        return None
-
-    # And finally if its not an editable install then the files can't have
-    # changed since we installed the package.
-    if not direct_url_dict.get("dir_info", {}).get("editable", False):
-        return None
-
-    return project_url.path
+def _dist_is_editable() -> bool:
+    """Is distribution an editable install?"""
+    for path_item in sys.path:
+        egg_link = os.path.join(path_item, "matrix-synapse.egg-link")
+        if os.path.isfile(egg_link):
+            return True
+    return False
@@ -47,6 +47,7 @@ class WheelTimer(Generic[T]):
        """
        self.bucket_size: int = bucket_size
        self.entries: List[_Entry[T]] = []
+        self.current_tick: int = 0

    def insert(self, now: int, obj: T, then: int) -> None:
        """Inserts object into timer.
@@ -77,10 +78,11 @@ class WheelTimer(Generic[T]):
                self.entries[max(min_key, then_key) - min_key].elements.add(obj)
                return

+        next_key = now_key + 1
        if self.entries:
-            last_key = self.entries[-1].end_key + 1
+            last_key = self.entries[-1].end_key
        else:
-            last_key = now_key + 1
+            last_key = next_key

        # Handle the case when `then` is in the past and `entries` is empty.
        then_key = max(last_key, then_key)
@@ -151,30 +151,18 @@ class E2eKeysHandlerTestCase(unittest.HomeserverTestCase):
    def test_claim_one_time_key(self) -> None:
        local_user = "@boris:" + self.hs.hostname
        device_id = "xyz"
+        keys = {"alg1:k1": "key1"}
+
        res = self.get_success(
            self.handler.upload_keys_for_user(
-                local_user, device_id, {"one_time_keys": {"alg1:k1": "key1"}}
+                local_user, device_id, {"one_time_keys": keys}
            )
        )
        self.assertDictEqual(
            res, {"one_time_key_counts": {"alg1": 1, "signed_curve25519": 0}}
        )

-        # Keys should be returned in the order they were uploaded. To test, advance time
-        # a little, then upload a second key with an earlier key ID; it should get
-        # returned second.
-        self.reactor.advance(1)
-        res = self.get_success(
-            self.handler.upload_keys_for_user(
-                local_user, device_id, {"one_time_keys": {"alg1:k0": "key0"}}
-            )
-        )
-        self.assertDictEqual(
-            res, {"one_time_key_counts": {"alg1": 2, "signed_curve25519": 0}}
-        )
-
-        # now claim both keys back. They should be in the same order
-        res = self.get_success(
+        res2 = self.get_success(
            self.handler.claim_one_time_keys(
                {local_user: {device_id: {"alg1": 1}}},
                self.requester,
@@ -183,27 +171,12 @@ class E2eKeysHandlerTestCase(unittest.HomeserverTestCase):
            )
        )
        self.assertEqual(
-            res,
+            res2,
            {
                "failures": {},
                "one_time_keys": {local_user: {device_id: {"alg1:k1": "key1"}}},
            },
        )
-        res = self.get_success(
-            self.handler.claim_one_time_keys(
-                {local_user: {device_id: {"alg1": 1}}},
-                self.requester,
-                timeout=None,
-                always_include_fallback_keys=False,
-            )
-        )
-        self.assertEqual(
-            res,
-            {
-                "failures": {},
-                "one_time_keys": {local_user: {device_id: {"alg1:k0": "key0"}}},
-            },
-        )

    def test_claim_one_time_key_bulk(self) -> None:
        """Like test_claim_one_time_key but claims multiple keys in one handler call."""
@@ -363,47 +336,6 @@ class E2eKeysHandlerTestCase(unittest.HomeserverTestCase):
                    counts_by_alg, expected_counts_by_alg, f"{user_id}:{device_id}"
                )

-    def test_claim_one_time_key_bulk_ordering(self) -> None:
-        """Keys returned by the bulk claim call should be returned in the correct order"""
-
-        # Alice has lots of keys, uploaded in a specific order
-        alice = f"@alice:{self.hs.hostname}"
-        alice_dev = "alice_dev_1"
-
-        self.get_success(
-            self.handler.upload_keys_for_user(
-                alice,
-                alice_dev,
-                {"one_time_keys": {"alg1:k20": 20, "alg1:k21": 21, "alg1:k22": 22}},
-            )
-        )
-        # Advance time by 1s, to ensure that there is a difference in upload time.
-        self.reactor.advance(1)
-        self.get_success(
-            self.handler.upload_keys_for_user(
-                alice,
-                alice_dev,
-                {"one_time_keys": {"alg1:k10": 10, "alg1:k11": 11, "alg1:k12": 12}},
-            )
-        )
-
-        # Now claim some, and check we get the right ones.
-        claim_res = self.get_success(
-            self.handler.claim_one_time_keys(
-                {alice: {alice_dev: {"alg1": 2}}},
-                self.requester,
-                timeout=None,
-                always_include_fallback_keys=False,
-            )
-        )
-        # We should get the first-uploaded keys, even though they have later key ids.
-        # We should get a random set of two of k20, k21, k22.
-        self.assertEqual(claim_res["failures"], {})
-        claimed_keys = claim_res["one_time_keys"]["@alice:test"]["alice_dev_1"]
-        self.assertEqual(len(claimed_keys), 2)
-        for key_id in claimed_keys.keys():
-            self.assertIn(key_id, ["alg1:k20", "alg1:k21", "alg1:k22"])
-
    def test_fallback_key(self) -> None:
        local_user = "@boris:" + self.hs.hostname
        device_id = "xyz"
@@ -661,12 +661,9 @@ class PartialJoinTestCase(unittest.FederatingHomeserverTestCase):
            )
        )

-        with (
-            patch.object(
-                fed_client, "make_membership_event", mock_make_membership_event
-            ),
-            patch.object(fed_client, "send_join", mock_send_join),
-        ):
+        with patch.object(
+            fed_client, "make_membership_event", mock_make_membership_event
+        ), patch.object(fed_client, "send_join", mock_send_join):
            # Join and check that our join event is rejected
            # (The join event is rejected because it doesn't have any signatures)
            join_exc = self.get_failure(
@@ -711,12 +708,9 @@ class PartialJoinTestCase(unittest.FederatingHomeserverTestCase):
        fed_handler = self.hs.get_federation_handler()
        store = self.hs.get_datastores().main

-        with (
-            patch.object(
-                fed_handler, "_sync_partial_state_room", mock_sync_partial_state_room
-            ),
-            patch.object(store, "is_partial_state_room", mock_is_partial_state_room),
-        ):
+        with patch.object(
+            fed_handler, "_sync_partial_state_room", mock_sync_partial_state_room
+        ), patch.object(store, "is_partial_state_room", mock_is_partial_state_room):
            # Start the partial state sync.
            fed_handler._start_partial_state_room_sync("hs1", {"hs2"}, "room_id")
            self.assertEqual(mock_sync_partial_state_room.call_count, 1)
@@ -766,12 +760,9 @@ class PartialJoinTestCase(unittest.FederatingHomeserverTestCase):
        fed_handler = self.hs.get_federation_handler()
        store = self.hs.get_datastores().main

-        with (
-            patch.object(
-                fed_handler, "_sync_partial_state_room", mock_sync_partial_state_room
-            ),
-            patch.object(store, "is_partial_state_room", mock_is_partial_state_room),
-        ):
+        with patch.object(
+            fed_handler, "_sync_partial_state_room", mock_sync_partial_state_room
+        ), patch.object(store, "is_partial_state_room", mock_is_partial_state_room):
            # Start the partial state sync.
            fed_handler._start_partial_state_room_sync("hs1", {"hs2"}, "room_id")
            self.assertEqual(mock_sync_partial_state_room.call_count, 1)
@@ -172,25 +172,20 @@ class TestJoinsLimitedByPerRoomRateLimiter(FederatingHomeserverTestCase):
            )
        )

-        with (
-            patch.object(
-                self.handler.federation_handler.federation_client,
-                "make_membership_event",
-                mock_make_membership_event,
-            ),
-            patch.object(
-                self.handler.federation_handler.federation_client,
-                "send_join",
-                mock_send_join,
-            ),
-            patch(
-                "synapse.event_auth._is_membership_change_allowed",
-                return_value=None,
-            ),
-            patch(
-                "synapse.handlers.federation_event.check_state_dependent_auth_rules",
-                return_value=None,
-            ),
+        with patch.object(
+            self.handler.federation_handler.federation_client,
+            "make_membership_event",
+            mock_make_membership_event,
+        ), patch.object(
+            self.handler.federation_handler.federation_client,
+            "send_join",
+            mock_send_join,
+        ), patch(
+            "synapse.event_auth._is_membership_change_allowed",
+            return_value=None,
+        ), patch(
+            "synapse.handlers.federation_event.check_state_dependent_auth_rules",
+            return_value=None,
        ):
            self.get_success(
                self.handler.update_membership(
@@ -33,7 +33,6 @@ from synapse.api.constants import (
 )
 from synapse.api.room_versions import RoomVersions
 from synapse.handlers.sliding_sync import (
-    MAX_NUMBER_PREVIOUS_STATE_KEYS_TO_REMEMBER,
    RoomsForUserType,
    RoomSyncConfig,
    StateValues,
@@ -3320,32 +3319,6 @@ class RequiredStateChangesTestCase(unittest.TestCase):
                    ),
                ),
            ),
-            (
-                "simple_retain_previous_state_keys",
-                """Test adding a state key to the config and retaining a previously sent state_key""",
-                RequiredStateChangesTestParameters(
-                    previous_required_state_map={"type": {"state_key1"}},
-                    request_required_state_map={"type": {"state_key2", "state_key3"}},
-                    state_deltas={("type", "state_key2"): "$event_id"},
-                    expected_with_state_deltas=(
-                        # We've added a key so we should persist the changed required state
-                        # config.
-                        #
-                        # Retain `state_key1` from the `previous_required_state_map`
-                        {"type": {"state_key1", "state_key2", "state_key3"}},
-                        # We should see the new state_keys added
-                        StateFilter.from_types(
-                            [("type", "state_key2"), ("type", "state_key3")]
-                        ),
-                    ),
-                    expected_without_state_deltas=(
-                        {"type": {"state_key1", "state_key2", "state_key3"}},
-                        StateFilter.from_types(
-                            [("type", "state_key2"), ("type", "state_key3")]
-                        ),
-                    ),
-                ),
-            ),
            (
                "simple_remove_type",
                """
@@ -3751,249 +3724,6 @@ class RequiredStateChangesTestCase(unittest.TestCase):
                    ),
                ),
            ),
-            (
-                "state_key_lazy_keep_previous_memberships_and_no_new_memberships",
-                """
-                This test mimics a request with lazy-loading room members enabled where
-                we have previously sent down user2 and user3's membership events and now
-                we're sending down another response without any timeline events.
-                """,
-                RequiredStateChangesTestParameters(
-                    previous_required_state_map={
-                        EventTypes.Member: {
-                            StateValues.LAZY,
-                            "@user2:test",
-                            "@user3:test",
-                        }
-                    },
-                    request_required_state_map={EventTypes.Member: {StateValues.LAZY}},
-                    state_deltas={(EventTypes.Member, "@user2:test"): "$event_id"},
-                    expected_with_state_deltas=(
-                        # Remove "@user2:test" since that state has changed and is no
-                        # longer being requested anymore. Since something was removed,
-                        # we should persist the changed to required state. That way next
-                        # time, they request "@user2:test", we see that we haven't sent
-                        # it before and send the new state. (we should still keep track
-                        # that we've sent specific `EventTypes.Member` before)
-                        {
-                            EventTypes.Member: {
-                                StateValues.LAZY,
-                                "@user3:test",
-                            }
-                        },
-                        # We don't need to request anything more if they are requesting
-                        # less state now
-                        StateFilter.none(),
-                    ),
-                    expected_without_state_deltas=(
-                        # We're not requesting any specific `EventTypes.Member` now but
-                        # since that state hasn't changed, nothing should change (we
-                        # should still keep track that we've sent specific
-                        # `EventTypes.Member` before).
-                        None,
-                        # We don't need to request anything more if they are requesting
-                        # less state now
-                        StateFilter.none(),
-                    ),
-                ),
-            ),
-            (
-                "state_key_lazy_keep_previous_memberships_with_new_memberships",
-                """
-                This test mimics a request with lazy-loading room members enabled where
-                we have previously sent down user2 and user3's membership events and now
-                we're sending down another response with a new event from user4.
-                """,
-                RequiredStateChangesTestParameters(
-                    previous_required_state_map={
-                        EventTypes.Member: {
-                            StateValues.LAZY,
-                            "@user2:test",
-                            "@user3:test",
-                        }
-                    },
-                    request_required_state_map={
-                        EventTypes.Member: {StateValues.LAZY, "@user4:test"}
-                    },
-                    state_deltas={(EventTypes.Member, "@user2:test"): "$event_id"},
-                    expected_with_state_deltas=(
-                        # Since "@user4:test" was added, we should persist the changed
-                        # required state config.
-                        #
-                        # Also remove "@user2:test" since that state has changed and is no
-                        # longer being requested anymore. Since something was removed,
-                        # we also should persist the changed to required state. That way next
-                        # time, they request "@user2:test", we see that we haven't sent
-                        # it before and send the new state. (we should still keep track
-                        # that we've sent specific `EventTypes.Member` before)
-                        {
-                            EventTypes.Member: {
-                                StateValues.LAZY,
-                                "@user3:test",
-                                "@user4:test",
-                            }
-                        },
-                        # We should see the new state_keys added
-                        StateFilter.from_types([(EventTypes.Member, "@user4:test")]),
-                    ),
-                    expected_without_state_deltas=(
-                        # Since "@user4:test" was added, we should persist the changed
-                        # required state config.
-                        {
-                            EventTypes.Member: {
-                                StateValues.LAZY,
-                                "@user2:test",
-                                "@user3:test",
-                                "@user4:test",
-                            }
-                        },
-                        # We should see the new state_keys added
-                        StateFilter.from_types([(EventTypes.Member, "@user4:test")]),
-                    ),
-                ),
-            ),
-            (
-                "state_key_expand_lazy_keep_previous_memberships",
-                """
-                Test expanding the `required_state` to lazy-loading room members.
-                """,
-                RequiredStateChangesTestParameters(
-                    previous_required_state_map={
-                        EventTypes.Member: {"@user2:test", "@user3:test"}
-                    },
-                    request_required_state_map={EventTypes.Member: {StateValues.LAZY}},
-                    state_deltas={(EventTypes.Member, "@user2:test"): "$event_id"},
-                    expected_with_state_deltas=(
-                        # Since `StateValues.LAZY` was added, we should persist the
-                        # changed required state config.
-                        #
-                        # Also remove "@user2:test" since that state has changed and is no
-                        # longer being requested anymore. Since something was removed,
-                        # we also should persist the changed to required state. That way next
-                        # time, they request "@user2:test", we see that we haven't sent
-                        # it before and send the new state. (we should still keep track
-                        # that we've sent specific `EventTypes.Member` before)
-                        {
-                            EventTypes.Member: {
-                                StateValues.LAZY,
-                                "@user3:test",
-                            }
-                        },
-                        # We don't need to request anything more if they are requesting
-                        # less state now
-                        StateFilter.none(),
-                    ),
-                    expected_without_state_deltas=(
-                        # Since `StateValues.LAZY` was added, we should persist the
-                        # changed required state config.
-                        {
-                            EventTypes.Member: {
-                                StateValues.LAZY,
-                                "@user2:test",
-                                "@user3:test",
-                            }
-                        },
-                        # We don't need to request anything more if they are requesting
-                        # less state now
-                        StateFilter.none(),
-                    ),
-                ),
-            ),
-            (
-                "state_key_retract_lazy_keep_previous_memberships_no_new_memberships",
-                """
-                Test retracting the `required_state` to no longer lazy-loading room members.
-                """,
-                RequiredStateChangesTestParameters(
-                    previous_required_state_map={
-                        EventTypes.Member: {
-                            StateValues.LAZY,
-                            "@user2:test",
-                            "@user3:test",
-                        }
-                    },
-                    request_required_state_map={},
-                    state_deltas={(EventTypes.Member, "@user2:test"): "$event_id"},
-                    expected_with_state_deltas=(
-                        # Remove `EventTypes.Member` since there's been a change to that
-                        # state, (persist the change to required state). That way next
-                        # time, they request `EventTypes.Member`, we see that we haven't
-                        # sent it before and send the new state. (if we were tracking
-                        # that we sent any other state, we should still keep track
-                        # that).
-                        #
-                        # This acts the same as the `simple_remove_type` test. It's
-                        # possible that we could remember the specific `state_keys` that
-                        # we have sent down before but this currently just acts the same
-                        # as if a whole `type` was removed. Perhaps it's good that we
-                        # "garbage collect" and forget what we've sent before for a
-                        # given `type`  when the client stops caring about a certain
-                        # `type`.
-                        {},
-                        # We don't need to request anything more if they are requesting
-                        # less state now
-                        StateFilter.none(),
-                    ),
-                    expected_without_state_deltas=(
-                        # `EventTypes.Member` is no longer requested but since that
-                        # state hasn't changed, nothing should change (we should still
-                        # keep track that we've sent `EventTypes.Member` before).
-                        None,
-                        # We don't need to request anything more if they are requesting
-                        # less state now
-                        StateFilter.none(),
-                    ),
-                ),
-            ),
-            (
-                "state_key_retract_lazy_keep_previous_memberships_with_new_memberships",
-                """
-                Test retracting the `required_state` to no longer lazy-loading room members.
-                """,
-                RequiredStateChangesTestParameters(
-                    previous_required_state_map={
-                        EventTypes.Member: {
-                            StateValues.LAZY,
-                            "@user2:test",
-                            "@user3:test",
-                        }
-                    },
-                    request_required_state_map={EventTypes.Member: {"@user4:test"}},
-                    state_deltas={(EventTypes.Member, "@user2:test"): "$event_id"},
-                    expected_with_state_deltas=(
-                        # Since "@user4:test" was added, we should persist the changed
-                        # required state config.
-                        #
-                        # Also remove "@user2:test" since that state has changed and is no
-                        # longer being requested anymore. Since something was removed,
-                        # we also should persist the changed to required state. That way next
-                        # time, they request "@user2:test", we see that we haven't sent
-                        # it before and send the new state. (we should still keep track
-                        # that we've sent specific `EventTypes.Member` before)
-                        {
-                            EventTypes.Member: {
-                                "@user3:test",
-                                "@user4:test",
-                            }
-                        },
-                        # We should see the new state_keys added
-                        StateFilter.from_types([(EventTypes.Member, "@user4:test")]),
-                    ),
-                    expected_without_state_deltas=(
-                        # Since "@user4:test" was added, we should persist the changed
-                        # required state config.
-                        {
-                            EventTypes.Member: {
-                                "@user2:test",
-                                "@user3:test",
-                                "@user4:test",
-                            }
-                        },
-                        # We should see the new state_keys added
-                        StateFilter.from_types([(EventTypes.Member, "@user4:test")]),
-                    ),
-                ),
-            ),
            (
                "type_wildcard_with_state_key_wildcard_to_explicit_state_keys",
                """
@@ -4094,7 +3824,7 @@ class RequiredStateChangesTestCase(unittest.TestCase):
                ),
            ),
            (
-                "explicit_state_keys_to_wildcard_state_key",
+                "state_key_wildcard_to_explicit_state_keys",
                """Test switching from a wildcard to explicit state keys with a concrete type""",
                RequiredStateChangesTestParameters(
                    previous_required_state_map={
@@ -4107,18 +3837,11 @@ class RequiredStateChangesTestCase(unittest.TestCase):
                    # request. And we need to request all of the state for that type
                    # because we previously, only sent down a few keys.
                    expected_with_state_deltas=(
-                        {"type1": {StateValues.WILDCARD, "state_key2", "state_key3"}},
+                        {"type1": {StateValues.WILDCARD}},
                        StateFilter.from_types([("type1", None)]),
                    ),
                    expected_without_state_deltas=(
-                        {
-                            "type1": {
-                                StateValues.WILDCARD,
-                                "state_key1",
-                                "state_key2",
-                                "state_key3",
-                            }
-                        },
+                        {"type1": {StateValues.WILDCARD}},
                        StateFilter.from_types([("type1", None)]),
                    ),
                ),
@@ -4134,8 +3857,14 @@ class RequiredStateChangesTestCase(unittest.TestCase):
        # Without `state_deltas`
        changed_required_state_map, added_state_filter = _required_state_changes(
            user_id="@user:test",
-            prev_required_state_map=test_parameters.previous_required_state_map,
-            request_required_state_map=test_parameters.request_required_state_map,
+            previous_room_config=RoomSyncConfig(
+                timeline_limit=0,
+                required_state_map=test_parameters.previous_required_state_map,
+            ),
+            room_sync_config=RoomSyncConfig(
+                timeline_limit=0,
+                required_state_map=test_parameters.request_required_state_map,
+            ),
            state_deltas={},
        )

@@ -4153,8 +3882,14 @@ class RequiredStateChangesTestCase(unittest.TestCase):
        # With `state_deltas`
        changed_required_state_map, added_state_filter = _required_state_changes(
            user_id="@user:test",
-            prev_required_state_map=test_parameters.previous_required_state_map,
-            request_required_state_map=test_parameters.request_required_state_map,
+            previous_room_config=RoomSyncConfig(
+                timeline_limit=0,
+                required_state_map=test_parameters.previous_required_state_map,
+            ),
+            room_sync_config=RoomSyncConfig(
+                timeline_limit=0,
+                required_state_map=test_parameters.request_required_state_map,
+            ),
            state_deltas=test_parameters.state_deltas,
        )

@@ -4168,121 +3903,3 @@ class RequiredStateChangesTestCase(unittest.TestCase):
            test_parameters.expected_with_state_deltas[1],
            "added_state_filter does not match (with state_deltas)",
        )
-
-    @parameterized.expand(
-        [
-            # Test with a normal arbitrary type (no special meaning)
-            ("arbitrary_type", "type", set()),
-            # Test with membership
-            ("membership", EventTypes.Member, set()),
-            # Test with lazy-loading room members
-            ("lazy_loading_membership", EventTypes.Member, {StateValues.LAZY}),
-        ]
-    )
-    def test_limit_retained_previous_state_keys(
-        self,
-        _test_label: str,
-        event_type: str,
-        extra_state_keys: Set[str],
-    ) -> None:
-        """
-        Test that we limit the number of state_keys that we remember but always include
-        the state_keys that we've just requested.
-        """
-        previous_required_state_map = {
-            event_type: {
-                # Prefix the state_keys we've "prev_"iously sent so they are easier to
-                # identify in our assertions.
-                f"prev_state_key{i}"
-                for i in range(MAX_NUMBER_PREVIOUS_STATE_KEYS_TO_REMEMBER - 30)
-            }
-            | extra_state_keys
-        }
-        request_required_state_map = {
-            event_type: {f"state_key{i}" for i in range(50)} | extra_state_keys
-        }
-
-        # (function under test)
-        changed_required_state_map, added_state_filter = _required_state_changes(
-            user_id="@user:test",
-            prev_required_state_map=previous_required_state_map,
-            request_required_state_map=request_required_state_map,
-            state_deltas={},
-        )
-        assert changed_required_state_map is not None
-
-        # We should only remember up to the maximum number of state keys
-        self.assertGreaterEqual(
-            len(changed_required_state_map[event_type]),
-            # Most of the time this will be `MAX_NUMBER_PREVIOUS_STATE_KEYS_TO_REMEMBER` but
-            # because we are just naively selecting enough previous state_keys to fill
-            # the limit, there might be some overlap in what's added back which means we
-            # might have slightly less than the limit.
-            #
-            # `extra_state_keys` overlaps in the previous and requested
-            # `required_state_map` so we might see this this scenario.
-            MAX_NUMBER_PREVIOUS_STATE_KEYS_TO_REMEMBER - len(extra_state_keys),
-        )
-
-        # Should include all of the requested state
-        self.assertIncludes(
-            changed_required_state_map[event_type],
-            request_required_state_map[event_type],
-        )
-        # And the rest is filled with the previous state keys
-        #
-        # We can't assert the exact state_keys since we don't know the order so we just
-        # check that they all start with "prev_" and that we have the correct amount.
-        remaining_state_keys = (
-            changed_required_state_map[event_type]
-            - request_required_state_map[event_type]
-        )
-        self.assertGreater(
-            len(remaining_state_keys),
-            0,
-        )
-        assert all(
-            state_key.startswith("prev_") for state_key in remaining_state_keys
-        ), "Remaining state_keys should be the previous state_keys"
-
-    def test_request_more_state_keys_than_remember_limit(self) -> None:
-        """
-        Test requesting more state_keys than fit in our limit to remember from previous
-        requests.
-        """
-        previous_required_state_map = {
-            "type": {
-                # Prefix the state_keys we've "prev_"iously sent so they are easier to
-                # identify in our assertions.
-                f"prev_state_key{i}"
-                for i in range(MAX_NUMBER_PREVIOUS_STATE_KEYS_TO_REMEMBER - 30)
-            }
-        }
-        request_required_state_map = {
-            "type": {
-                f"state_key{i}"
-                # Requesting more than the MAX_NUMBER_PREVIOUS_STATE_KEYS_TO_REMEMBER
-                for i in range(MAX_NUMBER_PREVIOUS_STATE_KEYS_TO_REMEMBER + 20)
-            }
-        }
-        # Ensure that we are requesting more than the limit
-        self.assertGreater(
-            len(request_required_state_map["type"]),
-            MAX_NUMBER_PREVIOUS_STATE_KEYS_TO_REMEMBER,
-        )
-
-        # (function under test)
-        changed_required_state_map, added_state_filter = _required_state_changes(
-            user_id="@user:test",
-            prev_required_state_map=previous_required_state_map,
-            request_required_state_map=request_required_state_map,
-            state_deltas={},
-        )
-        assert changed_required_state_map is not None
-
-        # Should include all of the requested state
-        self.assertIncludes(
-            changed_required_state_map["type"],
-            request_required_state_map["type"],
-            exact=True,
-        )
--- a/Show More
+++ b/Show More
Author	SHA1	Message	Date
Hugh Nimmo-Smith	98c27ab824	Lint	2024-10-25 17:39:55 +01:00
Hugh Nimmo-Smith	70d166e7b5	inductive sync state WIP	2024-10-25 17:31:06 +01:00
Hugh Nimmo-Smith	96425d4071	Test cases for sync of state from DAG branches	2024-10-25 16:13:46 +01:00
				`@@ -1 +0,0 @@`
				`Support [MSC4151](https://github.com/matrix-org/matrix-spec-proposals/pull/4151)'s stable report room API.`
				`@@ -0,0 +1 @@`
				Clarify when the `user_may_invite` and `user_may_send_3pid_invite` module callbacks are called.
				`@@ -1 +0,0 @@`
				`Bump actions/download-artifact and actions/upload-artifact from v3 -> v4.`
				`@@ -0,0 +1 @@`
				Added the `display_name_claim` option to the JWT configuration. This option allows specifying the claim key that contains the user's display name in the JWT payload.
				`@@ -0,0 +1 @@`
				`Slight optimization when fetching state/events for Sliding Sync.`
				`@@ -0,0 +1 @@`
				`Fix saving of PNG thumbnails, when the original image is in the CMYK color space.`
				`@@ -0,0 +1 @@`
				`Add Python 3.13 and Postgres 17 to the test matrix.`
				`@@ -0,0 +1 @@`
				`Implement [MSC4210](https://github.com/matrix-org/matrix-spec-proposals/pull/4210): Remove legacy mentions. Contributed by @tulir @ Beeper.`
				`@@ -0,0 +1 @@`
				Fix bug with sliding sync where the server would not return state that was added to the `required_state` config.