AAAA#

DBG cache all
DBG cache
2022-07-05 17:26:12 +01:00 · 2022-07-05 12:40:26 +01:00 · 2022-07-05 12:31:50 +01:00 · 2022-07-05 12:18:26 +01:00 · 2022-07-05 12:18:07 +01:00 · 2022-07-05 11:59:48 +01:00
98 changed files with 1068 additions and 2856 deletions
@@ -7,4 +7,3 @@ root = true
 [*.py]
 indent_style = space
 indent_size = 4
-max_line_length = 88
@@ -10,324 +10,15 @@ concurrency:
  cancel-in-progress: true

 jobs:
-  check-sampleconfig:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
-      - uses: actions/setup-python@v2
-      - run: pip install .
-      - run: scripts-dev/generate_sample_config.sh --check
-      - run: scripts-dev/config-lint.sh
-
-  check-schema-delta:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
-      - uses: actions/setup-python@v2
-      - run: "pip install 'click==8.1.1' 'GitPython>=3.1.20'"
-      - run: scripts-dev/check_schema_delta.py --force-colors
-
-  lint:
-    uses: "matrix-org/backend-meta/.github/workflows/python-poetry-ci.yml@v1"
-    with:
-      typechecking-extras: "all"
-
-  lint-crlf:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
-      - name: Check line endings
-        run: scripts-dev/check_line_terminators.sh
-
-  lint-newsfile:
-    if: ${{ github.base_ref == 'develop'  || contains(github.base_ref, 'release-') }}
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
-        with:
-          ref: ${{ github.event.pull_request.head.sha }}
-          fetch-depth: 0
-      - uses: actions/setup-python@v2
-      - run: "pip install 'towncrier>=18.6.0rc1'"
-      - run: scripts-dev/check-newsfragment.sh
-        env:
-          PULL_REQUEST_NUMBER: ${{ github.event.number }}
-
-  # Dummy step to gate other tests on without repeating the whole list
-  linting-done:
-    if: ${{ !cancelled() }} # Run this even if prior jobs were skipped
-    needs: [lint, lint-crlf, lint-newsfile, check-sampleconfig, check-schema-delta]
-    runs-on: ubuntu-latest
-    steps:
-      - run: "true"
-
-  trial:
-    if: ${{ !cancelled() && !failure() }} # Allow previous steps to be skipped, but not fail
-    needs: linting-done
-    runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        python-version: ["3.7", "3.8", "3.9", "3.10"]
-        database: ["sqlite"]
-        extras: ["all"]
-        include:
-          # Newest Python without optional deps
-          - python-version: "3.10"
-            extras: ""
-
-          # Oldest Python with PostgreSQL
-          - python-version: "3.7"
-            database: "postgres"
-            postgres-version: "10"
-            extras: "all"
-
-          # Newest Python with newest PostgreSQL
-          - python-version: "3.10"
-            database: "postgres"
-            postgres-version: "14"
-            extras: "all"
-
-    steps:
-      - uses: actions/checkout@v2
-      - run: sudo apt-get -qq install xmlsec1
-      - name: Set up PostgreSQL ${{ matrix.postgres-version }}
-        if: ${{ matrix.postgres-version }}
-        run: |
-          docker run -d -p 5432:5432 \
-            -e POSTGRES_PASSWORD=postgres \
-            -e POSTGRES_INITDB_ARGS="--lc-collate C --lc-ctype C --encoding UTF8" \
-            postgres:${{ matrix.postgres-version }}
-      - uses: matrix-org/setup-python-poetry@v1
-        with:
-          python-version: ${{ matrix.python-version }}
-          extras: ${{ matrix.extras }}
-      - name: Await PostgreSQL
-        if: ${{ matrix.postgres-version }}
-        timeout-minutes: 2
-        run: until pg_isready -h localhost; do sleep 1; done
-      - run: poetry run trial --jobs=2 tests
-        env:
-          SYNAPSE_POSTGRES: ${{ matrix.database == 'postgres' || '' }}
-          SYNAPSE_POSTGRES_HOST: localhost
-          SYNAPSE_POSTGRES_USER: postgres
-          SYNAPSE_POSTGRES_PASSWORD: postgres
-      - name: Dump logs
-        # Logs are most useful when the command fails, always include them.
-        if: ${{ always() }}
-        # Note: Dumps to workflow logs instead of using actions/upload-artifact
-        #       This keeps logs colocated with failing jobs
-        #       It also ignores find's exit code; this is a best effort affair
-        run: >-
-          find _trial_temp -name '*.log'
-          -exec echo "::group::{}" \;
-          -exec cat {} \;
-          -exec echo "::endgroup::" \;
-          || true
-
-  trial-olddeps:
-    # Note: sqlite only; no postgres
-    if: ${{ !cancelled() && !failure() }} # Allow previous steps to be skipped, but not fail
-    needs: linting-done
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
-      - name: Test with old deps
-        uses: docker://ubuntu:focal # For old python and sqlite
-        # Note: focal seems to be using 3.8, but the oldest is 3.7?
-        # See https://github.com/matrix-org/synapse/issues/12343
-        with:
-          workdir: /github/workspace
-          entrypoint: .ci/scripts/test_old_deps.sh
-      - name: Dump logs
-        # Logs are most useful when the command fails, always include them.
-        if: ${{ always() }}
-        # Note: Dumps to workflow logs instead of using actions/upload-artifact
-        #       This keeps logs colocated with failing jobs
-        #       It also ignores find's exit code; this is a best effort affair
-        run: >-
-          find _trial_temp -name '*.log'
-          -exec echo "::group::{}" \;
-          -exec cat {} \;
-          -exec echo "::endgroup::" \;
-          || true
-
-  trial-pypy:
-    # Very slow; only run if the branch name includes 'pypy'
-    # Note: sqlite only; no postgres. Completely untested since poetry move.
-    if: ${{ contains(github.ref, 'pypy') && !failure() && !cancelled() }}
-    needs: linting-done
-    runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        python-version: ["pypy-3.7"]
-        extras: ["all"]
-
-    steps:
-      - uses: actions/checkout@v2
-      # Install libs necessary for PyPy to build binary wheels for dependencies
-      - run: sudo apt-get -qq install xmlsec1 libxml2-dev libxslt-dev
-      - uses: matrix-org/setup-python-poetry@v1
-        with:
-          python-version: ${{ matrix.python-version }}
-          extras: ${{ matrix.extras }}
-      - run: poetry run trial --jobs=2 tests
-      - name: Dump logs
-        # Logs are most useful when the command fails, always include them.
-        if: ${{ always() }}
-        # Note: Dumps to workflow logs instead of using actions/upload-artifact
-        #       This keeps logs colocated with failing jobs
-        #       It also ignores find's exit code; this is a best effort affair
-        run: >-
-          find _trial_temp -name '*.log'
-          -exec echo "::group::{}" \;
-          -exec cat {} \;
-          -exec echo "::endgroup::" \;
-          || true
-
-  sytest:
-    if: ${{ !failure() && !cancelled() }}
-    needs: linting-done
-    runs-on: ubuntu-latest
-    container:
-      image: matrixdotorg/sytest-synapse:${{ matrix.sytest-tag }}
-      volumes:
-        - ${{ github.workspace }}:/src
-      env:
-        SYTEST_BRANCH: ${{ github.head_ref }}
-        POSTGRES: ${{ matrix.postgres && 1}}
-        MULTI_POSTGRES: ${{ (matrix.postgres == 'multi-postgres') && 1}}
-        WORKERS: ${{ matrix.workers && 1 }}
-        REDIS: ${{ matrix.redis && 1 }}
-        BLACKLIST: ${{ matrix.workers && 'synapse-blacklist-with-workers' }}
-        TOP: ${{ github.workspace }}
-
-    strategy:
-      fail-fast: false
-      matrix:
-        include:
-          - sytest-tag: focal
-
-          - sytest-tag: focal
-            postgres: postgres
-
-          - sytest-tag: testing
-            postgres: postgres
-
-          - sytest-tag: focal
-            postgres: multi-postgres
-            workers: workers
-
-          - sytest-tag: buster
-            postgres: multi-postgres
-            workers: workers
-
-          - sytest-tag: buster
-            postgres: postgres
-            workers: workers
-            redis: redis
-
-    steps:
-      - uses: actions/checkout@v2
-      - name: Prepare test blacklist
-        run: cat sytest-blacklist .ci/worker-blacklist > synapse-blacklist-with-workers
-      - name: Run SyTest
-        run: /bootstrap.sh synapse
-        working-directory: /src
-      - name: Summarise results.tap
-        if: ${{ always() }}
-        run: /sytest/scripts/tap_to_gha.pl /logs/results.tap
-      - name: Upload SyTest logs
-        uses: actions/upload-artifact@v2
-        if: ${{ always() }}
-        with:
-          name: Sytest Logs - ${{ job.status }} - (${{ join(matrix.*, ', ') }})
-          path: |
-            /logs/results.tap
-            /logs/**/*.log*
-
-  export-data:
-    if: ${{ !failure() && !cancelled() }} # Allow previous steps to be skipped, but not fail
-    needs: [linting-done, portdb]
-    runs-on: ubuntu-latest
-    env:
-      TOP: ${{ github.workspace }}
-
-    services:
-      postgres:
-        image: postgres
-        ports:
-          - 5432:5432
-        env:
-          POSTGRES_PASSWORD: "postgres"
-          POSTGRES_INITDB_ARGS: "--lc-collate C --lc-ctype C --encoding UTF8"
-        options: >-
-          --health-cmd pg_isready
-          --health-interval 10s
-          --health-timeout 5s
-          --health-retries 5
-
-    steps:
-      - uses: actions/checkout@v2
-      - run: sudo apt-get -qq install xmlsec1
-      - uses: matrix-org/setup-python-poetry@v1
-        with:
-          python-version: ${{ matrix.python-version }}
-          extras: "postgres"
-      - run: .ci/scripts/test_export_data_command.sh
-
-  portdb:
-    if: ${{ !failure() && !cancelled() }} # Allow previous steps to be skipped, but not fail
-    needs: linting-done
-    runs-on: ubuntu-latest
-    env:
-      TOP: ${{ github.workspace }}
-    strategy:
-      matrix:
-        include:
-          - python-version: "3.7"
-            postgres-version: "10"
-
-          - python-version: "3.10"
-            postgres-version: "14"
-
-    services:
-      postgres:
-        image: postgres:${{ matrix.postgres-version }}
-        ports:
-          - 5432:5432
-        env:
-          POSTGRES_PASSWORD: "postgres"
-          POSTGRES_INITDB_ARGS: "--lc-collate C --lc-ctype C --encoding UTF8"
-        options: >-
-          --health-cmd pg_isready
-          --health-interval 10s
-          --health-timeout 5s
-          --health-retries 5
-
-    steps:
-      - uses: actions/checkout@v2
-      - run: sudo apt-get -qq install xmlsec1
-      - uses: matrix-org/setup-python-poetry@v1
-        with:
-          python-version: ${{ matrix.python-version }}
-          extras: "postgres"
-      - run: .ci/scripts/test_synapse_port_db.sh

  complement:
    if: "${{ !failure() && !cancelled() }}"
-    needs: linting-done
    runs-on: ubuntu-latest

    strategy:
      fail-fast: false
      matrix:
        include:
-          - arrangement: monolith
-            database: SQLite
-
-          - arrangement: monolith
-            database: Postgres
-
          - arrangement: workers
            database: Postgres

@@ -342,7 +33,14 @@ jobs:

      - run: |
          set -o pipefail
-          POSTGRES=${{ (matrix.database == 'Postgres') && 1 || '' }} WORKERS=${{ (matrix.arrangement == 'workers') && 1 || '' }} COMPLEMENT_DIR=`pwd`/complement synapse/scripts-dev/complement.sh -json 2>&1 | gotestfmt
+          synapse/scripts-dev/complement.sh --build-only
+          while :; do
+            POSTGRES=${{ (matrix.database == 'Postgres') && 1 || '' }} WORKERS=${{ (matrix.arrangement == 'workers') && 1 || '' }} SYNAPSE_TEST_LOG_LEVEL=DEBUG COMPLEMENT_DIR=`pwd`/complement synapse/scripts-dev/complement.sh -f -run TestMembersLocal -json 2>&1 | gotestfmt | tee /tmp/xxx
+            if grep FAIL /tmp/xxx; then
+              break
+            fi
+          done
+
        shell: bash
        name: Run Complement Tests

@@ -350,15 +48,6 @@ jobs:
  tests-done:
    if: ${{ always() }}
    needs:
-      - check-sampleconfig
-      - lint
-      - lint-crlf
-      - lint-newsfile
-      - trial
-      - trial-olddeps
-      - sytest
-      - export-data
-      - portdb
      - complement
    runs-on: ubuntu-latest
    steps:
@@ -1,115 +1,3 @@
-Synapse 1.63.1 (2022-07-20)
-===========================
-
-Bugfixes
--------
-
- Fix a bug introduced in Synapse 1.63.0 where push actions were incorrectly calculated for appservice users. This caused performance issues on servers with large numbers of appservices. ([\#13332](https://github.com/matrix-org/synapse/issues/13332))
-
-
-Synapse 1.63.0 (2022-07-19)
-===========================
-
-Improved Documentation
----------------------
-
- Clarify that homeserver server names are included in the reported data when the `report_stats` config option is enabled. ([\#13321](https://github.com/matrix-org/synapse/issues/13321))
-
-
-Synapse 1.63.0rc1 (2022-07-12)
-==============================
-
-Features
--------
-
- Add a rate limit for local users sending invites. ([\#13125](https://github.com/matrix-org/synapse/issues/13125))
- Implement [MSC3827](https://github.com/matrix-org/matrix-spec-proposals/pull/3827): Filtering of `/publicRooms` by room type. ([\#13031](https://github.com/matrix-org/synapse/issues/13031))
- Improve validation logic in the account data REST endpoints. ([\#13148](https://github.com/matrix-org/synapse/issues/13148))
-
-
-Bugfixes
--------
-
- Fix a long-standing bug where application services were not able to join remote federated rooms without a profile. ([\#13131](https://github.com/matrix-org/synapse/issues/13131))
- Fix a long-standing bug where `_get_state_map_for_room` might raise errors when third party event rules callbacks are present. ([\#13174](https://github.com/matrix-org/synapse/issues/13174))
- Fix a long-standing bug where the `synapse_port_db` script could fail to copy rows with negative row ids. ([\#13226](https://github.com/matrix-org/synapse/issues/13226))
- Fix a bug introduced in 1.54.0 where appservices would not receive room-less EDUs, like presence, when both [MSC2409](https://github.com/matrix-org/matrix-spec-proposals/pull/2409) and [MSC3202](https://github.com/matrix-org/matrix-spec-proposals/pull/3202) are enabled. ([\#13236](https://github.com/matrix-org/synapse/issues/13236))
- Fix a bug introduced in 1.62.0 where rows were not deleted from `event_push_actions` table on large servers. ([\#13194](https://github.com/matrix-org/synapse/issues/13194))
- Fix a bug introduced in 1.62.0 where notification counts would get stuck after a highlighted message. ([\#13223](https://github.com/matrix-org/synapse/issues/13223))
- Fix exception when using experimental [MSC3030](https://github.com/matrix-org/matrix-spec-proposals/pull/3030) `/timestamp_to_event` endpoint to look for remote federated imported events before room creation. ([\#13197](https://github.com/matrix-org/synapse/issues/13197))
- Fix [MSC3202](https://github.com/matrix-org/matrix-spec-proposals/pull/3202)-enabled appservices not receiving to-device messages, preventing messages from being decrypted. ([\#13235](https://github.com/matrix-org/synapse/issues/13235))
-
-
-Updates to the Docker image
---------------------------
-
- Bump the version of `lxml` in matrix.org Docker images Debian packages from 4.8.0 to 4.9.1. ([\#13207](https://github.com/matrix-org/synapse/issues/13207))
-
-
-Improved Documentation
----------------------
-
- Add an explanation of the `--report-stats` argument to the docs. ([\#13029](https://github.com/matrix-org/synapse/issues/13029))
- Add a helpful example bash script to the contrib directory for creating multiple worker configuration files of the same type. Contributed by @villepeh. ([\#13032](https://github.com/matrix-org/synapse/issues/13032))
- Add missing links to config options. ([\#13166](https://github.com/matrix-org/synapse/issues/13166))
- Add documentation for homeserver usage statistics collection. ([\#13086](https://github.com/matrix-org/synapse/issues/13086))
- Add documentation for the existing `databases` option in the homeserver configuration manual. ([\#13212](https://github.com/matrix-org/synapse/issues/13212))
- Clean up references to sample configuration and redirect users to the configuration manual instead. ([\#13077](https://github.com/matrix-org/synapse/issues/13077), [\#13139](https://github.com/matrix-org/synapse/issues/13139))
- Document how the Synapse team does reviews. ([\#13132](https://github.com/matrix-org/synapse/issues/13132))
- Fix wrong section header for `allow_public_rooms_over_federation` in the homeserver config documentation. ([\#13116](https://github.com/matrix-org/synapse/issues/13116))
-
-
-Deprecations and Removals
-------------------------
-
- Remove obsolete and for 8 years unused `RoomEventsStoreTestCase`. Contributed by @arkamar. ([\#13200](https://github.com/matrix-org/synapse/issues/13200))
-
-
-Internal Changes
----------------
-
- Add type annotations to `synapse.logging`, `tests.server` and `tests.utils`. ([\#13028](https://github.com/matrix-org/synapse/issues/13028), [\#13103](https://github.com/matrix-org/synapse/issues/13103), [\#13159](https://github.com/matrix-org/synapse/issues/13159), [\#13136](https://github.com/matrix-org/synapse/issues/13136))
- Enforce type annotations for `tests.test_server`. ([\#13135](https://github.com/matrix-org/synapse/issues/13135))
- Support temporary experimental return values for spam checker module callbacks. ([\#13044](https://github.com/matrix-org/synapse/issues/13044))
- Add support to `complement.sh` for skipping the docker build. ([\#13143](https://github.com/matrix-org/synapse/issues/13143), [\#13158](https://github.com/matrix-org/synapse/issues/13158))
- Add support to `complement.sh` for setting the log level using the `SYNAPSE_TEST_LOG_LEVEL` environment variable. ([\#13152](https://github.com/matrix-org/synapse/issues/13152))
- Enable Complement testing in the 'Twisted Trunk' CI runs. ([\#13079](https://github.com/matrix-org/synapse/issues/13079), [\#13157](https://github.com/matrix-org/synapse/issues/13157))
- Improve startup times in Complement test runs against workers, particularly in CPU-constrained environments. ([\#13127](https://github.com/matrix-org/synapse/issues/13127))
- Update config used by Complement to allow device name lookup over federation. ([\#13167](https://github.com/matrix-org/synapse/issues/13167))
- Faster room joins: handle race between persisting an event and un-partial stating a room. ([\#13100](https://github.com/matrix-org/synapse/issues/13100))
- Faster room joins: fix race in recalculation of current room state. ([\#13151](https://github.com/matrix-org/synapse/issues/13151))
- Faster room joins: skip waiting for full state when processing incoming events over federation. ([\#13144](https://github.com/matrix-org/synapse/issues/13144))
- Raise a `DependencyError` on missing dependencies instead of a `ConfigError`. ([\#13113](https://github.com/matrix-org/synapse/issues/13113))
- Avoid stripping line breaks from SQL sent to the database. ([\#13129](https://github.com/matrix-org/synapse/issues/13129))
- Apply ratelimiting earlier in processing of `/send` requests. ([\#13134](https://github.com/matrix-org/synapse/issues/13134))
- Improve exception handling when processing events received over federation. ([\#13145](https://github.com/matrix-org/synapse/issues/13145))
- Check that `auto_vacuum` is disabled when porting a SQLite database to Postgres, as `VACUUM`s must not be performed between runs of the script. ([\#13195](https://github.com/matrix-org/synapse/issues/13195))
- Reduce DB usage of `/sync` when a large number of unread messages have recently been sent in a room. ([\#13119](https://github.com/matrix-org/synapse/issues/13119), [\#13153](https://github.com/matrix-org/synapse/issues/13153))
- Reduce memory consumption when processing incoming events in large rooms. ([\#13078](https://github.com/matrix-org/synapse/issues/13078), [\#13222](https://github.com/matrix-org/synapse/issues/13222))
- Reduce number of queries used to get profile information. Contributed by Nick @ Beeper (@fizzadar). ([\#13209](https://github.com/matrix-org/synapse/issues/13209))
- Reduce number of events queried during room creation. Contributed by Nick @ Beeper (@fizzadar). ([\#13210](https://github.com/matrix-org/synapse/issues/13210))
- More aggressively rotate push actions. ([\#13211](https://github.com/matrix-org/synapse/issues/13211))
- Add `max_line_length` setting for Python files to the `.editorconfig`. Contributed by @sumnerevans @ Beeper. ([\#13228](https://github.com/matrix-org/synapse/issues/13228))
-
-
-Synapse 1.62.0 (2022-07-05)
-===========================
-
-No significant changes since 1.62.0rc3.
-
-Authors of spam-checker plugins should consult the [upgrade notes](https://github.com/matrix-org/synapse/blob/release-v1.62/docs/upgrade.md#upgrading-to-v1620) to learn about the enriched signatures for spam checker callbacks, which are supported with this release of Synapse.
-
-
-Synapse 1.62.0rc3 (2022-07-04)
-==============================
-
-Bugfixes
--------
-
- Update the version of the [ldap3 plugin](https://github.com/matrix-org/matrix-synapse-ldap3/) included in the `matrixdotorg/synapse` DockerHub images and the Debian packages hosted on `packages.matrix.org` to 0.2.1. This fixes [a bug](https://github.com/matrix-org/matrix-synapse-ldap3/pull/163) with usernames containing uppercase characters. ([\#13156](https://github.com/matrix-org/synapse/issues/13156))
- Fix a bug introduced in Synapse 1.62.0rc1 affecting unread counts for users on small servers. ([\#13168](https://github.com/matrix-org/synapse/issues/13168))
-
-
 Synapse 1.62.0rc2 (2022-07-01)
 ==============================

@@ -34,14 +34,6 @@ additional-css = [
    "docs/website_files/table-of-contents.css",
    "docs/website_files/remove-nav-buttons.css",
    "docs/website_files/indent-section-headers.css",
-    "docs/website_files/version-picker.css",
 ]
-additional-js = [
-    "docs/website_files/table-of-contents.js",
-    "docs/website_files/version-picker.js",
-    "docs/website_files/version.js",
-]
-theme = "docs/website_files/theme"
-
-[preprocessor.schema_versions]
-command = "./scripts-dev/schema_versions.py"
+additional-js = ["docs/website_files/table-of-contents.js"]
+theme = "docs/website_files/theme"
@@ -0,0 +1 @@
+Add an explanation of the `--report-stats` argument to the docs.
@@ -0,0 +1 @@
+Implement [MSC3827](https://github.com/matrix-org/matrix-spec-proposals/pull/3827): Filtering of /publicRooms by room type.
@@ -0,0 +1,3 @@
+Clean up references to sample configuration and redirect users to the configuration manual instead.
+
+
@@ -0,0 +1 @@
+Enable Complement testing in the 'Twisted Trunk' CI runs.
@@ -0,0 +1 @@
+Add documentation for anonymised homeserver statistics collection.
@@ -0,0 +1 @@
+Add missing type hints to `synapse.logging`.
@@ -0,0 +1 @@
+Raise a `DependencyError` on missing dependencies instead of a `ConfigError`.
@@ -0,0 +1 @@
+Fix wrong section header for `allow_public_rooms_over_federation` in the homeserver config documentation.
@@ -0,0 +1 @@
+Reduce DB usage of `/sync` when a large number of unread messages have recently been sent in a room.
@@ -0,0 +1 @@
+Add a rate limit for local users sending invites.
@@ -0,0 +1 @@
+Improve startup times in Complement test runs against workers, particularly in CPU-constrained environments.
@@ -0,0 +1 @@
+Only one-line SQL statements for logging and tracing.
@@ -0,0 +1 @@
+Apply ratelimiting earlier in processing of /send request.
@@ -0,0 +1 @@
+Enforce type annotations for `tests.test_server`.
@@ -0,0 +1 @@
+Add a link to the configuration manual from the homeserver sample config documentation.
@@ -0,0 +1 @@
+Add support to `complement.sh` for skipping the docker build.
@@ -0,0 +1 @@
+Faster joins: skip waiting for full state when processing incoming events over federation.
@@ -0,0 +1 @@
+Improve exception handling when processing events received over federation.
@@ -0,0 +1 @@
+Improve validation logic in Synapse's REST endpoints.
@@ -0,0 +1 @@
+Enable Complement testing in the 'Twisted Trunk' CI runs.
@@ -0,0 +1 @@
+Add support to `complement.sh` for skipping the docker build.
@@ -1,31 +0,0 @@
-# Creating multiple workers with a bash script
-
-Setting up multiple worker configuration files manually can be time-consuming.
-You can alternatively create multiple worker configuration files with a simple `bash` script. For example:
-
-```sh
-#!/bin/bash
-for i in {1..5}
-do
-cat << EOF >> generic_worker$i.yaml
-worker_app: synapse.app.generic_worker
-worker_name: generic_worker$i
-
-# The replication listener on the main synapse process.
-worker_replication_host: 127.0.0.1
-worker_replication_http_port: 9093
-
-worker_listeners:
-  - type: http
-    port: 808$i
-    resources:
-      - names: [client, federation]
-
-worker_log_config: /etc/matrix-synapse/generic-worker-log.yaml
-EOF
-done
-```
-
-This would create five generic workers with a unique `worker_name` field in each file and listening on ports 8081-8085.
-
-Customise the script to your needs.
@@ -1,35 +1,3 @@
-matrix-synapse-py3 (1.63.1) stable; urgency=medium
-
-  * New Synapse release 1.63.1.
-
- -- Synapse Packaging team <packages@matrix.org>  Wed, 20 Jul 2022 13:36:52 +0100
-
-matrix-synapse-py3 (1.63.0) stable; urgency=medium
-
-  * Clarify that homeserver server names are included in the data reported
-    by opt-in server stats reporting (`report_stats` homeserver config option).
-  * New Synapse release 1.63.0.
-
- -- Synapse Packaging team <packages@matrix.org>  Tue, 19 Jul 2022 14:42:24 +0200
-
-matrix-synapse-py3 (1.63.0~rc1) stable; urgency=medium
-
-  * New Synapse release 1.63.0rc1.
-
- -- Synapse Packaging team <packages@matrix.org>  Tue, 12 Jul 2022 11:26:02 +0100
-
-matrix-synapse-py3 (1.62.0) stable; urgency=medium
-
-  * New Synapse release 1.62.0.
-
- -- Synapse Packaging team <packages@matrix.org>  Tue, 05 Jul 2022 11:14:15 +0100
-
-matrix-synapse-py3 (1.62.0~rc3) stable; urgency=medium
-
-  * New Synapse release 1.62.0rc3.
-
- -- Synapse Packaging team <packages@matrix.org>  Mon, 04 Jul 2022 16:07:01 +0100
-
 matrix-synapse-py3 (1.62.0~rc2) stable; urgency=medium

  * New Synapse release 1.62.0rc2.
@@ -31,7 +31,7 @@ EOF
 # This file is autogenerated, and will be recreated on upgrade if it is deleted.
 # Any changes you make will be preserved.

-# Whether to report homeserver usage statistics.
+# Whether to report anonymized homeserver usage statistics.
 report_stats: false
 EOF
    fi
@@ -37,7 +37,7 @@ msgstr ""
 #. Type: boolean
 #. Description
 #: ../templates:2001
-msgid "Report homeserver usage statistics?"
+msgid "Report anonymous statistics?"
 msgstr ""

 #. Type: boolean
@@ -45,11 +45,11 @@ msgstr ""
 #: ../templates:2001
 msgid ""
 "Developers of Matrix and Synapse really appreciate helping the project out "
-"by reporting homeserver usage statistics from this homeserver. Your "
-"homeserver's server name, along with very basic aggregate data (e.g. "
-"number of users) will be reported. But it helps track the growth of the "
-"Matrix community, and helps in making Matrix a success, as well as to "
-"convince other networks that they should peer with Matrix."
+"by reporting anonymized usage statistics from this homeserver. Only very "
+"basic aggregate data (e.g. number of users) will be reported, but it helps "
+"track the growth of the Matrix community, and helps in making Matrix a "
+"success, as well as to convince other networks that they should peer with "
+"Matrix."
 msgstr ""

 #. Type: boolean
@@ -10,13 +10,12 @@ _Description: Name of the server:
 Template: matrix-synapse/report-stats
 Type: boolean
 Default: false
-_Description: Report homeserver usage statistics?
+_Description: Report anonymous statistics?
 Developers of Matrix and Synapse really appreciate helping the
- project out by reporting homeserver usage statistics from this
- homeserver. Your homeserver's server name, along with very basic
- aggregate data (e.g. number of users) will be reported. But it
- helps track the growth of the Matrix community, and helps in
- making Matrix a success, as well as to convince other networks
- that they should peer with Matrix.
+ project out by reporting anonymized usage statistics from this
+ homeserver. Only very basic aggregate data (e.g. number of users)
+ will be reported, but it helps track the growth of the Matrix
+ community, and helps in making Matrix a success, as well as to
+ convince other networks that they should peer with Matrix.
 .
 Thank you.
@@ -67,13 +67,6 @@ The following environment variables are supported in `generate` mode:
 * `UID`, `GID`: the user id and group id to use for creating the data
  directories. If unset, and no user is set via `docker run --user`, defaults
  to `991`, `991`.
-* `SYNAPSE_LOG_LEVEL`: the log level to use (one of `DEBUG`, `INFO`, `WARNING` or `ERROR`).
-  Defaults to `INFO`.
-* `SYNAPSE_LOG_SENSITIVE`: if set and the log level is set to `DEBUG`, Synapse
-  will log sensitive information such as access tokens.
-  This should not be needed unless you are a developer attempting to debug something
-  particularly tricky.
-

 ## Postgres

@@ -81,8 +81,6 @@ rc_invites:

 federation_rr_transactions_per_room_per_second: 9999

-allow_device_name_lookup_over_federation: true
-
 ## Experimental Features ##

 experimental_features:
@@ -49,17 +49,11 @@ handlers:
    class: logging.StreamHandler
    formatter: precise

-{% if not SYNAPSE_LOG_SENSITIVE %}
-{#
-  If SYNAPSE_LOG_SENSITIVE is unset, then override synapse.storage.SQL to INFO
-  so that DEBUG entries (containing sensitive information) are not emitted.
-#}
 loggers:
    synapse.storage.SQL:
        # beware: increasing this to DEBUG will make synapse log sensitive
        # information such as access tokens.
        level: INFO
-{% endif %}

 root:
    level: {{ SYNAPSE_LOG_LEVEL or "INFO" }}
@@ -29,10 +29,6 @@
 #   * SYNAPSE_USE_EXPERIMENTAL_FORKING_LAUNCHER: Whether to use the forking launcher,
 #         only intended for usage in Complement at the moment.
 #         No stability guarantees are provided.
-#   * SYNAPSE_LOG_LEVEL: Set this to DEBUG, INFO, WARNING or ERROR to change the
-#         log level. INFO is the default.
-#   * SYNAPSE_LOG_SENSITIVE: If unset, SQL and SQL values won't be logged,
-#         regardless of the SYNAPSE_LOG_LEVEL setting.
 #
 # NOTE: According to Complement's ENTRYPOINT expectations for a homeserver image (as defined
 # in the project's README), this script may be run multiple times, and functionality should
@@ -42,7 +38,7 @@ import os
 import subprocess
 import sys
 from pathlib import Path
-from typing import Any, Dict, List, Mapping, MutableMapping, NoReturn, Optional, Set
+from typing import Any, Dict, List, Mapping, MutableMapping, NoReturn, Set

 import yaml
 from jinja2 import Environment, FileSystemLoader
@@ -556,17 +552,13 @@ def generate_worker_log_config(
    Returns: the path to the generated file
    """
    # Check whether we should write worker logs to disk, in addition to the console
-    extra_log_template_args: Dict[str, Optional[str]] = {}
+    extra_log_template_args = {}
    if environ.get("SYNAPSE_WORKERS_WRITE_LOGS_TO_DISK"):
-        extra_log_template_args["LOG_FILE_PATH"] = f"{data_dir}/logs/{worker_name}.log"
-
-    extra_log_template_args["SYNAPSE_LOG_LEVEL"] = environ.get("SYNAPSE_LOG_LEVEL")
-    extra_log_template_args["SYNAPSE_LOG_SENSITIVE"] = environ.get(
-        "SYNAPSE_LOG_SENSITIVE"
-    )
-
+        extra_log_template_args["LOG_FILE_PATH"] = "{dir}/logs/{name}.log".format(
+            dir=data_dir, name=worker_name
+        )
    # Render and write the file
-    log_config_filepath = f"/conf/workers/{worker_name}.log.config"
+    log_config_filepath = "/conf/workers/{name}.log.config".format(name=worker_name)
    convert(
        "/conf/log.config",
        log_config_filepath,
@@ -69,7 +69,7 @@
      - [Federation](usage/administration/admin_api/federation.md)
    - [Manhole](manhole.md)
    - [Monitoring](metrics-howto.md)
-      - [Reporting Homeserver Usage Statistics](usage/administration/monitoring/reporting_homeserver_usage_statistics.md)
+      - [Reporting Anonymised Statistics](usage/administration/monitoring/reporting_anonymised_statistics.md)
    - [Understanding Synapse Through Grafana Graphs](usage/administration/understanding_synapse_through_grafana_graphs.md)
    - [Useful SQL for Admins](usage/administration/useful_sql_for_admins.md)
    - [Database Maintenance Tools](usage/administration/database_maintenance_tools.md)
@@ -81,7 +81,6 @@
 # Development
  - [Contributing Guide](development/contributing_guide.md)
  - [Code Style](code_style.md)
-  - [Reviewing Code](development/reviews.md)
  - [Release Cycle](development/releases.md)
  - [Git Usage](development/git.md)
  - [Testing]()
@@ -309,10 +309,6 @@ The above will run a monolithic (single-process) Synapse with SQLite as the data
 - Passing `POSTGRES=1` as an environment variable to use the Postgres database instead.
 - Passing `WORKERS=1` as an environment variable to use a workerised setup instead. This option implies the use of Postgres.

-To increase the log level for the tests, set `SYNAPSE_TEST_LOG_LEVEL`, e.g:
-```sh
-SYNAPSE_TEST_LOG_LEVEL=DEBUG COMPLEMENT_DIR=../complement ./scripts-dev/complement.sh -run TestImportHistoricalMessages
-```

 ### Prettier formatting with `gotestfmt`

@@ -351,7 +347,7 @@ To prepare a Pull Request, please:
 3. `git push` your commit to your fork of Synapse;
 4. on GitHub, [create the Pull Request](https://docs.github.com/en/github/collaborating-with-issues-and-pull-requests/creating-a-pull-request);
 5. add a [changelog entry](#changelog) and push it to your Pull Request;
-6. that's it for now, a non-draft pull request will automatically request review from the team;
+6. for most contributors, that's all - however, if you are a member of the organization `matrix-org`, on GitHub, please request a review from `matrix.org / Synapse Core`.
 7. if you need to update your PR, please avoid rebasing and just add new commits to your branch.


@@ -527,13 +523,10 @@ From this point, you should:
 1. Look at the results of the CI pipeline.
    - If there is any error, fix the error.
 2. If a developer has requested changes, make these changes and let us know if it is ready for a developer to review again.
-   - A pull request is a conversation, if you disagree with the suggestions, please respond and discuss it.
 3. Create a new commit with the changes.
    - Please do NOT overwrite the history. New commits make the reviewer's life easier.
    - Push this commits to your Pull Request.
 4. Back to 1.
-5. Once the pull request is ready for review again please re-request review from whichever developer did your initial
-  review (or leave a comment in the pull request that you believe all required changes have been done).

 Once both the CI and the developers are happy, the patch will be merged into Synapse and released shortly!

@@ -1,41 +0,0 @@
-Some notes on how we do reviews
-===============================
-
-The Synapse team works off a shared review queue -- any new pull requests for
-Synapse (or related projects) has a review requested from the entire team. Team
-members should process this queue using the following rules:
-
-* Any high urgency pull requests (e.g. fixes for broken continuous integration
-  or fixes for release blockers);
-* Follow-up reviews for pull requests which have previously received reviews;
-* Any remaining pull requests.
-
-For the latter two categories above, older pull requests should be prioritised.
-
-It is explicit that there is no priority given to pull requests from the team
-(vs from the community). If a pull request requires a quick turn around, please
-explicitly communicate this via [#synapse-dev:matrix.org](https://matrix.to/#/#synapse-dev:matrix.org)
-or as a comment on the pull request.
-
-Once an initial review has been completed and the author has made additional changes,
-follow-up reviews should go back to the same reviewer. This helps build a shared
-context and conversation between author and reviewer.
-
-As a team we aim to keep the number of inflight pull requests to a minimum to ensure
-that ongoing work is finished before starting new work.
-
-Performing a review
-------------------
-
-To communicate to the rest of the team the status of each pull request, team
-members should do the following:
-
-* Assign themselves to the pull request (they should be left assigned to the
-  pull request until it is merged, closed, or are no longer the reviewer);
-* Review the pull request by leaving comments, questions, and suggestions;
-* Mark the pull request appropriately (as needing changes or accepted).
-
-If you are unsure about a particular part of the pull request (or are not confident
-in your understanding of part of the code) then ask questions or request review
-from the team again. When requesting review from the team be sure to leave a comment
-with the rationale on why you're putting it back in the queue.
@@ -143,14 +143,6 @@ to do step 2.

 It is safe to at any time kill the port script and restart it.

-However, under no circumstances should the SQLite database be `VACUUM`ed between
-multiple runs of the script. Doing so can lead to an inconsistent copy of your database
-into Postgres.
-To avoid accidental error, the script will check that SQLite's `auto_vacuum` mechanism
-is disabled, but the script is not able to protect against a manual `VACUUM` operation
-performed either by the administrator or by any automated task that the administrator
-may have configured.
-
 Note that the database may take up significantly more (25% - 100% more)
 space on disk after porting to Postgres.

@@ -1,11 +1,11 @@
-# Reporting Homeserver Usage Statistics
+# Reporting Anonymised Statistics

 When generating your Synapse configuration file, you are asked whether you
-would like to report usage statistics to Matrix.org. These statistics
+would like to report anonymised statistics to Matrix.org. These statistics
 provide the foundation a glimpse into the number of Synapse homeservers
 participating in the network, as well as statistics such as the number of
 rooms being created and messages being sent. This feature is sometimes
-affectionately called "phone home" stats. Reporting
+affectionately called "phone-home" stats. Reporting
 [is optional](../../configuration/config_documentation.md#report_stats)
 and the reporting endpoint
 [can be configured](../../configuration/config_documentation.md#report_stats_endpoint),
@@ -21,9 +21,9 @@ The following statistics are sent to the configured reporting endpoint:

 | Statistic Name             | Type   | Description                                                                                                                                                                                                                                                                                     |
 |----------------------------|--------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
-| `homeserver`               | string | The homeserver's server name.                                                                                                                                                                                                                                                                   |
 | `memory_rss`               | int    | The memory usage of the process (in kilobytes on Unix-based systems, bytes on MacOS).                                                                                                                                                                                                           |
 | `cpu_average`              | int    | CPU time in % of a single core (not % of all cores).                                                                                                                                                                                                                                            |              
+| `homeserver`               | string | The homeserver's server name.                                                                                                                                                                                                                                                                   |
 | `server_context`           | string | An arbitrary string used to group statistics from a set of homeservers.                                                                                                                                                                                                                         |
 | `timestamp`                | int    | The current time, represented as the number of seconds since the epoch.                                                                                                                                                                                                                         |                 
 | `uptime_seconds`           | int    | The number of seconds since the homeserver was last started.                                                                                                                                                                                                                                    |
@@ -591,7 +591,7 @@ Example configuration:
 dummy_events_threshold: 5
 ```
 ---
-### `delete_stale_devices_after`
+Config option `delete_stale_devices_after`

 An optional duration. If set, Synapse will run a daily background task to log out and
 delete any device that hasn't been accessed for more than the specified amount of time.
@@ -1257,98 +1257,6 @@ database:
    cp_max: 10
 ```
 ---
-### `databases`
-
-The `databases` option allows specifying a mapping between certain database tables and
-database host details, spreading the load of a single Synapse instance across multiple
-database backends. This is often referred to as "database sharding". This option is only
-supported for PostgreSQL database backends.
-
-**Important note:** This is a supported option, but is not currently used in production by the
-Matrix.org Foundation. Proceed with caution and always make backups.
-
-`databases` is a dictionary of arbitrarily-named database entries. Each entry is equivalent
-to the value of the `database` homeserver config option (see above), with the addition of
-a `data_stores` key. `data_stores` is an array of strings that specifies the data store(s)
-(a defined label for a set of tables) that should be stored on the associated database
-backend entry.
-
-The currently defined values for `data_stores` are:
-
-* `"state"`: Database that relates to state groups will be stored in this database.
-
-  Specifically, that means the following tables:
-  * `state_groups`
-  * `state_group_edges`
-  * `state_groups_state`
-
-  And the following sequences:
-  * `state_groups_seq_id`
-
-* `"main"`: All other database tables and sequences.
-
-All databases will end up with additional tables used for tracking database schema migrations
-and any pending background updates. Synapse will create these automatically on startup when checking for
-and/or performing database schema migrations.
-
-To migrate an existing database configuration (e.g. all tables on a single database) to a different
-configuration (e.g. the "main" data store on one database, and "state" on another), do the following:
-
-1. Take a backup of your existing database. Things can and do go wrong and database corruption is no joke!
-2. Ensure all pending database migrations have been applied and background updates have run. The simplest
-   way to do this is to use the `update_synapse_database` script supplied with your Synapse installation.
-
-   ```sh
-   update_synapse_database --database-config homeserver.yaml --run-background-updates
-   ```
-
-3. Copy over the necessary tables and sequences from one database to the other. Tables relating to database
-   migrations, schemas, schema versions and background updates should **not** be copied.
-
-   As an example, say that you'd like to split out the "state" data store from an existing database which
-   currently contains all data stores.
-
-   Simply copy the tables and sequences defined above for the "state" datastore from the existing database
-   to the secondary database. As noted above, additional tables will be created in the secondary database
-   when Synapse is started.
-
-4. Modify/create the `databases` option in your `homeserver.yaml` to match the desired database configuration.
-5. Start Synapse. Check that it starts up successfully and that things generally seem to be working.
-6. Drop the old tables that were copied in step 3.
-
-Only one of the options `database` or `databases` may be specified in your config, but not both.
-
-Example configuration:
-
-```yaml
-databases:
-  basement_box:
-    name: psycopg2
-    txn_limit: 10000
-    data_stores: ["main"]
-    args:
-      user: synapse_user
-      password: secretpassword
-      database: synapse_main
-      host: localhost
-      port: 5432
-      cp_min: 5
-      cp_max: 10
-
-  my_other_database:
-    name: psycopg2
-    txn_limit: 10000
-    data_stores: ["state"]
-    args:
-      user: synapse_user
-      password: secretpassword
-      database: synapse_state
-      host: localhost
-      port: 5432
-      cp_min: 5
-      cp_max: 10
-```
---
 ## Logging ##
 Config options related to logging. 

@@ -1935,7 +1843,7 @@ Example configuration:
 turn_shared_secret: "YOUR_SHARED_SECRET"
 ```
 ----
-### `turn_username` and `turn_password`
+Config options: `turn_username` and `turn_password`

 The Username and password if the TURN server needs them and does not use a token.

@@ -2409,14 +2317,9 @@ metrics_flags:
 ---
 ### `report_stats`

-Whether or not to report homeserver usage statistics. This is originally
+Whether or not to report anonymized homeserver usage statistics. This is originally
 set when generating the config. Set this option to true or false to change the current
-behavior. See
-[Reporting Homeserver Usage Statistics](../administration/monitoring/reporting_homeserver_usage_statistics.md)
-for information on what data is reported.
-
-Statistics will be reported 5 minutes after Synapse starts, and then every 3 hours
-after that.
+behavior. 

 Example configuration:
 ```yaml
@@ -2425,7 +2328,7 @@ report_stats: true
 ---
 ### `report_stats_endpoint`

-The endpoint to report homeserver usage statistics to.
+The endpoint to report the anonymized homeserver usage statistics to.
 Defaults to https://matrix.org/report-usage-stats/push

 Example configuration:
@@ -3470,7 +3373,7 @@ alias_creation_rules:
    action: deny
 ```
 ---
-### `room_list_publication_rules`
+Config options: `room_list_publication_rules`

 The `room_list_publication_rules` option controls who can publish and
 which rooms can be published in the public room list.
@@ -24,11 +24,6 @@ Finally, we also stylise the chapter titles in the left sidebar by indenting the
 slightly so that they are more visually distinguishable from the section headers
 (the bold titles). This is done through the `indent-section-headers.css` file.

-In addition to these modifications, we have added a version picker to the documentation.
-Users can switch between documentations for different versions of Synapse.
-This functionality was implemented through the `version-picker.js` and
-`version-picker.css` files.
-
 More information can be found in mdbook's official documentation for
 [injecting page JS/CSS](https://rust-lang.github.io/mdBook/format/config.html)
 and
@@ -131,18 +131,6 @@
                            <i class="fa fa-search"></i>
                        </button>
                        {{/if}}
-                        <div class="version-picker">
-                            <div class="dropdown">
-                                <div class="select">
-                                    <span></span>
-                                    <i class="fa fa-chevron-down"></i>
-                                </div>
-                                <input type="hidden" name="version">
-                                <ul class="dropdown-menu">
-                                    <!-- Versions will be added dynamically in version-picker.js -->
-                                </ul>
-                            </div>
-                        </div>      
                    </div>

                    <h1 class="menu-title">{{ book_title }}</h1>
@@ -321,4 +309,4 @@
        {{/if}}

    </body>
-</html>
+</html>
@@ -1,78 +0,0 @@
-.version-picker {
-    display: flex;
-    align-items: center;
-}
-
-.version-picker .dropdown {
-    width: 130px;
-    max-height: 29px;
-    margin-left: 10px;
-    display: inline-block;
-    border-radius: 4px;
-    border: 1px solid var(--theme-popup-border);
-    position: relative;
-    font-size: 13px;
-    color: var(--fg);
-    height: 100%;
-    text-align: left;
-}
-.version-picker .dropdown .select {
-    cursor: pointer;
-    display: block;
-    padding: 5px 2px 5px 15px;
-}
-.version-picker .dropdown .select > i {
-    font-size: 10px;
-    color: var(--fg);
-    cursor: pointer;
-    float: right;
-    line-height: 20px !important;
-}
-.version-picker .dropdown:hover {
-    border: 1px solid var(--theme-popup-border);
-}
-.version-picker .dropdown:active {
-    background-color: var(--theme-popup-bg);
-}
-.version-picker .dropdown.active:hover,
-.version-picker .dropdown.active {
-    border: 1px solid var(--theme-popup-border);
-    border-radius: 2px 2px 0 0;
-    background-color: var(--theme-popup-bg);
-}
-.version-picker .dropdown.active .select > i {
-    transform: rotate(-180deg);
-}
-.version-picker .dropdown .dropdown-menu {
-    position: absolute;
-    background-color: var(--theme-popup-bg);
-    width: 100%;
-    left: -1px;
-    right: 1px;
-    margin-top: 1px;
-    border: 1px solid var(--theme-popup-border);
-    border-radius: 0 0 4px 4px;
-    overflow: hidden;
-    display: none;
-    max-height: 300px;
-    overflow-y: auto;
-    z-index: 9;
-}
-.version-picker .dropdown .dropdown-menu li {
-    font-size: 12px;
-    padding: 6px 20px;
-    cursor: pointer;
-} 
-.version-picker .dropdown .dropdown-menu {
-    padding: 0;
-    list-style: none;
-}
-.version-picker .dropdown .dropdown-menu li:hover {
-    background-color: var(--theme-hover);
-}
-.version-picker .dropdown .dropdown-menu li.active::before {
-    display: inline-block;
-    content: "✓";
-    margin-inline-start: -14px;
-    width: 14px;
-}
@@ -1,127 +0,0 @@
-
-const dropdown = document.querySelector('.version-picker .dropdown');
-const dropdownMenu = dropdown.querySelector('.dropdown-menu');
-
-fetchVersions(dropdown, dropdownMenu).then(() => {
-    initializeVersionDropdown(dropdown, dropdownMenu);
-});
-
-/**
- * Initialize the dropdown functionality for version selection.
- * 
- * @param {Element} dropdown - The dropdown element.
- * @param {Element} dropdownMenu - The dropdown menu element.
- */
-function initializeVersionDropdown(dropdown, dropdownMenu) {
-    // Toggle the dropdown menu on click
-    dropdown.addEventListener('click', function () {
-        this.setAttribute('tabindex', 1);
-        this.classList.toggle('active');
-        dropdownMenu.style.display = (dropdownMenu.style.display === 'block') ? 'none' : 'block';
-    });
-  
-    // Remove the 'active' class and hide the dropdown menu on focusout
-    dropdown.addEventListener('focusout', function () {
-        this.classList.remove('active');
-        dropdownMenu.style.display = 'none';
-    });
-  
-    // Handle item selection within the dropdown menu
-    const dropdownMenuItems = dropdownMenu.querySelectorAll('li');    
-    dropdownMenuItems.forEach(function (item) {
-        item.addEventListener('click', function () {
-            dropdownMenuItems.forEach(function (item) {
-                item.classList.remove('active');
-            });
-            this.classList.add('active');
-            dropdown.querySelector('span').textContent = this.textContent;
-            dropdown.querySelector('input').value = this.getAttribute('id');
-
-            window.location.href = changeVersion(window.location.href, this.textContent);
-        });
-    });
-};
-
-/**
- * This function fetches the available versions from a GitHub repository
- * and inserts them into the version picker.
- * 
- * @param {Element} dropdown - The dropdown element.
- * @param {Element} dropdownMenu - The dropdown menu element.
- * @returns {Promise<Array<string>>} A promise that resolves with an array of available versions.
- */
-function fetchVersions(dropdown, dropdownMenu) {
-    return new Promise((resolve, reject) => {
-        window.addEventListener("load", () => {
-
-            fetch("https://api.github.com/repos/matrix-org/synapse/git/trees/gh-pages", {
-                cache: "force-cache",
-            }).then(res => 
-                res.json()
-            ).then(resObject => {
-                const excluded = ['dev-docs', 'v1.91.0', 'v1.80.0', 'v1.69.0'];
-                const tree = resObject.tree.filter(item => item.type === "tree" && !excluded.includes(item.path));
-                const versions = tree.map(item => item.path).sort(sortVersions);
-
-                // Create a list of <li> items for versions
-                versions.forEach((version) => {
-                    const li = document.createElement("li");
-                    li.textContent = version;
-                    li.id = version;
-    
-                    if (window.SYNAPSE_VERSION === version) {
-                        li.classList.add('active');
-                        dropdown.querySelector('span').textContent = version;
-                        dropdown.querySelector('input').value = version;
-                    }
-    
-                    dropdownMenu.appendChild(li);
-                });
-
-                resolve(versions);
-
-            }).catch(ex => {
-                console.error("Failed to fetch version data", ex);
-                reject(ex);
-            })
-        });
-    });
-}
-
-/**
- * Custom sorting function to sort an array of version strings.
- *
- * @param {string} a - The first version string to compare.
- * @param {string} b - The second version string to compare.
- * @returns {number} - A negative number if a should come before b, a positive number if b should come before a, or 0 if they are equal.
- */
-function sortVersions(a, b) {
-    // Put 'develop' and 'latest' at the top
-    if (a === 'develop' || a === 'latest') return -1;
-    if (b === 'develop' || b === 'latest') return 1;
-
-    const versionA = (a.match(/v\d+(\.\d+)+/) || [])[0];
-    const versionB = (b.match(/v\d+(\.\d+)+/) || [])[0];
-
-    return versionB.localeCompare(versionA);
-}
-
-/**
- * Change the version in a URL path.
- *
- * @param {string} url - The original URL to be modified.
- * @param {string} newVersion - The new version to replace the existing version in the URL.
- * @returns {string} The updated URL with the new version.
- */
-function changeVersion(url, newVersion) {
-    const parsedURL = new URL(url);
-    const pathSegments = parsedURL.pathname.split('/');
-  
-    // Modify the version
-    pathSegments[2] = newVersion;
-
-    // Reconstruct the URL
-    parsedURL.pathname = pathSegments.join('/');
-  
-    return parsedURL.href;
-}
@@ -1 +0,0 @@
-window.SYNAPSE_VERSION = 'v1.63';
@@ -73,6 +73,7 @@ exclude = (?x)
   |tests/util/test_lrucache.py
   |tests/util/test_rwlock.py
   |tests/util/test_wheel_timer.py
+   |tests/utils.py
   )$

 [mypy-synapse.federation.transport.client]
@@ -126,9 +127,6 @@ disallow_untyped_defs = True
 [mypy-tests.federation.transport.test_client]
 disallow_untyped_defs = True

-[mypy-tests.utils]
-disallow_untyped_defs = True
-

 ;; Dependencies without annotations
 ;; Before ignoring a module, check to see if type stubs are available.
@@ -502,7 +502,7 @@ pyasn1 = ">=0.4.6"

 [[package]]
 name = "lxml"
-version = "4.9.1"
+version = "4.8.0"
 description = "Powerful and Pythonic XML processing library combining libxml2/libxslt with the ElementTree API."
 category = "main"
 optional = true
@@ -540,7 +540,7 @@ test = ["tox", "twisted", "aiounittest"]

 [[package]]
 name = "matrix-synapse-ldap3"
-version = "0.2.1"
+version = "0.2.0"
 description = "An LDAP3 auth provider for Synapse"
 category = "main"
 optional = true
@@ -552,7 +552,7 @@ service-identity = "*"
 Twisted = ">=15.1.0"

 [package.extras]
-dev = ["matrix-synapse", "tox", "ldaptor", "mypy (==0.910)", "types-setuptools", "black (==22.3.0)", "flake8 (==4.0.1)", "isort (==5.9.3)"]
+dev = ["matrix-synapse", "tox", "ldaptor", "mypy (==0.910)", "types-setuptools", "black (==21.9b0)", "flake8 (==4.0.1)", "isort (==5.9.3)"]

 [[package]]
 name = "mccabe"
@@ -1937,76 +1937,67 @@ ldap3 = [
    {file = "ldap3-2.9.1.tar.gz", hash = "sha256:f3e7fc4718e3f09dda568b57100095e0ce58633bcabbed8667ce3f8fbaa4229f"},
 ]
 lxml = [
-    {file = "lxml-4.9.1-cp27-cp27m-macosx_10_15_x86_64.whl", hash = "sha256:98cafc618614d72b02185ac583c6f7796202062c41d2eeecdf07820bad3295ed"},
-    {file = "lxml-4.9.1-cp27-cp27m-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:c62e8dd9754b7debda0c5ba59d34509c4688f853588d75b53c3791983faa96fc"},
-    {file = "lxml-4.9.1-cp27-cp27m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:21fb3d24ab430fc538a96e9fbb9b150029914805d551deeac7d7822f64631dfc"},
-    {file = "lxml-4.9.1-cp27-cp27m-win32.whl", hash = "sha256:86e92728ef3fc842c50a5cb1d5ba2bc66db7da08a7af53fb3da79e202d1b2cd3"},
-    {file = "lxml-4.9.1-cp27-cp27m-win_amd64.whl", hash = "sha256:4cfbe42c686f33944e12f45a27d25a492cc0e43e1dc1da5d6a87cbcaf2e95627"},
-    {file = "lxml-4.9.1-cp27-cp27mu-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:dad7b164905d3e534883281c050180afcf1e230c3d4a54e8038aa5cfcf312b84"},
-    {file = "lxml-4.9.1-cp27-cp27mu-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:a614e4afed58c14254e67862456d212c4dcceebab2eaa44d627c2ca04bf86837"},
-    {file = "lxml-4.9.1-cp310-cp310-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_24_i686.whl", hash = "sha256:f9ced82717c7ec65a67667bb05865ffe38af0e835cdd78728f1209c8fffe0cad"},
-    {file = "lxml-4.9.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.manylinux_2_24_aarch64.whl", hash = "sha256:d9fc0bf3ff86c17348dfc5d322f627d78273eba545db865c3cd14b3f19e57fa5"},
-    {file = "lxml-4.9.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_24_x86_64.whl", hash = "sha256:e5f66bdf0976ec667fc4594d2812a00b07ed14d1b44259d19a41ae3fff99f2b8"},
-    {file = "lxml-4.9.1-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:fe17d10b97fdf58155f858606bddb4e037b805a60ae023c009f760d8361a4eb8"},
-    {file = "lxml-4.9.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:8caf4d16b31961e964c62194ea3e26a0e9561cdf72eecb1781458b67ec83423d"},
-    {file = "lxml-4.9.1-cp310-cp310-win32.whl", hash = "sha256:4780677767dd52b99f0af1f123bc2c22873d30b474aa0e2fc3fe5e02217687c7"},
-    {file = "lxml-4.9.1-cp310-cp310-win_amd64.whl", hash = "sha256:b122a188cd292c4d2fcd78d04f863b789ef43aa129b233d7c9004de08693728b"},
-    {file = "lxml-4.9.1-cp311-cp311-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_24_i686.whl", hash = "sha256:be9eb06489bc975c38706902cbc6888f39e946b81383abc2838d186f0e8b6a9d"},
-    {file = "lxml-4.9.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_24_x86_64.whl", hash = "sha256:f1be258c4d3dc609e654a1dc59d37b17d7fef05df912c01fc2e15eb43a9735f3"},
-    {file = "lxml-4.9.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:927a9dd016d6033bc12e0bf5dee1dde140235fc8d0d51099353c76081c03dc29"},
-    {file = "lxml-4.9.1-cp35-cp35m-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:9232b09f5efee6a495a99ae6824881940d6447debe272ea400c02e3b68aad85d"},
-    {file = "lxml-4.9.1-cp35-cp35m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:04da965dfebb5dac2619cb90fcf93efdb35b3c6994fea58a157a834f2f94b318"},
-    {file = "lxml-4.9.1-cp35-cp35m-win32.whl", hash = "sha256:4d5bae0a37af799207140652a700f21a85946f107a199bcb06720b13a4f1f0b7"},
-    {file = "lxml-4.9.1-cp35-cp35m-win_amd64.whl", hash = "sha256:4878e667ebabe9b65e785ac8da4d48886fe81193a84bbe49f12acff8f7a383a4"},
-    {file = "lxml-4.9.1-cp36-cp36m-macosx_10_15_x86_64.whl", hash = "sha256:1355755b62c28950f9ce123c7a41460ed9743c699905cbe664a5bcc5c9c7c7fb"},
-    {file = "lxml-4.9.1-cp36-cp36m-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_24_i686.whl", hash = "sha256:bcaa1c495ce623966d9fc8a187da80082334236a2a1c7e141763ffaf7a405067"},
-    {file = "lxml-4.9.1-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6eafc048ea3f1b3c136c71a86db393be36b5b3d9c87b1c25204e7d397cee9536"},
-    {file = "lxml-4.9.1-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_24_x86_64.whl", hash = "sha256:13c90064b224e10c14dcdf8086688d3f0e612db53766e7478d7754703295c7c8"},
-    {file = "lxml-4.9.1-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:206a51077773c6c5d2ce1991327cda719063a47adc02bd703c56a662cdb6c58b"},
-    {file = "lxml-4.9.1-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:e8f0c9d65da595cfe91713bc1222af9ecabd37971762cb830dea2fc3b3bb2acf"},
-    {file = "lxml-4.9.1-cp36-cp36m-musllinux_1_1_aarch64.whl", hash = "sha256:8f0a4d179c9a941eb80c3a63cdb495e539e064f8054230844dcf2fcb812b71d3"},
-    {file = "lxml-4.9.1-cp36-cp36m-musllinux_1_1_x86_64.whl", hash = "sha256:830c88747dce8a3e7525defa68afd742b4580df6aa2fdd6f0855481e3994d391"},
-    {file = "lxml-4.9.1-cp36-cp36m-win32.whl", hash = "sha256:1e1cf47774373777936c5aabad489fef7b1c087dcd1f426b621fda9dcc12994e"},
-    {file = "lxml-4.9.1-cp36-cp36m-win_amd64.whl", hash = "sha256:5974895115737a74a00b321e339b9c3f45c20275d226398ae79ac008d908bff7"},
-    {file = "lxml-4.9.1-cp37-cp37m-macosx_10_15_x86_64.whl", hash = "sha256:1423631e3d51008871299525b541413c9b6c6423593e89f9c4cfbe8460afc0a2"},
-    {file = "lxml-4.9.1-cp37-cp37m-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_24_i686.whl", hash = "sha256:2aaf6a0a6465d39b5ca69688fce82d20088c1838534982996ec46633dc7ad6cc"},
-    {file = "lxml-4.9.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.manylinux_2_24_aarch64.whl", hash = "sha256:9f36de4cd0c262dd9927886cc2305aa3f2210db437aa4fed3fb4940b8bf4592c"},
-    {file = "lxml-4.9.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_24_x86_64.whl", hash = "sha256:ae06c1e4bc60ee076292e582a7512f304abdf6c70db59b56745cca1684f875a4"},
-    {file = "lxml-4.9.1-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:57e4d637258703d14171b54203fd6822fda218c6c2658a7d30816b10995f29f3"},
-    {file = "lxml-4.9.1-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:6d279033bf614953c3fc4a0aa9ac33a21e8044ca72d4fa8b9273fe75359d5cca"},
-    {file = "lxml-4.9.1-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:a60f90bba4c37962cbf210f0188ecca87daafdf60271f4c6948606e4dabf8785"},
-    {file = "lxml-4.9.1-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:6ca2264f341dd81e41f3fffecec6e446aa2121e0b8d026fb5130e02de1402785"},
-    {file = "lxml-4.9.1-cp37-cp37m-win32.whl", hash = "sha256:27e590352c76156f50f538dbcebd1925317a0f70540f7dc8c97d2931c595783a"},
-    {file = "lxml-4.9.1-cp37-cp37m-win_amd64.whl", hash = "sha256:eea5d6443b093e1545ad0210e6cf27f920482bfcf5c77cdc8596aec73523bb7e"},
-    {file = "lxml-4.9.1-cp38-cp38-macosx_10_15_x86_64.whl", hash = "sha256:f05251bbc2145349b8d0b77c0d4e5f3b228418807b1ee27cefb11f69ed3d233b"},
-    {file = "lxml-4.9.1-cp38-cp38-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_24_i686.whl", hash = "sha256:487c8e61d7acc50b8be82bda8c8d21d20e133c3cbf41bd8ad7eb1aaeb3f07c97"},
-    {file = "lxml-4.9.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.manylinux_2_24_aarch64.whl", hash = "sha256:8d1a92d8e90b286d491e5626af53afef2ba04da33e82e30744795c71880eaa21"},
-    {file = "lxml-4.9.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_24_x86_64.whl", hash = "sha256:b570da8cd0012f4af9fa76a5635cd31f707473e65a5a335b186069d5c7121ff2"},
-    {file = "lxml-4.9.1-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:5ef87fca280fb15342726bd5f980f6faf8b84a5287fcc2d4962ea8af88b35130"},
-    {file = "lxml-4.9.1-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:93e414e3206779ef41e5ff2448067213febf260ba747fc65389a3ddaa3fb8715"},
-    {file = "lxml-4.9.1-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:6653071f4f9bac46fbc30f3c7838b0e9063ee335908c5d61fb7a4a86c8fd2036"},
-    {file = "lxml-4.9.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:32a73c53783becdb7eaf75a2a1525ea8e49379fb7248c3eeefb9412123536387"},
-    {file = "lxml-4.9.1-cp38-cp38-win32.whl", hash = "sha256:1a7c59c6ffd6ef5db362b798f350e24ab2cfa5700d53ac6681918f314a4d3b94"},
-    {file = "lxml-4.9.1-cp38-cp38-win_amd64.whl", hash = "sha256:1436cf0063bba7888e43f1ba8d58824f085410ea2025befe81150aceb123e345"},
-    {file = "lxml-4.9.1-cp39-cp39-macosx_10_15_x86_64.whl", hash = "sha256:4beea0f31491bc086991b97517b9683e5cfb369205dac0148ef685ac12a20a67"},
-    {file = "lxml-4.9.1-cp39-cp39-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_24_i686.whl", hash = "sha256:41fb58868b816c202e8881fd0f179a4644ce6e7cbbb248ef0283a34b73ec73bb"},
-    {file = "lxml-4.9.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.manylinux_2_24_aarch64.whl", hash = "sha256:bd34f6d1810d9354dc7e35158aa6cc33456be7706df4420819af6ed966e85448"},
-    {file = "lxml-4.9.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_24_x86_64.whl", hash = "sha256:edffbe3c510d8f4bf8640e02ca019e48a9b72357318383ca60e3330c23aaffc7"},
-    {file = "lxml-4.9.1-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:6d949f53ad4fc7cf02c44d6678e7ff05ec5f5552b235b9e136bd52e9bf730b91"},
-    {file = "lxml-4.9.1-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:079b68f197c796e42aa80b1f739f058dcee796dc725cc9a1be0cdb08fc45b000"},
-    {file = "lxml-4.9.1-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:9c3a88d20e4fe4a2a4a84bf439a5ac9c9aba400b85244c63a1ab7088f85d9d25"},
-    {file = "lxml-4.9.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:4e285b5f2bf321fc0857b491b5028c5f276ec0c873b985d58d7748ece1d770dd"},
-    {file = "lxml-4.9.1-cp39-cp39-win32.whl", hash = "sha256:ef72013e20dd5ba86a8ae1aed7f56f31d3374189aa8b433e7b12ad182c0d2dfb"},
-    {file = "lxml-4.9.1-cp39-cp39-win_amd64.whl", hash = "sha256:10d2017f9150248563bb579cd0d07c61c58da85c922b780060dcc9a3aa9f432d"},
-    {file = "lxml-4.9.1-pp37-pypy37_pp73-macosx_10_15_x86_64.whl", hash = "sha256:0538747a9d7827ce3e16a8fdd201a99e661c7dee3c96c885d8ecba3c35d1032c"},
-    {file = "lxml-4.9.1-pp37-pypy37_pp73-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_24_i686.whl", hash = "sha256:0645e934e940107e2fdbe7c5b6fb8ec6232444260752598bc4d09511bd056c0b"},
-    {file = "lxml-4.9.1-pp37-pypy37_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_24_x86_64.whl", hash = "sha256:6daa662aba22ef3258934105be2dd9afa5bb45748f4f702a3b39a5bf53a1f4dc"},
-    {file = "lxml-4.9.1-pp38-pypy38_pp73-macosx_10_15_x86_64.whl", hash = "sha256:603a464c2e67d8a546ddaa206d98e3246e5db05594b97db844c2f0a1af37cf5b"},
-    {file = "lxml-4.9.1-pp38-pypy38_pp73-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_24_i686.whl", hash = "sha256:c4b2e0559b68455c085fb0f6178e9752c4be3bba104d6e881eb5573b399d1eb2"},
-    {file = "lxml-4.9.1-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_24_x86_64.whl", hash = "sha256:0f3f0059891d3254c7b5fb935330d6db38d6519ecd238ca4fce93c234b4a0f73"},
-    {file = "lxml-4.9.1-pp39-pypy39_pp73-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_24_i686.whl", hash = "sha256:c852b1530083a620cb0de5f3cd6826f19862bafeaf77586f1aef326e49d95f0c"},
-    {file = "lxml-4.9.1-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_24_x86_64.whl", hash = "sha256:287605bede6bd36e930577c5925fcea17cb30453d96a7b4c63c14a257118dbb9"},
-    {file = "lxml-4.9.1.tar.gz", hash = "sha256:fe749b052bb7233fe5d072fcb549221a8cb1a16725c47c37e42b0b9cb3ff2c3f"},
+    {file = "lxml-4.8.0-cp27-cp27m-macosx_10_14_x86_64.whl", hash = "sha256:e1ab2fac607842ac36864e358c42feb0960ae62c34aa4caaf12ada0a1fb5d99b"},
+    {file = "lxml-4.8.0-cp27-cp27m-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:28d1af847786f68bec57961f31221125c29d6f52d9187c01cd34dc14e2b29430"},
+    {file = "lxml-4.8.0-cp27-cp27m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:b92d40121dcbd74831b690a75533da703750f7041b4bf951befc657c37e5695a"},
+    {file = "lxml-4.8.0-cp27-cp27m-win32.whl", hash = "sha256:e01f9531ba5420838c801c21c1b0f45dbc9607cb22ea2cf132844453bec863a5"},
+    {file = "lxml-4.8.0-cp27-cp27m-win_amd64.whl", hash = "sha256:6259b511b0f2527e6d55ad87acc1c07b3cbffc3d5e050d7e7bcfa151b8202df9"},
+    {file = "lxml-4.8.0-cp27-cp27mu-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:1010042bfcac2b2dc6098260a2ed022968dbdfaf285fc65a3acf8e4eb1ffd1bc"},
+    {file = "lxml-4.8.0-cp27-cp27mu-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:fa56bb08b3dd8eac3a8c5b7d075c94e74f755fd9d8a04543ae8d37b1612dd170"},
+    {file = "lxml-4.8.0-cp310-cp310-macosx_10_15_x86_64.whl", hash = "sha256:31ba2cbc64516dcdd6c24418daa7abff989ddf3ba6d3ea6f6ce6f2ed6e754ec9"},
+    {file = "lxml-4.8.0-cp310-cp310-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_24_i686.whl", hash = "sha256:31499847fc5f73ee17dbe1b8e24c6dafc4e8d5b48803d17d22988976b0171f03"},
+    {file = "lxml-4.8.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.manylinux_2_24_aarch64.whl", hash = "sha256:5f7d7d9afc7b293147e2d506a4596641d60181a35279ef3aa5778d0d9d9123fe"},
+    {file = "lxml-4.8.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_24_x86_64.whl", hash = "sha256:a3c5f1a719aa11866ffc530d54ad965063a8cbbecae6515acbd5f0fae8f48eaa"},
+    {file = "lxml-4.8.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:6268e27873a3d191849204d00d03f65c0e343b3bcb518a6eaae05677c95621d1"},
+    {file = "lxml-4.8.0-cp310-cp310-win32.whl", hash = "sha256:330bff92c26d4aee79c5bc4d9967858bdbe73fdbdbacb5daf623a03a914fe05b"},
+    {file = "lxml-4.8.0-cp310-cp310-win_amd64.whl", hash = "sha256:b2582b238e1658c4061ebe1b4df53c435190d22457642377fd0cb30685cdfb76"},
+    {file = "lxml-4.8.0-cp35-cp35m-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:a2bfc7e2a0601b475477c954bf167dee6d0f55cb167e3f3e7cefad906e7759f6"},
+    {file = "lxml-4.8.0-cp35-cp35m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:a1547ff4b8a833511eeaceacbcd17b043214fcdb385148f9c1bc5556ca9623e2"},
+    {file = "lxml-4.8.0-cp35-cp35m-win32.whl", hash = "sha256:a9f1c3489736ff8e1c7652e9dc39f80cff820f23624f23d9eab6e122ac99b150"},
+    {file = "lxml-4.8.0-cp35-cp35m-win_amd64.whl", hash = "sha256:530f278849031b0eb12f46cca0e5db01cfe5177ab13bd6878c6e739319bae654"},
+    {file = "lxml-4.8.0-cp36-cp36m-macosx_10_14_x86_64.whl", hash = "sha256:078306d19a33920004addeb5f4630781aaeabb6a8d01398045fcde085091a169"},
+    {file = "lxml-4.8.0-cp36-cp36m-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_24_i686.whl", hash = "sha256:86545e351e879d0b72b620db6a3b96346921fa87b3d366d6c074e5a9a0b8dadb"},
+    {file = "lxml-4.8.0-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:24f5c5ae618395ed871b3d8ebfcbb36e3f1091fd847bf54c4de623f9107942f3"},
+    {file = "lxml-4.8.0-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_24_x86_64.whl", hash = "sha256:bbab6faf6568484707acc052f4dfc3802bdb0cafe079383fbaa23f1cdae9ecd4"},
+    {file = "lxml-4.8.0-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:7993232bd4044392c47779a3c7e8889fea6883be46281d45a81451acfd704d7e"},
+    {file = "lxml-4.8.0-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:6d6483b1229470e1d8835e52e0ff3c6973b9b97b24cd1c116dca90b57a2cc613"},
+    {file = "lxml-4.8.0-cp36-cp36m-musllinux_1_1_x86_64.whl", hash = "sha256:ad4332a532e2d5acb231a2e5d33f943750091ee435daffca3fec0a53224e7e33"},
+    {file = "lxml-4.8.0-cp36-cp36m-win32.whl", hash = "sha256:db3535733f59e5605a88a706824dfcb9bd06725e709ecb017e165fc1d6e7d429"},
+    {file = "lxml-4.8.0-cp36-cp36m-win_amd64.whl", hash = "sha256:5f148b0c6133fb928503cfcdfdba395010f997aa44bcf6474fcdd0c5398d9b63"},
+    {file = "lxml-4.8.0-cp37-cp37m-macosx_10_14_x86_64.whl", hash = "sha256:8a31f24e2a0b6317f33aafbb2f0895c0bce772980ae60c2c640d82caac49628a"},
+    {file = "lxml-4.8.0-cp37-cp37m-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_24_i686.whl", hash = "sha256:719544565c2937c21a6f76d520e6e52b726d132815adb3447ccffbe9f44203c4"},
+    {file = "lxml-4.8.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.manylinux_2_24_aarch64.whl", hash = "sha256:c0b88ed1ae66777a798dc54f627e32d3b81c8009967c63993c450ee4cbcbec15"},
+    {file = "lxml-4.8.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_24_x86_64.whl", hash = "sha256:fa9b7c450be85bfc6cd39f6df8c5b8cbd76b5d6fc1f69efec80203f9894b885f"},
+    {file = "lxml-4.8.0-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:e9f84ed9f4d50b74fbc77298ee5c870f67cb7e91dcdc1a6915cb1ff6a317476c"},
+    {file = "lxml-4.8.0-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:1d650812b52d98679ed6c6b3b55cbb8fe5a5460a0aef29aeb08dc0b44577df85"},
+    {file = "lxml-4.8.0-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:80bbaddf2baab7e6de4bc47405e34948e694a9efe0861c61cdc23aa774fcb141"},
+    {file = "lxml-4.8.0-cp37-cp37m-win32.whl", hash = "sha256:6f7b82934c08e28a2d537d870293236b1000d94d0b4583825ab9649aef7ddf63"},
+    {file = "lxml-4.8.0-cp37-cp37m-win_amd64.whl", hash = "sha256:e1fd7d2fe11f1cb63d3336d147c852f6d07de0d0020d704c6031b46a30b02ca8"},
+    {file = "lxml-4.8.0-cp38-cp38-macosx_10_14_x86_64.whl", hash = "sha256:5045ee1ccd45a89c4daec1160217d363fcd23811e26734688007c26f28c9e9e7"},
+    {file = "lxml-4.8.0-cp38-cp38-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_24_i686.whl", hash = "sha256:0c1978ff1fd81ed9dcbba4f91cf09faf1f8082c9d72eb122e92294716c605428"},
+    {file = "lxml-4.8.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.manylinux_2_24_aarch64.whl", hash = "sha256:52cbf2ff155b19dc4d4100f7442f6a697938bf4493f8d3b0c51d45568d5666b5"},
+    {file = "lxml-4.8.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_24_x86_64.whl", hash = "sha256:ce13d6291a5f47c1c8dbd375baa78551053bc6b5e5c0e9bb8e39c0a8359fd52f"},
+    {file = "lxml-4.8.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:e11527dc23d5ef44d76fef11213215c34f36af1608074561fcc561d983aeb870"},
+    {file = "lxml-4.8.0-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:60d2f60bd5a2a979df28ab309352cdcf8181bda0cca4529769a945f09aba06f9"},
+    {file = "lxml-4.8.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:62f93eac69ec0f4be98d1b96f4d6b964855b8255c345c17ff12c20b93f247b68"},
+    {file = "lxml-4.8.0-cp38-cp38-win32.whl", hash = "sha256:20b8a746a026017acf07da39fdb10aa80ad9877046c9182442bf80c84a1c4696"},
+    {file = "lxml-4.8.0-cp38-cp38-win_amd64.whl", hash = "sha256:891dc8f522d7059ff0024cd3ae79fd224752676447f9c678f2a5c14b84d9a939"},
+    {file = "lxml-4.8.0-cp39-cp39-macosx_10_15_x86_64.whl", hash = "sha256:b6fc2e2fb6f532cf48b5fed57567ef286addcef38c28874458a41b7837a57807"},
+    {file = "lxml-4.8.0-cp39-cp39-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_24_i686.whl", hash = "sha256:74eb65ec61e3c7c019d7169387d1b6ffcfea1b9ec5894d116a9a903636e4a0b1"},
+    {file = "lxml-4.8.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.manylinux_2_24_aarch64.whl", hash = "sha256:627e79894770783c129cc5e89b947e52aa26e8e0557c7e205368a809da4b7939"},
+    {file = "lxml-4.8.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_24_x86_64.whl", hash = "sha256:545bd39c9481f2e3f2727c78c169425efbfb3fbba6e7db4f46a80ebb249819ca"},
+    {file = "lxml-4.8.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:5a58d0b12f5053e270510bf12f753a76aaf3d74c453c00942ed7d2c804ca845c"},
+    {file = "lxml-4.8.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:ec4b4e75fc68da9dc0ed73dcdb431c25c57775383fec325d23a770a64e7ebc87"},
+    {file = "lxml-4.8.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:5804e04feb4e61babf3911c2a974a5b86f66ee227cc5006230b00ac6d285b3a9"},
+    {file = "lxml-4.8.0-cp39-cp39-win32.whl", hash = "sha256:aa0cf4922da7a3c905d000b35065df6184c0dc1d866dd3b86fd961905bbad2ea"},
+    {file = "lxml-4.8.0-cp39-cp39-win_amd64.whl", hash = "sha256:dd10383f1d6b7edf247d0960a3db274c07e96cf3a3fc7c41c8448f93eac3fb1c"},
+    {file = "lxml-4.8.0-pp37-pypy37_pp73-macosx_10_14_x86_64.whl", hash = "sha256:2403a6d6fb61c285969b71f4a3527873fe93fd0abe0832d858a17fe68c8fa507"},
+    {file = "lxml-4.8.0-pp37-pypy37_pp73-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_24_i686.whl", hash = "sha256:986b7a96228c9b4942ec420eff37556c5777bfba6758edcb95421e4a614b57f9"},
+    {file = "lxml-4.8.0-pp37-pypy37_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_24_x86_64.whl", hash = "sha256:6fe4ef4402df0250b75ba876c3795510d782def5c1e63890bde02d622570d39e"},
+    {file = "lxml-4.8.0-pp38-pypy38_pp73-macosx_10_14_x86_64.whl", hash = "sha256:f10ce66fcdeb3543df51d423ede7e238be98412232fca5daec3e54bcd16b8da0"},
+    {file = "lxml-4.8.0-pp38-pypy38_pp73-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_24_i686.whl", hash = "sha256:730766072fd5dcb219dd2b95c4c49752a54f00157f322bc6d71f7d2a31fecd79"},
+    {file = "lxml-4.8.0-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_24_x86_64.whl", hash = "sha256:8b99ec73073b37f9ebe8caf399001848fced9c08064effdbfc4da2b5a8d07b93"},
+    {file = "lxml-4.8.0.tar.gz", hash = "sha256:f63f62fc60e6228a4ca9abae28228f35e1bd3ce675013d1dfb828688d50c6e23"},
 ]
 markupsafe = [
    {file = "MarkupSafe-2.1.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:3028252424c72b2602a323f70fbf50aa80a5d3aa616ea6add4ba21ae9cc9da4c"},
@@ -2055,8 +2046,8 @@ matrix-common = [
    {file = "matrix_common-1.2.1.tar.gz", hash = "sha256:a99dcf02a6bd95b24a5a61b354888a2ac92bf2b4b839c727b8dd9da2cdfa3853"},
 ]
 matrix-synapse-ldap3 = [
-    {file = "matrix-synapse-ldap3-0.2.1.tar.gz", hash = "sha256:bfb4390f4a262ffb0d6f057ff3aeb1e46d4e52ff420a064d795fb4f555f00285"},
-    {file = "matrix_synapse_ldap3-0.2.1-py3-none-any.whl", hash = "sha256:1b3310a60f1d06466f35905a269b6df95747fd1305f2b7fe638f373963b2aa2c"},
+    {file = "matrix-synapse-ldap3-0.2.0.tar.gz", hash = "sha256:91a0715b43a41ec3033244174fca20846836da98fda711fb01687f7199eecd2e"},
+    {file = "matrix_synapse_ldap3-0.2.0-py3-none-any.whl", hash = "sha256:0128ca7c3058987adc2e8a88463bb46879915bfd3d373309632813b353e30f9f"},
 ]
 mccabe = [
    {file = "mccabe-0.6.1-py2.py3-none-any.whl", hash = "sha256:ab8a6258860da4b6677da4bd2fe5dc2c659cff31b3ee4f7f5d64e79735b80d42"},
@@ -54,7 +54,7 @@ skip_gitignore = true

 [tool.poetry]
 name = "matrix-synapse"
-version = "1.63.1"
+version = "1.62.0rc2"
 description = "Homeserver for the Matrix decentralised comms protocol"
 authors = ["Matrix.org Team and Contributors <packages@matrix.org>"]
 license = "Apache-2.0"
@@ -46,12 +46,10 @@ Run the complement test suite on Synapse.

  -f, --fast
        Skip rebuilding the docker images, and just use the most recent
-        'complement-synapse:latest' image.
-        Conflicts with --build-only.
+        'complement-synapse:latest' image

  --build-only
        Only build the Docker images. Don't actually run Complement.
-        Conflicts with -f/--fast.

 For help on arguments to 'go test', run 'go help testflag'.
 EOF
@@ -160,18 +158,6 @@ else
  test_tags="$test_tags,faster_joins"
 fi

-
-if [[ -n "$SYNAPSE_TEST_LOG_LEVEL" ]]; then
-  # Set the log level to what is desired
-  export PASS_SYNAPSE_LOG_LEVEL="$SYNAPSE_TEST_LOG_LEVEL"
-
-  # Allow logging sensitive things (currently SQL queries & parameters).
-  # (This won't have any effect if we're not logging at DEBUG level overall.)
-  # Since this is just a test suite, this is fine and won't reveal anyone's
-  # personal information
-  export PASS_SYNAPSE_LOG_SENSITIVE=1
-fi
-
 # Run the tests!
 echo "Images built; running complement"
 cd "$COMPLEMENT_DIR"
@@ -33,7 +33,7 @@ def main() -> None:
    parser.add_argument(
        "--report-stats",
        action="store",
-        help="Whether the generated config reports homeserver usage statistics",
+        help="Whether the generated config reports anonymized usage statistics",
        choices=["yes", "no"],
    )

@@ -418,15 +418,12 @@ class Porter:
            self.progress.update(table, table_size)  # Mark table as done
            return

-        # We sweep over rowids in two directions: one forwards (rowids 1, 2, 3, ...)
-        # and another backwards (rowids 0, -1, -2, ...).
        forward_select = (
            "SELECT rowid, * FROM %s WHERE rowid >= ? ORDER BY rowid LIMIT ?" % (table,)
        )

        backward_select = (
-            "SELECT rowid, * FROM %s WHERE rowid <= ? ORDER BY rowid DESC LIMIT ?"
-            % (table,)
+            "SELECT rowid, * FROM %s WHERE rowid <= ? ORDER BY rowid LIMIT ?" % (table,)
        )

        do_forward = [True]
@@ -624,25 +621,6 @@ class Porter:
                self.postgres_store.db_pool.updates.has_completed_background_updates()
            )

-    @staticmethod
-    def _is_sqlite_autovacuum_enabled(txn: LoggingTransaction) -> bool:
-        """
-        Returns true if auto_vacuum is enabled in SQLite.
-        https://www.sqlite.org/pragma.html#pragma_auto_vacuum
-
-        Vacuuming changes the rowids on rows in the database.
-        Auto-vacuuming is therefore dangerous when used in conjunction with this script.
-
-        Note that the auto_vacuum setting can't be changed without performing
-        a VACUUM after trying to change the pragma.
-        """
-        txn.execute("PRAGMA auto_vacuum")
-        row = txn.fetchone()
-        assert row is not None, "`PRAGMA auto_vacuum` did not give a row."
-        (autovacuum_setting,) = row
-        # 0 means off. 1 means full. 2 means incremental.
-        return autovacuum_setting != 0
-
    async def run(self) -> None:
        """Ports the SQLite database to a PostgreSQL database.

@@ -659,21 +637,6 @@ class Porter:
                allow_outdated_version=True,
            )

-            # For safety, ensure auto_vacuums are disabled.
-            if await self.sqlite_store.db_pool.runInteraction(
-                "is_sqlite_autovacuum_enabled", self._is_sqlite_autovacuum_enabled
-            ):
-                end_error = (
-                    "auto_vacuum is enabled in the SQLite database."
-                    " (This is not the default configuration.)\n"
-                    " This script relies on rowids being consistent and must not"
-                    " be used if the database could be vacuumed between re-runs.\n"
-                    " To disable auto_vacuum, you need to stop Synapse and run the following SQL:\n"
-                    " PRAGMA auto_vacuum=off;\n"
-                    " VACUUM;"
-                )
-                return
-
            # Check if all background updates are done, abort if not.
            updates_complete = (
                await self.sqlite_store.db_pool.updates.has_completed_background_updates()
@@ -297,14 +297,8 @@ class AuthError(SynapseError):
    other poorly-defined times.
    """

-    def __init__(
-        self,
-        code: int,
-        msg: str,
-        errcode: str = Codes.FORBIDDEN,
-        additional_fields: Optional[dict] = None,
-    ):
-        super().__init__(code, msg, errcode, additional_fields)
+    def __init__(self, code: int, msg: str, errcode: str = Codes.FORBIDDEN):
+        super().__init__(code, msg, errcode)


 class InvalidClientCredentialsError(SynapseError):
@@ -39,7 +39,6 @@ from synapse.replication.slave.storage.push_rule import SlavedPushRuleStore
 from synapse.replication.slave.storage.receipts import SlavedReceiptsStore
 from synapse.replication.slave.storage.registration import SlavedRegistrationStore
 from synapse.server import HomeServer
-from synapse.storage.database import DatabasePool, LoggingDatabaseConnection
 from synapse.storage.databases.main.room import RoomWorkerStore
 from synapse.types import StateMap
 from synapse.util import SYNAPSE_VERSION
@@ -61,17 +60,7 @@ class AdminCmdSlavedStore(
    BaseSlavedStore,
    RoomWorkerStore,
 ):
-    def __init__(
-        self,
-        database: DatabasePool,
-        db_conn: LoggingDatabaseConnection,
-        hs: "HomeServer",
-    ):
-        super().__init__(database, db_conn, hs)
-
-        # Annoyingly `filter_events_for_client` assumes that this exists. We
-        # should refactor it to take a `Clock` directly.
-        self.clock = hs.get_clock()
+    pass


 class AdminCmdServer(HomeServer):
@@ -319,9 +319,7 @@ class _ServiceQueuer:
        rooms_of_interesting_users.update(event.room_id for event in events)
        # EDUs
        rooms_of_interesting_users.update(
-            ephemeral["room_id"]
-            for ephemeral in ephemerals
-            if ephemeral.get("room_id") is not None
+            ephemeral["room_id"] for ephemeral in ephemerals
        )

        # Look up the AS users in those rooms
@@ -331,9 +329,8 @@ class _ServiceQueuer:
            )

        # Add recipients of to-device messages.
-        users.update(
-            device_message["to_user_id"] for device_message in to_device_messages
-        )
+        # device_message["user_id"] is the ID of the recipient.
+        users.update(device_message["user_id"] for device_message in to_device_messages)

        # Compute and return the counts / fallback key usage states
        otk_counts = await self._store.count_bulk_e2e_one_time_keys_for_as(users)
@@ -97,16 +97,16 @@ def format_config_error(e: ConfigError) -> Iterator[str]:
 # We split these messages out to allow packages to override with package
 # specific instructions.
 MISSING_REPORT_STATS_CONFIG_INSTRUCTIONS = """\
-Please opt in or out of reporting homeserver usage statistics, by setting
-the `report_stats` key in your config file to either True or False.
+Please opt in or out of reporting anonymized homeserver usage statistics, by
+setting the `report_stats` key in your config file to either True or False.
 """

 MISSING_REPORT_STATS_SPIEL = """\
 We would really appreciate it if you could help our project out by reporting
-homeserver usage statistics from your homeserver. Your homeserver's server name,
-along with very basic aggregate data (e.g. number of users) will be reported. But
-it helps us to track the growth of the Matrix community, and helps us to make Matrix
-a success, as well as to convince other networks that they should peer with us.
+anonymized usage statistics from your homeserver. Only very basic aggregate
+data (e.g. number of users) will be reported, but it helps us to track the
+growth of the Matrix community, and helps us to make Matrix a success, as well
+as to convince other networks that they should peer with us.

 Thank you.
 """
@@ -621,7 +621,7 @@ class RootConfig:
        generate_group.add_argument(
            "--report-stats",
            action="store",
-            help="Whether the generated config reports homeserver usage statistics.",
+            help="Whether the generated config reports anonymized usage statistics.",
            choices=["yes", "no"],
        )
        generate_group.add_argument(
@@ -21,6 +21,7 @@ from typing import (
    Awaitable,
    Callable,
    Collection,
+    Dict,
    List,
    Optional,
    Tuple,
@@ -31,11 +32,10 @@ from typing import (
 from typing_extensions import Literal

 import synapse
-from synapse.api.errors import Codes
 from synapse.rest.media.v1._base import FileInfo
 from synapse.rest.media.v1.media_storage import ReadableFileWrapper
 from synapse.spam_checker_api import RegistrationBehaviour
-from synapse.types import JsonDict, RoomAlias, UserProfile
+from synapse.types import RoomAlias, UserProfile
 from synapse.util.async_helpers import delay_cancellation, maybe_awaitable
 from synapse.util.metrics import Measure

@@ -50,12 +50,12 @@ CHECK_EVENT_FOR_SPAM_CALLBACK = Callable[
    Awaitable[
        Union[
            str,
-            Codes,
+            "synapse.api.errors.Codes",
            # Highly experimental, not officially part of the spamchecker API, may
            # disappear without warning depending on the results of ongoing
            # experiments.
            # Use this to return additional information as part of an error.
-            Tuple[Codes, JsonDict],
+            Tuple["synapse.api.errors.Codes", Dict],
            # Deprecated
            bool,
        ]
@@ -70,12 +70,7 @@ USER_MAY_JOIN_ROOM_CALLBACK = Callable[
    Awaitable[
        Union[
            Literal["NOT_SPAM"],
-            Codes,
-            # Highly experimental, not officially part of the spamchecker API, may
-            # disappear without warning depending on the results of ongoing
-            # experiments.
-            # Use this to return additional information as part of an error.
-            Tuple[Codes, JsonDict],
+            "synapse.api.errors.Codes",
            # Deprecated
            bool,
        ]
@@ -86,12 +81,7 @@ USER_MAY_INVITE_CALLBACK = Callable[
    Awaitable[
        Union[
            Literal["NOT_SPAM"],
-            Codes,
-            # Highly experimental, not officially part of the spamchecker API, may
-            # disappear without warning depending on the results of ongoing
-            # experiments.
-            # Use this to return additional information as part of an error.
-            Tuple[Codes, JsonDict],
+            "synapse.api.errors.Codes",
            # Deprecated
            bool,
        ]
@@ -102,12 +92,7 @@ USER_MAY_SEND_3PID_INVITE_CALLBACK = Callable[
    Awaitable[
        Union[
            Literal["NOT_SPAM"],
-            Codes,
-            # Highly experimental, not officially part of the spamchecker API, may
-            # disappear without warning depending on the results of ongoing
-            # experiments.
-            # Use this to return additional information as part of an error.
-            Tuple[Codes, JsonDict],
+            "synapse.api.errors.Codes",
            # Deprecated
            bool,
        ]
@@ -118,12 +103,7 @@ USER_MAY_CREATE_ROOM_CALLBACK = Callable[
    Awaitable[
        Union[
            Literal["NOT_SPAM"],
-            Codes,
-            # Highly experimental, not officially part of the spamchecker API, may
-            # disappear without warning depending on the results of ongoing
-            # experiments.
-            # Use this to return additional information as part of an error.
-            Tuple[Codes, JsonDict],
+            "synapse.api.errors.Codes",
            # Deprecated
            bool,
        ]
@@ -134,12 +114,7 @@ USER_MAY_CREATE_ROOM_ALIAS_CALLBACK = Callable[
    Awaitable[
        Union[
            Literal["NOT_SPAM"],
-            Codes,
-            # Highly experimental, not officially part of the spamchecker API, may
-            # disappear without warning depending on the results of ongoing
-            # experiments.
-            # Use this to return additional information as part of an error.
-            Tuple[Codes, JsonDict],
+            "synapse.api.errors.Codes",
            # Deprecated
            bool,
        ]
@@ -150,12 +125,7 @@ USER_MAY_PUBLISH_ROOM_CALLBACK = Callable[
    Awaitable[
        Union[
            Literal["NOT_SPAM"],
-            Codes,
-            # Highly experimental, not officially part of the spamchecker API, may
-            # disappear without warning depending on the results of ongoing
-            # experiments.
-            # Use this to return additional information as part of an error.
-            Tuple[Codes, JsonDict],
+            "synapse.api.errors.Codes",
            # Deprecated
            bool,
        ]
@@ -184,12 +154,7 @@ CHECK_MEDIA_FILE_FOR_SPAM_CALLBACK = Callable[
    Awaitable[
        Union[
            Literal["NOT_SPAM"],
-            Codes,
-            # Highly experimental, not officially part of the spamchecker API, may
-            # disappear without warning depending on the results of ongoing
-            # experiments.
-            # Use this to return additional information as part of an error.
-            Tuple[Codes, JsonDict],
+            "synapse.api.errors.Codes",
            # Deprecated
            bool,
        ]
@@ -380,7 +345,7 @@ class SpamChecker:

    async def check_event_for_spam(
        self, event: "synapse.events.EventBase"
-    ) -> Union[Tuple[Codes, JsonDict], str]:
+    ) -> Union[Tuple["synapse.api.errors.Codes", Dict], str]:
        """Checks if a given event is considered "spammy" by this server.

        If the server considers an event spammy, then it will be rejected if
@@ -411,16 +376,7 @@ class SpamChecker:
                elif res is True:
                    # This spam-checker rejects the event with deprecated
                    # return value `True`
-                    return synapse.api.errors.Codes.FORBIDDEN, {}
-                elif (
-                    isinstance(res, tuple)
-                    and len(res) == 2
-                    and isinstance(res[0], synapse.api.errors.Codes)
-                    and isinstance(res[1], dict)
-                ):
-                    return res
-                elif isinstance(res, synapse.api.errors.Codes):
-                    return res, {}
+                    return (synapse.api.errors.Codes.FORBIDDEN, {})
                elif not isinstance(res, str):
                    # mypy complains that we can't reach this code because of the
                    # return type in CHECK_EVENT_FOR_SPAM_CALLBACK, but we don't know
@@ -466,7 +422,7 @@ class SpamChecker:

    async def user_may_join_room(
        self, user_id: str, room_id: str, is_invited: bool
-    ) -> Union[Tuple[Codes, JsonDict], Literal["NOT_SPAM"]]:
+    ) -> Union["synapse.api.errors.Codes", Literal["NOT_SPAM"]]:
        """Checks if a given users is allowed to join a room.
        Not called when a user creates a room.

@@ -476,7 +432,7 @@ class SpamChecker:
            is_invited: Whether the user is invited into the room

        Returns:
-            NOT_SPAM if the operation is permitted, [Codes, Dict] otherwise.
+            NOT_SPAM if the operation is permitted, Codes otherwise.
        """
        for callback in self._user_may_join_room_callbacks:
            with Measure(
@@ -487,28 +443,21 @@ class SpamChecker:
                if res is True or res is self.NOT_SPAM:
                    continue
                elif res is False:
-                    return synapse.api.errors.Codes.FORBIDDEN, {}
+                    return synapse.api.errors.Codes.FORBIDDEN
                elif isinstance(res, synapse.api.errors.Codes):
-                    return res, {}
-                elif (
-                    isinstance(res, tuple)
-                    and len(res) == 2
-                    and isinstance(res[0], synapse.api.errors.Codes)
-                    and isinstance(res[1], dict)
-                ):
                    return res
                else:
                    logger.warning(
                        "Module returned invalid value, rejecting join as spam"
                    )
-                    return synapse.api.errors.Codes.FORBIDDEN, {}
+                    return synapse.api.errors.Codes.FORBIDDEN

        # No spam-checker has rejected the request, let it pass.
        return self.NOT_SPAM

    async def user_may_invite(
        self, inviter_userid: str, invitee_userid: str, room_id: str
-    ) -> Union[Tuple[Codes, dict], Literal["NOT_SPAM"]]:
+    ) -> Union["synapse.api.errors.Codes", Literal["NOT_SPAM"]]:
        """Checks if a given user may send an invite

        Args:
@@ -530,28 +479,21 @@ class SpamChecker:
                if res is True or res is self.NOT_SPAM:
                    continue
                elif res is False:
-                    return synapse.api.errors.Codes.FORBIDDEN, {}
+                    return synapse.api.errors.Codes.FORBIDDEN
                elif isinstance(res, synapse.api.errors.Codes):
-                    return res, {}
-                elif (
-                    isinstance(res, tuple)
-                    and len(res) == 2
-                    and isinstance(res[0], synapse.api.errors.Codes)
-                    and isinstance(res[1], dict)
-                ):
                    return res
                else:
                    logger.warning(
                        "Module returned invalid value, rejecting invite as spam"
                    )
-                    return synapse.api.errors.Codes.FORBIDDEN, {}
+                    return synapse.api.errors.Codes.FORBIDDEN

        # No spam-checker has rejected the request, let it pass.
        return self.NOT_SPAM

    async def user_may_send_3pid_invite(
        self, inviter_userid: str, medium: str, address: str, room_id: str
-    ) -> Union[Tuple[Codes, dict], Literal["NOT_SPAM"]]:
+    ) -> Union["synapse.api.errors.Codes", Literal["NOT_SPAM"]]:
        """Checks if a given user may invite a given threepid into the room

        Note that if the threepid is already associated with a Matrix user ID, Synapse
@@ -577,27 +519,20 @@ class SpamChecker:
                if res is True or res is self.NOT_SPAM:
                    continue
                elif res is False:
-                    return synapse.api.errors.Codes.FORBIDDEN, {}
+                    return synapse.api.errors.Codes.FORBIDDEN
                elif isinstance(res, synapse.api.errors.Codes):
-                    return res, {}
-                elif (
-                    isinstance(res, tuple)
-                    and len(res) == 2
-                    and isinstance(res[0], synapse.api.errors.Codes)
-                    and isinstance(res[1], dict)
-                ):
                    return res
                else:
                    logger.warning(
                        "Module returned invalid value, rejecting 3pid invite as spam"
                    )
-                    return synapse.api.errors.Codes.FORBIDDEN, {}
+                    return synapse.api.errors.Codes.FORBIDDEN

        return self.NOT_SPAM

    async def user_may_create_room(
        self, userid: str
-    ) -> Union[Tuple[Codes, dict], Literal["NOT_SPAM"]]:
+    ) -> Union["synapse.api.errors.Codes", Literal["NOT_SPAM"]]:
        """Checks if a given user may create a room

        Args:
@@ -611,27 +546,20 @@ class SpamChecker:
                if res is True or res is self.NOT_SPAM:
                    continue
                elif res is False:
-                    return synapse.api.errors.Codes.FORBIDDEN, {}
+                    return synapse.api.errors.Codes.FORBIDDEN
                elif isinstance(res, synapse.api.errors.Codes):
-                    return res, {}
-                elif (
-                    isinstance(res, tuple)
-                    and len(res) == 2
-                    and isinstance(res[0], synapse.api.errors.Codes)
-                    and isinstance(res[1], dict)
-                ):
                    return res
                else:
                    logger.warning(
                        "Module returned invalid value, rejecting room creation as spam"
                    )
-                    return synapse.api.errors.Codes.FORBIDDEN, {}
+                    return synapse.api.errors.Codes.FORBIDDEN

        return self.NOT_SPAM

    async def user_may_create_room_alias(
        self, userid: str, room_alias: RoomAlias
-    ) -> Union[Tuple[Codes, dict], Literal["NOT_SPAM"]]:
+    ) -> Union["synapse.api.errors.Codes", Literal["NOT_SPAM"]]:
        """Checks if a given user may create a room alias

        Args:
@@ -647,27 +575,20 @@ class SpamChecker:
                if res is True or res is self.NOT_SPAM:
                    continue
                elif res is False:
-                    return synapse.api.errors.Codes.FORBIDDEN, {}
+                    return synapse.api.errors.Codes.FORBIDDEN
                elif isinstance(res, synapse.api.errors.Codes):
-                    return res, {}
-                elif (
-                    isinstance(res, tuple)
-                    and len(res) == 2
-                    and isinstance(res[0], synapse.api.errors.Codes)
-                    and isinstance(res[1], dict)
-                ):
                    return res
                else:
                    logger.warning(
                        "Module returned invalid value, rejecting room create as spam"
                    )
-                    return synapse.api.errors.Codes.FORBIDDEN, {}
+                    return synapse.api.errors.Codes.FORBIDDEN

        return self.NOT_SPAM

    async def user_may_publish_room(
        self, userid: str, room_id: str
-    ) -> Union[Tuple[Codes, dict], Literal["NOT_SPAM"]]:
+    ) -> Union["synapse.api.errors.Codes", Literal["NOT_SPAM"]]:
        """Checks if a given user may publish a room to the directory

        Args:
@@ -682,21 +603,14 @@ class SpamChecker:
                if res is True or res is self.NOT_SPAM:
                    continue
                elif res is False:
-                    return synapse.api.errors.Codes.FORBIDDEN, {}
+                    return synapse.api.errors.Codes.FORBIDDEN
                elif isinstance(res, synapse.api.errors.Codes):
-                    return res, {}
-                elif (
-                    isinstance(res, tuple)
-                    and len(res) == 2
-                    and isinstance(res[0], synapse.api.errors.Codes)
-                    and isinstance(res[1], dict)
-                ):
                    return res
                else:
                    logger.warning(
                        "Module returned invalid value, rejecting room publication as spam"
                    )
-                    return synapse.api.errors.Codes.FORBIDDEN, {}
+                    return synapse.api.errors.Codes.FORBIDDEN

        return self.NOT_SPAM

@@ -764,7 +678,7 @@ class SpamChecker:

    async def check_media_file_for_spam(
        self, file_wrapper: ReadableFileWrapper, file_info: FileInfo
-    ) -> Union[Tuple[Codes, dict], Literal["NOT_SPAM"]]:
+    ) -> Union["synapse.api.errors.Codes", Literal["NOT_SPAM"]]:
        """Checks if a piece of newly uploaded media should be blocked.

        This will be called for local uploads, downloads of remote media, each
@@ -801,20 +715,13 @@ class SpamChecker:
                if res is False or res is self.NOT_SPAM:
                    continue
                elif res is True:
-                    return synapse.api.errors.Codes.FORBIDDEN, {}
+                    return synapse.api.errors.Codes.FORBIDDEN
                elif isinstance(res, synapse.api.errors.Codes):
-                    return res, {}
-                elif (
-                    isinstance(res, tuple)
-                    and len(res) == 2
-                    and isinstance(res[0], synapse.api.errors.Codes)
-                    and isinstance(res[1], dict)
-                ):
                    return res
                else:
                    logger.warning(
                        "Module returned invalid value, rejecting media file as spam"
                    )
-                    return synapse.api.errors.Codes.FORBIDDEN, {}
+                    return synapse.api.errors.Codes.FORBIDDEN

        return self.NOT_SPAM
@@ -464,7 +464,14 @@ class ThirdPartyEventRules:
        Returns:
            A dict mapping (event type, state key) to state event.
        """
-        return await self._storage_controllers.state.get_current_state(room_id)
+        state_ids = await self._storage_controllers.state.get_current_state_ids(room_id)
+        room_state_events = await self.store.get_events(state_ids.values())
+
+        state_events = {}
+        for key, event_id in state_ids.items():
+            state_events[key] = room_state_events[event_id]
+
+        return state_events

    async def on_profile_update(
        self, user_id: str, new_profile: ProfileInfo, by_admin: bool, deactivation: bool
@@ -67,7 +67,6 @@ from synapse.replication.http.federation import (
    ReplicationFederationSendEduRestServlet,
    ReplicationGetQueryRestServlet,
 )
-from synapse.storage.databases.main.events import PartialStateConflictError
 from synapse.storage.databases.main.lock import Lock
 from synapse.types import JsonDict, StateMap, get_domain_from_id
 from synapse.util import json_decoder, unwrapFirstError
@@ -883,20 +882,9 @@ class FederationServer(FederationBase):
            logger.warning("%s", errmsg)
            raise SynapseError(403, errmsg, Codes.FORBIDDEN)

-        try:
-            return await self._federation_event_handler.on_send_membership_event(
-                origin, event
-            )
-        except PartialStateConflictError:
-            # The room was un-partial stated while we were persisting the event.
-            # Try once more, with full state this time.
-            logger.info(
-                "Room %s was un-partial stated during `on_send_membership_event`, trying again.",
-                room_id,
-            )
-            return await self._federation_event_handler.on_send_membership_event(
-                origin, event
-            )
+        return await self._federation_event_handler.on_send_membership_event(
+            origin, event
+        )

    async def on_event_auth(
        self, origin: str, room_id: str, event_id: str
@@ -149,8 +149,7 @@ class DirectoryHandler:
                raise AuthError(
                    403,
                    "This user is not permitted to create this alias",
-                    errcode=spam_check[0],
-                    additional_fields=spam_check[1],
+                    spam_check,
                )

            if not self.config.roomdirectory.is_alias_creation_allowed(
@@ -442,8 +441,7 @@ class DirectoryHandler:
            raise AuthError(
                403,
                "This user is not permitted to publish rooms to the room list",
-                errcode=spam_check[0],
-                additional_fields=spam_check[1],
+                spam_check,
            )

        if requester.is_guest:
@@ -45,7 +45,6 @@ from synapse.api.errors import (
    FederationDeniedError,
    FederationError,
    HttpResponseException,
-    LimitExceededError,
    NotFoundError,
    RequestSendFailed,
    SynapseError,
@@ -65,7 +64,6 @@ from synapse.replication.http.federation import (
    ReplicationCleanRoomRestServlet,
    ReplicationStoreRoomOnOutlierMembershipRestServlet,
 )
-from synapse.storage.databases.main.events import PartialStateConflictError
 from synapse.storage.databases.main.events_worker import EventRedactBehaviour
 from synapse.storage.state import StateFilter
 from synapse.types import JsonDict, StateMap, get_domain_from_id
@@ -551,29 +549,15 @@ class FederationHandler:
                #   https://github.com/matrix-org/synapse/issues/12998
                await self.store.store_partial_state_room(room_id, ret.servers_in_room)

-            try:
-                max_stream_id = (
-                    await self._federation_event_handler.process_remote_join(
-                        origin,
-                        room_id,
-                        auth_chain,
-                        state,
-                        event,
-                        room_version_obj,
-                        partial_state=ret.partial_state,
-                    )
-                )
-            except PartialStateConflictError as e:
-                # The homeserver was already in the room and it is no longer partial
-                # stated. We ought to be doing a local join instead. Turn the error into
-                # a 429, as a hint to the client to try again.
-                # TODO(faster_joins): `_should_perform_remote_join` suggests that we may
-                #   do a remote join for restricted rooms even if we have full state.
-                logger.error(
-                    "Room %s was un-partial stated while processing remote join.",
-                    room_id,
-                )
-                raise LimitExceededError(msg=e.msg, errcode=e.errcode, retry_after_ms=0)
+            max_stream_id = await self._federation_event_handler.process_remote_join(
+                origin,
+                room_id,
+                auth_chain,
+                state,
+                event,
+                room_version_obj,
+                partial_state=ret.partial_state,
+            )

            if ret.partial_state:
                # Kick off the process of asynchronously fetching the state for this
@@ -844,8 +828,7 @@ class FederationHandler:
            raise SynapseError(
                403,
                "This user is not permitted to send invites to this server/user",
-                errcode=spam_check[0],
-                additional_fields=spam_check[1],
+                spam_check,
            )

        membership = event.content.get("membership")
@@ -1560,9 +1543,14 @@ class FederationHandler:
                # all the events are updated, so we can update current state and
                # clear the lazy-loading flag.
                logger.info("Updating current state for %s", room_id)
-                # TODO(faster_joins): notify workers in notify_room_un_partial_stated
+                # TODO(faster_joins): support workers
                #   https://github.com/matrix-org/synapse/issues/12994
-                await self.state_handler.update_current_state(room_id)
+                assert (
+                    self._storage_controllers.persistence is not None
+                ), "worker-mode deployments not currently supported here"
+                await self._storage_controllers.persistence.update_current_state(
+                    room_id
+                )

                logger.info("Clearing partial-state flag for %s", room_id)
                success = await self.store.clear_partial_state_room(room_id)
@@ -1579,6 +1567,11 @@ class FederationHandler:

                # we raced against more events arriving with partial state. Go round
                # the loop again. We've already logged a warning, so no need for more.
+                # TODO(faster_joins): there is still a race here, whereby incoming events which raced
+                #   with us will fail to be persisted after the call to `clear_partial_state_room` due to
+                #   having partial state.
+                #   https://github.com/matrix-org/synapse/issues/12988
+                #
                continue

            events = await self.store.get_events_as_list(
@@ -64,7 +64,6 @@ from synapse.replication.http.federation import (
    ReplicationFederationSendEventsRestServlet,
 )
 from synapse.state import StateResolutionStore
-from synapse.storage.databases.main.events import PartialStateConflictError
 from synapse.storage.databases.main.events_worker import EventRedactBehaviour
 from synapse.storage.state import StateFilter
 from synapse.types import (
@@ -276,16 +275,7 @@ class FederationEventHandler:
                    affected=pdu.event_id,
                )

-        try:
-            await self._process_received_pdu(origin, pdu, state_ids=None)
-        except PartialStateConflictError:
-            # The room was un-partial stated while we were processing the PDU.
-            # Try once more, with full state this time.
-            logger.info(
-                "Room %s was un-partial stated while processing the PDU, trying again.",
-                room_id,
-            )
-            await self._process_received_pdu(origin, pdu, state_ids=None)
+        await self._process_received_pdu(origin, pdu, state_ids=None)

    async def on_send_membership_event(
        self, origin: str, event: EventBase
@@ -316,9 +306,6 @@ class FederationEventHandler:

        Raises:
            SynapseError if the event is not accepted into the room
-            PartialStateConflictError if the room was un-partial stated in between
-                computing the state at the event and persisting it. The caller should
-                retry exactly once in this case.
        """
        logger.debug(
            "on_send_membership_event: Got event: %s, signatures: %s",
@@ -436,8 +423,6 @@ class FederationEventHandler:

        Raises:
            SynapseError if the response is in some way invalid.
-            PartialStateConflictError if the homeserver is already in the room and it
-                has been un-partial stated.
        """
        create_event = None
        for e in state:
@@ -1099,14 +1084,10 @@ class FederationEventHandler:

            state_ids: Normally None, but if we are handling a gap in the graph
                (ie, we are missing one or more prev_events), the resolved state at the
-                event. Must not be partial state.
+                event

            backfilled: True if this is part of a historical batch of events (inhibits
                notification to clients, and validation of device keys.)
-
-        PartialStateConflictError: if the room was un-partial stated in between
-            computing the state at the event and persisting it. The caller should retry
-            exactly once in this case. Will never be raised if `state_ids` is provided.
        """
        logger.debug("Processing event: %s", event)
        assert not event.internal_metadata.outlier
@@ -1952,9 +1933,6 @@ class FederationEventHandler:
            event: The event itself.
            context: The event context.
            backfilled: True if the event was backfilled.
-
-        PartialStateConflictError: if attempting to persist a partial state event in
-            a room that has been un-partial stated.
        """
        # this method should not be called on outliers (those code paths call
        # persist_events_and_notify directly.)
@@ -2007,10 +1985,6 @@ class FederationEventHandler:

        Returns:
            The stream ID after which all events have been persisted.
-
-        Raises:
-            PartialStateConflictError: if attempting to persist a partial state event in
-                a room that has been un-partial stated.
        """
        if not event_and_contexts:
            return self._store.get_room_max_stream_ordering()
@@ -2019,19 +1993,14 @@ class FederationEventHandler:
        if instance != self._instance_name:
            # Limit the number of events sent over replication. We choose 200
            # here as that is what we default to in `max_request_body_size(..)`
-            try:
-                for batch in batch_iter(event_and_contexts, 200):
-                    result = await self._send_events(
-                        instance_name=instance,
-                        store=self._store,
-                        room_id=room_id,
-                        event_and_contexts=batch,
-                        backfilled=backfilled,
-                    )
-            except SynapseError as e:
-                if e.code == HTTPStatus.CONFLICT:
-                    raise PartialStateConflictError()
-                raise
+            for batch in batch_iter(event_and_contexts, 200):
+                result = await self._send_events(
+                    instance_name=instance,
+                    store=self._store,
+                    room_id=room_id,
+                    event_and_contexts=batch,
+                    backfilled=backfilled,
+                )
            return result["max_stream_id"]
        else:
            assert self._storage_controllers.persistence
@@ -37,7 +37,6 @@ from synapse.api.errors import (
    AuthError,
    Codes,
    ConsentNotGivenError,
-    LimitExceededError,
    NotFoundError,
    ShadowBanError,
    SynapseError,
@@ -54,7 +53,6 @@ from synapse.handlers.directory import DirectoryHandler
 from synapse.logging.context import make_deferred_yieldable, run_in_background
 from synapse.metrics.background_process_metrics import run_as_background_process
 from synapse.replication.http.send_event import ReplicationSendEventRestServlet
-from synapse.storage.databases.main.events import PartialStateConflictError
 from synapse.storage.databases.main.events_worker import EventRedactBehaviour
 from synapse.storage.state import StateFilter
 from synapse.types import (
@@ -1252,8 +1250,6 @@ class EventCreationHandler:

        Raises:
            ShadowBanError if the requester has been shadow-banned.
-            SynapseError(503) if attempting to persist a partial state event in
-                a room that has been un-partial stated.
        """
        extra_users = extra_users or []

@@ -1304,35 +1300,24 @@ class EventCreationHandler:

        # We now persist the event (and update the cache in parallel, since we
        # don't want to block on it).
-        try:
-            result, _ = await make_deferred_yieldable(
-                gather_results(
-                    (
-                        run_in_background(
-                            self._persist_event,
-                            requester=requester,
-                            event=event,
-                            context=context,
-                            ratelimit=ratelimit,
-                            extra_users=extra_users,
-                        ),
-                        run_in_background(
-                            self.cache_joined_hosts_for_event, event, context
-                        ).addErrback(
-                            log_failure, "cache_joined_hosts_for_event failed"
-                        ),
+        result, _ = await make_deferred_yieldable(
+            gather_results(
+                (
+                    run_in_background(
+                        self._persist_event,
+                        requester=requester,
+                        event=event,
+                        context=context,
+                        ratelimit=ratelimit,
+                        extra_users=extra_users,
                    ),
-                    consumeErrors=True,
-                )
-            ).addErrback(unwrapFirstError)
-        except PartialStateConflictError as e:
-            # The event context needs to be recomputed.
-            # Turn the error into a 429, as a hint to the client to try again.
-            logger.info(
-                "Room %s was un-partial stated while persisting client event.",
-                event.room_id,
+                    run_in_background(
+                        self.cache_joined_hosts_for_event, event, context
+                    ).addErrback(log_failure, "cache_joined_hosts_for_event failed"),
+                ),
+                consumeErrors=True,
            )
-            raise LimitExceededError(msg=e.msg, errcode=e.errcode, retry_after_ms=0)
+        ).addErrback(unwrapFirstError)

        return result

@@ -1347,9 +1332,6 @@ class EventCreationHandler:
        """Actually persists the event. Should only be called by
        `handle_new_client_event`, and see its docstring for documentation of
        the arguments.
-
-        PartialStateConflictError: if attempting to persist a partial state event in
-            a room that has been un-partial stated.
        """

        # Skip push notification actions for historical messages
@@ -1366,21 +1348,16 @@ class EventCreationHandler:
            # If we're a worker we need to hit out to the master.
            writer_instance = self._events_shard_config.get_instance(event.room_id)
            if writer_instance != self._instance_name:
-                try:
-                    result = await self.send_event(
-                        instance_name=writer_instance,
-                        event_id=event.event_id,
-                        store=self.store,
-                        requester=requester,
-                        event=event,
-                        context=context,
-                        ratelimit=ratelimit,
-                        extra_users=extra_users,
-                    )
-                except SynapseError as e:
-                    if e.code == HTTPStatus.CONFLICT:
-                        raise PartialStateConflictError()
-                    raise
+                result = await self.send_event(
+                    instance_name=writer_instance,
+                    event_id=event.event_id,
+                    store=self.store,
+                    requester=requester,
+                    event=event,
+                    context=context,
+                    ratelimit=ratelimit,
+                    extra_users=extra_users,
+                )
                stream_id = result["stream_id"]
                event_id = result["event_id"]
                if event_id != event.event_id:
@@ -1508,10 +1485,6 @@ class EventCreationHandler:
            The persisted event. This may be different than the given event if
            it was de-duplicated (e.g. because we had already persisted an
            event with the same transaction ID.)
-
-        Raises:
-            PartialStateConflictError: if attempting to persist a partial state event in
-                a room that has been un-partial stated.
        """
        extra_users = extra_users or []

@@ -67,14 +67,19 @@ class ProfileHandler:
        target_user = UserID.from_string(user_id)

        if self.hs.is_mine(target_user):
-            profileinfo = await self.store.get_profileinfo(target_user.localpart)
-            if profileinfo.display_name is None:
-                raise SynapseError(404, "Profile was not found", Codes.NOT_FOUND)
+            try:
+                displayname = await self.store.get_profile_displayname(
+                    target_user.localpart
+                )
+                avatar_url = await self.store.get_profile_avatar_url(
+                    target_user.localpart
+                )
+            except StoreError as e:
+                if e.code == 404:
+                    raise SynapseError(404, "Profile was not found", Codes.NOT_FOUND)
+                raise

-            return {
-                "displayname": profileinfo.display_name,
-                "avatar_url": profileinfo.avatar_url,
-            }
+            return {"displayname": displayname, "avatar_url": avatar_url}
        else:
            try:
                result = await self.federation.make_query(
@@ -440,12 +440,7 @@ class RoomCreationHandler:

        spam_check = await self.spam_checker.user_may_create_room(user_id)
        if spam_check != NOT_SPAM:
-            raise SynapseError(
-                403,
-                "You are not permitted to create rooms",
-                errcode=spam_check[0],
-                additional_fields=spam_check[1],
-            )
+            raise SynapseError(403, "You are not permitted to create rooms", spam_check)

        creation_content: JsonDict = {
            "room_version": new_room_version.identifier,
@@ -736,10 +731,7 @@ class RoomCreationHandler:
            spam_check = await self.spam_checker.user_may_create_room(user_id)
            if spam_check != NOT_SPAM:
                raise SynapseError(
-                    403,
-                    "You are not permitted to create rooms",
-                    errcode=spam_check[0],
-                    additional_fields=spam_check[1],
+                    403, "You are not permitted to create rooms", spam_check
                )

        if ratelimit:
@@ -1019,8 +1011,6 @@ class RoomCreationHandler:

        event_keys = {"room_id": room_id, "sender": creator_id, "state_key": ""}

-        last_sent_event_id: Optional[str] = None
-
        def create(etype: str, content: JsonDict, **kwargs: Any) -> JsonDict:
            e = {"type": etype, "content": content}

@@ -1030,27 +1020,19 @@ class RoomCreationHandler:
            return e

        async def send(etype: str, content: JsonDict, **kwargs: Any) -> int:
-            nonlocal last_sent_event_id
-
            event = create(etype, content, **kwargs)
            logger.debug("Sending %s in new room", etype)
            # Allow these events to be sent even if the user is shadow-banned to
            # allow the room creation to complete.
            (
-                sent_event,
+                _,
                last_stream_id,
            ) = await self.event_creation_handler.create_and_send_nonmember_event(
                creator,
                event,
                ratelimit=False,
                ignore_shadow_ban=True,
-                # Note: we don't pass state_event_ids here because this triggers
-                # an additional query per event to look them up from the events table.
-                prev_event_ids=[last_sent_event_id] if last_sent_event_id else [],
            )
-
-            last_sent_event_id = sent_event.event_id
-
            return last_stream_id

        try:
@@ -1064,9 +1046,7 @@ class RoomCreationHandler:
        await send(etype=EventTypes.Create, content=creation_content)

        logger.debug("Sending %s in new room", EventTypes.Member)
-        # Room create event must exist at this point
-        assert last_sent_event_id is not None
-        member_event_id, _ = await self.room_member_handler.update_membership(
+        await self.room_member_handler.update_membership(
            creator,
            creator.user,
            room_id,
@@ -1074,9 +1054,7 @@ class RoomCreationHandler:
            ratelimit=ratelimit,
            content=creator_join_profile,
            new_room=True,
-            prev_event_ids=[last_sent_event_id],
        )
-        last_sent_event_id = member_event_id

        # We treat the power levels override specially as this needs to be one
        # of the first events that get sent into a room.
@@ -1397,7 +1375,6 @@ class TimestampLookupHandler:
        # the timestamp given and the event we were able to find locally
        is_event_next_to_backward_gap = False
        is_event_next_to_forward_gap = False
-        local_event = None
        if local_event_id:
            local_event = await self.store.get_event(
                local_event_id, allow_none=False, allow_rejected=False
@@ -1484,10 +1461,7 @@ class TimestampLookupHandler:
                        ex.args,
                    )

-        # To appease mypy, we have to add both of these conditions to check for
-        # `None`. We only expect `local_event` to be `None` when
-        # `local_event_id` is `None` but mypy isn't as smart and assuming as us.
-        if not local_event_id or not local_event:
+        if not local_event_id:
            raise SynapseError(
                404,
                "Unable to find event from %s in direction %s" % (timestamp, direction),
@@ -685,7 +685,7 @@ class RoomMemberHandler(metaclass=abc.ABCMeta):
            if target_id == self._server_notices_mxid:
                raise SynapseError(HTTPStatus.FORBIDDEN, "Cannot invite this user")

-            block_invite_result = None
+            block_invite_code = None

            if (
                self._server_notices_mxid is not None
@@ -703,21 +703,18 @@ class RoomMemberHandler(metaclass=abc.ABCMeta):
                        "Blocking invite: user is not admin and non-admin "
                        "invites disabled"
                    )
-                    block_invite_result = (Codes.FORBIDDEN, {})
+                    block_invite_code = Codes.FORBIDDEN

                spam_check = await self.spam_checker.user_may_invite(
                    requester.user.to_string(), target_id, room_id
                )
                if spam_check != NOT_SPAM:
                    logger.info("Blocking invite due to spam checker")
-                    block_invite_result = spam_check
+                    block_invite_code = spam_check

-            if block_invite_result is not None:
+            if block_invite_code is not None:
                raise SynapseError(
-                    403,
-                    "Invites have been disabled on this server",
-                    errcode=block_invite_result[0],
-                    additional_fields=block_invite_result[1],
+                    403, "Invites have been disabled on this server", block_invite_code
                )

        # An empty prev_events list is allowed as long as the auth_event_ids are present
@@ -831,12 +828,7 @@ class RoomMemberHandler(metaclass=abc.ABCMeta):
                    target.to_string(), room_id, is_invited=inviter is not None
                )
                if spam_check != NOT_SPAM:
-                    raise SynapseError(
-                        403,
-                        "Not allowed to join this room",
-                        errcode=spam_check[0],
-                        additional_fields=spam_check[1],
-                    )
+                    raise SynapseError(403, "Not allowed to join this room", spam_check)

            # Check if a remote join should be performed.
            remote_join, remote_room_hosts = await self._should_perform_remote_join(
@@ -854,17 +846,10 @@ class RoomMemberHandler(metaclass=abc.ABCMeta):

                content["membership"] = Membership.JOIN

-                try:
-                    profile = self.profile_handler
-                    if not content_specified:
-                        content["displayname"] = await profile.get_displayname(target)
-                        content["avatar_url"] = await profile.get_avatar_url(target)
-                except Exception as e:
-                    logger.info(
-                        "Failed to get profile information while processing remote join for %r: %s",
-                        target,
-                        e,
-                    )
+                profile = self.profile_handler
+                if not content_specified:
+                    content["displayname"] = await profile.get_displayname(target)
+                    content["avatar_url"] = await profile.get_avatar_url(target)

                if requester.is_guest:
                    content["kind"] = "guest"
@@ -941,18 +926,11 @@ class RoomMemberHandler(metaclass=abc.ABCMeta):

                content["membership"] = Membership.KNOCK

-                try:
-                    profile = self.profile_handler
-                    if "displayname" not in content:
-                        content["displayname"] = await profile.get_displayname(target)
-                    if "avatar_url" not in content:
-                        content["avatar_url"] = await profile.get_avatar_url(target)
-                except Exception as e:
-                    logger.info(
-                        "Failed to get profile information while processing remote knock for %r: %s",
-                        target,
-                        e,
-                    )
+                profile = self.profile_handler
+                if "displayname" not in content:
+                    content["displayname"] = await profile.get_displayname(target)
+                if "avatar_url" not in content:
+                    content["avatar_url"] = await profile.get_avatar_url(target)

                return await self.remote_knock(
                    remote_room_hosts, room_id, target, content
@@ -1395,12 +1373,7 @@ class RoomMemberHandler(metaclass=abc.ABCMeta):
                room_id=room_id,
            )
            if spam_check != NOT_SPAM:
-                raise SynapseError(
-                    403,
-                    "Cannot send threepid invite",
-                    errcode=spam_check[0],
-                    additional_fields=spam_check[1],
-                )
+                raise SynapseError(403, "Cannot send threepid invite", spam_check)

            stream_id = await self._make_and_store_3pid_invite(
                requester,
@@ -35,7 +35,6 @@ from typing_extensions import ParamSpec
 from twisted.internet import defer
 from twisted.web.resource import Resource

-from synapse.api import errors
 from synapse.api.errors import SynapseError
 from synapse.events import EventBase
 from synapse.events.presence_router import (
@@ -17,6 +17,7 @@ import itertools
 import logging
 from typing import TYPE_CHECKING, Any, Dict, Iterable, List, Optional, Set, Tuple, Union

+import attr
 from prometheus_client import Counter

 from synapse.api.constants import EventTypes, Membership, RelationTypes
@@ -25,11 +26,13 @@ from synapse.events import EventBase, relation_from_event
 from synapse.events.snapshot import EventContext
 from synapse.state import POWER_KEY
 from synapse.storage.databases.main.roommember import EventIdMembership
-from synapse.storage.state import StateFilter
-from synapse.util.caches import register_cache
+from synapse.util.async_helpers import Linearizer
+from synapse.util.caches import CacheMetric, register_cache
+from synapse.util.caches.descriptors import lru_cache
+from synapse.util.caches.lrucache import LruCache
 from synapse.util.metrics import measure_func
-from synapse.visibility import filter_event_for_clients_with_state

+from ..storage.state import StateFilter
 from .push_rule_evaluator import PushRuleEvaluatorForEvent

 if TYPE_CHECKING:
@@ -45,6 +48,15 @@ push_rules_state_size_counter = Counter(
    "synapse_push_bulk_push_rule_evaluator_push_rules_state_size_counter", ""
 )

+# Measures whether we use the fast path of using state deltas, or if we have to
+# recalculate from scratch
+push_rules_delta_state_cache_metric = register_cache(
+    "cache",
+    "push_rules_delta_state_cache_metric",
+    cache=[],  # Meaningless size, as this isn't a cache that stores values
+    resizable=False,
+)
+

 STATE_EVENT_TYPES_TO_MARK_UNREAD = {
    EventTypes.Topic,
@@ -99,6 +111,10 @@ class BulkPushRuleEvaluator:
        self.clock = hs.get_clock()
        self._event_auth_handler = hs.get_event_auth_handler()

+        # Used by `RulesForRoom` to ensure only one thing mutates the cache at a
+        # time. Keyed off room_id.
+        self._rules_linearizer = Linearizer(name="rules_for_room")
+
        self.room_push_rule_cache_metrics = register_cache(
            "cache",
            "room_push_rule_cache",
@@ -110,55 +126,48 @@ class BulkPushRuleEvaluator:
        self._relations_match_enabled = self.hs.config.experimental.msc3772_enabled

    async def _get_rules_for_event(
-        self,
-        event: EventBase,
+        self, event: EventBase, context: EventContext
    ) -> Dict[str, List[Dict[str, Any]]]:
-        """Get the push rules for all users who may need to be notified about
-        the event.
-
-        Note: this does not check if the user is allowed to see the event.
+        """This gets the rules for all users in the room at the time of the event,
+        as well as the push rules for the invitee if the event is an invite.

        Returns:
-            Mapping of user ID to their push rules.
+            dict of user_id -> push_rules
        """
-        # We get the users who may need to be notified by first fetching the
-        # local users currently in the room, finding those that have push rules,
-        # and *then* checking which users are actually allowed to see the event.
-        #
-        # The alternative is to first fetch all users that were joined at the
-        # event, but that requires fetching the full state at the event, which
-        # may be expensive for large rooms with few local users.
+        room_id = event.room_id

-        local_users = await self.store.get_local_users_in_room(event.room_id)
+        rules_for_room_data = self._get_rules_for_room(room_id)
+        rules_for_room = RulesForRoom(
+            hs=self.hs,
+            room_id=room_id,
+            rules_for_room_cache=self._get_rules_for_room.cache,
+            room_push_rule_cache_metrics=self.room_push_rule_cache_metrics,
+            linearizer=self._rules_linearizer,
+            cached_data=rules_for_room_data,
+        )

-        # Filter out appservice users.
-        local_users = [
-            u
-            for u in local_users
-            if not self.store.get_if_app_services_interested_in_user(u)
-        ]
+        rules_by_user = await rules_for_room.get_rules(event, context)

        # if this event is an invite event, we may need to run rules for the user
        # who's been invited, otherwise they won't get told they've been invited
-        if event.type == EventTypes.Member and event.membership == Membership.INVITE:
+        if event.type == "m.room.member" and event.content["membership"] == "invite":
            invited = event.state_key
-            if invited and self.hs.is_mine_id(invited) and invited not in local_users:
-                local_users = list(local_users)
-                local_users.append(invited)
-
-        rules_by_user = await self.store.bulk_get_push_rules(local_users)
-
-        logger.debug("Users in room: %s", local_users)
-
-        if logger.isEnabledFor(logging.DEBUG):
-            logger.debug(
-                "Returning push rules for %r %r",
-                event.room_id,
-                list(rules_by_user.keys()),
-            )
+            if invited and self.hs.is_mine_id(invited):
+                rules_by_user = dict(rules_by_user)
+                rules_by_user[invited] = await self.store.get_push_rules_for_user(
+                    invited
+                )

        return rules_by_user

+    @lru_cache()
+    def _get_rules_for_room(self, room_id: str) -> "RulesForRoomData":
+        """Get the current RulesForRoomData object for the given room id"""
+        # It's important that the RulesForRoomData object gets added to self._get_rules_for_room.cache
+        # before any lookup methods get called on it as otherwise there may be
+        # a race if invalidate_all gets called (which assumes its in the cache)
+        return RulesForRoomData()
+
    async def _get_power_levels_and_sender_level(
        self, event: EventBase, context: EventContext
    ) -> Tuple[dict, int]:
@@ -253,12 +262,10 @@ class BulkPushRuleEvaluator:

        count_as_unread = _should_count_as_unread(event, context)

-        rules_by_user = await self._get_rules_for_event(event)
+        rules_by_user = await self._get_rules_for_event(event, context)
        actions_by_user: Dict[str, List[Union[dict, str]]] = {}

-        room_member_count = await self.store.get_number_joined_users_in_room(
-            event.room_id
-        )
+        room_members = await self.store.get_joined_users_from_context(event, context)

        (
            power_levels,
@@ -271,36 +278,30 @@ class BulkPushRuleEvaluator:

        evaluator = PushRuleEvaluatorForEvent(
            event,
-            room_member_count,
+            len(room_members),
            sender_power_level,
            power_levels,
            relations,
            self._relations_match_enabled,
        )

-        users = rules_by_user.keys()
-        profiles = await self.store.get_subset_users_in_room_with_profiles(
-            event.room_id, users
-        )
-
-        # This is a check for the case where user joins a room without being
-        # allowed to see history, and then the server receives a delayed event
-        # from before the user joined, which they should not be pushed for
-        uids_with_visibility = await filter_event_for_clients_with_state(
-            self.store, users, event, context
-        )
+        # If the event is not a state event check if any users ignore the sender.
+        if not event.is_state():
+            ignorers = await self.store.ignored_by(event.sender)
+        else:
+            ignorers = frozenset()

        for uid, rules in rules_by_user.items():
            if event.sender == uid:
                continue

-            if uid not in uids_with_visibility:
+            if uid in ignorers:
                continue

            display_name = None
-            profile = profiles.get(uid)
-            if profile:
-                display_name = profile.display_name
+            profile_info = room_members.get(uid)
+            if profile_info:
+                display_name = profile_info.display_name

            if not display_name:
                # Handle the case where we are pushing a membership event to
@@ -345,3 +346,283 @@ MemberMap = Dict[str, Optional[EventIdMembership]]
 Rule = Dict[str, dict]
 RulesByUser = Dict[str, List[Rule]]
 StateGroup = Union[object, int]
+
+
+@attr.s(slots=True, auto_attribs=True)
+class RulesForRoomData:
+    """The data stored in the cache by `RulesForRoom`.
+
+    We don't store `RulesForRoom` directly in the cache as we want our caches to
+    *only* include data, and not references to e.g. the data stores.
+    """
+
+    # event_id -> EventIdMembership
+    member_map: MemberMap = attr.Factory(dict)
+    # user_id -> rules
+    rules_by_user: RulesByUser = attr.Factory(dict)
+
+    # The last state group we updated the caches for. If the state_group of
+    # a new event comes along, we know that we can just return the cached
+    # result.
+    # On invalidation of the rules themselves (if the user changes them),
+    # we invalidate everything and set state_group to `object()`
+    state_group: StateGroup = attr.Factory(object)
+
+    # A sequence number to keep track of when we're allowed to update the
+    # cache. We bump the sequence number when we invalidate the cache. If
+    # the sequence number changes while we're calculating stuff we should
+    # not update the cache with it.
+    sequence: int = 0
+
+    # A cache of user_ids that we *know* aren't interesting, e.g. user_ids
+    # owned by AS's, or remote users, etc. (I.e. users we will never need to
+    # calculate push for)
+    # These never need to be invalidated as we will never set up push for
+    # them.
+    uninteresting_user_set: Set[str] = attr.Factory(set)
+
+
+class RulesForRoom:
+    """Caches push rules for users in a room.
+
+    This efficiently handles users joining/leaving the room by not invalidating
+    the entire cache for the room.
+
+    A new instance is constructed for each call to
+    `BulkPushRuleEvaluator._get_rules_for_event`, with the cached data from
+    previous calls passed in.
+    """
+
+    def __init__(
+        self,
+        hs: "HomeServer",
+        room_id: str,
+        rules_for_room_cache: LruCache,
+        room_push_rule_cache_metrics: CacheMetric,
+        linearizer: Linearizer,
+        cached_data: RulesForRoomData,
+    ):
+        """
+        Args:
+            hs: The HomeServer object.
+            room_id: The room ID.
+            rules_for_room_cache: The cache object that caches these
+                RoomsForUser objects.
+            room_push_rule_cache_metrics: The metrics object
+            linearizer: The linearizer used to ensure only one thing mutates
+                the cache at a time. Keyed off room_id
+            cached_data: Cached data from previous calls to `self.get_rules`,
+                can be mutated.
+        """
+        self.room_id = room_id
+        self.is_mine_id = hs.is_mine_id
+        self.store = hs.get_datastores().main
+        self.room_push_rule_cache_metrics = room_push_rule_cache_metrics
+
+        # Used to ensure only one thing mutates the cache at a time. Keyed off
+        # room_id.
+        self.linearizer = linearizer
+
+        self.data = cached_data
+
+        # We need to be clever on the invalidating caches callbacks, as
+        # otherwise the invalidation callback holds a reference to the object,
+        # potentially causing it to leak.
+        # To get around this we pass a function that on invalidations looks ups
+        # the RoomsForUser entry in the cache, rather than keeping a reference
+        # to self around in the callback.
+        self.invalidate_all_cb = _Invalidation(rules_for_room_cache, room_id)
+
+    async def get_rules(
+        self, event: EventBase, context: EventContext
+    ) -> Dict[str, List[Dict[str, dict]]]:
+        """Given an event context return the rules for all users who are
+        currently in the room.
+        """
+        state_group = context.state_group
+
+        if state_group and self.data.state_group == state_group:
+            logger.debug("Using cached rules for %r", self.room_id)
+            self.room_push_rule_cache_metrics.inc_hits()
+            return self.data.rules_by_user
+
+        async with self.linearizer.queue(self.room_id):
+            if state_group and self.data.state_group == state_group:
+                logger.debug("Using cached rules for %r", self.room_id)
+                self.room_push_rule_cache_metrics.inc_hits()
+                return self.data.rules_by_user
+
+            self.room_push_rule_cache_metrics.inc_misses()
+
+            ret_rules_by_user = {}
+            missing_member_event_ids = {}
+            if state_group and self.data.state_group == context.prev_group:
+                # If we have a simple delta then we can reuse most of the previous
+                # results.
+                ret_rules_by_user = self.data.rules_by_user
+                current_state_ids = context.delta_ids
+
+                push_rules_delta_state_cache_metric.inc_hits()
+            else:
+                current_state_ids = await context.get_current_state_ids()
+                push_rules_delta_state_cache_metric.inc_misses()
+            # Ensure the state IDs exist.
+            assert current_state_ids is not None
+
+            push_rules_state_size_counter.inc(len(current_state_ids))
+
+            logger.debug(
+                "Looking for member changes in %r %r", state_group, current_state_ids
+            )
+
+            # Loop through to see which member events we've seen and have rules
+            # for and which we need to fetch
+            for key in current_state_ids:
+                typ, user_id = key
+                if typ != EventTypes.Member:
+                    continue
+
+                if user_id in self.data.uninteresting_user_set:
+                    continue
+
+                if not self.is_mine_id(user_id):
+                    self.data.uninteresting_user_set.add(user_id)
+                    continue
+
+                if self.store.get_if_app_services_interested_in_user(user_id):
+                    self.data.uninteresting_user_set.add(user_id)
+                    continue
+
+                event_id = current_state_ids[key]
+
+                res = self.data.member_map.get(event_id, None)
+                if res:
+                    if res.membership == Membership.JOIN:
+                        rules = self.data.rules_by_user.get(res.user_id, None)
+                        if rules:
+                            ret_rules_by_user[res.user_id] = rules
+                    continue
+
+                # If a user has left a room we remove their push rule. If they
+                # joined then we re-add it later in _update_rules_with_member_event_ids
+                ret_rules_by_user.pop(user_id, None)
+                missing_member_event_ids[user_id] = event_id
+
+            if missing_member_event_ids:
+                # If we have some member events we haven't seen, look them up
+                # and fetch push rules for them if appropriate.
+                logger.debug("Found new member events %r", missing_member_event_ids)
+                await self._update_rules_with_member_event_ids(
+                    ret_rules_by_user, missing_member_event_ids, state_group, event
+                )
+            else:
+                # The push rules didn't change but lets update the cache anyway
+                self.update_cache(
+                    self.data.sequence,
+                    members={},  # There were no membership changes
+                    rules_by_user=ret_rules_by_user,
+                    state_group=state_group,
+                )
+
+        if logger.isEnabledFor(logging.DEBUG):
+            logger.debug(
+                "Returning push rules for %r %r", self.room_id, ret_rules_by_user.keys()
+            )
+        return ret_rules_by_user
+
+    async def _update_rules_with_member_event_ids(
+        self,
+        ret_rules_by_user: Dict[str, list],
+        member_event_ids: Dict[str, str],
+        state_group: Optional[int],
+        event: EventBase,
+    ) -> None:
+        """Update the partially filled rules_by_user dict by fetching rules for
+        any newly joined users in the `member_event_ids` list.
+
+        Args:
+            ret_rules_by_user: Partially filled dict of push rules. Gets
+                updated with any new rules.
+            member_event_ids: Dict of user id to event id for membership events
+                that have happened since the last time we filled rules_by_user
+            state_group: The state group we are currently computing push rules
+                for. Used when updating the cache.
+            event: The event we are currently computing push rules for.
+        """
+        sequence = self.data.sequence
+
+        members = await self.store.get_membership_from_event_ids(
+            member_event_ids.values()
+        )
+
+        # If the event is a join event then it will be in current state events
+        # map but not in the DB, so we have to explicitly insert it.
+        if event.type == EventTypes.Member:
+            for event_id in member_event_ids.values():
+                if event_id == event.event_id:
+                    members[event_id] = EventIdMembership(
+                        user_id=event.state_key, membership=event.membership
+                    )
+
+        if logger.isEnabledFor(logging.DEBUG):
+            logger.debug("Found members %r: %r", self.room_id, members.values())
+
+        joined_user_ids = {
+            entry.user_id
+            for entry in members.values()
+            if entry and entry.membership == Membership.JOIN
+        }
+
+        logger.debug("Joined: %r", joined_user_ids)
+
+        # Previously we only considered users with pushers or read receipts in that
+        # room. We can't do this anymore because we use push actions to calculate unread
+        # counts, which don't rely on the user having pushers or sent a read receipt into
+        # the room. Therefore we just need to filter for local users here.
+        user_ids = list(filter(self.is_mine_id, joined_user_ids))
+
+        rules_by_user = await self.store.bulk_get_push_rules(
+            user_ids, on_invalidate=self.invalidate_all_cb
+        )
+
+        ret_rules_by_user.update(
+            item for item in rules_by_user.items() if item[0] is not None
+        )
+
+        self.update_cache(sequence, members, ret_rules_by_user, state_group)
+
+    def update_cache(
+        self,
+        sequence: int,
+        members: MemberMap,
+        rules_by_user: RulesByUser,
+        state_group: StateGroup,
+    ) -> None:
+        if sequence == self.data.sequence:
+            self.data.member_map.update(members)
+            self.data.rules_by_user = rules_by_user
+            self.data.state_group = state_group
+
+
+@attr.attrs(slots=True, frozen=True, auto_attribs=True)
+class _Invalidation:
+    # _Invalidation is passed as an `on_invalidate` callback to bulk_get_push_rules,
+    # which means that it it is stored on the bulk_get_push_rules cache entry. In order
+    # to ensure that we don't accumulate lots of redundant callbacks on the cache entry,
+    # we need to ensure that two _Invalidation objects are "equal" if they refer to the
+    # same `cache` and `room_id`.
+    #
+    # attrs provides suitable __hash__ and __eq__ methods, provided we remember to
+    # set `frozen=True`.
+
+    cache: LruCache
+    room_id: str
+
+    def __call__(self) -> None:
+        rules_data = self.cache.get(self.room_id, None, update_metrics=False)
+        if rules_data:
+            rules_data.sequence += 1
+            rules_data.state_group = object()
+            rules_data.member_map = {}
+            rules_data.rules_by_user = {}
+            push_rules_invalidation_counter.inc()
@@ -25,7 +25,6 @@ from synapse.replication.http import (
    push,
    register,
    send_event,
-    state,
    streams,
 )

@@ -49,7 +48,6 @@ class ReplicationRestResource(JsonResource):
        streams.register_servlets(hs, self)
        account_data.register_servlets(hs, self)
        push.register_servlets(hs, self)
-        state.register_servlets(hs, self)

        # The following can't currently be instantiated on workers.
        if hs.config.worker.worker_app is None:
@@ -60,9 +60,6 @@ class ReplicationFederationSendEventsRestServlet(ReplicationEndpoint):
        {
            "max_stream_id": 32443,
        }
-
-    Responds with a 409 when a `PartialStateConflictError` is raised due to an event
-    context that needs to be recomputed due to the un-partial stating of a room.
    """

    NAME = "fed_send_events"
@@ -59,9 +59,6 @@ class ReplicationSendEventRestServlet(ReplicationEndpoint):

        { "stream_id": 12345, "event_id": "$abcdef..." }

-    Responds with a 409 when a `PartialStateConflictError` is raised due to an event
-    context that needs to be recomputed due to the un-partial stating of a room.
-
    The returned event ID may not match the sent event if it was deduplicated.
    """

@@ -1,75 +0,0 @@
-# Copyright 2022 The Matrix.org Foundation C.I.C.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import logging
-from typing import TYPE_CHECKING, Tuple
-
-from twisted.web.server import Request
-
-from synapse.api.errors import SynapseError
-from synapse.http.server import HttpServer
-from synapse.replication.http._base import ReplicationEndpoint
-from synapse.types import JsonDict
-
-if TYPE_CHECKING:
-    from synapse.server import HomeServer
-
-logger = logging.getLogger(__name__)
-
-
-class ReplicationUpdateCurrentStateRestServlet(ReplicationEndpoint):
-    """Recalculates the current state for a room, and persists it.
-
-    The API looks like:
-
-        POST /_synapse/replication/update_current_state/:room_id
-
-        {}
-
-        200 OK
-
-        {}
-    """
-
-    NAME = "update_current_state"
-    PATH_ARGS = ("room_id",)
-
-    def __init__(self, hs: "HomeServer"):
-        super().__init__(hs)
-
-        self._state_handler = hs.get_state_handler()
-        self._events_shard_config = hs.config.worker.events_shard_config
-        self._instance_name = hs.get_instance_name()
-
-    @staticmethod
-    async def _serialize_payload(room_id: str) -> JsonDict:  # type: ignore[override]
-        return {}
-
-    async def _handle_request(  # type: ignore[override]
-        self, request: Request, room_id: str
-    ) -> Tuple[int, JsonDict]:
-        writer_instance = self._events_shard_config.get_instance(room_id)
-        if writer_instance != self._instance_name:
-            raise SynapseError(
-                400, "/update_current_state request was routed to the wrong worker"
-            )
-
-        await self._state_handler.update_current_state(room_id)
-
-        return 200, {}
-
-
-def register_servlets(hs: "HomeServer", http_server: HttpServer) -> None:
-    if hs.get_instance_name() in hs.config.worker.writers.events:
-        ReplicationUpdateCurrentStateRestServlet(hs).register(http_server)
@@ -154,9 +154,7 @@ class MediaStorage:
                        # Note that we'll delete the stored media, due to the
                        # try/except below. The media also won't be stored in
                        # the DB.
-                        # We currently ignore any additional field returned by
-                        # the spam-check API.
-                        raise SpamMediaException(errcode=spam_check[0])
+                        raise SpamMediaException(errcode=spam_check)

                    for provider in self.storage_providers:
                        await provider.store_file(path, file_info)
@@ -43,7 +43,6 @@ from synapse.api.room_versions import KNOWN_ROOM_VERSIONS, StateResolutionVersio
 from synapse.events import EventBase
 from synapse.events.snapshot import EventContext
 from synapse.logging.context import ContextResourceUsage
-from synapse.replication.http.state import ReplicationUpdateCurrentStateRestServlet
 from synapse.state import v1, v2
 from synapse.storage.databases.main.events_worker import EventRedactBehaviour
 from synapse.storage.roommember import ProfileInfo
@@ -130,12 +129,6 @@ class StateHandler:
        self.hs = hs
        self._state_resolution_handler = hs.get_state_resolution_handler()
        self._storage_controllers = hs.get_storage_controllers()
-        self._events_shard_config = hs.config.worker.events_shard_config
-        self._instance_name = hs.get_instance_name()
-
-        self._update_current_state_client = (
-            ReplicationUpdateCurrentStateRestServlet.make_client(hs)
-        )

    async def get_current_state_ids(
        self,
@@ -430,24 +423,6 @@ class StateHandler:

        return {key: state_map[ev_id] for key, ev_id in new_state.items()}

-    async def update_current_state(self, room_id: str) -> None:
-        """Recalculates the current state for a room, and persists it.
-
-        Raises:
-            SynapseError(502): if all attempts to connect to the event persister worker
-                fail
-        """
-        writer_instance = self._events_shard_config.get_instance(room_id)
-        if writer_instance != self._instance_name:
-            await self._update_current_state_client(
-                instance_name=writer_instance,
-                room_id=room_id,
-            )
-            return
-
-        assert self._storage_controllers.persistence is not None
-        await self._storage_controllers.persistence.update_current_state(room_id)
-

@attr.s(slots=True, auto_attribs=True)
 class _StateResMetrics:
@@ -75,15 +75,6 @@ class SQLBaseStore(metaclass=ABCMeta):
            self._attempt_to_invalidate_cache(
                "get_users_in_room_with_profiles", (room_id,)
            )
-            self._attempt_to_invalidate_cache(
-                "get_number_joined_users_in_room", (room_id,)
-            )
-            self._attempt_to_invalidate_cache("get_local_users_in_room", (room_id,))
-
-        for user_id in members_changed:
-            self._attempt_to_invalidate_cache(
-                "get_user_in_room_with_profile", (room_id, user_id)
-            )

        # Purge other caches based on room state.
        self._attempt_to_invalidate_cache("get_room_summary", (room_id,))
@@ -22,7 +22,6 @@ from typing import (
    Any,
    Awaitable,
    Callable,
-    ClassVar,
    Collection,
    Deque,
    Dict,
@@ -34,7 +33,6 @@ from typing import (
    Set,
    Tuple,
    TypeVar,
-    Union,
 )

 import attr
@@ -113,43 +111,9 @@ times_pruned_extremities = Counter(


@attr.s(auto_attribs=True, slots=True)
-class _PersistEventsTask:
-    """A batch of events to persist."""
-
-    name: ClassVar[str] = "persist_event_batch"  # used for opentracing
-
+class _EventPersistQueueItem:
    events_and_contexts: List[Tuple[EventBase, EventContext]]
    backfilled: bool
-
-    def try_merge(self, task: "_EventPersistQueueTask") -> bool:
-        """Batches events with the same backfilled option together."""
-        if (
-            not isinstance(task, _PersistEventsTask)
-            or self.backfilled != task.backfilled
-        ):
-            return False
-
-        self.events_and_contexts.extend(task.events_and_contexts)
-        return True
-
-
-@attr.s(auto_attribs=True, slots=True)
-class _UpdateCurrentStateTask:
-    """A room whose current state needs recalculating."""
-
-    name: ClassVar[str] = "update_current_state"  # used for opentracing
-
-    def try_merge(self, task: "_EventPersistQueueTask") -> bool:
-        """Deduplicates consecutive recalculations of current state."""
-        return isinstance(task, _UpdateCurrentStateTask)
-
-
-_EventPersistQueueTask = Union[_PersistEventsTask, _UpdateCurrentStateTask]
-
-
-@attr.s(auto_attribs=True, slots=True)
-class _EventPersistQueueItem:
-    task: _EventPersistQueueTask
    deferred: ObservableDeferred

    parent_opentracing_span_contexts: List = attr.ib(factory=list)
@@ -163,16 +127,14 @@ _PersistResult = TypeVar("_PersistResult")


 class _EventPeristenceQueue(Generic[_PersistResult]):
-    """Queues up tasks so that they can be processed with only one concurrent
-    transaction per room.
-
-    Tasks can be bulk persistence of events or recalculation of a room's current state.
+    """Queues up events so that they can be persisted in bulk with only one
+    concurrent transaction per room.
    """

    def __init__(
        self,
        per_item_callback: Callable[
-            [str, _EventPersistQueueTask],
+            [List[Tuple[EventBase, EventContext]], bool],
            Awaitable[_PersistResult],
        ],
    ):
@@ -188,17 +150,18 @@ class _EventPeristenceQueue(Generic[_PersistResult]):
    async def add_to_queue(
        self,
        room_id: str,
-        task: _EventPersistQueueTask,
+        events_and_contexts: Iterable[Tuple[EventBase, EventContext]],
+        backfilled: bool,
    ) -> _PersistResult:
-        """Add a task to the queue.
+        """Add events to the queue, with the given persist_event options.

-        If we are not already processing tasks in this room, starts off a background
+        If we are not already processing events in this room, starts off a background
        process to to so, calling the per_item_callback for each item.

        Args:
            room_id (str):
-            task (_EventPersistQueueTask): A _PersistEventsTask or
-                _UpdateCurrentStateTask to process.
+            events_and_contexts (list[(EventBase, EventContext)]):
+            backfilled (bool):

        Returns:
            the result returned by the `_per_item_callback` passed to
@@ -206,20 +169,26 @@ class _EventPeristenceQueue(Generic[_PersistResult]):
        """
        queue = self._event_persist_queues.setdefault(room_id, deque())

-        if queue and queue[-1].task.try_merge(task):
-            # the new task has been merged into the last task in the queue
+        # if the last item in the queue has the same `backfilled` setting,
+        # we can just add these new events to that item.
+        if queue and queue[-1].backfilled == backfilled:
            end_item = queue[-1]
        else:
+            # need to make a new queue item
            deferred: ObservableDeferred[_PersistResult] = ObservableDeferred(
                defer.Deferred(), consumeErrors=True
            )

            end_item = _EventPersistQueueItem(
-                task=task,
+                events_and_contexts=[],
+                backfilled=backfilled,
                deferred=deferred,
            )
            queue.append(end_item)

+        # add our events to the queue item
+        end_item.events_and_contexts.extend(events_and_contexts)
+
        # also add our active opentracing span to the item so that we get a link back
        span = opentracing.active_span()
        if span:
@@ -233,7 +202,7 @@ class _EventPeristenceQueue(Generic[_PersistResult]):

        # add another opentracing span which links to the persist trace.
        with opentracing.start_active_span_follows_from(
-            f"{task.name}_complete", (end_item.opentracing_span_context,)
+            "persist_event_batch_complete", (end_item.opentracing_span_context,)
        ):
            pass

@@ -265,14 +234,16 @@ class _EventPeristenceQueue(Generic[_PersistResult]):
                for item in queue:
                    try:
                        with opentracing.start_active_span_follows_from(
-                            item.task.name,
+                            "persist_event_batch",
                            item.parent_opentracing_span_contexts,
                            inherit_force_tracing=True,
                        ) as scope:
                            if scope:
                                item.opentracing_span_context = scope.span.context

-                            ret = await self._per_item_callback(room_id, item.task)
+                            ret = await self._per_item_callback(
+                                item.events_and_contexts, item.backfilled
+                            )
                    except Exception:
                        with PreserveLoggingContext():
                            item.deferred.errback()
@@ -321,32 +292,9 @@ class EventsPersistenceStorageController:
        self._clock = hs.get_clock()
        self._instance_name = hs.get_instance_name()
        self.is_mine_id = hs.is_mine_id
-        self._event_persist_queue = _EventPeristenceQueue(
-            self._process_event_persist_queue_task
-        )
+        self._event_persist_queue = _EventPeristenceQueue(self._persist_event_batch)
        self._state_resolution_handler = hs.get_state_resolution_handler()

-    async def _process_event_persist_queue_task(
-        self,
-        room_id: str,
-        task: _EventPersistQueueTask,
-    ) -> Dict[str, str]:
-        """Callback for the _event_persist_queue
-
-        Returns:
-            A dictionary of event ID to event ID we didn't persist as we already
-            had another event persisted with the same TXN ID.
-        """
-        if isinstance(task, _PersistEventsTask):
-            return await self._persist_event_batch(room_id, task)
-        elif isinstance(task, _UpdateCurrentStateTask):
-            await self._update_current_state(room_id, task)
-            return {}
-        else:
-            raise AssertionError(
-                f"Found an unexpected task type in event persistence queue: {task}"
-            )
-
    @opentracing.trace
    async def persist_events(
        self,
@@ -367,10 +315,6 @@ class EventsPersistenceStorageController:
            if they were deduplicated due to an event already existing that
            matched the transaction ID; the existing event is returned in such
            a case.
-
-        Raises:
-            PartialStateConflictError: if attempting to persist a partial state event in
-                a room that has been un-partial stated.
        """
        partitioned: Dict[str, List[Tuple[EventBase, EventContext]]] = {}
        for event, ctx in events_and_contexts:
@@ -381,8 +325,7 @@ class EventsPersistenceStorageController:
        ) -> Dict[str, str]:
            room_id, evs_ctxs = item
            return await self._event_persist_queue.add_to_queue(
-                room_id,
-                _PersistEventsTask(events_and_contexts=evs_ctxs, backfilled=backfilled),
+                room_id, evs_ctxs, backfilled=backfilled
            )

        ret_vals = await yieldable_gather_results(enqueue, partitioned.items())
@@ -420,19 +363,12 @@ class EventsPersistenceStorageController:
            latest persisted event. The returned event may not match the given
            event if it was deduplicated due to an existing event matching the
            transaction ID.
-
-        Raises:
-            PartialStateConflictError: if attempting to persist a partial state event in
-                a room that has been un-partial stated.
        """
        # add_to_queue returns a map from event ID to existing event ID if the
        # event was deduplicated. (The dict may also include other entries if
        # the event was persisted in a batch with other events.)
        replaced_events = await self._event_persist_queue.add_to_queue(
-            event.room_id,
-            _PersistEventsTask(
-                events_and_contexts=[(event, context)], backfilled=backfilled
-            ),
+            event.room_id, [(event, context)], backfilled=backfilled
        )
        replaced_event = replaced_events.get(event.event_id)
        if replaced_event:
@@ -447,22 +383,20 @@ class EventsPersistenceStorageController:

    async def update_current_state(self, room_id: str) -> None:
        """Recalculate the current state for a room, and persist it"""
-        await self._event_persist_queue.add_to_queue(
-            room_id,
-            _UpdateCurrentStateTask(),
-        )
-
-    async def _update_current_state(
-        self, room_id: str, _task: _UpdateCurrentStateTask
-    ) -> None:
-        """Callback for the _event_persist_queue
-
-        Recalculates the current state for a room, and persists it.
-        """
        state = await self._calculate_current_state(room_id)
        delta = await self._calculate_state_delta(room_id, state)

-        await self.persist_events_store.update_current_state(room_id, delta)
+        # TODO(faster_joins): get a real stream ordering, to make this work correctly
+        #    across workers.
+        #    https://github.com/matrix-org/synapse/issues/12994
+        #
+        # TODO(faster_joins): this can race against event persistence, in which case we
+        #    will end up with incorrect state. Perhaps we should make this a job we
+        #    farm out to the event persister thread, somehow.
+        #    https://github.com/matrix-org/synapse/issues/13007
+        #
+        stream_id = self.main_store.get_room_max_stream_ordering()
+        await self.persist_events_store.update_current_state(room_id, delta, stream_id)

    async def _calculate_current_state(self, room_id: str) -> StateMap[str]:
        """Calculate the current state of a room, based on the forward extremities
@@ -507,7 +441,9 @@ class EventsPersistenceStorageController:
        return res.state

    async def _persist_event_batch(
-        self, _room_id: str, task: _PersistEventsTask
+        self,
+        events_and_contexts: List[Tuple[EventBase, EventContext]],
+        backfilled: bool = False,
    ) -> Dict[str, str]:
        """Callback for the _event_persist_queue

@@ -517,14 +453,7 @@ class EventsPersistenceStorageController:
        Returns:
            A dictionary of event ID to event ID we didn't persist as we already
            had another event persisted with the same TXN ID.
-
-        Raises:
-            PartialStateConflictError: if attempting to persist a partial state event in
-                a room that has been un-partial stated.
        """
-        events_and_contexts = task.events_and_contexts
-        backfilled = task.backfilled
-
        replaced_events: Dict[str, str] = {}
        if not events_and_contexts:
            return replaced_events
@@ -143,6 +143,7 @@ class EventPushActionsWorkerStore(ReceiptsWorkerStore, StreamWorkerStore, SQLBas
            self._find_stream_orderings_for_times, 10 * 60 * 1000
        )

+        self._rotate_delay = 3
        self._rotate_count = 10000
        self._doing_notif_rotation = False
        if hs.config.worker.run_background_tasks:
@@ -846,6 +847,7 @@ class EventPushActionsWorkerStore(ReceiptsWorkerStore, StreamWorkerStore, SQLBas
                )
                if caught_up:
                    break
+                await self.hs.get_clock().sleep(self._rotate_delay)

            # Finally we clear out old event push actions.
            await self._remove_old_push_actions_that_have_rotated()
@@ -980,12 +982,7 @@ class EventPushActionsWorkerStore(ReceiptsWorkerStore, StreamWorkerStore, SQLBas
        stream_row = txn.fetchone()
        if stream_row:
            (offset_stream_ordering,) = stream_row
-
-            # We need to bound by the current token to ensure that we handle
-            # out-of-order writes correctly.
-            rotate_to_stream_ordering = min(
-                offset_stream_ordering, self._stream_id_gen.get_current_token()
-            )
+            rotate_to_stream_ordering = offset_stream_ordering
            caught_up = False
        else:
            rotate_to_stream_ordering = self._stream_id_gen.get_current_token()
@@ -1011,17 +1008,13 @@ class EventPushActionsWorkerStore(ReceiptsWorkerStore, StreamWorkerStore, SQLBas
        sql = """
            SELECT user_id, room_id,
                coalesce(old.%s, 0) + upd.cnt,
-                upd.stream_ordering
+                upd.stream_ordering,
+                old.user_id
            FROM (
                SELECT user_id, room_id, count(*) as cnt,
-                    max(ea.stream_ordering) as stream_ordering
-                FROM event_push_actions AS ea
-                LEFT JOIN event_push_summary AS old USING (user_id, room_id)
-                WHERE ? < ea.stream_ordering AND ea.stream_ordering <= ?
-                    AND (
-                        old.last_receipt_stream_ordering IS NULL
-                        OR old.last_receipt_stream_ordering < ea.stream_ordering
-                    )
+                    max(stream_ordering) as stream_ordering
+                FROM event_push_actions
+                WHERE ? <= stream_ordering AND stream_ordering < ?
                    AND %s = 1
                GROUP BY user_id, room_id
            ) AS upd
@@ -1044,6 +1037,7 @@ class EventPushActionsWorkerStore(ReceiptsWorkerStore, StreamWorkerStore, SQLBas
            summaries[(row[0], row[1])] = _EventPushSummary(
                unread_count=row[2],
                stream_ordering=row[3],
+                old_user_id=row[4],
                notif_count=0,
            )

@@ -1064,27 +1058,57 @@ class EventPushActionsWorkerStore(ReceiptsWorkerStore, StreamWorkerStore, SQLBas
                summaries[(row[0], row[1])] = _EventPushSummary(
                    unread_count=0,
                    stream_ordering=row[3],
+                    old_user_id=row[4],
                    notif_count=row[2],
                )

        logger.info("Rotating notifications, handling %d rows", len(summaries))

-        self.db_pool.simple_upsert_many_txn(
+        # If the `old.user_id` above is NULL then we know there isn't already an
+        # entry in the table, so we simply insert it. Otherwise we update the
+        # existing table.
+        self.db_pool.simple_insert_many_txn(
            txn,
            table="event_push_summary",
-            key_names=("user_id", "room_id"),
-            key_values=[(user_id, room_id) for user_id, room_id in summaries],
-            value_names=("notif_count", "unread_count", "stream_ordering"),
-            value_values=[
+            keys=(
+                "user_id",
+                "room_id",
+                "notif_count",
+                "unread_count",
+                "stream_ordering",
+            ),
+            values=[
                (
+                    user_id,
+                    room_id,
                    summary.notif_count,
                    summary.unread_count,
                    summary.stream_ordering,
                )
-                for summary in summaries.values()
+                for ((user_id, room_id), summary) in summaries.items()
+                if summary.old_user_id is None
            ],
        )

+        txn.execute_batch(
+            """
+                UPDATE event_push_summary
+                SET notif_count = ?, unread_count = ?, stream_ordering = ?
+                WHERE user_id = ? AND room_id = ?
+            """,
+            (
+                (
+                    summary.notif_count,
+                    summary.unread_count,
+                    summary.stream_ordering,
+                    user_id,
+                    room_id,
+                )
+                for ((user_id, room_id), summary) in summaries.items()
+                if summary.old_user_id is not None
+            ),
+        )
+
        txn.execute(
            "UPDATE event_push_summary_stream_ordering SET stream_ordering = ?",
            (rotate_to_stream_ordering,),
@@ -1112,12 +1136,12 @@ class EventPushActionsWorkerStore(ReceiptsWorkerStore, StreamWorkerStore, SQLBas
        ) -> bool:
            # We don't want to clear out too much at a time, so we bound our
            # deletes.
-            batch_size = self._rotate_count
+            batch_size = 10000

            txn.execute(
                """
                SELECT stream_ordering FROM event_push_actions
-                WHERE stream_ordering <= ? AND highlight = 0
+                WHERE stream_ordering < ? AND highlight = 0
                ORDER BY stream_ordering ASC LIMIT 1 OFFSET ?
            """,
                (
@@ -1132,12 +1156,10 @@ class EventPushActionsWorkerStore(ReceiptsWorkerStore, StreamWorkerStore, SQLBas
            else:
                stream_ordering = max_stream_ordering_to_delete

-            # We need to use a inclusive bound here to handle the case where a
-            # single stream ordering has more than `batch_size` rows.
            txn.execute(
                """
                DELETE FROM event_push_actions
-                WHERE stream_ordering <= ? AND highlight = 0
+                WHERE stream_ordering < ? AND highlight = 0
                """,
                (stream_ordering,),
            )
@@ -1266,4 +1288,5 @@ class _EventPushSummary:

    unread_count: int
    stream_ordering: int
+    old_user_id: str
    notif_count: int
@@ -16,7 +16,6 @@
 import itertools
 import logging
 from collections import OrderedDict
-from http import HTTPStatus
 from typing import (
    TYPE_CHECKING,
    Any,
@@ -36,7 +35,6 @@ from prometheus_client import Counter

 import synapse.metrics
 from synapse.api.constants import EventContentFields, EventTypes, RelationTypes
-from synapse.api.errors import Codes, SynapseError
 from synapse.api.room_versions import RoomVersions
 from synapse.events import EventBase, relation_from_event
 from synapse.events.snapshot import EventContext
@@ -71,24 +69,6 @@ event_counter = Counter(
 )


-class PartialStateConflictError(SynapseError):
-    """An internal error raised when attempting to persist an event with partial state
-    after the room containing the event has been un-partial stated.
-
-    This error should be handled by recomputing the event context and trying again.
-
-    This error has an HTTP status code so that it can be transported over replication.
-    It should not be exposed to clients.
-    """
-
-    def __init__(self) -> None:
-        super().__init__(
-            HTTPStatus.CONFLICT,
-            msg="Cannot persist partial state event in un-partial stated room",
-            errcode=Codes.UNKNOWN,
-        )
-
-
@attr.s(slots=True, auto_attribs=True)
 class DeltaState:
    """Deltas to use to update the `current_state_events` table.
@@ -174,10 +154,6 @@ class PersistEventsStore:

        Returns:
            Resolves when the events have been persisted
-
-        Raises:
-            PartialStateConflictError: if attempting to persist a partial state event in
-                a room that has been un-partial stated.
        """

        # We want to calculate the stream orderings as late as possible, as
@@ -378,9 +354,6 @@ class PersistEventsStore:
                For each room, a list of the event ids which are the forward
                extremities.

-        Raises:
-            PartialStateConflictError: if attempting to persist a partial state event in
-                a room that has been un-partial stated.
        """
        state_delta_for_room = state_delta_for_room or {}
        new_forward_extremities = new_forward_extremities or {}
@@ -1007,16 +980,16 @@ class PersistEventsStore:
        self,
        room_id: str,
        state_delta: DeltaState,
+        stream_id: int,
    ) -> None:
        """Update the current state stored in the datatabase for the given room"""

-        async with self._stream_id_gen.get_next() as stream_ordering:
-            await self.db_pool.runInteraction(
-                "update_current_state",
-                self._update_current_state_txn,
-                state_delta_by_room={room_id: state_delta},
-                stream_id=stream_ordering,
-            )
+        await self.db_pool.runInteraction(
+            "update_current_state",
+            self._update_current_state_txn,
+            state_delta_by_room={room_id: state_delta},
+            stream_id=stream_id,
+        )

    def _update_current_state_txn(
        self,
@@ -1331,10 +1304,6 @@ class PersistEventsStore:

        Returns:
            new list, without events which are already in the events table.
-
-        Raises:
-            PartialStateConflictError: if attempting to persist a partial state event in
-                a room that has been un-partial stated.
        """
        txn.execute(
            "SELECT event_id, outlier FROM events WHERE event_id in (%s)"
@@ -1797,18 +1766,6 @@ class PersistEventsStore:
                self.store.get_invited_rooms_for_local_user.invalidate,
                (event.state_key,),
            )
-            txn.call_after(
-                self.store.get_local_users_in_room.invalidate,
-                (event.room_id,),
-            )
-            txn.call_after(
-                self.store.get_number_joined_users_in_room.invalidate,
-                (event.room_id,),
-            )
-            txn.call_after(
-                self.store.get_user_in_room_with_profile.invalidate,
-                (event.room_id, event.state_key),
-            )

            # The `_get_membership_from_event_id` is immutable, except for the
            # case where we look up an event *before* persisting it.
@@ -2258,11 +2215,6 @@ class PersistEventsStore:
        txn: LoggingTransaction,
        events_and_contexts: Collection[Tuple[EventBase, EventContext]],
    ) -> None:
-        """
-        Raises:
-            PartialStateConflictError: if attempting to persist a partial state event in
-                a room that has been un-partial stated.
-        """
        state_groups = {}
        for event, context in events_and_contexts:
            if event.internal_metadata.is_outlier():
@@ -2287,37 +2239,19 @@ class PersistEventsStore:
        # if we have partial state for these events, record the fact. (This happens
        # here rather than in _store_event_txn because it also needs to happen when
        # we de-outlier an event.)
-        try:
-            self.db_pool.simple_insert_many_txn(
-                txn,
-                table="partial_state_events",
-                keys=("room_id", "event_id"),
-                values=[
-                    (
-                        event.room_id,
-                        event.event_id,
-                    )
-                    for event, ctx in events_and_contexts
-                    if ctx.partial_state
-                ],
-            )
-        except self.db_pool.engine.module.IntegrityError:
-            logger.info(
-                "Cannot persist events %s in rooms %s: room has been un-partial stated",
-                [
-                    event.event_id
-                    for event, ctx in events_and_contexts
-                    if ctx.partial_state
-                ],
-                list(
-                    {
-                        event.room_id
-                        for event, ctx in events_and_contexts
-                        if ctx.partial_state
-                    }
-                ),
-            )
-            raise PartialStateConflictError()
+        self.db_pool.simple_insert_many_txn(
+            txn,
+            table="partial_state_events",
+            keys=("room_id", "event_id"),
+            values=[
+                (
+                    event.room_id,
+                    event.event_id,
+                )
+                for event, ctx in events_and_contexts
+                if ctx.partial_state
+            ],
+        )

        self.db_pool.simple_upsert_many_txn(
            txn,
@@ -1156,25 +1156,19 @@ class RoomWorkerStore(CacheInvalidationWorkerStore):
        return room_servers

    async def clear_partial_state_room(self, room_id: str) -> bool:
-        """Clears the partial state flag for a room.
-
-        Args:
-            room_id: The room whose partial state flag is to be cleared.
-
-        Returns:
-            `True` if the partial state flag has been cleared successfully.
-
-            `False` if the partial state flag could not be cleared because the room
-            still contains events with partial state.
-        """
+        # this can race with incoming events, so we watch out for FK errors.
+        # TODO(faster_joins): this still doesn't completely fix the race, since the persist process
+        #   is not atomic. I fear we need an application-level lock.
+        #   https://github.com/matrix-org/synapse/issues/12988
        try:
            await self.db_pool.runInteraction(
                "clear_partial_state_room", self._clear_partial_state_room_txn, room_id
            )
            return True
-        except self.db_pool.engine.module.IntegrityError as e:
-            # Assume that any `IntegrityError`s are due to partial state events.
-            logger.info(
+        except self.db_pool.engine.module.DatabaseError as e:
+            # TODO(faster_joins): how do we distinguish between FK errors and other errors?
+            #   https://github.com/matrix-org/synapse/issues/12988
+            logger.warning(
                "Exception while clearing lazy partial-state-room %s, retrying: %s",
                room_id,
                e,
@@ -212,60 +212,6 @@ class RoomMemberWorkerStore(EventsWorkerStore):
        txn.execute(sql, (room_id, Membership.JOIN))
        return [r[0] for r in txn]

-    @cached()
-    def get_user_in_room_with_profile(
-        self, room_id: str, user_id: str
-    ) -> Dict[str, ProfileInfo]:
-        raise NotImplementedError()
-
-    @cachedList(
-        cached_method_name="get_user_in_room_with_profile", list_name="user_ids"
-    )
-    async def get_subset_users_in_room_with_profiles(
-        self, room_id: str, user_ids: Collection[str]
-    ) -> Dict[str, ProfileInfo]:
-        """Get a mapping from user ID to profile information for a list of users
-        in a given room.
-
-        The profile information comes directly from this room's `m.room.member`
-        events, and so may be specific to this room rather than part of a user's
-        global profile. To avoid privacy leaks, the profile data should only be
-        revealed to users who are already in this room.
-
-        Args:
-            room_id: The ID of the room to retrieve the users of.
-            user_ids: a list of users in the room to run the query for
-
-        Returns:
-                A mapping from user ID to ProfileInfo.
-        """
-
-        def _get_subset_users_in_room_with_profiles(
-            txn: LoggingTransaction,
-        ) -> Dict[str, ProfileInfo]:
-            clause, ids = make_in_list_sql_clause(
-                self.database_engine, "m.user_id", user_ids
-            )
-
-            sql = """
-                SELECT state_key, display_name, avatar_url FROM room_memberships as m
-                INNER JOIN current_state_events as c
-                ON m.event_id = c.event_id
-                AND m.room_id = c.room_id
-                AND m.user_id = c.state_key
-                WHERE c.type = 'm.room.member' AND c.room_id = ? AND m.membership = ? AND %s
-            """ % (
-                clause,
-            )
-            txn.execute(sql, (room_id, Membership.JOIN, *ids))
-
-            return {r[0]: ProfileInfo(display_name=r[1], avatar_url=r[2]) for r in txn}
-
-        return await self.db_pool.runInteraction(
-            "get_subset_users_in_room_with_profiles",
-            _get_subset_users_in_room_with_profiles,
-        )
-
    @cached(max_entries=100000, iterable=True)
    async def get_users_in_room_with_profiles(
        self, room_id: str
@@ -391,15 +337,6 @@ class RoomMemberWorkerStore(EventsWorkerStore):
            "get_room_summary", _get_room_summary_txn
        )

-    @cached()
-    async def get_number_joined_users_in_room(self, room_id: str) -> int:
-        return await self.db_pool.simple_select_one_onecol(
-            table="current_state_events",
-            keyvalues={"room_id": room_id, "membership": Membership.JOIN},
-            retcol="COUNT(*)",
-            desc="get_number_joined_users_in_room",
-        )
-
    @cached()
    async def get_invited_rooms_for_local_user(
        self, user_id: str
@@ -479,17 +416,6 @@ class RoomMemberWorkerStore(EventsWorkerStore):
        user_id: str,
        membership_list: List[str],
    ) -> List[RoomsForUser]:
-        """Get all the rooms for this *local* user where the membership for this user
-        matches one in the membership list.
-
-        Args:
-            user_id: The user ID.
-            membership_list: A list of synapse.api.constants.Membership
-                    values which the user must be in.
-
-        Returns:
-            The RoomsForUser that the user matches the membership types.
-        """
        # Paranoia check.
        if not self.hs.is_mine_id(user_id):
            raise Exception(
@@ -518,18 +444,6 @@ class RoomMemberWorkerStore(EventsWorkerStore):

        return results

-    @cached(iterable=True)
-    async def get_local_users_in_room(self, room_id: str) -> List[str]:
-        """
-        Retrieves a list of the current roommembers who are local to the server.
-        """
-        return await self.db_pool.simple_select_onecol(
-            table="local_current_membership",
-            keyvalues={"room_id": room_id, "membership": Membership.JOIN},
-            retcol="user_id",
-            desc="get_local_users_in_room",
-        )
-
    async def get_local_current_membership_for_user_in_room(
        self, user_id: str, room_id: str
    ) -> Tuple[Optional[str], Optional[str]]:
@@ -562,7 +476,7 @@ class RoomMemberWorkerStore(EventsWorkerStore):

        return results_dict.get("membership"), results_dict.get("event_id")

-    @cached(max_entries=500000, iterable=True, prune_unread_entries=False)
+    @cached(max_entries=500000, iterable=True, prune_unread_entries=False, debug_invalidations=True)
    async def get_rooms_for_user_with_stream_ordering(
        self, user_id: str
    ) -> FrozenSet[GetRoomsForUserWithStreamOrdering]:
@@ -20,7 +20,6 @@ from typing import Any, Callable, Dict, Generator, Optional
 import attr
 from frozendict import frozendict
 from matrix_common.versionstring import get_distribution_version_string
-from typing_extensions import ParamSpec

 from twisted.internet import defer, task
 from twisted.internet.defer import Deferred
@@ -83,9 +82,6 @@ def unwrapFirstError(failure: Failure) -> Failure:
    return failure.value.subFailure  # type: ignore[union-attr]  # Issue in Twisted's annotations


-P = ParamSpec("P")
-
-
@attr.s(slots=True)
 class Clock:
    """
@@ -114,7 +110,7 @@ class Clock:
        return int(self.time() * 1000)

    def looping_call(
-        self, f: Callable[P, object], msec: float, *args: P.args, **kwargs: P.kwargs
+        self, f: Callable, msec: float, *args: Any, **kwargs: Any
    ) -> LoopingCall:
        """Call a function repeatedly.

@@ -15,6 +15,7 @@
 # limitations under the License.

 import enum
+import logging
 import threading
 from typing import (
    Callable,
@@ -66,6 +67,7 @@ class DeferredCache(Generic[KT, VT]):
        "cache",
        "thread",
        "_pending_deferred_cache",
+        "debug_invalidations"
    )

    def __init__(
@@ -76,6 +78,7 @@ class DeferredCache(Generic[KT, VT]):
        iterable: bool = False,
        apply_cache_factor_from_config: bool = True,
        prune_unread_entries: bool = True,
+        debug_invalidations: bool = False,
    ):
        """
        Args:
@@ -119,6 +122,7 @@ class DeferredCache(Generic[KT, VT]):
        )

        self.thread: Optional[threading.Thread] = None
+        self.debug_invalidations = debug_invalidations

    @property
    def max_entries(self) -> int:
@@ -310,6 +314,9 @@ class DeferredCache(Generic[KT, VT]):
        self.check_thread()
        self.cache.del_multi(key)

+        if self.debug_invalidations:
+            logging.debug("Invalidating key %r in cache", key)
+
        # if we have a pending lookup for this key, remove it from the
        # _pending_deferred_cache, which will (a) stop it being returned
        # for future queries and (b) stop it being persisted as a proper entry
@@ -326,6 +333,8 @@ class DeferredCache(Generic[KT, VT]):
                entry.invalidate()

    def invalidate_all(self) -> None:
+        if self.debug_invalidations:
+            logging.debug("Invalidating ALL keys")
        self.check_thread()
        self.cache.clear()
        for entry in self._pending_deferred_cache.values():
@@ -301,6 +301,7 @@ class DeferredCacheDescriptor(_CacheDescriptorBase):
        cache_context: bool = False,
        iterable: bool = False,
        prune_unread_entries: bool = True,
+        debug_invalidations: bool = False
    ):
        super().__init__(
            orig,
@@ -318,6 +319,7 @@ class DeferredCacheDescriptor(_CacheDescriptorBase):
        self.tree = tree
        self.iterable = iterable
        self.prune_unread_entries = prune_unread_entries
+        self.debug_invalidations = debug_invalidations

    def __get__(self, obj: Optional[Any], owner: Optional[Type]) -> Callable[..., Any]:
        cache: DeferredCache[CacheKey, Any] = DeferredCache(
@@ -326,6 +328,7 @@ class DeferredCacheDescriptor(_CacheDescriptorBase):
            tree=self.tree,
            iterable=self.iterable,
            prune_unread_entries=self.prune_unread_entries,
+            debug_invalidations=self.debug_invalidations,
        )

        get_cache_key = self.cache_key_builder
@@ -577,6 +580,7 @@ def cached(
    cache_context: bool = False,
    iterable: bool = False,
    prune_unread_entries: bool = True,
+    debug_invalidations: bool = False,
 ) -> Callable[[F], _CachedFunction[F]]:
    func = lambda orig: DeferredCacheDescriptor(
        orig,
@@ -587,6 +591,7 @@ def cached(
        cache_context=cache_context,
        iterable=iterable,
        prune_unread_entries=prune_unread_entries,
+        debug_invalidations=debug_invalidations
    )

    return cast(Callable[[F], _CachedFunction[F]], func)
@@ -109,7 +109,7 @@ GLOBAL_ROOT = ListNode["_Node"].create_root_node()

@wrap_as_background_process("LruCache._expire_old_entries")
 async def _expire_old_entries(
-    clock: Clock, expiry_seconds: float, autotune_config: Optional[dict]
+    clock: Clock, expiry_seconds: int, autotune_config: Optional[dict]
 ) -> None:
    """Walks the global cache list to find cache entries that haven't been
    accessed in the given number of seconds, or if a given memory threshold has been breached.
@@ -13,21 +13,16 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 import logging
-from enum import Enum, auto
 from typing import Collection, Dict, FrozenSet, List, Optional, Tuple

-import attr
 from typing_extensions import Final

 from synapse.api.constants import EventTypes, HistoryVisibility, Membership
 from synapse.events import EventBase
-from synapse.events.snapshot import EventContext
 from synapse.events.utils import prune_event
 from synapse.storage.controllers import StorageControllers
-from synapse.storage.databases.main import DataStore
 from synapse.storage.state import StateFilter
 from synapse.types import RetentionPolicy, StateMap, get_domain_from_id
-from synapse.util import Clock

 logger = logging.getLogger(__name__)

@@ -107,18 +102,153 @@ async def filter_events_for_client(
            ] = await storage.main.get_retention_policy_for_room(room_id)

    def allowed(event: EventBase) -> Optional[EventBase]:
-        return _check_client_allowed_to_see_event(
-            user_id=user_id,
-            event=event,
-            clock=storage.main.clock,
-            filter_send_to_client=filter_send_to_client,
-            sender_ignored=event.sender in ignore_list,
-            always_include_ids=always_include_ids,
-            retention_policy=retention_policies[room_id],
-            state=event_id_to_state.get(event.event_id),
-            is_peeking=is_peeking,
-            sender_erased=erased_senders.get(event.sender, False),
-        )
+        """
+        Args:
+            event: event to check
+
+        Returns:
+           None if the user cannot see this event at all
+
+           a redacted copy of the event if they can only see a redacted
+           version
+
+           the original event if they can see it as normal.
+        """
+        # Only run some checks if these events aren't about to be sent to clients. This is
+        # because, if this is not the case, we're probably only checking if the users can
+        # see events in the room at that point in the DAG, and that shouldn't be decided
+        # on those checks.
+        if filter_send_to_client:
+            if event.type == EventTypes.Dummy:
+                return None
+
+            if not event.is_state() and event.sender in ignore_list:
+                return None
+
+            # Until MSC2261 has landed we can't redact malicious alias events, so for
+            # now we temporarily filter out m.room.aliases entirely to mitigate
+            # abuse, while we spec a better solution to advertising aliases
+            # on rooms.
+            if event.type == EventTypes.Aliases:
+                return None
+
+            # Don't try to apply the room's retention policy if the event is a state
+            # event, as MSC1763 states that retention is only considered for non-state
+            # events.
+            if not event.is_state():
+                retention_policy = retention_policies[event.room_id]
+                max_lifetime = retention_policy.max_lifetime
+
+                if max_lifetime is not None:
+                    oldest_allowed_ts = storage.main.clock.time_msec() - max_lifetime
+
+                    if event.origin_server_ts < oldest_allowed_ts:
+                        return None
+
+        if event.event_id in always_include_ids:
+            return event
+
+        # we need to handle outliers separately, since we don't have the room state.
+        if event.internal_metadata.outlier:
+            # Normally these can't be seen by clients, but we make an exception for
+            # for out-of-band membership events (eg, incoming invites, or rejections of
+            # said invite) for the user themselves.
+            if event.type == EventTypes.Member and event.state_key == user_id:
+                logger.debug("Returning out-of-band-membership event %s", event)
+                return event
+
+            return None
+
+        state = event_id_to_state[event.event_id]
+
+        # get the room_visibility at the time of the event.
+        visibility = get_effective_room_visibility_from_state(state)
+
+        # Always allow history visibility events on boundaries. This is done
+        # by setting the effective visibility to the least restrictive
+        # of the old vs new.
+        if event.type == EventTypes.RoomHistoryVisibility:
+            prev_content = event.unsigned.get("prev_content", {})
+            prev_visibility = prev_content.get("history_visibility", None)
+
+            if prev_visibility not in VISIBILITY_PRIORITY:
+                prev_visibility = HistoryVisibility.SHARED
+
+            new_priority = VISIBILITY_PRIORITY.index(visibility)
+            old_priority = VISIBILITY_PRIORITY.index(prev_visibility)
+            if old_priority < new_priority:
+                visibility = prev_visibility
+
+        # likewise, if the event is the user's own membership event, use
+        # the 'most joined' membership
+        membership = None
+        if event.type == EventTypes.Member and event.state_key == user_id:
+            membership = event.content.get("membership", None)
+            if membership not in MEMBERSHIP_PRIORITY:
+                membership = "leave"
+
+            prev_content = event.unsigned.get("prev_content", {})
+            prev_membership = prev_content.get("membership", None)
+            if prev_membership not in MEMBERSHIP_PRIORITY:
+                prev_membership = "leave"
+
+            # Always allow the user to see their own leave events, otherwise
+            # they won't see the room disappear if they reject the invite
+            #
+            # (Note this doesn't work for out-of-band invite rejections, which don't
+            # have prev_state populated. They are handled above in the outlier code.)
+            if membership == "leave" and (
+                prev_membership == "join" or prev_membership == "invite"
+            ):
+                return event
+
+            new_priority = MEMBERSHIP_PRIORITY.index(membership)
+            old_priority = MEMBERSHIP_PRIORITY.index(prev_membership)
+            if old_priority < new_priority:
+                membership = prev_membership
+
+        # otherwise, get the user's membership at the time of the event.
+        if membership is None:
+            membership_event = state.get((EventTypes.Member, user_id), None)
+            if membership_event:
+                membership = membership_event.membership
+
+        # if the user was a member of the room at the time of the event,
+        # they can see it.
+        if membership == Membership.JOIN:
+            return event
+
+        # otherwise, it depends on the room visibility.
+
+        if visibility == HistoryVisibility.JOINED:
+            # we weren't a member at the time of the event, so we can't
+            # see this event.
+            return None
+
+        elif visibility == HistoryVisibility.INVITED:
+            # user can also see the event if they were *invited* at the time
+            # of the event.
+            return event if membership == Membership.INVITE else None
+
+        elif visibility == HistoryVisibility.SHARED and is_peeking:
+            # if the visibility is shared, users cannot see the event unless
+            # they have *subsequently* joined the room (or were members at the
+            # time, of course)
+            #
+            # XXX: if the user has subsequently joined and then left again,
+            # ideally we would share history up to the point they left. But
+            # we don't know when they left. We just treat it as though they
+            # never joined, and restrict access.
+            return None
+
+        # the visibility is either shared or world_readable, and the user was
+        # not a member at the time. We allow it, provided the original sender
+        # has not requested their data to be erased, in which case, we return
+        # a redacted version.
+        if erased_senders[event.sender]:
+            return prune_event(event)
+
+        return event

    # Check each event: gives an iterable of None or (a potentially modified)
    # EventBase.
@@ -128,389 +258,9 @@ async def filter_events_for_client(
    return [ev for ev in filtered_events if ev]


-async def filter_event_for_clients_with_state(
-    store: DataStore,
-    user_ids: Collection[str],
-    event: EventBase,
-    context: EventContext,
-    is_peeking: bool = False,
-    filter_send_to_client: bool = True,
-) -> Collection[str]:
-    """
-    Checks to see if an event is visible to the users in the list at the time of
-    the event.
-
-    Note: This does *not* check if the sender of the event was erased.
-
-    Args:
-        store: databases
-        user_ids: user_ids to be checked
-        event: the event to be checked
-        context: EventContext for the event to be checked
-        is_peeking: Whether the users are peeking into the room, ie not
-            currently joined
-        filter_send_to_client: Whether we're checking an event that's going to be
-            sent to a client. This might not always be the case since this function can
-            also be called to check whether a user can see the state at a given point.
-
-    Returns:
-        Collection of user IDs for whom the event is visible
-    """
-    # None of the users should see the event if it is soft_failed
-    if event.internal_metadata.is_soft_failed():
-        return []
-
-    # Make a set for all user IDs that haven't been filtered out by a check.
-    allowed_user_ids = set(user_ids)
-
-    # Only run some checks if these events aren't about to be sent to clients. This is
-    # because, if this is not the case, we're probably only checking if the users can
-    # see events in the room at that point in the DAG, and that shouldn't be decided
-    # on those checks.
-    if filter_send_to_client:
-        ignored_by = await store.ignored_by(event.sender)
-        retention_policy = await store.get_retention_policy_for_room(event.room_id)
-
-        for user_id in user_ids:
-            if (
-                _check_filter_send_to_client(
-                    event,
-                    store.clock,
-                    retention_policy,
-                    sender_ignored=user_id in ignored_by,
-                )
-                == _CheckFilter.DENIED
-            ):
-                allowed_user_ids.discard(user_id)
-
-    if event.internal_metadata.outlier:
-        # Normally these can't be seen by clients, but we make an exception for
-        # for out-of-band membership events (eg, incoming invites, or rejections of
-        # said invite) for the user themselves.
-        if event.type == EventTypes.Member and event.state_key in allowed_user_ids:
-            logger.debug("Returning out-of-band-membership event %s", event)
-            return {event.state_key}
-
-        return set()
-
-    # First we get just the history visibility in case its shared/world-readable
-    # room.
-    visibility_state_map = await _get_state_map(
-        store, event, context, StateFilter.from_types([_HISTORY_VIS_KEY])
-    )
-
-    visibility = get_effective_room_visibility_from_state(visibility_state_map)
-    if (
-        _check_history_visibility(event, visibility, is_peeking=is_peeking)
-        == _CheckVisibility.ALLOWED
-    ):
-        return allowed_user_ids
-
-    # The history visibility isn't lax, so we now need to fetch the membership
-    # events of all the users.
-
-    filter_list = []
-    for user_id in allowed_user_ids:
-        filter_list.append((EventTypes.Member, user_id))
-    filter_list.append((EventTypes.RoomHistoryVisibility, ""))
-
-    state_filter = StateFilter.from_types(filter_list)
-    state_map = await _get_state_map(store, event, context, state_filter)
-
-    # Now we check whether the membership allows each user to see the event.
-    return {
-        user_id
-        for user_id in allowed_user_ids
-        if _check_membership(user_id, event, visibility, state_map, is_peeking).allowed
-    }
-
-
-async def _get_state_map(
-    store: DataStore, event: EventBase, context: EventContext, state_filter: StateFilter
-) -> StateMap[EventBase]:
-    """Helper function for getting a `StateMap[EventBase]` from an `EventContext`"""
-    state_map = await context.get_prev_state_ids(state_filter)
-
-    # Use events rather than event ids as content from the events are needed in
-    # _check_visibility
-    event_map = await store.get_events(state_map.values(), get_prev_content=False)
-
-    updated_state_map = {}
-    for state_key, event_id in state_map.items():
-        state_event = event_map.get(event_id)
-        if state_event:
-            updated_state_map[state_key] = state_event
-
-    if event.is_state():
-        current_state_key = (event.type, event.state_key)
-        # Add current event to updated_state_map, we need to do this here as it
-        # may not have been persisted to the db yet
-        updated_state_map[current_state_key] = event
-
-    return updated_state_map
-
-
-def _check_client_allowed_to_see_event(
-    user_id: str,
-    event: EventBase,
-    clock: Clock,
-    filter_send_to_client: bool,
-    is_peeking: bool,
-    always_include_ids: FrozenSet[str],
-    sender_ignored: bool,
-    retention_policy: RetentionPolicy,
-    state: Optional[StateMap[EventBase]],
-    sender_erased: bool,
-) -> Optional[EventBase]:
-    """Check with the given user is allowed to see the given event
-
-    See `filter_events_for_client` for details about args
-
-    Args:
-        user_id
-        event
-        clock
-        filter_send_to_client
-        is_peeking
-        always_include_ids
-        sender_ignored: Whether the user is ignoring the event sender
-        retention_policy: The retention policy of the room
-        state: The state at the event, unless its an outlier
-        sender_erased: Whether the event sender has been marked as "erased"
-
-    Returns:
-        None if the user cannot see this event at all
-
-        a redacted copy of the event if they can only see a redacted
-        version
-
-        the original event if they can see it as normal.
-    """
-    # Only run some checks if these events aren't about to be sent to clients. This is
-    # because, if this is not the case, we're probably only checking if the users can
-    # see events in the room at that point in the DAG, and that shouldn't be decided
-    # on those checks.
-    if filter_send_to_client:
-        if (
-            _check_filter_send_to_client(event, clock, retention_policy, sender_ignored)
-            == _CheckFilter.DENIED
-        ):
-            return None
-
-    if event.event_id in always_include_ids:
-        return event
-
-    # we need to handle outliers separately, since we don't have the room state.
-    if event.internal_metadata.outlier:
-        # Normally these can't be seen by clients, but we make an exception for
-        # for out-of-band membership events (eg, incoming invites, or rejections of
-        # said invite) for the user themselves.
-        if event.type == EventTypes.Member and event.state_key == user_id:
-            logger.debug("Returning out-of-band-membership event %s", event)
-            return event
-
-        return None
-
-    if state is None:
-        raise Exception("Missing state for non-outlier event")
-
-    # get the room_visibility at the time of the event.
-    visibility = get_effective_room_visibility_from_state(state)
-
-    # Check if the room has lax history visibility, allowing us to skip
-    # membership checks.
-    #
-    # We can only do this check if the sender has *not* been erased, as if they
-    # have we need to check the user's membership.
-    if (
-        not sender_erased
-        and _check_history_visibility(event, visibility, is_peeking)
-        == _CheckVisibility.ALLOWED
-    ):
-        return event
-
-    membership_result = _check_membership(user_id, event, visibility, state, is_peeking)
-    if not membership_result.allowed:
-        return None
-
-    # If the sender has been erased and the user was not joined at the time, we
-    # must only return the redacted form.
-    if sender_erased and not membership_result.joined:
-        event = prune_event(event)
-
-    return event
-
-
-@attr.s(frozen=True, slots=True, auto_attribs=True)
-class _CheckMembershipReturn:
-    "Return value of _check_membership"
-    allowed: bool
-    joined: bool
-
-
-def _check_membership(
-    user_id: str,
-    event: EventBase,
-    visibility: str,
-    state: StateMap[EventBase],
-    is_peeking: bool,
-) -> _CheckMembershipReturn:
-    """Check whether the user can see the event due to their membership
-
-    Returns:
-        True if they can, False if they can't, plus the membership of the user
-        at the event.
-    """
-    # If the event is the user's own membership event, use the 'most joined'
-    # membership
-    membership = None
-    if event.type == EventTypes.Member and event.state_key == user_id:
-        membership = event.content.get("membership", None)
-        if membership not in MEMBERSHIP_PRIORITY:
-            membership = "leave"
-
-        prev_content = event.unsigned.get("prev_content", {})
-        prev_membership = prev_content.get("membership", None)
-        if prev_membership not in MEMBERSHIP_PRIORITY:
-            prev_membership = "leave"
-
-        # Always allow the user to see their own leave events, otherwise
-        # they won't see the room disappear if they reject the invite
-        #
-        # (Note this doesn't work for out-of-band invite rejections, which don't
-        # have prev_state populated. They are handled above in the outlier code.)
-        if membership == "leave" and (
-            prev_membership == "join" or prev_membership == "invite"
-        ):
-            return _CheckMembershipReturn(True, membership == Membership.JOIN)
-
-        new_priority = MEMBERSHIP_PRIORITY.index(membership)
-        old_priority = MEMBERSHIP_PRIORITY.index(prev_membership)
-        if old_priority < new_priority:
-            membership = prev_membership
-
-    # otherwise, get the user's membership at the time of the event.
-    if membership is None:
-        membership_event = state.get((EventTypes.Member, user_id), None)
-        if membership_event:
-            membership = membership_event.membership
-
-    # if the user was a member of the room at the time of the event,
-    # they can see it.
-    if membership == Membership.JOIN:
-        return _CheckMembershipReturn(True, True)
-
-    # otherwise, it depends on the room visibility.
-
-    if visibility == HistoryVisibility.JOINED:
-        # we weren't a member at the time of the event, so we can't
-        # see this event.
-        return _CheckMembershipReturn(False, False)
-
-    elif visibility == HistoryVisibility.INVITED:
-        # user can also see the event if they were *invited* at the time
-        # of the event.
-        return _CheckMembershipReturn(membership == Membership.INVITE, False)
-
-    elif visibility == HistoryVisibility.SHARED and is_peeking:
-        # if the visibility is shared, users cannot see the event unless
-        # they have *subsequently* joined the room (or were members at the
-        # time, of course)
-        #
-        # XXX: if the user has subsequently joined and then left again,
-        # ideally we would share history up to the point they left. But
-        # we don't know when they left. We just treat it as though they
-        # never joined, and restrict access.
-        return _CheckMembershipReturn(False, False)
-
-    # The visibility is either shared or world_readable, and the user was
-    # not a member at the time. We allow it.
-    return _CheckMembershipReturn(True, False)
-
-
-class _CheckFilter(Enum):
-    MAYBE_ALLOWED = auto()
-    DENIED = auto()
-
-
-def _check_filter_send_to_client(
-    event: EventBase,
-    clock: Clock,
-    retention_policy: RetentionPolicy,
-    sender_ignored: bool,
-) -> _CheckFilter:
-    """Apply checks for sending events to client
-
-    Returns:
-        True if might be allowed to be sent to clients, False if definitely not.
-    """
-
-    if event.type == EventTypes.Dummy:
-        return _CheckFilter.DENIED
-
-    if not event.is_state() and sender_ignored:
-        return _CheckFilter.DENIED
-
-    # Until MSC2261 has landed we can't redact malicious alias events, so for
-    # now we temporarily filter out m.room.aliases entirely to mitigate
-    # abuse, while we spec a better solution to advertising aliases
-    # on rooms.
-    if event.type == EventTypes.Aliases:
-        return _CheckFilter.DENIED
-
-    # Don't try to apply the room's retention policy if the event is a state
-    # event, as MSC1763 states that retention is only considered for non-state
-    # events.
-    if not event.is_state():
-        max_lifetime = retention_policy.max_lifetime
-
-        if max_lifetime is not None:
-            oldest_allowed_ts = clock.time_msec() - max_lifetime
-
-            if event.origin_server_ts < oldest_allowed_ts:
-                return _CheckFilter.DENIED
-
-    return _CheckFilter.MAYBE_ALLOWED
-
-
-class _CheckVisibility(Enum):
-    ALLOWED = auto()
-    MAYBE_DENIED = auto()
-
-
-def _check_history_visibility(
-    event: EventBase, visibility: str, is_peeking: bool
-) -> _CheckVisibility:
-    """Check if event is allowed to be seen due to lax history visibility.
-
-    Returns:
-        True if user can definitely see the event, False if maybe not.
-    """
-    # Always allow history visibility events on boundaries. This is done
-    # by setting the effective visibility to the least restrictive
-    # of the old vs new.
-    if event.type == EventTypes.RoomHistoryVisibility:
-        prev_content = event.unsigned.get("prev_content", {})
-        prev_visibility = prev_content.get("history_visibility", None)
-
-        if prev_visibility not in VISIBILITY_PRIORITY:
-            prev_visibility = HistoryVisibility.SHARED
-
-        new_priority = VISIBILITY_PRIORITY.index(visibility)
-        old_priority = VISIBILITY_PRIORITY.index(prev_visibility)
-        if old_priority < new_priority:
-            visibility = prev_visibility
-
-    if visibility == HistoryVisibility.SHARED and not is_peeking:
-        return _CheckVisibility.ALLOWED
-    elif visibility == HistoryVisibility.WORLD_READABLE:
-        return _CheckVisibility.ALLOWED
-
-    return _CheckVisibility.MAYBE_DENIED
-
-
 def get_effective_room_visibility_from_state(state: StateMap[EventBase]) -> str:
    """Get the actual history vis, from a state map including the history_visibility event
+
    Handles missing and invalid history visibility events.
    """
    visibility_event = state.get(_HISTORY_VIS_KEY, None)
@@ -16,23 +16,13 @@ from typing import Dict, Optional, Set, Tuple, Union

 import frozendict

-from twisted.test.proto_helpers import MemoryReactor
-
-import synapse.rest.admin
-from synapse.api.constants import EventTypes, Membership
 from synapse.api.room_versions import RoomVersions
-from synapse.appservice import ApplicationService
 from synapse.events import FrozenEvent
 from synapse.push import push_rule_evaluator
 from synapse.push.push_rule_evaluator import PushRuleEvaluatorForEvent
-from synapse.rest.client import login, register, room
-from synapse.server import HomeServer
-from synapse.storage.databases.main.appservice import _make_exclusive_regex
 from synapse.types import JsonDict
-from synapse.util import Clock

 from tests import unittest
-from tests.test_utils.event_injection import create_event, inject_member_event


 class PushRuleEvaluatorTestCase(unittest.TestCase):
@@ -364,78 +354,3 @@ class PushRuleEvaluatorTestCase(unittest.TestCase):
            "event_type": "*.reaction",
        }
        self.assertTrue(evaluator.matches(condition, "@user:test", "foo"))
-
-
-class TestBulkPushRuleEvaluator(unittest.HomeserverTestCase):
-    """Tests for the bulk push rule evaluator"""
-
-    servlets = [
-        synapse.rest.admin.register_servlets_for_client_rest_resource,
-        login.register_servlets,
-        register.register_servlets,
-        room.register_servlets,
-    ]
-
-    def prepare(self, reactor: MemoryReactor, clock: Clock, homeserver: HomeServer):
-        # Define an application service so that we can register appservice users
-        self._service_token = "some_token"
-        self._service = ApplicationService(
-            self._service_token,
-            "as1",
-            "@as.sender:test",
-            namespaces={
-                "users": [
-                    {"regex": "@_as_.*:test", "exclusive": True},
-                    {"regex": "@as.sender:test", "exclusive": True},
-                ]
-            },
-            msc3202_transaction_extensions=True,
-        )
-        self.hs.get_datastores().main.services_cache = [self._service]
-        self.hs.get_datastores().main.exclusive_user_regex = _make_exclusive_regex(
-            [self._service]
-        )
-
-        self._as_user, _ = self.register_appservice_user(
-            "_as_user", self._service_token
-        )
-
-        self.evaluator = self.hs.get_bulk_push_rule_evaluator()
-
-    def test_ignore_appservice_users(self) -> None:
-        "Test that we don't generate push for appservice users"
-
-        user_id = self.register_user("user", "pass")
-        token = self.login("user", "pass")
-
-        room_id = self.helper.create_room_as(user_id, tok=token)
-        self.get_success(
-            inject_member_event(self.hs, room_id, self._as_user, Membership.JOIN)
-        )
-
-        event, context = self.get_success(
-            create_event(
-                self.hs,
-                type=EventTypes.Message,
-                room_id=room_id,
-                sender=user_id,
-                content={"body": "test", "msgtype": "m.text"},
-            )
-        )
-
-        # Assert the returned push rules do not contain the app service user
-        rules = self.get_success(self.evaluator._get_rules_for_event(event))
-        self.assertTrue(self._as_user not in rules)
-
-        # Assert that no push actions have been added to the staging table (the
-        # sender should not be pushed for the event)
-        users_with_push_actions = self.get_success(
-            self.hs.get_datastores().main.db_pool.simple_select_onecol(
-                table="event_push_actions_staging",
-                keyvalues={"event_id": event.event_id},
-                retcol="user_id",
-                desc="test_ignore_appservice_users",
-            )
-        )
-
-        self.assertEqual(len(users_with_push_actions), 0)
@@ -1579,8 +1579,8 @@ class RoomTestCase(unittest.HomeserverTestCase):
            access_token=self.admin_user_tok,
        )
        self.assertEqual(HTTPStatus.OK, channel.code, msg=channel.json_body)
-        self.assertEqual(room_id, channel.json_body["rooms"][0].get("room_id"))
-        self.assertEqual("ж", channel.json_body["rooms"][0].get("name"))
+        self.assertEqual(room_id, channel.json_body.get("rooms")[0].get("room_id"))
+        self.assertEqual("ж", channel.json_body.get("rooms")[0].get("name"))

    def test_single_room(self) -> None:
        """Test that a single room can be requested correctly"""
@@ -1488,7 +1488,7 @@ class UserRestTestCase(unittest.HomeserverTestCase):

        if channel.code != HTTPStatus.OK:
            raise HttpResponseException(
-                channel.code, channel.result["reason"], channel.result["body"]
+                channel.code, channel.result["reason"], channel.json_body
            )

        # Set monthly active users to the limit
@@ -949,7 +949,7 @@ class ThreepidEmailRestTestCase(unittest.HomeserverTestCase):
        client_secret: str,
        next_link: Optional[str] = None,
        expect_code: int = 200,
-    ) -> Optional[str]:
+    ) -> str:
        """Request a validation token to add an email address to a user's account

        Args:
@@ -959,8 +959,7 @@ class ThreepidEmailRestTestCase(unittest.HomeserverTestCase):
            expect_code: Expected return code of the call

        Returns:
-            The ID of the new threepid validation session, or None if the response
-            did not contain a session ID.
+            The ID of the new threepid validation session
        """
        body = {"client_secret": client_secret, "email": email, "send_attempt": 1}
        if next_link:
@@ -153,22 +153,18 @@ class ProfileTestCase(unittest.HomeserverTestCase):
        )
        self.assertEqual(channel.code, 400, channel.result)

-    def _get_displayname(self, name: Optional[str] = None) -> Optional[str]:
+    def _get_displayname(self, name: Optional[str] = None) -> str:
        channel = self.make_request(
            "GET", "/profile/%s/displayname" % (name or self.owner,)
        )
        self.assertEqual(channel.code, 200, channel.result)
-        # FIXME: If a user has no displayname set, Synapse returns 200 and omits a
-        # displayname from the response. This contradicts the spec, see #13137.
-        return channel.json_body.get("displayname")
+        return channel.json_body["displayname"]

-    def _get_avatar_url(self, name: Optional[str] = None) -> Optional[str]:
+    def _get_avatar_url(self, name: Optional[str] = None) -> str:
        channel = self.make_request(
            "GET", "/profile/%s/avatar_url" % (name or self.owner,)
        )
        self.assertEqual(channel.code, 200, channel.result)
-        # FIXME: If a user has no avatar set, Synapse returns 200 and omits an
-        # avatar_url from the response. This contradicts the spec, see #13137.
        return channel.json_body.get("avatar_url")

    @unittest.override_config({"max_avatar_size": 50})
@@ -800,7 +800,7 @@ class RelationPaginationTestCase(BaseRelationsTestCase):
            )
            expected_event_ids.append(channel.json_body["event_id"])

-        prev_token: Optional[str] = ""
+        prev_token = ""
        found_event_ids: List[str] = []
        for _ in range(20):
            from_token = ""
@@ -22,7 +22,7 @@ from typing import Any, Dict, Iterable, List, Optional, Tuple, Union
 from unittest.mock import Mock, call
 from urllib import parse as urlparse

-from parameterized import param, parameterized
+# `Literal` appears with Python 3.8.
 from typing_extensions import Literal

 from twisted.test.proto_helpers import MemoryReactor
@@ -708,21 +708,6 @@ class RoomsCreateTestCase(RoomBase):

        self.assertEqual(200, channel.code, channel.result)
        self.assertTrue("room_id" in channel.json_body)
-        assert channel.resource_usage is not None
-        self.assertEqual(37, channel.resource_usage.db_txn_count)
-
-    def test_post_room_initial_state(self) -> None:
-        # POST with initial_state config key, expect new room id
-        channel = self.make_request(
-            "POST",
-            "/createRoom",
-            b'{"initial_state":[{"type": "m.bridge", "content": {}}]}',
-        )
-
-        self.assertEqual(200, channel.code, channel.result)
-        self.assertTrue("room_id" in channel.json_body)
-        assert channel.resource_usage is not None
-        self.assertEqual(41, channel.resource_usage.db_txn_count)

    def test_post_room_visibility_key(self) -> None:
        # POST with visibility config key, expect new room id
@@ -830,14 +815,14 @@ class RoomsCreateTestCase(RoomBase):
        In this test, we use the more recent API in which callbacks return a `Union[Codes, Literal["NOT_SPAM"]]`.
        """

-        async def user_may_join_room_codes(
+        async def user_may_join_room(
            mxid: str,
            room_id: str,
            is_invite: bool,
        ) -> Codes:
            return Codes.CONSENT_NOT_GIVEN

-        join_mock = Mock(side_effect=user_may_join_room_codes)
+        join_mock = Mock(side_effect=user_may_join_room)
        self.hs.get_spam_checker()._user_may_join_room_callbacks.append(join_mock)

        channel = self.make_request(
@@ -849,25 +834,6 @@ class RoomsCreateTestCase(RoomBase):

        self.assertEqual(join_mock.call_count, 0)

-        # Now change the return value of the callback to deny any join. Since we're
-        # creating the room, despite the return value, we should be able to join.
-        async def user_may_join_room_tuple(
-            mxid: str,
-            room_id: str,
-            is_invite: bool,
-        ) -> Tuple[Codes, dict]:
-            return Codes.INCOMPATIBLE_ROOM_VERSION, {}
-
-        join_mock.side_effect = user_may_join_room_tuple
-
-        channel = self.make_request(
-            "POST",
-            "/createRoom",
-            {},
-        )
-        self.assertEqual(channel.code, 200, channel.json_body)
-        self.assertEqual(join_mock.call_count, 0)
-

 class RoomTopicTestCase(RoomBase):
    """Tests /rooms/$room_id/topic REST events."""
@@ -1147,15 +1113,13 @@ class RoomJoinTestCase(RoomBase):
        """

        # Register a dummy callback. Make it allow all room joins for now.
-        return_value: Union[
-            Literal["NOT_SPAM"], Tuple[Codes, dict], Codes
-        ] = synapse.module_api.NOT_SPAM
+        return_value: Union[Literal["NOT_SPAM"], Codes] = synapse.module_api.NOT_SPAM

        async def user_may_join_room(
            userid: str,
            room_id: str,
            is_invited: bool,
-        ) -> Union[Literal["NOT_SPAM"], Tuple[Codes, dict], Codes]:
+        ) -> Union[Literal["NOT_SPAM"], Codes]:
            return return_value

        # `spec` argument is needed for this function mock to have `__qualname__`, which
@@ -1199,28 +1163,8 @@ class RoomJoinTestCase(RoomBase):
        )

        # Now make the callback deny all room joins, and check that a join actually fails.
-        # We pick an arbitrary Codes rather than the default `Codes.FORBIDDEN`.
        return_value = Codes.CONSENT_NOT_GIVEN
-        self.helper.invite(self.room3, self.user1, self.user2, tok=self.tok1)
-        self.helper.join(
-            self.room3,
-            self.user2,
-            expect_code=403,
-            expect_errcode=return_value,
-            tok=self.tok2,
-        )
-
-        # Now make the callback deny all room joins, and check that a join actually fails.
-        # As above, with the experimental extension that lets us return dictionaries.
-        return_value = (Codes.BAD_ALIAS, {"another_field": "12345"})
-        self.helper.join(
-            self.room3,
-            self.user2,
-            expect_code=403,
-            expect_errcode=return_value[0],
-            tok=self.tok2,
-            expect_additional_fields=return_value[1],
-        )
+        self.helper.join(self.room3, self.user2, expect_code=403, tok=self.tok2)


 class RoomJoinRatelimitTestCase(RoomBase):
@@ -1370,97 +1314,6 @@ class RoomMessagesTestCase(RoomBase):
        channel = self.make_request("PUT", path, content)
        self.assertEqual(200, channel.code, msg=channel.result["body"])

-    @parameterized.expand(
-        [
-            # Allow
-            param(
-                name="NOT_SPAM", value="NOT_SPAM", expected_code=200, expected_fields={}
-            ),
-            param(name="False", value=False, expected_code=200, expected_fields={}),
-            # Block
-            param(
-                name="scalene string",
-                value="ANY OTHER STRING",
-                expected_code=403,
-                expected_fields={"errcode": "M_FORBIDDEN"},
-            ),
-            param(
-                name="True",
-                value=True,
-                expected_code=403,
-                expected_fields={"errcode": "M_FORBIDDEN"},
-            ),
-            param(
-                name="Code",
-                value=Codes.LIMIT_EXCEEDED,
-                expected_code=403,
-                expected_fields={"errcode": "M_LIMIT_EXCEEDED"},
-            ),
-            param(
-                name="Tuple",
-                value=(Codes.SERVER_NOT_TRUSTED, {"additional_field": "12345"}),
-                expected_code=403,
-                expected_fields={
-                    "errcode": "M_SERVER_NOT_TRUSTED",
-                    "additional_field": "12345",
-                },
-            ),
-        ]
-    )
-    def test_spam_checker_check_event_for_spam(
-        self,
-        name: str,
-        value: Union[str, bool, Codes, Tuple[Codes, JsonDict]],
-        expected_code: int,
-        expected_fields: dict,
-    ) -> None:
-        class SpamCheck:
-            mock_return_value: Union[
-                str, bool, Codes, Tuple[Codes, JsonDict], bool
-            ] = "NOT_SPAM"
-            mock_content: Optional[JsonDict] = None
-
-            async def check_event_for_spam(
-                self,
-                event: synapse.events.EventBase,
-            ) -> Union[str, Codes, Tuple[Codes, JsonDict], bool]:
-                self.mock_content = event.content
-                return self.mock_return_value
-
-        spam_checker = SpamCheck()
-
-        self.hs.get_spam_checker()._check_event_for_spam_callbacks.append(
-            spam_checker.check_event_for_spam
-        )
-
-        # Inject `value` as mock_return_value
-        spam_checker.mock_return_value = value
-        path = "/rooms/%s/send/m.room.message/check_event_for_spam_%s" % (
-            urlparse.quote(self.room_id),
-            urlparse.quote(name),
-        )
-        body = "test-%s" % name
-        content = '{"body":"%s","msgtype":"m.text"}' % body
-        channel = self.make_request("PUT", path, content)
-
-        # Check that the callback has witnessed the correct event.
-        self.assertIsNotNone(spam_checker.mock_content)
-        if (
-            spam_checker.mock_content is not None
-        ):  # Checked just above, but mypy doesn't know about that.
-            self.assertEqual(
-                spam_checker.mock_content["body"], body, spam_checker.mock_content
-            )
-
-        # Check that we have the correct result.
-        self.assertEqual(expected_code, channel.code, msg=channel.result["body"])
-        for expected_key, expected_value in expected_fields.items():
-            self.assertEqual(
-                channel.json_body.get(expected_key, None),
-                expected_value,
-                "Field %s absent or invalid " % expected_key,
-            )
-

 class RoomPowerLevelOverridesTestCase(RoomBase):
    """Tests that the power levels can be overridden with server config."""
@@ -3382,8 +3235,7 @@ class ThreepidInviteTestCase(unittest.HomeserverTestCase):
        make_invite_mock.assert_called_once()

        # Now change the return value of the callback to deny any invite and test that
-        # we can't send the invite. We pick an arbitrary error code to be able to check
-        # that the same code has been returned
+        # we can't send the invite.
        mock.return_value = make_awaitable(Codes.CONSENT_NOT_GIVEN)
        channel = self.make_request(
            method="POST",
@@ -3397,27 +3249,6 @@ class ThreepidInviteTestCase(unittest.HomeserverTestCase):
            access_token=self.tok,
        )
        self.assertEqual(channel.code, 403)
-        self.assertEqual(channel.json_body["errcode"], Codes.CONSENT_NOT_GIVEN)
-
-        # Also check that it stopped before calling _make_and_store_3pid_invite.
-        make_invite_mock.assert_called_once()
-
-        # Run variant with `Tuple[Codes, dict]`.
-        mock.return_value = make_awaitable((Codes.EXPIRED_ACCOUNT, {"field": "value"}))
-        channel = self.make_request(
-            method="POST",
-            path="/rooms/" + self.room_id + "/invite",
-            content={
-                "id_server": "example.com",
-                "id_access_token": "sometoken",
-                "medium": "email",
-                "address": email_to_invite,
-            },
-            access_token=self.tok,
-        )
-        self.assertEqual(channel.code, 403)
-        self.assertEqual(channel.json_body["errcode"], Codes.EXPIRED_ACCOUNT)
-        self.assertEqual(channel.json_body["field"], "value")

        # Also check that it stopped before calling _make_and_store_3pid_invite.
        make_invite_mock.assert_called_once()
@@ -41,7 +41,6 @@ from twisted.web.resource import Resource
 from twisted.web.server import Site

 from synapse.api.constants import Membership
-from synapse.api.errors import Codes
 from synapse.server import HomeServer
 from synapse.types import JsonDict

@@ -172,8 +171,6 @@ class RestHelper:
        expect_code: int = HTTPStatus.OK,
        tok: Optional[str] = None,
        appservice_user_id: Optional[str] = None,
-        expect_errcode: Optional[Codes] = None,
-        expect_additional_fields: Optional[dict] = None,
    ) -> None:
        self.change_membership(
            room=room,
@@ -183,8 +180,6 @@ class RestHelper:
            appservice_user_id=appservice_user_id,
            membership=Membership.JOIN,
            expect_code=expect_code,
-            expect_errcode=expect_errcode,
-            expect_additional_fields=expect_additional_fields,
        )

    def knock(
@@ -268,7 +263,6 @@ class RestHelper:
        appservice_user_id: Optional[str] = None,
        expect_code: int = HTTPStatus.OK,
        expect_errcode: Optional[str] = None,
-        expect_additional_fields: Optional[dict] = None,
    ) -> None:
        """
        Send a membership state event into a room.
@@ -329,21 +323,6 @@ class RestHelper:
                channel.result["body"],
            )

-        if expect_additional_fields is not None:
-            for expect_key, expect_value in expect_additional_fields.items():
-                assert expect_key in channel.json_body, "Expected field %s, got %s" % (
-                    expect_key,
-                    channel.json_body,
-                )
-                assert (
-                    channel.json_body[expect_key] == expect_value
-                ), "Expected: %s at %s, got: %s, resp: %s" % (
-                    expect_value,
-                    expect_key,
-                    channel.json_body[expect_key],
-                    channel.json_body,
-                )
-
        self.auth_user_id = temp_id

    def send(
@@ -23,13 +23,11 @@ from urllib import parse
 import attr
 from parameterized import parameterized, parameterized_class
 from PIL import Image as Image
-from typing_extensions import Literal

 from twisted.internet import defer
 from twisted.internet.defer import Deferred
 from twisted.test.proto_helpers import MemoryReactor

-from synapse.api.errors import Codes
 from synapse.events import EventBase
 from synapse.events.spamcheck import load_legacy_spam_checkers
 from synapse.logging.context import make_deferred_yieldable
@@ -572,11 +570,9 @@ class MediaRepoTests(unittest.HomeserverTestCase):
        )


-class TestSpamCheckerLegacy:
+class TestSpamChecker:
    """A spam checker module that rejects all media that includes the bytes
    `evil`.
-
-    Uses the legacy Spam-Checker API.
    """

    def __init__(self, config: Dict[str, Any], api: ModuleApi) -> None:
@@ -617,7 +613,7 @@ class TestSpamCheckerLegacy:
        return b"evil" in buf.getvalue()


-class SpamCheckerTestCaseLegacy(unittest.HomeserverTestCase):
+class SpamCheckerTestCase(unittest.HomeserverTestCase):
    servlets = [
        login.register_servlets,
        admin.register_servlets,
@@ -641,8 +637,7 @@ class SpamCheckerTestCaseLegacy(unittest.HomeserverTestCase):
            {
                "spam_checker": [
                    {
-                        "module": TestSpamCheckerLegacy.__module__
-                        + ".TestSpamCheckerLegacy",
+                        "module": TestSpamChecker.__module__ + ".TestSpamChecker",
                        "config": {},
                    }
                ]
@@ -667,62 +662,3 @@ class SpamCheckerTestCaseLegacy(unittest.HomeserverTestCase):
        self.helper.upload_media(
            self.upload_resource, data, tok=self.tok, expect_code=400
        )
-
-
-EVIL_DATA = b"Some evil data"
-EVIL_DATA_EXPERIMENT = b"Some evil data to trigger the experimental tuple API"
-
-
-class SpamCheckerTestCase(unittest.HomeserverTestCase):
-    servlets = [
-        login.register_servlets,
-        admin.register_servlets,
-    ]
-
-    def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None:
-        self.user = self.register_user("user", "pass")
-        self.tok = self.login("user", "pass")
-
-        # Allow for uploading and downloading to/from the media repo
-        self.media_repo = hs.get_media_repository_resource()
-        self.download_resource = self.media_repo.children[b"download"]
-        self.upload_resource = self.media_repo.children[b"upload"]
-
-        hs.get_module_api().register_spam_checker_callbacks(
-            check_media_file_for_spam=self.check_media_file_for_spam
-        )
-
-    async def check_media_file_for_spam(
-        self, file_wrapper: ReadableFileWrapper, file_info: FileInfo
-    ) -> Union[Codes, Literal["NOT_SPAM"]]:
-        buf = BytesIO()
-        await file_wrapper.write_chunks_to(buf.write)
-
-        if buf.getvalue() == EVIL_DATA:
-            return Codes.FORBIDDEN
-        elif buf.getvalue() == EVIL_DATA_EXPERIMENT:
-            return (Codes.FORBIDDEN, {})
-        else:
-            return "NOT_SPAM"
-
-    def test_upload_innocent(self) -> None:
-        """Attempt to upload some innocent data that should be allowed."""
-        self.helper.upload_media(
-            self.upload_resource, SMALL_PNG, tok=self.tok, expect_code=200
-        )
-
-    def test_upload_ban(self) -> None:
-        """Attempt to upload some data that includes bytes "evil", which should
-        get rejected by the spam checker.
-        """
-
-        self.helper.upload_media(
-            self.upload_resource, EVIL_DATA, tok=self.tok, expect_code=400
-        )
-
-        self.helper.upload_media(
-            self.upload_resource,
-            EVIL_DATA_EXPERIMENT,
-            tok=self.tok,
-            expect_code=400,
-        )
@@ -43,7 +43,6 @@ from twisted.internet.defer import Deferred, fail, maybeDeferred, succeed
 from twisted.internet.error import DNSLookupError
 from twisted.internet.interfaces import (
    IAddress,
-    IConsumer,
    IHostnameResolver,
    IProtocol,
    IPullProducer,
@@ -54,7 +53,11 @@ from twisted.internet.interfaces import (
    ITransport,
 )
 from twisted.python.failure import Failure
-from twisted.test.proto_helpers import AccumulatingProtocol, MemoryReactorClock
+from twisted.test.proto_helpers import (
+    AccumulatingProtocol,
+    MemoryReactor,
+    MemoryReactorClock,
+)
 from twisted.web.http_headers import Headers
 from twisted.web.resource import IResource
 from twisted.web.server import Request, Site
@@ -93,7 +96,6 @@ class TimedOutException(Exception):
    """


-@implementer(IConsumer)
@attr.s(auto_attribs=True)
 class FakeChannel:
    """
@@ -102,7 +104,7 @@ class FakeChannel:
    """

    site: Union[Site, "FakeSite"]
-    _reactor: MemoryReactorClock
+    _reactor: MemoryReactor
    result: dict = attr.Factory(dict)
    _ip: str = "127.0.0.1"
    _producer: Optional[Union[IPullProducer, IPushProducer]] = None
@@ -120,7 +122,7 @@ class FakeChannel:
        self._request = request

    @property
-    def json_body(self) -> JsonDict:
+    def json_body(self):
        return json.loads(self.text_body)

    @property
@@ -138,7 +140,7 @@ class FakeChannel:
        return self.result.get("done", False)

    @property
-    def code(self) -> int:
+    def code(self):
        if not self.result:
            raise Exception("No result yet.")
        return int(self.result["code"])
@@ -158,7 +160,7 @@ class FakeChannel:
        self.result["reason"] = reason
        self.result["headers"] = headers

-    def write(self, content: bytes) -> None:
+    def write(self, content):
        assert isinstance(content, bytes), "Should be bytes! " + repr(content)

        if "body" not in self.result:
@@ -166,16 +168,11 @@ class FakeChannel:

        self.result["body"] += content

-    # Type ignore: mypy doesn't like the fact that producer isn't an IProducer.
-    def registerProducer(  # type: ignore[override]
-        self,
-        producer: Union[IPullProducer, IPushProducer],
-        streaming: bool,
-    ) -> None:
+    def registerProducer(self, producer, streaming):
        self._producer = producer
        self.producerStreaming = streaming

-        def _produce() -> None:
+        def _produce():
            if self._producer:
                self._producer.resumeProducing()
                self._reactor.callLater(0.1, _produce)
@@ -183,32 +180,31 @@ class FakeChannel:
        if not streaming:
            self._reactor.callLater(0.0, _produce)

-    def unregisterProducer(self) -> None:
+    def unregisterProducer(self):
        if self._producer is None:
            return

        self._producer = None

-    def requestDone(self, _self: Request) -> None:
+    def requestDone(self, _self):
        self.result["done"] = True
        if isinstance(_self, SynapseRequest):
-            assert _self.logcontext is not None
            self.resource_usage = _self.logcontext.get_resource_usage()

-    def getPeer(self) -> IAddress:
+    def getPeer(self):
        # We give an address so that getClientAddress/getClientIP returns a non null entry,
        # causing us to record the MAU
        return address.IPv4Address("TCP", self._ip, 3423)

-    def getHost(self) -> IAddress:
+    def getHost(self):
        # this is called by Request.__init__ to configure Request.host.
        return address.IPv4Address("TCP", "127.0.0.1", 8888)

-    def isSecure(self) -> bool:
+    def isSecure(self):
        return False

    @property
-    def transport(self) -> "FakeChannel":
+    def transport(self):
        return self

    def await_result(self, timeout_ms: int = 1000) -> None:
@@ -834,6 +830,7 @@ def setup_test_homeserver(

    # Mock TLS
    hs.tls_server_context_factory = Mock()
+    hs.tls_client_options_factory = Mock()

    hs.setup()
    if homeserver_to_use == TestHomeServer:
@@ -148,12 +148,12 @@ class EventPushActionsStoreTestCase(HomeserverTestCase):
        _assert_counts(0, 0)
        _inject_actions(1, PlAIN_NOTIF)
        _assert_counts(1, 0)
-        _rotate(1)
+        _rotate(2)
        _assert_counts(1, 0)

        _inject_actions(3, PlAIN_NOTIF)
        _assert_counts(2, 0)
-        _rotate(3)
+        _rotate(4)
        _assert_counts(2, 0)

        _inject_actions(5, PlAIN_NOTIF)
@@ -164,7 +164,7 @@ class EventPushActionsStoreTestCase(HomeserverTestCase):
        _assert_counts(0, 0)

        _inject_actions(6, PlAIN_NOTIF)
-        _rotate(6)
+        _rotate(7)
        _assert_counts(1, 0)

        self.get_success(
@@ -180,13 +180,13 @@ class EventPushActionsStoreTestCase(HomeserverTestCase):

        _inject_actions(8, HIGHLIGHT)
        _assert_counts(1, 1)
-        _rotate(8)
+        _rotate(9)
        _assert_counts(1, 1)

        # Check that adding another notification and rotating after highlight
        # works.
        _inject_actions(10, PlAIN_NOTIF)
-        _rotate(10)
+        _rotate(11)
        _assert_counts(2, 1)

        # Check that sending read receipts at different points results in the
@@ -196,13 +196,6 @@ class EventPushActionsStoreTestCase(HomeserverTestCase):
        _mark_read(10, 10)
        _assert_counts(0, 0)

-        _inject_actions(11, HIGHLIGHT)
-        _assert_counts(1, 1)
-        _mark_read(11, 11)
-        _assert_counts(0, 0)
-        _rotate(11)
-        _assert_counts(0, 0)
-
    def test_find_first_stream_ordering_after_ts(self) -> None:
        def add_event(so: int, ts: int) -> None:
            self.get_success(
@@ -12,6 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.

+from synapse.api.constants import EventTypes
 from synapse.api.room_versions import RoomVersions
 from synapse.types import RoomAlias, RoomID, UserID

@@ -64,3 +65,71 @@ class RoomStoreTestCase(HomeserverTestCase):
        self.assertIsNone(
            (self.get_success(self.store.get_room_with_stats("!uknown:test"))),
        )
+
+
+class RoomEventsStoreTestCase(HomeserverTestCase):
+    def prepare(self, reactor, clock, hs):
+        # Room events need the full datastore, for persist_event() and
+        # get_room_state()
+        self.store = hs.get_datastores().main
+        self._storage_controllers = hs.get_storage_controllers()
+        self.event_factory = hs.get_event_factory()
+
+        self.room = RoomID.from_string("!abcde:test")
+
+        self.get_success(
+            self.store.store_room(
+                self.room.to_string(),
+                room_creator_user_id="@creator:text",
+                is_public=True,
+                room_version=RoomVersions.V1,
+            )
+        )
+
+    def inject_room_event(self, **kwargs):
+        self.get_success(
+            self._storage_controllers.persistence.persist_event(
+                self.event_factory.create_event(room_id=self.room.to_string(), **kwargs)
+            )
+        )
+
+    def STALE_test_room_name(self):
+        name = "A-Room-Name"
+
+        self.inject_room_event(
+            etype=EventTypes.Name, name=name, content={"name": name}, depth=1
+        )
+
+        state = self.get_success(
+            self._storage_controllers.state.get_current_state(
+                room_id=self.room.to_string()
+            )
+        )
+
+        self.assertEqual(1, len(state))
+        self.assertObjectHasAttributes(
+            {"type": "m.room.name", "room_id": self.room.to_string(), "name": name},
+            state[0],
+        )
+
+    def STALE_test_room_topic(self):
+        topic = "A place for things"
+
+        self.inject_room_event(
+            etype=EventTypes.Topic, topic=topic, content={"topic": topic}, depth=1
+        )
+
+        state = self.get_success(
+            self._storage_controllers.state.get_current_state(
+                room_id=self.room.to_string()
+            )
+        )
+
+        self.assertEqual(1, len(state))
+        self.assertObjectHasAttributes(
+            {"type": "m.room.topic", "room_id": self.room.to_string(), "topic": topic},
+            state[0],
+        )
+
+    # Not testing the various 'level' methods for now because there's lots
+    # of them and need coalescing; see JIRA SPEC-11
@@ -195,8 +195,6 @@ class StateTestCase(unittest.TestCase):
                "get_state_resolution_handler",
                "get_account_validity_handler",
                "get_macaroon_generator",
-                "get_instance_name",
-                "get_simple_http_client",
                "hostname",
            ]
        )
@@ -15,17 +15,12 @@

 import atexit
 import os
-from typing import Any, Callable, Dict, List, Tuple, Union, overload
-
-import attr
-from typing_extensions import Literal, ParamSpec

 from synapse.api.constants import EventTypes
 from synapse.api.room_versions import RoomVersions
 from synapse.config.homeserver import HomeServerConfig
 from synapse.config.server import DEFAULT_ROOM_VERSION
 from synapse.logging.context import current_context, set_current_context
-from synapse.server import HomeServer
 from synapse.storage.database import LoggingDatabaseConnection
 from synapse.storage.engines import create_engine
 from synapse.storage.prepare_database import prepare_database
@@ -55,11 +50,12 @@ SQLITE_PERSIST_DB = os.environ.get("SYNAPSE_TEST_PERSIST_SQLITE_DB") is not None
 POSTGRES_DBNAME_FOR_INITIAL_CREATE = "postgres"


-def setupdb() -> None:
+def setupdb():
    # If we're using PostgreSQL, set up the db once
    if USE_POSTGRES_FOR_TESTS:
        # create a PostgresEngine
        db_engine = create_engine({"name": "psycopg2", "args": {}})
+
        # connect to postgres to create the base database.
        db_conn = db_engine.module.connect(
            user=POSTGRES_USER,
@@ -68,7 +64,7 @@ def setupdb() -> None:
            password=POSTGRES_PASSWORD,
            dbname=POSTGRES_DBNAME_FOR_INITIAL_CREATE,
        )
-        db_engine.attempt_to_set_autocommit(db_conn, autocommit=True)
+        db_conn.autocommit = True
        cur = db_conn.cursor()
        cur.execute("DROP DATABASE IF EXISTS %s;" % (POSTGRES_BASE_DB,))
        cur.execute(
@@ -86,11 +82,11 @@ def setupdb() -> None:
            port=POSTGRES_PORT,
            password=POSTGRES_PASSWORD,
        )
-        logging_conn = LoggingDatabaseConnection(db_conn, db_engine, "tests")
-        prepare_database(logging_conn, db_engine, None)
-        logging_conn.close()
+        db_conn = LoggingDatabaseConnection(db_conn, db_engine, "tests")
+        prepare_database(db_conn, db_engine, None)
+        db_conn.close()

-        def _cleanup() -> None:
+        def _cleanup():
            db_conn = db_engine.module.connect(
                user=POSTGRES_USER,
                host=POSTGRES_HOST,
@@ -98,7 +94,7 @@ def setupdb() -> None:
                password=POSTGRES_PASSWORD,
                dbname=POSTGRES_DBNAME_FOR_INITIAL_CREATE,
            )
-            db_engine.attempt_to_set_autocommit(db_conn, autocommit=True)
+            db_conn.autocommit = True
            cur = db_conn.cursor()
            cur.execute("DROP DATABASE IF EXISTS %s;" % (POSTGRES_BASE_DB,))
            cur.close()
@@ -107,19 +103,7 @@ def setupdb() -> None:
        atexit.register(_cleanup)


-@overload
-def default_config(name: str, parse: Literal[False] = ...) -> Dict[str, object]:
-    ...
-
-
-@overload
-def default_config(name: str, parse: Literal[True]) -> HomeServerConfig:
-    ...
-
-
-def default_config(
-    name: str, parse: bool = False
-) -> Union[Dict[str, object], HomeServerConfig]:
+def default_config(name, parse=False):
    """
    Create a reasonable test config.
    """
@@ -197,122 +181,90 @@ def default_config(
    return config_dict


-def mock_getRawHeaders(headers=None):  # type: ignore[no-untyped-def]
+def mock_getRawHeaders(headers=None):
    headers = headers if headers is not None else {}

-    def getRawHeaders(name, default=None):  # type: ignore[no-untyped-def]
-        # If the requested header is present, the real twisted function returns
-        # List[str] if name is a str and List[bytes] if name is a bytes.
-        # This mock doesn't support that behaviour.
-        # Fortunately, none of the current callers of mock_getRawHeaders() provide a
-        # headers dict, so we don't encounter this discrepancy in practice.
+    def getRawHeaders(name, default=None):
        return headers.get(name, default)

    return getRawHeaders


-P = ParamSpec("P")
-
-
-@attr.s(slots=True, auto_attribs=True)
-class Timer:
-    absolute_time: float
-    callback: Callable[[], None]
-    expired: bool
-
-
-# TODO: Make this generic over a ParamSpec?
-@attr.s(slots=True, auto_attribs=True)
-class Looper:
-    func: Callable[..., Any]
-    interval: float  # seconds
-    last: float
-    args: Tuple[object, ...]
-    kwargs: Dict[str, object]
-
-
 class MockClock:
-    now = 1000.0
+    now = 1000

-    def __init__(self) -> None:
-        # Timers in no particular order
-        self.timers: List[Timer] = []
-        self.loopers: List[Looper] = []
+    def __init__(self):
+        # list of lists of [absolute_time, callback, expired] in no particular
+        # order
+        self.timers = []
+        self.loopers = []

-    def time(self) -> float:
+    def time(self):
        return self.now

-    def time_msec(self) -> int:
-        return int(self.time() * 1000)
+    def time_msec(self):
+        return self.time() * 1000

-    def call_later(
-        self,
-        delay: float,
-        callback: Callable[P, object],
-        *args: P.args,
-        **kwargs: P.kwargs,
-    ) -> Timer:
+    def call_later(self, delay, callback, *args, **kwargs):
        ctx = current_context()

-        def wrapped_callback() -> None:
+        def wrapped_callback():
            set_current_context(ctx)
            callback(*args, **kwargs)

-        t = Timer(self.now + delay, wrapped_callback, False)
+        t = [self.now + delay, wrapped_callback, False]
        self.timers.append(t)

        return t

-    def looping_call(
-        self,
-        function: Callable[P, object],
-        interval: float,
-        *args: P.args,
-        **kwargs: P.kwargs,
-    ) -> None:
-        # This type-ignore should be redundant once we use a mypy release with
-        # https://github.com/python/mypy/pull/12668.
-        self.loopers.append(Looper(function, interval / 1000.0, self.now, args, kwargs))  # type: ignore[arg-type]
+    def looping_call(self, function, interval, *args, **kwargs):
+        self.loopers.append([function, interval / 1000.0, self.now, args, kwargs])

-    def cancel_call_later(self, timer: Timer, ignore_errs: bool = False) -> None:
-        if timer.expired:
+    def cancel_call_later(self, timer, ignore_errs=False):
+        if timer[2]:
            if not ignore_errs:
                raise Exception("Cannot cancel an expired timer")

-        timer.expired = True
+        timer[2] = True
        self.timers = [t for t in self.timers if t != timer]

    # For unit testing
-    def advance_time(self, secs: float) -> None:
+    def advance_time(self, secs):
        self.now += secs

        timers = self.timers
        self.timers = []

        for t in timers:
-            if t.expired:
+            time, callback, expired = t
+
+            if expired:
                raise Exception("Timer already expired")

-            if self.now >= t.absolute_time:
-                t.expired = True
-                t.callback()
+            if self.now >= time:
+                t[2] = True
+                callback()
            else:
                self.timers.append(t)

        for looped in self.loopers:
-            if looped.last + looped.interval < self.now:
-                looped.func(*looped.args, **looped.kwargs)
-                looped.last = self.now
+            func, interval, last, args, kwargs = looped
+            if last + interval < self.now:
+                func(*args, **kwargs)
+                looped[2] = self.now

-    def advance_time_msec(self, ms: float) -> None:
+    def advance_time_msec(self, ms):
        self.advance_time(ms / 1000.0)

+    def time_bound_deferred(self, d, *args, **kwargs):
+        # We don't bother timing things out for now.
+        return d

-async def create_room(hs: HomeServer, room_id: str, creator_id: str) -> None:
+
+async def create_room(hs, room_id: str, creator_id: str):
    """Creates and persist a creation event for the given room"""

    persistence_store = hs.get_storage_controllers().persistence
-    assert persistence_store is not None
    store = hs.get_datastores().main
    event_builder_factory = hs.get_event_builder_factory()
    event_creation_handler = hs.get_event_creation_handler()
Author	SHA1	Message	Date
Olivier Wilkinson (reivilibre)	c747b4b37d	AAAA#	2022-07-05 17:26:12 +01:00
Olivier Wilkinson (reivilibre)	7094bcf251	DBG cache all	2022-07-05 12:40:26 +01:00
Olivier Wilkinson (reivilibre)	5af3ce8bf8	DBG cache	2022-07-05 12:31:50 +01:00
Olivier Wilkinson (reivilibre)	7fd218ad8b	DBG retry until fail	2022-07-05 12:18:26 +01:00
Olivier Wilkinson (reivilibre)	30627554d4	DBG set debug	2022-07-05 12:18:07 +01:00
Olivier Wilkinson (reivilibre)	3c41609a9e	DBG limit	2022-07-05 11:59:48 +01:00
Olivier Wilkinson (reivilibre)	18695c7631	DBG	2022-07-05 11:37:14 +01:00
Olivier Wilkinson (reivilibre)	c76da0d948	TMP flake debug code	2022-07-05 11:25:57 +01:00
Olivier Wilkinson (reivilibre)	3f203e1b10	Newsfile Signed-off-by: Olivier Wilkinson (reivilibre) <oliverw@matrix.org>	2022-07-04 12:01:09 +01:00
Olivier Wilkinson (reivilibre)	ae7f8cccc2	Give -f a long option --fast as it's confusing otherwise (and I might remember it better)	2022-07-04 12:01:09 +01:00
Olivier Wilkinson (reivilibre)	37112f76f7	Add a --build-only argument to complement.sh	2022-07-04 12:01:09 +01:00
Olivier Wilkinson (reivilibre)	f61931fb8a	Newsfile Signed-off-by: Olivier Wilkinson (reivilibre) <oliverw@matrix.org>	2022-07-01 16:55:14 +01:00
Olivier Wilkinson (reivilibre)	ae1712dd71	Factor out Complement checking-out	2022-07-01 16:55:14 +01:00
Olivier Wilkinson (reivilibre)	c1229e0218	Factor out some common commands into a script	2022-07-01 16:55:14 +01:00
				`@@ -0,0 +1 @@`
				Add an explanation of the `--report-stats` argument to the docs.
				`@@ -0,0 +1 @@`
				`Implement [MSC3827](https://github.com/matrix-org/matrix-spec-proposals/pull/3827): Filtering of /publicRooms by room type.`
				`@@ -0,0 +1,3 @@`
				`Clean up references to sample configuration and redirect users to the configuration manual instead.`
				`@@ -0,0 +1 @@`
				`Enable Complement testing in the 'Twisted Trunk' CI runs.`
				`@@ -0,0 +1 @@`
				`Add documentation for anonymised homeserver statistics collection.`
				`@@ -0,0 +1 @@`
				Raise a `DependencyError` on missing dependencies instead of a `ConfigError`.
				`@@ -0,0 +1 @@`
				Fix wrong section header for `allow_public_rooms_over_federation` in the homeserver config documentation.
				`@@ -0,0 +1 @@`
				Reduce DB usage of `/sync` when a large number of unread messages have recently been sent in a room.
				`@@ -0,0 +1 @@`
				`Add a rate limit for local users sending invites.`