Revert "dodgy"

This reverts commit 6c579370d4.
dodgy
2022-12-19 22:30:33 +00:00 · 2022-12-19 22:05:54 +00:00 · 2022-12-19 21:48:00 +00:00 · 2022-12-19 20:55:56 +00:00 · 2022-12-19 20:55:56 +00:00 · 2022-12-19 20:20:35 +00:00
157 changed files with 4089 additions and 6046 deletions
@@ -0,0 +1,18 @@
+# TODO: incorporate this into pyproject.toml if flake8 supports it in the future.
+# See https://github.com/PyCQA/flake8/issues/234
+[flake8]
+# see https://pycodestyle.readthedocs.io/en/latest/intro.html#error-codes
+# for error codes. The ones we ignore are:
+#  W503: line break before binary operator
+#  W504: line break after binary operator
+#  E203: whitespace before ':' (which is contrary to pep8?)
+#  E731: do not assign a lambda expression, use a def
+#  E501: Line too long (black enforces this for us)
+#
+# flake8-bugbear runs extra checks. Its error codes are described at
+# https://github.com/PyCQA/flake8-bugbear#list-of-warnings
+#  B019: Use of functools.lru_cache or functools.cache on methods can lead to memory leaks
+#  B023: Functions defined inside a loop must not use variables redefined in the loop
+#  B024: Abstract base class with no abstract method.
+
+ignore=W503,W504,E203,E731,E501,B019,B023,B024
@@ -6,7 +6,7 @@ on:
      - reopened  # For debugging!

 permissions:
-  # Needed to be able to push the commit. See
+  # Needed to be able to push the commit. See 
  #     https://docs.github.com/en/code-security/dependabot/working-with-dependabot/automating-dependabot-with-github-actions#enable-auto-merge-on-a-pull-request
  # for a similar example
  contents: write
@@ -20,11 +20,8 @@ jobs:
        with:
          ref: ${{ github.event.pull_request.head.ref }}
      - name: Write, commit and push changelog
-        env:
-          PR_TITLE: ${{ github.event.pull_request.title }}
-          PR_NUMBER: ${{ github.event.pull_request.number }}
        run: |
-          echo "${PR_TITLE}." > "changelog.d/${PR_NUMBER}".misc
+          echo "${{ github.event.pull_request.title }}." > "changelog.d/${{ github.event.pull_request.number }}".misc
          git add changelog.d
          git config user.email "github-actions[bot]@users.noreply.github.com"
          git config user.name "GitHub Actions"
@@ -14,7 +14,7 @@ jobs:
      # There's a 'download artifact' action, but it hasn't been updated for the workflow_run action
      # (https://github.com/actions/download-artifact/issues/60) so instead we get this mess:
      - name: 📥 Download artifact
-        uses: dawidd6/action-download-artifact@bd10f381a96414ce2b13a11bfa89902ba7cea07f # v2.24.3
+        uses: dawidd6/action-download-artifact@e6e25ac3a2b93187502a8be1ef9e9603afc34925 # v2.24.2
        with:
          workflow: docs-pr.yaml
          run_id: ${{ github.event.workflow_run.id }}
@@ -4,8 +4,6 @@ on:
  pull_request:
    paths:
      - docs/**
-      - book.toml
-      - .github/workflows/docs-pr.yaml

 jobs:
  pages:
@@ -34,27 +32,3 @@ jobs:
          path: book
          # We'll only use this in a workflow_run, then we're done with it
          retention-days: 1
-
-  link-check:
-    name: Check links in documentation
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
-
-      - name: Setup mdbook
-        uses: peaceiris/actions-mdbook@adeb05db28a0c0004681db83893d56c0388ea9ea # v1.2.0
-        with:
-          mdbook-version: '0.4.17'
-
-      - name: Setup htmltest
-        run: |
-          wget https://github.com/wjdp/htmltest/releases/download/v0.17.0/htmltest_0.17.0_linux_amd64.tar.gz
-          echo '775c597ee74899d6002cd2d93076f897f4ba68686bceabe2e5d72e84c57bc0fb  htmltest_0.17.0_linux_amd64.tar.gz' | sha256sum -c
-          tar zxf htmltest_0.17.0_linux_amd64.tar.gz
-
-      - name: Test links with htmltest
-        # Build the book with `./` as the site URL (to make checks on 404.html possible)
-        # Then run htmltest (without checking external links since that involves the network and is slow).
-        run: |
-          MDBOOK_OUTPUT__HTML__SITE_URL="./" mdbook build
-          ./htmltest book --skip-external
@@ -58,7 +58,7 @@ jobs:
          
      # Deploy to the target directory.
      - name: Deploy to gh pages
-        uses: peaceiris/actions-gh-pages@64b46b4226a4a12da2239ba3ea5aa73e3163c75b # v3.9.1
+        uses: peaceiris/actions-gh-pages@de7ea6f8efb354206b205ef54722213d99067935 # v3.9.0
        with:
          github_token: ${{ secrets.GITHUB_TOKEN }}
          publish_dir: ./book
@@ -208,7 +208,7 @@ jobs:

    steps:
      - uses: actions/checkout@v3
-      - uses: JasonEtco/create-an-issue@e27dddc79c92bc6e4562f268fffa5ed752639abd # v2.9.1
+      - uses: JasonEtco/create-an-issue@77399b6110ef82b94c1c9f9f615acf9e604f7f56 # v2.5.0, 2020-12-06
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        with:
@@ -148,7 +148,7 @@ jobs:
        env:
          # Skip testing for platforms which various libraries don't have wheels
          # for, and so need extra build deps.
-          CIBW_TEST_SKIP: pp3{7,9}-* *i686* *musl*
+          CIBW_TEST_SKIP: pp39-* *i686* *musl* pp37-macosx*
          # Fix Rust OOM errors on emulated aarch64: https://github.com/rust-lang/cargo/issues/10583
          CARGO_NET_GIT_FETCH_WITH_CLI: true
          CIBW_ENVIRONMENT_PASS_LINUX: CARGO_NET_GIT_FETCH_WITH_CLI
@@ -53,7 +53,7 @@ jobs:
      - run: scripts-dev/check_schema_delta.py --force-colors

  lint:
-    uses: "matrix-org/backend-meta/.github/workflows/python-poetry-ci.yml@v2"
+    uses: "matrix-org/backend-meta/.github/workflows/python-poetry-ci.yml@v1"
    with:
      typechecking-extras: "all"

@@ -174,7 +174,7 @@ jobs:

    steps:
      - uses: actions/checkout@v3
-      - uses: JasonEtco/create-an-issue@e27dddc79c92bc6e4562f268fffa5ed752639abd # v2.9.1
+      - uses: JasonEtco/create-an-issue@77399b6110ef82b94c1c9f9f615acf9e604f7f56 # v2.5.0, 2020-12-06
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        with:
@@ -36,7 +36,6 @@ __pycache__/

 # For direnv users
 /.envrc
-.direnv/

 # IDEs
 /.idea/
@@ -1,122 +1,3 @@
-Synapse 1.75.0 (2023-01-17)
-===========================
-
-No significant changes since 1.75.0rc2.
-
-
-Synapse 1.75.0rc2 (2023-01-12)
-==============================
-
-Bugfixes
--------
-
- Fix a bug introduced in Synapse 1.75.0rc1 where device lists could be miscalculated with some sync filters. ([\#14810](https://github.com/matrix-org/synapse/issues/14810))
- Fix race where calling `/members` or `/state` with an `at` parameter could fail for newly created rooms, when using multiple workers. ([\#14817](https://github.com/matrix-org/synapse/issues/14817))
-
-
-Synapse 1.75.0rc1 (2023-01-10)
-==============================
-
-Features
--------
-
- Add a `cached` function to `synapse.module_api` that returns a decorator to cache return values of functions. ([\#14663](https://github.com/matrix-org/synapse/issues/14663))
- Add experimental support for [MSC3391](https://github.com/matrix-org/matrix-spec-proposals/pull/3391) (removing account data). ([\#14714](https://github.com/matrix-org/synapse/issues/14714))
- Support [RFC7636](https://datatracker.ietf.org/doc/html/rfc7636) Proof Key for Code Exchange for OAuth single sign-on. ([\#14750](https://github.com/matrix-org/synapse/issues/14750))
- Support non-OpenID compliant userinfo claims for subject and picture. ([\#14753](https://github.com/matrix-org/synapse/issues/14753))
- Improve performance of `/sync` when filtering all rooms, message types, or senders. ([\#14786](https://github.com/matrix-org/synapse/issues/14786))
- Improve performance of the `/hierarchy` endpoint. ([\#14263](https://github.com/matrix-org/synapse/issues/14263))
-
-
-Bugfixes
--------
-
- Fix the *MAU Limits* section of the Grafana dashboard relying on a specific `job` name for the workers of a Synapse deployment. ([\#14644](https://github.com/matrix-org/synapse/issues/14644))
- Fix a bug introduced in Synapse 1.70.0 which could cause spurious `UNIQUE constraint failed` errors in the `rotate_notifs` background job. ([\#14669](https://github.com/matrix-org/synapse/issues/14669))
- Ensure stream IDs are always updated after caches get invalidated with workers. Contributed by Nick @ Beeper (@fizzadar). ([\#14723](https://github.com/matrix-org/synapse/issues/14723))
- Remove the unspecced `device` field from `/pushrules` responses. ([\#14727](https://github.com/matrix-org/synapse/issues/14727))
- Fix a bug introduced in Synapse 1.73.0 where the `picture_claim` configured under `oidc_providers` was unused (the default value of `"picture"` was used instead). ([\#14751](https://github.com/matrix-org/synapse/issues/14751))
- Unescape HTML entities in URL preview titles making use of oEmbed responses. ([\#14781](https://github.com/matrix-org/synapse/issues/14781))
- Disable sending confirmation email when 3pid is disabled. ([\#14725](https://github.com/matrix-org/synapse/issues/14725))
-
-
-Improved Documentation
----------------------
-
- Declare support for Python 3.11. ([\#14673](https://github.com/matrix-org/synapse/issues/14673))
- Fix `target_memory_usage` being used in the description for the actual `cache_autotune` sub-option `target_cache_memory_usage`. ([\#14674](https://github.com/matrix-org/synapse/issues/14674))
- Move `email` to Server section in config file documentation. ([\#14730](https://github.com/matrix-org/synapse/issues/14730))
- Fix broken links in the Synapse documentation. ([\#14744](https://github.com/matrix-org/synapse/issues/14744))
- Add missing worker settings to shared configuration documentation. ([\#14748](https://github.com/matrix-org/synapse/issues/14748))
- Document using Twitter as a OAuth 2.0 authentication provider. ([\#14778](https://github.com/matrix-org/synapse/issues/14778))
- Fix Synapse 1.74 upgrade notes to correctly explain how to install pyICU when installing Synapse from PyPI. ([\#14797](https://github.com/matrix-org/synapse/issues/14797))
- Update link to towncrier in contribution guide. ([\#14801](https://github.com/matrix-org/synapse/issues/14801))
- Use `htmltest` to check links in the Synapse documentation. ([\#14743](https://github.com/matrix-org/synapse/issues/14743))
-
-
-Internal Changes
----------------
-
- Faster remote room joins: stream the un-partial-stating of events over replication. ([\#14545](https://github.com/matrix-org/synapse/issues/14545), [\#14546](https://github.com/matrix-org/synapse/issues/14546))
- Use [ruff](https://github.com/charliermarsh/ruff/) instead of flake8. ([\#14633](https://github.com/matrix-org/synapse/issues/14633), [\#14741](https://github.com/matrix-org/synapse/issues/14741))
- Change `handle_new_client_event` signature so that a 429 does not reach clients on `PartialStateConflictError`, and internally retry when needed instead. ([\#14665](https://github.com/matrix-org/synapse/issues/14665))
- Remove dependency on jQuery on reCAPTCHA page. ([\#14672](https://github.com/matrix-org/synapse/issues/14672))
- Faster joins: make `compute_state_after_events` consistent with other state-fetching functions that take a `StateFilter`. ([\#14676](https://github.com/matrix-org/synapse/issues/14676))
- Add missing type hints. ([\#14680](https://github.com/matrix-org/synapse/issues/14680), [\#14681](https://github.com/matrix-org/synapse/issues/14681), [\#14687](https://github.com/matrix-org/synapse/issues/14687))
- Improve type annotations for the helper methods on a `CachedFunction`. ([\#14685](https://github.com/matrix-org/synapse/issues/14685))
- Check that the SQLite database file exists before porting to PostgreSQL. ([\#14692](https://github.com/matrix-org/synapse/issues/14692))
- Add `.direnv/` directory to .gitignore to prevent local state generated by the [direnv](https://direnv.net/) development tool from being committed. ([\#14707](https://github.com/matrix-org/synapse/issues/14707))
- Batch up replication requests to request the resyncing of remote users's devices. ([\#14716](https://github.com/matrix-org/synapse/issues/14716))
- If debug logging is enabled, log the `msgid`s of any to-device messages that are returned over `/sync`. ([\#14724](https://github.com/matrix-org/synapse/issues/14724))
- Change GHA CI job to follow best practices. ([\#14772](https://github.com/matrix-org/synapse/issues/14772))
- Switch to our fork of `dh-virtualenv` to work around an upstream Python 3.11 incompatibility. ([\#14774](https://github.com/matrix-org/synapse/issues/14774))
- Skip testing built wheels for PyPy 3.7 on Linux x86_64 as we lack new required dependencies in the build environment. ([\#14802](https://github.com/matrix-org/synapse/issues/14802))
-
-### Dependabot updates
-
-<details>
-
- Bump JasonEtco/create-an-issue from 2.8.1 to 2.8.2. ([\#14693](https://github.com/matrix-org/synapse/issues/14693))
- Bump anyhow from 1.0.66 to 1.0.68. ([\#14694](https://github.com/matrix-org/synapse/issues/14694))
- Bump blake2 from 0.10.5 to 0.10.6. ([\#14695](https://github.com/matrix-org/synapse/issues/14695))
- Bump serde_json from 1.0.89 to 1.0.91. ([\#14696](https://github.com/matrix-org/synapse/issues/14696))
- Bump serde from 1.0.150 to 1.0.151. ([\#14697](https://github.com/matrix-org/synapse/issues/14697))
- Bump lxml from 4.9.1 to 4.9.2. ([\#14698](https://github.com/matrix-org/synapse/issues/14698))
- Bump types-jsonschema from 4.17.0.1 to 4.17.0.2. ([\#14700](https://github.com/matrix-org/synapse/issues/14700))
- Bump sentry-sdk from 1.11.1 to 1.12.0. ([\#14701](https://github.com/matrix-org/synapse/issues/14701))
- Bump types-setuptools from 65.6.0.1 to 65.6.0.2. ([\#14702](https://github.com/matrix-org/synapse/issues/14702))
- Bump minimum PyYAML to 3.13. ([\#14720](https://github.com/matrix-org/synapse/issues/14720))
- Bump JasonEtco/create-an-issue from 2.8.2 to 2.9.1. ([\#14731](https://github.com/matrix-org/synapse/issues/14731))
- Bump towncrier from 22.8.0 to 22.12.0. ([\#14732](https://github.com/matrix-org/synapse/issues/14732))
- Bump isort from 5.10.1 to 5.11.4. ([\#14733](https://github.com/matrix-org/synapse/issues/14733))
- Bump attrs from 22.1.0 to 22.2.0. ([\#14734](https://github.com/matrix-org/synapse/issues/14734))
- Bump black from 22.10.0 to 22.12.0. ([\#14735](https://github.com/matrix-org/synapse/issues/14735))
- Bump sentry-sdk from 1.12.0 to 1.12.1. ([\#14736](https://github.com/matrix-org/synapse/issues/14736))
- Bump setuptools from 65.3.0 to 65.5.1. ([\#14738](https://github.com/matrix-org/synapse/issues/14738))
- Bump serde from 1.0.151 to 1.0.152. ([\#14758](https://github.com/matrix-org/synapse/issues/14758))
- Bump ruff from 0.0.189 to 0.0.206. ([\#14759](https://github.com/matrix-org/synapse/issues/14759))
- Bump pydantic from 1.10.2 to 1.10.4. ([\#14760](https://github.com/matrix-org/synapse/issues/14760))
- Bump gitpython from 3.1.29 to 3.1.30. ([\#14761](https://github.com/matrix-org/synapse/issues/14761))
- Bump pillow from 9.3.0 to 9.4.0. ([\#14762](https://github.com/matrix-org/synapse/issues/14762))
- Bump types-requests from 2.28.11.5 to 2.28.11.7. ([\#14763](https://github.com/matrix-org/synapse/issues/14763))
- Bump dawidd6/action-download-artifact from 2.24.2 to 2.24.3. ([\#14779](https://github.com/matrix-org/synapse/issues/14779))
- Bump peaceiris/actions-gh-pages from 3.9.0 to 3.9.1. ([\#14791](https://github.com/matrix-org/synapse/issues/14791))
- Bump types-pillow from 9.3.0.4 to 9.4.0.0. ([\#14792](https://github.com/matrix-org/synapse/issues/14792))
- Bump pyopenssl from 22.1.0 to 23.0.0. ([\#14793](https://github.com/matrix-org/synapse/issues/14793))
- Bump types-setuptools from 65.6.0.2 to 65.6.0.3. ([\#14794](https://github.com/matrix-org/synapse/issues/14794))
- Bump importlib-metadata from 4.2.0 to 6.0.0. ([\#14795](https://github.com/matrix-org/synapse/issues/14795))
- Bump ruff from 0.0.206 to 0.0.215. ([\#14796](https://github.com/matrix-org/synapse/issues/14796))
-</details>
-
-Synapse 1.74.0 (2022-12-20)
-===========================
-
-Improved Documentation
----------------------
-
- Add release note and update documentation regarding optional ICU support in user search. ([\#14712](https://github.com/matrix-org/synapse/issues/14712))
-
-
 Synapse 1.74.0rc1 (2022-12-13)
 ==============================

@@ -13,9 +13,9 @@ dependencies = [

 [[package]]
 name = "anyhow"
-version = "1.0.68"
+version = "1.0.66"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2cb2f989d18dd141ab8ae82f64d1a8cdd37e0840f73a406896cf5e99502fab61"
+checksum = "216261ddc8289130e551ddcd5ce8a064710c0d064a4d2895c67151c92b5443f6"

 [[package]]
 name = "arc-swap"
@@ -37,9 +37,9 @@ checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a"

 [[package]]
 name = "blake2"
-version = "0.10.6"
+version = "0.10.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "46502ad458c9a52b69d4d4d32775c788b7a1b85e8bc9d482d92250fc0e3f8efe"
+checksum = "b12e5fd123190ce1c2e559308a94c9bacad77907d4c6005d9e58fe1a0689e55e"
 dependencies = [
 "digest",
 ]
@@ -323,18 +323,18 @@ checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd"

 [[package]]
 name = "serde"
-version = "1.0.152"
+version = "1.0.150"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bb7d1f0d3021d347a83e556fc4683dea2ea09d87bccdf88ff5c12545d89d5efb"
+checksum = "e326c9ec8042f1b5da33252c8a37e9ffbd2c9bef0155215b6e6c80c790e05f91"
 dependencies = [
 "serde_derive",
 ]

 [[package]]
 name = "serde_derive"
-version = "1.0.152"
+version = "1.0.150"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "af487d118eecd09402d70a5d72551860e788df87b464af30e5ea6a38c75c541e"
+checksum = "42a3df25b0713732468deadad63ab9da1f1fd75a48a15024b50363f128db627e"
 dependencies = [
 "proc-macro2",
 "quote",
@@ -343,9 +343,9 @@ dependencies = [

 [[package]]
 name = "serde_json"
-version = "1.0.91"
+version = "1.0.89"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "877c235533714907a8c2464236f5c4b2a17262ef1bd71f38f35ea592c8da6883"
+checksum = "020ff22c755c2ed3f8cf162dbb41a7268d934702f3ed3631656ea597e08fc3db"
 dependencies = [
 "itoa",
 "ryu",
@@ -34,14 +34,6 @@ additional-css = [
    "docs/website_files/table-of-contents.css",
    "docs/website_files/remove-nav-buttons.css",
    "docs/website_files/indent-section-headers.css",
-    "docs/website_files/version-picker.css",
 ]
-additional-js = [
-    "docs/website_files/table-of-contents.js",
-    "docs/website_files/version-picker.js",
-    "docs/website_files/version.js",
-]
-theme = "docs/website_files/theme"
-
-[preprocessor.schema_versions]
-command = "./scripts-dev/schema_versions.py"
+additional-js = ["docs/website_files/table-of-contents.js"]
+theme = "docs/website_files/theme"
@@ -1008,7 +1008,8 @@
                "mode": "absolute",
                "steps": [
                  {
-                    "color": "green"
+                    "color": "green",
+                    "value": null
                  },
                  {
                    "color": "red",
@@ -1680,7 +1681,8 @@
                "mode": "absolute",
                "steps": [
                  {
-                    "color": "green"
+                    "color": "green",
+                    "value": null
                  },
                  {
                    "color": "red",
@@ -2531,7 +2533,8 @@
                "mode": "absolute",
                "steps": [
                  {
-                    "color": "green"
+                    "color": "green",
+                    "value": null
                  },
                  {
                    "color": "red",
@@ -11293,7 +11296,7 @@
                "uid": "$datasource"
              },
              "editorMode": "code",
-              "expr": "max(synapse_admin_mau_max{instance=\"$instance\"})",
+              "expr": "synapse_admin_mau_max{instance=\"$instance\", job=~\"(hhs_)?synapse\"}",
              "format": "time_series",
              "interval": "",
              "intervalFactor": 1,
@@ -11307,7 +11310,7 @@
                "uid": "$datasource"
              },
              "editorMode": "code",
-              "expr": "max(synapse_admin_mau_current{instance=\"$instance\"})",
+              "expr": "synapse_admin_mau_current{instance=\"$instance\", job=~\"(hhs_)?synapse\"}",
              "hide": false,
              "legendFormat": "Current",
              "range": true,
@@ -12757,6 +12760,6 @@
  "timezone": "",
  "title": "Synapse",
  "uid": "000000012",
-  "version": 150,
+  "version": 149,
  "weekStart": ""
 }
@@ -1,27 +1,3 @@
-matrix-synapse-py3 (1.75.0) stable; urgency=medium
-
-  * New Synapse release 1.75.0.
-
- -- Synapse Packaging team <packages@matrix.org>  Tue, 17 Jan 2023 11:36:02 +0000
-
-matrix-synapse-py3 (1.75.0~rc2) stable; urgency=medium
-
-  * New Synapse release 1.75.0rc2.
-
- -- Synapse Packaging team <packages@matrix.org>  Thu, 12 Jan 2023 10:30:15 -0800
-
-matrix-synapse-py3 (1.75.0~rc1) stable; urgency=medium
-
-  * New Synapse release 1.75.0rc1.
-
- -- Synapse Packaging team <packages@matrix.org>  Tue, 10 Jan 2023 12:18:27 +0000
-
-matrix-synapse-py3 (1.74.0) stable; urgency=medium
-
-  * New Synapse release 1.74.0.
-
- -- Synapse Packaging team <packages@matrix.org>  Tue, 20 Dec 2022 16:07:38 +0000
-
 matrix-synapse-py3 (1.74.0~rc1) stable; urgency=medium

  * New dependency on libicu-dev to provide improved results for user
@@ -167,7 +167,6 @@ RUN \
    libwebp6 \
    xmlsec1 \
    libjemalloc2 \
-    libicu67 \
    libssl-dev \
    openssl \
    && rm -rf /var/lib/apt/lists/*
@@ -36,10 +36,8 @@ RUN env DEBIAN_FRONTEND=noninteractive apt-get install \
        wget

 # fetch and unpack the package
-# We are temporarily using a fork of dh-virtualenv due to an incompatibility with Python 3.11, which ships with
-# Debian sid. TODO: Switch back to upstream once https://github.com/spotify/dh-virtualenv/pull/354 has merged.
 RUN mkdir /dh-virtualenv
-RUN wget -q -O /dh-virtualenv.tar.gz https://github.com/matrix-org/dh-virtualenv/archive/refs/tags/matrixorg-2023010302.tar.gz
+RUN wget -q -O /dh-virtualenv.tar.gz https://github.com/spotify/dh-virtualenv/archive/refs/tags/1.2.2.tar.gz
 RUN tar -xv --strip-components=1 -C /dh-virtualenv -f /dh-virtualenv.tar.gz

 # install its build deps. We do another apt-cache-update here, because we might
@@ -102,8 +102,6 @@ experimental_features:
  {% endif %}
  # Filtering /messages by relation type.
  msc3874_enabled: true
-  # Enable removing account data support
-  msc3391_enabled: true

 server_notices:
  system_mxid_localpart: _server
@@ -5,7 +5,7 @@ use it, you must enable the account validity feature (under
 `account_validity`) in Synapse's configuration.

 To use it, you will need to authenticate by providing an `access_token`
-for a server admin: see [Admin API](../usage/administration/admin_api/).
+for a server admin: see [Admin API](../usage/administration/admin_api).

 ## Renew account

@@ -3,7 +3,7 @@
 This API returns information about reported events.

 To use it, you will need to authenticate by providing an `access_token`
-for a server admin: see [Admin API](../usage/administration/admin_api/).
+for a server admin: see [Admin API](../usage/administration/admin_api).

 The api is:
 ```
@@ -6,7 +6,7 @@ Details about the format of the `media_id` and storage of the media in the file
 are documented under [media repository](../media_repository.md).

 To use it, you will need to authenticate by providing an `access_token`
-for a server admin: see [Admin API](../usage/administration/admin_api/).
+for a server admin: see [Admin API](../usage/administration/admin_api).

 ## List all media in a room

@@ -11,7 +11,7 @@ Note that Synapse requires at least one message in each room, so it will never
 delete the last message in a room.

 To use it, you will need to authenticate by providing an `access_token`
-for a server admin: see [Admin API](../usage/administration/admin_api/).
+for a server admin: see [Admin API](../usage/administration/admin_api).

 The API is:

@@ -6,7 +6,7 @@ local users. The server administrator must be in the room and have permission to
 invite users.

 To use it, you will need to authenticate by providing an `access_token`
-for a server admin: see [Admin API](../usage/administration/admin_api/).
+for a server admin: see [Admin API](../usage/administration/admin_api).

 ## Parameters

@@ -5,7 +5,7 @@ server. There are various parameters available that allow for filtering and
 sorting the returned list. This API supports pagination.

 To use it, you will need to authenticate by providing an `access_token`
-for a server admin: see [Admin API](../usage/administration/admin_api/).
+for a server admin: see [Admin API](../usage/administration/admin_api).

 **Parameters**

@@ -400,7 +400,7 @@ sent to a room in a given timeframe. There are various parameters available
 that allow for filtering and ordering the returned list. This API supports pagination.

 To use it, you will need to authenticate by providing an `access_token`
-for a server admin: see [Admin API](../usage/administration/admin_api/).
+for a server admin: see [Admin API](../usage/administration/admin_api).

 This endpoint mirrors the [Matrix Spec defined Messages API](https://spec.matrix.org/v1.1/client-server-api/#get_matrixclientv3roomsroomidmessages).

@@ -4,7 +4,7 @@ Returns information about all local media usage of users. Gives the
 possibility to filter them by time and user.

 To use it, you will need to authenticate by providing an `access_token`
-for a server admin: see [Admin API](../usage/administration/admin_api/).
+for a server admin: see [Admin API](../usage/administration/admin_api).

 The API is:

@@ -1,7 +1,7 @@
 # User Admin API

 To use it, you will need to authenticate by providing an `access_token`
-for a server admin: see [Admin API](../usage/administration/admin_api/).
+for a server admin: see [Admin API](../usage/administration/admin_api).

 ## Query User Account

@@ -10,7 +10,7 @@ The necessary tools are:

 - [black](https://black.readthedocs.io/en/stable/), a source code formatter;
 - [isort](https://pycqa.github.io/isort/), which organises each file's imports;
- [ruff](https://github.com/charliermarsh/ruff), which can spot common errors; and
+- [flake8](https://flake8.pycqa.org/en/latest/), which can spot common errors; and
 - [mypy](https://mypy.readthedocs.io/en/stable/), a type checker.

 Install them with:
@@ -28,7 +28,7 @@ scripts-dev/lint.sh
 It's worth noting that modern IDEs and text editors can run these tools
 automatically on save. It may be worth looking into whether this
 functionality is supported in your editor for a more convenient
-development workflow. It is not, however, recommended to run `mypy`
+development workflow. It is not, however, recommended to run `flake8` or `mypy`
 on save as they take a while and can be very resource intensive.

 ## General rules
@@ -24,8 +24,6 @@ The code of Synapse is written in Python 3. To do pretty much anything, you'll n

 Synapse can connect to PostgreSQL via the [psycopg2](https://pypi.org/project/psycopg2/) Python library. Building this library from source requires access to PostgreSQL's C header files. On Debian or Ubuntu Linux, these can be installed with `sudo apt install libpq-dev`.

-Synapse has an optional, improved user search with better Unicode support. For that you need the development package of `libicu`. On Debian or Ubuntu Linux, this can be installed with `sudo apt install libicu-dev`.
-
 The source code of Synapse is hosted on GitHub. You will also need [a recent version of git](https://github.com/git-guides/install-git).

 For some tests, you will need [a recent version of Docker](https://docs.docker.com/get-docker/).
@@ -106,8 +104,8 @@ regarding Synapse's Admin API, which is used mostly by sysadmins and external
 service developers.

 Synapse's code style is documented [here](../code_style.md). Please follow
-it, including the conventions for [configuration
-options and documentation](../code_style.md#configuration-code-and-documentation-format).
+it, including the conventions for the [sample configuration
+file](../code_style.md#configuration-file-format).

 We welcome improvements and additions to our documentation itself! When
 writing new pages, please
@@ -126,7 +124,7 @@ changes to the Rust code.


 # 8. Test, test, test!
-<a name="test-test-test" id="test-test-test"></a>
+<a name="test-test-test"></a>

 While you're developing and before submitting a patch, you'll
 want to test your code.
@@ -382,7 +380,7 @@ To prepare a Pull Request, please:
 ## Changelog

 All changes, even minor ones, need a corresponding changelog / newsfragment
-entry. These are managed by [Towncrier](https://github.com/twisted/towncrier).
+entry. These are managed by [Towncrier](https://github.com/hawkowl/towncrier).

 To create a changelog entry, make a new file in the `changelog.d` directory named
 in the format of `PRnumber.type`. The type can be one of the following:
@@ -424,7 +422,8 @@ chicken-and-egg problem.
 There are two options for solving this:

 1. Open the PR without a changelog file, see what number you got, and *then*
-   add the changelog file to your branch, or:
+   add the changelog file to your branch (see [Updating your pull
+   request](#updating-your-pull-request)), or:

 1. Look at the [list of all
   issues/PRs](https://github.com/matrix-org/synapse/issues?q=), add one to the
@@ -59,8 +59,8 @@ namespace (such as anything under `/_matrix/client` for example). It is strongly
 recommended that modules register their web resources under the `/_synapse/client`
 namespace.

-The provided resource is a Python class that implements Twisted's [IResource](https://docs.twistedmatrix.com/en/stable/api/twisted.web.resource.IResource.html)
-interface (such as [Resource](https://docs.twistedmatrix.com/en/stable/api/twisted.web.resource.Resource.html)).
+The provided resource is a Python class that implements Twisted's [IResource](https://twistedmatrix.com/documents/current/api/twisted.web.resource.IResource.html)
+interface (such as [Resource](https://twistedmatrix.com/documents/current/api/twisted.web.resource.Resource.html)).

 Only one resource can be registered for a given path. If several modules attempt to
 register a resource for the same path, the module that appears first in Synapse's
@@ -82,4 +82,4 @@ the callback name as the argument name and the function as its value. A
 `register_[...]_callbacks` method exists for each category.

 Callbacks for each category can be found on their respective page of the
-[Synapse documentation website](https://matrix-org.github.io/synapse).
+[Synapse documentation website](https://matrix-org.github.io/synapse).
@@ -88,41 +88,98 @@ oidc_providers:
        display_name_template: "{{ user.name }}"
 ```

-### Apple
+### Dex

-Configuring "Sign in with Apple" (SiWA) requires an Apple Developer account.
+[Dex][dex-idp] is a simple, open-source OpenID Connect Provider.
+Although it is designed to help building a full-blown provider with an
+external database, it can be configured with static passwords in a config file.

-You will need to create a new "Services ID" for SiWA, and create and download a
-private key with "SiWA" enabled.
+Follow the [Getting Started guide](https://dexidp.io/docs/getting-started/)
+to install Dex.

-As well as the private key file, you will need:
- * Client ID: the "identifier" you gave the "Services ID"
- * Team ID: a 10-character ID associated with your developer account.
- * Key ID: the 10-character identifier for the key.
-
-[Apple's developer documentation](https://help.apple.com/developer-account/?lang=en#/dev77c875b7e)
-has more information on setting up SiWA.
-
-The synapse config will look like this:
+Edit `examples/config-dev.yaml` config file from the Dex repo to add a client:

 ```yaml
-  - idp_id: apple
-    idp_name: Apple
-    issuer: "https://appleid.apple.com"
-    client_id: "your-client-id" # Set to the "identifier" for your "ServicesID"
-    client_auth_method: "client_secret_post"
-    client_secret_jwt_key:
-      key_file: "/path/to/AuthKey_KEYIDCODE.p8"  # point to your key file
-      jwt_header:
-        alg: ES256
-        kid: "KEYIDCODE"   # Set to the 10-char Key ID
-      jwt_payload:
-        iss: TEAMIDCODE    # Set to the 10-char Team ID
-    scopes: ["name", "email", "openid"]
-    authorization_endpoint: https://appleid.apple.com/auth/authorize?response_mode=form_post
+staticClients:
+- id: synapse
+  secret: secret
+  redirectURIs:
+  - '[synapse public baseurl]/_synapse/client/oidc/callback'
+  name: 'Synapse'
+```
+
+Run with `dex serve examples/config-dev.yaml`.
+
+Synapse config:
+
+```yaml
+oidc_providers:
+  - idp_id: dex
+    idp_name: "My Dex server"
+    skip_verification: true # This is needed as Dex is served on an insecure endpoint
+    issuer: "http://127.0.0.1:5556/dex"
+    client_id: "synapse"
+    client_secret: "secret"
+    scopes: ["openid", "profile"]
    user_mapping_provider:
      config:
-        email_template: "{{ user.email }}"
+        localpart_template: "{{ user.name }}"
+        display_name_template: "{{ user.name|capitalize }}"
+```
+### Keycloak
+
+[Keycloak][keycloak-idp] is an opensource IdP maintained by Red Hat.
+
+Keycloak supports OIDC Back-Channel Logout, which sends logout notification to Synapse, so that Synapse users get logged out when they log out from Keycloak.
+This can be optionally enabled by setting `backchannel_logout_enabled` to `true` in the Synapse configuration, and by setting the "Backchannel Logout URL" in Keycloak.
+
+Follow the [Getting Started Guide](https://www.keycloak.org/getting-started) to install Keycloak and set up a realm.
+
+1. Click `Clients` in the sidebar and click `Create`
+
+2. Fill in the fields as below:
+
+| Field | Value |
+|-----------|-----------|
+| Client ID | `synapse` |
+| Client Protocol | `openid-connect` |
+
+3. Click `Save`
+4. Fill in the fields as below:
+
+| Field | Value |
+|-----------|-----------|
+| Client ID | `synapse` |
+| Enabled | `On` |
+| Client Protocol | `openid-connect` |
+| Access Type | `confidential` |
+| Valid Redirect URIs | `[synapse public baseurl]/_synapse/client/oidc/callback` |
+| Backchannel Logout URL (optional) | `[synapse public baseurl]/_synapse/client/oidc/backchannel_logout` |
+| Backchannel Logout Session Required (optional) | `On` |
+
+5. Click `Save`
+6. On the Credentials tab, update the fields:
+
+| Field | Value |
+|-------|-------|
+| Client Authenticator | `Client ID and Secret` |
+
+7. Click `Regenerate Secret`
+8. Copy Secret
+
+```yaml
+oidc_providers:
+  - idp_id: keycloak
+    idp_name: "My KeyCloak server"
+    issuer: "https://127.0.0.1:8443/realms/{realm_name}"
+    client_id: "synapse"
+    client_secret: "copy secret generated from above"
+    scopes: ["openid", "profile"]
+    user_mapping_provider:
+      config:
+        localpart_template: "{{ user.preferred_username }}"
+        display_name_template: "{{ user.name }}"
+    backchannel_logout_enabled: true # Optional
 ```

 ### Auth0
@@ -205,43 +262,285 @@ oidc_providers:
        display_name_template: "{{ user.preferred_username|capitalize }}" # TO BE FILLED: If your users have names in Authentik and you want those in Synapse, this should be replaced with user.name|capitalize.
 ```

-### Dex
+### LemonLDAP

-[Dex][dex-idp] is a simple, open-source OpenID Connect Provider.
-Although it is designed to help building a full-blown provider with an
-external database, it can be configured with static passwords in a config file.
+[LemonLDAP::NG][lemonldap] is an open-source IdP solution.

-Follow the [Getting Started guide](https://dexidp.io/docs/getting-started/)
-to install Dex.
-
-Edit `examples/config-dev.yaml` config file from the Dex repo to add a client:
+1. Create an OpenID Connect Relying Parties in LemonLDAP::NG
+2. The parameters are:
+- Client ID under the basic menu of the new Relying Parties (`Options > Basic >
+  Client ID`)
+- Client secret (`Options > Basic > Client secret`)
+- JWT Algorithm: RS256 within the security menu of the new Relying Parties
+  (`Options > Security > ID Token signature algorithm` and `Options > Security >
+  Access Token signature algorithm`)
+- Scopes: OpenID, Email and Profile
+- Allowed redirection addresses for login (`Options > Basic > Allowed
+  redirection addresses for login` ) :
+  `[synapse public baseurl]/_synapse/client/oidc/callback`

+Synapse config:
 ```yaml
-staticClients:
- id: synapse
-  secret: secret
-  redirectURIs:
-  - '[synapse public baseurl]/_synapse/client/oidc/callback'
-  name: 'Synapse'
+oidc_providers:
+  - idp_id: lemonldap
+    idp_name: lemonldap
+    discover: true
+    issuer: "https://auth.example.org/" # TO BE FILLED: replace with your domain
+    client_id: "your client id" # TO BE FILLED
+    client_secret: "your client secret" # TO BE FILLED
+    scopes:
+      - "openid"
+      - "profile"
+      - "email"
+    user_mapping_provider:
+      config:
+        localpart_template: "{{ user.preferred_username }}}"
+        # TO BE FILLED: If your users have names in LemonLDAP::NG and you want those in Synapse, this should be replaced with user.name|capitalize or any valid filter.
+        display_name_template: "{{ user.preferred_username|capitalize }}"
 ```

-Run with `dex serve examples/config-dev.yaml`.
+### GitHub
+
+[GitHub][github-idp] is a bit special as it is not an OpenID Connect compliant provider, but
+just a regular OAuth2 provider.
+
+The [`/user` API endpoint](https://developer.github.com/v3/users/#get-the-authenticated-user)
+can be used to retrieve information on the authenticated user. As the Synapse
+login mechanism needs an attribute to uniquely identify users, and that endpoint
+does not return a `sub` property, an alternative `subject_claim` has to be set.
+
+1. Create a new OAuth application: [https://github.com/settings/applications/new](https://github.com/settings/applications/new).
+2. Set the callback URL to `[synapse public baseurl]/_synapse/client/oidc/callback`.

 Synapse config:

 ```yaml
 oidc_providers:
-  - idp_id: dex
-    idp_name: "My Dex server"
-    skip_verification: true # This is needed as Dex is served on an insecure endpoint
-    issuer: "http://127.0.0.1:5556/dex"
-    client_id: "synapse"
-    client_secret: "secret"
-    scopes: ["openid", "profile"]
+  - idp_id: github
+    idp_name: Github
+    idp_brand: "github"  # optional: styling hint for clients
+    discover: false
+    issuer: "https://github.com/"
+    client_id: "your-client-id" # TO BE FILLED
+    client_secret: "your-client-secret" # TO BE FILLED
+    authorization_endpoint: "https://github.com/login/oauth/authorize"
+    token_endpoint: "https://github.com/login/oauth/access_token"
+    userinfo_endpoint: "https://api.github.com/user"
+    scopes: ["read:user"]
    user_mapping_provider:
      config:
-        localpart_template: "{{ user.name }}"
-        display_name_template: "{{ user.name|capitalize }}"
+        subject_claim: "id"
+        localpart_template: "{{ user.login }}"
+        display_name_template: "{{ user.name }}"
+```
+
+### Google
+
+[Google][google-idp] is an OpenID certified authentication and authorisation provider.
+
+1. Set up a project in the Google API Console (see 
+   [documentation](https://developers.google.com/identity/protocols/oauth2/openid-connect#appsetup)).
+3. Add an "OAuth Client ID" for a Web Application under "Credentials".
+4. Copy the Client ID and Client Secret, and add the following to your synapse config:
+   ```yaml
+   oidc_providers:
+     - idp_id: google
+       idp_name: Google
+       idp_brand: "google"  # optional: styling hint for clients
+       issuer: "https://accounts.google.com/"
+       client_id: "your-client-id" # TO BE FILLED
+       client_secret: "your-client-secret" # TO BE FILLED
+       scopes: ["openid", "profile", "email"] # email is optional, read below
+       user_mapping_provider:
+         config:
+           localpart_template: "{{ user.given_name|lower }}"
+           display_name_template: "{{ user.name }}"
+           email_template: "{{ user.email }}" # needs "email" in scopes above
+   ```
+4. Back in the Google console, add this Authorized redirect URI: `[synapse
+   public baseurl]/_synapse/client/oidc/callback`.
+
+### Twitch
+
+1. Setup a developer account on [Twitch](https://dev.twitch.tv/)
+2. Obtain the OAuth 2.0 credentials by [creating an app](https://dev.twitch.tv/console/apps/)
+3. Add this OAuth Redirect URL: `[synapse public baseurl]/_synapse/client/oidc/callback`
+
+Synapse config:
+
+```yaml
+oidc_providers:
+  - idp_id: twitch
+    idp_name: Twitch
+    issuer: "https://id.twitch.tv/oauth2/"
+    client_id: "your-client-id" # TO BE FILLED
+    client_secret: "your-client-secret" # TO BE FILLED
+    client_auth_method: "client_secret_post"
+    user_mapping_provider:
+      config:
+        localpart_template: "{{ user.preferred_username }}"
+        display_name_template: "{{ user.name }}"
+```
+
+### GitLab
+
+1. Create a [new application](https://gitlab.com/profile/applications).
+2. Add the `read_user` and `openid` scopes.
+3. Add this Callback URL: `[synapse public baseurl]/_synapse/client/oidc/callback`
+
+Synapse config:
+
+```yaml
+oidc_providers:
+  - idp_id: gitlab
+    idp_name: Gitlab
+    idp_brand: "gitlab"  # optional: styling hint for clients
+    issuer: "https://gitlab.com/"
+    client_id: "your-client-id" # TO BE FILLED
+    client_secret: "your-client-secret" # TO BE FILLED
+    client_auth_method: "client_secret_post"
+    scopes: ["openid", "read_user"]
+    user_profile_method: "userinfo_endpoint"
+    user_mapping_provider:
+      config:
+        localpart_template: '{{ user.nickname }}'
+        display_name_template: '{{ user.name }}'
+```
+
+### Facebook
+
+0. You will need a Facebook developer account. You can register for one
+   [here](https://developers.facebook.com/async/registration/).
+1. On the [apps](https://developers.facebook.com/apps/) page of the developer
+   console, "Create App", and choose "Build Connected Experiences".
+2. Once the app is created, add "Facebook Login" and choose "Web". You don't
+   need to go through the whole form here.
+3. In the left-hand menu, open "Products"/"Facebook Login"/"Settings".
+   * Add `[synapse public baseurl]/_synapse/client/oidc/callback` as an OAuth Redirect
+     URL.
+4. In the left-hand menu, open "Settings/Basic". Here you can copy the "App ID"
+   and "App Secret" for use below.
+
+Synapse config:
+
+```yaml
+  - idp_id: facebook
+    idp_name: Facebook
+    idp_brand: "facebook"  # optional: styling hint for clients
+    discover: false
+    issuer: "https://www.facebook.com"
+    client_id: "your-client-id" # TO BE FILLED
+    client_secret: "your-client-secret" # TO BE FILLED
+    scopes: ["openid", "email"]
+    authorization_endpoint: "https://facebook.com/dialog/oauth"
+    token_endpoint: "https://graph.facebook.com/v9.0/oauth/access_token"
+    jwks_uri: "https://www.facebook.com/.well-known/oauth/openid/jwks/"
+    user_mapping_provider:
+      config:
+        display_name_template: "{{ user.name }}"
+        email_template: "{{ user.email }}"
+```
+
+Relevant documents:
+ * [Manually Build a Login Flow](https://developers.facebook.com/docs/facebook-login/manually-build-a-login-flow)
+ * [Using Facebook's Graph API](https://developers.facebook.com/docs/graph-api/using-graph-api/)
+ * [Reference to the User endpoint](https://developers.facebook.com/docs/graph-api/reference/user)
+
+Facebook do have an [OIDC discovery endpoint](https://www.facebook.com/.well-known/openid-configuration),
+but it has a `response_types_supported` which excludes "code" (which we rely on, and
+is even mentioned in their [documentation](https://developers.facebook.com/docs/facebook-login/manually-build-a-login-flow#login)),
+so we have to disable discovery and configure the URIs manually.
+
+### Gitea
+
+Gitea is, like Github, not an OpenID provider, but just an OAuth2 provider.
+
+The [`/user` API endpoint](https://try.gitea.io/api/swagger#/user/userGetCurrent)
+can be used to retrieve information on the authenticated user. As the Synapse
+login mechanism needs an attribute to uniquely identify users, and that endpoint
+does not return a `sub` property, an alternative `subject_claim` has to be set.
+
+1. Create a new application.
+2. Add this Callback URL: `[synapse public baseurl]/_synapse/client/oidc/callback`
+
+Synapse config:
+
+```yaml
+oidc_providers:
+  - idp_id: gitea
+    idp_name: Gitea
+    discover: false
+    issuer: "https://your-gitea.com/"
+    client_id: "your-client-id" # TO BE FILLED
+    client_secret: "your-client-secret" # TO BE FILLED
+    client_auth_method: client_secret_post
+    scopes: [] # Gitea doesn't support Scopes
+    authorization_endpoint: "https://your-gitea.com/login/oauth/authorize"
+    token_endpoint: "https://your-gitea.com/login/oauth/access_token"
+    userinfo_endpoint: "https://your-gitea.com/api/v1/user"
+    user_mapping_provider:
+      config:
+        subject_claim: "id"
+        localpart_template: "{{ user.login }}"
+        display_name_template: "{{ user.full_name }}"
+```
+
+### XWiki
+
+Install [OpenID Connect Provider](https://extensions.xwiki.org/xwiki/bin/view/Extension/OpenID%20Connect/OpenID%20Connect%20Provider/) extension in your [XWiki](https://www.xwiki.org) instance.
+
+Synapse config:
+
+```yaml
+oidc_providers:
+  - idp_id: xwiki
+    idp_name: "XWiki"
+    issuer: "https://myxwikihost/xwiki/oidc/"
+    client_id: "your-client-id" # TO BE FILLED
+    client_auth_method: none
+    scopes: ["openid", "profile"]
+    user_profile_method: "userinfo_endpoint"
+    user_mapping_provider:
+      config:
+        localpart_template: "{{ user.preferred_username }}"
+        display_name_template: "{{ user.name }}"
+```
+
+### Apple
+
+Configuring "Sign in with Apple" (SiWA) requires an Apple Developer account.
+
+You will need to create a new "Services ID" for SiWA, and create and download a
+private key with "SiWA" enabled.
+
+As well as the private key file, you will need:
+ * Client ID: the "identifier" you gave the "Services ID"
+ * Team ID: a 10-character ID associated with your developer account.
+ * Key ID: the 10-character identifier for the key.
+
+[Apple's developer documentation](https://help.apple.com/developer-account/?lang=en#/dev77c875b7e)
+has more information on setting up SiWA.
+
+The synapse config will look like this:
+
+```yaml
+  - idp_id: apple
+    idp_name: Apple
+    issuer: "https://appleid.apple.com"
+    client_id: "your-client-id" # Set to the "identifier" for your "ServicesID"
+    client_auth_method: "client_secret_post"
+    client_secret_jwt_key:
+      key_file: "/path/to/AuthKey_KEYIDCODE.p8"  # point to your key file
+      jwt_header:
+        alg: ES256
+        kid: "KEYIDCODE"   # Set to the 10-char Key ID
+      jwt_payload:
+        iss: TEAMIDCODE    # Set to the 10-char Team ID
+    scopes: ["name", "email", "openid"]
+    authorization_endpoint: https://appleid.apple.com/auth/authorize?response_mode=form_post
+    user_mapping_provider:
+      config:
+        email_template: "{{ user.email }}"
 ```

 ### Django OAuth Toolkit
@@ -292,263 +591,6 @@ oidc_providers:
        email_template: "{{ user.email }}"
 ```

-### Facebook
-
-0. You will need a Facebook developer account. You can register for one
-   [here](https://developers.facebook.com/async/registration/).
-1. On the [apps](https://developers.facebook.com/apps/) page of the developer
-   console, "Create App", and choose "Build Connected Experiences".
-2. Once the app is created, add "Facebook Login" and choose "Web". You don't
-   need to go through the whole form here.
-3. In the left-hand menu, open "Products"/"Facebook Login"/"Settings".
-   * Add `[synapse public baseurl]/_synapse/client/oidc/callback` as an OAuth Redirect
-     URL.
-4. In the left-hand menu, open "Settings/Basic". Here you can copy the "App ID"
-   and "App Secret" for use below.
-
-Synapse config:
-
-```yaml
-  - idp_id: facebook
-    idp_name: Facebook
-    idp_brand: "facebook"  # optional: styling hint for clients
-    discover: false
-    issuer: "https://www.facebook.com"
-    client_id: "your-client-id" # TO BE FILLED
-    client_secret: "your-client-secret" # TO BE FILLED
-    scopes: ["openid", "email"]
-    authorization_endpoint: "https://facebook.com/dialog/oauth"
-    token_endpoint: "https://graph.facebook.com/v9.0/oauth/access_token"
-    jwks_uri: "https://www.facebook.com/.well-known/oauth/openid/jwks/"
-    user_mapping_provider:
-      config:
-        display_name_template: "{{ user.name }}"
-        email_template: "{{ user.email }}"
-```
-
-Relevant documents:
- * [Manually Build a Login Flow](https://developers.facebook.com/docs/facebook-login/manually-build-a-login-flow)
- * [Using Facebook's Graph API](https://developers.facebook.com/docs/graph-api/using-graph-api/)
- * [Reference to the User endpoint](https://developers.facebook.com/docs/graph-api/reference/user)
-
-Facebook do have an [OIDC discovery endpoint](https://www.facebook.com/.well-known/openid-configuration),
-but it has a `response_types_supported` which excludes "code" (which we rely on, and
-is even mentioned in their [documentation](https://developers.facebook.com/docs/facebook-login/manually-build-a-login-flow#login)),
-so we have to disable discovery and configure the URIs manually.
-
-### GitHub
-
-[GitHub][github-idp] is a bit special as it is not an OpenID Connect compliant provider, but
-just a regular OAuth2 provider.
-
-The [`/user` API endpoint](https://developer.github.com/v3/users/#get-the-authenticated-user)
-can be used to retrieve information on the authenticated user. As the Synapse
-login mechanism needs an attribute to uniquely identify users, and that endpoint
-does not return a `sub` property, an alternative `subject_claim` has to be set.
-
-1. Create a new OAuth application: [https://github.com/settings/applications/new](https://github.com/settings/applications/new).
-2. Set the callback URL to `[synapse public baseurl]/_synapse/client/oidc/callback`.
-
-Synapse config:
-
-```yaml
-oidc_providers:
-  - idp_id: github
-    idp_name: Github
-    idp_brand: "github"  # optional: styling hint for clients
-    discover: false
-    issuer: "https://github.com/"
-    client_id: "your-client-id" # TO BE FILLED
-    client_secret: "your-client-secret" # TO BE FILLED
-    authorization_endpoint: "https://github.com/login/oauth/authorize"
-    token_endpoint: "https://github.com/login/oauth/access_token"
-    userinfo_endpoint: "https://api.github.com/user"
-    scopes: ["read:user"]
-    user_mapping_provider:
-      config:
-        subject_claim: "id"
-        localpart_template: "{{ user.login }}"
-        display_name_template: "{{ user.name }}"
-```
-
-### GitLab
-
-1. Create a [new application](https://gitlab.com/profile/applications).
-2. Add the `read_user` and `openid` scopes.
-3. Add this Callback URL: `[synapse public baseurl]/_synapse/client/oidc/callback`
-
-Synapse config:
-
-```yaml
-oidc_providers:
-  - idp_id: gitlab
-    idp_name: Gitlab
-    idp_brand: "gitlab"  # optional: styling hint for clients
-    issuer: "https://gitlab.com/"
-    client_id: "your-client-id" # TO BE FILLED
-    client_secret: "your-client-secret" # TO BE FILLED
-    client_auth_method: "client_secret_post"
-    scopes: ["openid", "read_user"]
-    user_profile_method: "userinfo_endpoint"
-    user_mapping_provider:
-      config:
-        localpart_template: '{{ user.nickname }}'
-        display_name_template: '{{ user.name }}'
-```
-
-### Gitea
-
-Gitea is, like Github, not an OpenID provider, but just an OAuth2 provider.
-
-The [`/user` API endpoint](https://try.gitea.io/api/swagger#/user/userGetCurrent)
-can be used to retrieve information on the authenticated user. As the Synapse
-login mechanism needs an attribute to uniquely identify users, and that endpoint
-does not return a `sub` property, an alternative `subject_claim` has to be set.
-
-1. Create a new application.
-2. Add this Callback URL: `[synapse public baseurl]/_synapse/client/oidc/callback`
-
-Synapse config:
-
-```yaml
-oidc_providers:
-  - idp_id: gitea
-    idp_name: Gitea
-    discover: false
-    issuer: "https://your-gitea.com/"
-    client_id: "your-client-id" # TO BE FILLED
-    client_secret: "your-client-secret" # TO BE FILLED
-    client_auth_method: client_secret_post
-    scopes: [] # Gitea doesn't support Scopes
-    authorization_endpoint: "https://your-gitea.com/login/oauth/authorize"
-    token_endpoint: "https://your-gitea.com/login/oauth/access_token"
-    userinfo_endpoint: "https://your-gitea.com/api/v1/user"
-    user_mapping_provider:
-      config:
-        subject_claim: "id"
-        localpart_template: "{{ user.login }}"
-        display_name_template: "{{ user.full_name }}"
-```
-
-### Google
-
-[Google][google-idp] is an OpenID certified authentication and authorisation provider.
-
-1. Set up a project in the Google API Console (see 
-   [documentation](https://developers.google.com/identity/protocols/oauth2/openid-connect#appsetup)).
-3. Add an "OAuth Client ID" for a Web Application under "Credentials".
-4. Copy the Client ID and Client Secret, and add the following to your synapse config:
-   ```yaml
-   oidc_providers:
-     - idp_id: google
-       idp_name: Google
-       idp_brand: "google"  # optional: styling hint for clients
-       issuer: "https://accounts.google.com/"
-       client_id: "your-client-id" # TO BE FILLED
-       client_secret: "your-client-secret" # TO BE FILLED
-       scopes: ["openid", "profile", "email"] # email is optional, read below
-       user_mapping_provider:
-         config:
-           localpart_template: "{{ user.given_name|lower }}"
-           display_name_template: "{{ user.name }}"
-           email_template: "{{ user.email }}" # needs "email" in scopes above
-   ```
-4. Back in the Google console, add this Authorized redirect URI: `[synapse
-   public baseurl]/_synapse/client/oidc/callback`.
-
-### Keycloak
-
-[Keycloak][keycloak-idp] is an opensource IdP maintained by Red Hat.
-
-Keycloak supports OIDC Back-Channel Logout, which sends logout notification to Synapse, so that Synapse users get logged out when they log out from Keycloak.
-This can be optionally enabled by setting `backchannel_logout_enabled` to `true` in the Synapse configuration, and by setting the "Backchannel Logout URL" in Keycloak.
-
-Follow the [Getting Started Guide](https://www.keycloak.org/guides) to install Keycloak and set up a realm.
-
-1. Click `Clients` in the sidebar and click `Create`
-
-2. Fill in the fields as below:
-
-| Field | Value |
-|-----------|-----------|
-| Client ID | `synapse` |
-| Client Protocol | `openid-connect` |
-
-3. Click `Save`
-4. Fill in the fields as below:
-
-| Field | Value |
-|-----------|-----------|
-| Client ID | `synapse` |
-| Enabled | `On` |
-| Client Protocol | `openid-connect` |
-| Access Type | `confidential` |
-| Valid Redirect URIs | `[synapse public baseurl]/_synapse/client/oidc/callback` |
-| Backchannel Logout URL (optional) | `[synapse public baseurl]/_synapse/client/oidc/backchannel_logout` |
-| Backchannel Logout Session Required (optional) | `On` |
-
-5. Click `Save`
-6. On the Credentials tab, update the fields:
-
-| Field | Value |
-|-------|-------|
-| Client Authenticator | `Client ID and Secret` |
-
-7. Click `Regenerate Secret`
-8. Copy Secret
-
-```yaml
-oidc_providers:
-  - idp_id: keycloak
-    idp_name: "My KeyCloak server"
-    issuer: "https://127.0.0.1:8443/realms/{realm_name}"
-    client_id: "synapse"
-    client_secret: "copy secret generated from above"
-    scopes: ["openid", "profile"]
-    user_mapping_provider:
-      config:
-        localpart_template: "{{ user.preferred_username }}"
-        display_name_template: "{{ user.name }}"
-    backchannel_logout_enabled: true # Optional
-```
-
-### LemonLDAP
-
-[LemonLDAP::NG][lemonldap] is an open-source IdP solution.
-
-1. Create an OpenID Connect Relying Parties in LemonLDAP::NG
-2. The parameters are:
- Client ID under the basic menu of the new Relying Parties (`Options > Basic >
-  Client ID`)
- Client secret (`Options > Basic > Client secret`)
- JWT Algorithm: RS256 within the security menu of the new Relying Parties
-  (`Options > Security > ID Token signature algorithm` and `Options > Security >
-  Access Token signature algorithm`)
- Scopes: OpenID, Email and Profile
- Allowed redirection addresses for login (`Options > Basic > Allowed
-  redirection addresses for login` ) :
-  `[synapse public baseurl]/_synapse/client/oidc/callback`
-
-Synapse config:
-```yaml
-oidc_providers:
-  - idp_id: lemonldap
-    idp_name: lemonldap
-    discover: true
-    issuer: "https://auth.example.org/" # TO BE FILLED: replace with your domain
-    client_id: "your client id" # TO BE FILLED
-    client_secret: "your client secret" # TO BE FILLED
-    scopes:
-      - "openid"
-      - "profile"
-      - "email"
-    user_mapping_provider:
-      config:
-        localpart_template: "{{ user.preferred_username }}}"
-        # TO BE FILLED: If your users have names in LemonLDAP::NG and you want those in Synapse, this should be replaced with user.name|capitalize or any valid filter.
-        display_name_template: "{{ user.preferred_username|capitalize }}"
-```
-
 ### Mastodon

 [Mastodon](https://docs.joinmastodon.org/) instances provide an [OAuth API](https://docs.joinmastodon.org/spec/oauth/), allowing those instances to be used as a single sign-on provider for Synapse.
@@ -589,81 +631,3 @@ oidc_providers:
 ```

 Note that the fields `client_id` and `client_secret` are taken from the CURL response above.
-
-### Twitch
-
-1. Setup a developer account on [Twitch](https://dev.twitch.tv/)
-2. Obtain the OAuth 2.0 credentials by [creating an app](https://dev.twitch.tv/console/apps/)
-3. Add this OAuth Redirect URL: `[synapse public baseurl]/_synapse/client/oidc/callback`
-
-Synapse config:
-
-```yaml
-oidc_providers:
-  - idp_id: twitch
-    idp_name: Twitch
-    issuer: "https://id.twitch.tv/oauth2/"
-    client_id: "your-client-id" # TO BE FILLED
-    client_secret: "your-client-secret" # TO BE FILLED
-    client_auth_method: "client_secret_post"
-    user_mapping_provider:
-      config:
-        localpart_template: "{{ user.preferred_username }}"
-        display_name_template: "{{ user.name }}"
-```
-
-### Twitter
-
-*Using Twitter as an identity provider requires using Synapse 1.75.0 or later.*
-
-1. Setup a developer account on [Twitter](https://developer.twitter.com/en/portal/dashboard)
-2. Create a project & app.
-3. Enable user authentication and under "Type of App" choose "Web App, Automated App or Bot".
-4. Under "App info" set the callback URL to `[synapse public baseurl]/_synapse/client/oidc/callback`.
-5. Obtain the OAuth 2.0 credentials under the "Keys and tokens" tab, copy the "OAuth 2.0 Client ID and Client Secret"
-
-Synapse config:
-
-```yaml
-oidc_providers:
-  - idp_id: twitter
-    idp_name: Twitter
-    idp_brand: "twitter"  # optional: styling hint for clients
-    discover: false  # Twitter is not OpenID compliant.
-    issuer: "https://twitter.com/"
-    client_id: "your-client-id" # TO BE FILLED
-    client_secret: "your-client-secret" # TO BE FILLED
-    pkce_method: "always"
-    # offline.access providers refresh tokens, tweet.read and users.read needed for userinfo request.
-    scopes: ["offline.access", "tweet.read", "users.read"]
-    authorization_endpoint: https://twitter.com/i/oauth2/authorize
-    token_endpoint: https://api.twitter.com/2/oauth2/token
-    userinfo_endpoint: https://api.twitter.com/2/users/me?user.fields=profile_image_url
-    user_mapping_provider:
-      config:
-        subject_template: "{{ user.data.id }}"
-        localpart_template: "{{ user.data.username }}"
-        display_name_template: "{{ user.data.name }}"
-        picture_template: "{{ user.data.profile_image_url }}"
-```
-
-### XWiki
-
-Install [OpenID Connect Provider](https://extensions.xwiki.org/xwiki/bin/view/Extension/OpenID%20Connect/OpenID%20Connect%20Provider/) extension in your [XWiki](https://www.xwiki.org) instance.
-
-Synapse config:
-
-```yaml
-oidc_providers:
-  - idp_id: xwiki
-    idp_name: "XWiki"
-    issuer: "https://myxwikihost/xwiki/oidc/"
-    client_id: "your-client-id" # TO BE FILLED
-    client_auth_method: none
-    scopes: ["openid", "profile"]
-    user_profile_method: "userinfo_endpoint"
-    user_mapping_provider:
-      config:
-        localpart_template: "{{ user.preferred_username }}"
-        display_name_template: "{{ user.name }}"
-```
@@ -16,7 +16,7 @@ connect to a postgres database.
 -   For other pre-built packages, please consult the documentation from
    the relevant package.
 -   If you installed synapse [in a
-    virtualenv](setup/installation.md#installing-as-a-python-module-from-pypi), you can install
+    virtualenv](setup/installation.md#installing-from-source), you can install
    the library with:

        ~/synapse/env/bin/pip install "matrix-synapse[postgres]"
@@ -46,7 +46,7 @@ when using a containerized Synapse, as that will prevent it from responding
 to proxied traffic.)

 Optionally, you can also set
-[`request_id_header`](./usage/configuration/config_documentation.md#listeners)
+[`request_id_header`](../usage/configuration/config_documentation.md#listeners)
 so that the server extracts and re-uses the same request ID format that the
 reverse proxy is using.

@@ -136,7 +136,7 @@ Unofficial package are built for SLES 15 in the openSUSE:Backports:SLE-15 reposi
 #### ArchLinux

 The quickest way to get up and running with ArchLinux is probably with the community package
-<https://archlinux.org/packages/community/x86_64/matrix-synapse/>, which should pull in most of
+<https://www.archlinux.org/packages/community/any/matrix-synapse/>, which should pull in most of
 the necessary dependencies.

 pip may be outdated (6.0.7-1 and needs to be upgraded to 6.0.8-1 ):
@@ -200,7 +200,7 @@ When following this route please make sure that the [Platform-specific prerequis
 System requirements:

 - POSIX-compliant system (tested on Linux & OS X)
- Python 3.7 or later, up to Python 3.11.
+- Python 3.7 or later, up to Python 3.10.
 - At least 1GB of free RAM if you want to join large public rooms like #matrix:matrix.org

 If building on an uncommon architecture for which pre-built wheels are
@@ -278,7 +278,7 @@ Installing prerequisites on Ubuntu or Debian:
 ```sh
 sudo apt install build-essential python3-dev libffi-dev \
                     python3-pip python3-setuptools sqlite3 \
-                     libssl-dev virtualenv libjpeg-dev libxslt1-dev libicu-dev
+                     libssl-dev virtualenv libjpeg-dev libxslt1-dev
 ```

 ##### ArchLinux
@@ -287,7 +287,7 @@ Installing prerequisites on ArchLinux:

 ```sh
 sudo pacman -S base-devel python python-pip \
-               python-setuptools python-virtualenv sqlite3 icu
+               python-setuptools python-virtualenv sqlite3
 ```

 ##### CentOS/Fedora
@@ -297,8 +297,7 @@ Installing prerequisites on CentOS or Fedora Linux:
 ```sh
 sudo dnf install libtiff-devel libjpeg-devel libzip-devel freetype-devel \
                 libwebp-devel libxml2-devel libxslt-devel libpq-devel \
-                 python3-virtualenv libffi-devel openssl-devel python3-devel \
-                 libicu-devel
+                 python3-virtualenv libffi-devel openssl-devel python3-devel
 sudo dnf groupinstall "Development Tools"
 ```

@@ -311,12 +310,8 @@ You may need to install the latest Xcode developer tools:
 xcode-select --install
 ```

-Some extra dependencies may be needed. You can use Homebrew (https://brew.sh) for them.
-
-You may need to install icu, and make the icu binaries and libraries accessible.
-Please follow [the official instructions of PyICU](https://pypi.org/project/PyICU/) to do so.
-
-On ARM-based Macs you may also need to install libjpeg and libpq:
+On ARM-based Macs you may need to install libjpeg and libpq. 
+You can use Homebrew (https://brew.sh):
 ```sh
 brew install jpeg libpq
 ```
@@ -337,8 +332,7 @@ Installing prerequisites on openSUSE:
 ```sh
 sudo zypper in -t pattern devel_basis
 sudo zypper in python-pip python-setuptools sqlite3 python-virtualenv \
-               python-devel libffi-devel libopenssl-devel libjpeg62-devel \
-               libicu-devel
+               python-devel libffi-devel libopenssl-devel libjpeg62-devel
 ```

 ##### OpenBSD
@@ -120,7 +120,7 @@ specified in the config. It is located at
 ## SAML Mapping Providers

 The SAML mapping provider can be customized by editing the
-[`saml2_config.user_mapping_provider.module`](usage/configuration/config_documentation.md#saml2_config)
+[`saml2_config.user_mapping_provider.module`](docs/usage/configuration/config_documentation.md#saml2_config)
 config option.

 `saml2_config.user_mapping_provider.config` allows you to provide custom
@@ -88,22 +88,6 @@ process, for example:
    dpkg -i matrix-synapse-py3_1.3.0+stretch1_amd64.deb
    ```

-# Upgrading to v1.74.0
-
-## Unicode support in user search
-
-This version introduces optional support for an [improved user search dealing with Unicode characters](https://github.com/matrix-org/synapse/pull/14464).
-
-If you want to take advantage of this feature you need to install PyICU,
-the ICU native dependency and its development headers
-so that PyICU can build since no prebuilt wheels are available.
-
-You can follow [the PyICU documentation](https://pypi.org/project/PyICU/) to do so,
-and then do `pip install matrix-synapse[user-search]` for a PyPI install.
-
-Docker images and Debian packages need nothing specific as they already
-include or specify ICU as an explicit dependency.
-
 # Upgrading to v1.73.0

 ## Legacy Prometheus metric names have now been removed
@@ -889,8 +873,8 @@ Any scripts still using the above APIs should be converted to use the
 ## User-interactive authentication fallback templates can now display errors

 This may affect you if you make use of custom HTML templates for the
-[reCAPTCHA (`synapse/res/templates/recaptcha.html`)](https://github.com/matrix-org/synapse/tree/develop/synapse/res/templates/recaptcha.html) or
-[terms (`synapse/res/templates/terms.html`)](https://github.com/matrix-org/synapse/tree/develop/synapse/res/templates/terms.html) fallback pages.
+[reCAPTCHA](../synapse/res/templates/recaptcha.html) or
+[terms](../synapse/res/templates/terms.html) fallback pages.

 The template is now provided an `error` variable if the authentication
 process failed. See the default templates linked above for an example.
@@ -1488,7 +1472,7 @@ New templates (`sso_auth_confirm.html`, `sso_auth_success.html`, and
 is configured to use SSO and a custom
 `sso_redirect_confirm_template_dir` configuration then these templates
 will need to be copied from
-[`synapse/res/templates`](https://github.com/matrix-org/synapse/tree/develop/synapse/res/templates) into that directory.
+[synapse/res/templates](synapse/res/templates) into that directory.

 ## Synapse SSO Plugins Method Deprecation

@@ -7,7 +7,7 @@ server admin. (Note that a server admin is distinct from a room admin.)

 An existing user can be marked as a server admin by updating the database directly.

-Check your [database settings](../../configuration/config_documentation.md#database) in the configuration file, connect to the correct database using either `psql [database name]` (if using PostgreSQL) or `sqlite3 path/to/your/database.db` (if using SQLite) and elevate the user `@foo:bar.com` to administrator.
+Check your [database settings](config_documentation.md#database) in the configuration file, connect to the correct database using either `psql [database name]` (if using PostgreSQL) or `sqlite3 path/to/your/database.db` (if using SQLite) and elevate the user `@foo:bar.com` to administrator.
 ```sql
 UPDATE users SET admin = 1 WHERE name = '@foo:bar.com';
 ```
@@ -32,10 +32,10 @@ curl --header "Authorization: Bearer <access_token>" <the_rest_of_your_API_reque
 ```

 For example, suppose we want to
-[query the account](../../../admin_api/user_admin_api.md#query-user-account) of the user
+[query the account](user_admin_api.md#query-user-account) of the user
 `@foo:bar.com`. We need an admin access token (e.g.
 `syt_AjfVef2_L33JNpafeif_0feKJfeaf0CQpoZk`), and we need to know which port
-Synapse's [`client` listener](../../configuration/config_documentation.md#listeners) is listening
+Synapse's [`client` listener](config_documentation.md#listeners) is listening
 on (e.g. `8008`). Then we can use the following command to request the account
 information from the Admin API.

@@ -81,7 +81,7 @@ The following fields are returned in the JSON response body:
  - `failure_ts` - nullable integer - The first time Synapse tried and failed to reach the
    remote server, in ms. This is `null` if communication with the remote server has never failed.
  - `last_successful_stream_ordering` - nullable integer - The stream ordering of the most
-    recent successfully-sent [PDU](../understanding_synapse_through_grafana_graphs.md#federation)
+    recent successfully-sent [PDU](understanding_synapse_through_grafana_graphs.md#federation)
    to this destination, or `null` if this information has not been tracked yet.
 - `next_token`: string representing a positive integer - Indication for pagination. See above.
 - `total` - integer - Total number of destinations.
@@ -174,7 +174,7 @@ The following fields are returned in the JSON response body:
  Room objects contain the following fields:
  - `room_id` - string - The ID of the room.
  - `stream_ordering` - integer -  The stream ordering of the most recent
-    successfully-sent [PDU](../understanding_synapse_through_grafana_graphs.md#federation)
+    successfully-sent [PDU](understanding_synapse_through_grafana_graphs.md#federation)
    to this destination in this room.
 - `next_token`: string representing a positive integer - Indication for pagination. See above.
 - `total` - integer - Total number of destinations.
@@ -6,7 +6,7 @@ registration requests, as proposed in
 and stabilised in version 1.2 of the Matrix specification.
 To use it, you will need to enable the `registration_requires_token` config
 option, and authenticate by providing an `access_token` for a server admin:
-see [Admin API](../admin_api/).
+see [Admin API](../admin_api).


 ## Registration token objects
@@ -2,7 +2,7 @@

 How do I become a server admin?
 ---
-If your server already has an admin account you should use the [User Admin API](../../admin_api/user_admin_api.md#change-whether-a-user-is-a-server-administrator-or-not) to promote other accounts to become admins.
+If your server already has an admin account you should use the [User Admin API](../../admin_api/user_admin_api.md#Change-whether-a-user-is-a-server-administrator-or-not) to promote other accounts to become admins.

 If you don't have any admin accounts yet you won't be able to use the admin API, so you'll have to edit the database manually. Manually editing the database is generally not recommended so once you have an admin account: use the admin APIs to make further changes.

@@ -115,7 +115,7 @@ something like the following in their logs:

    2019-09-11 19:32:04,271 - synapse.federation.transport.server - 288 - WARNING - GET-11752 - authenticate_request failed: 401: Invalid signature for server <server> with key ed25519:a_EqML: Unable to verify signature for <server>

-This is normally caused by a misconfiguration in your reverse-proxy. See [the reverse proxy docs](../../reverse_proxy.md) and double-check that your settings are correct.
+This is normally caused by a misconfiguration in your reverse-proxy. See [the reverse proxy docs](docs/reverse_proxy.md) and double-check that your settings are correct.


 Help!! Synapse is slow and eats all my RAM/CPU!
@@ -78,4 +78,4 @@ If you would like to set up your own statistics collection server and send metri
 consider using one of the following known implementations:

 * [Matrix.org's Panopticon](https://github.com/matrix-org/panopticon)
-* [Famedly's Barad-dûr](https://gitlab.com/famedly/infra/services/barad-dur)
+* [Famedly's Barad-dûr](https://gitlab.com/famedly/company/devops/services/barad-dur)
@@ -1,6 +1,6 @@
 # Request log format

-HTTP request logs are written by synapse (see [`synapse/http/site.py`](https://github.com/matrix-org/synapse/tree/develop/synapse/http/site.py) for details).
+HTTP request logs are written by synapse (see [`site.py`](../synapse/http/site.py) for details).

 See the following for how to decode the dense data available from the default logging configuration.

@@ -569,115 +569,6 @@ Example configuration:
 ```yaml
 delete_stale_devices_after: 1y
 ```
---
-### `email`
-
-Configuration for sending emails from Synapse.
-
-Server admins can configure custom templates for email content. See
-[here](../../templates.md) for more information.
-
-This setting has the following sub-options:
-* `smtp_host`: The hostname of the outgoing SMTP server to use. Defaults to 'localhost'.
-* `smtp_port`: The port on the mail server for outgoing SMTP. Defaults to 465 if `force_tls` is true, else 25.
-
-  _Changed in Synapse 1.64.0:_ the default port is now aware of `force_tls`.
-* `smtp_user` and `smtp_pass`: Username/password for authentication to the SMTP server. By default, no
-   authentication is attempted.
-* `force_tls`: By default, Synapse connects over plain text and then optionally upgrades
-   to TLS via STARTTLS. If this option is set to true, TLS is used from the start (Implicit TLS),
-   and the option `require_transport_security` is ignored.
-   It is recommended to enable this if supported by your mail server.
-
-  _New in Synapse 1.64.0._
-* `require_transport_security`: Set to true to require TLS transport security for SMTP.
-   By default, Synapse will connect over plain text, and will then switch to
-   TLS via STARTTLS *if the SMTP server supports it*. If this option is set,
-   Synapse will refuse to connect unless the server supports STARTTLS.
-* `enable_tls`: By default, if the server supports TLS, it will be used, and the server
-   must present a certificate that is valid for 'smtp_host'. If this option
-   is set to false, TLS will not be used.
-* `notif_from`: defines the "From" address to use when sending emails.
-    It must be set if email sending is enabled. The placeholder '%(app)s' will be replaced by the application name,
-    which is normally set in `app_name`, but may be overridden by the
-    Matrix client application. Note that the placeholder must be written '%(app)s', including the
-    trailing 's'.
-* `app_name`: `app_name` defines the default value for '%(app)s' in `notif_from` and email
-   subjects. It defaults to 'Matrix'.
-* `enable_notifs`: Set to true to enable sending emails for messages that the user
-   has missed. Disabled by default.
-* `notif_for_new_users`: Set to false to disable automatic subscription to email
-   notifications for new users. Enabled by default.
-* `client_base_url`: Custom URL for client links within the email notifications. By default
-   links will be based on "https://matrix.to". (This setting used to be called `riot_base_url`;
-   the old name is still supported for backwards-compatibility but is now deprecated.)
-* `validation_token_lifetime`: Configures the time that a validation email will expire after sending.
-   Defaults to 1h.
-* `invite_client_location`: The web client location to direct users to during an invite. This is passed
-   to the identity server as the `org.matrix.web_client_location` key. Defaults
-   to unset, giving no guidance to the identity server.
-* `subjects`: Subjects to use when sending emails from Synapse. The placeholder '%(app)s' will
-   be replaced with the value of the `app_name` setting, or by a value dictated by the Matrix client application.
-   In addition, each subject can use the following placeholders: '%(person)s', which will be replaced by the displayname
-   of the user(s) that sent the message(s), e.g. "Alice and Bob", and '%(room)s', which will be replaced by the name of the room the
-   message(s) have been sent to, e.g. "My super room". In addition, emails related to account administration will
-   can use the '%(server_name)s' placeholder, which will be replaced by the value of the
-   `server_name` setting in your Synapse configuration.
-
-   Here is a list of subjects for notification emails that can be set:
-     * `message_from_person_in_room`: Subject to use to notify about one message from one or more user(s) in a
-        room which has a name. Defaults to "[%(app)s] You have a message on %(app)s from %(person)s in the %(room)s room..."
-     * `message_from_person`: Subject to use to notify about one message from one or more user(s) in a
-        room which doesn't have a name. Defaults to "[%(app)s] You have a message on %(app)s from %(person)s..."
-     * `messages_from_person`: Subject to use to notify about multiple messages from one or more users in
-        a room which doesn't have a name. Defaults to "[%(app)s] You have messages on %(app)s from %(person)s..."
-     * `messages_in_room`: Subject to use to notify about multiple messages in a room which has a
-        name. Defaults to "[%(app)s] You have messages on %(app)s in the %(room)s room..."
-     * `messages_in_room_and_others`: Subject to use to notify about multiple messages in multiple rooms.
-        Defaults to "[%(app)s] You have messages on %(app)s in the %(room)s room and others..."
-     * `messages_from_person_and_others`: Subject to use to notify about multiple messages from multiple persons in
-        multiple rooms. This is similar to the setting above except it's used when
-        the room in which the notification was triggered has no name. Defaults to
-        "[%(app)s] You have messages on %(app)s from %(person)s and others..."
-     * `invite_from_person_to_room`: Subject to use to notify about an invite to a room which has a name.
-        Defaults to  "[%(app)s] %(person)s has invited you to join the %(room)s room on %(app)s..."
-     * `invite_from_person`: Subject to use to notify about an invite to a room which doesn't have a
-        name. Defaults to "[%(app)s] %(person)s has invited you to chat on %(app)s..."
-     * `password_reset`: Subject to use when sending a password reset email. Defaults to "[%(server_name)s] Password reset"
-     * `email_validation`: Subject to use when sending a verification email to assert an address's
-        ownership. Defaults to "[%(server_name)s] Validate your email"
-
-Example configuration:
-
-```yaml
-email:
-  smtp_host: mail.server
-  smtp_port: 587
-  smtp_user: "exampleusername"
-  smtp_pass: "examplepassword"
-  force_tls: true
-  require_transport_security: true
-  enable_tls: false
-  notif_from: "Your Friendly %(app)s homeserver <noreply@example.com>"
-  app_name: my_branded_matrix_server
-  enable_notifs: true
-  notif_for_new_users: false
-  client_base_url: "http://localhost/riot"
-  validation_token_lifetime: 15m
-  invite_client_location: https://app.element.io
-
-  subjects:
-    message_from_person_in_room: "[%(app)s] You have a message on %(app)s from %(person)s in the %(room)s room..."
-    message_from_person: "[%(app)s] You have a message on %(app)s from %(person)s..."
-    messages_from_person: "[%(app)s] You have messages on %(app)s from %(person)s..."
-    messages_in_room: "[%(app)s] You have messages on %(app)s in the %(room)s room..."
-    messages_in_room_and_others: "[%(app)s] You have messages on %(app)s in the %(room)s room and others..."
-    messages_from_person_and_others: "[%(app)s] You have messages on %(app)s from %(person)s and others..."
-    invite_from_person_to_room: "[%(app)s] %(person)s has invited you to join the %(room)s room on %(app)s..."
-    invite_from_person: "[%(app)s] %(person)s has invited you to chat on %(app)s..."
-    password_reset: "[%(server_name)s] Password reset"
-    email_validation: "[%(server_name)s] Validate your email"
-```

 ## Homeserver blocking
 Useful options for Synapse admins.
@@ -1257,7 +1148,7 @@ number of entries that can be stored.
     * `max_cache_memory_usage` sets a ceiling on how much memory the cache can use before caches begin to be continuously evicted.
        They will continue to be evicted until the memory usage drops below the `target_memory_usage`, set in
        the setting below, or until the `min_cache_ttl` is hit. There is no default value for this option.
-     * `target_cache_memory_usage` sets a rough target for the desired memory usage of the caches. There is no default value
+     * `target_memory_usage` sets a rough target for the desired memory usage of the caches. There is no default value
        for this option.
     * `min_cache_ttl` sets a limit under which newer cache entries are not evicted and is only applied when
        caches are actively being evicted/`max_cache_memory_usage` has been exceeded. This is to protect hot caches
@@ -1321,7 +1212,7 @@ Associated sub-options:
  connection pool. For a reference to valid arguments, see:
    * for [sqlite](https://docs.python.org/3/library/sqlite3.html#sqlite3.connect)
    * for [postgres](https://www.postgresql.org/docs/current/libpq-connect.html#LIBPQ-PARAMKEYWORDS)
-    * for [the connection pool](https://docs.twistedmatrix.com/en/stable/api/twisted.enterprise.adbapi.ConnectionPool.html#__init__)
+    * for [the connection pool](https://twistedmatrix.com/documents/current/api/twisted.enterprise.adbapi.ConnectionPool.html#__init__)

 For more information on using Synapse with Postgres,
 see [here](../../postgres.md).
@@ -2623,18 +2514,18 @@ state events are shared with users:
 - `m.room.topic`

 To change the default behavior, use the following sub-options:
-* `disable_default_event_types`: boolean. Set to `true` to disable the above
+* `disable_default_event_types`: boolean. Set to `true` to disable the above 
  defaults. If this is enabled, only the event types listed in
  `additional_event_types` are shared. Defaults to `false`.
-* `additional_event_types`: A list of additional state events to include in the
-  events to be shared. By default, this list is empty (so only the default event
+* `additional_event_types`: A list of additional state events to include in the 
+  events to be shared. By default, this list is empty (so only the default event 
  types are shared).

  Each entry in this list should be either a single string or a list of two
-  strings.
+  strings. 
  * A standalone string `t` represents all events with type `t` (i.e.
    with no restrictions on state keys).
-  * A pair of strings `[t, s]` represents a single event with type `t` and
+  * A pair of strings `[t, s]` represents a single event with type `t` and 
    state key `s`. The same type can appear in two entries with different state
    keys: in this situation, both state keys are included in prejoin state.

@@ -3053,13 +2944,8 @@ Options for each entry include:
   values are `client_secret_basic` (default), `client_secret_post` and
   `none`.

-* `pkce_method`: Whether to use proof key for code exchange when requesting
-   and exchanging the token. Valid values are: `auto`, `always`, or `never`. Defaults
-   to `auto`, which uses PKCE if supported during metadata discovery. Set to `always`
-   to force enable PKCE or `never` to force disable PKCE.
-
 * `scopes`: list of scopes to request. This should normally include the "openid"
-   scope. Defaults to `["openid"]`.
+   scope. Defaults to ["openid"].

 * `authorization_endpoint`: the oauth2 authorization endpoint. Required if
   provider discovery is disabled.
@@ -3103,35 +2989,17 @@ Options for each entry include:

        For the default provider, the following settings are available:

-       * `subject_template`: Jinja2 template for a unique identifier for the user.
-         Defaults to `{{ user.sub }}`, which OpenID Connect compliant providers should provide.
-
-         This replaces and overrides `subject_claim`.
-
       * `subject_claim`: name of the claim containing a unique identifier
         for the user. Defaults to 'sub', which OpenID Connect
         compliant providers should provide.

-         *Deprecated in Synapse v1.75.0.*
-
-       * `picture_template`: Jinja2 template for an url for the user's profile picture.
-         Defaults to `{{ user.picture }}`, which OpenID Connect compliant providers should
-         provide and has to refer to a direct image file such as PNG, JPEG, or GIF image file.
-
-         This replaces and overrides `picture_claim`.
-
-         Currently only supported in monolithic (single-process) server configurations
-         where the media repository runs within the Synapse process.
-
       * `picture_claim`: name of the claim containing an url for the user's profile picture.
         Defaults to 'picture', which OpenID Connect compliant providers should provide
         and has to refer to a direct image file such as PNG, JPEG, or GIF image file.
-
+         
         Currently only supported in monolithic (single-process) server configurations
         where the media repository runs within the Synapse process.

-         *Deprecated in Synapse v1.75.0.*
-
       * `localpart_template`: Jinja2 template for the localpart of the MXID.
          If this is not set, the user will be prompted to choose their
          own username (see the documentation for the `sso_auth_account_details.html`
@@ -3391,6 +3259,114 @@ ui_auth:
    session_timeout: "15s"
 ```
 ---
+### `email`
+
+Configuration for sending emails from Synapse.
+
+Server admins can configure custom templates for email content. See
+[here](../../templates.md) for more information.
+
+This setting has the following sub-options:
+* `smtp_host`: The hostname of the outgoing SMTP server to use. Defaults to 'localhost'.
+* `smtp_port`: The port on the mail server for outgoing SMTP. Defaults to 465 if `force_tls` is true, else 25.
+
+  _Changed in Synapse 1.64.0:_ the default port is now aware of `force_tls`.
+* `smtp_user` and `smtp_pass`: Username/password for authentication to the SMTP server. By default, no
+   authentication is attempted.
+* `force_tls`: By default, Synapse connects over plain text and then optionally upgrades
+   to TLS via STARTTLS. If this option is set to true, TLS is used from the start (Implicit TLS),
+   and the option `require_transport_security` is ignored.
+   It is recommended to enable this if supported by your mail server.
+
+  _New in Synapse 1.64.0._
+* `require_transport_security`: Set to true to require TLS transport security for SMTP.
+   By default, Synapse will connect over plain text, and will then switch to
+   TLS via STARTTLS *if the SMTP server supports it*. If this option is set,
+   Synapse will refuse to connect unless the server supports STARTTLS.
+* `enable_tls`: By default, if the server supports TLS, it will be used, and the server
+   must present a certificate that is valid for 'smtp_host'. If this option
+   is set to false, TLS will not be used.
+* `notif_from`: defines the "From" address to use when sending emails.
+    It must be set if email sending is enabled. The placeholder '%(app)s' will be replaced by the application name,
+    which is normally set in `app_name`, but may be overridden by the
+    Matrix client application. Note that the placeholder must be written '%(app)s', including the
+    trailing 's'.
+* `app_name`: `app_name` defines the default value for '%(app)s' in `notif_from` and email
+   subjects. It defaults to 'Matrix'.
+* `enable_notifs`: Set to true to enable sending emails for messages that the user
+   has missed. Disabled by default.
+* `notif_for_new_users`: Set to false to disable automatic subscription to email
+   notifications for new users. Enabled by default.
+* `client_base_url`: Custom URL for client links within the email notifications. By default
+   links will be based on "https://matrix.to". (This setting used to be called `riot_base_url`;
+   the old name is still supported for backwards-compatibility but is now deprecated.)
+* `validation_token_lifetime`: Configures the time that a validation email will expire after sending.
+   Defaults to 1h.
+* `invite_client_location`: The web client location to direct users to during an invite. This is passed
+   to the identity server as the `org.matrix.web_client_location` key. Defaults
+   to unset, giving no guidance to the identity server.
+* `subjects`: Subjects to use when sending emails from Synapse. The placeholder '%(app)s' will
+   be replaced with the value of the `app_name` setting, or by a value dictated by the Matrix client application.
+   In addition, each subject can use the following placeholders: '%(person)s', which will be replaced by the displayname
+   of the user(s) that sent the message(s), e.g. "Alice and Bob", and '%(room)s', which will be replaced by the name of the room the
+   message(s) have been sent to, e.g. "My super room". In addition, emails related to account administration will
+   can use the '%(server_name)s' placeholder, which will be replaced by the value of the
+   `server_name` setting in your Synapse configuration.
+
+   Here is a list of subjects for notification emails that can be set:
+     * `message_from_person_in_room`: Subject to use to notify about one message from one or more user(s) in a
+        room which has a name. Defaults to "[%(app)s] You have a message on %(app)s from %(person)s in the %(room)s room..."
+     * `message_from_person`: Subject to use to notify about one message from one or more user(s) in a
+        room which doesn't have a name. Defaults to "[%(app)s] You have a message on %(app)s from %(person)s..."
+     * `messages_from_person`: Subject to use to notify about multiple messages from one or more users in
+        a room which doesn't have a name. Defaults to "[%(app)s] You have messages on %(app)s from %(person)s..."
+     * `messages_in_room`: Subject to use to notify about multiple messages in a room which has a
+        name. Defaults to "[%(app)s] You have messages on %(app)s in the %(room)s room..."
+     * `messages_in_room_and_others`: Subject to use to notify about multiple messages in multiple rooms.
+        Defaults to "[%(app)s] You have messages on %(app)s in the %(room)s room and others..."
+     * `messages_from_person_and_others`: Subject to use to notify about multiple messages from multiple persons in
+        multiple rooms. This is similar to the setting above except it's used when
+        the room in which the notification was triggered has no name. Defaults to
+        "[%(app)s] You have messages on %(app)s from %(person)s and others..."
+     * `invite_from_person_to_room`: Subject to use to notify about an invite to a room which has a name.
+        Defaults to  "[%(app)s] %(person)s has invited you to join the %(room)s room on %(app)s..."
+     * `invite_from_person`: Subject to use to notify about an invite to a room which doesn't have a
+        name. Defaults to "[%(app)s] %(person)s has invited you to chat on %(app)s..."
+     * `password_reset`: Subject to use when sending a password reset email. Defaults to "[%(server_name)s] Password reset"
+     * `email_validation`: Subject to use when sending a verification email to assert an address's
+        ownership. Defaults to "[%(server_name)s] Validate your email"
+
+Example configuration:
+```yaml
+email:
+  smtp_host: mail.server
+  smtp_port: 587
+  smtp_user: "exampleusername"
+  smtp_pass: "examplepassword"
+  force_tls: true
+  require_transport_security: true
+  enable_tls: false
+  notif_from: "Your Friendly %(app)s homeserver <noreply@example.com>"
+  app_name: my_branded_matrix_server
+  enable_notifs: true
+  notif_for_new_users: false
+  client_base_url: "http://localhost/riot"
+  validation_token_lifetime: 15m
+  invite_client_location: https://app.element.io
+
+  subjects:
+    message_from_person_in_room: "[%(app)s] You have a message on %(app)s from %(person)s in the %(room)s room..."
+    message_from_person: "[%(app)s] You have a message on %(app)s from %(person)s..."
+    messages_from_person: "[%(app)s] You have messages on %(app)s from %(person)s..."
+    messages_in_room: "[%(app)s] You have messages on %(app)s in the %(room)s room..."
+    messages_in_room_and_others: "[%(app)s] You have messages on %(app)s in the %(room)s room and others..."
+    messages_from_person_and_others: "[%(app)s] You have messages on %(app)s from %(person)s and others..."
+    invite_from_person_to_room: "[%(app)s] %(person)s has invited you to join the %(room)s room on %(app)s..."
+    invite_from_person: "[%(app)s] %(person)s has invited you to chat on %(app)s..."
+    password_reset: "[%(server_name)s] Password reset"
+    email_validation: "[%(server_name)s] Validate your email"
+```
+---
 ## Push
 Configuration settings related to push notifications

@@ -3864,48 +3840,6 @@ Example configuration:
 ```yaml
 run_background_tasks_on: worker1
 ```
---
-### `update_user_directory_from_worker`
-
-The [worker](../../workers.md#updating-the-user-directory) that is used to
-update the user directory. If not provided this defaults to the main process.
-
-Example configuration:
-```yaml
-update_user_directory_from_worker: worker1
-```
-
-_Added in Synapse 1.59.0._
-
---
-### `notify_appservices_from_worker`
-
-The [worker](../../workers.md#notifying-application-services) that is used to
-send output traffic to Application Services. If not provided this defaults
-to the main process.
-
-Example configuration:
-```yaml
-notify_appservices_from_worker: worker1
-```
-
-_Added in Synapse 1.59.0._
-
---
-### `media_instance_running_background_jobs`
-
-The [worker](../../workers.md#synapseappmedia_repository) that is used to run
-background tasks for media repository. If running multiple media repositories
-you must configure a single instance to run the background tasks. If not provided
-this defaults to the main process or your single `media_repository` worker.
-
-Example configuration:
-```yaml
-media_instance_running_background_jobs: worker1
-```
-
-_Added in Synapse 1.16.0._
-
 ---
 ### `redis`

@@ -4023,7 +3957,7 @@ worker_listeners:
 ### `worker_daemonize`

 Specifies whether the worker should be started as a daemon process.
-If Synapse is being managed by [systemd](../../systemd-with-workers/), this option
+If Synapse is being managed by [systemd](../../systemd-with-workers/README.md), this option
 must be omitted or set to `false`.

 Defaults to `false`.
@@ -24,11 +24,6 @@ Finally, we also stylise the chapter titles in the left sidebar by indenting the
 slightly so that they are more visually distinguishable from the section headers
 (the bold titles). This is done through the `indent-section-headers.css` file.

-In addition to these modifications, we have added a version picker to the documentation.
-Users can switch between documentations for different versions of Synapse.
-This functionality was implemented through the `version-picker.js` and
-`version-picker.css` files.
-
 More information can be found in mdbook's official documentation for
 [injecting page JS/CSS](https://rust-lang.github.io/mdBook/format/config.html)
 and
@@ -131,18 +131,6 @@
                            <i class="fa fa-search"></i>
                        </button>
                        {{/if}}
-                        <div class="version-picker">
-                            <div class="dropdown">
-                                <div class="select">
-                                    <span></span>
-                                    <i class="fa fa-chevron-down"></i>
-                                </div>
-                                <input type="hidden" name="version">
-                                <ul class="dropdown-menu">
-                                    <!-- Versions will be added dynamically in version-picker.js -->
-                                </ul>
-                            </div>
-                        </div>      
                    </div>

                    <h1 class="menu-title">{{ book_title }}</h1>
@@ -321,4 +309,4 @@
        {{/if}}

    </body>
-</html>
+</html>
@@ -1,78 +0,0 @@
-.version-picker {
-    display: flex;
-    align-items: center;
-}
-
-.version-picker .dropdown {
-    width: 130px;
-    max-height: 29px;
-    margin-left: 10px;
-    display: inline-block;
-    border-radius: 4px;
-    border: 1px solid var(--theme-popup-border);
-    position: relative;
-    font-size: 13px;
-    color: var(--fg);
-    height: 100%;
-    text-align: left;
-}
-.version-picker .dropdown .select {
-    cursor: pointer;
-    display: block;
-    padding: 5px 2px 5px 15px;
-}
-.version-picker .dropdown .select > i {
-    font-size: 10px;
-    color: var(--fg);
-    cursor: pointer;
-    float: right;
-    line-height: 20px !important;
-}
-.version-picker .dropdown:hover {
-    border: 1px solid var(--theme-popup-border);
-}
-.version-picker .dropdown:active {
-    background-color: var(--theme-popup-bg);
-}
-.version-picker .dropdown.active:hover,
-.version-picker .dropdown.active {
-    border: 1px solid var(--theme-popup-border);
-    border-radius: 2px 2px 0 0;
-    background-color: var(--theme-popup-bg);
-}
-.version-picker .dropdown.active .select > i {
-    transform: rotate(-180deg);
-}
-.version-picker .dropdown .dropdown-menu {
-    position: absolute;
-    background-color: var(--theme-popup-bg);
-    width: 100%;
-    left: -1px;
-    right: 1px;
-    margin-top: 1px;
-    border: 1px solid var(--theme-popup-border);
-    border-radius: 0 0 4px 4px;
-    overflow: hidden;
-    display: none;
-    max-height: 300px;
-    overflow-y: auto;
-    z-index: 9;
-}
-.version-picker .dropdown .dropdown-menu li {
-    font-size: 12px;
-    padding: 6px 20px;
-    cursor: pointer;
-} 
-.version-picker .dropdown .dropdown-menu {
-    padding: 0;
-    list-style: none;
-}
-.version-picker .dropdown .dropdown-menu li:hover {
-    background-color: var(--theme-hover);
-}
-.version-picker .dropdown .dropdown-menu li.active::before {
-    display: inline-block;
-    content: "✓";
-    margin-inline-start: -14px;
-    width: 14px;
-}
@@ -1,127 +0,0 @@
-
-const dropdown = document.querySelector('.version-picker .dropdown');
-const dropdownMenu = dropdown.querySelector('.dropdown-menu');
-
-fetchVersions(dropdown, dropdownMenu).then(() => {
-    initializeVersionDropdown(dropdown, dropdownMenu);
-});
-
-/**
- * Initialize the dropdown functionality for version selection.
- * 
- * @param {Element} dropdown - The dropdown element.
- * @param {Element} dropdownMenu - The dropdown menu element.
- */
-function initializeVersionDropdown(dropdown, dropdownMenu) {
-    // Toggle the dropdown menu on click
-    dropdown.addEventListener('click', function () {
-        this.setAttribute('tabindex', 1);
-        this.classList.toggle('active');
-        dropdownMenu.style.display = (dropdownMenu.style.display === 'block') ? 'none' : 'block';
-    });
-  
-    // Remove the 'active' class and hide the dropdown menu on focusout
-    dropdown.addEventListener('focusout', function () {
-        this.classList.remove('active');
-        dropdownMenu.style.display = 'none';
-    });
-  
-    // Handle item selection within the dropdown menu
-    const dropdownMenuItems = dropdownMenu.querySelectorAll('li');    
-    dropdownMenuItems.forEach(function (item) {
-        item.addEventListener('click', function () {
-            dropdownMenuItems.forEach(function (item) {
-                item.classList.remove('active');
-            });
-            this.classList.add('active');
-            dropdown.querySelector('span').textContent = this.textContent;
-            dropdown.querySelector('input').value = this.getAttribute('id');
-
-            window.location.href = changeVersion(window.location.href, this.textContent);
-        });
-    });
-};
-
-/**
- * This function fetches the available versions from a GitHub repository
- * and inserts them into the version picker.
- * 
- * @param {Element} dropdown - The dropdown element.
- * @param {Element} dropdownMenu - The dropdown menu element.
- * @returns {Promise<Array<string>>} A promise that resolves with an array of available versions.
- */
-function fetchVersions(dropdown, dropdownMenu) {
-    return new Promise((resolve, reject) => {
-        window.addEventListener("load", () => {
-
-            fetch("https://api.github.com/repos/matrix-org/synapse/git/trees/gh-pages", {
-                cache: "force-cache",
-            }).then(res => 
-                res.json()
-            ).then(resObject => {
-                const excluded = ['dev-docs', 'v1.91.0', 'v1.80.0', 'v1.69.0'];
-                const tree = resObject.tree.filter(item => item.type === "tree" && !excluded.includes(item.path));
-                const versions = tree.map(item => item.path).sort(sortVersions);
-
-                // Create a list of <li> items for versions
-                versions.forEach((version) => {
-                    const li = document.createElement("li");
-                    li.textContent = version;
-                    li.id = version;
-    
-                    if (window.SYNAPSE_VERSION === version) {
-                        li.classList.add('active');
-                        dropdown.querySelector('span').textContent = version;
-                        dropdown.querySelector('input').value = version;
-                    }
-    
-                    dropdownMenu.appendChild(li);
-                });
-
-                resolve(versions);
-
-            }).catch(ex => {
-                console.error("Failed to fetch version data", ex);
-                reject(ex);
-            })
-        });
-    });
-}
-
-/**
- * Custom sorting function to sort an array of version strings.
- *
- * @param {string} a - The first version string to compare.
- * @param {string} b - The second version string to compare.
- * @returns {number} - A negative number if a should come before b, a positive number if b should come before a, or 0 if they are equal.
- */
-function sortVersions(a, b) {
-    // Put 'develop' and 'latest' at the top
-    if (a === 'develop' || a === 'latest') return -1;
-    if (b === 'develop' || b === 'latest') return 1;
-
-    const versionA = (a.match(/v\d+(\.\d+)+/) || [])[0];
-    const versionB = (b.match(/v\d+(\.\d+)+/) || [])[0];
-
-    return versionB.localeCompare(versionA);
-}
-
-/**
- * Change the version in a URL path.
- *
- * @param {string} url - The original URL to be modified.
- * @param {string} newVersion - The new version to replace the existing version in the URL.
- * @returns {string} The updated URL with the new version.
- */
-function changeVersion(url, newVersion) {
-    const parsedURL = new URL(url);
-    const pathSegments = parsedURL.pathname.split('/');
-  
-    // Modify the version
-    pathSegments[2] = newVersion;
-
-    // Reconstruct the URL
-    parsedURL.pathname = pathSegments.join('/');
-  
-    return parsedURL.href;
-}
@@ -1 +0,0 @@
-window.SYNAPSE_VERSION = 'v1.75';
@@ -157,7 +157,7 @@ Finally, you need to start your worker processes. This can be done with either
 `synctl` or your distribution's preferred service manager such as `systemd`. We
 recommend the use of `systemd` where available: for information on setting up
 `systemd` to start synapse workers, see
-[Systemd with Workers](systemd-with-workers/). To use `synctl`, see
+[Systemd with Workers](systemd-with-workers). To use `synctl`, see
 [Using synctl with Workers](synctl_workers.md).


@@ -386,7 +386,7 @@ so. It will then pass those events over HTTP replication to any configured event
 persisters (or the main process if none are configured).

 Note that `event_creator`s and `event_persister`s are implemented using the same
-[`synapse.app.generic_worker`](#synapseappgeneric_worker).
+[`synapse.app.generic_worker`](#synapse.app.generic_worker).

 An example [`stream_writers`](usage/configuration/config_documentation.md#stream_writers)
 configuration with multiple writers:
@@ -465,8 +465,7 @@ An example for a dedicated background worker instance:

 You can designate one generic worker to update the user directory.

-Specify its name in the [shared configuration](usage/configuration/config_documentation.md#update_user_directory_from_worker)
-as follows:
+Specify its name in the shared configuration as follows:

 ```yaml
 update_user_directory_from_worker: worker_name
@@ -491,8 +490,7 @@ worker application type.

 You can designate one generic worker to send output traffic to Application Services.
 Doesn't handle any REST endpoints itself, but you should specify its name in the
-[shared configuration](usage/configuration/config_documentation.md#notify_appservices_from_worker)
-as follows:
+shared configuration as follows:

 ```yaml
 notify_appservices_from_worker: worker_name
@@ -504,38 +502,11 @@ after setting this option in the shared configuration!
 This style of configuration supersedes the legacy `synapse.app.appservice`
 worker application type.

-#### Push Notifications
-
-You can designate generic worker to sending push notifications to
-a [push gateway](https://spec.matrix.org/v1.5/push-gateway-api/) such as
-[sygnal](https://github.com/matrix-org/sygnal) and email.
-
-This will stop the main process sending push notifications.
-
-The workers responsible for sending push notifications can be defined using the
-[`pusher_instances`](usage/configuration/config_documentation.md#pusher_instances)
-option. For example:
-
-```yaml
-pusher_instances:
-  - pusher_worker1
-  - pusher_worker2
-```
-
-Multiple workers can be added to this map, in which case the work is balanced
-across them. Ensure the main process and all pusher workers are restarted after changing
-this option.
-
-These workers don't need to accept incoming HTTP requests to send push notifications,
-so no additional reverse proxy configuration is required for pusher workers.
-
-This style of configuration supersedes the legacy `synapse.app.pusher`
-worker application type.

 ### `synapse.app.pusher`

 It is likely this option will be deprecated in the future and is not recommended for new
-installations. Instead, [use `synapse.app.generic_worker` with the `pusher_instances`](#push-notifications).
+installations. Instead, [use `synapse.app.generic_worker` with the `pusher_instances`](usage/configuration/config_documentation.md#pusher_instances).

 Handles sending push notifications to sygnal and email. Doesn't handle any
 REST endpoints itself, but you should set
@@ -576,7 +547,7 @@ Note this worker cannot be load-balanced: only one instance should be active.
 ### `synapse.app.federation_sender`

 It is likely this option will be deprecated in the future and not recommended for
-new installations. Instead, [use `synapse.app.generic_worker` with the `federation_sender_instances`](usage/configuration/config_documentation.md#federation_sender_instances).
+new installations. Instead, [use `synapse.app.generic_worker` with the `federation_sender_instances`](usage/configuration/config_documentation.md#federation_sender_instances). 

 Handles sending federation traffic to other servers. Doesn't handle any
 REST endpoints itself, but you should set
@@ -635,9 +606,7 @@ expose the `media` resource. For example:
 ```

 Note that if running multiple media repositories they must be on the same server
-and you must specify a single instance to run the background tasks in the
-[shared configuration](usage/configuration/config_documentation.md#media_instance_running_background_jobs),
-e.g.:
+and you must configure a single instance to run the background tasks, e.g.:

 ```yaml
 media_instance_running_background_jobs: "media-repository-1"
@@ -36,6 +36,9 @@ exclude = (?x)
   |tests/api/test_ratelimiting.py
   |tests/app/test_openid_listener.py
   |tests/appservice/test_scheduler.py
+   |tests/config/test_cache.py
+   |tests/config/test_tls.py
+   |tests/crypto/test_keyring.py
   |tests/events/test_presence_router.py
   |tests/events/test_utils.py
   |tests/federation/test_federation_catch_up.py
@@ -86,19 +89,19 @@ disallow_untyped_defs = False
 [mypy-tests.*]
 disallow_untyped_defs = False

-[mypy-tests.config.*]
-disallow_untyped_defs = True
-
-[mypy-tests.crypto.*]
+[mypy-tests.config.test_api]
 disallow_untyped_defs = True

 [mypy-tests.federation.transport.test_client]
 disallow_untyped_defs = True

-[mypy-tests.handlers.*]
+[mypy-tests.handlers.test_sso]
 disallow_untyped_defs = True

-[mypy-tests.metrics.*]
+[mypy-tests.handlers.test_user_directory]
+disallow_untyped_defs = True
+
+[mypy-tests.metrics.test_background_process_metrics]
 disallow_untyped_defs = True

 [mypy-tests.push.test_bulk_push_rule_evaluator]
@@ -174,6 +177,9 @@ ignore_missing_imports = True
 [mypy-saml2.*]
 ignore_missing_imports = True

+[mypy-scalene.*]
+ignore_missing_imports = True
+
 [mypy-service_identity.*]
 ignore_missing_imports = True

@@ -40,46 +40,6 @@ target-version = ['py37', 'py38', 'py39', 'py310']
 # https://black.readthedocs.io/en/stable/usage_and_configuration/file_collection_and_discovery.html#gitignore
 # Use `extend-exclude` if you want to exclude something in addition to this.

-[tool.ruff]
-line-length = 88
-
-# See https://github.com/charliermarsh/ruff/#pycodestyle
-# for error codes. The ones we ignore are:
-#  E731: do not assign a lambda expression, use a def
-#  E501: Line too long (black enforces this for us)
-#
-# See https://github.com/charliermarsh/ruff/#pyflakes
-#  F401: unused import
-#  F811: Redefinition of unused
-#  F821: Undefined name
-#
-# flake8-bugbear compatible checks. Its error codes are described at
-# https://github.com/charliermarsh/ruff/#flake8-bugbear
-#  B019: Use of functools.lru_cache or functools.cache on methods can lead to memory leaks
-#  B023: Functions defined inside a loop must not use variables redefined in the loop
-#  B024: Abstract base class with no abstract method.
-ignore = [
-    "B019",
-    "B023",
-    "B024",
-    "E501",
-    "E731",
-    "F401",
-    "F811",
-    "F821",
-]
-select = [
-    # pycodestyle checks.
-    "E",
-    "W",
-    # pyflakes checks.
-    "F",
-    # flake8-bugbear checks.
-    "B0",
-    # flake8-comprehensions checks.
-    "C4",
-]
-
 [tool.isort]
 line_length = 88
 sections = ["FUTURE", "STDLIB", "THIRDPARTY", "TWISTED", "FIRSTPARTY", "TESTS", "LOCALFOLDER"]
@@ -97,7 +57,7 @@ manifest-path = "rust/Cargo.toml"

 [tool.poetry]
 name = "matrix-synapse"
-version = "1.75.0"
+version = "1.74.0rc1"
 description = "Homeserver for the Matrix decentralised comms protocol"
 authors = ["Matrix.org Team and Contributors <packages@matrix.org>"]
 license = "Apache-2.0"
@@ -176,7 +136,7 @@ Twisted = {extras = ["tls"], version = ">=18.9.0"}
 treq = ">=15.1"
 # Twisted has required pyopenssl 16.0 since about Twisted 16.6.
 pyOpenSSL = ">=16.0.0"
-PyYAML = ">=3.13"
+PyYAML = ">=3.11"
 pyasn1 = ">=0.1.9"
 pyasn1-modules = ">=0.0.7"
 bcrypt = ">=3.1.7"
@@ -250,6 +210,8 @@ parameterized = { version = ">=0.7.4", optional = true }
 idna = { version = ">=2.5", optional = true }
 pyicu = { version = ">=2.10.2", optional = true }

+scalene = { version = ">=1.5.16", optional = true, python = ">=3.8,<4.0.0" }
+
 [tool.poetry.extras]
 # NB: Packages that should be part of `pip install matrix-synapse[all]` need to be specified
 # twice: once here, and once in the `all` extra.
@@ -276,6 +238,8 @@ test = ["parameterized", "idna"]
 # Debian-based distributions).
 user-search = ["pyicu"]

+scalene = ["scalene"]
+
 # The duplication here is awful. I hate hate hate hate hate it. However, for now I want
 # to ensure you can still `pip install matrix-synapse[all]` like today. Two motivations:
 # 1) for new installations, I want instructions in existing documentation and tutorials
@@ -314,10 +278,12 @@ all = [
 ]

 [tool.poetry.dev-dependencies]
-# We pin black so that our tests don't start failing on new releases.
+## We pin black so that our tests don't start failing on new releases.
 isort = ">=5.10.1"
 black = ">=22.3.0"
-ruff = "0.0.215"
+flake8-comprehensions = "*"
+flake8-bugbear = ">=21.3.2"
+flake8 = "*"

 # Typechecking
 mypy = "*"
@@ -190,7 +190,7 @@ fi

 extra_test_args=()

-test_tags="synapse_blacklist,msc3787,msc3874,msc3391"
+test_tags="synapse_blacklist,msc3787,msc3874"

 # All environment variables starting with PASS_ will be shared.
 # (The prefix is stripped off before reaching the container.)
@@ -1,8 +1,9 @@
 #!/usr/bin/env bash
 #
 # Runs linting scripts over the local Synapse checkout
+# isort - sorts import statements
 # black - opinionated code formatter
-# ruff - lints and finds mistakes
+# flake8 - lints and finds mistakes

 set -e

@@ -104,7 +105,6 @@ set -x
 isort "${files[@]}"
 python3 -m black "${files[@]}"
 ./scripts-dev/config-lint.sh
-# --quiet suppresses the update check.
-ruff --quiet "${files[@]}"
+flake8 "${files[@]}"
 ./scripts-dev/check_pydantic_models.py lint
 mypy
@@ -14,8 +14,6 @@

 # Stub for frozendict.

-from __future__ import annotations
-
 from typing import Any, Hashable, Iterable, Iterator, Mapping, Tuple, TypeVar, overload

 _KT = TypeVar("_KT", bound=Hashable)  # Key type.
@@ -14,8 +14,6 @@

 # Stub for PyICU.

-from __future__ import annotations
-
 class Locale:
    @staticmethod
    def getDefault() -> Locale: ...
@@ -2,8 +2,6 @@
 # https://github.com/grantjenks/python-sortedcontainers/blob/eea42df1f7bad2792e8da77335ff888f04b9e5ae/sortedcontainers/sorteddict.pyi
 # (from https://github.com/grantjenks/python-sortedcontainers/pull/107)

-from __future__ import annotations
-
 from typing import (
    Any,
    Callable,
@@ -2,8 +2,6 @@
 # https://github.com/grantjenks/python-sortedcontainers/blob/a419ffbd2b1c935b09f11f0971696e537fd0c510/sortedcontainers/sortedlist.pyi
 # (from https://github.com/grantjenks/python-sortedcontainers/pull/107)

-from __future__ import annotations
-
 from typing import (
    Any,
    Callable,
@@ -2,8 +2,6 @@
 # https://github.com/grantjenks/python-sortedcontainers/blob/d0a225d7fd0fb4c54532b8798af3cbeebf97e2d5/sortedcontainers/sortedset.pyi
 # (from https://github.com/grantjenks/python-sortedcontainers/pull/107)

-from __future__ import annotations
-
 from typing import (
    AbstractSet,
    Any,
@@ -1307,7 +1307,7 @@ def main() -> None:
    sqlite_config = {
        "name": "sqlite3",
        "args": {
-            "database": "file:{}?mode=rw".format(args.sqlite_database),
+            "database": args.sqlite_database,
            "cp_min": 1,
            "cp_max": 1,
            "check_same_thread": False,
@@ -351,13 +351,13 @@ class Filter:
            self.not_rel_types = filter_json.get("org.matrix.msc3874.not_rel_types", [])

    def filters_all_types(self) -> bool:
-        return self.types == [] or "*" in self.not_types
+        return "*" in self.not_types

    def filters_all_senders(self) -> bool:
-        return self.senders == [] or "*" in self.not_senders
+        return "*" in self.not_senders

    def filters_all_rooms(self) -> bool:
-        return self.rooms == [] or "*" in self.not_rooms
+        return "*" in self.not_rooms

    def _check(self, event: FilterEvent) -> bool:
        """Checks whether the filter matches the given event.
@@ -450,8 +450,8 @@ class Filter:
            if any(map(match_func, disallowed_values)):
                return False

-            # Otherwise if the event does not match at least one of the allowed
-            # values, reject it.
+            # Other the event does not match at least one of the allowed values,
+            # reject it.
            allowed_values = getattr(self, name)
            if allowed_values is not None:
                if not any(map(match_func, allowed_values)):
@@ -1,5 +1,3 @@
-from __future__ import annotations
-
 import argparse
 from typing import (
    Any,
@@ -16,7 +16,7 @@ import logging
 import os
 import re
 import threading
-from typing import Any, Callable, Dict, Mapping, Optional
+from typing import Any, Callable, Dict, Optional

 import attr

@@ -94,7 +94,7 @@ def add_resizable_cache(

 class CacheConfig(Config):
    section = "caches"
-    _environ: Mapping[str, str] = os.environ
+    _environ = os.environ

    event_cache_size: int
    cache_factors: Dict[str, float]
@@ -136,6 +136,3 @@ class ExperimentalConfig(Config):
            # Enable room version (and thus applicable push rules from MSC3931/3932)
            version_id = RoomVersions.MSC1767v10.identifier
            KNOWN_ROOM_VERSIONS[version_id] = RoomVersions.MSC1767v10
-
-        # MSC3391: Removing account data.
-        self.msc3391_enabled = experimental.get("msc3391_enabled", False)
@@ -117,7 +117,6 @@ OIDC_PROVIDER_CONFIG_SCHEMA = {
            # to avoid importing authlib here.
            "enum": ["client_secret_basic", "client_secret_post", "none"],
        },
-        "pkce_method": {"type": "string", "enum": ["auto", "always", "never"]},
        "scopes": {"type": "array", "items": {"type": "string"}},
        "authorization_endpoint": {"type": "string"},
        "token_endpoint": {"type": "string"},
@@ -290,7 +289,6 @@ def _parse_oidc_config_dict(
        client_secret=oidc_config.get("client_secret"),
        client_secret_jwt_key=client_secret_jwt_key,
        client_auth_method=oidc_config.get("client_auth_method", "client_secret_basic"),
-        pkce_method=oidc_config.get("pkce_method", "auto"),
        scopes=oidc_config.get("scopes", ["openid"]),
        authorization_endpoint=oidc_config.get("authorization_endpoint"),
        token_endpoint=oidc_config.get("token_endpoint"),
@@ -359,10 +357,6 @@ class OidcProviderConfig:
    # 'none'.
    client_auth_method: str

-    # Whether to enable PKCE when exchanging the authorization & token.
-    # Valid values are 'auto', 'always', and 'never'.
-    pkce_method: str
-
    # list of scopes to request
    scopes: Collection[str]

@@ -0,0 +1,13 @@
+# Copyright 2022 The Matrix.org Foundation C.I.C.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
@@ -0,0 +1,109 @@
+# Copyright 2022 The Matrix.org Foundation C.I.C.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import logging
+import os
+import time
+from typing import Any, Callable, Dict, Optional
+
+from scalene import scalene_profiler
+
+logger = logging.getLogger(__name__)
+
+
+class ProfilingDecider:
+    INSTANCES: Dict[str, "ProfilingDecider"] = {}
+
+    def __init__(self, name: str, cond: Callable[[], bool]) -> None:
+        ProfilingDecider.INSTANCES[name] = self
+
+        logger.warning("Setting up profiler %r", name)
+
+        # Default to being armed if SCALENE is available as an env var.
+        self.armed = b"SCALENE" in os.environb
+
+        self._cond = cond
+
+    def decide(self) -> bool:
+        logger.warning("Decide? Armed? %r", self.armed)
+        if not self.armed:
+            return False
+
+        if not self._cond():
+            logger.warning("Cond fail")
+            return False
+
+        self.armed = False
+
+        return True
+
+
+class CpuUtimeTracker:
+    def __init__(self) -> None:
+        self._update_times(time.time())
+
+    def _update_times(self, now_wall: float) -> None:
+        utime, _, _, _, elapsed = os.times()
+        self._last_utime = utime
+        self._last_elapsed = elapsed
+        self._last_wall = now_wall
+
+        self.min_elapse = 0.5
+        self.max_elapse = 120.0
+
+    def update_return_utime(self) -> Optional[float]:
+        """
+        Returns CPU usage over this period, provided at least `min_elapse` have
+        elapsed.
+        """
+        wall = time.time()
+        elapsed = wall - self._last_wall
+        if elapsed < self.min_elapse:
+            logger.warning("Not enough elapsed %r", elapsed)
+            return None
+
+        last_utime = self._last_utime
+        last_elapsed = self._last_elapsed
+
+        self._update_times(wall)
+
+        if elapsed > self.max_elapse:
+            # the average will be a bit skewy if so much time has elapsed. Ignore.
+            logger.warning("Too much elapsed %r", elapsed)
+            return None
+
+        usage = (self._last_utime - last_utime) / (self._last_elapsed - last_elapsed)
+        logger.info("Usage %r", usage)
+        return usage
+
+
+class SelectiveProfiling:
+    def __init__(self, decider: ProfilingDecider, enable: bool = False):
+        self._decider = decider
+        self._enable = enable
+        logger.info("selective enable %r", enable)
+
+    def __enter__(self) -> None:
+        if not self._enable:
+            return
+        if not self._decider.decide():
+            self._enable = False
+            return
+        logger.info("STARTING")
+        scalene_profiler.start()
+
+    def __exit__(self, exc_type: Any, exc_val: Any, exc_tb: Any) -> None:
+        if not self._enable:
+            return
+        scalene_profiler.stop()
+        logger.info("STOPPED")
@@ -17,12 +17,10 @@ import random
 from typing import TYPE_CHECKING, Awaitable, Callable, Collection, List, Optional, Tuple

 from synapse.replication.http.account_data import (
-    ReplicationAddRoomAccountDataRestServlet,
    ReplicationAddTagRestServlet,
-    ReplicationAddUserAccountDataRestServlet,
-    ReplicationRemoveRoomAccountDataRestServlet,
    ReplicationRemoveTagRestServlet,
-    ReplicationRemoveUserAccountDataRestServlet,
+    ReplicationRoomAccountDataRestServlet,
+    ReplicationUserAccountDataRestServlet,
 )
 from synapse.streams import EventSource
 from synapse.types import JsonDict, StreamKeyType, UserID
@@ -43,18 +41,8 @@ class AccountDataHandler:
        self._instance_name = hs.get_instance_name()
        self._notifier = hs.get_notifier()

-        self._add_user_data_client = (
-            ReplicationAddUserAccountDataRestServlet.make_client(hs)
-        )
-        self._remove_user_data_client = (
-            ReplicationRemoveUserAccountDataRestServlet.make_client(hs)
-        )
-        self._add_room_data_client = (
-            ReplicationAddRoomAccountDataRestServlet.make_client(hs)
-        )
-        self._remove_room_data_client = (
-            ReplicationRemoveRoomAccountDataRestServlet.make_client(hs)
-        )
+        self._user_data_client = ReplicationUserAccountDataRestServlet.make_client(hs)
+        self._room_data_client = ReplicationRoomAccountDataRestServlet.make_client(hs)
        self._add_tag_client = ReplicationAddTagRestServlet.make_client(hs)
        self._remove_tag_client = ReplicationRemoveTagRestServlet.make_client(hs)
        self._account_data_writers = hs.config.worker.writers.account_data
@@ -124,7 +112,7 @@ class AccountDataHandler:

            return max_stream_id
        else:
-            response = await self._add_room_data_client(
+            response = await self._room_data_client(
                instance_name=random.choice(self._account_data_writers),
                user_id=user_id,
                room_id=room_id,
@@ -133,59 +121,15 @@ class AccountDataHandler:
            )
            return response["max_stream_id"]

-    async def remove_account_data_for_room(
-        self, user_id: str, room_id: str, account_data_type: str
-    ) -> Optional[int]:
-        """
-        Deletes the room account data for the given user and account data type.
-
-        "Deleting" account data merely means setting the content of the account data
-        to an empty JSON object: {}.
-
-        Args:
-            user_id: The user ID to remove room account data for.
-            room_id: The room ID to target.
-            account_data_type: The account data type to remove.
-
-        Returns:
-            The maximum stream ID, or None if the room account data item did not exist.
-        """
-        if self._instance_name in self._account_data_writers:
-            max_stream_id = await self._store.remove_account_data_for_room(
-                user_id, room_id, account_data_type
-            )
-            if max_stream_id is None:
-                # The referenced account data did not exist, so no delete occurred.
-                return None
-
-            self._notifier.on_new_event(
-                StreamKeyType.ACCOUNT_DATA, max_stream_id, users=[user_id]
-            )
-
-            # Notify Synapse modules that the content of the type has changed to an
-            # empty dictionary.
-            await self._notify_modules(user_id, room_id, account_data_type, {})
-
-            return max_stream_id
-        else:
-            response = await self._remove_room_data_client(
-                instance_name=random.choice(self._account_data_writers),
-                user_id=user_id,
-                room_id=room_id,
-                account_data_type=account_data_type,
-                content={},
-            )
-            return response["max_stream_id"]
-
    async def add_account_data_for_user(
        self, user_id: str, account_data_type: str, content: JsonDict
    ) -> int:
        """Add some global account_data for a user.

        Args:
-            user_id: The user to add some account data for.
+            user_id: The user to add a tag for.
            account_data_type: The type of account_data to add.
-            content: The content json dictionary.
+            content: A json object to associate with the tag.

        Returns:
            The maximum stream ID.
@@ -204,7 +148,7 @@ class AccountDataHandler:

            return max_stream_id
        else:
-            response = await self._add_user_data_client(
+            response = await self._user_data_client(
                instance_name=random.choice(self._account_data_writers),
                user_id=user_id,
                account_data_type=account_data_type,
@@ -212,45 +156,6 @@ class AccountDataHandler:
            )
            return response["max_stream_id"]

-    async def remove_account_data_for_user(
-        self, user_id: str, account_data_type: str
-    ) -> Optional[int]:
-        """Removes a piece of global account_data for a user.
-
-        Args:
-            user_id: The user to remove account data for.
-            account_data_type: The type of account_data to remove.
-
-        Returns:
-            The maximum stream ID, or None if the room account data item did not exist.
-        """
-
-        if self._instance_name in self._account_data_writers:
-            max_stream_id = await self._store.remove_account_data_for_user(
-                user_id, account_data_type
-            )
-            if max_stream_id is None:
-                # The referenced account data did not exist, so no delete occurred.
-                return None
-
-            self._notifier.on_new_event(
-                StreamKeyType.ACCOUNT_DATA, max_stream_id, users=[user_id]
-            )
-
-            # Notify Synapse modules that the content of the type has changed to an
-            # empty dictionary.
-            await self._notify_modules(user_id, None, account_data_type, {})
-
-            return max_stream_id
-        else:
-            response = await self._remove_user_data_client(
-                instance_name=random.choice(self._account_data_writers),
-                user_id=user_id,
-                account_data_type=account_data_type,
-                content={},
-            )
-            return response["max_stream_id"]
-
    async def add_tag_to_room(
        self, user_id: str, room_id: str, tag: str, content: JsonDict
    ) -> int:
@@ -2031,7 +2031,7 @@ class PasswordAuthProvider:
        self.is_3pid_allowed_callbacks: List[IS_3PID_ALLOWED_CALLBACK] = []

        # Mapping from login type to login parameters
-        self._supported_login_types: Dict[str, Tuple[str, ...]] = {}
+        self._supported_login_types: Dict[str, Iterable[str]] = {}

        # Mapping from login type to auth checker callbacks
        self.auth_checker_callbacks: Dict[str, List[CHECK_AUTH_CALLBACK]] = {}
@@ -14,7 +14,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 import logging
-from http import HTTPStatus
 from typing import (
    TYPE_CHECKING,
    Any,
@@ -34,7 +33,6 @@ from synapse.api.errors import (
    Codes,
    FederationDeniedError,
    HttpResponseException,
-    InvalidAPICallError,
    RequestSendFailed,
    SynapseError,
 )
@@ -47,7 +45,6 @@ from synapse.types import (
    JsonDict,
    StreamKeyType,
    StreamToken,
-    UserID,
    get_domain_from_id,
    get_verify_key_from_cross_signing_key,
 )
@@ -896,47 +893,12 @@ class DeviceListWorkerUpdater:

    def __init__(self, hs: "HomeServer"):
        from synapse.replication.http.devices import (
-            ReplicationMultiUserDevicesResyncRestServlet,
            ReplicationUserDevicesResyncRestServlet,
        )

        self._user_device_resync_client = (
            ReplicationUserDevicesResyncRestServlet.make_client(hs)
        )
-        self._multi_user_device_resync_client = (
-            ReplicationMultiUserDevicesResyncRestServlet.make_client(hs)
-        )
-
-    async def multi_user_device_resync(
-        self, user_ids: List[str], mark_failed_as_stale: bool = True
-    ) -> Dict[str, Optional[JsonDict]]:
-        """
-        Like `user_device_resync` but operates on multiple users **from the same origin**
-        at once.
-
-        Returns:
-            Dict from User ID to the same Dict as `user_device_resync`.
-        """
-        # mark_failed_as_stale is not sent. Ensure this doesn't break expectations.
-        assert mark_failed_as_stale
-
-        if not user_ids:
-            # Shortcut empty requests
-            return {}
-
-        try:
-            return await self._multi_user_device_resync_client(user_ids=user_ids)
-        except SynapseError as err:
-            if not (
-                err.code == HTTPStatus.NOT_FOUND and err.errcode == Codes.UNRECOGNIZED
-            ):
-                raise
-
-            # Fall back to single requests
-            result: Dict[str, Optional[JsonDict]] = {}
-            for user_id in user_ids:
-                result[user_id] = await self._user_device_resync_client(user_id=user_id)
-            return result

    async def user_device_resync(
        self, user_id: str, mark_failed_as_stale: bool = True
@@ -951,10 +913,8 @@ class DeviceListWorkerUpdater:
            A dict with device info as under the "devices" in the result of this
            request:
            https://matrix.org/docs/spec/server_server/r0.1.2#get-matrix-federation-v1-user-devices-userid
-            None when we weren't able to fetch the device info for some reason,
-            e.g. due to a connection problem.
        """
-        return (await self.multi_user_device_resync([user_id]))[user_id]
+        return await self._user_device_resync_client(user_id=user_id)


 class DeviceListUpdater(DeviceListWorkerUpdater):
@@ -1200,66 +1160,19 @@ class DeviceListUpdater(DeviceListWorkerUpdater):
            # Allow future calls to retry resyncinc out of sync device lists.
            self._resync_retry_in_progress = False

-    async def multi_user_device_resync(
-        self, user_ids: List[str], mark_failed_as_stale: bool = True
-    ) -> Dict[str, Optional[JsonDict]]:
-        """
-        Like `user_device_resync` but operates on multiple users **from the same origin**
-        at once.
-
-        Returns:
-            Dict from User ID to the same Dict as `user_device_resync`.
-        """
-        if not user_ids:
-            return {}
-
-        origins = {UserID.from_string(user_id).domain for user_id in user_ids}
-
-        if len(origins) != 1:
-            raise InvalidAPICallError(f"Only one origin permitted, got {origins!r}")
-
-        result = {}
-        failed = set()
-        # TODO(Perf): Actually batch these up
-        for user_id in user_ids:
-            user_result, user_failed = await self._user_device_resync_returning_failed(
-                user_id
-            )
-            result[user_id] = user_result
-            if user_failed:
-                failed.add(user_id)
-
-        if mark_failed_as_stale:
-            await self.store.mark_remote_users_device_caches_as_stale(failed)
-
-        return result
-
    async def user_device_resync(
        self, user_id: str, mark_failed_as_stale: bool = True
    ) -> Optional[JsonDict]:
-        result, failed = await self._user_device_resync_returning_failed(user_id)
-
-        if failed and mark_failed_as_stale:
-            # Mark the remote user's device list as stale so we know we need to retry
-            # it later.
-            await self.store.mark_remote_users_device_caches_as_stale((user_id,))
-
-        return result
-
-    async def _user_device_resync_returning_failed(
-        self, user_id: str
-    ) -> Tuple[Optional[JsonDict], bool]:
        """Fetches all devices for a user and updates the device cache with them.

        Args:
            user_id: The user's id whose device_list will be updated.
+            mark_failed_as_stale: Whether to mark the user's device list as stale
+                if the attempt to resync failed.
        Returns:
-            - A dict with device info as under the "devices" in the result of this
-              request:
-              https://matrix.org/docs/spec/server_server/r0.1.2#get-matrix-federation-v1-user-devices-userid
-              None when we weren't able to fetch the device info for some reason,
-              e.g. due to a connection problem.
-            - True iff the resync failed and the device list should be marked as stale.
+            A dict with device info as under the "devices" in the result of this
+            request:
+            https://matrix.org/docs/spec/server_server/r0.1.2#get-matrix-federation-v1-user-devices-userid
        """
        logger.debug("Attempting to resync the device list for %s", user_id)
        log_kv({"message": "Doing resync to update device list."})
@@ -1268,7 +1181,12 @@ class DeviceListUpdater(DeviceListWorkerUpdater):
        try:
            result = await self.federation.query_user_devices(origin, user_id)
        except NotRetryingDestination:
-            return None, True
+            if mark_failed_as_stale:
+                # Mark the remote user's device list as stale so we know we need to retry
+                # it later.
+                await self.store.mark_remote_user_device_cache_as_stale(user_id)
+
+            return None
        except (RequestSendFailed, HttpResponseException) as e:
            logger.warning(
                "Failed to handle device list update for %s: %s",
@@ -1276,18 +1194,23 @@ class DeviceListUpdater(DeviceListWorkerUpdater):
                e,
            )

+            if mark_failed_as_stale:
+                # Mark the remote user's device list as stale so we know we need to retry
+                # it later.
+                await self.store.mark_remote_user_device_cache_as_stale(user_id)
+
            # We abort on exceptions rather than accepting the update
            # as otherwise synapse will 'forget' that its device list
            # is out of date. If we bail then we will retry the resync
            # next time we get a device list update for this user_id.
            # This makes it more likely that the device lists will
            # eventually become consistent.
-            return None, True
+            return None
        except FederationDeniedError as e:
            set_tag("error", True)
            log_kv({"reason": "FederationDeniedError"})
            logger.info(e)
-            return None, False
+            return None
        except Exception as e:
            set_tag("error", True)
            log_kv(
@@ -1295,7 +1218,12 @@ class DeviceListUpdater(DeviceListWorkerUpdater):
            )
            logger.exception("Failed to handle device list update for %s", user_id)

-            return None, True
+            if mark_failed_as_stale:
+                # Mark the remote user's device list as stale so we know we need to retry
+                # it later.
+                await self.store.mark_remote_user_device_cache_as_stale(user_id)
+
+            return None
        log_kv({"result": result})
        stream_id = result["stream_id"]
        devices = result["devices"]
@@ -1377,7 +1305,7 @@ class DeviceListUpdater(DeviceListWorkerUpdater):
        # point.
        self._seen_updates[user_id] = {stream_id}

-        return result, False
+        return result

    async def process_cross_signing_key_update(
        self,
@@ -195,7 +195,7 @@ class DeviceMessageHandler:
                sender_user_id,
                unknown_devices,
            )
-            await self.store.mark_remote_users_device_caches_as_stale((sender_user_id,))
+            await self.store.mark_remote_user_device_cache_as_stale(sender_user_id)

            # Immediately attempt a resync in the background
            run_in_background(self._user_device_resync, user_id=sender_user_id)
@@ -36,8 +36,8 @@ from synapse.types import (
    get_domain_from_id,
    get_verify_key_from_cross_signing_key,
 )
-from synapse.util import json_decoder
-from synapse.util.async_helpers import Linearizer, concurrently_execute
+from synapse.util import json_decoder, unwrapFirstError
+from synapse.util.async_helpers import Linearizer, delay_cancellation
 from synapse.util.cancellation import cancellable
 from synapse.util.retryutils import NotRetryingDestination

@@ -238,28 +238,24 @@ class E2eKeysHandler:
            # Now fetch any devices that we don't have in our cache
            # TODO It might make sense to propagate cancellations into the
            #      deferreds which are querying remote homeservers.
-            logger.debug(
-                "%d destinations to query devices for", len(remote_queries_not_in_cache)
-            )
-
-            async def _query(
-                destination_queries: Tuple[str, Dict[str, Iterable[str]]]
-            ) -> None:
-                destination, queries = destination_queries
-                return await self._query_devices_for_destination(
-                    results,
-                    cross_signing_keys,
-                    failures,
-                    destination,
-                    queries,
-                    timeout,
+            await make_deferred_yieldable(
+                delay_cancellation(
+                    defer.gatherResults(
+                        [
+                            run_in_background(
+                                self._query_devices_for_destination,
+                                results,
+                                cross_signing_keys,
+                                failures,
+                                destination,
+                                queries,
+                                timeout,
+                            )
+                            for destination, queries in remote_queries_not_in_cache.items()
+                        ],
+                        consumeErrors=True,
+                    ).addErrback(unwrapFirstError)
                )
-
-            await concurrently_execute(
-                _query,
-                remote_queries_not_in_cache.items(),
-                10,
-                delay_cancellation=True,
            )

            ret = {"device_keys": results, "failures": failures}
@@ -320,25 +316,18 @@ class E2eKeysHandler:
            destination,
        )

-        try:
-            user_resync_results = (
-                await self.device_handler.device_list_updater.multi_user_device_resync(
-                    list(users_to_resync_devices)
-                )
-            )
-            for user_id in users_to_resync_devices:
-                resync_results = user_resync_results[user_id]
-
-                if resync_results is None:
-                    # TODO: It's weird that we'll store a failure against a
-                    #       destination, yet continue processing users from that
-                    #       destination.
-                    #       We might want to consider changing this, but for now
-                    #       I'm leaving it as I found it.
-                    failures[destination] = _exception_to_failure(
-                        ValueError(f"Device resync failed for {user_id!r}")
+        for user_id in users_to_resync_devices:
+            # We've decided we're sharing a room with this user and should
+            # probably be tracking their device lists. However, we haven't
+            # done an initial sync on the device list so we do it now.
+            try:
+                resync_results = (
+                    await self.device_handler.device_list_updater.user_device_resync(
+                        user_id
                    )
-                    continue
+                )
+                if resync_results is None:
+                    raise ValueError("Device resync failed")

                # Add the device keys to the results.
                user_devices = resync_results["devices"]
@@ -356,8 +345,8 @@ class E2eKeysHandler:

                if self_signing_key:
                    cross_signing_keys["self_signing_keys"][user_id] = self_signing_key
-        except Exception as e:
-            failures[destination] = _exception_to_failure(e)
+            except Exception as e:
+                failures[destination] = _exception_to_failure(e)

        if len(destination_query) == len(user_ids_updated):
            # We've updated all the users in the query and we do not need to
@@ -1343,53 +1343,32 @@ class FederationHandler:
            )

            EventValidator().validate_builder(builder)
+            event, context = await self.event_creation_handler.create_new_client_event(
+                builder=builder
+            )

-            # Try several times, it could fail with PartialStateConflictError
-            # in send_membership_event, cf comment in except block.
-            max_retries = 5
-            for i in range(max_retries):
-                try:
-                    (
-                        event,
-                        context,
-                    ) = await self.event_creation_handler.create_new_client_event(
-                        builder=builder
-                    )
+            event, context = await self.add_display_name_to_third_party_invite(
+                room_version_obj, event_dict, event, context
+            )

-                    event, context = await self.add_display_name_to_third_party_invite(
-                        room_version_obj, event_dict, event, context
-                    )
+            EventValidator().validate_new(event, self.config)

-                    EventValidator().validate_new(event, self.config)
+            # We need to tell the transaction queue to send this out, even
+            # though the sender isn't a local user.
+            event.internal_metadata.send_on_behalf_of = self.hs.hostname

-                    # We need to tell the transaction queue to send this out, even
-                    # though the sender isn't a local user.
-                    event.internal_metadata.send_on_behalf_of = self.hs.hostname
+            try:
+                validate_event_for_room_version(event)
+                await self._event_auth_handler.check_auth_rules_from_context(event)
+            except AuthError as e:
+                logger.warning("Denying new third party invite %r because %s", event, e)
+                raise e

-                    try:
-                        validate_event_for_room_version(event)
-                        await self._event_auth_handler.check_auth_rules_from_context(
-                            event
-                        )
-                    except AuthError as e:
-                        logger.warning(
-                            "Denying new third party invite %r because %s", event, e
-                        )
-                        raise e
+            await self._check_signature(event, context)

-                    await self._check_signature(event, context)
-
-                    # We retrieve the room member handler here as to not cause a cyclic dependency
-                    member_handler = self.hs.get_room_member_handler()
-                    await member_handler.send_membership_event(None, event, context)
-
-                    break
-                except PartialStateConflictError as e:
-                    # Persisting couldn't happen because the room got un-partial stated
-                    # in the meantime and context needs to be recomputed, so let's do so.
-                    if i == max_retries - 1:
-                        raise e
-                    pass
+            # We retrieve the room member handler here as to not cause a cyclic dependency
+            member_handler = self.hs.get_room_member_handler()
+            await member_handler.send_membership_event(None, event, context)
        else:
            destinations = {x.split(":", 1)[-1] for x in (sender_user_id, room_id)}

@@ -1421,46 +1400,28 @@ class FederationHandler:
            room_version_obj, event_dict
        )

-        # Try several times, it could fail with PartialStateConflictError
-        # in send_membership_event, cf comment in except block.
-        max_retries = 5
-        for i in range(max_retries):
-            try:
-                (
-                    event,
-                    context,
-                ) = await self.event_creation_handler.create_new_client_event(
-                    builder=builder
-                )
-                event, context = await self.add_display_name_to_third_party_invite(
-                    room_version_obj, event_dict, event, context
-                )
+        event, context = await self.event_creation_handler.create_new_client_event(
+            builder=builder
+        )
+        event, context = await self.add_display_name_to_third_party_invite(
+            room_version_obj, event_dict, event, context
+        )

-                try:
-                    validate_event_for_room_version(event)
-                    await self._event_auth_handler.check_auth_rules_from_context(event)
-                except AuthError as e:
-                    logger.warning("Denying third party invite %r because %s", event, e)
-                    raise e
-                await self._check_signature(event, context)
+        try:
+            validate_event_for_room_version(event)
+            await self._event_auth_handler.check_auth_rules_from_context(event)
+        except AuthError as e:
+            logger.warning("Denying third party invite %r because %s", event, e)
+            raise e
+        await self._check_signature(event, context)

-                # We need to tell the transaction queue to send this out, even
-                # though the sender isn't a local user.
-                event.internal_metadata.send_on_behalf_of = get_domain_from_id(
-                    event.sender
-                )
+        # We need to tell the transaction queue to send this out, even
+        # though the sender isn't a local user.
+        event.internal_metadata.send_on_behalf_of = get_domain_from_id(event.sender)

-                # We retrieve the room member handler here as to not cause a cyclic dependency
-                member_handler = self.hs.get_room_member_handler()
-                await member_handler.send_membership_event(None, event, context)
-
-                break
-            except PartialStateConflictError as e:
-                # Persisting couldn't happen because the room got un-partial stated
-                # in the meantime and context needs to be recomputed, so let's do so.
-                if i == max_retries - 1:
-                    raise e
-                pass
+        # We retrieve the room member handler here as to not cause a cyclic dependency
+        member_handler = self.hs.get_room_member_handler()
+        await member_handler.send_membership_event(None, event, context)

    async def add_display_name_to_third_party_invite(
        self,
@@ -610,8 +610,6 @@ class FederationEventHandler:
            self._state_storage_controller.notify_event_un_partial_stated(
                event.event_id
            )
-            # Notify that there's a new row in the un_partial_stated_events stream.
-            self._notifier.notify_replication()

    @trace
    async def backfill(
@@ -1423,7 +1421,7 @@ class FederationEventHandler:
        """

        try:
-            await self._store.mark_remote_users_device_caches_as_stale((sender,))
+            await self._store.mark_remote_user_device_cache_as_stale(sender)

            # Immediately attempt a resync in the background
            if self._config.worker.worker_app:
@@ -37,6 +37,7 @@ from synapse.api.errors import (
    AuthError,
    Codes,
    ConsentNotGivenError,
+    LimitExceededError,
    NotFoundError,
    ShadowBanError,
    SynapseError,
@@ -998,73 +999,60 @@ class EventCreationHandler:
                        event.internal_metadata.stream_ordering,
                    )

-        # Try several times, it could fail with PartialStateConflictError
-        # in handle_new_client_event, cf comment in except block.
-        max_retries = 5
-        for i in range(max_retries):
-            try:
-                event, context = await self.create_event(
-                    requester,
-                    event_dict,
-                    txn_id=txn_id,
-                    allow_no_prev_events=allow_no_prev_events,
-                    prev_event_ids=prev_event_ids,
-                    state_event_ids=state_event_ids,
-                    outlier=outlier,
-                    historical=historical,
-                    depth=depth,
+            event, context = await self.create_event(
+                requester,
+                event_dict,
+                txn_id=txn_id,
+                allow_no_prev_events=allow_no_prev_events,
+                prev_event_ids=prev_event_ids,
+                state_event_ids=state_event_ids,
+                outlier=outlier,
+                historical=historical,
+                depth=depth,
+            )
+
+            assert self.hs.is_mine_id(event.sender), "User must be our own: %s" % (
+                event.sender,
+            )
+
+            spam_check_result = await self.spam_checker.check_event_for_spam(event)
+            if spam_check_result != self.spam_checker.NOT_SPAM:
+                if isinstance(spam_check_result, tuple):
+                    try:
+                        [code, dict] = spam_check_result
+                        raise SynapseError(
+                            403,
+                            "This message had been rejected as probable spam",
+                            code,
+                            dict,
+                        )
+                    except ValueError:
+                        logger.error(
+                            "Spam-check module returned invalid error value. Expecting [code, dict], got %s",
+                            spam_check_result,
+                        )
+
+                        raise SynapseError(
+                            403,
+                            "This message has been rejected as probable spam",
+                            Codes.FORBIDDEN,
+                        )
+
+                # Backwards compatibility: if the return value is not an error code, it
+                # means the module returned an error message to be included in the
+                # SynapseError (which is now deprecated).
+                raise SynapseError(
+                    403,
+                    spam_check_result,
+                    Codes.FORBIDDEN,
                )

-                assert self.hs.is_mine_id(event.sender), "User must be our own: %s" % (
-                    event.sender,
-                )
-
-                spam_check_result = await self.spam_checker.check_event_for_spam(event)
-                if spam_check_result != self.spam_checker.NOT_SPAM:
-                    if isinstance(spam_check_result, tuple):
-                        try:
-                            [code, dict] = spam_check_result
-                            raise SynapseError(
-                                403,
-                                "This message had been rejected as probable spam",
-                                code,
-                                dict,
-                            )
-                        except ValueError:
-                            logger.error(
-                                "Spam-check module returned invalid error value. Expecting [code, dict], got %s",
-                                spam_check_result,
-                            )
-
-                            raise SynapseError(
-                                403,
-                                "This message has been rejected as probable spam",
-                                Codes.FORBIDDEN,
-                            )
-
-                    # Backwards compatibility: if the return value is not an error code, it
-                    # means the module returned an error message to be included in the
-                    # SynapseError (which is now deprecated).
-                    raise SynapseError(
-                        403,
-                        spam_check_result,
-                        Codes.FORBIDDEN,
-                    )
-
-                ev = await self.handle_new_client_event(
-                    requester=requester,
-                    events_and_context=[(event, context)],
-                    ratelimit=ratelimit,
-                    ignore_shadow_ban=ignore_shadow_ban,
-                )
-
-                break
-            except PartialStateConflictError as e:
-                # Persisting couldn't happen because the room got un-partial stated
-                # in the meantime and context needs to be recomputed, so let's do so.
-                if i == max_retries - 1:
-                    raise e
-                pass
+            ev = await self.handle_new_client_event(
+                requester=requester,
+                events_and_context=[(event, context)],
+                ratelimit=ratelimit,
+                ignore_shadow_ban=ignore_shadow_ban,
+            )

        # we know it was persisted, so must have a stream ordering
        assert ev.internal_metadata.stream_ordering
@@ -1368,7 +1356,7 @@ class EventCreationHandler:

        Raises:
            ShadowBanError if the requester has been shadow-banned.
-            PartialStateConflictError if attempting to persist a partial state event in
+            SynapseError(503) if attempting to persist a partial state event in
                a room that has been un-partial stated.
        """
        extra_users = extra_users or []
@@ -1430,23 +1418,34 @@ class EventCreationHandler:
        # We now persist the event (and update the cache in parallel, since we
        # don't want to block on it).
        event, context = events_and_context[0]
-        result, _ = await make_deferred_yieldable(
-            gather_results(
-                (
-                    run_in_background(
-                        self._persist_events,
-                        requester=requester,
-                        events_and_context=events_and_context,
-                        ratelimit=ratelimit,
-                        extra_users=extra_users,
+        try:
+            result, _ = await make_deferred_yieldable(
+                gather_results(
+                    (
+                        run_in_background(
+                            self._persist_events,
+                            requester=requester,
+                            events_and_context=events_and_context,
+                            ratelimit=ratelimit,
+                            extra_users=extra_users,
+                        ),
+                        run_in_background(
+                            self.cache_joined_hosts_for_events, events_and_context
+                        ).addErrback(
+                            log_failure, "cache_joined_hosts_for_event failed"
+                        ),
                    ),
-                    run_in_background(
-                        self.cache_joined_hosts_for_events, events_and_context
-                    ).addErrback(log_failure, "cache_joined_hosts_for_event failed"),
-                ),
-                consumeErrors=True,
+                    consumeErrors=True,
+                )
+            ).addErrback(unwrapFirstError)
+        except PartialStateConflictError as e:
+            # The event context needs to be recomputed.
+            # Turn the error into a 429, as a hint to the client to try again.
+            logger.info(
+                "Room %s was un-partial stated while persisting client event.",
+                event.room_id,
            )
-        ).addErrback(unwrapFirstError)
+            raise LimitExceededError(msg=e.msg, errcode=e.errcode, retry_after_ms=0)

        return result

@@ -2013,39 +2012,26 @@ class EventCreationHandler:
        for user_id in members:
            requester = create_requester(user_id, authenticated_entity=self.server_name)
            try:
-                # Try several times, it could fail with PartialStateConflictError
-                # in handle_new_client_event, cf comment in except block.
-                max_retries = 5
-                for i in range(max_retries):
-                    try:
-                        event, context = await self.create_event(
-                            requester,
-                            {
-                                "type": EventTypes.Dummy,
-                                "content": {},
-                                "room_id": room_id,
-                                "sender": user_id,
-                            },
-                        )
+                event, context = await self.create_event(
+                    requester,
+                    {
+                        "type": EventTypes.Dummy,
+                        "content": {},
+                        "room_id": room_id,
+                        "sender": user_id,
+                    },
+                )

-                        event.internal_metadata.proactively_send = False
+                event.internal_metadata.proactively_send = False

-                        # Since this is a dummy-event it is OK if it is sent by a
-                        # shadow-banned user.
-                        await self.handle_new_client_event(
-                            requester,
-                            events_and_context=[(event, context)],
-                            ratelimit=False,
-                            ignore_shadow_ban=True,
-                        )
-
-                        break
-                    except PartialStateConflictError as e:
-                        # Persisting couldn't happen because the room got un-partial stated
-                        # in the meantime and context needs to be recomputed, so let's do so.
-                        if i == max_retries - 1:
-                            raise e
-                        pass
+                # Since this is a dummy-event it is OK if it is sent by a
+                # shadow-banned user.
+                await self.handle_new_client_event(
+                    requester,
+                    events_and_context=[(event, context)],
+                    ratelimit=False,
+                    ignore_shadow_ban=True,
+                )
                return True
            except AuthError:
                logger.info(
@@ -36,7 +36,6 @@ from authlib.jose import JsonWebToken, JWTClaims
 from authlib.jose.errors import InvalidClaimError, JoseError, MissingClaimError
 from authlib.oauth2.auth import ClientAuth
 from authlib.oauth2.rfc6749.parameters import prepare_grant_uri
-from authlib.oauth2.rfc7636.challenge import create_s256_code_challenge
 from authlib.oidc.core import CodeIDToken, UserInfo
 from authlib.oidc.discovery import OpenIDProviderMetadata, get_well_known_url
 from jinja2 import Environment, Template
@@ -476,16 +475,6 @@ class OidcProvider:
                    )
                )

-        # If PKCE support is advertised ensure the wanted method is available.
-        if m.get("code_challenge_methods_supported") is not None:
-            m.validate_code_challenge_methods_supported()
-            if "S256" not in m["code_challenge_methods_supported"]:
-                raise ValueError(
-                    '"S256" not in "code_challenge_methods_supported" ({supported!r})'.format(
-                        supported=m["code_challenge_methods_supported"],
-                    )
-                )
-
        if m.get("response_types_supported") is not None:
            m.validate_response_types_supported()

@@ -613,11 +602,6 @@ class OidcProvider:
        if self._config.jwks_uri:
            metadata["jwks_uri"] = self._config.jwks_uri

-        if self._config.pkce_method == "always":
-            metadata["code_challenge_methods_supported"] = ["S256"]
-        elif self._config.pkce_method == "never":
-            metadata.pop("code_challenge_methods_supported", None)
-
        self._validate_metadata(metadata)

        return metadata
@@ -669,7 +653,7 @@ class OidcProvider:

        return jwk_set

-    async def _exchange_code(self, code: str, code_verifier: str) -> Token:
+    async def _exchange_code(self, code: str) -> Token:
        """Exchange an authorization code for a token.

        This calls the ``token_endpoint`` with the authorization code we
@@ -682,7 +666,6 @@ class OidcProvider:

        Args:
            code: The authorization code we got from the callback.
-            code_verifier: The PKCE code verifier to send, blank if unused.

        Returns:
            A dict containing various tokens.
@@ -713,8 +696,6 @@ class OidcProvider:
            "code": code,
            "redirect_uri": self._callback_url,
        }
-        if code_verifier:
-            args["code_verifier"] = code_verifier
        body = urlencode(args, True)

        # Fill the body/headers with credentials
@@ -933,14 +914,11 @@ class OidcProvider:
          - ``scope``: the list of scopes set in ``oidc_config.scopes``
          - ``state``: a random string
          - ``nonce``: a random string
-          - ``code_challenge``: a RFC7636 code challenge (if PKCE is supported)

-        In addition to generating a redirect URL, we are setting a cookie with
-        a signed macaroon token containing the state, the nonce, the
-        client_redirect_url, and (optionally) the code_verifier params. The state,
-        nonce, and client_redirect_url are then checked when the client comes back
-        from the provider. The code_verifier is passed back to the server during
-        the token exchange and compared to the code_challenge sent in this request.
+        In addition generating a redirect URL, we are setting a cookie with
+        a signed macaroon token containing the state, the nonce and the
+        client_redirect_url params. Those are then checked when the client
+        comes back from the provider.

        Args:
            request: the incoming request from the browser.
@@ -957,25 +935,10 @@ class OidcProvider:

        state = generate_token()
        nonce = generate_token()
-        code_verifier = ""

        if not client_redirect_url:
            client_redirect_url = b""

-        metadata = await self.load_metadata()
-
-        # Automatically enable PKCE if it is supported.
-        extra_grant_values = {}
-        if metadata.get("code_challenge_methods_supported"):
-            code_verifier = generate_token(48)
-
-            # Note that we verified the server supports S256 earlier (in
-            # OidcProvider._validate_metadata).
-            extra_grant_values = {
-                "code_challenge_method": "S256",
-                "code_challenge": create_s256_code_challenge(code_verifier),
-            }
-
        cookie = self._macaroon_generaton.generate_oidc_session_token(
            state=state,
            session_data=OidcSessionData(
@@ -983,7 +946,6 @@ class OidcProvider:
                nonce=nonce,
                client_redirect_url=client_redirect_url.decode(),
                ui_auth_session_id=ui_auth_session_id or "",
-                code_verifier=code_verifier,
            ),
        )

@@ -1004,6 +966,7 @@ class OidcProvider:
                )
            )

+        metadata = await self.load_metadata()
        authorization_endpoint = metadata.get("authorization_endpoint")
        return prepare_grant_uri(
            authorization_endpoint,
@@ -1013,7 +976,6 @@ class OidcProvider:
            scope=self._scopes,
            state=state,
            nonce=nonce,
-            **extra_grant_values,
        )

    async def handle_oidc_callback(
@@ -1041,9 +1003,7 @@ class OidcProvider:
        # Exchange the code with the provider
        try:
            logger.debug("Exchanging OAuth2 code for a token")
-            token = await self._exchange_code(
-                code, code_verifier=session_data.code_verifier
-            )
+            token = await self._exchange_code(code)
        except OidcError as e:
            logger.warning("Could not exchange OAuth2 code: %s", e)
            self._sso_handler.render_error(request, e.error, e.error_description)
@@ -1560,8 +1520,8 @@ env.filters.update(

@attr.s(slots=True, frozen=True, auto_attribs=True)
 class JinjaOidcMappingConfig:
-    subject_template: Template
-    picture_template: Template
+    subject_claim: str
+    picture_claim: str
    localpart_template: Optional[Template]
    display_name_template: Optional[Template]
    email_template: Optional[Template]
@@ -1580,23 +1540,8 @@ class JinjaOidcMappingProvider(OidcMappingProvider[JinjaOidcMappingConfig]):

    @staticmethod
    def parse_config(config: dict) -> JinjaOidcMappingConfig:
-        def parse_template_config_with_claim(
-            option_name: str, default_claim: str
-        ) -> Template:
-            template_name = f"{option_name}_template"
-            template = config.get(template_name)
-            if not template:
-                # Convert the legacy subject_claim into a template.
-                claim = config.get(f"{option_name}_claim", default_claim)
-                template = "{{ user.%s }}" % (claim,)
-
-            try:
-                return env.from_string(template)
-            except Exception as e:
-                raise ConfigError("invalid jinja template", path=[template_name]) from e
-
-        subject_template = parse_template_config_with_claim("subject", "sub")
-        picture_template = parse_template_config_with_claim("picture", "picture")
+        subject_claim = config.get("subject_claim", "sub")
+        picture_claim = config.get("picture_claim", "picture")

        def parse_template_config(option_name: str) -> Optional[Template]:
            if option_name not in config:
@@ -1629,8 +1574,8 @@ class JinjaOidcMappingProvider(OidcMappingProvider[JinjaOidcMappingConfig]):
            raise ConfigError("must be a bool", path=["confirm_localpart"])

        return JinjaOidcMappingConfig(
-            subject_template=subject_template,
-            picture_template=picture_template,
+            subject_claim=subject_claim,
+            picture_claim=picture_claim,
            localpart_template=localpart_template,
            display_name_template=display_name_template,
            email_template=email_template,
@@ -1639,7 +1584,7 @@ class JinjaOidcMappingProvider(OidcMappingProvider[JinjaOidcMappingConfig]):
        )

    def get_remote_user_id(self, userinfo: UserInfo) -> str:
-        return self._config.subject_template.render(user=userinfo).strip()
+        return userinfo[self._config.subject_claim]

    async def map_user_attributes(
        self, userinfo: UserInfo, token: Token, failures: int
@@ -1670,7 +1615,7 @@ class JinjaOidcMappingProvider(OidcMappingProvider[JinjaOidcMappingConfig]):
        if email:
            emails.append(email)

-        picture = self._config.picture_template.render(user=userinfo).strip()
+        picture = userinfo.get("picture")

        return UserAttributeDict(
            localpart=localpart,
@@ -62,7 +62,6 @@ from synapse.events.utils import copy_and_fixup_power_levels_contents
 from synapse.handlers.relations import BundledAggregations
 from synapse.module_api import NOT_SPAM
 from synapse.rest.admin._base import assert_user_is_admin
-from synapse.storage.databases.main.events import PartialStateConflictError
 from synapse.streams import EventSource
 from synapse.types import (
    JsonDict,
@@ -208,64 +207,46 @@ class RoomCreationHandler:

        new_room_id = self._generate_room_id()

-        # Try several times, it could fail with PartialStateConflictError
-        # in _upgrade_room, cf comment in except block.
-        max_retries = 5
-        for i in range(max_retries):
-            try:
-                # Check whether the user has the power level to carry out the upgrade.
-                # `check_auth_rules_from_context` will check that they are in the room and have
-                # the required power level to send the tombstone event.
-                (
-                    tombstone_event,
-                    tombstone_context,
-                ) = await self.event_creation_handler.create_event(
-                    requester,
-                    {
-                        "type": EventTypes.Tombstone,
-                        "state_key": "",
-                        "room_id": old_room_id,
-                        "sender": user_id,
-                        "content": {
-                            "body": "This room has been replaced",
-                            "replacement_room": new_room_id,
-                        },
-                    },
-                )
-                validate_event_for_room_version(tombstone_event)
-                await self._event_auth_handler.check_auth_rules_from_context(
-                    tombstone_event
-                )
+        # Check whether the user has the power level to carry out the upgrade.
+        # `check_auth_rules_from_context` will check that they are in the room and have
+        # the required power level to send the tombstone event.
+        (
+            tombstone_event,
+            tombstone_context,
+        ) = await self.event_creation_handler.create_event(
+            requester,
+            {
+                "type": EventTypes.Tombstone,
+                "state_key": "",
+                "room_id": old_room_id,
+                "sender": user_id,
+                "content": {
+                    "body": "This room has been replaced",
+                    "replacement_room": new_room_id,
+                },
+            },
+        )
+        validate_event_for_room_version(tombstone_event)
+        await self._event_auth_handler.check_auth_rules_from_context(tombstone_event)

-                # Upgrade the room
-                #
-                # If this user has sent multiple upgrade requests for the same room
-                # and one of them is not complete yet, cache the response and
-                # return it to all subsequent requests
-                ret = await self._upgrade_response_cache.wrap(
-                    (old_room_id, user_id),
-                    self._upgrade_room,
-                    requester,
-                    old_room_id,
-                    old_room,  # args for _upgrade_room
-                    new_room_id,
-                    new_version,
-                    tombstone_event,
-                    tombstone_context,
-                )
+        # Upgrade the room
+        #
+        # If this user has sent multiple upgrade requests for the same room
+        # and one of them is not complete yet, cache the response and
+        # return it to all subsequent requests
+        ret = await self._upgrade_response_cache.wrap(
+            (old_room_id, user_id),
+            self._upgrade_room,
+            requester,
+            old_room_id,
+            old_room,  # args for _upgrade_room
+            new_room_id,
+            new_version,
+            tombstone_event,
+            tombstone_context,
+        )

-                return ret
-            except PartialStateConflictError as e:
-                # Clean up the cache so we can retry properly
-                self._upgrade_response_cache.unset((old_room_id, user_id))
-                # Persisting couldn't happen because the room got un-partial stated
-                # in the meantime and context needs to be recomputed, so let's do so.
-                if i == max_retries - 1:
-                    raise e
-                pass
-
-        # This is to satisfy mypy and should never happen
-        raise PartialStateConflictError()
+        return ret

    async def _upgrade_room(
        self,
@@ -375,8 +375,6 @@ class RoomBatchHandler:
        # Events are sorted by (topological_ordering, stream_ordering)
        # where topological_ordering is just depth.
        for (event, context) in reversed(events_to_persist):
-            # This call can't raise `PartialStateConflictError` since we forbid
-            # use of the historical batch API during partial state
            await self.event_creation_handler.handle_new_client_event(
                await self.create_requester_for_user_id_from_app_service(
                    event.sender, app_service_requester.app_service
@@ -34,7 +34,6 @@ from synapse.events.snapshot import EventContext
 from synapse.handlers.profile import MAX_AVATAR_URL_LEN, MAX_DISPLAYNAME_LEN
 from synapse.logging import opentracing
 from synapse.module_api import NOT_SPAM
-from synapse.storage.databases.main.events import PartialStateConflictError
 from synapse.types import (
    JsonDict,
    Requester,
@@ -393,81 +392,60 @@ class RoomMemberHandler(metaclass=abc.ABCMeta):
                event_pos = await self.store.get_position_for_event(existing_event_id)
                return existing_event_id, event_pos.stream

-        # Try several times, it could fail with PartialStateConflictError,
-        # in handle_new_client_event, cf comment in except block.
-        max_retries = 5
-        for i in range(max_retries):
-            try:
-                event, context = await self.event_creation_handler.create_event(
-                    requester,
-                    {
-                        "type": EventTypes.Member,
-                        "content": content,
-                        "room_id": room_id,
-                        "sender": requester.user.to_string(),
-                        "state_key": user_id,
-                        # For backwards compatibility:
-                        "membership": membership,
-                        "origin_server_ts": origin_server_ts,
-                    },
-                    txn_id=txn_id,
-                    allow_no_prev_events=allow_no_prev_events,
-                    prev_event_ids=prev_event_ids,
-                    state_event_ids=state_event_ids,
-                    depth=depth,
-                    require_consent=require_consent,
-                    outlier=outlier,
-                    historical=historical,
+        event, context = await self.event_creation_handler.create_event(
+            requester,
+            {
+                "type": EventTypes.Member,
+                "content": content,
+                "room_id": room_id,
+                "sender": requester.user.to_string(),
+                "state_key": user_id,
+                # For backwards compatibility:
+                "membership": membership,
+                "origin_server_ts": origin_server_ts,
+            },
+            txn_id=txn_id,
+            allow_no_prev_events=allow_no_prev_events,
+            prev_event_ids=prev_event_ids,
+            state_event_ids=state_event_ids,
+            depth=depth,
+            require_consent=require_consent,
+            outlier=outlier,
+            historical=historical,
+        )
+
+        prev_state_ids = await context.get_prev_state_ids(
+            StateFilter.from_types([(EventTypes.Member, None)])
+        )
+
+        prev_member_event_id = prev_state_ids.get((EventTypes.Member, user_id), None)
+
+        if event.membership == Membership.JOIN:
+            newly_joined = True
+            if prev_member_event_id:
+                prev_member_event = await self.store.get_event(prev_member_event_id)
+                newly_joined = prev_member_event.membership != Membership.JOIN
+
+            # Only rate-limit if the user actually joined the room, otherwise we'll end
+            # up blocking profile updates.
+            if newly_joined and ratelimit:
+                await self._join_rate_limiter_local.ratelimit(requester)
+                await self._join_rate_per_room_limiter.ratelimit(
+                    requester, key=room_id, update=False
                )
+        with opentracing.start_active_span("handle_new_client_event"):
+            result_event = await self.event_creation_handler.handle_new_client_event(
+                requester,
+                events_and_context=[(event, context)],
+                extra_users=[target],
+                ratelimit=ratelimit,
+            )

-                prev_state_ids = await context.get_prev_state_ids(
-                    StateFilter.from_types([(EventTypes.Member, None)])
-                )
-
-                prev_member_event_id = prev_state_ids.get(
-                    (EventTypes.Member, user_id), None
-                )
-
-                if event.membership == Membership.JOIN:
-                    newly_joined = True
-                    if prev_member_event_id:
-                        prev_member_event = await self.store.get_event(
-                            prev_member_event_id
-                        )
-                        newly_joined = prev_member_event.membership != Membership.JOIN
-
-                    # Only rate-limit if the user actually joined the room, otherwise we'll end
-                    # up blocking profile updates.
-                    if newly_joined and ratelimit:
-                        await self._join_rate_limiter_local.ratelimit(requester)
-                        await self._join_rate_per_room_limiter.ratelimit(
-                            requester, key=room_id, update=False
-                        )
-                with opentracing.start_active_span("handle_new_client_event"):
-                    result_event = (
-                        await self.event_creation_handler.handle_new_client_event(
-                            requester,
-                            events_and_context=[(event, context)],
-                            extra_users=[target],
-                            ratelimit=ratelimit,
-                        )
-                    )
-
-                if event.membership == Membership.LEAVE:
-                    if prev_member_event_id:
-                        prev_member_event = await self.store.get_event(
-                            prev_member_event_id
-                        )
-                        if prev_member_event.membership == Membership.JOIN:
-                            await self._user_left_room(target, room_id)
-
-                break
-            except PartialStateConflictError as e:
-                # Persisting couldn't happen because the room got un-partial stated
-                # in the meantime and context needs to be recomputed, so let's do so.
-                if i == max_retries - 1:
-                    raise e
-                pass
+        if event.membership == Membership.LEAVE:
+            if prev_member_event_id:
+                prev_member_event = await self.store.get_event(prev_member_event_id)
+                if prev_member_event.membership == Membership.JOIN:
+                    await self._user_left_room(target, room_id)

        # we know it was persisted, so should have a stream ordering
        assert result_event.internal_metadata.stream_ordering
@@ -1256,8 +1234,6 @@ class RoomMemberHandler(metaclass=abc.ABCMeta):
            ratelimit: Whether to rate limit this request.
        Raises:
            SynapseError if there was a problem changing the membership.
-            PartialStateConflictError: if attempting to persist a partial state event in
-                a room that has been un-partial stated.
        """
        target_user = UserID.from_string(event.state_key)
        room_id = event.room_id
@@ -1887,37 +1863,21 @@ class RoomMemberMasterHandler(RoomMemberHandler):
            list(previous_membership_event.auth_event_ids()) + prev_event_ids
        )

-        # Try several times, it could fail with PartialStateConflictError
-        # in handle_new_client_event, cf comment in except block.
-        max_retries = 5
-        for i in range(max_retries):
-            try:
-                event, context = await self.event_creation_handler.create_event(
-                    requester,
-                    event_dict,
-                    txn_id=txn_id,
-                    prev_event_ids=prev_event_ids,
-                    auth_event_ids=auth_event_ids,
-                    outlier=True,
-                )
-                event.internal_metadata.out_of_band_membership = True
-
-                result_event = (
-                    await self.event_creation_handler.handle_new_client_event(
-                        requester,
-                        events_and_context=[(event, context)],
-                        extra_users=[UserID.from_string(target_user)],
-                    )
-                )
-
-                break
-            except PartialStateConflictError as e:
-                # Persisting couldn't happen because the room got un-partial stated
-                # in the meantime and context needs to be recomputed, so let's do so.
-                if i == max_retries - 1:
-                    raise e
-                pass
+        event, context = await self.event_creation_handler.create_event(
+            requester,
+            event_dict,
+            txn_id=txn_id,
+            prev_event_ids=prev_event_ids,
+            auth_event_ids=auth_event_ids,
+            outlier=True,
+        )
+        event.internal_metadata.out_of_band_membership = True

+        result_event = await self.event_creation_handler.handle_new_client_event(
+            requester,
+            events_and_context=[(event, context)],
+            extra_users=[UserID.from_string(target_user)],
+        )
        # we know it was persisted, so must have a stream ordering
        assert result_event.internal_metadata.stream_ordering

@@ -20,6 +20,7 @@ from typing import TYPE_CHECKING, Dict, Iterable, List, Optional, Sequence, Set,
 import attr

 from synapse.api.constants import (
+    EventContentFields,
    EventTypes,
    HistoryVisibility,
    JoinRules,
@@ -700,6 +701,13 @@ class RoomSummaryHandler:
        # there should always be an entry
        assert stats is not None, "unable to retrieve stats for %s" % (room_id,)

+        current_state_ids = await self._storage_controllers.state.get_current_state_ids(
+            room_id
+        )
+        create_event = await self._store.get_event(
+            current_state_ids[(EventTypes.Create, "")]
+        )
+
        entry = {
            "room_id": stats["room_id"],
            "name": stats["name"],
@@ -712,7 +720,7 @@ class RoomSummaryHandler:
                stats["history_visibility"] == HistoryVisibility.WORLD_READABLE
            ),
            "guest_can_join": stats["guest_access"] == "can_join",
-            "room_type": stats["room_type"],
+            "room_type": create_event.content.get(EventContentFields.ROOM_TYPE),
        }

        if self._msc3266_enabled:
@@ -722,11 +730,7 @@ class RoomSummaryHandler:
        # Federation requests need to provide additional information so the
        # requested server is able to filter the response appropriately.
        if for_federation:
-            current_state_ids = (
-                await self._storage_controllers.state.get_current_state_ids(room_id)
-            )
            room_version = await self._store.get_room_version(room_id)
-
            if await self._event_auth_handler.has_restricted_join_rules(
                current_state_ids, room_version
            ):
@@ -275,7 +275,7 @@ class SearchHandler:
        )
        room_ids = {r.room_id for r in rooms}

-        # If doing a subset of all rooms search, check if any of the rooms
+        # If doing a subset of all rooms seearch, check if any of the rooms
        # are from an upgraded room, and search their contents as well
        if search_filter.rooms:
            historical_room_ids: List[str] = []
@@ -37,7 +37,6 @@ from synapse.api.presence import UserPresenceState
 from synapse.api.room_versions import KNOWN_ROOM_VERSIONS
 from synapse.events import EventBase
 from synapse.handlers.relations import BundledAggregations
-from synapse.logging import issue9533_logger
 from synapse.logging.context import current_context
 from synapse.logging.opentracing import (
    SynapseTags,
@@ -1403,14 +1402,11 @@ class SyncHandler:

        logger.debug("Fetching room data")

-        (
-            newly_joined_rooms,
-            newly_joined_or_invited_or_knocked_users,
-            newly_left_rooms,
-            newly_left_users,
-        ) = await self._generate_sync_entry_for_rooms(
+        res = await self._generate_sync_entry_for_rooms(
            sync_result_builder, account_data_by_room
        )
+        newly_joined_rooms, newly_joined_or_invited_or_knocked_users, _, _ = res
+        _, _, newly_left_rooms, newly_left_users = res

        block_all_presence_data = (
            since_token is None and sync_config.filter_collection.blocks_all_presence()
@@ -1627,18 +1623,13 @@ class SyncHandler:
                    }
                )

-            if messages and issue9533_logger.isEnabledFor(logging.DEBUG):
-                issue9533_logger.debug(
-                    "Returning to-device messages with stream_ids (%d, %d]; now: %d;"
-                    " msgids: %s",
-                    since_stream_id,
-                    stream_id,
-                    now_token.to_device_key,
-                    [
-                        message["content"].get(EventContentFields.TO_DEVICE_MSGID)
-                        for message in messages
-                    ],
-                )
+            logger.debug(
+                "Returning %d to-device messages between %d and %d (current token: %d)",
+                len(messages),
+                since_stream_id,
+                stream_id,
+                now_token.to_device_key,
+            )
            sync_result_builder.now_token = now_token.copy_and_replace(
                StreamKeyType.TO_DEVICE, stream_id
            )
@@ -1792,7 +1783,6 @@ class SyncHandler:
            - newly_left_rooms
            - newly_left_users
        """
-
        since_token = sync_result_builder.since_token

        # 1. Start by fetching all ephemeral events in rooms we've joined (if required).
@@ -18,7 +18,6 @@ from typing import (
    TYPE_CHECKING,
    Any,
    Callable,
-    Collection,
    Dict,
    Generator,
    Iterable,
@@ -127,7 +126,7 @@ from synapse.types import (
 from synapse.types.state import StateFilter
 from synapse.util import Clock
 from synapse.util.async_helpers import maybe_awaitable
-from synapse.util.caches.descriptors import CachedFunction, cached as _cached
+from synapse.util.caches.descriptors import CachedFunction, cached
 from synapse.util.frozenutils import freeze

 if TYPE_CHECKING:
@@ -137,7 +136,6 @@ if TYPE_CHECKING:

 T = TypeVar("T")
 P = ParamSpec("P")
-F = TypeVar("F", bound=Callable[..., Any])

 """
 This package defines the 'stable' API which can be used by extension modules which
@@ -187,42 +185,6 @@ class UserIpAndAgent:
    last_seen: int


-def cached(
-    *,
-    max_entries: int = 1000,
-    num_args: Optional[int] = None,
-    uncached_args: Optional[Collection[str]] = None,
-) -> Callable[[F], CachedFunction[F]]:
-    """Returns a decorator that applies a memoizing cache around the function. This
-    decorator behaves similarly to functools.lru_cache.
-
-    Example:
-
-        @cached()
-        def foo('a', 'b'):
-            ...
-
-    Added in Synapse v1.74.0.
-
-    Args:
-        max_entries: The maximum number of entries in the cache. If the cache is full
-            and a new entry is added, the least recently accessed entry will be evicted
-            from the cache.
-        num_args: The number of positional arguments (excluding `self`) to use as cache
-            keys. Defaults to all named args of the function.
-        uncached_args: A list of argument names to not use as the cache key. (`self` is
-            always ignored.) Cannot be used with num_args.
-
-    Returns:
-        A decorator that applies a memoizing cache around the function.
-    """
-    return _cached(
-        max_entries=max_entries,
-        num_args=num_args,
-        uncached_args=uncached_args,
-    )
-
-
 class ModuleApi:
    """A proxy object that gets passed to various plugin modules so they
    can register new users etc if necessary.
@@ -26,7 +26,10 @@ def format_push_rules_for_user(
    """Converts a list of rawrules and a enabled map into nested dictionaries
    to match the Matrix client-server format for push rules"""

-    rules: Dict[str, Dict[str, List[Dict[str, Any]]]] = {"global": {}}
+    rules: Dict[str, Dict[str, List[Dict[str, Any]]]] = {
+        "global": {},
+        "device": {},
+    }

    rules["global"] = _add_empty_priority_class_arrays(rules["global"])

@@ -28,7 +28,7 @@ if TYPE_CHECKING:
 logger = logging.getLogger(__name__)


-class ReplicationAddUserAccountDataRestServlet(ReplicationEndpoint):
+class ReplicationUserAccountDataRestServlet(ReplicationEndpoint):
    """Add user account data on the appropriate account data worker.

    Request format:
@@ -49,6 +49,7 @@ class ReplicationAddUserAccountDataRestServlet(ReplicationEndpoint):
        super().__init__(hs)

        self.handler = hs.get_account_data_handler()
+        self.clock = hs.get_clock()

    @staticmethod
    async def _serialize_payload(  # type: ignore[override]
@@ -72,45 +73,7 @@ class ReplicationAddUserAccountDataRestServlet(ReplicationEndpoint):
        return 200, {"max_stream_id": max_stream_id}


-class ReplicationRemoveUserAccountDataRestServlet(ReplicationEndpoint):
-    """Remove user account data on the appropriate account data worker.
-
-    Request format:
-
-        POST /_synapse/replication/remove_user_account_data/:user_id/:type
-
-        {
-            "content": { ... },
-        }
-
-    """
-
-    NAME = "remove_user_account_data"
-    PATH_ARGS = ("user_id", "account_data_type")
-    CACHE = False
-
-    def __init__(self, hs: "HomeServer"):
-        super().__init__(hs)
-
-        self.handler = hs.get_account_data_handler()
-
-    @staticmethod
-    async def _serialize_payload(  # type: ignore[override]
-        user_id: str, account_data_type: str
-    ) -> JsonDict:
-        return {}
-
-    async def _handle_request(  # type: ignore[override]
-        self, request: Request, user_id: str, account_data_type: str
-    ) -> Tuple[int, JsonDict]:
-        max_stream_id = await self.handler.remove_account_data_for_user(
-            user_id, account_data_type
-        )
-
-        return 200, {"max_stream_id": max_stream_id}
-
-
-class ReplicationAddRoomAccountDataRestServlet(ReplicationEndpoint):
+class ReplicationRoomAccountDataRestServlet(ReplicationEndpoint):
    """Add room account data on the appropriate account data worker.

    Request format:
@@ -131,6 +94,7 @@ class ReplicationAddRoomAccountDataRestServlet(ReplicationEndpoint):
        super().__init__(hs)

        self.handler = hs.get_account_data_handler()
+        self.clock = hs.get_clock()

    @staticmethod
    async def _serialize_payload(  # type: ignore[override]
@@ -154,44 +118,6 @@ class ReplicationAddRoomAccountDataRestServlet(ReplicationEndpoint):
        return 200, {"max_stream_id": max_stream_id}


-class ReplicationRemoveRoomAccountDataRestServlet(ReplicationEndpoint):
-    """Remove room account data on the appropriate account data worker.
-
-    Request format:
-
-        POST /_synapse/replication/remove_room_account_data/:user_id/:room_id/:account_data_type
-
-        {
-            "content": { ... },
-        }
-
-    """
-
-    NAME = "remove_room_account_data"
-    PATH_ARGS = ("user_id", "room_id", "account_data_type")
-    CACHE = False
-
-    def __init__(self, hs: "HomeServer"):
-        super().__init__(hs)
-
-        self.handler = hs.get_account_data_handler()
-
-    @staticmethod
-    async def _serialize_payload(  # type: ignore[override]
-        user_id: str, room_id: str, account_data_type: str, content: JsonDict
-    ) -> JsonDict:
-        return {}
-
-    async def _handle_request(  # type: ignore[override]
-        self, request: Request, user_id: str, room_id: str, account_data_type: str
-    ) -> Tuple[int, JsonDict]:
-        max_stream_id = await self.handler.remove_account_data_for_room(
-            user_id, room_id, account_data_type
-        )
-
-        return 200, {"max_stream_id": max_stream_id}
-
-
 class ReplicationAddTagRestServlet(ReplicationEndpoint):
    """Add tag on the appropriate account data worker.

@@ -213,6 +139,7 @@ class ReplicationAddTagRestServlet(ReplicationEndpoint):
        super().__init__(hs)

        self.handler = hs.get_account_data_handler()
+        self.clock = hs.get_clock()

    @staticmethod
    async def _serialize_payload(  # type: ignore[override]
@@ -259,6 +186,7 @@ class ReplicationRemoveTagRestServlet(ReplicationEndpoint):
        super().__init__(hs)

        self.handler = hs.get_account_data_handler()
+        self.clock = hs.get_clock()

    @staticmethod
    async def _serialize_payload(user_id: str, room_id: str, tag: str) -> JsonDict:  # type: ignore[override]
@@ -278,11 +206,7 @@ class ReplicationRemoveTagRestServlet(ReplicationEndpoint):


 def register_servlets(hs: "HomeServer", http_server: HttpServer) -> None:
-    ReplicationAddUserAccountDataRestServlet(hs).register(http_server)
-    ReplicationAddRoomAccountDataRestServlet(hs).register(http_server)
+    ReplicationUserAccountDataRestServlet(hs).register(http_server)
+    ReplicationRoomAccountDataRestServlet(hs).register(http_server)
    ReplicationAddTagRestServlet(hs).register(http_server)
    ReplicationRemoveTagRestServlet(hs).register(http_server)
-
-    if hs.config.experimental.msc3391_enabled:
-        ReplicationRemoveUserAccountDataRestServlet(hs).register(http_server)
-        ReplicationRemoveRoomAccountDataRestServlet(hs).register(http_server)
@@ -13,13 +13,12 @@
 # limitations under the License.

 import logging
-from typing import TYPE_CHECKING, Dict, List, Optional, Tuple
+from typing import TYPE_CHECKING, Optional, Tuple

 from twisted.web.server import Request

 from synapse.http.server import HttpServer
 from synapse.http.servlet import parse_json_object_from_request
-from synapse.logging.opentracing import active_span
 from synapse.replication.http._base import ReplicationEndpoint
 from synapse.types import JsonDict

@@ -85,76 +84,6 @@ class ReplicationUserDevicesResyncRestServlet(ReplicationEndpoint):
        return 200, user_devices


-class ReplicationMultiUserDevicesResyncRestServlet(ReplicationEndpoint):
-    """Ask master to resync the device list for multiple users from the same
-    remote server by contacting their server.
-
-    This must happen on master so that the results can be correctly cached in
-    the database and streamed to workers.
-
-    Request format:
-
-        POST /_synapse/replication/multi_user_device_resync
-
-        {
-            "user_ids": ["@alice:example.org", "@bob:example.org", ...]
-        }
-
-    Response is roughly equivalent to ` /_matrix/federation/v1/user/devices/:user_id`
-    response, but there is a map from user ID to response, e.g.:
-
-        {
-            "@alice:example.org": {
-                "devices": [
-                    {
-                        "device_id": "JLAFKJWSCS",
-                        "keys": { ... },
-                        "device_display_name": "Alice's Mobile Phone"
-                    }
-                ]
-            },
-            ...
-        }
-    """
-
-    NAME = "multi_user_device_resync"
-    PATH_ARGS = ()
-    CACHE = False
-
-    def __init__(self, hs: "HomeServer"):
-        super().__init__(hs)
-
-        from synapse.handlers.device import DeviceHandler
-
-        handler = hs.get_device_handler()
-        assert isinstance(handler, DeviceHandler)
-        self.device_list_updater = handler.device_list_updater
-
-        self.store = hs.get_datastores().main
-        self.clock = hs.get_clock()
-
-    @staticmethod
-    async def _serialize_payload(user_ids: List[str]) -> JsonDict:  # type: ignore[override]
-        return {"user_ids": user_ids}
-
-    async def _handle_request(  # type: ignore[override]
-        self, request: Request
-    ) -> Tuple[int, Dict[str, Optional[JsonDict]]]:
-        content = parse_json_object_from_request(request)
-        user_ids: List[str] = content["user_ids"]
-
-        logger.info("Resync for %r", user_ids)
-        span = active_span()
-        if span:
-            span.set_tag("user_ids", f"{user_ids!r}")
-
-        multi_user_devices = await self.device_list_updater.multi_user_device_resync(
-            user_ids
-        )
-
-        return 200, multi_user_devices
-
-
 class ReplicationUploadKeysForUserRestServlet(ReplicationEndpoint):
    """Ask master to upload keys for the user and send them out over federation to
    update other servers.
@@ -222,5 +151,4 @@ class ReplicationUploadKeysForUserRestServlet(ReplicationEndpoint):

 def register_servlets(hs: "HomeServer", http_server: HttpServer) -> None:
    ReplicationUserDevicesResyncRestServlet(hs).register(http_server)
-    ReplicationMultiUserDevicesResyncRestServlet(hs).register(http_server)
    ReplicationUploadKeysForUserRestServlet(hs).register(http_server)
@@ -36,7 +36,6 @@ from synapse.replication.tcp.streams import (
    TagAccountDataStream,
    ToDeviceStream,
    TypingStream,
-    UnPartialStatedEventStream,
    UnPartialStatedRoomStream,
 )
 from synapse.replication.tcp.streams.events import (
@@ -44,10 +43,7 @@ from synapse.replication.tcp.streams.events import (
    EventsStreamEventRow,
    EventsStreamRow,
 )
-from synapse.replication.tcp.streams.partial_state import (
-    UnPartialStatedEventStreamRow,
-    UnPartialStatedRoomStreamRow,
-)
+from synapse.replication.tcp.streams.partial_state import UnPartialStatedRoomStreamRow
 from synapse.types import PersistedEventPosition, ReadReceipt, StreamKeyType, UserID
 from synapse.util.async_helpers import Linearizer, timeout_deferred
 from synapse.util.metrics import Measure
@@ -152,9 +148,6 @@ class ReplicationDataHandler:
            rows: a list of Stream.ROW_TYPE objects as returned by Stream.parse_row.
        """
        self.store.process_replication_rows(stream_name, instance_name, token, rows)
-        # NOTE: this must be called after process_replication_rows to ensure any
-        # cache invalidations are first handled before any stream ID advances.
-        self.store.process_replication_position(stream_name, instance_name, token)

        if self.send_handler:
            await self.send_handler.process_replication_rows(stream_name, token, rows)
@@ -254,14 +247,6 @@ class ReplicationDataHandler:
                self._state_storage_controller.notify_room_un_partial_stated(
                    row.room_id
                )
-        elif stream_name == UnPartialStatedEventStream.NAME:
-            for row in rows:
-                assert isinstance(row, UnPartialStatedEventStreamRow)
-
-                # Wake up any tasks waiting for the event to be un-partial-stated.
-                self._state_storage_controller.notify_event_un_partial_stated(
-                    row.event_id
-                )

        await self._presence_handler.process_replication_rows(
            stream_name, instance_name, token, rows
@@ -42,10 +42,7 @@ from synapse.replication.tcp.streams._base import (
 )
 from synapse.replication.tcp.streams.events import EventsStream
 from synapse.replication.tcp.streams.federation import FederationStream
-from synapse.replication.tcp.streams.partial_state import (
-    UnPartialStatedEventStream,
-    UnPartialStatedRoomStream,
-)
+from synapse.replication.tcp.streams.partial_state import UnPartialStatedRoomStream

 STREAMS_MAP = {
    stream.NAME: stream
@@ -66,7 +63,6 @@ STREAMS_MAP = {
        AccountDataStream,
        UserSignatureStream,
        UnPartialStatedRoomStream,
-        UnPartialStatedEventStream,
    )
 }

@@ -87,5 +83,4 @@ __all__ = [
    "AccountDataStream",
    "UserSignatureStream",
    "UnPartialStatedRoomStream",
-    "UnPartialStatedEventStream",
 ]
@@ -46,31 +46,3 @@ class UnPartialStatedRoomStream(Stream):
            current_token_without_instance(store.get_un_partial_stated_rooms_token),
            store.get_un_partial_stated_rooms_from_stream,
        )
-
-
-@attr.s(slots=True, frozen=True, auto_attribs=True)
-class UnPartialStatedEventStreamRow:
-    # ID of the event that has been un-partial-stated.
-    event_id: str
-
-    # True iff the rejection status of the event changed as a result of being
-    # un-partial-stated.
-    rejection_status_changed: bool
-
-
-class UnPartialStatedEventStream(Stream):
-    """
-    Stream to notify about events becoming un-partial-stated.
-    """
-
-    NAME = "un_partial_stated_event"
-    ROW_TYPE = UnPartialStatedEventStreamRow
-
-    def __init__(self, hs: "HomeServer"):
-        store = hs.get_datastores().main
-        super().__init__(
-            hs.get_instance_name(),
-            # TODO(faster_joins, multiple writers): we need to account for instance names
-            current_token_without_instance(store.get_un_partial_stated_events_token),
-            store.get_un_partial_stated_events_from_stream,
-        )
@@ -3,10 +3,11 @@

 {% block header %}
 <script src="https://www.recaptcha.net/recaptcha/api.js" async defer></script>
+<script src="//code.jquery.com/jquery-1.11.2.min.js"></script>
 <link rel="stylesheet" href="/_matrix/static/client/register/style.css">
 <script>
 function captchaDone() {
-    document.getElementById('registrationForm').submit(); 
+    $('#registrationForm').submit();
 }
 </script>
 {% endblock %}
@@ -338,11 +338,6 @@ class EmailThreepidRequestTokenRestServlet(RestServlet):
            )

    async def on_POST(self, request: SynapseRequest) -> Tuple[int, JsonDict]:
-        if not self.hs.config.registration.enable_3pid_changes:
-            raise SynapseError(
-                400, "3PID changes are disabled on this server", Codes.FORBIDDEN
-            )
-
        if not self.config.email.can_verify_email:
            logger.warning(
                "Adding emails have been disabled due to lack of an email config"
@@ -41,7 +41,6 @@ class AccountDataServlet(RestServlet):

    def __init__(self, hs: "HomeServer"):
        super().__init__()
-        self._hs = hs
        self.auth = hs.get_auth()
        self.store = hs.get_datastores().main
        self.handler = hs.get_account_data_handler()
@@ -55,16 +54,6 @@ class AccountDataServlet(RestServlet):

        body = parse_json_object_from_request(request)

-        # If experimental support for MSC3391 is enabled, then providing an empty dict
-        # as the value for an account data type should be functionally equivalent to
-        # calling the DELETE method on the same type.
-        if self._hs.config.experimental.msc3391_enabled:
-            if body == {}:
-                await self.handler.remove_account_data_for_user(
-                    user_id, account_data_type
-                )
-                return 200, {}
-
        await self.handler.add_account_data_for_user(user_id, account_data_type, body)

        return 200, {}
@@ -83,48 +72,9 @@ class AccountDataServlet(RestServlet):
        if event is None:
            raise NotFoundError("Account data not found")

-        # If experimental support for MSC3391 is enabled, then this endpoint should
-        # return a 404 if the content for an account data type is an empty dict.
-        if self._hs.config.experimental.msc3391_enabled and event == {}:
-            raise NotFoundError("Account data not found")
-
        return 200, event


-class UnstableAccountDataServlet(RestServlet):
-    """
-    Contains an unstable endpoint for removing user account data, as specified by
-    MSC3391. If that MSC is accepted, this code should have unstable prefixes removed
-    and become incorporated into AccountDataServlet above.
-    """
-
-    PATTERNS = client_patterns(
-        "/org.matrix.msc3391/user/(?P<user_id>[^/]*)"
-        "/account_data/(?P<account_data_type>[^/]*)",
-        unstable=True,
-        releases=(),
-    )
-
-    def __init__(self, hs: "HomeServer"):
-        super().__init__()
-        self.auth = hs.get_auth()
-        self.handler = hs.get_account_data_handler()
-
-    async def on_DELETE(
-        self,
-        request: SynapseRequest,
-        user_id: str,
-        account_data_type: str,
-    ) -> Tuple[int, JsonDict]:
-        requester = await self.auth.get_user_by_req(request)
-        if user_id != requester.user.to_string():
-            raise AuthError(403, "Cannot delete account data for other users.")
-
-        await self.handler.remove_account_data_for_user(user_id, account_data_type)
-
-        return 200, {}
-
-
 class RoomAccountDataServlet(RestServlet):
    """
    PUT /user/{user_id}/rooms/{room_id}/account_data/{account_dataType} HTTP/1.1
@@ -139,7 +89,6 @@ class RoomAccountDataServlet(RestServlet):

    def __init__(self, hs: "HomeServer"):
        super().__init__()
-        self._hs = hs
        self.auth = hs.get_auth()
        self.store = hs.get_datastores().main
        self.handler = hs.get_account_data_handler()
@@ -172,16 +121,6 @@ class RoomAccountDataServlet(RestServlet):
                Codes.BAD_JSON,
            )

-        # If experimental support for MSC3391 is enabled, then providing an empty dict
-        # as the value for an account data type should be functionally equivalent to
-        # calling the DELETE method on the same type.
-        if self._hs.config.experimental.msc3391_enabled:
-            if body == {}:
-                await self.handler.remove_account_data_for_room(
-                    user_id, room_id, account_data_type
-                )
-                return 200, {}
-
        await self.handler.add_account_data_to_room(
            user_id, room_id, account_data_type, body
        )
@@ -213,63 +152,9 @@ class RoomAccountDataServlet(RestServlet):
        if event is None:
            raise NotFoundError("Room account data not found")

-        # If experimental support for MSC3391 is enabled, then this endpoint should
-        # return a 404 if the content for an account data type is an empty dict.
-        if self._hs.config.experimental.msc3391_enabled and event == {}:
-            raise NotFoundError("Room account data not found")
-
        return 200, event


-class UnstableRoomAccountDataServlet(RestServlet):
-    """
-    Contains an unstable endpoint for removing room account data, as specified by
-    MSC3391. If that MSC is accepted, this code should have unstable prefixes removed
-    and become incorporated into RoomAccountDataServlet above.
-    """
-
-    PATTERNS = client_patterns(
-        "/org.matrix.msc3391/user/(?P<user_id>[^/]*)"
-        "/rooms/(?P<room_id>[^/]*)"
-        "/account_data/(?P<account_data_type>[^/]*)",
-        unstable=True,
-        releases=(),
-    )
-
-    def __init__(self, hs: "HomeServer"):
-        super().__init__()
-        self.auth = hs.get_auth()
-        self.handler = hs.get_account_data_handler()
-
-    async def on_DELETE(
-        self,
-        request: SynapseRequest,
-        user_id: str,
-        room_id: str,
-        account_data_type: str,
-    ) -> Tuple[int, JsonDict]:
-        requester = await self.auth.get_user_by_req(request)
-        if user_id != requester.user.to_string():
-            raise AuthError(403, "Cannot delete account data for other users.")
-
-        if not RoomID.is_valid(room_id):
-            raise SynapseError(
-                400,
-                f"{room_id} is not a valid room ID",
-                Codes.INVALID_PARAM,
-            )
-
-        await self.handler.remove_account_data_for_room(
-            user_id, room_id, account_data_type
-        )
-
-        return 200, {}
-
-
 def register_servlets(hs: "HomeServer", http_server: HttpServer) -> None:
    AccountDataServlet(hs).register(http_server)
    RoomAccountDataServlet(hs).register(http_server)
-
-    if hs.config.experimental.msc3391_enabled:
-        UnstableAccountDataServlet(hs).register(http_server)
-        UnstableRoomAccountDataServlet(hs).register(http_server)
@@ -38,6 +38,11 @@ from synapse.api.errors import (
 )
 from synapse.api.filtering import Filter
 from synapse.events.utils import format_event_for_client_v2
+from synapse.hacks.selective_scalene_profiling import (
+    CpuUtimeTracker,
+    ProfilingDecider,
+    SelectiveProfiling,
+)
 from synapse.http.server import HttpServer
 from synapse.http.servlet import (
    ResolveRoomIdMixin,
@@ -313,6 +318,19 @@ class RoomStateEventRestServlet(TransactionRestServlet):
        return 200, ret


+_sep_cpu = CpuUtimeTracker()
+
+
+def _sep_cond() -> bool:
+    utime = _sep_cpu.update_return_utime()
+    if utime is None:
+        return False
+    return True
+
+
+send_event_profiler = ProfilingDecider("send_event", _sep_cond)
+
+
 # TODO: Needs unit testing for generic events + feedback
 class RoomSendEventRestServlet(TransactionRestServlet):
    def __init__(self, hs: "HomeServer"):
@@ -333,33 +351,38 @@ class RoomSendEventRestServlet(TransactionRestServlet):
        txn_id: Optional[str] = None,
    ) -> Tuple[int, JsonDict]:
        requester = await self.auth.get_user_by_req(request, allow_guest=True)
-        content = parse_json_object_from_request(request)
+        logger.info("SP!!!")
+        with SelectiveProfiling(
+            send_event_profiler,
+            enable=requester.user.to_string() == "@reivilibre.element:librepush.net",
+        ):
+            content = parse_json_object_from_request(request)

-        event_dict: JsonDict = {
-            "type": event_type,
-            "content": content,
-            "room_id": room_id,
-            "sender": requester.user.to_string(),
-        }
+            event_dict: JsonDict = {
+                "type": event_type,
+                "content": content,
+                "room_id": room_id,
+                "sender": requester.user.to_string(),
+            }

-        if requester.app_service:
-            origin_server_ts = parse_integer(request, "ts")
-            if origin_server_ts is not None:
-                event_dict["origin_server_ts"] = origin_server_ts
+            if requester.app_service:
+                origin_server_ts = parse_integer(request, "ts")
+                if origin_server_ts is not None:
+                    event_dict["origin_server_ts"] = origin_server_ts

-        try:
-            (
-                event,
-                _,
-            ) = await self.event_creation_handler.create_and_send_nonmember_event(
-                requester, event_dict, txn_id=txn_id
-            )
-            event_id = event.event_id
-        except ShadowBanError:
-            event_id = "$" + random_string(43)
+            try:
+                (
+                    event,
+                    _,
+                ) = await self.event_creation_handler.create_and_send_nonmember_event(
+                    requester, event_dict, txn_id=txn_id
+                )
+                event_id = event.event_id
+            except ShadowBanError:
+                event_id = "$" + random_string(43)

-        set_tag("event_id", event_id)
-        return 200, {"event_id": event_id}
+            set_tag("event_id", event_id)
+            return 200, {"event_id": event_id}

    def on_GET(
        self, request: SynapseRequest, room_id: str, event_type: str, txn_id: str
@@ -11,7 +11,6 @@
 #  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 #  See the License for the specific language governing permissions and
 #  limitations under the License.
-import html
 import logging
 import urllib.parse
 from typing import TYPE_CHECKING, List, Optional
@@ -162,9 +161,7 @@ class OEmbedProvider:

        title = oembed.get("title")
        if title and isinstance(title, str):
-            # A common WordPress plug-in seems to incorrectly escape entities
-            # in the oEmbed response.
-            open_graph_response["og:title"] = html.unescape(title)
+            open_graph_response["og:title"] = title

        author_name = oembed.get("author_name")
        if not isinstance(author_name, str):
@@ -183,9 +180,9 @@ class OEmbedProvider:
        # Process each type separately.
        oembed_type = oembed.get("type")
        if oembed_type == "rich":
-            html_str = oembed.get("html")
-            if isinstance(html_str, str):
-                calc_description_and_urls(open_graph_response, html_str)
+            html = oembed.get("html")
+            if isinstance(html, str):
+                calc_description_and_urls(open_graph_response, html)

        elif oembed_type == "photo":
            # If this is a photo, use the full image, not the thumbnail.
@@ -195,8 +192,8 @@ class OEmbedProvider:

        elif oembed_type == "video":
            open_graph_response["og:type"] = "video.other"
-            html_str = oembed.get("html")
-            if html_str and isinstance(html_str, str):
+            html = oembed.get("html")
+            if html and isinstance(html, str):
                calc_description_and_urls(open_graph_response, oembed["html"])
            for size in ("width", "height"):
                val = oembed.get(size)
@@ -202,20 +202,14 @@ class StateHandler:
            room_id: the room_id containing the given events.
            event_ids: the events whose state should be fetched and resolved.
            await_full_state: if `True`, will block if we do not yet have complete state
-                at these events and `state_filter` is not satisfied by partial state.
-                Defaults to `True`.
+                at the given `event_id`s, regardless of whether `state_filter` is
+                satisfied by partial state.

        Returns:
            the state dict (a mapping from (event_type, state_key) -> event_id) which
            holds the resolution of the states after the given event IDs.
        """
        logger.debug("calling resolve_state_groups from compute_state_after_events")
-        if (
-            await_full_state
-            and state_filter
-            and not state_filter.must_await_full_state(self.hs.is_mine_id)
-        ):
-            await_full_state = False
        ret = await self.resolve_state_groups_for_events(
            room_id, event_ids, await_full_state
        )
@@ -57,22 +57,7 @@ class SQLBaseStore(metaclass=ABCMeta):
        token: int,
        rows: Iterable[Any],
    ) -> None:
-        """
-        Used by storage classes to invalidate caches based on incoming replication data. These
-        must not update any ID generators, use `process_replication_position`.
-        """
-
-    def process_replication_position(  # noqa: B027 (no-op by design)
-        self,
-        stream_name: str,
-        instance_name: str,
-        token: int,
-    ) -> None:
-        """
-        Used by storage classes to advance ID generators based on incoming replication data. This
-        is called after process_replication_rows such that caches are invalidated before any token
-        positions advance.
-        """
+        pass

    def _invalidate_state_caches(
        self, room_id: str, members_changed: Collection[str]
@@ -1762,8 +1762,7 @@ class DatabasePool:
            desc: description of the transaction, for logging and metrics

        Returns:
-            A list of dictionaries, one per result row, each a mapping between the
-            column names from `retcols` and that column's value for the row.
+            A list of dictionaries.
        """
        return await self.runInteraction(
            desc,
@@ -1792,10 +1791,6 @@ class DatabasePool:
                column names and values to select the rows with, or None to not
                apply a WHERE clause.
            retcols: the names of the columns to return
-
-        Returns:
-            A list of dictionaries, one per result row, each a mapping between the
-            column names from `retcols` and that column's value for the row.
        """
        if keyvalues:
            sql = "SELECT %s FROM %s WHERE %s" % (
@@ -1903,19 +1898,6 @@ class DatabasePool:
        updatevalues: Dict[str, Any],
        desc: str,
    ) -> int:
-        """
-        Update rows in the given database table.
-        If the given keyvalues don't match anything, nothing will be updated.
-
-        Args:
-            table: The database table to update.
-            keyvalues: A mapping of column name to value to match rows on.
-            updatevalues: A mapping of column name to value to replace in any matched rows.
-            desc: description of the transaction, for logging and metrics.
-
-        Returns:
-            The number of rows that were updated. Will be 0 if no matching rows were found.
-        """
        return await self.runInteraction(
            desc, self.simple_update_txn, table, keyvalues, updatevalues
        )
@@ -1927,19 +1909,6 @@ class DatabasePool:
        keyvalues: Dict[str, Any],
        updatevalues: Dict[str, Any],
    ) -> int:
-        """
-        Update rows in the given database table.
-        If the given keyvalues don't match anything, nothing will be updated.
-
-        Args:
-            txn: The database transaction object.
-            table: The database table to update.
-            keyvalues: A mapping of column name to value to match rows on.
-            updatevalues: A mapping of column name to value to replace in any matched rows.
-
-        Returns:
-            The number of rows that were updated. Will be 0 if no matching rows were found.
-        """
        if keyvalues:
            where = "WHERE %s" % " AND ".join("%s = ?" % k for k in keyvalues.keys())
        else:
@@ -123,11 +123,7 @@ class AccountDataWorkerStore(PushRulesWorkerStore, CacheInvalidationWorkerStore)
    async def get_account_data_for_user(
        self, user_id: str
    ) -> Tuple[Dict[str, JsonDict], Dict[str, Dict[str, JsonDict]]]:
-        """
-        Get all the client account_data for a user.
-
-        If experimental MSC3391 support is enabled, any entries with an empty
-        content body are excluded; as this means they have been deleted.
+        """Get all the client account_data for a user.

        Args:
            user_id: The user to get the account_data for.
@@ -139,48 +135,27 @@ class AccountDataWorkerStore(PushRulesWorkerStore, CacheInvalidationWorkerStore)
        def get_account_data_for_user_txn(
            txn: LoggingTransaction,
        ) -> Tuple[Dict[str, JsonDict], Dict[str, Dict[str, JsonDict]]]:
-            # The 'content != '{}' condition below prevents us from using
-            # `simple_select_list_txn` here, as it doesn't support conditions
-            # other than 'equals'.
-            sql = """
-                SELECT account_data_type, content FROM account_data
-                WHERE user_id = ?
-            """
-
-            # If experimental MSC3391 support is enabled, then account data entries
-            # with an empty content are considered "deleted". So skip adding them to
-            # the results.
-            if self.hs.config.experimental.msc3391_enabled:
-                sql += " AND content != '{}'"
-
-            txn.execute(sql, (user_id,))
-            rows = self.db_pool.cursor_to_dict(txn)
+            rows = self.db_pool.simple_select_list_txn(
+                txn,
+                "account_data",
+                {"user_id": user_id},
+                ["account_data_type", "content"],
+            )

            global_account_data = {
                row["account_data_type"]: db_to_json(row["content"]) for row in rows
            }

-            # The 'content != '{}' condition below prevents us from using
-            # `simple_select_list_txn` here, as it doesn't support conditions
-            # other than 'equals'.
-            sql = """
-                SELECT room_id, account_data_type, content FROM room_account_data
-                WHERE user_id = ?
-            """
-
-            # If experimental MSC3391 support is enabled, then account data entries
-            # with an empty content are considered "deleted". So skip adding them to
-            # the results.
-            if self.hs.config.experimental.msc3391_enabled:
-                sql += " AND content != '{}'"
-
-            txn.execute(sql, (user_id,))
-            rows = self.db_pool.cursor_to_dict(txn)
+            rows = self.db_pool.simple_select_list_txn(
+                txn,
+                "room_account_data",
+                {"user_id": user_id},
+                ["room_id", "account_data_type", "content"],
+            )

            by_room: Dict[str, Dict[str, JsonDict]] = {}
            for row in rows:
                room_data = by_room.setdefault(row["room_id"], {})
-
                room_data[row["account_data_type"]] = db_to_json(row["content"])

            return global_account_data, by_room
@@ -436,7 +411,10 @@ class AccountDataWorkerStore(PushRulesWorkerStore, CacheInvalidationWorkerStore)
        token: int,
        rows: Iterable[Any],
    ) -> None:
-        if stream_name == AccountDataStream.NAME:
+        if stream_name == TagAccountDataStream.NAME:
+            self._account_data_id_gen.advance(instance_name, token)
+        elif stream_name == AccountDataStream.NAME:
+            self._account_data_id_gen.advance(instance_name, token)
            for row in rows:
                if not row.room_id:
                    self.get_global_account_data_by_type_for_user.invalidate(
@@ -451,15 +429,6 @@ class AccountDataWorkerStore(PushRulesWorkerStore, CacheInvalidationWorkerStore)

        super().process_replication_rows(stream_name, instance_name, token, rows)

-    def process_replication_position(
-        self, stream_name: str, instance_name: str, token: int
-    ) -> None:
-        if stream_name == TagAccountDataStream.NAME:
-            self._account_data_id_gen.advance(instance_name, token)
-        elif stream_name == AccountDataStream.NAME:
-            self._account_data_id_gen.advance(instance_name, token)
-        super().process_replication_position(stream_name, instance_name, token)
-
    async def add_account_data_to_room(
        self, user_id: str, room_id: str, account_data_type: str, content: JsonDict
    ) -> int:
@@ -500,72 +469,6 @@ class AccountDataWorkerStore(PushRulesWorkerStore, CacheInvalidationWorkerStore)

        return self._account_data_id_gen.get_current_token()

-    async def remove_account_data_for_room(
-        self, user_id: str, room_id: str, account_data_type: str
-    ) -> Optional[int]:
-        """Delete the room account data for the user of a given type.
-
-        Args:
-            user_id: The user to remove account_data for.
-            room_id: The room ID to scope the request to.
-            account_data_type: The account data type to delete.
-
-        Returns:
-            The maximum stream position, or None if there was no matching room account
-            data to delete.
-        """
-        assert self._can_write_to_account_data
-        assert isinstance(self._account_data_id_gen, AbstractStreamIdGenerator)
-
-        def _remove_account_data_for_room_txn(
-            txn: LoggingTransaction, next_id: int
-        ) -> bool:
-            """
-            Args:
-                txn: The transaction object.
-                next_id: The stream_id to update any existing rows to.
-
-            Returns:
-                True if an entry in room_account_data had its content set to '{}',
-                otherwise False. This informs callers of whether there actually was an
-                existing room account data entry to delete, or if the call was a no-op.
-            """
-            # We can't use `simple_update` as it doesn't have the ability to specify
-            # where clauses other than '=', which we need for `content != '{}'` below.
-            sql = """
-                UPDATE room_account_data
-                    SET stream_id = ?, content = '{}'
-                WHERE user_id = ?
-                    AND room_id = ?
-                    AND account_data_type = ?
-                    AND content != '{}'
-            """
-            txn.execute(
-                sql,
-                (next_id, user_id, room_id, account_data_type),
-            )
-            # Return true if any rows were updated.
-            return txn.rowcount != 0
-
-        async with self._account_data_id_gen.get_next() as next_id:
-            row_updated = await self.db_pool.runInteraction(
-                "remove_account_data_for_room",
-                _remove_account_data_for_room_txn,
-                next_id,
-            )
-
-            if not row_updated:
-                return None
-
-            self._account_data_stream_cache.entity_has_changed(user_id, next_id)
-            self.get_account_data_for_user.invalidate((user_id,))
-            self.get_account_data_for_room.invalidate((user_id, room_id))
-            self.get_account_data_for_room_and_type.prefill(
-                (user_id, room_id, account_data_type), {}
-            )
-
-        return self._account_data_id_gen.get_current_token()
-
    async def add_account_data_for_user(
        self, user_id: str, account_data_type: str, content: JsonDict
    ) -> int:
@@ -666,108 +569,6 @@ class AccountDataWorkerStore(PushRulesWorkerStore, CacheInvalidationWorkerStore)
            self._invalidate_cache_and_stream(txn, self.ignored_by, (ignored_user_id,))
        self._invalidate_cache_and_stream(txn, self.ignored_users, (user_id,))

-    async def remove_account_data_for_user(
-        self,
-        user_id: str,
-        account_data_type: str,
-    ) -> Optional[int]:
-        """
-        Delete a single piece of user account data by type.
-
-        A "delete" is performed by updating a potentially existing row in the
-        "account_data" database table for (user_id, account_data_type) and
-        setting its content to "{}".
-
-        Args:
-            user_id: The user ID to modify the account data of.
-            account_data_type: The type to remove.
-
-        Returns:
-            The maximum stream position, or None if there was no matching account data
-            to delete.
-        """
-        assert self._can_write_to_account_data
-        assert isinstance(self._account_data_id_gen, AbstractStreamIdGenerator)
-
-        def _remove_account_data_for_user_txn(
-            txn: LoggingTransaction, next_id: int
-        ) -> bool:
-            """
-            Args:
-                txn: The transaction object.
-                next_id: The stream_id to update any existing rows to.
-
-            Returns:
-                True if an entry in account_data had its content set to '{}', otherwise
-                False. This informs callers of whether there actually was an existing
-                account data entry to delete, or if the call was a no-op.
-            """
-            # We can't use `simple_update` as it doesn't have the ability to specify
-            # where clauses other than '=', which we need for `content != '{}'` below.
-            sql = """
-                UPDATE account_data
-                    SET stream_id = ?, content = '{}'
-                WHERE user_id = ?
-                    AND account_data_type = ?
-                    AND content != '{}'
-            """
-            txn.execute(sql, (next_id, user_id, account_data_type))
-            if txn.rowcount == 0:
-                # We didn't update any rows. This means that there was no matching room
-                # account data entry to delete in the first place.
-                return False
-
-            # Ignored users get denormalized into a separate table as an optimisation.
-            if account_data_type == AccountDataTypes.IGNORED_USER_LIST:
-                # If this method was called with the ignored users account data type, we
-                # simply delete all ignored users.
-
-                # First pull all the users that this user ignores.
-                previously_ignored_users = set(
-                    self.db_pool.simple_select_onecol_txn(
-                        txn,
-                        table="ignored_users",
-                        keyvalues={"ignorer_user_id": user_id},
-                        retcol="ignored_user_id",
-                    )
-                )
-
-                # Then delete them from the database.
-                self.db_pool.simple_delete_txn(
-                    txn,
-                    table="ignored_users",
-                    keyvalues={"ignorer_user_id": user_id},
-                )
-
-                # Invalidate the cache for ignored users which were removed.
-                for ignored_user_id in previously_ignored_users:
-                    self._invalidate_cache_and_stream(
-                        txn, self.ignored_by, (ignored_user_id,)
-                    )
-
-                # Invalidate for this user the cache tracking ignored users.
-                self._invalidate_cache_and_stream(txn, self.ignored_users, (user_id,))
-
-            return True
-
-        async with self._account_data_id_gen.get_next() as next_id:
-            row_updated = await self.db_pool.runInteraction(
-                "remove_account_data_for_user",
-                _remove_account_data_for_user_txn,
-                next_id,
-            )
-
-            if not row_updated:
-                return None
-
-            self._account_data_stream_cache.entity_has_changed(user_id, next_id)
-            self.get_account_data_for_user.invalidate((user_id,))
-            self.get_global_account_data_by_type_for_user.prefill(
-                (user_id, account_data_type), {}
-            )
-
-        return self._account_data_id_gen.get_current_token()
-
    async def purge_account_data_for_user(self, user_id: str) -> None:
        """
        Removes ALL the account data for a user.
@@ -164,6 +164,9 @@ class CacheInvalidationWorkerStore(SQLBaseStore):
                    backfilled=True,
                )
        elif stream_name == CachesStream.NAME:
+            if self._cache_id_gen:
+                self._cache_id_gen.advance(instance_name, token)
+
            for row in rows:
                if row.cache_func == CURRENT_STATE_CACHE_NAME:
                    if row.keys is None:
@@ -179,14 +182,6 @@ class CacheInvalidationWorkerStore(SQLBaseStore):

        super().process_replication_rows(stream_name, instance_name, token, rows)

-    def process_replication_position(
-        self, stream_name: str, instance_name: str, token: int
-    ) -> None:
-        if stream_name == CachesStream.NAME:
-            if self._cache_id_gen:
-                self._cache_id_gen.advance(instance_name, token)
-        super().process_replication_position(stream_name, instance_name, token)
-
    def _process_event_stream_row(self, token: int, row: EventsStreamRow) -> None:
        data = row.data

@@ -157,13 +157,6 @@ class DeviceInboxWorkerStore(SQLBaseStore):
                    )
        return super().process_replication_rows(stream_name, instance_name, token, rows)

-    def process_replication_position(
-        self, stream_name: str, instance_name: str, token: int
-    ) -> None:
-        if stream_name == ToDeviceStream.NAME:
-            self._device_inbox_id_gen.advance(instance_name, token)
-        super().process_replication_position(stream_name, instance_name, token)
-
    def get_to_device_stream_token(self) -> int:
        return self._device_inbox_id_gen.get_current_token()

@@ -54,7 +54,7 @@ from synapse.storage.util.id_generators import (
    AbstractStreamIdTracker,
    StreamIdGenerator,
 )
-from synapse.types import JsonDict, StrCollection, get_verify_key_from_cross_signing_key
+from synapse.types import JsonDict, get_verify_key_from_cross_signing_key
 from synapse.util import json_decoder, json_encoder
 from synapse.util.caches.descriptors import cached, cachedList
 from synapse.util.caches.lrucache import LruCache
@@ -162,21 +162,14 @@ class DeviceWorkerStore(RoomMemberWorkerStore, EndToEndKeyWorkerStore):
        self, stream_name: str, instance_name: str, token: int, rows: Iterable[Any]
    ) -> None:
        if stream_name == DeviceListsStream.NAME:
+            self._device_list_id_gen.advance(instance_name, token)
            self._invalidate_caches_for_devices(token, rows)
        elif stream_name == UserSignatureStream.NAME:
+            self._device_list_id_gen.advance(instance_name, token)
            for row in rows:
                self._user_signature_stream_cache.entity_has_changed(row.user_id, token)
        return super().process_replication_rows(stream_name, instance_name, token, rows)

-    def process_replication_position(
-        self, stream_name: str, instance_name: str, token: int
-    ) -> None:
-        if stream_name == DeviceListsStream.NAME:
-            self._device_list_id_gen.advance(instance_name, token)
-        elif stream_name == UserSignatureStream.NAME:
-            self._device_list_id_gen.advance(instance_name, token)
-        super().process_replication_position(stream_name, instance_name, token)
-
    def _invalidate_caches_for_devices(
        self, token: int, rows: Iterable[DeviceListsStream.DeviceListsStreamRow]
    ) -> None:
@@ -1069,30 +1062,16 @@ class DeviceWorkerStore(RoomMemberWorkerStore, EndToEndKeyWorkerStore):

        return {row["user_id"] for row in rows}

-    async def mark_remote_users_device_caches_as_stale(
-        self, user_ids: StrCollection
-    ) -> None:
+    async def mark_remote_user_device_cache_as_stale(self, user_id: str) -> None:
        """Records that the server has reason to believe the cache of the devices
        for the remote users is out of date.
        """
-
-        def _mark_remote_users_device_caches_as_stale_txn(
-            txn: LoggingTransaction,
-        ) -> None:
-            # TODO add insertion_values support to simple_upsert_many and use
-            #      that!
-            for user_id in user_ids:
-                self.db_pool.simple_upsert_txn(
-                    txn,
-                    table="device_lists_remote_resync",
-                    keyvalues={"user_id": user_id},
-                    values={},
-                    insertion_values={"added_ts": self._clock.time_msec()},
-                )
-
-        await self.db_pool.runInteraction(
-            "mark_remote_users_device_caches_as_stale",
-            _mark_remote_users_device_caches_as_stale_txn,
+        await self.db_pool.simple_upsert(
+            table="device_lists_remote_resync",
+            keyvalues={"user_id": user_id},
+            values={},
+            insertion_values={"added_ts": self._clock.time_msec()},
+            desc="mark_remote_user_device_cache_as_stale",
        )

    async def mark_remote_user_device_cache_as_valid(self, user_id: str) -> None:
--- a/Show More
+++ b/Show More
Author	SHA1	Message	Date
Olivier Wilkinson (reivilibre)	d80b3596b5	Revert "dodgy" This reverts commit `6c579370d4`.	2022-12-19 22:30:33 +00:00
Olivier Wilkinson (reivilibre)	6c579370d4	dodgy	2022-12-19 22:05:54 +00:00
Olivier Wilkinson (reivilibre)	37153a5478	Add logging	2022-12-19 21:48:00 +00:00
Olivier Wilkinson (reivilibre)	5e56736313	Example: room send event profiling	2022-12-19 20:55:56 +00:00
Olivier Wilkinson (reivilibre)	bd39e8363c	Add helpers	2022-12-19 20:55:56 +00:00
Olivier Wilkinson (reivilibre)	5ebd31c349	Add Scalene	2022-12-19 20:20:35 +00:00
Olivier Wilkinson (reivilibre)	16d70d0627	Add log line so we can see what's going on	2022-12-19 20:20:14 +00:00
Olivier Wilkinson (reivilibre)	68ecfc346b	Build a set of who we are interested in first and foremost	2022-12-19 20:20:14 +00:00