Version picker added for v1.85 docs

.ci/scripts/calculate_jobs.py

@@ -29,12 +29,11 @@ IS_PR = os.environ["GITHUB_REF"].startswith("refs/pull/")
 
 # First calculate the various trial jobs.
 #
-# For PRs, we only run each type of test with the oldest Python version supported (which
-# is Python 3.8 right now)
+# For each type of test we only run on Py3.7 on PRs
 
 trial_sqlite_tests = [
     {
-        "python-version": "3.8",
+        "python-version": "3.7",
         "database": "sqlite",
         "extras": "all",
     }
@@ -47,13 +46,13 @@ if not IS_PR:
             "database": "sqlite",
             "extras": "all",
         }
-        for version in ("3.9", "3.10", "3.11")
+        for version in ("3.8", "3.9", "3.10", "3.11")
     )
 
 
 trial_postgres_tests = [
     {
-        "python-version": "3.8",
+        "python-version": "3.7",
         "database": "postgres",
         "postgres-version": "11",
         "extras": "all",
@@ -72,7 +71,7 @@ if not IS_PR:
 
 trial_no_extra_tests = [
     {
-        "python-version": "3.8",
+        "python-version": "3.7",
         "database": "sqlite",
         "extras": "",
     }
@@ -134,6 +133,11 @@ if not IS_PR:
                 "sytest-tag": "testing",
                 "postgres": "postgres",
             },
+            {
+                "sytest-tag": "buster",
+                "postgres": "multi-postgres",
+                "workers": "workers",
+            },
         ]
     )
 

.github/workflows/latest_deps.yml

@@ -22,21 +22,7 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
-  check_repo:
-    # Prevent this workflow from running on any fork of Synapse other than matrix-org/synapse, as it is
-    # only useful to the Synapse core team.
-    # All other workflow steps depend on this one, thus if 'should_run_workflow' is not 'true', the rest
-    # of the workflow will be skipped as well.
-    runs-on: ubuntu-latest
-    outputs:
-      should_run_workflow: ${{ steps.check_condition.outputs.should_run_workflow }}
-    steps:
-      - id: check_condition
-        run: echo "should_run_workflow=${{ github.repository == 'matrix-org/synapse' }}" >> "$GITHUB_OUTPUT"
-
   mypy:
-    needs: check_repo
-    if: needs.check_repo.outputs.should_run_workflow == 'true'
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3
@@ -61,8 +47,6 @@ jobs:
         run: sed '/warn_unused_ignores = True/d' -i mypy.ini
       - run: poetry run mypy
   trial:
-    needs: check_repo
-    if: needs.check_repo.outputs.should_run_workflow == 'true'
     runs-on: ubuntu-latest
     strategy:
       matrix:
@@ -121,8 +105,6 @@ jobs:
 
 
   sytest:
-    needs: check_repo
-    if: needs.check_repo.outputs.should_run_workflow == 'true'
     runs-on: ubuntu-latest
     container:
       image: matrixdotorg/sytest-synapse:testing
@@ -174,8 +156,7 @@ jobs:
 
 
   complement:
-    needs: check_repo
-    if: "!failure() && !cancelled() && needs.check_repo.outputs.should_run_workflow == 'true'"
+    if: "${{ !failure() && !cancelled() }}"
     runs-on: ubuntu-latest
 
     strategy:
@@ -211,7 +192,7 @@ jobs:
   # Open an issue if the build fails, so we know about it.
   # Only do this if we're not experimenting with this action in a PR.
   open-issue:
-    if: "failure() && github.event_name != 'push' && github.event_name != 'pull_request' && needs.check_repo.outputs.should_run_workflow == 'true'"
+    if: "failure() && github.event_name != 'push' && github.event_name != 'pull_request'"
     needs:
       # TODO: should mypy be included here? It feels more brittle than the others.
       - mypy

.github/workflows/release-artifacts.yml

@@ -34,7 +34,6 @@ jobs:
       - id: set-distros
         run: |
           # if we're running from a tag, get the full list of distros; otherwise just use debian:sid
-          # NOTE: inside the actual Dockerfile-dhvirtualenv, the image name is expanded into its full image path
           dists='["debian:sid"]'
           if [[ $GITHUB_REF == refs/tags/* ]]; then
               dists=$(scripts-dev/build_debian_packages.py --show-dists-json)
@@ -144,7 +143,7 @@ jobs:
 
       - name: Only build a single wheel on PR
         if: startsWith(github.ref, 'refs/pull/')
-        run: echo "CIBW_BUILD="cp38-manylinux_${{ matrix.arch }}"" >> $GITHUB_ENV
+        run: echo "CIBW_BUILD="cp37-manylinux_${{ matrix.arch }}"" >> $GITHUB_ENV
 
       - name: Build wheels
         run: python -m cibuildwheel --output-dir wheelhouse

.github/workflows/tests.yml

@@ -35,7 +35,7 @@ jobs:
     steps:
       - uses: actions/checkout@v3
       - name: Install Rust
-        uses: dtolnay/rust-toolchain@1.60.0
+        uses: dtolnay/rust-toolchain@1.58.1
       - uses: Swatinem/rust-cache@v2
       - uses: matrix-org/setup-python-poetry@v1
         with:
@@ -92,10 +92,6 @@ jobs:
       - name: Checkout repository
         uses: actions/checkout@v3
 
-      - name: Install Rust
-        uses: dtolnay/rust-toolchain@1.60.0
-      - uses: Swatinem/rust-cache@v2
-
       - name: Setup Poetry
         uses: matrix-org/setup-python-poetry@v1
         with:
@@ -107,6 +103,10 @@ jobs:
           # To make CI green, err towards caution and install the project.
           install-project: "true"
 
+      - name: Install Rust
+        uses: dtolnay/rust-toolchain@1.58.1
+      - uses: Swatinem/rust-cache@v2
+
       # Cribbed from
       # https://github.com/AustinScola/mypy-cache-github-action/blob/85ea4f2972abed39b33bd02c36e341b28ca59213/src/restore.ts#L10-L17
       - name: Restore/persist mypy's cache
@@ -150,7 +150,7 @@ jobs:
         with:
           ref: ${{ github.event.pull_request.head.sha }}
       - name: Install Rust
-        uses: dtolnay/rust-toolchain@1.60.0
+        uses: dtolnay/rust-toolchain@1.58.1
       - uses: Swatinem/rust-cache@v2
       - uses: matrix-org/setup-python-poetry@v1
         with:
@@ -167,7 +167,7 @@ jobs:
       - uses: actions/checkout@v3
 
       - name: Install Rust
-        uses: dtolnay/rust-toolchain@1.60.0
+        uses: dtolnay/rust-toolchain@1.58.1
         with:
             components: clippy
       - uses: Swatinem/rust-cache@v2
@@ -268,7 +268,7 @@ jobs:
             postgres:${{ matrix.job.postgres-version }}
 
       - name: Install Rust
-        uses: dtolnay/rust-toolchain@1.60.0
+        uses: dtolnay/rust-toolchain@1.58.1
       - uses: Swatinem/rust-cache@v2
 
       - uses: matrix-org/setup-python-poetry@v1
@@ -308,7 +308,7 @@ jobs:
       - uses: actions/checkout@v3
 
       - name: Install Rust
-        uses: dtolnay/rust-toolchain@1.60.0
+        uses: dtolnay/rust-toolchain@1.58.1
       - uses: Swatinem/rust-cache@v2
 
       # There aren't wheels for some of the older deps, so we need to install
@@ -320,7 +320,7 @@ jobs:
 
       - uses: actions/setup-python@v4
         with:
-          python-version: '3.8'
+          python-version: '3.7'
 
       - name: Prepare old deps
         if: steps.cache-poetry-old-deps.outputs.cache-hit != 'true'
@@ -362,7 +362,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: ["pypy-3.8"]
+        python-version: ["pypy-3.7"]
         extras: ["all"]
 
     steps:
@@ -399,8 +399,8 @@ jobs:
       env:
         SYTEST_BRANCH: ${{ github.head_ref }}
         POSTGRES: ${{ matrix.job.postgres && 1}}
-        MULTI_POSTGRES: ${{ (matrix.job.postgres == 'multi-postgres') || '' }}
-        ASYNCIO_REACTOR: ${{ (matrix.job.reactor == 'asyncio') || '' }}
+        MULTI_POSTGRES: ${{ (matrix.job.postgres == 'multi-postgres') && 1}}
+        ASYNCIO_REACTOR: ${{ (matrix.job.reactor == 'asyncio') && 1 }}
         WORKERS: ${{ matrix.job.workers && 1 }}
         BLACKLIST: ${{ matrix.job.workers && 'synapse-blacklist-with-workers' }}
         TOP: ${{ github.workspace }}
@@ -416,7 +416,7 @@ jobs:
         run: cat sytest-blacklist .ci/worker-blacklist > synapse-blacklist-with-workers
 
       - name: Install Rust
-        uses: dtolnay/rust-toolchain@1.60.0
+        uses: dtolnay/rust-toolchain@1.58.1
       - uses: Swatinem/rust-cache@v2
 
       - name: Run SyTest
@@ -477,7 +477,7 @@ jobs:
     strategy:
       matrix:
         include:
-          - python-version: "3.8"
+          - python-version: "3.7"
             postgres-version: "11"
 
           - python-version: "3.11"
@@ -556,7 +556,7 @@ jobs:
           path: synapse
 
       - name: Install Rust
-        uses: dtolnay/rust-toolchain@1.60.0
+        uses: dtolnay/rust-toolchain@1.58.1
       - uses: Swatinem/rust-cache@v2
 
       - uses: actions/setup-go@v4
@@ -584,7 +584,7 @@ jobs:
       - uses: actions/checkout@v3
 
       - name: Install Rust
-        uses: dtolnay/rust-toolchain@1.60.0
+        uses: dtolnay/rust-toolchain@1.58.1
       - uses: Swatinem/rust-cache@v2
 
       - run: cargo test

.github/workflows/twisted_trunk.yml

@@ -18,22 +18,7 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
-  check_repo:
-    # Prevent this workflow from running on any fork of Synapse other than matrix-org/synapse, as it is
-    # only useful to the Synapse core team.
-    # All other workflow steps depend on this one, thus if 'should_run_workflow' is not 'true', the rest
-    # of the workflow will be skipped as well.
-    if: github.repository == 'matrix-org/synapse'
-    runs-on: ubuntu-latest
-    outputs:
-      should_run_workflow: ${{ steps.check_condition.outputs.should_run_workflow }}
-    steps:
-      - id: check_condition
-        run: echo "should_run_workflow=${{ github.repository == 'matrix-org/synapse' }}" >> "$GITHUB_OUTPUT"
-
   mypy:
-    needs: check_repo
-    if: needs.check_repo.outputs.should_run_workflow == 'true'
     runs-on: ubuntu-latest
 
     steps:
@@ -56,8 +41,6 @@ jobs:
       - run: poetry run mypy
 
   trial:
-    needs: check_repo
-    if: needs.check_repo.outputs.should_run_workflow == 'true'
     runs-on: ubuntu-latest
 
     steps:
@@ -92,15 +75,9 @@ jobs:
           || true
 
   sytest:
-    needs: check_repo
-    if: needs.check_repo.outputs.should_run_workflow == 'true'
     runs-on: ubuntu-latest
     container:
-      # We're using ubuntu:focal because it uses Python 3.8 which is our minimum supported Python version.
-      # This job is a canary to warn us about unreleased twisted changes that would cause problems for us if
-      # they were to be released immediately. For simplicity's sake (and to save CI runners) we use the oldest
-      # version, assuming that any incompatibilities on newer versions would also be present on the oldest.
-      image: matrixdotorg/sytest-synapse:focal
+      image: matrixdotorg/sytest-synapse:buster
       volumes:
         - ${{ github.workspace }}:/src
 
@@ -142,8 +119,7 @@ jobs:
             /logs/**/*.log*
 
   complement:
-    needs: check_repo
-    if: "!failure() && !cancelled() && needs.check_repo.outputs.should_run_workflow == 'true'"
+    if: "${{ !failure() && !cancelled() }}"
     runs-on: ubuntu-latest
 
     strategy:
@@ -190,7 +166,7 @@ jobs:
 
   # open an issue if the build fails, so we know about it.
   open-issue:
-    if: failure() && needs.check_repo.outputs.should_run_workflow == 'true'
+    if: failure()
     needs:
       - mypy
       - trial

.gitignore

@@ -34,7 +34,6 @@ __pycache__/
 /logs
 /media_store/
 /uploads
-/homeserver-config-overrides.d
 
 # For direnv users
 /.envrc

Cargo.lock

@@ -4,9 +4,9 @@ version = 3
 
 [[package]]
 name = "aho-corasick"
-version = "1.0.2"
+version = "0.7.19"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "43f6cb1bf222025340178f382c426f13757b2960e89779dfcb319c32542a5a41"
+checksum = "b4f55bd91a0978cbfd91c457a164bab8b4001c833b7f323132c0a4e1922dd44e"
 dependencies = [
  "memchr",
 ]
@@ -132,9 +132,9 @@ dependencies = [
 
 [[package]]
 name = "log"
-version = "0.4.19"
+version = "0.4.18"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b06a4cde4c0f271a446782e3eff8de789548ce57dbc8eca9292c27f4a42004b4"
+checksum = "518ef76f2f87365916b142844c16d8fefd85039bc5699050210a7778ee1cd1de"
 
 [[package]]
 name = "memchr"
@@ -182,9 +182,9 @@ dependencies = [
 
 [[package]]
 name = "proc-macro2"
-version = "1.0.64"
+version = "1.0.52"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "78803b62cbf1f46fde80d7c0e803111524b9877184cfe7c3033659490ac7a7da"
+checksum = "1d0e1ae9e836cc3beddd63db0df682593d7e2d3d891ae8c9083d2113e1744224"
 dependencies = [
  "unicode-ident",
 ]
@@ -229,9 +229,9 @@ dependencies = [
 
 [[package]]
 name = "pyo3-log"
-version = "0.8.2"
+version = "0.8.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c94ff6535a6bae58d7d0b85e60d4c53f7f84d0d0aa35d6a28c3f3e70bfe51444"
+checksum = "f9c8b57fe71fb5dcf38970ebedc2b1531cf1c14b1b9b4c560a182a57e115575c"
 dependencies = [
  "arc-swap",
  "log",
@@ -273,9 +273,9 @@ dependencies = [
 
 [[package]]
 name = "quote"
-version = "1.0.29"
+version = "1.0.26"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "573015e8ab27661678357f27dc26460738fd2b6c86e46f386fde94cb5d913105"
+checksum = "4424af4bf778aae2051a77b60283332f386554255d722233d09fbfc7e30da2fc"
 dependencies = [
  "proc-macro2",
 ]
@@ -291,21 +291,9 @@ dependencies = [
 
 [[package]]
 name = "regex"
-version = "1.9.1"
+version = "1.7.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b2eae68fc220f7cf2532e4494aded17545fce192d59cd996e0fe7887f4ceb575"
-dependencies = [
- "aho-corasick",
- "memchr",
- "regex-automata",
- "regex-syntax",
-]
-
-[[package]]
-name = "regex-automata"
-version = "0.3.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "83d3daa6976cffb758ec878f108ba0e062a45b2d6ca3a2cca965338855476caf"
+checksum = "8b1f693b24f6ac912f4893ef08244d70b6067480d2f1a46e950c9691e6749d1d"
 dependencies = [
  "aho-corasick",
  "memchr",
@@ -314,9 +302,9 @@ dependencies = [
 
 [[package]]
 name = "regex-syntax"
-version = "0.7.3"
+version = "0.6.29"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2ab07dc67230e4a4718e70fd5c20055a4334b121f1f9db8fe63ef39ce9b8c846"
+checksum = "f162c6dd7b008981e4d40210aca20b4bd0f9b60ca9271061b07f78537722f2e1"
 
 [[package]]
 name = "ryu"
@@ -332,29 +320,29 @@ checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd"
 
 [[package]]
 name = "serde"
-version = "1.0.171"
+version = "1.0.163"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "30e27d1e4fd7659406c492fd6cfaf2066ba8773de45ca75e855590f856dc34a9"
+checksum = "2113ab51b87a539ae008b5c6c02dc020ffa39afd2d83cffcb3f4eb2722cebec2"
 dependencies = [
  "serde_derive",
 ]
 
 [[package]]
 name = "serde_derive"
-version = "1.0.171"
+version = "1.0.163"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "389894603bd18c46fa56231694f8d827779c0951a667087194cf9de94ed24682"
+checksum = "8c805777e3930c8883389c602315a24224bcc738b63905ef87cd1420353ea93e"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.25",
+ "syn 2.0.10",
 ]
 
 [[package]]
 name = "serde_json"
-version = "1.0.100"
+version = "1.0.96"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0f1e14e89be7aa4c4b78bdbdc9eb5bf8517829a600ae8eaa39a6e1d960b5185c"
+checksum = "057d394a50403bcac12672b2b18fb387ab6d289d957dab67dd201875391e52f1"
 dependencies = [
  "itoa",
  "ryu",
@@ -386,9 +374,9 @@ dependencies = [
 
 [[package]]
 name = "syn"
-version = "2.0.25"
+version = "2.0.10"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "15e3fc8c0c74267e2df136e5e5fb656a464158aa57624053375eb9c8c6e25ae2"
+checksum = "5aad1363ed6d37b84299588d62d3a7d95b5a5c2d9aad5c85609fda12afaa1f40"
 dependencies = [
  "proc-macro2",
  "quote",

Cargo.toml

@@ -3,4 +3,3 @@
 
 [workspace]
 members = ["rust"]
-resolver = "2"

book.toml

@@ -34,6 +34,14 @@ additional-css = [
     "docs/website_files/table-of-contents.css",
     "docs/website_files/remove-nav-buttons.css",
     "docs/website_files/indent-section-headers.css",
+    "docs/website_files/version-picker.css",
 ]
-additional-js = ["docs/website_files/table-of-contents.js"]
-theme = "docs/website_files/theme"
+additional-js = [
+    "docs/website_files/table-of-contents.js",
+    "docs/website_files/version-picker.js",
+    "docs/website_files/version.js",
+]
+theme = "docs/website_files/theme"
+
+[preprocessor.schema_versions]
+command = "./scripts-dev/schema_versions.py"

changelog.d/15708.feature

@@ -1 +0,0 @@
-Add Unix Socket support for HTTP Replication Listeners. Document and provide usage instructions for utilizing Unix sockets in Synapse. Contributed by Jason Little.

changelog.d/15820.bugfix

@@ -1 +0,0 @@
-Fix long-standing bug where remote invites weren't correctly pushed.

changelog.d/15884.misc

@@ -1 +0,0 @@
-Mark `get_user_in_directory` private since it is only used in tests. Also remove the cache from it.

changelog.d/15909.misc

@@ -1 +0,0 @@
-Document which Python version runs on a given Linux distribution so we can more easily clean up later.

changelog.d/15911.feature

@@ -1 +0,0 @@
-Allow `+` in Matrix IDs, per [MSC4009](https://github.com/matrix-org/matrix-spec-proposals/pull/4009).

changelog.d/15921.doc

@@ -1 +0,0 @@
-Better clarify how to run a worker instance (pass both configs).

changelog.d/15922.misc

@@ -1 +0,0 @@
-Add details to warning in log when we fail to fetch an alias.

changelog.d/15924.feature

@@ -1 +0,0 @@
-Add Unix Socket support for HTTP Replication Listeners. Document and provide usage instructions for utilizing Unix sockets in Synapse. Contributed by Jason Little.

changelog.d/15925.bugfix

@@ -1 +0,0 @@
-Fix a bug introduced in 1.86.0 where Synapse starting with an empty `experimental_features` configuration setting.

contrib/lnav/synapse-log-format.json

@@ -29,7 +29,7 @@
         "level": "error"
       },
       {
-        "line": "my-matrix-server-federation-sender-1 | 2023-01-25 20:56:20,995 - synapse.http.matrixfederationclient - 709 - WARNING - federation_transaction_transmission_loop-3 - {PUT-O-3} [example.com] Request failed: PUT matrix-federation://example.com/_matrix/federation/v1/send/1674680155797: HttpResponseException('403: Forbidden')",
+        "line": "my-matrix-server-federation-sender-1 | 2023-01-25 20:56:20,995 - synapse.http.matrixfederationclient - 709 - WARNING - federation_transaction_transmission_loop-3 - {PUT-O-3} [example.com] Request failed: PUT matrix://example.com/_matrix/federation/v1/send/1674680155797: HttpResponseException('403: Forbidden')",
         "level": "warning"
       },
       {

debian/changelog

@@ -1,39 +1,3 @@
-matrix-synapse-py3 (1.88.0~rc1) stable; urgency=medium
-
-  * New Synapse release 1.88.0rc1.
-
- -- Synapse Packaging team <packages@matrix.org>  Tue, 11 Jul 2023 10:20:19 +0100
-
-matrix-synapse-py3 (1.87.0) stable; urgency=medium
-
-  * New Synapse release 1.87.0.
-
- -- Synapse Packaging team <packages@matrix.org>  Tue, 04 Jul 2023 16:24:00 +0100
-
-matrix-synapse-py3 (1.87.0~rc1) stable; urgency=medium
-
-  * New synapse release 1.87.0rc1.
-
- -- Synapse Packaging team <packages@matrix.org>  Tue, 27 Jun 2023 15:27:04 +0000
-
-matrix-synapse-py3 (1.86.0) stable; urgency=medium
-
-  * New Synapse release 1.86.0.
-
- -- Synapse Packaging team <packages@matrix.org>  Tue, 20 Jun 2023 17:22:46 +0200
-
-matrix-synapse-py3 (1.86.0~rc2) stable; urgency=medium
-
-  * New Synapse release 1.86.0rc2.
-
- -- Synapse Packaging team <packages@matrix.org>  Wed, 14 Jun 2023 12:16:27 +0200
-
-matrix-synapse-py3 (1.86.0~rc1) stable; urgency=medium
-
-  * New Synapse release 1.86.0rc1.
-
- -- Synapse Packaging team <packages@matrix.org>  Tue, 13 Jun 2023 14:30:45 +0200
-
 matrix-synapse-py3 (1.85.2) stable; urgency=medium
 
   * New Synapse release 1.85.2.

docker/Dockerfile

@@ -27,7 +27,7 @@ ARG PYTHON_VERSION=3.11
 ###
 # We hardcode the use of Debian bullseye here because this could change upstream
 # and other Dockerfiles used for testing are expecting bullseye.
-FROM docker.io/library/python:${PYTHON_VERSION}-slim-bullseye as requirements
+FROM docker.io/python:${PYTHON_VERSION}-slim-bullseye as requirements
 
 # RUN --mount is specific to buildkit and is documented at
 # https://github.com/moby/buildkit/blob/master/frontend/dockerfile/docs/syntax.md#build-mounts-run---mount.
@@ -87,7 +87,7 @@ RUN if [ -z "$TEST_ONLY_IGNORE_POETRY_LOCKFILE" ]; then \
 ###
 ### Stage 1: builder
 ###
-FROM docker.io/library/python:${PYTHON_VERSION}-slim-bullseye as builder
+FROM docker.io/python:${PYTHON_VERSION}-slim-bullseye as builder
 
 # install the OS build deps
 RUN \
@@ -158,7 +158,7 @@ RUN --mount=type=cache,target=/synapse/target,sharing=locked \
 ### Stage 2: runtime
 ###
 
-FROM docker.io/library/python:${PYTHON_VERSION}-slim-bullseye
+FROM docker.io/python:${PYTHON_VERSION}-slim-bullseye
 
 LABEL org.opencontainers.image.url='https://matrix.org/docs/projects/server/synapse'
 LABEL org.opencontainers.image.documentation='https://github.com/matrix-org/synapse/blob/master/docker/README.md'

docker/Dockerfile-dhvirtualenv

@@ -24,16 +24,16 @@ ARG distro=""
 # https://launchpad.net/~jyrki-pulliainen/+archive/ubuntu/dh-virtualenv, but
 # it's not obviously easier to use that than to build our own.)
 
-FROM docker.io/library/${distro} as builder
+FROM ${distro} as builder
 
 RUN apt-get update -qq -o Acquire::Languages=none
 RUN env DEBIAN_FRONTEND=noninteractive apt-get install \
-    -yqq --no-install-recommends \
-    build-essential \
-    ca-certificates \
-    devscripts \
-    equivs \
-    wget
+        -yqq --no-install-recommends \
+        build-essential \
+        ca-certificates \
+        devscripts \
+        equivs \
+        wget
 
 # fetch and unpack the package
 # We are temporarily using a fork of dh-virtualenv due to an incompatibility with Python 3.11, which ships with
@@ -55,36 +55,40 @@ RUN cd /dh-virtualenv && DEB_BUILD_OPTIONS=nodoc dpkg-buildpackage -us -uc -b
 ###
 ### Stage 1
 ###
-FROM docker.io/library/${distro}
+FROM ${distro}
 
 # Get the distro we want to pull from as a dynamic build variable
 # (We need to define it in each build stage)
 ARG distro=""
 ENV distro ${distro}
 
+# Python < 3.7 assumes LANG="C" means ASCII-only and throws on printing unicode
+# http://bugs.python.org/issue19846
+ENV LANG C.UTF-8
+
 # Install the build dependencies
 #
 # NB: keep this list in sync with the list of build-deps in debian/control
 # TODO: it would be nice to do that automatically.
 RUN apt-get update -qq -o Acquire::Languages=none \
     && env DEBIAN_FRONTEND=noninteractive apt-get install \
-    -yqq --no-install-recommends -o Dpkg::Options::=--force-unsafe-io \
-    build-essential \
-    curl \
-    debhelper \
-    devscripts \
-    libsystemd-dev \
-    lsb-release \
-    pkg-config \
-    python3-dev \
-    python3-pip \
-    python3-setuptools \
-    python3-venv \
-    sqlite3 \
-    libpq-dev \
-    libicu-dev \
-    pkg-config \
-    xmlsec1
+        -yqq --no-install-recommends -o Dpkg::Options::=--force-unsafe-io \
+        build-essential \
+        curl \
+        debhelper \
+        devscripts \
+        libsystemd-dev \
+        lsb-release \
+        pkg-config \
+        python3-dev \
+        python3-pip \
+        python3-setuptools \
+        python3-venv \
+        sqlite3 \
+        libpq-dev \
+        libicu-dev \
+        pkg-config \
+        xmlsec1
 
 # Install rust and ensure it's in the PATH
 ENV RUSTUP_HOME=/rust

docker/Dockerfile-workers

@@ -7,7 +7,7 @@ ARG FROM=matrixdotorg/synapse:$SYNAPSE_VERSION
 # target image. For repeated rebuilds, this is much faster than apt installing
 # each time.
 
-FROM docker.io/library/debian:bullseye-slim AS deps_base
+FROM debian:bullseye-slim AS deps_base
     RUN \
        --mount=type=cache,target=/var/cache/apt,sharing=locked \
        --mount=type=cache,target=/var/lib/apt,sharing=locked \
@@ -21,7 +21,7 @@ FROM docker.io/library/debian:bullseye-slim AS deps_base
 # which makes it much easier to copy (but we need to make sure we use an image
 # based on the same debian version as the synapse image, to make sure we get
 # the expected version of libc.
-FROM docker.io/library/redis:7-bullseye AS redis_base
+FROM redis:6-bullseye AS redis_base
 
 # now build the final image, based on the the regular Synapse docker image
 FROM $FROM

docker/README.md

@@ -73,8 +73,7 @@ The following environment variables are supported in `generate` mode:
   will log sensitive information such as access tokens.
   This should not be needed unless you are a developer attempting to debug something
   particularly tricky.
-* `SYNAPSE_LOG_TESTING`: if set, Synapse will log additional information useful
-  for testing.
+
 
 ## Postgres
 

docker/complement/Dockerfile

@@ -7,7 +7,6 @@
 # https://github.com/matrix-org/synapse/blob/develop/docker/README-testing.md#testing-with-postgresql-and-single-or-multi-process-synapse
 
 ARG SYNAPSE_VERSION=latest
-# This is an intermediate image, to be built locally (not pulled from a registry).
 ARG FROM=matrixdotorg/synapse-workers:$SYNAPSE_VERSION
 
 FROM $FROM
@@ -20,8 +19,8 @@ FROM $FROM
     # the same debian version as Synapse's docker image (so the versions of the
     # shared libraries match).
     RUN adduser --system --uid 999 postgres --home /var/lib/postgresql
-    COPY --from=docker.io/library/postgres:13-bullseye /usr/lib/postgresql /usr/lib/postgresql
-    COPY --from=docker.io/library/postgres:13-bullseye /usr/share/postgresql /usr/share/postgresql
+    COPY --from=postgres:13-bullseye /usr/lib/postgresql /usr/lib/postgresql
+    COPY --from=postgres:13-bullseye /usr/share/postgresql /usr/share/postgresql
     RUN mkdir /var/run/postgresql && chown postgres /var/run/postgresql
     ENV PATH="${PATH}:/usr/lib/postgresql/13/bin"
     ENV PGDATA=/var/lib/postgresql/data

docker/complement/conf/workers-shared-extra.yaml.j2

@@ -92,6 +92,8 @@ allow_device_name_lookup_over_federation: true
 ## Experimental Features ##
 
 experimental_features:
+  # Enable history backfilling support
+  msc2716_enabled: true
   # client-side support for partial state in /send_join responses
   faster_joins: true
   # Enable support for polls

docker/conf-workers/nginx.conf.j2

@@ -35,11 +35,7 @@ server {
 
     # Send all other traffic to the main process
     location ~* ^(\\/_matrix|\\/_synapse) {
-{% if using_unix_sockets %}
-        proxy_pass http://unix:/run/main_public.sock;
-{% else %}
         proxy_pass http://localhost:8080;
-{% endif %}
         proxy_set_header X-Forwarded-For $remote_addr;
         proxy_set_header X-Forwarded-Proto $scheme;
         proxy_set_header Host $host;

docker/conf-workers/shared.yaml.j2

@@ -6,9 +6,6 @@
 {% if enable_redis %}
 redis:
     enabled: true
-    {% if using_unix_sockets %}
-    path: /tmp/redis.sock
-    {% endif %}
 {% endif %}
 
 {% if appservice_registrations is not none %}

docker/conf-workers/supervisord.conf.j2

@@ -19,11 +19,7 @@ username=www-data
 autorestart=true
 
 [program:redis]
-{% if using_unix_sockets %}
-command=/usr/local/bin/prefix-log /usr/local/bin/redis-server --unixsocket /tmp/redis.sock
-{% else %}
 command=/usr/local/bin/prefix-log /usr/local/bin/redis-server
-{% endif %}
 priority=1
 stdout_logfile=/dev/stdout
 stdout_logfile_maxbytes=0

docker/conf-workers/worker.yaml.j2

@@ -8,11 +8,7 @@ worker_name: "{{ name }}"
 
 worker_listeners:
   - type: http
-{% if using_unix_sockets %}
-    path: "/run/worker.{{ port }}"
-{% else %}
     port: {{ port }}
-{% endif %}
 {% if listener_resources %}
     resources:
       - names:

docker/conf/homeserver.yaml

@@ -36,17 +36,12 @@ listeners:
 
   # Allow configuring in case we want to reverse proxy 8008
   # using another process in the same container
-{% if SYNAPSE_USE_UNIX_SOCKET %}
-  # Unix sockets don't care about TLS or IP addresses or ports
-  - path: '/run/main_public.sock'
-    type: http
-{% else %}
   - port: {{ SYNAPSE_HTTP_PORT or 8008 }}
     tls: false
     bind_addresses: ['::']
     type: http
     x_forwarded: false
-{% endif %}
+
     resources:
       - names: [client]
         compress: true
@@ -62,11 +57,8 @@ database:
     user: "{{ POSTGRES_USER or "synapse" }}"
     password: "{{ POSTGRES_PASSWORD }}"
     database: "{{ POSTGRES_DB or "synapse" }}"
-{% if not SYNAPSE_USE_UNIX_SOCKET %}
-{# Synapse will use a default unix socket for Postgres when host/port is not specified (behavior from `psycopg2`). #}
     host: "{{ POSTGRES_HOST or "db" }}"
     port: "{{ POSTGRES_PORT or "5432" }}"
-{% endif %}
     cp_min: 5
     cp_max: 10
 {% else %}

docker/conf/log.config

@@ -49,35 +49,17 @@ handlers:
     class: logging.StreamHandler
     formatter: precise
 
+{% if not SYNAPSE_LOG_SENSITIVE %}
+{#
+  If SYNAPSE_LOG_SENSITIVE is unset, then override synapse.storage.SQL to INFO
+  so that DEBUG entries (containing sensitive information) are not emitted.
+#}
 loggers:
-    # This is just here so we can leave `loggers` in the config regardless of whether
-    # we configure other loggers below (avoid empty yaml dict error).
-    _placeholder:
-        level: "INFO"
-
-    {% if not SYNAPSE_LOG_SENSITIVE %}
-    {#
-      If SYNAPSE_LOG_SENSITIVE is unset, then override synapse.storage.SQL to INFO
-      so that DEBUG entries (containing sensitive information) are not emitted.
-    #}
     synapse.storage.SQL:
         # beware: increasing this to DEBUG will make synapse log sensitive
         # information such as access tokens.
         level: INFO
-    {% endif %}
-
-    {% if SYNAPSE_LOG_TESTING %}
-    {#
-      If Synapse is under test, log a few more useful things for a developer
-      attempting to debug something particularly tricky.
-
-      With `synapse.visibility.filtered_event_debug`, it logs when events are (maybe
-      unexpectedly) filtered out of responses in tests. It's just nice to be able to
-      look at the CI log and figure out why an event isn't being returned.
-    #}
-    synapse.visibility.filtered_event_debug:
-        level: DEBUG
-    {% endif %}
+{% endif %}
 
 root:
     level: {{ SYNAPSE_LOG_LEVEL or "INFO" }}

docker/configure_workers_and_start.py

@@ -40,8 +40,6 @@
 #         log level. INFO is the default.
 #   * SYNAPSE_LOG_SENSITIVE: If unset, SQL and SQL values won't be logged,
 #         regardless of the SYNAPSE_LOG_LEVEL setting.
-#   * SYNAPSE_LOG_TESTING: if set, Synapse will log additional information useful
-#     for testing.
 #
 # NOTE: According to Complement's ENTRYPOINT expectations for a homeserver image (as defined
 # in the project's README), this script may be run multiple times, and functionality should
@@ -74,9 +72,6 @@ MAIN_PROCESS_HTTP_LISTENER_PORT = 8080
 MAIN_PROCESS_INSTANCE_NAME = "main"
 MAIN_PROCESS_LOCALHOST_ADDRESS = "127.0.0.1"
 MAIN_PROCESS_REPLICATION_PORT = 9093
-# Obviously, these would only be used with the UNIX socket option
-MAIN_PROCESS_UNIX_SOCKET_PUBLIC_PATH = "/run/main_public.sock"
-MAIN_PROCESS_UNIX_SOCKET_PRIVATE_PATH = "/run/main_private.sock"
 
 # A simple name used as a placeholder in the WORKERS_CONFIG below. This will be replaced
 # during processing with the name of the worker.
@@ -247,6 +242,7 @@ WORKERS_CONFIG: Dict[str, Dict[str, Any]] = {
             "^/_matrix/client/(api/v1|r0|v3|unstable)/join/",
             "^/_matrix/client/(api/v1|r0|v3|unstable)/knock/",
             "^/_matrix/client/(api/v1|r0|v3|unstable)/profile/",
+            "^/_matrix/client/(v1|unstable/org.matrix.msc2716)/rooms/.*/batch_send",
         ],
         "shared_extra_conf": {},
         "worker_extra_conf": "",
@@ -410,15 +406,11 @@ def add_worker_roles_to_shared_config(
         )
 
         # Map of stream writer instance names to host/ports combos
-        if os.environ.get("SYNAPSE_USE_UNIX_SOCKET", False):
-            instance_map[worker_name] = {
-                "path": f"/run/worker.{worker_port}",
-            }
-        else:
-            instance_map[worker_name] = {
-                "host": "localhost",
-                "port": worker_port,
-            }
+        instance_map[worker_name] = {
+            "host": "localhost",
+            "port": worker_port,
+        }
+
     # Update the list of stream writers. It's convenient that the name of the worker
     # type is the same as the stream to write. Iterate over the whole list in case there
     # is more than one.
@@ -430,15 +422,10 @@ def add_worker_roles_to_shared_config(
 
             # Map of stream writer instance names to host/ports combos
             # For now, all stream writers need http replication ports
-            if os.environ.get("SYNAPSE_USE_UNIX_SOCKET", False):
-                instance_map[worker_name] = {
-                    "path": f"/run/worker.{worker_port}",
-                }
-            else:
-                instance_map[worker_name] = {
-                    "host": "localhost",
-                    "port": worker_port,
-                }
+            instance_map[worker_name] = {
+                "host": "localhost",
+                "port": worker_port,
+            }
 
 
 def merge_worker_template_configs(
@@ -730,29 +717,17 @@ def generate_worker_files(
     # Note that yaml cares about indentation, so care should be taken to insert lines
     # into files at the correct indentation below.
 
-    # Convenience helper for if using unix sockets instead of host:port
-    using_unix_sockets = environ.get("SYNAPSE_USE_UNIX_SOCKET", False)
     # First read the original config file and extract the listeners block. Then we'll
     # add another listener for replication. Later we'll write out the result to the
     # shared config file.
-    listeners: List[Any]
-    if using_unix_sockets:
-        listeners = [
-            {
-                "path": MAIN_PROCESS_UNIX_SOCKET_PRIVATE_PATH,
-                "type": "http",
-                "resources": [{"names": ["replication"]}],
-            }
-        ]
-    else:
-        listeners = [
-            {
-                "port": MAIN_PROCESS_REPLICATION_PORT,
-                "bind_address": MAIN_PROCESS_LOCALHOST_ADDRESS,
-                "type": "http",
-                "resources": [{"names": ["replication"]}],
-            }
-        ]
+    listeners = [
+        {
+            "port": MAIN_PROCESS_REPLICATION_PORT,
+            "bind_address": MAIN_PROCESS_LOCALHOST_ADDRESS,
+            "type": "http",
+            "resources": [{"names": ["replication"]}],
+        }
+    ]
     with open(config_path) as file_stream:
         original_config = yaml.safe_load(file_stream)
         original_listeners = original_config.get("listeners")
@@ -793,17 +768,7 @@ def generate_worker_files(
 
     # A list of internal endpoints to healthcheck, starting with the main process
     # which exists even if no workers do.
-    # This list ends up being part of the command line to curl, (curl added support for
-    # Unix sockets in version 7.40).
-    if using_unix_sockets:
-        healthcheck_urls = [
-            f"--unix-socket {MAIN_PROCESS_UNIX_SOCKET_PUBLIC_PATH} "
-            # The scheme and hostname from the following URL are ignored.
-            # The only thing that matters is the path `/health`
-            "http://localhost/health"
-        ]
-    else:
-        healthcheck_urls = ["http://localhost:8080/health"]
+    healthcheck_urls = ["http://localhost:8080/health"]
 
     # Get the set of all worker types that we have configured
     all_worker_types_in_use = set(chain(*requested_worker_types.values()))
@@ -840,12 +805,8 @@ def generate_worker_files(
         # given worker_type needs to stay assigned and not be replaced.
         worker_config["shared_extra_conf"].update(shared_config)
         shared_config = worker_config["shared_extra_conf"]
-        if using_unix_sockets:
-            healthcheck_urls.append(
-                f"--unix-socket /run/worker.{worker_port} http://localhost/health"
-            )
-        else:
-            healthcheck_urls.append("http://localhost:%d/health" % (worker_port,))
+
+        healthcheck_urls.append("http://localhost:%d/health" % (worker_port,))
 
         # Update the shared config with sharding-related options if necessary
         add_worker_roles_to_shared_config(
@@ -864,7 +825,6 @@ def generate_worker_files(
             "/conf/workers/{name}.yaml".format(name=worker_name),
             **worker_config,
             worker_log_config_filepath=log_config_filepath,
-            using_unix_sockets=using_unix_sockets,
         )
 
         # Save this worker's port number to the correct nginx upstreams
@@ -885,13 +845,8 @@ def generate_worker_files(
     nginx_upstream_config = ""
     for upstream_worker_base_name, upstream_worker_ports in nginx_upstreams.items():
         body = ""
-        if using_unix_sockets:
-            for port in upstream_worker_ports:
-                body += f"    server unix:/run/worker.{port};\n"
-
-        else:
-            for port in upstream_worker_ports:
-                body += f"    server localhost:{port};\n"
+        for port in upstream_worker_ports:
+            body += f"    server localhost:{port};\n"
 
         # Add to the list of configured upstreams
         nginx_upstream_config += NGINX_UPSTREAM_CONFIG_BLOCK.format(
@@ -921,15 +876,10 @@ def generate_worker_files(
     # If there are workers, add the main process to the instance_map too.
     if workers_in_use:
         instance_map = shared_config.setdefault("instance_map", {})
-        if using_unix_sockets:
-            instance_map[MAIN_PROCESS_INSTANCE_NAME] = {
-                "path": MAIN_PROCESS_UNIX_SOCKET_PRIVATE_PATH,
-            }
-        else:
-            instance_map[MAIN_PROCESS_INSTANCE_NAME] = {
-                "host": MAIN_PROCESS_LOCALHOST_ADDRESS,
-                "port": MAIN_PROCESS_REPLICATION_PORT,
-            }
+        instance_map[MAIN_PROCESS_INSTANCE_NAME] = {
+            "host": MAIN_PROCESS_LOCALHOST_ADDRESS,
+            "port": MAIN_PROCESS_REPLICATION_PORT,
+        }
 
     # Shared homeserver config
     convert(
@@ -939,7 +889,6 @@ def generate_worker_files(
         appservice_registrations=appservice_registrations,
         enable_redis=workers_in_use,
         workers_in_use=workers_in_use,
-        using_unix_sockets=using_unix_sockets,
     )
 
     # Nginx config
@@ -950,7 +899,6 @@ def generate_worker_files(
         upstream_directives=nginx_upstream_config,
         tls_cert_path=os.environ.get("SYNAPSE_TLS_CERT"),
         tls_key_path=os.environ.get("SYNAPSE_TLS_KEY"),
-        using_unix_sockets=using_unix_sockets,
     )
 
     # Supervisord config
@@ -960,7 +908,6 @@ def generate_worker_files(
         "/etc/supervisor/supervisord.conf",
         main_config_path=config_path,
         enable_redis=workers_in_use,
-        using_unix_sockets=using_unix_sockets,
     )
 
     convert(
@@ -1000,7 +947,6 @@ def generate_worker_log_config(
     extra_log_template_args["SYNAPSE_LOG_SENSITIVE"] = environ.get(
         "SYNAPSE_LOG_SENSITIVE"
     )
-    extra_log_template_args["SYNAPSE_LOG_TESTING"] = environ.get("SYNAPSE_LOG_TESTING")
 
     # Render and write the file
     log_config_filepath = f"/conf/workers/{worker_name}.log.config"

docker/editable.Dockerfile

@@ -10,7 +10,7 @@ ARG PYTHON_VERSION=3.9
 ###
 # We hardcode the use of Debian bullseye here because this could change upstream
 # and other Dockerfiles used for testing are expecting bullseye.
-FROM docker.io/library/python:${PYTHON_VERSION}-slim-bullseye
+FROM docker.io/python:${PYTHON_VERSION}-slim-bullseye
 
 # Install Rust and other dependencies (stolen from normal Dockerfile)
 # install the OS build deps

docs/admin_api/rooms.md

@@ -419,7 +419,7 @@ The following query parameters are available:
 
 * `from` (required) - The token to start returning events from. This token can be obtained from a prev_batch
   or next_batch token returned by the /sync endpoint, or from an end token returned by a previous request to this endpoint.
-* `to` - The token to stop returning events at.
+* `to` - The token to spot returning events at.
 * `limit` - The maximum number of events to return. Defaults to `10`.
 * `filter` - A JSON RoomEventFilter to filter returned events with.
 * `dir` - The direction to return events from. Either `f` for forwards or `b` for backwards. Setting

docs/admin_api/user_admin_api.md

@@ -242,9 +242,6 @@ The following parameters should be set in the URL:
 
 - `dir` - Direction of media order. Either `f` for forwards or `b` for backwards.
   Setting this value to `b` will reverse the above sort order. Defaults to `f`.
-- `not_user_type` - Exclude certain user types, such as bot users, from the request.
-   Can be provided multiple times. Possible values are `bot`, `support` or "empty string".
-   "empty string" here means to exclude users without a type.
 
 Caution. The database only has indexes on the columns `name` and `creation_ts`.
 This means that if a different sort order is used (`is_guest`, `admin`,
@@ -1183,7 +1180,7 @@ The following parameters should be set in the URL:
 - `user_id` - The fully qualified MXID: for example, `@user:server.com`. The user must
   be local.
 
-## Check username availability
+### Check username availability
 
 Checks to see if a username is available, and valid, for the server. See [the client-server 
 API](https://matrix.org/docs/spec/client_server/r0.6.0#get-matrix-client-r0-register-available)
@@ -1201,7 +1198,7 @@ GET /_synapse/admin/v1/username_available?username=$localpart
 The request and response format is the same as the
 [/_matrix/client/r0/register/available](https://matrix.org/docs/spec/client_server/r0.6.0#get-matrix-client-r0-register-available) API.
 
-## Find a user based on their ID in an auth provider
+### Find a user based on their ID in an auth provider
 
 The API is:
 
@@ -1240,7 +1237,7 @@ Returns a `404` HTTP status code if no user was found, with a response body like
 _Added in Synapse 1.68.0._
 
 
-## Find a user based on their Third Party ID (ThreePID or 3PID)
+### Find a user based on their Third Party ID (ThreePID or 3PID)
 
 The API is:
 

docs/deprecation_policy.md

@@ -23,7 +23,7 @@ people building from source should ensure they can fetch recent versions of Rust
 (e.g. by using [rustup](https://rustup.rs/)).
 
 The oldest supported version of SQLite is the version
-[provided](https://packages.debian.org/bullseye/libsqlite3-0) by
+[provided](https://packages.debian.org/buster/libsqlite3-0) by
 [Debian oldstable](https://wiki.debian.org/DebianOldStable).
 
 Context

docs/development/contributing_guide.md

@@ -322,7 +322,7 @@ The following command will let you run the integration test with the most common
 configuration:
 
 ```sh
-$ docker run --rm -it -v /path/where/you/have/cloned/the/repository\:/src:ro -v /path/to/where/you/want/logs\:/logs matrixdotorg/sytest-synapse:focal
+$ docker run --rm -it -v /path/where/you/have/cloned/the/repository\:/src:ro -v /path/to/where/you/want/logs\:/logs matrixdotorg/sytest-synapse:buster
 ```
 (Note that the paths must be full paths! You could also write `$(realpath relative/path)` if needed.)
 
@@ -370,7 +370,6 @@ The above will run a monolithic (single-process) Synapse with SQLite as the data
     See the [worker documentation](../workers.md) for additional information on workers.
 - Passing `ASYNCIO_REACTOR=1` as an environment variable to use the Twisted asyncio reactor instead of the default one.
 - Passing `PODMAN=1` will use the [podman](https://podman.io/) container runtime, instead of docker.
-- Passing `UNIX_SOCKETS=1` will utilise Unix socket functionality for Synapse, Redis, and Postgres(when applicable).
 
 To increase the log level for the tests, set `SYNAPSE_TEST_LOG_LEVEL`, e.g:
 ```sh

docs/development/synapse_architecture/faster_joins.md

@@ -6,7 +6,7 @@ This is a work-in-progress set of notes with two goals:
 
 See also [MSC3902](https://github.com/matrix-org/matrix-spec-proposals/pull/3902).
 
-The key idea is described by [MSC3706](https://github.com/matrix-org/matrix-spec-proposals/pull/3706). This allows servers to
+The key idea is described by [MSC706](https://github.com/matrix-org/matrix-spec-proposals/pull/3902). This allows servers to
 request a lightweight response to the federation `/send_join` endpoint.
 This is called a **faster join**, also known as a **partial join**. In these
 notes we'll usually use the word "partial" as it matches the database schema.

docs/modules/spam_checker_callbacks.md

@@ -348,42 +348,6 @@ callback returns `False`, Synapse falls through to the next one. The value of th
 callback that does not return `False` will be used. If this happens, Synapse will not call
 any of the subsequent implementations of this callback.
 
-
-### `check_login_for_spam`
-
-_First introduced in Synapse v1.87.0_
-
-```python
-async def check_login_for_spam(
-    user_id: str,
-    device_id: Optional[str],
-    initial_display_name: Optional[str],
-    request_info: Collection[Tuple[Optional[str], str]],
-    auth_provider_id: Optional[str] = None,
-) -> Union["synapse.module_api.NOT_SPAM", "synapse.module_api.errors.Codes"]
-```
-
-Called when a user logs in.
-
-The arguments passed to this callback are:
-
-* `user_id`: The user ID the user is logging in with
-* `device_id`: The device ID the user is re-logging into.
-* `initial_display_name`: The device display name, if any.
-* `request_info`: A collection of tuples, which first item is a user agent, and which
-  second item is an IP address. These user agents and IP addresses are the ones that were
-  used during the login process.
-* `auth_provider_id`: The identifier of the SSO authentication provider, if any.
-
-If multiple modules implement this callback, they will be considered in order. If a
-callback returns `synapse.module_api.NOT_SPAM`, Synapse falls through to the next one.
-The value of the first callback that does not return `synapse.module_api.NOT_SPAM` will
-be used. If this happens, Synapse will not call any of the subsequent implementations of
-this callback.
-
-*Note:* This will not be called when a user registers.
-
-
 ## Example
 
 The example below is a module that implements the spam checker callback

docs/setup/installation.md

@@ -200,7 +200,7 @@ When following this route please make sure that the [Platform-specific prerequis
 System requirements:
 
 - POSIX-compliant system (tested on Linux & OS X)
-- Python 3.8 or later, up to Python 3.11.
+- Python 3.7 or later, up to Python 3.11.
 - At least 1GB of free RAM if you want to join large public rooms like #matrix:matrix.org
 
 If building on an uncommon architecture for which pre-built wheels are

docs/systemd-with-workers/workers/background_worker.yaml

@@ -1,4 +1,8 @@
 worker_app: synapse.app.generic_worker
 worker_name: background_worker
 
+# The replication listener on the main synapse process.
+worker_replication_host: 127.0.0.1
+worker_replication_http_port: 9093
+
 worker_log_config: /etc/matrix-synapse/background-worker-log.yaml

docs/systemd-with-workers/workers/event_persister.yaml

@@ -1,5 +1,9 @@
 worker_app: synapse.app.generic_worker
-worker_name: event_persister1
+worker_name: event_persister1 
+
+# The replication listener on the main synapse process.
+worker_replication_host: 127.0.0.1
+worker_replication_http_port: 9093
 
 worker_listeners:
   - type: http

docs/systemd-with-workers/workers/federation_sender.yaml

@@ -1,4 +1,8 @@
 worker_app: synapse.app.federation_sender
 worker_name: federation_sender1
 
+# The replication listener on the main synapse process.
+worker_replication_host: 127.0.0.1
+worker_replication_http_port: 9093
+
 worker_log_config: /etc/matrix-synapse/federation-sender-log.yaml

docs/systemd-with-workers/workers/media_worker.yaml

@@ -1,6 +1,10 @@
 worker_app: synapse.app.media_repository
 worker_name: media_worker
 
+# The replication listener on the main synapse process.
+worker_replication_host: 127.0.0.1
+worker_replication_http_port: 9093
+
 worker_listeners:
   - type: http
     port: 8085

docs/systemd-with-workers/workers/pusher_worker.yaml

@@ -1,4 +1,8 @@
 worker_app: synapse.app.pusher
 worker_name: pusher_worker1
 
+# The replication listener on the main synapse process.
+worker_replication_host: 127.0.0.1
+worker_replication_http_port: 9093
+
 worker_log_config: /etc/matrix-synapse/pusher-worker-log.yaml

docs/upgrade.md

@@ -88,41 +88,6 @@ process, for example:
     dpkg -i matrix-synapse-py3_1.3.0+stretch1_amd64.deb
     ```
 
-# Upgrading to v1.88.0
-
-## Minimum supported Python version
-
-The minimum supported Python version has been increased from v3.7 to v3.8.
-You will need Python 3.8 to run Synapse v1.88.0 (due out July 18th, 2023).
-
-If you use current versions of the Matrix.org-distributed Debian
-packages or Docker images, no action is required.
-
-## Removal of `worker_replication_*` settings
-
-As mentioned previously in [Upgrading to v1.84.0](#upgrading-to-v1840), the following deprecated settings
-are being removed in this release of Synapse:
-
-* [`worker_replication_host`](https://matrix-org.github.io/synapse/v1.86/usage/configuration/config_documentation.html#worker_replication_host)
-* [`worker_replication_http_port`](https://matrix-org.github.io/synapse/v1.86/usage/configuration/config_documentation.html#worker_replication_http_port)
-* [`worker_replication_http_tls`](https://matrix-org.github.io/synapse/v1.86/usage/configuration/config_documentation.html#worker_replication_http_tls)
-
-Please ensure that you have migrated to using `main` on your shared configuration's `instance_map`
-(or create one if necessary). This is required if you have ***any*** workers at all;
-administrators of single-process (monolith) installations don't need to do anything.
-
-For an illustrative example, please see [Upgrading to v1.84.0](#upgrading-to-v1840) below.
-
-
-# Upgrading to v1.86.0
-
-## Minimum supported Rust version
-
-The minimum supported Rust version has been increased from v1.58.1 to v1.60.0.
-Users building from source will need to ensure their `rustc` version is up to
-date.
-
-
 # Upgrading to v1.85.0
 
 ## Application service registration with "user" property deprecation

docs/usage/administration/admin_faq.md

@@ -27,8 +27,9 @@ What servers are currently participating in this room?
 Run this sql query on your db:
 ```sql
 SELECT DISTINCT split_part(state_key, ':', 2)
-FROM current_state_events
-WHERE room_id = '!cURbafjkfsMDVwdRDQ:matrix.org' AND membership = 'join';
+    FROM current_state_events AS c
+    INNER JOIN room_memberships AS m USING (room_id, event_id)
+    WHERE room_id = '!cURbafjkfsMDVwdRDQ:matrix.org' AND membership = 'join';
 ```
 
 What users are registered on my server?

docs/usage/configuration/config_documentation.md

@@ -462,20 +462,6 @@ See the docs [request log format](../administration/request_log.md).
 * `additional_resources`: Only valid for an 'http' listener. A map of
    additional endpoints which should be loaded via dynamic modules.
 
-Unix socket support (_Added in Synapse 1.89.0_):
-* `path`: A path and filename for a Unix socket. Make sure it is located in a
-  directory with read and write permissions, and that it already exists (the directory
-  will not be created). Defaults to `None`.
-  * **Note**: The use of both `path` and `port` options for the same `listener` is not
-    compatible.
-  * The `x_forwarded` option defaults to true  when using Unix sockets and can be omitted.
-  * Other options that would not make sense to use with a UNIX socket, such as 
-    `bind_addresses` and `tls` will be ignored and can be removed.
-* `mode`: The file permissions to set on the UNIX socket. Defaults to `666`
-* **Note:** Must be set as `type: http` (does not support `metrics` and `manhole`). 
-  Also make sure that `metrics` is not included in `resources` -> `names`
-
-
 Valid resource names are:
 
 * `client`: the client-server API (/_matrix/client), and the synapse admin API (/_synapse/admin). Also implies `media` and `static`.
@@ -488,7 +474,7 @@ Valid resource names are:
 
 * `media`: the media API (/_matrix/media).
 
-* `metrics`: the metrics interface. See [here](../../metrics-howto.md). (Not compatible with Unix sockets)
+* `metrics`: the metrics interface. See [here](../../metrics-howto.md).
 
 * `openid`: OpenID authentication. See [here](../../openid.md).
 
@@ -547,22 +533,6 @@ listeners:
     bind_addresses: ['::1', '127.0.0.1']
     type: manhole
 ```
-Example configuration #3:
-```yaml
-listeners:
-  # Unix socket listener: Ideal for Synapse deployments behind a reverse proxy, offering
-  # lightweight interprocess communication without TCP/IP overhead, avoid port
-  # conflicts, and providing enhanced security through system file permissions.
-  #
-  # Note that x_forwarded will default to true, when using a UNIX socket. Please see
-  # https://matrix-org.github.io/synapse/latest/reverse_proxy.html.
-  #
-  - path: /var/run/synapse/main_public.sock
-    type: http
-    resources:
-      - names: [client, federation]
-```
-
 ---
 ### `manhole_settings`
 
@@ -1226,32 +1196,6 @@ Example configuration:
 allow_device_name_lookup_over_federation: true
 ```
 ---
-### `federation`
-
-The federation section defines some sub-options related to federation.
-
-The following options are related to configuring timeout and retry logic for one request,
-independently of the others.
-Short retry algorithm is used when something or someone will wait for the request to have an
-answer, while long retry is used for requests that happen in the background,
-like sending a federation transaction.
-
-* `client_timeout`: timeout for the federation requests. Default to 60s.
-* `max_short_retry_delay`: maximum delay to be used for the short retry algo. Default to 2s.
-* `max_long_retry_delay`: maximum delay to be used for the short retry algo. Default to 60s.
-* `max_short_retries`: maximum number of retries for the short retry algo. Default to 3 attempts.
-* `max_long_retries`: maximum number of retries for the long retry algo. Default to 10 attempts.
-
-Example configuration:
-```yaml
-federation:
-  client_timeout: 180s
-  max_short_retry_delay: 7s
-  max_long_retry_delay: 100s
-  max_short_retries: 5
-  max_long_retries: 20
-```
----
 ## Caching
 
 Options related to caching.
@@ -2626,50 +2570,7 @@ Example configuration:
 ```yaml
 nonrefreshable_access_token_lifetime: 24h
 ```
----
-### `ui_auth`
 
-The amount of time to allow a user-interactive authentication session to be active.
-
-This defaults to 0, meaning the user is queried for their credentials
-before every action, but this can be overridden to allow a single
-validation to be re-used.  This weakens the protections afforded by
-the user-interactive authentication process, by allowing for multiple
-(and potentially different) operations to use the same validation session.
-
-This is ignored for potentially "dangerous" operations (including
-deactivating an account, modifying an account password, adding a 3PID,
-and minting additional login tokens).
-
-Use the `session_timeout` sub-option here to change the time allowed for credential validation.
-
-Example configuration:
-```yaml
-ui_auth:
-    session_timeout: "15s"
-```
----
-### `login_via_existing_session`
-
-Matrix supports the ability of an existing session to mint a login token for
-another client.
-
-Synapse disables this by default as it has security ramifications -- a malicious
-client could use the mechanism to spawn more than one session.
-
-The duration of time the generated token is valid for can be configured with the
-`token_timeout` sub-option.
-
-User-interactive authentication is required when this is enabled unless the
-`require_ui_auth` sub-option is set to `False`.
-
-Example configuration:
-```yaml
-login_via_existing_session:
-    enabled: true
-    require_ui_auth: false
-    token_timeout: "5m"
-```
 ---
 ## Metrics
 Config options related to metrics.
@@ -3514,6 +3415,28 @@ password_config:
       require_uppercase: true
 ```
 ---
+### `ui_auth`
+
+The amount of time to allow a user-interactive authentication session to be active.
+
+This defaults to 0, meaning the user is queried for their credentials
+before every action, but this can be overridden to allow a single
+validation to be re-used.  This weakens the protections afforded by
+the user-interactive authentication process, by allowing for multiple
+(and potentially different) operations to use the same validation session.
+
+This is ignored for potentially "dangerous" operations (including
+deactivating an account, modifying an account password, and
+adding a 3PID).
+
+Use the `session_timeout` sub-option here to change the time allowed for credential validation.
+
+Example configuration:
+```yaml
+ui_auth:
+    session_timeout: "15s"
+```
+---
 ## Push
 Configuration settings related to push notifications
 
@@ -3979,14 +3902,6 @@ instance_map:
     host: localhost
     port: 8034
 ```
-Example configuration(#2, for UNIX sockets):
-```yaml
-instance_map:
-  main:
-    path: /var/run/synapse/main_replication.sock
-  worker1:
-    path: /var/run/synapse/worker1_replication.sock
-```
 ---
 ### `stream_writers`
 
@@ -4128,6 +4043,51 @@ Example configuration:
 worker_name: generic_worker1
 ```
 ---
+### `worker_replication_host`
+*Deprecated as of version 1.84.0. Place `host` under `main` entry on the [`instance_map`](#instance_map) in your shared yaml configuration instead.*
+
+The HTTP replication endpoint that it should talk to on the main Synapse process.
+The main Synapse process defines this with a `replication` resource in
+[`listeners` option](#listeners).
+
+Example configuration:
+```yaml
+worker_replication_host: 127.0.0.1
+```
+---
+### `worker_replication_http_port`
+*Deprecated as of version 1.84.0. Place `port` under `main` entry on the [`instance_map`](#instance_map) in your shared yaml configuration instead.*
+
+The HTTP replication port that it should talk to on the main Synapse process.
+The main Synapse process defines this with a `replication` resource in
+[`listeners` option](#listeners).
+
+Example configuration:
+```yaml
+worker_replication_http_port: 9093
+```
+---
+### `worker_replication_http_tls`
+*Deprecated as of version 1.84.0. Place `tls` under `main` entry on the [`instance_map`](#instance_map) in your shared yaml configuration instead.*
+
+Whether TLS should be used for talking to the HTTP replication port on the main
+Synapse process.
+The main Synapse process defines this with the `tls` option on its [listener](#listeners) that
+has the `replication` resource enabled.
+
+**Please note:** by default, it is not safe to expose replication ports to the
+public Internet, even with TLS enabled.
+See [`worker_replication_secret`](#worker_replication_secret).
+
+Defaults to `false`.
+
+*Added in Synapse 1.72.0.*
+
+Example configuration:
+```yaml
+worker_replication_http_tls: true
+```
+---
 ### `worker_listeners`
 
 A worker can handle HTTP requests. To do so, a `worker_listeners` option
@@ -4146,18 +4106,6 @@ worker_listeners:
     resources:
       - names: [client, federation]
 ```
-Example configuration(#2, using UNIX sockets with a `replication` listener):
-```yaml
-worker_listeners:
-  - type: http
-    path: /var/run/synapse/worker_public.sock
-    resources:
-      - names: [client, federation]
-  - type: http
-    path: /var/run/synapse/worker_replication.sock
-    resources:
-      - names: [replication]
-```
 ---
 ### `worker_manhole`
 

docs/website_files/README.md

@@ -24,6 +24,11 @@ Finally, we also stylise the chapter titles in the left sidebar by indenting the
 slightly so that they are more visually distinguishable from the section headers
 (the bold titles). This is done through the `indent-section-headers.css` file.
 
+In addition to these modifications, we have added a version picker to the documentation.
+Users can switch between documentations for different versions of Synapse.
+This functionality was implemented through the `version-picker.js` and
+`version-picker.css` files.
+
 More information can be found in mdbook's official documentation for
 [injecting page JS/CSS](https://rust-lang.github.io/mdBook/format/config.html)
 and

docs/website_files/theme/index.hbs

@@ -131,6 +131,18 @@
                             <i class="fa fa-search"></i>
                         </button>
                         {{/if}}
+                        <div class="version-picker">
+                            <div class="dropdown">
+                                <div class="select">
+                                    <span></span>
+                                    <i class="fa fa-chevron-down"></i>
+                                </div>
+                                <input type="hidden" name="version">
+                                <ul class="dropdown-menu">
+                                    <!-- Versions will be added dynamically in version-picker.js -->
+                                </ul>
+                            </div>
+                        </div>      
                     </div>
 
                     <h1 class="menu-title">{{ book_title }}</h1>
@@ -309,4 +321,4 @@
         {{/if}}
 
     </body>
-</html>
+</html>

docs/website_files/version-picker.css

@@ -0,0 +1,78 @@
+.version-picker {
+    display: flex;
+    align-items: center;
+}
+
+.version-picker .dropdown {
+    width: 130px;
+    max-height: 29px;
+    margin-left: 10px;
+    display: inline-block;
+    border-radius: 4px;
+    border: 1px solid var(--theme-popup-border);
+    position: relative;
+    font-size: 13px;
+    color: var(--fg);
+    height: 100%;
+    text-align: left;
+}
+.version-picker .dropdown .select {
+    cursor: pointer;
+    display: block;
+    padding: 5px 2px 5px 15px;
+}
+.version-picker .dropdown .select > i {
+    font-size: 10px;
+    color: var(--fg);
+    cursor: pointer;
+    float: right;
+    line-height: 20px !important;
+}
+.version-picker .dropdown:hover {
+    border: 1px solid var(--theme-popup-border);
+}
+.version-picker .dropdown:active {
+    background-color: var(--theme-popup-bg);
+}
+.version-picker .dropdown.active:hover,
+.version-picker .dropdown.active {
+    border: 1px solid var(--theme-popup-border);
+    border-radius: 2px 2px 0 0;
+    background-color: var(--theme-popup-bg);
+}
+.version-picker .dropdown.active .select > i {
+    transform: rotate(-180deg);
+}
+.version-picker .dropdown .dropdown-menu {
+    position: absolute;
+    background-color: var(--theme-popup-bg);
+    width: 100%;
+    left: -1px;
+    right: 1px;
+    margin-top: 1px;
+    border: 1px solid var(--theme-popup-border);
+    border-radius: 0 0 4px 4px;
+    overflow: hidden;
+    display: none;
+    max-height: 300px;
+    overflow-y: auto;
+    z-index: 9;
+}
+.version-picker .dropdown .dropdown-menu li {
+    font-size: 12px;
+    padding: 6px 20px;
+    cursor: pointer;
+} 
+.version-picker .dropdown .dropdown-menu {
+    padding: 0;
+    list-style: none;
+}
+.version-picker .dropdown .dropdown-menu li:hover {
+    background-color: var(--theme-hover);
+}
+.version-picker .dropdown .dropdown-menu li.active::before {
+    display: inline-block;
+    content: "✓";
+    margin-inline-start: -14px;
+    width: 14px;
+}

docs/website_files/version-picker.js

@@ -0,0 +1,127 @@
+
+const dropdown = document.querySelector('.version-picker .dropdown');
+const dropdownMenu = dropdown.querySelector('.dropdown-menu');
+
+fetchVersions(dropdown, dropdownMenu).then(() => {
+    initializeVersionDropdown(dropdown, dropdownMenu);
+});
+
+/**
+ * Initialize the dropdown functionality for version selection.
+ * 
+ * @param {Element} dropdown - The dropdown element.
+ * @param {Element} dropdownMenu - The dropdown menu element.
+ */
+function initializeVersionDropdown(dropdown, dropdownMenu) {
+    // Toggle the dropdown menu on click
+    dropdown.addEventListener('click', function () {
+        this.setAttribute('tabindex', 1);
+        this.classList.toggle('active');
+        dropdownMenu.style.display = (dropdownMenu.style.display === 'block') ? 'none' : 'block';
+    });
+  
+    // Remove the 'active' class and hide the dropdown menu on focusout
+    dropdown.addEventListener('focusout', function () {
+        this.classList.remove('active');
+        dropdownMenu.style.display = 'none';
+    });
+  
+    // Handle item selection within the dropdown menu
+    const dropdownMenuItems = dropdownMenu.querySelectorAll('li');    
+    dropdownMenuItems.forEach(function (item) {
+        item.addEventListener('click', function () {
+            dropdownMenuItems.forEach(function (item) {
+                item.classList.remove('active');
+            });
+            this.classList.add('active');
+            dropdown.querySelector('span').textContent = this.textContent;
+            dropdown.querySelector('input').value = this.getAttribute('id');
+
+            window.location.href = changeVersion(window.location.href, this.textContent);
+        });
+    });
+};
+
+/**
+ * This function fetches the available versions from a GitHub repository
+ * and inserts them into the version picker.
+ * 
+ * @param {Element} dropdown - The dropdown element.
+ * @param {Element} dropdownMenu - The dropdown menu element.
+ * @returns {Promise<Array<string>>} A promise that resolves with an array of available versions.
+ */
+function fetchVersions(dropdown, dropdownMenu) {
+    return new Promise((resolve, reject) => {
+        window.addEventListener("load", () => {
+
+            fetch("https://api.github.com/repos/matrix-org/synapse/git/trees/gh-pages", {
+                cache: "force-cache",
+            }).then(res => 
+                res.json()
+            ).then(resObject => {
+                const excluded = ['dev-docs', 'v1.91.0', 'v1.80.0', 'v1.69.0'];
+                const tree = resObject.tree.filter(item => item.type === "tree" && !excluded.includes(item.path));
+                const versions = tree.map(item => item.path).sort(sortVersions);
+
+                // Create a list of <li> items for versions
+                versions.forEach((version) => {
+                    const li = document.createElement("li");
+                    li.textContent = version;
+                    li.id = version;
+    
+                    if (window.SYNAPSE_VERSION === version) {
+                        li.classList.add('active');
+                        dropdown.querySelector('span').textContent = version;
+                        dropdown.querySelector('input').value = version;
+                    }
+    
+                    dropdownMenu.appendChild(li);
+                });
+
+                resolve(versions);
+
+            }).catch(ex => {
+                console.error("Failed to fetch version data", ex);
+                reject(ex);
+            })
+        });
+    });
+}
+
+/**
+ * Custom sorting function to sort an array of version strings.
+ *
+ * @param {string} a - The first version string to compare.
+ * @param {string} b - The second version string to compare.
+ * @returns {number} - A negative number if a should come before b, a positive number if b should come before a, or 0 if they are equal.
+ */
+function sortVersions(a, b) {
+    // Put 'develop' and 'latest' at the top
+    if (a === 'develop' || a === 'latest') return -1;
+    if (b === 'develop' || b === 'latest') return 1;
+
+    const versionA = (a.match(/v\d+(\.\d+)+/) || [])[0];
+    const versionB = (b.match(/v\d+(\.\d+)+/) || [])[0];
+
+    return versionB.localeCompare(versionA);
+}
+
+/**
+ * Change the version in a URL path.
+ *
+ * @param {string} url - The original URL to be modified.
+ * @param {string} newVersion - The new version to replace the existing version in the URL.
+ * @returns {string} The updated URL with the new version.
+ */
+function changeVersion(url, newVersion) {
+    const parsedURL = new URL(url);
+    const pathSegments = parsedURL.pathname.split('/');
+  
+    // Modify the version
+    pathSegments[2] = newVersion;
+
+    // Reconstruct the URL
+    parsedURL.pathname = pathSegments.join('/');
+  
+    return parsedURL.href;
+}

docs/website_files/version.js

@@ -0,0 +1 @@
+window.SYNAPSE_VERSION = 'v1.85';

docs/workers.md

@@ -95,12 +95,9 @@ for the main process
 * Secondly, you need to enable
 [redis-based replication](usage/configuration/config_documentation.md#redis)
 * You will need to add an [`instance_map`](usage/configuration/config_documentation.md#instance_map) 
-with the `main` process defined, as well as the relevant connection information from
-it's HTTP `replication` listener (defined in step 1 above).
-  * Note that the `host` defined is the address the worker needs to look for the `main`
-  process at, not necessarily the same address that is bound to.
-  * If you are using Unix sockets for the `replication` resource, make sure to
-  use a `path` to the socket file instead of a `port`.
+with the `main` process defined, as well as the relevant connection information from 
+it's HTTP `replication` listener (defined in step 1 above). Note that the `host` defined 
+is the address the worker needs to look for the `main` process at, not necessarily the same address that is bound to.
 * Optionally, a [shared secret](usage/configuration/config_documentation.md#worker_replication_secret)
 can be used to authenticate HTTP traffic between workers. For example:
 
@@ -148,6 +145,9 @@ In the config file for each worker, you must specify:
    with an `http` listener.
  * **Synapse 1.72 and older:** if handling the `^/_matrix/client/v3/keys/upload` endpoint, the HTTP URI for
    the main process (`worker_main_http_uri`). This config option is no longer required and is ignored when running Synapse 1.73 and newer.
+ * **Synapse 1.83 and older:** The HTTP replication endpoint that the worker should talk to on the main synapse process
+   ([`worker_replication_host`](usage/configuration/config_documentation.md#worker_replication_host) and
+   [`worker_replication_http_port`](usage/configuration/config_documentation.md#worker_replication_http_port)). If using Synapse 1.84 and newer, these are not needed if `main` is defined on the [shared configuration](#shared-configuration) `instance_map`
 
 For example:
 
@@ -177,11 +177,11 @@ The following applies to Synapse installations that have been installed from sou
 
 You can start the main Synapse process with Poetry by running the following command:
 ```console
-poetry run synapse_homeserver --config-file [your homeserver.yaml]
+poetry run synapse_homeserver -c [your homeserver.yaml]
 ```
 For worker setups, you can run the following command
 ```console
-poetry run synapse_worker --config-file [your homeserver.yaml] --config-file [your worker.yaml]
+poetry run synapse_worker -c [your worker.yaml]
 ```
 ## Available worker applications
 
@@ -232,6 +232,7 @@ information.
     ^/_matrix/client/v1/rooms/.*/hierarchy$
     ^/_matrix/client/(v1|unstable)/rooms/.*/relations/
     ^/_matrix/client/v1/rooms/.*/threads$
+    ^/_matrix/client/unstable/org.matrix.msc2716/rooms/.*/batch_send$
     ^/_matrix/client/unstable/im.nheko.summary/rooms/.*/summary$
     ^/_matrix/client/(r0|v3|unstable)/account/3pid$
     ^/_matrix/client/(r0|v3|unstable)/account/whoami$

flake.nix

@@ -178,7 +178,7 @@
                   EOF
                 '';
                 # Start synapse when `devenv up` is run.
-                processes.synapse.exec = "poetry run python -m synapse.app.homeserver -c homeserver.yaml -c homeserver-config-overrides.d";
+                processes.synapse.exec = "poetry run python -m synapse.app.homeserver -c homeserver.yaml --config-directory homeserver-config-overrides.d";
 
                 # Define the perl modules we require to run SyTest.
                 #

mypy.ini

@@ -2,29 +2,17 @@
 namespace_packages = True
 plugins = pydantic.mypy, mypy_zope:plugin, scripts-dev/mypy_synapse_plugin.py
 follow_imports = normal
+check_untyped_defs = True
 show_error_codes = True
 show_traceback = True
 mypy_path = stubs
 warn_unreachable = True
+warn_unused_ignores = True
 local_partial_types = True
 no_implicit_optional = True
-
-# Strict checks, see mypy --help
-warn_unused_configs = True
-# disallow_any_generics = True
-disallow_subclassing_any = True
-# disallow_untyped_calls = True
 disallow_untyped_defs = True
-disallow_incomplete_defs = True
-# check_untyped_defs = True
-# disallow_untyped_decorators = True
-warn_redundant_casts = True
-warn_unused_ignores = True
-# warn_return_any = True
-# no_implicit_reexport = True
 strict_equality = True
-strict_concatenate = True
-
+warn_redundant_casts = True
 # Run mypy type checking with the minimum supported Python version to catch new usage
 # that isn't backwards-compatible (types, overloads, etc).
 python_version = 3.8
@@ -43,7 +31,6 @@ warn_unused_ignores = False
 
 [mypy-synapse.util.caches.treecache]
 disallow_untyped_defs = False
-disallow_incomplete_defs = False
 
 ;; Dependencies without annotations
 ;; Before ignoring a module, check to see if type stubs are available.
@@ -53,18 +40,18 @@ disallow_incomplete_defs = False
 ;; which we can pull in as a dev dependency by adding to `pyproject.toml`'s
 ;; `[tool.poetry.dev-dependencies]` list.
 
-# https://github.com/lepture/authlib/issues/460
 [mypy-authlib.*]
 ignore_missing_imports = True
 
 [mypy-ijson.*]
 ignore_missing_imports = True
 
-# https://github.com/msgpack/msgpack-python/issues/448
+[mypy-lxml]
+ignore_missing_imports = True
+
 [mypy-msgpack]
 ignore_missing_imports = True
 
-# https://github.com/wolever/parameterized/issues/143
 [mypy-parameterized.*]
 ignore_missing_imports = True
 
@@ -86,7 +73,6 @@ ignore_missing_imports = True
 [mypy-srvlookup.*]
 ignore_missing_imports = True
 
-# https://github.com/twisted/treq/pull/366
 [mypy-treq.*]
 ignore_missing_imports = True
 

pyproject.toml

@@ -89,7 +89,7 @@ manifest-path = "rust/Cargo.toml"
 
 [tool.poetry]
 name = "matrix-synapse"
-version = "1.88.0rc1"
+version = "1.85.2"
 description = "Homeserver for the Matrix decentralised comms protocol"
 authors = ["Matrix.org Team and Contributors <packages@matrix.org>"]
 license = "Apache-2.0"
@@ -147,7 +147,7 @@ synapse_review_recent_signups = "synapse._scripts.review_recent_signups:main"
 update_synapse_database = "synapse._scripts.update_synapse_database:main"
 
 [tool.poetry.dependencies]
-python = "^3.8.0"
+python = "^3.7.1"
 
 # Mandatory Dependencies
 # ----------------------
@@ -203,9 +203,11 @@ ijson = ">=3.1.4"
 matrix-common = "^1.3.0"
 # We need packaging.requirements.Requirement, added in 16.1.
 packaging = ">=16.1"
+# At the time of writing, we only use functions from the version `importlib.metadata`
+# which shipped in Python 3.8. This corresponds to version 1.4 of the backport.
+importlib_metadata = { version = ">=1.4", python = "<3.8" }
 # This is the most recent version of Pydantic with available on common distros.
-# We are currently incompatible with >=2.0.0: (https://github.com/matrix-org/synapse/issues/15858)
-pydantic = "^1.7.4"
+pydantic = ">=1.7.4"
 
 # This is for building the rust components during "poetry install", which
 # currently ignores the `build-system.requires` directive (c.f.
@@ -309,10 +311,9 @@ all = [
 # We pin black so that our tests don't start failing on new releases.
 isort = ">=5.10.1"
 black = ">=22.3.0"
-ruff = "0.0.277"
+ruff = "0.0.265"
 
 # Typechecking
-lxml-stubs = ">=0.4.0"
 mypy = "*"
 mypy-zope = "*"
 types-bleach = ">=4.1.0"
@@ -373,15 +374,7 @@ build-backend = "poetry.core.masonry.api"
 
 [tool.cibuildwheel]
 # Skip unsupported platforms (by us or by Rust).
-# See https://cibuildwheel.readthedocs.io/en/stable/options/#build-skip for the list of build targets.
-# We skip:
-#  - CPython 3.6 and 3.7: EOLed
-#  - PyPy 3.7: we only support Python 3.8+
-#  - musllinux i686: excluded to reduce number of wheels we build.
-#    c.f. https://github.com/matrix-org/synapse/pull/12595#discussion_r963107677
-#  - PyPy on Aarch64 and musllinux on aarch64: too slow to build.
-#    c.f. https://github.com/matrix-org/synapse/pull/14259
-skip = "cp36* cp37* pp37* *-musllinux_i686 pp*aarch64 *-musllinux_aarch64"
+skip = "cp36* *-musllinux_i686 pp*aarch64 *-musllinux_aarch64"
 
 # We need a rust compiler
 before-all =  "curl https://sh.rustup.rs -sSf | sh -s -- --default-toolchain stable -y --profile minimal"

rust/Cargo.toml

@@ -7,7 +7,7 @@ name = "synapse"
 version = "0.1.0"
 
 edition = "2021"
-rust-version = "1.60.0"
+rust-version = "1.58.1"
 
 [lib]
 name = "synapse"

rust/benches/evaluator.rs

@@ -13,6 +13,8 @@
 // limitations under the License.
 
 #![feature(test)]
+use std::collections::BTreeSet;
+
 use synapse::push::{
     evaluator::PushRuleEvaluator, Condition, EventMatchCondition, FilteredPushRules, JsonValue,
     PushRules, SimpleJsonValue,
@@ -195,6 +197,7 @@ fn bench_eval_message(b: &mut Bencher) {
         false,
         false,
         false,
+        false,
     );
 
     b.iter(|| eval.run(&rules, Some("bob"), Some("person")));

rust/src/push/base_rules.rs

@@ -142,11 +142,11 @@ pub const BASE_APPEND_OVERRIDE_RULES: &[PushRule] = &[
         default_enabled: true,
     },
     PushRule {
-        rule_id: Cow::Borrowed("global/override/.m.rule.is_user_mention"),
+        rule_id: Cow::Borrowed(".org.matrix.msc3952.is_user_mention"),
         priority_class: 5,
         conditions: Cow::Borrowed(&[Condition::Known(
             KnownCondition::ExactEventPropertyContainsType(EventPropertyIsTypeCondition {
-                key: Cow::Borrowed("content.m\\.mentions.user_ids"),
+                key: Cow::Borrowed("content.org\\.matrix\\.msc3952\\.mentions.user_ids"),
                 value_type: Cow::Borrowed(&EventMatchPatternType::UserId),
             }),
         )]),
@@ -163,11 +163,11 @@ pub const BASE_APPEND_OVERRIDE_RULES: &[PushRule] = &[
         default_enabled: true,
     },
     PushRule {
-        rule_id: Cow::Borrowed("global/override/.m.rule.is_room_mention"),
+        rule_id: Cow::Borrowed(".org.matrix.msc3952.is_room_mention"),
         priority_class: 5,
         conditions: Cow::Borrowed(&[
             Condition::Known(KnownCondition::EventPropertyIs(EventPropertyIsCondition {
-                key: Cow::Borrowed("content.m\\.mentions.room"),
+                key: Cow::Borrowed("content.org\\.matrix\\.msc3952\\.mentions.room"),
                 value: Cow::Borrowed(&SimpleJsonValue::Bool(true)),
             })),
             Condition::Known(KnownCondition::SenderNotificationPermission {

rust/src/push/evaluator.rs

@@ -70,9 +70,7 @@ pub struct PushRuleEvaluator {
     /// The "content.body", if any.
     body: String,
 
-    /// True if the event has a m.mentions property. (Note that this is a separate
-    /// flag instead of checking flattened_keys since the m.mentions property
-    /// might be an empty map and not appear in flattened_keys.
+    /// True if the event has a mentions property and MSC3952 support is enabled.
     has_mentions: bool,
 
     /// The number of users in the room.
@@ -157,7 +155,9 @@ impl PushRuleEvaluator {
             let rule_id = &push_rule.rule_id().to_string();
 
             // For backwards-compatibility the legacy mention rules are disabled
-            // if the event contains the 'm.mentions' property.
+            // if the event contains the 'm.mentions' property (and if the
+            // experimental feature is enabled, both of these are represented
+            // by the has_mentions flag).
             if self.has_mentions
                 && (rule_id == "global/override/.m.rule.contains_display_name"
                     || rule_id == "global/content/.m.rule.contains_user_name"
@@ -562,7 +562,7 @@ fn test_requires_room_version_supports_condition() {
     };
     let rules = PushRules::new(vec![custom_rule]);
     result = evaluator.run(
-        &FilteredPushRules::py_new(rules, BTreeMap::new(), true, false, true, false),
+        &FilteredPushRules::py_new(rules, BTreeMap::new(), true, false, true, false, false),
         None,
         None,
     );

rust/src/push/mod.rs

@@ -527,6 +527,7 @@ pub struct FilteredPushRules {
     msc1767_enabled: bool,
     msc3381_polls_enabled: bool,
     msc3664_enabled: bool,
+    msc3952_intentional_mentions: bool,
     msc3958_suppress_edits_enabled: bool,
 }
 
@@ -539,6 +540,7 @@ impl FilteredPushRules {
         msc1767_enabled: bool,
         msc3381_polls_enabled: bool,
         msc3664_enabled: bool,
+        msc3952_intentional_mentions: bool,
         msc3958_suppress_edits_enabled: bool,
     ) -> Self {
         Self {
@@ -547,6 +549,7 @@ impl FilteredPushRules {
             msc1767_enabled,
             msc3381_polls_enabled,
             msc3664_enabled,
+            msc3952_intentional_mentions,
             msc3958_suppress_edits_enabled,
         }
     }
@@ -584,6 +587,10 @@ impl FilteredPushRules {
                     return false;
                 }
 
+                if !self.msc3952_intentional_mentions && rule.rule_id.contains("org.matrix.msc3952")
+                {
+                    return false;
+                }
                 if !self.msc3958_suppress_edits_enabled
                     && rule.rule_id == "global/override/.com.beeper.suppress_edits"
                 {

scripts-dev/build_debian_packages.py

@@ -20,20 +20,15 @@ from concurrent.futures import ThreadPoolExecutor
 from types import FrameType
 from typing import Collection, Optional, Sequence, Set
 
-# These are expanded inside the dockerfile to be a fully qualified image name.
-# e.g. docker.io/library/debian:bullseye
-#
-# If an EOL is forced by a Python version and we're dropping support for it, make sure
-# to remove references to the distibution across Synapse (search for "bullseye" for
-# example)
 DISTS = (
-    "debian:bullseye",  # (EOL ~2024-07) (our EOL forced by Python 3.9 is 2025-10-05)
-    "debian:bookworm",  # (EOL not specified yet) (our EOL forced by Python 3.11 is 2027-10-24)
-    "debian:sid",  # (EOL not specified yet) (our EOL forced by Python 3.11 is 2027-10-24)
-    "ubuntu:focal",  # 20.04 LTS (EOL 2025-04) (our EOL forced by Python 3.8 is 2024-10-14)
-    "ubuntu:jammy",  # 22.04 LTS (EOL 2027-04) (our EOL forced by Python 3.10 is 2026-10-04)
-    "ubuntu:kinetic",  # 22.10 (EOL 2023-07-20) (our EOL forced by Python 3.10 is 2026-10-04)
-    "ubuntu:lunar",  # 23.04 (EOL 2024-01) (our EOL forced by Python 3.11 is 2027-10-24)
+    "debian:buster",  # oldstable: EOL 2022-08
+    "debian:bullseye",
+    "debian:bookworm",
+    "debian:sid",
+    "ubuntu:focal",  # 20.04 LTS (our EOL forced by Py38 on 2024-10-14)
+    "ubuntu:jammy",  # 22.04 LTS (EOL 2027-04)
+    "ubuntu:kinetic",  # 22.10 (EOL 2023-07-20)
+    "ubuntu:lunar",  # 23.04 (EOL 2024-01)
 )
 
 DESC = """\

scripts-dev/complement.sh

@@ -246,6 +246,10 @@ else
   else
     export PASS_SYNAPSE_COMPLEMENT_DATABASE=sqlite
   fi
+
+  # The tests for importing historical messages (MSC2716)
+  # only pass with monoliths, currently.
+  test_tags="$test_tags,msc2716"
 fi
 
 if [[ -n "$ASYNCIO_REACTOR" ]]; then
@@ -253,10 +257,6 @@ if [[ -n "$ASYNCIO_REACTOR" ]]; then
   export PASS_SYNAPSE_COMPLEMENT_USE_ASYNCIO_REACTOR=true
 fi
 
-if [[ -n "$UNIX_SOCKETS" ]]; then
-  # Enable full on Unix socket mode for Synapse, Redis and Postgresql
-  export PASS_SYNAPSE_USE_UNIX_SOCKET=1
-fi
 
 if [[ -n "$SYNAPSE_TEST_LOG_LEVEL" ]]; then
   # Set the log level to what is desired
@@ -269,10 +269,6 @@ if [[ -n "$SYNAPSE_TEST_LOG_LEVEL" ]]; then
   export PASS_SYNAPSE_LOG_SENSITIVE=1
 fi
 
-# Log a few more useful things for a developer attempting to debug something
-# particularly tricky.
-export PASS_SYNAPSE_LOG_TESTING=1
-
 # Run the tests!
 echo "Images built; running complement"
 cd "$COMPLEMENT_DIR"

scripts-dev/federation_client.py

@@ -136,11 +136,11 @@ def request(
         authorization_headers.append(header)
         print("Authorization: %s" % header, file=sys.stderr)
 
-    dest = "matrix-federation://%s%s" % (destination, path)
+    dest = "matrix://%s%s" % (destination, path)
     print("Requesting %s" % dest, file=sys.stderr)
 
     s = requests.Session()
-    s.mount("matrix-federation://", MatrixConnectionAdapter())
+    s.mount("matrix://", MatrixConnectionAdapter())
 
     headers: Dict[str, str] = {
         "Authorization": authorization_headers[0],

stubs/synapse/synapse_rust/push.pyi

@@ -46,6 +46,7 @@ class FilteredPushRules:
         msc1767_enabled: bool,
         msc3381_polls_enabled: bool,
         msc3664_enabled: bool,
+        msc3952_intentional_mentions: bool,
         msc3958_suppress_edits_enabled: bool,
     ): ...
     def rules(self) -> Collection[Tuple[PushRule, bool]]: ...

synapse/__init__.py

@@ -25,8 +25,8 @@ from synapse.util.rust import check_rust_lib_up_to_date
 from synapse.util.stringutils import strtobool
 
 # Check that we're not running on an unsupported Python version.
-if sys.version_info < (3, 8):
-    print("Synapse requires Python 3.8 or above.")
+if sys.version_info < (3, 7):
+    print("Synapse requires Python 3.7 or above.")
     sys.exit(1)
 
 # Allow using the asyncio reactor via env var.

synapse/_scripts/synapse_port_db.py

@@ -61,7 +61,6 @@ from synapse.storage.databases.main.deviceinbox import DeviceInboxBackgroundUpda
 from synapse.storage.databases.main.devices import DeviceBackgroundUpdateStore
 from synapse.storage.databases.main.e2e_room_keys import EndToEndRoomKeyBackgroundStore
 from synapse.storage.databases.main.end_to_end_keys import EndToEndKeyBackgroundStore
-from synapse.storage.databases.main.event_federation import EventFederationWorkerStore
 from synapse.storage.databases.main.event_push_actions import EventPushActionsStore
 from synapse.storage.databases.main.events_bg_updates import (
     EventsBackgroundUpdatesStore,
@@ -197,11 +196,6 @@ IGNORED_TABLES = {
     "ui_auth_sessions",
     "ui_auth_sessions_credentials",
     "ui_auth_sessions_ips",
-    # Ignore the worker locks table, as a) there shouldn't be any acquired locks
-    # after porting, and b) the circular foreign key constraints make it hard to
-    # port.
-    "worker_read_write_locks_mode",
-    "worker_read_write_locks",
 }
 
 
@@ -245,7 +239,6 @@ class Store(
     PresenceBackgroundUpdateStore,
     ReceiptsBackgroundUpdateStore,
     RelationsWorkerStore,
-    EventFederationWorkerStore,
 ):
     def execute(self, f: Callable[..., R], *args: Any, **kwargs: Any) -> Awaitable[R]:
         return self.db_pool.runInteraction(f.__name__, f, *args, **kwargs)
@@ -810,9 +803,7 @@ class Porter:
             )
             # Map from table name to args passed to `handle_table`, i.e. a tuple
             # of: `postgres_size`, `table_size`, `forward_chunk`, `backward_chunk`.
-            tables_to_port_info_map = {
-                r[0]: r[1:] for r in setup_res if r[0] not in IGNORED_TABLES
-            }
+            tables_to_port_info_map = {r[0]: r[1:] for r in setup_res}
 
             # Step 5. Do the copying.
             #
@@ -1378,9 +1369,6 @@ def main() -> None:
         sys.stderr.write("Database must use the 'psycopg2' connector.\n")
         sys.exit(3)
 
-    # Don't run the background tasks that get started by the data stores.
-    hs_config["run_background_tasks_on"] = "some_other_process"
-
     config = HomeServerConfig()
     config.parse_config_dict(hs_config, "", "")
 

synapse/api/auth.py

@@ -1,4 +1,4 @@
-# Copyright 2023 The Matrix.org Foundation.
+# Copyright 2014 - 2016 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -14,6 +14,7 @@
 import logging
 from typing import TYPE_CHECKING, Optional, Tuple
 
+import pymacaroons
 from netaddr import IPAddress
 
 from twisted.web.server import Request
@@ -23,11 +24,19 @@ from synapse.api.constants import EventTypes, HistoryVisibility, Membership
 from synapse.api.errors import (
     AuthError,
     Codes,
+    InvalidClientTokenError,
     MissingClientTokenError,
     UnstableSpecAuthError,
 )
 from synapse.appservice import ApplicationService
-from synapse.logging.opentracing import trace
+from synapse.http import get_request_user_agent
+from synapse.http.site import SynapseRequest
+from synapse.logging.opentracing import (
+    active_span,
+    force_tracing,
+    start_active_span,
+    trace,
+)
 from synapse.types import Requester, create_requester
 from synapse.util.cancellation import cancellable
 
@@ -37,13 +46,26 @@ if TYPE_CHECKING:
 logger = logging.getLogger(__name__)
 
 
-class BaseAuth:
-    """Common base class for all auth implementations."""
+# guests always get this device id.
+GUEST_DEVICE_ID = "guest_device"
+
+
+class Auth:
+    """
+    This class contains functions for authenticating users of our client-server API.
+    """
 
     def __init__(self, hs: "HomeServer"):
         self.hs = hs
+        self.clock = hs.get_clock()
         self.store = hs.get_datastores().main
+        self._account_validity_handler = hs.get_account_validity_handler()
         self._storage_controllers = hs.get_storage_controllers()
+        self._macaroon_generator = hs.get_macaroon_generator()
+
+        self._track_appservice_user_ips = hs.config.appservice.track_appservice_user_ips
+        self._track_puppeted_user_ips = hs.config.api.track_puppeted_user_ips
+        self._force_tracing_for_users = hs.config.tracing.force_tracing_for_users
 
     async def check_user_in_room(
         self,
@@ -97,49 +119,139 @@ class BaseAuth:
             errcode=Codes.NOT_JOINED,
         )
 
-    @trace
-    async def check_user_in_room_or_world_readable(
-        self, room_id: str, requester: Requester, allow_departed_users: bool = False
-    ) -> Tuple[str, Optional[str]]:
-        """Checks that the user is or was in the room or the room is world
-        readable. If it isn't then an exception is raised.
+    @cancellable
+    async def get_user_by_req(
+        self,
+        request: SynapseRequest,
+        allow_guest: bool = False,
+        allow_expired: bool = False,
+    ) -> Requester:
+        """Get a registered user's ID.
 
         Args:
-            room_id: room to check
-            user_id: user to check
-            allow_departed_users: if True, accept users that were previously
-                members but have now departed
+            request: An HTTP request with an access_token query parameter.
+            allow_guest: If False, will raise an AuthError if the user making the
+                request is a guest.
+            allow_expired: If True, allow the request through even if the account
+                is expired, or session token lifetime has ended. Note that
+                /login will deliver access tokens regardless of expiration.
 
         Returns:
-            Resolves to the current membership of the user in the room and the
-            membership event ID of the user. If the user is not in the room and
-            never has been, then `(Membership.JOIN, None)` is returned.
+            Resolves to the requester
+        Raises:
+            InvalidClientCredentialsError if no user by that token exists or the token
+                is invalid.
+            AuthError if access is denied for the user in the access token
         """
+        parent_span = active_span()
+        with start_active_span("get_user_by_req"):
+            requester = await self._wrapped_get_user_by_req(
+                request, allow_guest, allow_expired
+            )
 
+            if parent_span:
+                if requester.authenticated_entity in self._force_tracing_for_users:
+                    # request tracing is enabled for this user, so we need to force it
+                    # tracing on for the parent span (which will be the servlet span).
+                    #
+                    # It's too late for the get_user_by_req span to inherit the setting,
+                    # so we also force it on for that.
+                    force_tracing()
+                    force_tracing(parent_span)
+                parent_span.set_tag(
+                    "authenticated_entity", requester.authenticated_entity
+                )
+                parent_span.set_tag("user_id", requester.user.to_string())
+                if requester.device_id is not None:
+                    parent_span.set_tag("device_id", requester.device_id)
+                if requester.app_service is not None:
+                    parent_span.set_tag("appservice_id", requester.app_service.id)
+            return requester
+
+    @cancellable
+    async def _wrapped_get_user_by_req(
+        self,
+        request: SynapseRequest,
+        allow_guest: bool,
+        allow_expired: bool,
+    ) -> Requester:
+        """Helper for get_user_by_req
+
+        Once get_user_by_req has set up the opentracing span, this does the actual work.
+        """
         try:
-            # check_user_in_room will return the most recent membership
-            # event for the user if:
-            #  * The user is a non-guest user, and was ever in the room
-            #  * The user is a guest user, and has joined the room
-            # else it will throw.
-            return await self.check_user_in_room(
-                room_id, requester, allow_departed_users=allow_departed_users
-            )
-        except AuthError:
-            visibility = await self._storage_controllers.state.get_current_state_event(
-                room_id, EventTypes.RoomHistoryVisibility, ""
-            )
-            if (
-                visibility
-                and visibility.content.get("history_visibility")
-                == HistoryVisibility.WORLD_READABLE
+            ip_addr = request.getClientAddress().host
+            user_agent = get_request_user_agent(request)
+
+            access_token = self.get_access_token_from_request(request)
+
+            # First check if it could be a request from an appservice
+            requester = await self._get_appservice_user(request)
+            if not requester:
+                # If not, it should be from a regular user
+                requester = await self.get_user_by_access_token(
+                    access_token, allow_expired=allow_expired
+                )
+
+                # Deny the request if the user account has expired.
+                # This check is only done for regular users, not appservice ones.
+                if not allow_expired:
+                    if await self._account_validity_handler.is_user_expired(
+                        requester.user.to_string()
+                    ):
+                        # Raise the error if either an account validity module has determined
+                        # the account has expired, or the legacy account validity
+                        # implementation is enabled and determined the account has expired
+                        raise AuthError(
+                            403,
+                            "User account has expired",
+                            errcode=Codes.EXPIRED_ACCOUNT,
+                        )
+
+            if ip_addr and (
+                not requester.app_service or self._track_appservice_user_ips
             ):
-                return Membership.JOIN, None
-            raise AuthError(
-                403,
-                "User %r not in room %s, and room previews are disabled"
-                % (requester.user, room_id),
-            )
+                # XXX(quenting): I'm 95% confident that we could skip setting the
+                # device_id to "dummy-device" for appservices, and that the only impact
+                # would be some rows which whould not deduplicate in the 'user_ips'
+                # table during the transition
+                recorded_device_id = (
+                    "dummy-device"
+                    if requester.device_id is None and requester.app_service is not None
+                    else requester.device_id
+                )
+                await self.store.insert_client_ip(
+                    user_id=requester.authenticated_entity,
+                    access_token=access_token,
+                    ip=ip_addr,
+                    user_agent=user_agent,
+                    device_id=recorded_device_id,
+                )
+
+                # Track also the puppeted user client IP if enabled and the user is puppeting
+                if (
+                    requester.user.to_string() != requester.authenticated_entity
+                    and self._track_puppeted_user_ips
+                ):
+                    await self.store.insert_client_ip(
+                        user_id=requester.user.to_string(),
+                        access_token=access_token,
+                        ip=ip_addr,
+                        user_agent=user_agent,
+                        device_id=requester.device_id,
+                    )
+
+            if requester.is_guest and not allow_guest:
+                raise AuthError(
+                    403,
+                    "Guest access not allowed",
+                    errcode=Codes.GUEST_ACCESS_FORBIDDEN,
+                )
+
+            request.requester = requester
+            return requester
+        except KeyError:
+            raise MissingClientTokenError()
 
     async def validate_appservice_can_control_user_id(
         self, app_service: ApplicationService, user_id: str
@@ -172,16 +284,184 @@ class BaseAuth:
                 403, "Application service has not registered this user (%s)" % user_id
             )
 
+    @cancellable
+    async def _get_appservice_user(self, request: Request) -> Optional[Requester]:
+        """
+        Given a request, reads the request parameters to determine:
+        - whether it's an application service that's making this request
+        - what user the application service should be treated as controlling
+          (the user_id URI parameter allows an application service to masquerade
+          any applicable user in its namespace)
+        - what device the application service should be treated as controlling
+          (the device_id[^1] URI parameter allows an application service to masquerade
+          as any device that exists for the relevant user)
+
+        [^1] Unstable and provided by MSC3202.
+             Must use `org.matrix.msc3202.device_id` in place of `device_id` for now.
+
+        Returns:
+            the application service `Requester` of that request
+
+        Postconditions:
+        - The `app_service` field in the returned `Requester` is set
+        - The `user_id` field in the returned `Requester` is either the application
+          service sender or the controlled user set by the `user_id` URI parameter
+        - The returned application service is permitted to control the returned user ID.
+        - The returned device ID, if present, has been checked to be a valid device ID
+          for the returned user ID.
+        """
+        DEVICE_ID_ARG_NAME = b"org.matrix.msc3202.device_id"
+
+        app_service = self.store.get_app_service_by_token(
+            self.get_access_token_from_request(request)
+        )
+        if app_service is None:
+            return None
+
+        if app_service.ip_range_whitelist:
+            ip_address = IPAddress(request.getClientAddress().host)
+            if ip_address not in app_service.ip_range_whitelist:
+                return None
+
+        # This will always be set by the time Twisted calls us.
+        assert request.args is not None
+
+        if b"user_id" in request.args:
+            effective_user_id = request.args[b"user_id"][0].decode("utf8")
+            await self.validate_appservice_can_control_user_id(
+                app_service, effective_user_id
+            )
+        else:
+            effective_user_id = app_service.sender
+
+        effective_device_id: Optional[str] = None
+
+        if (
+            self.hs.config.experimental.msc3202_device_masquerading_enabled
+            and DEVICE_ID_ARG_NAME in request.args
+        ):
+            effective_device_id = request.args[DEVICE_ID_ARG_NAME][0].decode("utf8")
+            # We only just set this so it can't be None!
+            assert effective_device_id is not None
+            device_opt = await self.store.get_device(
+                effective_user_id, effective_device_id
+            )
+            if device_opt is None:
+                # For now, use 400 M_EXCLUSIVE if the device doesn't exist.
+                # This is an open thread of discussion on MSC3202 as of 2021-12-09.
+                raise AuthError(
+                    400,
+                    f"Application service trying to use a device that doesn't exist ('{effective_device_id}' for {effective_user_id})",
+                    Codes.EXCLUSIVE,
+                )
+
+        return create_requester(
+            effective_user_id, app_service=app_service, device_id=effective_device_id
+        )
+
+    async def get_user_by_access_token(
+        self,
+        token: str,
+        allow_expired: bool = False,
+    ) -> Requester:
+        """Validate access token and get user_id from it
+
+        Args:
+            token: The access token to get the user by
+            allow_expired: If False, raises an InvalidClientTokenError
+                if the token is expired
+
+        Raises:
+            InvalidClientTokenError if a user by that token exists, but the token is
+                expired
+            InvalidClientCredentialsError if no user by that token exists or the token
+                is invalid
+        """
+
+        # First look in the database to see if the access token is present
+        # as an opaque token.
+        user_info = await self.store.get_user_by_access_token(token)
+        if user_info:
+            valid_until_ms = user_info.valid_until_ms
+            if (
+                not allow_expired
+                and valid_until_ms is not None
+                and valid_until_ms < self.clock.time_msec()
+            ):
+                # there was a valid access token, but it has expired.
+                # soft-logout the user.
+                raise InvalidClientTokenError(
+                    msg="Access token has expired", soft_logout=True
+                )
+
+            # Mark the token as used. This is used to invalidate old refresh
+            # tokens after some time.
+            await self.store.mark_access_token_as_used(user_info.token_id)
+
+            requester = create_requester(
+                user_id=user_info.user_id,
+                access_token_id=user_info.token_id,
+                is_guest=user_info.is_guest,
+                shadow_banned=user_info.shadow_banned,
+                device_id=user_info.device_id,
+                authenticated_entity=user_info.token_owner,
+            )
+
+            return requester
+
+        # If the token isn't found in the database, then it could still be a
+        # macaroon for a guest, so we check that here.
+        try:
+            user_id = self._macaroon_generator.verify_guest_token(token)
+
+            # Guest access tokens are not stored in the database (there can
+            # only be one access token per guest, anyway).
+            #
+            # In order to prevent guest access tokens being used as regular
+            # user access tokens (and hence getting around the invalidation
+            # process), we look up the user id and check that it is indeed
+            # a guest user.
+            #
+            # It would of course be much easier to store guest access
+            # tokens in the database as well, but that would break existing
+            # guest tokens.
+            stored_user = await self.store.get_user_by_id(user_id)
+            if not stored_user:
+                raise InvalidClientTokenError("Unknown user_id %s" % user_id)
+            if not stored_user["is_guest"]:
+                raise InvalidClientTokenError(
+                    "Guest access token used for regular user"
+                )
+
+            return create_requester(
+                user_id=user_id,
+                is_guest=True,
+                # all guests get the same device id
+                device_id=GUEST_DEVICE_ID,
+                authenticated_entity=user_id,
+            )
+        except (
+            pymacaroons.exceptions.MacaroonException,
+            TypeError,
+            ValueError,
+        ) as e:
+            logger.warning(
+                "Invalid access token in auth: %s %s.",
+                type(e),
+                e,
+            )
+            raise InvalidClientTokenError("Invalid access token passed.")
+
     async def is_server_admin(self, requester: Requester) -> bool:
         """Check if the given user is a local server admin.
 
         Args:
-            requester: user to check
+            requester: The user making the request, according to the access token.
 
         Returns:
             True if the user is an admin
         """
-        raise NotImplementedError()
+        return await self.store.is_server_admin(requester.user)
 
     async def check_can_change_room_list(
         self, room_id: str, requester: Requester
@@ -190,8 +470,8 @@ class BaseAuth:
         published room list.
 
         Args:
-            room_id
-            user
+            room_id: The room to check.
+            requester: The user making the request, according to the access token.
         """
 
         is_admin = await self.is_server_admin(requester)
@@ -238,6 +518,7 @@ class BaseAuth:
         return bool(query_params) or bool(auth_headers)
 
     @staticmethod
+    @cancellable
     def get_access_token_from_request(request: Request) -> str:
         """Extracts the access_token from the request.
 
@@ -275,77 +556,47 @@ class BaseAuth:
 
             return query_params[0].decode("ascii")
 
-    @cancellable
-    async def get_appservice_user(
-        self, request: Request, access_token: str
-    ) -> Optional[Requester]:
-        """
-        Given a request, reads the request parameters to determine:
-        - whether it's an application service that's making this request
-        - what user the application service should be treated as controlling
-          (the user_id URI parameter allows an application service to masquerade
-          any applicable user in its namespace)
-        - what device the application service should be treated as controlling
-          (the device_id[^1] URI parameter allows an application service to masquerade
-          as any device that exists for the relevant user)
+    @trace
+    async def check_user_in_room_or_world_readable(
+        self, room_id: str, requester: Requester, allow_departed_users: bool = False
+    ) -> Tuple[str, Optional[str]]:
+        """Checks that the user is or was in the room or the room is world
+        readable. If it isn't then an exception is raised.
 
-        [^1] Unstable and provided by MSC3202.
-             Must use `org.matrix.msc3202.device_id` in place of `device_id` for now.
+        Args:
+            room_id: The room to check.
+            requester: The user making the request, according to the access token.
+            allow_departed_users: If True, accept users that were previously
+                members but have now departed.
 
         Returns:
-            the application service `Requester` of that request
-
-        Postconditions:
-        - The `app_service` field in the returned `Requester` is set
-        - The `user_id` field in the returned `Requester` is either the application
-          service sender or the controlled user set by the `user_id` URI parameter
-        - The returned application service is permitted to control the returned user ID.
-        - The returned device ID, if present, has been checked to be a valid device ID
-          for the returned user ID.
+            Resolves to the current membership of the user in the room and the
+            membership event ID of the user. If the user is not in the room and
+            never has been, then `(Membership.JOIN, None)` is returned.
         """
-        DEVICE_ID_ARG_NAME = b"org.matrix.msc3202.device_id"
 
-        app_service = self.store.get_app_service_by_token(access_token)
-        if app_service is None:
-            return None
-
-        if app_service.ip_range_whitelist:
-            ip_address = IPAddress(request.getClientAddress().host)
-            if ip_address not in app_service.ip_range_whitelist:
-                return None
-
-        # This will always be set by the time Twisted calls us.
-        assert request.args is not None
-
-        if b"user_id" in request.args:
-            effective_user_id = request.args[b"user_id"][0].decode("utf8")
-            await self.validate_appservice_can_control_user_id(
-                app_service, effective_user_id
+        try:
+            # check_user_in_room will return the most recent membership
+            # event for the user if:
+            #  * The user is a non-guest user, and was ever in the room
+            #  * The user is a guest user, and has joined the room
+            # else it will throw.
+            return await self.check_user_in_room(
+                room_id, requester, allow_departed_users=allow_departed_users
             )
-        else:
-            effective_user_id = app_service.sender
-
-        effective_device_id: Optional[str] = None
-
-        if (
-            self.hs.config.experimental.msc3202_device_masquerading_enabled
-            and DEVICE_ID_ARG_NAME in request.args
-        ):
-            effective_device_id = request.args[DEVICE_ID_ARG_NAME][0].decode("utf8")
-            # We only just set this so it can't be None!
-            assert effective_device_id is not None
-            device_opt = await self.store.get_device(
-                effective_user_id, effective_device_id
+        except AuthError:
+            visibility = await self._storage_controllers.state.get_current_state_event(
+                room_id, EventTypes.RoomHistoryVisibility, ""
+            )
+            if (
+                visibility
+                and visibility.content.get("history_visibility")
+                == HistoryVisibility.WORLD_READABLE
+            ):
+                return Membership.JOIN, None
+            raise UnstableSpecAuthError(
+                403,
+                "User %s not in room %s, and room previews are disabled"
+                % (requester.user, room_id),
+                errcode=Codes.NOT_JOINED,
             )
-            if device_opt is None:
-                # For now, use 400 M_EXCLUSIVE if the device doesn't exist.
-                # This is an open thread of discussion on MSC3202 as of 2021-12-09.
-                raise AuthError(
-                    400,
-                    f"Application service trying to use a device that doesn't exist ('{effective_device_id}' for {effective_user_id})",
-                    Codes.EXCLUSIVE,
-                )
-
-        return create_requester(
-            effective_user_id, app_service=app_service, device_id=effective_device_id
-        )

synapse/api/auth/__init__.py

@@ -1,175 +0,0 @@
-# Copyright 2023 The Matrix.org Foundation.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-from typing import Optional, Tuple
-
-from typing_extensions import Protocol
-
-from twisted.web.server import Request
-
-from synapse.appservice import ApplicationService
-from synapse.http.site import SynapseRequest
-from synapse.types import Requester
-
-# guests always get this device id.
-GUEST_DEVICE_ID = "guest_device"
-
-
-class Auth(Protocol):
-    """The interface that an auth provider must implement."""
-
-    async def check_user_in_room(
-        self,
-        room_id: str,
-        requester: Requester,
-        allow_departed_users: bool = False,
-    ) -> Tuple[str, Optional[str]]:
-        """Check if the user is in the room, or was at some point.
-        Args:
-            room_id: The room to check.
-
-            user_id: The user to check.
-
-            current_state: Optional map of the current state of the room.
-                If provided then that map is used to check whether they are a
-                member of the room. Otherwise the current membership is
-                loaded from the database.
-
-            allow_departed_users: if True, accept users that were previously
-                members but have now departed.
-
-        Raises:
-            AuthError if the user is/was not in the room.
-        Returns:
-            The current membership of the user in the room and the
-            membership event ID of the user.
-        """
-
-    async def get_user_by_req(
-        self,
-        request: SynapseRequest,
-        allow_guest: bool = False,
-        allow_expired: bool = False,
-    ) -> Requester:
-        """Get a registered user's ID.
-
-        Args:
-            request: An HTTP request with an access_token query parameter.
-            allow_guest: If False, will raise an AuthError if the user making the
-                request is a guest.
-            allow_expired: If True, allow the request through even if the account
-                is expired, or session token lifetime has ended. Note that
-                /login will deliver access tokens regardless of expiration.
-
-        Returns:
-            Resolves to the requester
-        Raises:
-            InvalidClientCredentialsError if no user by that token exists or the token
-                is invalid.
-            AuthError if access is denied for the user in the access token
-        """
-
-    async def validate_appservice_can_control_user_id(
-        self, app_service: ApplicationService, user_id: str
-    ) -> None:
-        """Validates that the app service is allowed to control
-        the given user.
-
-        Args:
-            app_service: The app service that controls the user
-            user_id: The author MXID that the app service is controlling
-
-        Raises:
-            AuthError: If the application service is not allowed to control the user
-                (user namespace regex does not match, wrong homeserver, etc)
-                or if the user has not been registered yet.
-        """
-
-    async def get_user_by_access_token(
-        self,
-        token: str,
-        allow_expired: bool = False,
-    ) -> Requester:
-        """Validate access token and get user_id from it
-
-        Args:
-            token: The access token to get the user by
-            allow_expired: If False, raises an InvalidClientTokenError
-                if the token is expired
-
-        Raises:
-            InvalidClientTokenError if a user by that token exists, but the token is
-                expired
-            InvalidClientCredentialsError if no user by that token exists or the token
-                is invalid
-        """
-
-    async def is_server_admin(self, requester: Requester) -> bool:
-        """Check if the given user is a local server admin.
-
-        Args:
-            requester: user to check
-
-        Returns:
-            True if the user is an admin
-        """
-
-    async def check_can_change_room_list(
-        self, room_id: str, requester: Requester
-    ) -> bool:
-        """Determine whether the user is allowed to edit the room's entry in the
-        published room list.
-
-        Args:
-            room_id
-            user
-        """
-
-    @staticmethod
-    def has_access_token(request: Request) -> bool:
-        """Checks if the request has an access_token.
-
-        Returns:
-            False if no access_token was given, True otherwise.
-        """
-
-    @staticmethod
-    def get_access_token_from_request(request: Request) -> str:
-        """Extracts the access_token from the request.
-
-        Args:
-            request: The http request.
-        Returns:
-            The access_token
-        Raises:
-            MissingClientTokenError: If there isn't a single access_token in the
-                request
-        """
-
-    async def check_user_in_room_or_world_readable(
-        self, room_id: str, requester: Requester, allow_departed_users: bool = False
-    ) -> Tuple[str, Optional[str]]:
-        """Checks that the user is or was in the room or the room is world
-        readable. If it isn't then an exception is raised.
-
-        Args:
-            room_id: room to check
-            user_id: user to check
-            allow_departed_users: if True, accept users that were previously
-                members but have now departed
-
-        Returns:
-            Resolves to the current membership of the user in the room and the
-            membership event ID of the user. If the user is not in the room and
-            never has been, then `(Membership.JOIN, None)` is returned.
-        """

synapse/api/auth/internal.py

@@ -1,291 +0,0 @@
-# Copyright 2023 The Matrix.org Foundation.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-import logging
-from typing import TYPE_CHECKING
-
-import pymacaroons
-
-from synapse.api.errors import (
-    AuthError,
-    Codes,
-    InvalidClientTokenError,
-    MissingClientTokenError,
-)
-from synapse.http import get_request_user_agent
-from synapse.http.site import SynapseRequest
-from synapse.logging.opentracing import active_span, force_tracing, start_active_span
-from synapse.types import Requester, create_requester
-from synapse.util.cancellation import cancellable
-
-from . import GUEST_DEVICE_ID
-from .base import BaseAuth
-
-if TYPE_CHECKING:
-    from synapse.server import HomeServer
-
-logger = logging.getLogger(__name__)
-
-
-class InternalAuth(BaseAuth):
-    """
-    This class contains functions for authenticating users of our client-server API.
-    """
-
-    def __init__(self, hs: "HomeServer"):
-        super().__init__(hs)
-        self.clock = hs.get_clock()
-        self._account_validity_handler = hs.get_account_validity_handler()
-        self._macaroon_generator = hs.get_macaroon_generator()
-
-        self._track_appservice_user_ips = hs.config.appservice.track_appservice_user_ips
-        self._track_puppeted_user_ips = hs.config.api.track_puppeted_user_ips
-        self._force_tracing_for_users = hs.config.tracing.force_tracing_for_users
-
-    @cancellable
-    async def get_user_by_req(
-        self,
-        request: SynapseRequest,
-        allow_guest: bool = False,
-        allow_expired: bool = False,
-    ) -> Requester:
-        """Get a registered user's ID.
-
-        Args:
-            request: An HTTP request with an access_token query parameter.
-            allow_guest: If False, will raise an AuthError if the user making the
-                request is a guest.
-            allow_expired: If True, allow the request through even if the account
-                is expired, or session token lifetime has ended. Note that
-                /login will deliver access tokens regardless of expiration.
-
-        Returns:
-            Resolves to the requester
-        Raises:
-            InvalidClientCredentialsError if no user by that token exists or the token
-                is invalid.
-            AuthError if access is denied for the user in the access token
-        """
-        parent_span = active_span()
-        with start_active_span("get_user_by_req"):
-            requester = await self._wrapped_get_user_by_req(
-                request, allow_guest, allow_expired
-            )
-
-            if parent_span:
-                if requester.authenticated_entity in self._force_tracing_for_users:
-                    # request tracing is enabled for this user, so we need to force it
-                    # tracing on for the parent span (which will be the servlet span).
-                    #
-                    # It's too late for the get_user_by_req span to inherit the setting,
-                    # so we also force it on for that.
-                    force_tracing()
-                    force_tracing(parent_span)
-                parent_span.set_tag(
-                    "authenticated_entity", requester.authenticated_entity
-                )
-                parent_span.set_tag("user_id", requester.user.to_string())
-                if requester.device_id is not None:
-                    parent_span.set_tag("device_id", requester.device_id)
-                if requester.app_service is not None:
-                    parent_span.set_tag("appservice_id", requester.app_service.id)
-            return requester
-
-    @cancellable
-    async def _wrapped_get_user_by_req(
-        self,
-        request: SynapseRequest,
-        allow_guest: bool,
-        allow_expired: bool,
-    ) -> Requester:
-        """Helper for get_user_by_req
-
-        Once get_user_by_req has set up the opentracing span, this does the actual work.
-        """
-        try:
-            ip_addr = request.getClientAddress().host
-            user_agent = get_request_user_agent(request)
-
-            access_token = self.get_access_token_from_request(request)
-
-            # First check if it could be a request from an appservice
-            requester = await self.get_appservice_user(request, access_token)
-            if not requester:
-                # If not, it should be from a regular user
-                requester = await self.get_user_by_access_token(
-                    access_token, allow_expired=allow_expired
-                )
-
-                # Deny the request if the user account has expired.
-                # This check is only done for regular users, not appservice ones.
-                if not allow_expired:
-                    if await self._account_validity_handler.is_user_expired(
-                        requester.user.to_string()
-                    ):
-                        # Raise the error if either an account validity module has determined
-                        # the account has expired, or the legacy account validity
-                        # implementation is enabled and determined the account has expired
-                        raise AuthError(
-                            403,
-                            "User account has expired",
-                            errcode=Codes.EXPIRED_ACCOUNT,
-                        )
-
-            if ip_addr and (
-                not requester.app_service or self._track_appservice_user_ips
-            ):
-                # XXX(quenting): I'm 95% confident that we could skip setting the
-                # device_id to "dummy-device" for appservices, and that the only impact
-                # would be some rows which whould not deduplicate in the 'user_ips'
-                # table during the transition
-                recorded_device_id = (
-                    "dummy-device"
-                    if requester.device_id is None and requester.app_service is not None
-                    else requester.device_id
-                )
-                await self.store.insert_client_ip(
-                    user_id=requester.authenticated_entity,
-                    access_token=access_token,
-                    ip=ip_addr,
-                    user_agent=user_agent,
-                    device_id=recorded_device_id,
-                )
-
-                # Track also the puppeted user client IP if enabled and the user is puppeting
-                if (
-                    requester.user.to_string() != requester.authenticated_entity
-                    and self._track_puppeted_user_ips
-                ):
-                    await self.store.insert_client_ip(
-                        user_id=requester.user.to_string(),
-                        access_token=access_token,
-                        ip=ip_addr,
-                        user_agent=user_agent,
-                        device_id=requester.device_id,
-                    )
-
-            if requester.is_guest and not allow_guest:
-                raise AuthError(
-                    403,
-                    "Guest access not allowed",
-                    errcode=Codes.GUEST_ACCESS_FORBIDDEN,
-                )
-
-            request.requester = requester
-            return requester
-        except KeyError:
-            raise MissingClientTokenError()
-
-    async def get_user_by_access_token(
-        self,
-        token: str,
-        allow_expired: bool = False,
-    ) -> Requester:
-        """Validate access token and get user_id from it
-
-        Args:
-            token: The access token to get the user by
-            allow_expired: If False, raises an InvalidClientTokenError
-                if the token is expired
-
-        Raises:
-            InvalidClientTokenError if a user by that token exists, but the token is
-                expired
-            InvalidClientCredentialsError if no user by that token exists or the token
-                is invalid
-        """
-
-        # First look in the database to see if the access token is present
-        # as an opaque token.
-        user_info = await self.store.get_user_by_access_token(token)
-        if user_info:
-            valid_until_ms = user_info.valid_until_ms
-            if (
-                not allow_expired
-                and valid_until_ms is not None
-                and valid_until_ms < self.clock.time_msec()
-            ):
-                # there was a valid access token, but it has expired.
-                # soft-logout the user.
-                raise InvalidClientTokenError(
-                    msg="Access token has expired", soft_logout=True
-                )
-
-            # Mark the token as used. This is used to invalidate old refresh
-            # tokens after some time.
-            await self.store.mark_access_token_as_used(user_info.token_id)
-
-            requester = create_requester(
-                user_id=user_info.user_id,
-                access_token_id=user_info.token_id,
-                is_guest=user_info.is_guest,
-                shadow_banned=user_info.shadow_banned,
-                device_id=user_info.device_id,
-                authenticated_entity=user_info.token_owner,
-            )
-
-            return requester
-
-        # If the token isn't found in the database, then it could still be a
-        # macaroon for a guest, so we check that here.
-        try:
-            user_id = self._macaroon_generator.verify_guest_token(token)
-
-            # Guest access tokens are not stored in the database (there can
-            # only be one access token per guest, anyway).
-            #
-            # In order to prevent guest access tokens being used as regular
-            # user access tokens (and hence getting around the invalidation
-            # process), we look up the user id and check that it is indeed
-            # a guest user.
-            #
-            # It would of course be much easier to store guest access
-            # tokens in the database as well, but that would break existing
-            # guest tokens.
-            stored_user = await self.store.get_user_by_id(user_id)
-            if not stored_user:
-                raise InvalidClientTokenError("Unknown user_id %s" % user_id)
-            if not stored_user["is_guest"]:
-                raise InvalidClientTokenError(
-                    "Guest access token used for regular user"
-                )
-
-            return create_requester(
-                user_id=user_id,
-                is_guest=True,
-                # all guests get the same device id
-                device_id=GUEST_DEVICE_ID,
-                authenticated_entity=user_id,
-            )
-        except (
-            pymacaroons.exceptions.MacaroonException,
-            TypeError,
-            ValueError,
-        ) as e:
-            logger.warning(
-                "Invalid access token in auth: %s %s.",
-                type(e),
-                e,
-            )
-            raise InvalidClientTokenError("Invalid access token passed.")
-
-    async def is_server_admin(self, requester: Requester) -> bool:
-        """Check if the given user is a local server admin.
-
-        Args:
-            requester: The user making the request, according to the access token.
-
-        Returns:
-            True if the user is an admin
-        """
-        return await self.store.is_server_admin(requester.user)

synapse/api/auth/msc3861_delegated.py

@@ -1,352 +0,0 @@
-# Copyright 2023 The Matrix.org Foundation.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-import logging
-from typing import TYPE_CHECKING, Any, Dict, List, Optional
-from urllib.parse import urlencode
-
-from authlib.oauth2 import ClientAuth
-from authlib.oauth2.auth import encode_client_secret_basic, encode_client_secret_post
-from authlib.oauth2.rfc7523 import ClientSecretJWT, PrivateKeyJWT, private_key_jwt_sign
-from authlib.oauth2.rfc7662 import IntrospectionToken
-from authlib.oidc.discovery import OpenIDProviderMetadata, get_well_known_url
-
-from twisted.web.client import readBody
-from twisted.web.http_headers import Headers
-
-from synapse.api.auth.base import BaseAuth
-from synapse.api.errors import (
-    AuthError,
-    HttpResponseException,
-    InvalidClientTokenError,
-    OAuthInsufficientScopeError,
-    StoreError,
-    SynapseError,
-)
-from synapse.http.site import SynapseRequest
-from synapse.logging.context import make_deferred_yieldable
-from synapse.types import Requester, UserID, create_requester
-from synapse.util import json_decoder
-from synapse.util.caches.cached_call import RetryOnExceptionCachedCall
-
-if TYPE_CHECKING:
-    from synapse.server import HomeServer
-
-logger = logging.getLogger(__name__)
-
-# Scope as defined by MSC2967
-# https://github.com/matrix-org/matrix-spec-proposals/pull/2967
-SCOPE_MATRIX_API = "urn:matrix:org.matrix.msc2967.client:api:*"
-SCOPE_MATRIX_GUEST = "urn:matrix:org.matrix.msc2967.client:api:guest"
-SCOPE_MATRIX_DEVICE_PREFIX = "urn:matrix:org.matrix.msc2967.client:device:"
-
-# Scope which allows access to the Synapse admin API
-SCOPE_SYNAPSE_ADMIN = "urn:synapse:admin:*"
-
-
-def scope_to_list(scope: str) -> List[str]:
-    """Convert a scope string to a list of scope tokens"""
-    return scope.strip().split(" ")
-
-
-class PrivateKeyJWTWithKid(PrivateKeyJWT):  # type: ignore[misc]
-    """An implementation of the private_key_jwt client auth method that includes a kid header.
-
-    This is needed because some providers (Keycloak) require the kid header to figure
-    out which key to use to verify the signature.
-    """
-
-    def sign(self, auth: Any, token_endpoint: str) -> bytes:
-        return private_key_jwt_sign(
-            auth.client_secret,
-            client_id=auth.client_id,
-            token_endpoint=token_endpoint,
-            claims=self.claims,
-            header={"kid": auth.client_secret["kid"]},
-        )
-
-
-class MSC3861DelegatedAuth(BaseAuth):
-    AUTH_METHODS = {
-        "client_secret_post": encode_client_secret_post,
-        "client_secret_basic": encode_client_secret_basic,
-        "client_secret_jwt": ClientSecretJWT(),
-        "private_key_jwt": PrivateKeyJWTWithKid(),
-    }
-
-    EXTERNAL_ID_PROVIDER = "oauth-delegated"
-
-    def __init__(self, hs: "HomeServer"):
-        super().__init__(hs)
-
-        self._config = hs.config.experimental.msc3861
-        auth_method = MSC3861DelegatedAuth.AUTH_METHODS.get(
-            self._config.client_auth_method.value, None
-        )
-        # Those assertions are already checked when parsing the config
-        assert self._config.enabled, "OAuth delegation is not enabled"
-        assert self._config.issuer, "No issuer provided"
-        assert self._config.client_id, "No client_id provided"
-        assert auth_method is not None, "Invalid client_auth_method provided"
-
-        self._http_client = hs.get_proxied_http_client()
-        self._hostname = hs.hostname
-        self._admin_token = self._config.admin_token
-
-        self._issuer_metadata = RetryOnExceptionCachedCall(self._load_metadata)
-
-        if isinstance(auth_method, PrivateKeyJWTWithKid):
-            # Use the JWK as the client secret when using the private_key_jwt method
-            assert self._config.jwk, "No JWK provided"
-            self._client_auth = ClientAuth(
-                self._config.client_id, self._config.jwk, auth_method
-            )
-        else:
-            # Else use the client secret
-            assert self._config.client_secret, "No client_secret provided"
-            self._client_auth = ClientAuth(
-                self._config.client_id, self._config.client_secret, auth_method
-            )
-
-    async def _load_metadata(self) -> OpenIDProviderMetadata:
-        if self._config.issuer_metadata is not None:
-            return OpenIDProviderMetadata(**self._config.issuer_metadata)
-        url = get_well_known_url(self._config.issuer, external=True)
-        response = await self._http_client.get_json(url)
-        metadata = OpenIDProviderMetadata(**response)
-        # metadata.validate_introspection_endpoint()
-        return metadata
-
-    async def _introspect_token(self, token: str) -> IntrospectionToken:
-        """
-        Send a token to the introspection endpoint and returns the introspection response
-
-        Parameters:
-            token: The token to introspect
-
-        Raises:
-            HttpResponseException: If the introspection endpoint returns a non-2xx response
-            ValueError: If the introspection endpoint returns an invalid JSON response
-            JSONDecodeError: If the introspection endpoint returns a non-JSON response
-            Exception: If the HTTP request fails
-
-        Returns:
-            The introspection response
-        """
-        metadata = await self._issuer_metadata.get()
-        introspection_endpoint = metadata.get("introspection_endpoint")
-        raw_headers: Dict[str, str] = {
-            "Content-Type": "application/x-www-form-urlencoded",
-            "User-Agent": str(self._http_client.user_agent, "utf-8"),
-            "Accept": "application/json",
-        }
-
-        args = {"token": token, "token_type_hint": "access_token"}
-        body = urlencode(args, True)
-
-        # Fill the body/headers with credentials
-        uri, raw_headers, body = self._client_auth.prepare(
-            method="POST", uri=introspection_endpoint, headers=raw_headers, body=body
-        )
-        headers = Headers({k: [v] for (k, v) in raw_headers.items()})
-
-        # Do the actual request
-        # We're not using the SimpleHttpClient util methods as we don't want to
-        # check the HTTP status code, and we do the body encoding ourselves.
-        response = await self._http_client.request(
-            method="POST",
-            uri=uri,
-            data=body.encode("utf-8"),
-            headers=headers,
-        )
-
-        resp_body = await make_deferred_yieldable(readBody(response))
-
-        if response.code < 200 or response.code >= 300:
-            raise HttpResponseException(
-                response.code,
-                response.phrase.decode("ascii", errors="replace"),
-                resp_body,
-            )
-
-        resp = json_decoder.decode(resp_body.decode("utf-8"))
-
-        if not isinstance(resp, dict):
-            raise ValueError(
-                "The introspection endpoint returned an invalid JSON response."
-            )
-
-        return IntrospectionToken(**resp)
-
-    async def is_server_admin(self, requester: Requester) -> bool:
-        return "urn:synapse:admin:*" in requester.scope
-
-    async def get_user_by_req(
-        self,
-        request: SynapseRequest,
-        allow_guest: bool = False,
-        allow_expired: bool = False,
-    ) -> Requester:
-        access_token = self.get_access_token_from_request(request)
-
-        requester = await self.get_appservice_user(request, access_token)
-        if not requester:
-            # TODO: we probably want to assert the allow_guest inside this call
-            # so that we don't provision the user if they don't have enough permission:
-            requester = await self.get_user_by_access_token(access_token, allow_expired)
-
-        if not allow_guest and requester.is_guest:
-            raise OAuthInsufficientScopeError([SCOPE_MATRIX_API])
-
-        request.requester = requester
-
-        return requester
-
-    async def get_user_by_access_token(
-        self,
-        token: str,
-        allow_expired: bool = False,
-    ) -> Requester:
-        if self._admin_token is not None and token == self._admin_token:
-            # XXX: This is a temporary solution so that the admin API can be called by
-            # the OIDC provider. This will be removed once we have OIDC client
-            # credentials grant support in matrix-authentication-service.
-            logging.info("Admin toked used")
-            # XXX: that user doesn't exist and won't be provisioned.
-            # This is mostly fine for admin calls, but we should also think about doing
-            # requesters without a user_id.
-            admin_user = UserID("__oidc_admin", self._hostname)
-            return create_requester(
-                user_id=admin_user,
-                scope=["urn:synapse:admin:*"],
-            )
-
-        try:
-            introspection_result = await self._introspect_token(token)
-        except Exception:
-            logger.exception("Failed to introspect token")
-            raise SynapseError(503, "Unable to introspect the access token")
-
-        logger.info(f"Introspection result: {introspection_result!r}")
-
-        # TODO: introspection verification should be more extensive, especially:
-        #   - verify the audience
-        if not introspection_result.get("active"):
-            raise InvalidClientTokenError("Token is not active")
-
-        # Let's look at the scope
-        scope: List[str] = scope_to_list(introspection_result.get("scope", ""))
-
-        # Determine type of user based on presence of particular scopes
-        has_user_scope = SCOPE_MATRIX_API in scope
-        has_guest_scope = SCOPE_MATRIX_GUEST in scope
-
-        if not has_user_scope and not has_guest_scope:
-            raise InvalidClientTokenError("No scope in token granting user rights")
-
-        # Match via the sub claim
-        sub: Optional[str] = introspection_result.get("sub")
-        if sub is None:
-            raise InvalidClientTokenError(
-                "Invalid sub claim in the introspection result"
-            )
-
-        user_id_str = await self.store.get_user_by_external_id(
-            MSC3861DelegatedAuth.EXTERNAL_ID_PROVIDER, sub
-        )
-        if user_id_str is None:
-            # If we could not find a user via the external_id, it either does not exist,
-            # or the external_id was never recorded
-
-            # TODO: claim mapping should be configurable
-            username: Optional[str] = introspection_result.get("username")
-            if username is None or not isinstance(username, str):
-                raise AuthError(
-                    500,
-                    "Invalid username claim in the introspection result",
-                )
-            user_id = UserID(username, self._hostname)
-
-            # First try to find a user from the username claim
-            user_info = await self.store.get_userinfo_by_id(user_id=user_id.to_string())
-            if user_info is None:
-                # If the user does not exist, we should create it on the fly
-                # TODO: we could use SCIM to provision users ahead of time and listen
-                # for SCIM SET events if those ever become standard:
-                # https://datatracker.ietf.org/doc/html/draft-hunt-scim-notify-00
-
-                # TODO: claim mapping should be configurable
-                # If present, use the name claim as the displayname
-                name: Optional[str] = introspection_result.get("name")
-
-                await self.store.register_user(
-                    user_id=user_id.to_string(), create_profile_with_displayname=name
-                )
-
-            # And record the sub as external_id
-            await self.store.record_user_external_id(
-                MSC3861DelegatedAuth.EXTERNAL_ID_PROVIDER, sub, user_id.to_string()
-            )
-        else:
-            user_id = UserID.from_string(user_id_str)
-
-        # Find device_ids in scope
-        # We only allow a single device_id in the scope, so we find them all in the
-        # scope list, and raise if there are more than one. The OIDC server should be
-        # the one enforcing valid scopes, so we raise a 500 if we find an invalid scope.
-        device_ids = [
-            tok[len(SCOPE_MATRIX_DEVICE_PREFIX) :]
-            for tok in scope
-            if tok.startswith(SCOPE_MATRIX_DEVICE_PREFIX)
-        ]
-
-        if len(device_ids) > 1:
-            raise AuthError(
-                500,
-                "Multiple device IDs in scope",
-            )
-
-        device_id = device_ids[0] if device_ids else None
-        if device_id is not None:
-            # Sanity check the device_id
-            if len(device_id) > 255 or len(device_id) < 1:
-                raise AuthError(
-                    500,
-                    "Invalid device ID in scope",
-                )
-
-            # Create the device on the fly if it does not exist
-            try:
-                await self.store.get_device(
-                    user_id=user_id.to_string(), device_id=device_id
-                )
-            except StoreError:
-                await self.store.store_device(
-                    user_id=user_id.to_string(),
-                    device_id=device_id,
-                    initial_device_display_name="OIDC-native client",
-                )
-
-        # TODO: there is a few things missing in the requester here, which still need
-        # to be figured out, like:
-        #   - impersonation, with the `authenticated_entity`, which is used for
-        #     rate-limiting, MAU limits, etc.
-        #   - shadow-banning, with the `shadow_banned` flag
-        #   - a proper solution for appservices, which still needs to be figured out in
-        #     the context of MSC3861
-        return create_requester(
-            user_id=user_id,
-            device_id=device_id,
-            scope=scope,
-            is_guest=(has_guest_scope and not has_user_scope),
-        )

synapse/api/constants.py

@@ -123,6 +123,10 @@ class EventTypes:
     SpaceChild: Final = "m.space.child"
     SpaceParent: Final = "m.space.parent"
 
+    MSC2716_INSERTION: Final = "org.matrix.msc2716.insertion"
+    MSC2716_BATCH: Final = "org.matrix.msc2716.batch"
+    MSC2716_MARKER: Final = "org.matrix.msc2716.marker"
+
     Reaction: Final = "m.reaction"
 
 
@@ -218,11 +222,21 @@ class EventContentFields:
     # Used in m.room.guest_access events.
     GUEST_ACCESS: Final = "guest_access"
 
+    # Used on normal messages to indicate they were historically imported after the fact
+    MSC2716_HISTORICAL: Final = "org.matrix.msc2716.historical"
+    # For "insertion" events to indicate what the next batch ID should be in
+    # order to connect to it
+    MSC2716_NEXT_BATCH_ID: Final = "next_batch_id"
+    # Used on "batch" events to indicate which insertion event it connects to
+    MSC2716_BATCH_ID: Final = "batch_id"
+    # For "marker" events
+    MSC2716_INSERTION_EVENT_REFERENCE: Final = "insertion_event_reference"
+
     # The authorising user for joining a restricted room.
     AUTHORISING_USER: Final = "join_authorised_via_users_server"
 
     # Use for mentioning users.
-    MENTIONS: Final = "m.mentions"
+    MSC3952_MENTIONS: Final = "org.matrix.msc3952.mentions"
 
     # an unspecced field added to to-device messages to identify them uniquely-ish
     TO_DEVICE_MSGID: Final = "org.matrix.msgid"

synapse/api/errors.py

@@ -119,20 +119,14 @@ class Codes(str, Enum):
 
 
 class CodeMessageException(RuntimeError):
-    """An exception with integer code, a message string attributes and optional headers.
+    """An exception with integer code and message string attributes.
 
     Attributes:
         code: HTTP error code
         msg: string describing the error
-        headers: optional response headers to send
     """
 
-    def __init__(
-        self,
-        code: Union[int, HTTPStatus],
-        msg: str,
-        headers: Optional[Dict[str, str]] = None,
-    ):
+    def __init__(self, code: Union[int, HTTPStatus], msg: str):
         super().__init__("%d: %s" % (code, msg))
 
         # Some calls to this method pass instances of http.HTTPStatus for `code`.
@@ -143,7 +137,6 @@ class CodeMessageException(RuntimeError):
         # To eliminate this behaviour, we convert them to their integer equivalents here.
         self.code = int(code)
         self.msg = msg
-        self.headers = headers
 
 
 class RedirectException(CodeMessageException):
@@ -189,7 +182,6 @@ class SynapseError(CodeMessageException):
         msg: str,
         errcode: str = Codes.UNKNOWN,
         additional_fields: Optional[Dict] = None,
-        headers: Optional[Dict[str, str]] = None,
     ):
         """Constructs a synapse error.
 
@@ -198,7 +190,7 @@ class SynapseError(CodeMessageException):
             msg: The human-readable error message.
             errcode: The matrix error code e.g 'M_FORBIDDEN'
         """
-        super().__init__(code, msg, headers)
+        super().__init__(code, msg)
         self.errcode = errcode
         if additional_fields is None:
             self._additional_fields: Dict = {}
@@ -343,20 +335,6 @@ class AuthError(SynapseError):
         super().__init__(code, msg, errcode, additional_fields)
 
 
-class OAuthInsufficientScopeError(SynapseError):
-    """An error raised when the caller does not have sufficient scope to perform the requested action"""
-
-    def __init__(
-        self,
-        required_scopes: List[str],
-    ):
-        headers = {
-            "WWW-Authenticate": 'Bearer error="insufficient_scope", scope="%s"'
-            % (" ".join(required_scopes))
-        }
-        super().__init__(401, "Insufficient scope", Codes.FORBIDDEN, None, headers)
-
-
 class UnstableSpecAuthError(AuthError):
     """An error raised when a new error code is being proposed to replace a previous one.
     This error will return a "org.matrix.unstable.errcode" property with the new error code,

synapse/api/filtering.py

@@ -152,9 +152,9 @@ class Filtering:
         self.DEFAULT_FILTER_COLLECTION = FilterCollection(hs, {})
 
     async def get_user_filter(
-        self, user_id: UserID, filter_id: Union[int, str]
+        self, user_localpart: str, filter_id: Union[int, str]
     ) -> "FilterCollection":
-        result = await self.store.get_user_filter(user_id, filter_id)
+        result = await self.store.get_user_filter(user_localpart, filter_id)
         return FilterCollection(self._hs, result)
 
     def add_user_filter(self, user_id: UserID, user_filter: JsonDict) -> Awaitable[int]:

synapse/api/room_versions.py

@@ -91,6 +91,11 @@ class RoomVersion:
     # MSC2403: Allows join_rules to be set to 'knock', changes auth rules to allow sending
     # m.room.membership event with membership 'knock'.
     msc2403_knocking: bool
+    # MSC2716: Adds m.room.power_levels -> content.historical field to control
+    # whether "insertion", "chunk", "marker" events can be sent
+    msc2716_historical: bool
+    # MSC2716: Adds support for redacting "insertion", "chunk", and "marker" events
+    msc2716_redactions: bool
     # MSC3389: Protect relation information from redaction.
     msc3389_relation_redactions: bool
     # MSC3787: Adds support for a `knock_restricted` join rule, mixing concepts of
@@ -125,6 +130,8 @@ class RoomVersions:
         msc3083_join_rules=False,
         msc3375_redaction_rules=False,
         msc2403_knocking=False,
+        msc2716_historical=False,
+        msc2716_redactions=False,
         msc3389_relation_redactions=False,
         msc3787_knock_restricted_join_rule=False,
         msc3667_int_only_power_levels=False,
@@ -146,6 +153,8 @@ class RoomVersions:
         msc3083_join_rules=False,
         msc3375_redaction_rules=False,
         msc2403_knocking=False,
+        msc2716_historical=False,
+        msc2716_redactions=False,
         msc3389_relation_redactions=False,
         msc3787_knock_restricted_join_rule=False,
         msc3667_int_only_power_levels=False,
@@ -167,6 +176,8 @@ class RoomVersions:
         msc3083_join_rules=False,
         msc3375_redaction_rules=False,
         msc2403_knocking=False,
+        msc2716_historical=False,
+        msc2716_redactions=False,
         msc3389_relation_redactions=False,
         msc3787_knock_restricted_join_rule=False,
         msc3667_int_only_power_levels=False,
@@ -188,6 +199,8 @@ class RoomVersions:
         msc3083_join_rules=False,
         msc3375_redaction_rules=False,
         msc2403_knocking=False,
+        msc2716_historical=False,
+        msc2716_redactions=False,
         msc3389_relation_redactions=False,
         msc3787_knock_restricted_join_rule=False,
         msc3667_int_only_power_levels=False,
@@ -209,6 +222,8 @@ class RoomVersions:
         msc3083_join_rules=False,
         msc3375_redaction_rules=False,
         msc2403_knocking=False,
+        msc2716_historical=False,
+        msc2716_redactions=False,
         msc3389_relation_redactions=False,
         msc3787_knock_restricted_join_rule=False,
         msc3667_int_only_power_levels=False,
@@ -230,6 +245,8 @@ class RoomVersions:
         msc3083_join_rules=False,
         msc3375_redaction_rules=False,
         msc2403_knocking=False,
+        msc2716_historical=False,
+        msc2716_redactions=False,
         msc3389_relation_redactions=False,
         msc3787_knock_restricted_join_rule=False,
         msc3667_int_only_power_levels=False,
@@ -251,6 +268,8 @@ class RoomVersions:
         msc3083_join_rules=False,
         msc3375_redaction_rules=False,
         msc2403_knocking=False,
+        msc2716_historical=False,
+        msc2716_redactions=False,
         msc3389_relation_redactions=False,
         msc3787_knock_restricted_join_rule=False,
         msc3667_int_only_power_levels=False,
@@ -272,6 +291,8 @@ class RoomVersions:
         msc3083_join_rules=False,
         msc3375_redaction_rules=False,
         msc2403_knocking=True,
+        msc2716_historical=False,
+        msc2716_redactions=False,
         msc3389_relation_redactions=False,
         msc3787_knock_restricted_join_rule=False,
         msc3667_int_only_power_levels=False,
@@ -293,6 +314,8 @@ class RoomVersions:
         msc3083_join_rules=True,
         msc3375_redaction_rules=False,
         msc2403_knocking=True,
+        msc2716_historical=False,
+        msc2716_redactions=False,
         msc3389_relation_redactions=False,
         msc3787_knock_restricted_join_rule=False,
         msc3667_int_only_power_levels=False,
@@ -314,6 +337,8 @@ class RoomVersions:
         msc3083_join_rules=True,
         msc3375_redaction_rules=True,
         msc2403_knocking=True,
+        msc2716_historical=False,
+        msc2716_redactions=False,
         msc3389_relation_redactions=False,
         msc3787_knock_restricted_join_rule=False,
         msc3667_int_only_power_levels=False,
@@ -335,6 +360,8 @@ class RoomVersions:
         msc3083_join_rules=True,
         msc3375_redaction_rules=True,
         msc2403_knocking=True,
+        msc2716_historical=False,
+        msc2716_redactions=False,
         msc3389_relation_redactions=False,
         msc3787_knock_restricted_join_rule=True,
         msc3667_int_only_power_levels=False,
@@ -356,6 +383,8 @@ class RoomVersions:
         msc3083_join_rules=True,
         msc3375_redaction_rules=True,
         msc2403_knocking=True,
+        msc2716_historical=False,
+        msc2716_redactions=False,
         msc3389_relation_redactions=False,
         msc3787_knock_restricted_join_rule=False,
         msc3667_int_only_power_levels=False,
@@ -377,6 +406,8 @@ class RoomVersions:
         msc3083_join_rules=True,
         msc3375_redaction_rules=True,
         msc2403_knocking=True,
+        msc2716_historical=False,
+        msc2716_redactions=False,
         msc3389_relation_redactions=False,
         msc3787_knock_restricted_join_rule=True,
         msc3667_int_only_power_levels=True,
@@ -384,6 +415,29 @@ class RoomVersions:
         msc3931_push_features=(),
         msc3989_redaction_rules=False,
     )
+    MSC2716v4 = RoomVersion(
+        "org.matrix.msc2716v4",
+        RoomDisposition.UNSTABLE,
+        EventFormatVersions.ROOM_V4_PLUS,
+        StateResolutionVersions.V2,
+        enforce_key_validity=True,
+        special_case_aliases_auth=False,
+        strict_canonicaljson=True,
+        limit_notifications_power_levels=True,
+        msc2175_implicit_room_creator=False,
+        msc2176_redaction_rules=False,
+        msc3083_join_rules=False,
+        msc3375_redaction_rules=False,
+        msc2403_knocking=True,
+        msc2716_historical=True,
+        msc2716_redactions=True,
+        msc3389_relation_redactions=False,
+        msc3787_knock_restricted_join_rule=False,
+        msc3667_int_only_power_levels=False,
+        msc3821_redaction_rules=False,
+        msc3931_push_features=(),
+        msc3989_redaction_rules=False,
+    )
     MSC1767v10 = RoomVersion(
         # MSC1767 (Extensible Events) based on room version "10"
         "org.matrix.msc1767.10",
@@ -399,6 +453,8 @@ class RoomVersions:
         msc3083_join_rules=True,
         msc3375_redaction_rules=True,
         msc2403_knocking=True,
+        msc2716_historical=False,
+        msc2716_redactions=False,
         msc3389_relation_redactions=False,
         msc3787_knock_restricted_join_rule=True,
         msc3667_int_only_power_levels=True,
@@ -420,6 +476,8 @@ class RoomVersions:
         msc3083_join_rules=True,
         msc3375_redaction_rules=True,
         msc2403_knocking=True,
+        msc2716_historical=False,
+        msc2716_redactions=False,
         msc3389_relation_redactions=False,
         msc3787_knock_restricted_join_rule=True,
         msc3667_int_only_power_levels=True,
@@ -442,6 +500,8 @@ class RoomVersions:
         msc3083_join_rules=True,
         msc3375_redaction_rules=True,
         msc2403_knocking=True,
+        msc2716_historical=False,
+        msc2716_redactions=False,
         msc3389_relation_redactions=False,
         msc3787_knock_restricted_join_rule=True,
         msc3667_int_only_power_levels=True,
@@ -466,6 +526,7 @@ KNOWN_ROOM_VERSIONS: Dict[str, RoomVersion] = {
         RoomVersions.V9,
         RoomVersions.MSC3787,
         RoomVersions.V10,
+        RoomVersions.MSC2716v4,
         RoomVersions.MSC3989,
         RoomVersions.MSC3820opt2,
     )

synapse/app/generic_worker.py

@@ -83,6 +83,7 @@ from synapse.storage.databases.main.receipts import ReceiptsWorkerStore
 from synapse.storage.databases.main.registration import RegistrationWorkerStore
 from synapse.storage.databases.main.relations import RelationsWorkerStore
 from synapse.storage.databases.main.room import RoomWorkerStore
+from synapse.storage.databases.main.room_batch import RoomBatchStore
 from synapse.storage.databases.main.roommember import RoomMemberWorkerStore
 from synapse.storage.databases.main.search import SearchStore
 from synapse.storage.databases.main.session import SessionStore
@@ -119,6 +120,7 @@ class GenericWorkerStore(
     # the races it creates aren't too bad.
     KeyStore,
     RoomWorkerStore,
+    RoomBatchStore,
     DirectoryWorkerStore,
     PushRulesWorkerStore,
     ApplicationServiceTransactionWorkerStore,

synapse/config/auth.py

@@ -29,14 +29,7 @@ class AuthConfig(Config):
         if password_config is None:
             password_config = {}
 
-        # The default value of password_config.enabled is True, unless msc3861 is enabled.
-        msc3861_enabled = (
-            (config.get("experimental_features") or {})
-            .get("msc3861", {})
-            .get("enabled", False)
-        )
-        passwords_enabled = password_config.get("enabled", not msc3861_enabled)
-
+        passwords_enabled = password_config.get("enabled", True)
         # 'only_for_reauth' allows users who have previously set a password to use it,
         # even though passwords would otherwise be disabled.
         passwords_for_reauth_only = passwords_enabled == "only_for_reauth"
@@ -60,13 +53,3 @@ class AuthConfig(Config):
         self.ui_auth_session_timeout = self.parse_duration(
             ui_auth.get("session_timeout", 0)
         )
-
-        # Logging in with an existing session.
-        login_via_existing = config.get("login_via_existing_session", {})
-        self.login_via_existing_enabled = login_via_existing.get("enabled", False)
-        self.login_via_existing_require_ui_auth = login_via_existing.get(
-            "require_ui_auth", True
-        )
-        self.login_via_existing_token_timeout = self.parse_duration(
-            login_via_existing.get("token_timeout", "5m")
-        )

synapse/config/experimental.py

@@ -12,216 +12,15 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import enum
-from typing import TYPE_CHECKING, Any, Optional
+from typing import Any, Optional
 
 import attr
-import attr.validators
 
 from synapse.api.room_versions import KNOWN_ROOM_VERSIONS, RoomVersions
 from synapse.config import ConfigError
-from synapse.config._base import Config, RootConfig
+from synapse.config._base import Config
 from synapse.types import JsonDict
 
-# Determine whether authlib is installed.
-try:
-    import authlib  # noqa: F401
-
-    HAS_AUTHLIB = True
-except ImportError:
-    HAS_AUTHLIB = False
-
-if TYPE_CHECKING:
-    # Only import this if we're type checking, as it might not be installed at runtime.
-    from authlib.jose.rfc7517 import JsonWebKey
-
-
-class ClientAuthMethod(enum.Enum):
-    """List of supported client auth methods."""
-
-    CLIENT_SECRET_POST = "client_secret_post"
-    CLIENT_SECRET_BASIC = "client_secret_basic"
-    CLIENT_SECRET_JWT = "client_secret_jwt"
-    PRIVATE_KEY_JWT = "private_key_jwt"
-
-
-def _parse_jwks(jwks: Optional[JsonDict]) -> Optional["JsonWebKey"]:
-    """A helper function to parse a JWK dict into a JsonWebKey."""
-
-    if jwks is None:
-        return None
-
-    from authlib.jose.rfc7517 import JsonWebKey
-
-    return JsonWebKey.import_key(jwks)
-
-
-@attr.s(slots=True, frozen=True)
-class MSC3861:
-    """Configuration for MSC3861: Matrix architecture change to delegate authentication via OIDC"""
-
-    enabled: bool = attr.ib(default=False, validator=attr.validators.instance_of(bool))
-    """Whether to enable MSC3861 auth delegation."""
-
-    @enabled.validator
-    def _check_enabled(self, attribute: attr.Attribute, value: bool) -> None:
-        # Only allow enabling MSC3861 if authlib is installed
-        if value and not HAS_AUTHLIB:
-            raise ConfigError(
-                "MSC3861 is enabled but authlib is not installed. "
-                "Please install authlib to use MSC3861.",
-                ("experimental", "msc3861", "enabled"),
-            )
-
-    issuer: str = attr.ib(default="", validator=attr.validators.instance_of(str))
-    """The URL of the OIDC Provider."""
-
-    issuer_metadata: Optional[JsonDict] = attr.ib(default=None)
-    """The issuer metadata to use, otherwise discovered from /.well-known/openid-configuration as per MSC2965."""
-
-    client_id: str = attr.ib(
-        default="",
-        validator=attr.validators.instance_of(str),
-    )
-    """The client ID to use when calling the introspection endpoint."""
-
-    client_auth_method: ClientAuthMethod = attr.ib(
-        default=ClientAuthMethod.CLIENT_SECRET_POST, converter=ClientAuthMethod
-    )
-    """The auth method used when calling the introspection endpoint."""
-
-    client_secret: Optional[str] = attr.ib(
-        default=None,
-        validator=attr.validators.optional(attr.validators.instance_of(str)),
-    )
-    """
-    The client secret to use when calling the introspection endpoint,
-    when using any of the client_secret_* client auth methods.
-    """
-
-    jwk: Optional["JsonWebKey"] = attr.ib(default=None, converter=_parse_jwks)
-    """
-    The JWKS to use when calling the introspection endpoint,
-    when using the private_key_jwt client auth method.
-    """
-
-    @client_auth_method.validator
-    def _check_client_auth_method(
-        self, attribute: attr.Attribute, value: ClientAuthMethod
-    ) -> None:
-        # Check that the right client credentials are provided for the client auth method.
-        if not self.enabled:
-            return
-
-        if value == ClientAuthMethod.PRIVATE_KEY_JWT and self.jwk is None:
-            raise ConfigError(
-                "A JWKS must be provided when using the private_key_jwt client auth method",
-                ("experimental", "msc3861", "client_auth_method"),
-            )
-
-        if (
-            value
-            in (
-                ClientAuthMethod.CLIENT_SECRET_POST,
-                ClientAuthMethod.CLIENT_SECRET_BASIC,
-                ClientAuthMethod.CLIENT_SECRET_JWT,
-            )
-            and self.client_secret is None
-        ):
-            raise ConfigError(
-                f"A client secret must be provided when using the {value} client auth method",
-                ("experimental", "msc3861", "client_auth_method"),
-            )
-
-    account_management_url: Optional[str] = attr.ib(
-        default=None,
-        validator=attr.validators.optional(attr.validators.instance_of(str)),
-    )
-    """The URL of the My Account page on the OIDC Provider as per MSC2965."""
-
-    admin_token: Optional[str] = attr.ib(
-        default=None,
-        validator=attr.validators.optional(attr.validators.instance_of(str)),
-    )
-    """
-    A token that should be considered as an admin token.
-    This is used by the OIDC provider, to make admin calls to Synapse.
-    """
-
-    def check_config_conflicts(self, root: RootConfig) -> None:
-        """Checks for any configuration conflicts with other parts of Synapse.
-
-        Raises:
-            ConfigError: If there are any configuration conflicts.
-        """
-
-        if not self.enabled:
-            return
-
-        if (
-            root.auth.password_enabled_for_reauth
-            or root.auth.password_enabled_for_login
-        ):
-            raise ConfigError(
-                "Password auth cannot be enabled when OAuth delegation is enabled",
-                ("password_config", "enabled"),
-            )
-
-        if root.registration.enable_registration:
-            raise ConfigError(
-                "Registration cannot be enabled when OAuth delegation is enabled",
-                ("enable_registration",),
-            )
-
-        if (
-            root.oidc.oidc_enabled
-            or root.saml2.saml2_enabled
-            or root.cas.cas_enabled
-            or root.jwt.jwt_enabled
-        ):
-            raise ConfigError("SSO cannot be enabled when OAuth delegation is enabled")
-
-        if bool(root.authproviders.password_providers):
-            raise ConfigError(
-                "Password auth providers cannot be enabled when OAuth delegation is enabled"
-            )
-
-        if root.captcha.enable_registration_captcha:
-            raise ConfigError(
-                "CAPTCHA cannot be enabled when OAuth delegation is enabled",
-                ("captcha", "enable_registration_captcha"),
-            )
-
-        if root.auth.login_via_existing_enabled:
-            raise ConfigError(
-                "Login via existing session cannot be enabled when OAuth delegation is enabled",
-                ("login_via_existing_session", "enabled"),
-            )
-
-        if root.registration.refresh_token_lifetime:
-            raise ConfigError(
-                "refresh_token_lifetime cannot be set when OAuth delegation is enabled",
-                ("refresh_token_lifetime",),
-            )
-
-        if root.registration.nonrefreshable_access_token_lifetime:
-            raise ConfigError(
-                "nonrefreshable_access_token_lifetime cannot be set when OAuth delegation is enabled",
-                ("nonrefreshable_access_token_lifetime",),
-            )
-
-        if root.registration.session_lifetime:
-            raise ConfigError(
-                "session_lifetime cannot be set when OAuth delegation is enabled",
-                ("session_lifetime",),
-            )
-
-        if not root.experimental.msc3970_enabled:
-            raise ConfigError(
-                "experimental_features.msc3970_enabled must be 'true' when OAuth delegation is enabled",
-                ("experimental_features", "msc3970_enabled"),
-            )
-
 
 @attr.s(auto_attribs=True, frozen=True, slots=True)
 class MSC3866Config:
@@ -247,6 +46,9 @@ class ExperimentalConfig(Config):
         # MSC3026 (busy presence state)
         self.msc3026_enabled: bool = experimental.get("msc3026_enabled", False)
 
+        # MSC2716 (importing historical messages)
+        self.msc2716_enabled: bool = experimental.get("msc2716_enabled", False)
+
         # MSC3244 (room version capabilities)
         self.msc3244_enabled: bool = experimental.get("msc3244_enabled", True)
 
@@ -316,6 +118,13 @@ class ExperimentalConfig(Config):
         # MSC3881: Remotely toggle push notifications for another client
         self.msc3881_enabled: bool = experimental.get("msc3881_enabled", False)
 
+        # MSC3882: Allow an existing session to sign in a new session
+        self.msc3882_enabled: bool = experimental.get("msc3882_enabled", False)
+        self.msc3882_ui_auth: bool = experimental.get("msc3882_ui_auth", True)
+        self.msc3882_token_timeout = self.parse_duration(
+            experimental.get("msc3882_token_timeout", "5m")
+        )
+
         # MSC3874: Filtering /messages with rel_types / not_rel_types.
         self.msc3874_enabled: bool = experimental.get("msc3874_enabled", False)
 
@@ -355,6 +164,11 @@ class ExperimentalConfig(Config):
         # MSC3391: Removing account data.
         self.msc3391_enabled = experimental.get("msc3391_enabled", False)
 
+        # MSC3952: Intentional mentions, this depends on MSC3966.
+        self.msc3952_intentional_mentions = experimental.get(
+            "msc3952_intentional_mentions", False
+        )
+
         # MSC3959: Do not generate notifications for edits.
         self.msc3958_supress_edit_notifs = experimental.get(
             "msc3958_supress_edit_notifs", False
@@ -368,19 +182,11 @@ class ExperimentalConfig(Config):
             "msc3981_recurse_relations", False
         )
 
-        # MSC3861: Matrix architecture change to delegate authentication via OIDC
-        try:
-            self.msc3861 = MSC3861(**experimental.get("msc3861", {}))
-        except ValueError as exc:
-            raise ConfigError(
-                "Invalid MSC3861 configuration", ("experimental", "msc3861")
-            ) from exc
-
         # MSC3970: Scope transaction IDs to devices
-        self.msc3970_enabled = experimental.get("msc3970_enabled", self.msc3861.enabled)
+        self.msc3970_enabled = experimental.get("msc3970_enabled", False)
 
-        # Check that none of the other config options conflict with MSC3861 when enabled
-        self.msc3861.check_config_conflicts(self.root)
+        # MSC4009: E.164 Matrix IDs
+        self.msc4009_e164_mxids = experimental.get("msc4009_e164_mxids", False)
 
         # MSC4010: Do not allow setting m.push_rules account data.
         self.msc4010_push_rules_account_data = experimental.get(

synapse/config/federation.py

@@ -22,8 +22,6 @@ class FederationConfig(Config):
     section = "federation"
 
     def read_config(self, config: JsonDict, **kwargs: Any) -> None:
-        federation_config = config.setdefault("federation", {})
-
         # FIXME: federation_domain_whitelist needs sytests
         self.federation_domain_whitelist: Optional[dict] = None
         federation_domain_whitelist = config.get("federation_domain_whitelist", None)
@@ -51,19 +49,5 @@ class FederationConfig(Config):
             "allow_device_name_lookup_over_federation", False
         )
 
-        # Allow for the configuration of timeout, max request retries
-        # and min/max retry delays in the matrix federation client.
-        self.client_timeout_ms = Config.parse_duration(
-            federation_config.get("client_timeout", "60s")
-        )
-        self.max_long_retry_delay_ms = Config.parse_duration(
-            federation_config.get("max_long_retry_delay", "60s")
-        )
-        self.max_short_retry_delay_ms = Config.parse_duration(
-            federation_config.get("max_short_retry_delay", "2s")
-        )
-        self.max_long_retries = federation_config.get("max_long_retries", 10)
-        self.max_short_retries = federation_config.get("max_short_retries", 3)
-
 
 _METRICS_FOR_DOMAINS_SCHEMA = {"type": "array", "items": {"type": "string"}}

synapse/config/workers.py

@@ -41,17 +41,11 @@ Synapse version. Please use ``%s: name_of_worker`` instead.
 
 _MISSING_MAIN_PROCESS_INSTANCE_MAP_DATA = """
 Missing data for a worker to connect to main process. Please include '%s' in the
-`instance_map` declared in your shared yaml configuration as defined in configuration
-documentation here:
-`https://matrix-org.github.io/synapse/latest/usage/configuration/config_documentation.html#instance_map`
-"""
-
-WORKER_REPLICATION_SETTING_DEPRECATED_MESSAGE = """
-'%s' is no longer a supported worker setting, please place '%s' onto your shared
-configuration under `main` inside the `instance_map`. See workers documentation here:
+`instance_map` declared in your shared yaml configuration, or optionally(as a deprecated
+solution) in every worker's yaml as various `worker_replication_*` settings as defined
+in workers documentation here:
 `https://matrix-org.github.io/synapse/latest/workers.html#worker-configuration`
 """
-
 # This allows for a handy knob when it's time to change from 'master' to
 # something with less 'history'
 MAIN_PROCESS_INSTANCE_NAME = "master"
@@ -94,7 +88,7 @@ class ConfigModel(BaseModel):
         allow_mutation = False
 
 
-class InstanceTcpLocationConfig(ConfigModel):
+class InstanceLocationConfig(ConfigModel):
     """The host and port to talk to an instance via HTTP replication."""
 
     host: StrictStr
@@ -110,23 +104,6 @@ class InstanceTcpLocationConfig(ConfigModel):
         return f"{self.host}:{self.port}"
 
 
-class InstanceUnixLocationConfig(ConfigModel):
-    """The socket file to talk to an instance via HTTP replication."""
-
-    path: StrictStr
-
-    def scheme(self) -> str:
-        """Hardcode a retrievable scheme"""
-        return "unix"
-
-    def netloc(self) -> str:
-        """Nicely format the address location data"""
-        return f"{self.path}"
-
-
-InstanceLocationConfig = Union[InstanceTcpLocationConfig, InstanceUnixLocationConfig]
-
-
 @attr.s
 class WriterLocations:
     """Specifies the instances that write various streams.
@@ -239,37 +216,22 @@ class WorkerConfig(Config):
         )
 
         # A map from instance name to host/port of their HTTP replication endpoint.
-        # Check if the main process is declared. The main process itself doesn't need
-        # this data as it would never have to talk to itself.
+        # Check if the main process is declared. Inject it into the map if it's not,
+        # based first on if a 'main' block is declared then on 'worker_replication_*'
+        # data. If both are available, default to instance_map. The main process
+        # itself doesn't need this data as it would never have to talk to itself.
         instance_map: Dict[str, Any] = config.get("instance_map", {})
 
         if self.instance_name is not MAIN_PROCESS_INSTANCE_NAME:
-            # TODO: The next 3 condition blocks can be deleted after some time has
-            #  passed and we're ready to stop checking for these settings.
             # The host used to connect to the main synapse
             main_host = config.get("worker_replication_host", None)
-            if main_host:
-                raise ConfigError(
-                    WORKER_REPLICATION_SETTING_DEPRECATED_MESSAGE
-                    % ("worker_replication_host", main_host)
-                )
 
             # The port on the main synapse for HTTP replication endpoint
             main_port = config.get("worker_replication_http_port")
-            if main_port:
-                raise ConfigError(
-                    WORKER_REPLICATION_SETTING_DEPRECATED_MESSAGE
-                    % ("worker_replication_http_port", main_port)
-                )
 
             # The tls mode on the main synapse for HTTP replication endpoint.
             # For backward compatibility this defaults to False.
             main_tls = config.get("worker_replication_http_tls", False)
-            if main_tls:
-                raise ConfigError(
-                    WORKER_REPLICATION_SETTING_DEPRECATED_MESSAGE
-                    % ("worker_replication_http_tls", main_tls)
-                )
 
             # For now, accept 'main' in the instance_map, but the replication system
             # expects 'master', force that into being until it's changed later.
@@ -279,20 +241,30 @@ class WorkerConfig(Config):
                 ]
                 del instance_map[MAIN_PROCESS_INSTANCE_MAP_NAME]
 
+            # This is the backwards compatibility bit that handles the
+            # worker_replication_* bits using setdefault() to not overwrite anything.
+            elif main_host is not None and main_port is not None:
+                instance_map.setdefault(
+                    MAIN_PROCESS_INSTANCE_NAME,
+                    {
+                        "host": main_host,
+                        "port": main_port,
+                        "tls": main_tls,
+                    },
+                )
+
             else:
                 # If we've gotten here, it means that the main process is not on the
-                # instance_map.
+                # instance_map and that not enough worker_replication_* variables
+                # were declared in the worker's yaml.
                 raise ConfigError(
                     _MISSING_MAIN_PROCESS_INSTANCE_MAP_DATA
                     % MAIN_PROCESS_INSTANCE_MAP_NAME
                 )
 
-        # type-ignore: the expression `Union[A, B]` is not a Type[Union[A, B]] currently
         self.instance_map: Dict[
             str, InstanceLocationConfig
-        ] = parse_and_validate_mapping(
-            instance_map, InstanceLocationConfig  # type: ignore[arg-type]
-        )
+        ] = parse_and_validate_mapping(instance_map, InstanceLocationConfig)
 
         # Map from type of streams to source, c.f. WriterLocations.
         writers = config.get("stream_writers") or {}

synapse/event_auth.py

@@ -339,6 +339,13 @@ def check_state_dependent_auth_rules(
     if event.type == EventTypes.Redaction:
         check_redaction(event.room_version, event, auth_dict)
 
+    if (
+        event.type == EventTypes.MSC2716_INSERTION
+        or event.type == EventTypes.MSC2716_BATCH
+        or event.type == EventTypes.MSC2716_MARKER
+    ):
+        check_historical(event.room_version, event, auth_dict)
+
     logger.debug("Allowing! %s", event)
 
 
@@ -358,6 +365,7 @@ LENIENT_EVENT_BYTE_LIMITS_ROOM_VERSIONS = {
     RoomVersions.V9,
     RoomVersions.MSC3787,
     RoomVersions.V10,
+    RoomVersions.MSC2716v4,
     RoomVersions.MSC1767v10,
 }
 
@@ -815,6 +823,38 @@ def check_redaction(
     raise AuthError(403, "You don't have permission to redact events")
 
 
+def check_historical(
+    room_version_obj: RoomVersion,
+    event: "EventBase",
+    auth_events: StateMap["EventBase"],
+) -> None:
+    """Check whether the event sender is allowed to send historical related
+    events like "insertion", "batch", and "marker".
+
+    Returns:
+        None
+
+    Raises:
+        AuthError if the event sender is not allowed to send historical related events
+        ("insertion", "batch", and "marker").
+    """
+    # Ignore the auth checks in room versions that do not support historical
+    # events
+    if not room_version_obj.msc2716_historical:
+        return
+
+    user_level = get_user_power_level(event.user_id, auth_events)
+
+    historical_level = get_named_level(auth_events, "historical", 100)
+
+    if user_level < historical_level:
+        raise UnstableSpecAuthError(
+            403,
+            'You don\'t have permission to send send historical related events ("insertion", "batch", and "marker")',
+            errcode=Codes.INSUFFICIENT_POWER,
+        )
+
+
 def _check_power_levels(
     room_version_obj: RoomVersion,
     event: "EventBase",

synapse/events/__init__.py

@@ -198,6 +198,7 @@ class _EventInternalMetadata:
     soft_failed: DictProperty[bool] = DictProperty("soft_failed")
     proactively_send: DictProperty[bool] = DictProperty("proactively_send")
     redacted: DictProperty[bool] = DictProperty("redacted")
+    historical: DictProperty[bool] = DictProperty("historical")
 
     txn_id: DictProperty[str] = DictProperty("txn_id")
     """The transaction ID, if it was set when the event was created."""
@@ -287,6 +288,14 @@ class _EventInternalMetadata:
         """
         return self._dict.get("redacted", False)
 
+    def is_historical(self) -> bool:
+        """Whether this is a historical message.
+        This is used by the batchsend historical message endpoint and
+        is needed to and mark the event as backfilled and skip some checks
+        like push notifications.
+        """
+        return self._dict.get("historical", False)
+
     def is_notifiable(self) -> bool:
         """Whether this event can trigger a push notification"""
         return not self.is_outlier() or self.is_out_of_band_membership()

synapse/events/snapshot.py

@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 from abc import ABC, abstractmethod
-from typing import TYPE_CHECKING, Dict, List, Optional, Tuple
+from typing import TYPE_CHECKING, List, Optional, Tuple
 
 import attr
 from immutabledict import immutabledict
@@ -107,32 +107,33 @@ class EventContext(UnpersistedEventContextBase):
         state_delta_due_to_event: If `state_group` and `state_group_before_event` are not None
             then this is the delta of the state between the two groups.
 
-        state_group_deltas: If not empty, this is a dict collecting a mapping of the state
-            difference between state groups.
+        prev_group: If it is known, ``state_group``'s prev_group. Note that this being
+            None does not necessarily mean that ``state_group`` does not have
+            a prev_group!
 
-            The keys are a tuple of two integers: the initial group and final state group.
-            The corresponding value is a state map representing the state delta between
-            these state groups.
+            If the event is a state event, this is normally the same as
+            ``state_group_before_event``.
 
-            The dictionary is expected to have at most two entries with state groups of:
+            If ``state_group`` is None (ie, the event is an outlier), ``prev_group``
+            will always also be ``None``.
 
-            1. The state group before the event and after the event.
-            2. The state group preceding the state group before the event and the
-               state group before the event.
+            Note that this *not* (necessarily) the state group associated with
+            ``_prev_state_ids``.
 
-            This information is collected and stored as part of an optimization for persisting
-            events.
+        delta_ids: If ``prev_group`` is not None, the state delta between ``prev_group``
+            and ``state_group``.
 
         partial_state: if True, we may be storing this event with a temporary,
             incomplete state.
     """
 
     _storage: "StorageControllers"
-    state_group_deltas: Dict[Tuple[int, int], StateMap[str]]
     rejected: Optional[str] = None
     _state_group: Optional[int] = None
     state_group_before_event: Optional[int] = None
     _state_delta_due_to_event: Optional[StateMap[str]] = None
+    prev_group: Optional[int] = None
+    delta_ids: Optional[StateMap[str]] = None
     app_service: Optional[ApplicationService] = None
 
     partial_state: bool = False
@@ -144,14 +145,16 @@ class EventContext(UnpersistedEventContextBase):
         state_group_before_event: Optional[int],
         state_delta_due_to_event: Optional[StateMap[str]],
         partial_state: bool,
-        state_group_deltas: Dict[Tuple[int, int], StateMap[str]],
+        prev_group: Optional[int] = None,
+        delta_ids: Optional[StateMap[str]] = None,
     ) -> "EventContext":
         return EventContext(
             storage=storage,
             state_group=state_group,
             state_group_before_event=state_group_before_event,
             state_delta_due_to_event=state_delta_due_to_event,
-            state_group_deltas=state_group_deltas,
+            prev_group=prev_group,
+            delta_ids=delta_ids,
             partial_state=partial_state,
         )
 
@@ -160,7 +163,7 @@ class EventContext(UnpersistedEventContextBase):
         storage: "StorageControllers",
     ) -> "EventContext":
         """Return an EventContext instance suitable for persisting an outlier event"""
-        return EventContext(storage=storage, state_group_deltas={})
+        return EventContext(storage=storage)
 
     async def persist(self, event: EventBase) -> "EventContext":
         return self
@@ -180,15 +183,13 @@ class EventContext(UnpersistedEventContextBase):
             "state_group": self._state_group,
             "state_group_before_event": self.state_group_before_event,
             "rejected": self.rejected,
-            "state_group_deltas": _encode_state_group_delta(self.state_group_deltas),
+            "prev_group": self.prev_group,
             "state_delta_due_to_event": _encode_state_dict(
                 self._state_delta_due_to_event
             ),
+            "delta_ids": _encode_state_dict(self.delta_ids),
             "app_service_id": self.app_service.id if self.app_service else None,
             "partial_state": self.partial_state,
-            # add dummy delta_ids and prev_group for backwards compatibility
-            "delta_ids": None,
-            "prev_group": None,
         }
 
     @staticmethod
@@ -203,24 +204,17 @@ class EventContext(UnpersistedEventContextBase):
         Returns:
             The event context.
         """
-        # workaround for backwards/forwards compatibility: if the input doesn't have a value
-        # for "state_group_deltas" just assign an empty dict
-        state_group_deltas = input.get("state_group_deltas", None)
-        if state_group_deltas:
-            state_group_deltas = _decode_state_group_delta(state_group_deltas)
-        else:
-            state_group_deltas = {}
-
         context = EventContext(
             # We use the state_group and prev_state_id stuff to pull the
             # current_state_ids out of the DB and construct prev_state_ids.
             storage=storage,
             state_group=input["state_group"],
             state_group_before_event=input["state_group_before_event"],
-            state_group_deltas=state_group_deltas,
+            prev_group=input["prev_group"],
             state_delta_due_to_event=_decode_state_dict(
                 input["state_delta_due_to_event"]
             ),
+            delta_ids=_decode_state_dict(input["delta_ids"]),
             rejected=input["rejected"],
             partial_state=input.get("partial_state", False),
         )
@@ -355,7 +349,7 @@ class UnpersistedEventContext(UnpersistedEventContextBase):
     _storage: "StorageControllers"
     state_group_before_event: Optional[int]
     state_group_after_event: Optional[int]
-    state_delta_due_to_event: Optional[StateMap[str]]
+    state_delta_due_to_event: Optional[dict]
     prev_group_for_state_group_before_event: Optional[int]
     delta_ids_to_state_group_before_event: Optional[StateMap[str]]
     partial_state: bool
@@ -386,16 +380,26 @@ class UnpersistedEventContext(UnpersistedEventContextBase):
 
         events_and_persisted_context = []
         for event, unpersisted_context in amended_events_and_context:
-            state_group_deltas = unpersisted_context._build_state_group_deltas()
-
-            context = EventContext(
-                storage=unpersisted_context._storage,
-                state_group=unpersisted_context.state_group_after_event,
-                state_group_before_event=unpersisted_context.state_group_before_event,
-                state_delta_due_to_event=unpersisted_context.state_delta_due_to_event,
-                partial_state=unpersisted_context.partial_state,
-                state_group_deltas=state_group_deltas,
-            )
+            if event.is_state():
+                context = EventContext(
+                    storage=unpersisted_context._storage,
+                    state_group=unpersisted_context.state_group_after_event,
+                    state_group_before_event=unpersisted_context.state_group_before_event,
+                    state_delta_due_to_event=unpersisted_context.state_delta_due_to_event,
+                    partial_state=unpersisted_context.partial_state,
+                    prev_group=unpersisted_context.state_group_before_event,
+                    delta_ids=unpersisted_context.state_delta_due_to_event,
+                )
+            else:
+                context = EventContext(
+                    storage=unpersisted_context._storage,
+                    state_group=unpersisted_context.state_group_after_event,
+                    state_group_before_event=unpersisted_context.state_group_before_event,
+                    state_delta_due_to_event=unpersisted_context.state_delta_due_to_event,
+                    partial_state=unpersisted_context.partial_state,
+                    prev_group=unpersisted_context.prev_group_for_state_group_before_event,
+                    delta_ids=unpersisted_context.delta_ids_to_state_group_before_event,
+                )
             events_and_persisted_context.append((event, context))
         return events_and_persisted_context
 
@@ -448,11 +452,11 @@ class UnpersistedEventContext(UnpersistedEventContextBase):
 
         # if the event isn't a state event the state group doesn't change
         if not self.state_delta_due_to_event:
-            self.state_group_after_event = self.state_group_before_event
+            state_group_after_event = self.state_group_before_event
 
         # otherwise if it is a state event we need to get a state group for it
         else:
-            self.state_group_after_event = await self._storage.state.store_state_group(
+            state_group_after_event = await self._storage.state.store_state_group(
                 event.event_id,
                 event.room_id,
                 prev_group=self.state_group_before_event,
@@ -460,81 +464,16 @@ class UnpersistedEventContext(UnpersistedEventContextBase):
                 current_state_ids=None,
             )
 
-        state_group_deltas = self._build_state_group_deltas()
-
         return EventContext.with_state(
             storage=self._storage,
-            state_group=self.state_group_after_event,
+            state_group=state_group_after_event,
             state_group_before_event=self.state_group_before_event,
             state_delta_due_to_event=self.state_delta_due_to_event,
-            state_group_deltas=state_group_deltas,
             partial_state=self.partial_state,
+            prev_group=self.state_group_before_event,
+            delta_ids=self.state_delta_due_to_event,
         )
 
-    def _build_state_group_deltas(self) -> Dict[Tuple[int, int], StateMap]:
-        """
-        Collect deltas between the state groups associated with this context
-        """
-        state_group_deltas = {}
-
-        # if we know the state group before the event and after the event, add them and the
-        # state delta between them to state_group_deltas
-        if self.state_group_before_event and self.state_group_after_event:
-            # if we have the state groups we should have the delta
-            assert self.state_delta_due_to_event is not None
-            state_group_deltas[
-                (
-                    self.state_group_before_event,
-                    self.state_group_after_event,
-                )
-            ] = self.state_delta_due_to_event
-
-        # the state group before the event may also have a state group which precedes it, if
-        # we have that and the state group before the event, add them and the state
-        # delta between them to state_group_deltas
-        if (
-            self.prev_group_for_state_group_before_event
-            and self.state_group_before_event
-        ):
-            # if we have both state groups we should have the delta between them
-            assert self.delta_ids_to_state_group_before_event is not None
-            state_group_deltas[
-                (
-                    self.prev_group_for_state_group_before_event,
-                    self.state_group_before_event,
-                )
-            ] = self.delta_ids_to_state_group_before_event
-
-        return state_group_deltas
-
-
-def _encode_state_group_delta(
-    state_group_delta: Dict[Tuple[int, int], StateMap[str]]
-) -> List[Tuple[int, int, Optional[List[Tuple[str, str, str]]]]]:
-    if not state_group_delta:
-        return []
-
-    state_group_delta_encoded = []
-    for key, value in state_group_delta.items():
-        state_group_delta_encoded.append((key[0], key[1], _encode_state_dict(value)))
-
-    return state_group_delta_encoded
-
-
-def _decode_state_group_delta(
-    input: List[Tuple[int, int, List[Tuple[str, str, str]]]]
-) -> Dict[Tuple[int, int], StateMap[str]]:
-    if not input:
-        return {}
-
-    state_group_deltas = {}
-    for state_group_1, state_group_2, state_dict in input:
-        state_map = _decode_state_dict(state_dict)
-        assert state_map is not None
-        state_group_deltas[(state_group_1, state_group_2)] = state_map
-
-    return state_group_deltas
-
 
 def _encode_state_dict(
     state_dict: Optional[StateMap[str]],

synapse/events/utils.py

@@ -164,12 +164,21 @@ def prune_event_dict(room_version: RoomVersion, event_dict: JsonDict) -> JsonDic
         if room_version.msc2176_redaction_rules:
             add_fields("invite")
 
+        if room_version.msc2716_historical:
+            add_fields("historical")
+
     elif event_type == EventTypes.Aliases and room_version.special_case_aliases_auth:
         add_fields("aliases")
     elif event_type == EventTypes.RoomHistoryVisibility:
         add_fields("history_visibility")
     elif event_type == EventTypes.Redaction and room_version.msc2176_redaction_rules:
         add_fields("redacts")
+    elif room_version.msc2716_redactions and event_type == EventTypes.MSC2716_INSERTION:
+        add_fields(EventContentFields.MSC2716_NEXT_BATCH_ID)
+    elif room_version.msc2716_redactions and event_type == EventTypes.MSC2716_BATCH:
+        add_fields(EventContentFields.MSC2716_BATCH_ID)
+    elif room_version.msc2716_redactions and event_type == EventTypes.MSC2716_MARKER:
+        add_fields(EventContentFields.MSC2716_INSERTION_EVENT_REFERENCE)
 
     # Protect the rel_type and event_id fields under the m.relates_to field.
     if room_version.msc3389_relation_redactions:

synapse/events/validator.py

@@ -134,8 +134,13 @@ class EventValidator:
                 )
 
         # If the event contains a mentions key, validate it.
-        if EventContentFields.MENTIONS in event.content:
-            validate_json_object(event.content[EventContentFields.MENTIONS], Mentions)
+        if (
+            EventContentFields.MSC3952_MENTIONS in event.content
+            and config.experimental.msc3952_intentional_mentions
+        ):
+            validate_json_object(
+                event.content[EventContentFields.MSC3952_MENTIONS], Mentions
+            )
 
     def _validate_retention(self, event: EventBase) -> None:
         """Checks that an event that defines the retention policy for a room respects the

synapse/federation/federation_client.py

@@ -260,9 +260,7 @@ class FederationClient(FederationBase):
         use_unstable = False
         for user_id, one_time_keys in query.items():
             for device_id, algorithms in one_time_keys.items():
-                # If more than one algorithm is requested, attempt to use the unstable
-                # endpoint.
-                if sum(algorithms.values()) > 1:
+                if any(count > 1 for count in algorithms.values()):
                     use_unstable = True
                 if algorithms:
                     # For the stable query, choose only the first algorithm.
@@ -298,7 +296,6 @@ class FederationClient(FederationBase):
         else:
             logger.debug("Skipping unstable claim client keys API")
 
-        # TODO Potentially attempt multiple queries and combine the results?
         return await self.transport_layer.claim_client_keys(
             user, destination, content, timeout
         )

synapse/federation/federation_server.py

@@ -515,7 +515,7 @@ class FederationServer(FederationBase):
                     logger.error(
                         "Failed to handle PDU %s",
                         event_id,
-                        exc_info=(f.type, f.value, f.getTracebackObject()),
+                        exc_info=(f.type, f.value, f.getTracebackObject()),  # type: ignore
                     )
                     return {"error": str(e)}
 
@@ -944,7 +944,7 @@ class FederationServer(FederationBase):
             if not self._is_mine_server_name(authorising_server):
                 raise SynapseError(
                     400,
-                    f"Cannot authorise membership event for {authorising_server}. We can only authorise requests from our own homeserver",
+                    f"Cannot authorise request from resident server: {authorising_server}",
                 )
 
             event.signatures.update(
@@ -1016,9 +1016,7 @@ class FederationServer(FederationBase):
             for user_id, device_keys in result.items():
                 for device_id, keys in device_keys.items():
                     for key_id, key in keys.items():
-                        json_result.setdefault(user_id, {}).setdefault(device_id, {})[
-                            key_id
-                        ] = key
+                        json_result.setdefault(user_id, {})[device_id] = {key_id: key}
 
         logger.info(
             "Claimed one-time-keys: %s",
@@ -1249,7 +1247,7 @@ class FederationServer(FederationBase):
                     logger.error(
                         "Failed to handle PDU %s",
                         event.event_id,
-                        exc_info=(f.type, f.value, f.getTracebackObject()),
+                        exc_info=(f.type, f.value, f.getTracebackObject()),  # type: ignore
                     )
 
                 received_ts = await self.store.remove_received_event_from_staging(
@@ -1293,6 +1291,9 @@ class FederationServer(FederationBase):
                 return
             lock = new_lock
 
+    def __str__(self) -> str:
+        return "<ReplicationLayer(%s)>" % self.server_name
+
     async def exchange_third_party_invite(
         self, sender_user_id: str, target_user_id: str, room_id: str, signed: Dict
     ) -> None:

synapse/federation/sender/__init__.py

@@ -109,8 +109,10 @@ was enabled*, Catch-Up Mode is exited and we return to `_transaction_transmissio
 
 If a remote server is unreachable over federation, we back off from that server,
 with an exponentially-increasing retry interval.
-We automatically retry after the retry interval expires (roughly, the logic to do so
-being triggered every minute).
+Whilst we don't automatically retry after the interval, we prevent making new attempts
+until such time as the back-off has cleared.
+Once the back-off is cleared and a new PDU or EDU arrives for transmission, the transmission
+loop resumes and empties the queue by making federation requests.
 
 If the backoff grows too large (> 1 hour), the in-memory queue is emptied (to prevent
 unbounded growth) and Catch-Up Mode is entered.
@@ -143,6 +145,7 @@ from prometheus_client import Counter
 from typing_extensions import Literal
 
 from twisted.internet import defer
+from twisted.internet.interfaces import IDelayedCall
 
 import synapse.metrics
 from synapse.api.presence import UserPresenceState
@@ -181,18 +184,14 @@ sent_pdus_destination_dist_total = Counter(
     "Total number of PDUs queued for sending across all destinations",
 )
 
-# Time (in s) to wait before trying to wake up destinations that have
-# catch-up outstanding. This will also be the delay applied at startup
-# before trying the same.
-# Please note that rate limiting still applies, so while the loop is
-# executed every X seconds the destinations may not be wake up because
-# they are being rate limited following previous attempt failures.
-WAKEUP_RETRY_PERIOD_SEC = 60
+# Time (in s) after Synapse's startup that we will begin to wake up destinations
+# that have catch-up outstanding.
+CATCH_UP_STARTUP_DELAY_SEC = 15
 
 # Time (in s) to wait in between waking up each destination, i.e. one destination
-# will be woken up every <x> seconds until we have woken every destination
-# has outstanding catch-up.
-WAKEUP_INTERVAL_BETWEEN_DESTINATIONS_SEC = 5
+# will be woken up every <x> seconds after Synapse's startup until we have woken
+# every destination has outstanding catch-up.
+CATCH_UP_STARTUP_INTERVAL_SEC = 5
 
 
 class AbstractFederationSender(metaclass=abc.ABCMeta):
@@ -416,10 +415,12 @@ class FederationSender(AbstractFederationSender):
             / hs.config.ratelimiting.federation_rr_transactions_per_room_per_second
         )
 
-        # Regularly wake up destinations that have outstanding PDUs to be caught up
-        self.clock.looping_call(
+        # wake up destinations that have outstanding PDUs to be caught up
+        self._catchup_after_startup_timer: Optional[
+            IDelayedCall
+        ] = self.clock.call_later(
+            CATCH_UP_STARTUP_DELAY_SEC,
             run_as_background_process,
-            WAKEUP_RETRY_PERIOD_SEC * 1000.0,
             "wake_destinations_needing_catchup",
             self._wake_destinations_needing_catchup,
         )
@@ -965,6 +966,7 @@ class FederationSender(AbstractFederationSender):
 
             if not destinations_to_wake:
                 # finished waking all destinations!
+                self._catchup_after_startup_timer = None
                 break
 
             last_processed = destinations_to_wake[-1]
@@ -981,4 +983,4 @@ class FederationSender(AbstractFederationSender):
                     last_processed,
                 )
                 self.wake_destination(destination)
-                await self.clock.sleep(WAKEUP_INTERVAL_BETWEEN_DESTINATIONS_SEC)
+                await self.clock.sleep(CATCH_UP_STARTUP_INTERVAL_SEC)

synapse/handlers/account_validity.py

@@ -164,7 +164,7 @@ class AccountValidityHandler:
 
         try:
             user_display_name = await self.store.get_profile_displayname(
-                UserID.from_string(user_id)
+                UserID.from_string(user_id).localpart
             )
             if user_display_name is None:
                 user_display_name = user_id

synapse/handlers/admin.py

@@ -89,7 +89,7 @@ class AdminHandler:
         }
 
         # Add additional user metadata
-        profile = await self._store.get_profileinfo(user)
+        profile = await self._store.get_profileinfo(user.localpart)
         threepids = await self._store.user_get_threepids(user.to_string())
         external_ids = [
             ({"auth_provider": auth_provider, "external_id": external_id})

synapse/handlers/auth.py

@@ -274,8 +274,6 @@ class AuthHandler:
         # response.
         self._extra_attributes: Dict[str, SsoLoginExtraAttributes] = {}
 
-        self.msc3861_oauth_delegation_enabled = hs.config.experimental.msc3861.enabled
-
     async def validate_user_via_ui_auth(
         self,
         requester: Requester,
@@ -324,12 +322,8 @@ class AuthHandler:
 
             LimitExceededError if the ratelimiter's failed request count for this
                 user is too high to proceed
-        """
-        if self.msc3861_oauth_delegation_enabled:
-            raise SynapseError(
-                HTTPStatus.INTERNAL_SERVER_ERROR, "UIA shouldn't be used with MSC3861"
-            )
 
+        """
         if not requester.access_token_id:
             raise ValueError("Cannot validate a user without an access token")
         if can_skip_ui_auth and self._ui_auth_session_timeout:
@@ -1759,7 +1753,7 @@ class AuthHandler:
             return
 
         user_profile_data = await self.store.get_profileinfo(
-            UserID.from_string(registered_user_id)
+            UserID.from_string(registered_user_id).localpart
         )
 
         # Store any extra attributes which will be passed in the login response.

synapse/handlers/deactivate_account.py

@@ -297,5 +297,5 @@ class DeactivateAccountHandler:
         # Add the user to the directory, if necessary. Note that
         # this must be done after the user is re-activated, because
         # deactivated users are excluded from the user directory.
-        profile = await self.store.get_profileinfo(user)
+        profile = await self.store.get_profileinfo(user.localpart)
         await self.user_directory_handler.handle_local_profile_change(user_id, profile)

synapse/handlers/directory.py

@@ -277,9 +277,7 @@ class DirectoryHandler:
             except RequestSendFailed:
                 raise SynapseError(502, "Failed to fetch alias")
             except CodeMessageException as e:
-                logging.warning(
-                    "Error retrieving alias %s -> %s %s", room_alias, e.code, e.msg
-                )
+                logging.warning("Error retrieving alias")
                 if e.code == 404:
                     fed_result = None
                 else:

synapse/handlers/federation.py

@@ -105,12 +105,14 @@ backfill_processing_before_timer = Histogram(
 )
 
 
-# TODO: We can refactor this away now that there is only one backfill point again
 class _BackfillPointType(Enum):
     # a regular backwards extremity (ie, an event which we don't yet have, but which
     # is referred to by other events in the DAG)
     BACKWARDS_EXTREMITY = enum.auto()
 
+    # an MSC2716 "insertion event"
+    INSERTION_PONT = enum.auto()
+
 
 @attr.s(slots=True, auto_attribs=True, frozen=True)
 class _BackfillPoint:
@@ -198,7 +200,6 @@ class FederationHandler:
             )
 
     @trace
-    @tag_args
     async def maybe_backfill(
         self, room_id: str, current_depth: int, limit: int
     ) -> bool:
@@ -213,9 +214,6 @@ class FederationHandler:
             limit: The number of events that the pagination request will
                 return. This is used as part of the heuristic to decide if we
                 should back paginate.
-
-        Returns:
-            True if we actually tried to backfill something, otherwise False.
         """
         # Starting the processing time here so we can include the room backfill
         # linearizer lock queue in the timing
@@ -229,8 +227,6 @@ class FederationHandler:
                 processing_start_time=processing_start_time,
             )
 
-    @trace
-    @tag_args
     async def _maybe_backfill_inner(
         self,
         room_id: str,
@@ -251,9 +247,6 @@ class FederationHandler:
             limit: The max number of events to request from the remote federated server.
             processing_start_time: The time when `maybe_backfill` started processing.
                 Only used for timing. If `None`, no timing observation will be made.
-
-        Returns:
-            True if we actually tried to backfill something, otherwise False.
         """
         backwards_extremities = [
             _BackfillPoint(event_id, depth, _BackfillPointType.BACKWARDS_EXTREMITY)
@@ -271,10 +264,32 @@ class FederationHandler:
             )
         ]
 
+        insertion_events_to_be_backfilled: List[_BackfillPoint] = []
+        if self.hs.config.experimental.msc2716_enabled:
+            insertion_events_to_be_backfilled = [
+                _BackfillPoint(event_id, depth, _BackfillPointType.INSERTION_PONT)
+                for event_id, depth in await self.store.get_insertion_event_backward_extremities_in_room(
+                    room_id=room_id,
+                    current_depth=current_depth,
+                    # We only need to end up with 5 extremities combined with
+                    # the backfill points to make the `/backfill` request ...
+                    # (see the other comment above for more context).
+                    limit=50,
+                )
+            ]
+        logger.debug(
+            "_maybe_backfill_inner: backwards_extremities=%s insertion_events_to_be_backfilled=%s",
+            backwards_extremities,
+            insertion_events_to_be_backfilled,
+        )
+
         # we now have a list of potential places to backpaginate from. We prefer to
         # start with the most recent (ie, max depth), so let's sort the list.
         sorted_backfill_points: List[_BackfillPoint] = sorted(
-            backwards_extremities,
+            itertools.chain(
+                backwards_extremities,
+                insertion_events_to_be_backfilled,
+            ),
             key=lambda e: -int(e.depth),
         )
 
@@ -287,30 +302,15 @@ class FederationHandler:
             len(sorted_backfill_points),
             sorted_backfill_points,
         )
-        set_tag(
-            SynapseTags.RESULT_PREFIX + "sorted_backfill_points",
-            str(sorted_backfill_points),
-        )
-        set_tag(
-            SynapseTags.RESULT_PREFIX + "sorted_backfill_points.length",
-            str(len(sorted_backfill_points)),
-        )
 
-        # If we have no backfill points lower than the `current_depth` then either we
-        # can a) bail or b) still attempt to backfill. We opt to try backfilling anyway
-        # just in case we do get relevant events. This is good for eventual consistency
-        # sake but we don't need to block the client for something that is just as
-        # likely not to return anything relevant so we backfill in the background. The
-        # only way, this could return something relevant is if we discover a new branch
-        # of history that extends all the way back to where we are currently paginating
-        # and it's within the 100 events that are returned from `/backfill`.
+        # If we have no backfill points lower than the `current_depth` then
+        # either we can a) bail or b) still attempt to backfill. We opt to try
+        # backfilling anyway just in case we do get relevant events.
         if not sorted_backfill_points and current_depth != MAX_DEPTH:
             logger.debug(
                 "_maybe_backfill_inner: all backfill points are *after* current depth. Trying again with later backfill points."
             )
-            run_as_background_process(
-                "_maybe_backfill_inner_anyway_with_max_depth",
-                self._maybe_backfill_inner,
+            return await self._maybe_backfill_inner(
                 room_id=room_id,
                 # We use `MAX_DEPTH` so that we find all backfill points next
                 # time (all events are below the `MAX_DEPTH`)
@@ -321,9 +321,6 @@ class FederationHandler:
                 # overall otherwise the smaller one will throw off the results.
                 processing_start_time=None,
             )
-            # We return `False` because we're backfilling in the background and there is
-            # no new events immediately for the caller to know about yet.
-            return False
 
         # Even after recursing with `MAX_DEPTH`, we didn't find any
         # backward extremities to backfill from.
@@ -387,7 +384,10 @@ class FederationHandler:
             #   event but not anything before it. This would require looking at the
             #   state *before* the event, ignoring the special casing certain event
             #   types have.
-            event_ids_to_check = await self.store.get_successor_events(bp.event_id)
+            if bp.type == _BackfillPointType.INSERTION_PONT:
+                event_ids_to_check = [bp.event_id]
+            else:
+                event_ids_to_check = await self.store.get_successor_events(bp.event_id)
 
             events_to_check = await self.store.get_events_as_list(
                 event_ids_to_check,