Set right content type header

Fixup3
fixup
2023-07-12 15:32:17 +01:00 · 2023-07-12 15:24:29 +01:00 · 2023-07-12 15:11:59 +01:00 · 2023-07-12 15:07:13 +01:00 · 2023-07-12 14:46:42 +01:00 · 2023-07-12 12:39:25 +00:00
305 changed files with 13176 additions and 7900 deletions
@@ -29,11 +29,12 @@ IS_PR = os.environ["GITHUB_REF"].startswith("refs/pull/")

 # First calculate the various trial jobs.
 #
-# For each type of test we only run on Py3.7 on PRs
+# For PRs, we only run each type of test with the oldest Python version supported (which
+# is Python 3.8 right now)

 trial_sqlite_tests = [
    {
-        "python-version": "3.7",
+        "python-version": "3.8",
        "database": "sqlite",
        "extras": "all",
    }
@@ -46,13 +47,13 @@ if not IS_PR:
            "database": "sqlite",
            "extras": "all",
        }
-        for version in ("3.8", "3.9", "3.10", "3.11")
+        for version in ("3.9", "3.10", "3.11")
    )


 trial_postgres_tests = [
    {
-        "python-version": "3.7",
+        "python-version": "3.8",
        "database": "postgres",
        "postgres-version": "11",
        "extras": "all",
@@ -71,7 +72,7 @@ if not IS_PR:

 trial_no_extra_tests = [
    {
-        "python-version": "3.7",
+        "python-version": "3.8",
        "database": "sqlite",
        "extras": "",
    }
@@ -133,11 +134,6 @@ if not IS_PR:
                "sytest-tag": "testing",
                "postgres": "postgres",
            },
-            {
-                "sytest-tag": "buster",
-                "postgres": "multi-postgres",
-                "workers": "workers",
-            },
        ]
    )

@@ -1,49 +0,0 @@
-name: Write changelog for dependabot PR
-on:
-  pull_request:
-    types:
-      - opened
-      - reopened  # For debugging!
-
-permissions:
-  # Needed to be able to push the commit. See
-  #     https://docs.github.com/en/code-security/dependabot/working-with-dependabot/automating-dependabot-with-github-actions#enable-auto-merge-on-a-pull-request
-  # for a similar example
-  contents: write
-
-jobs:
-  add-changelog:
-    runs-on: 'ubuntu-latest'
-    if: ${{ github.actor == 'dependabot[bot]' }}
-    steps:
-      - uses: actions/checkout@v3
-        with:
-          ref: ${{ github.event.pull_request.head.ref }}
-      - name: Write, commit and push changelog
-        env:
-          PR_TITLE: ${{ github.event.pull_request.title }}
-          PR_NUMBER: ${{ github.event.pull_request.number }}
-        run: |
-          echo "${PR_TITLE}." > "changelog.d/${PR_NUMBER}".misc
-          git add changelog.d
-          git config user.email "github-actions[bot]@users.noreply.github.com"
-          git config user.name "GitHub Actions"
-          git commit -m "Changelog"
-          git push
-        shell: bash
-      # The `git push` above does not trigger CI on the dependabot PR.
-      #
-      # By default, workflows can't trigger other workflows when they're just using the
-      # default `GITHUB_TOKEN` access token. (This is intended to stop you from writing
-      # recursive workflow loops by accident, because that'll get very expensive very
-      # quickly.) Instead, you have to manually call out to another workflow, or else
-      # make your changes (i.e. the `git push` above) using a personal access token.
-      # See
-      # https://docs.github.com/en/actions/using-workflows/triggering-a-workflow#triggering-a-workflow-from-a-workflow
-      #
-      # I have tried and failed to find a way to trigger CI on the "merge ref" of the PR.
-      # See git commit history for previous attempts. If anyone desperately wants to try
-      # again in the future, make a matrix-bot account and use its access token to git push.
-
-  # THIS WORKFLOW HAS WRITE PERMISSIONS---do not add other jobs here unless they
-  # are sufficiently locked down to dependabot only as above.
@@ -22,7 +22,21 @@ concurrency:
  cancel-in-progress: true

 jobs:
+  check_repo:
+    # Prevent this workflow from running on any fork of Synapse other than matrix-org/synapse, as it is
+    # only useful to the Synapse core team.
+    # All other workflow steps depend on this one, thus if 'should_run_workflow' is not 'true', the rest
+    # of the workflow will be skipped as well.
+    runs-on: ubuntu-latest
+    outputs:
+      should_run_workflow: ${{ steps.check_condition.outputs.should_run_workflow }}
+    steps:
+      - id: check_condition
+        run: echo "should_run_workflow=${{ github.repository == 'matrix-org/synapse' }}" >> "$GITHUB_OUTPUT"
+
  mypy:
+    needs: check_repo
+    if: needs.check_repo.outputs.should_run_workflow == 'true'
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v3
@@ -47,6 +61,8 @@ jobs:
        run: sed '/warn_unused_ignores = True/d' -i mypy.ini
      - run: poetry run mypy
  trial:
+    needs: check_repo
+    if: needs.check_repo.outputs.should_run_workflow == 'true'
    runs-on: ubuntu-latest
    strategy:
      matrix:
@@ -105,6 +121,8 @@ jobs:


  sytest:
+    needs: check_repo
+    if: needs.check_repo.outputs.should_run_workflow == 'true'
    runs-on: ubuntu-latest
    container:
      image: matrixdotorg/sytest-synapse:testing
@@ -156,7 +174,8 @@ jobs:


  complement:
-    if: "${{ !failure() && !cancelled() }}"
+    needs: check_repo
+    if: "!failure() && !cancelled() && needs.check_repo.outputs.should_run_workflow == 'true'"
    runs-on: ubuntu-latest

    strategy:
@@ -192,7 +211,7 @@ jobs:
  # Open an issue if the build fails, so we know about it.
  # Only do this if we're not experimenting with this action in a PR.
  open-issue:
-    if: "failure() && github.event_name != 'push' && github.event_name != 'pull_request'"
+    if: "failure() && github.event_name != 'push' && github.event_name != 'pull_request' && needs.check_repo.outputs.should_run_workflow == 'true'"
    needs:
      # TODO: should mypy be included here? It feels more brittle than the others.
      - mypy
@@ -34,6 +34,7 @@ jobs:
      - id: set-distros
        run: |
          # if we're running from a tag, get the full list of distros; otherwise just use debian:sid
+          # NOTE: inside the actual Dockerfile-dhvirtualenv, the image name is expanded into its full image path
          dists='["debian:sid"]'
          if [[ $GITHUB_REF == refs/tags/* ]]; then
              dists=$(scripts-dev/build_debian_packages.py --show-dists-json)
@@ -143,7 +144,7 @@ jobs:

      - name: Only build a single wheel on PR
        if: startsWith(github.ref, 'refs/pull/')
-        run: echo "CIBW_BUILD="cp37-manylinux_${{ matrix.arch }}"" >> $GITHUB_ENV
+        run: echo "CIBW_BUILD="cp38-manylinux_${{ matrix.arch }}"" >> $GITHUB_ENV

      - name: Build wheels
        run: python -m cibuildwheel --output-dir wheelhouse
@@ -35,7 +35,7 @@ jobs:
    steps:
      - uses: actions/checkout@v3
      - name: Install Rust
-        uses: dtolnay/rust-toolchain@1.58.1
+        uses: dtolnay/rust-toolchain@1.60.0
      - uses: Swatinem/rust-cache@v2
      - uses: matrix-org/setup-python-poetry@v1
        with:
@@ -92,6 +92,10 @@ jobs:
      - name: Checkout repository
        uses: actions/checkout@v3

+      - name: Install Rust
+        uses: dtolnay/rust-toolchain@1.60.0
+      - uses: Swatinem/rust-cache@v2
+
      - name: Setup Poetry
        uses: matrix-org/setup-python-poetry@v1
        with:
@@ -103,10 +107,6 @@ jobs:
          # To make CI green, err towards caution and install the project.
          install-project: "true"

-      - name: Install Rust
-        uses: dtolnay/rust-toolchain@1.58.1
-      - uses: Swatinem/rust-cache@v2
-
      # Cribbed from
      # https://github.com/AustinScola/mypy-cache-github-action/blob/85ea4f2972abed39b33bd02c36e341b28ca59213/src/restore.ts#L10-L17
      - name: Restore/persist mypy's cache
@@ -150,7 +150,7 @@ jobs:
        with:
          ref: ${{ github.event.pull_request.head.sha }}
      - name: Install Rust
-        uses: dtolnay/rust-toolchain@1.58.1
+        uses: dtolnay/rust-toolchain@1.60.0
      - uses: Swatinem/rust-cache@v2
      - uses: matrix-org/setup-python-poetry@v1
        with:
@@ -167,7 +167,7 @@ jobs:
      - uses: actions/checkout@v3

      - name: Install Rust
-        uses: dtolnay/rust-toolchain@1.58.1
+        uses: dtolnay/rust-toolchain@1.60.0
        with:
            components: clippy
      - uses: Swatinem/rust-cache@v2
@@ -268,7 +268,7 @@ jobs:
            postgres:${{ matrix.job.postgres-version }}

      - name: Install Rust
-        uses: dtolnay/rust-toolchain@1.58.1
+        uses: dtolnay/rust-toolchain@1.60.0
      - uses: Swatinem/rust-cache@v2

      - uses: matrix-org/setup-python-poetry@v1
@@ -308,19 +308,19 @@ jobs:
      - uses: actions/checkout@v3

      - name: Install Rust
-        uses: dtolnay/rust-toolchain@1.58.1
+        uses: dtolnay/rust-toolchain@1.60.0
      - uses: Swatinem/rust-cache@v2

      # There aren't wheels for some of the older deps, so we need to install
      # their build dependencies
      - run: |
-          sudo apt update
+          sudo apt-get -qq update
          sudo apt-get -qq install build-essential libffi-dev python-dev \
            libxml2-dev libxslt-dev xmlsec1 zlib1g-dev libjpeg-dev libwebp-dev

      - uses: actions/setup-python@v4
        with:
-          python-version: '3.7'
+          python-version: '3.8'

      - name: Prepare old deps
        if: steps.cache-poetry-old-deps.outputs.cache-hit != 'true'
@@ -362,7 +362,7 @@ jobs:
    runs-on: ubuntu-latest
    strategy:
      matrix:
-        python-version: ["pypy-3.7"]
+        python-version: ["pypy-3.8"]
        extras: ["all"]

    steps:
@@ -399,8 +399,8 @@ jobs:
      env:
        SYTEST_BRANCH: ${{ github.head_ref }}
        POSTGRES: ${{ matrix.job.postgres && 1}}
-        MULTI_POSTGRES: ${{ (matrix.job.postgres == 'multi-postgres') && 1}}
-        ASYNCIO_REACTOR: ${{ (matrix.job.reactor == 'asyncio') && 1 }}
+        MULTI_POSTGRES: ${{ (matrix.job.postgres == 'multi-postgres') || '' }}
+        ASYNCIO_REACTOR: ${{ (matrix.job.reactor == 'asyncio') || '' }}
        WORKERS: ${{ matrix.job.workers && 1 }}
        BLACKLIST: ${{ matrix.job.workers && 'synapse-blacklist-with-workers' }}
        TOP: ${{ github.workspace }}
@@ -416,7 +416,7 @@ jobs:
        run: cat sytest-blacklist .ci/worker-blacklist > synapse-blacklist-with-workers

      - name: Install Rust
-        uses: dtolnay/rust-toolchain@1.58.1
+        uses: dtolnay/rust-toolchain@1.60.0
      - uses: Swatinem/rust-cache@v2

      - name: Run SyTest
@@ -477,7 +477,7 @@ jobs:
    strategy:
      matrix:
        include:
-          - python-version: "3.7"
+          - python-version: "3.8"
            postgres-version: "11"

          - python-version: "3.11"
@@ -556,7 +556,7 @@ jobs:
          path: synapse

      - name: Install Rust
-        uses: dtolnay/rust-toolchain@1.58.1
+        uses: dtolnay/rust-toolchain@1.60.0
      - uses: Swatinem/rust-cache@v2

      - uses: actions/setup-go@v4
@@ -584,7 +584,7 @@ jobs:
      - uses: actions/checkout@v3

      - name: Install Rust
-        uses: dtolnay/rust-toolchain@1.58.1
+        uses: dtolnay/rust-toolchain@1.60.0
      - uses: Swatinem/rust-cache@v2

      - run: cargo test
@@ -18,7 +18,22 @@ concurrency:
  cancel-in-progress: true

 jobs:
+  check_repo:
+    # Prevent this workflow from running on any fork of Synapse other than matrix-org/synapse, as it is
+    # only useful to the Synapse core team.
+    # All other workflow steps depend on this one, thus if 'should_run_workflow' is not 'true', the rest
+    # of the workflow will be skipped as well.
+    if: github.repository == 'matrix-org/synapse'
+    runs-on: ubuntu-latest
+    outputs:
+      should_run_workflow: ${{ steps.check_condition.outputs.should_run_workflow }}
+    steps:
+      - id: check_condition
+        run: echo "should_run_workflow=${{ github.repository == 'matrix-org/synapse' }}" >> "$GITHUB_OUTPUT"
+
  mypy:
+    needs: check_repo
+    if: needs.check_repo.outputs.should_run_workflow == 'true'
    runs-on: ubuntu-latest

    steps:
@@ -41,6 +56,8 @@ jobs:
      - run: poetry run mypy

  trial:
+    needs: check_repo
+    if: needs.check_repo.outputs.should_run_workflow == 'true'
    runs-on: ubuntu-latest

    steps:
@@ -75,9 +92,15 @@ jobs:
          || true

  sytest:
+    needs: check_repo
+    if: needs.check_repo.outputs.should_run_workflow == 'true'
    runs-on: ubuntu-latest
    container:
-      image: matrixdotorg/sytest-synapse:buster
+      # We're using ubuntu:focal because it uses Python 3.8 which is our minimum supported Python version.
+      # This job is a canary to warn us about unreleased twisted changes that would cause problems for us if
+      # they were to be released immediately. For simplicity's sake (and to save CI runners) we use the oldest
+      # version, assuming that any incompatibilities on newer versions would also be present on the oldest.
+      image: matrixdotorg/sytest-synapse:focal
      volumes:
        - ${{ github.workspace }}:/src

@@ -119,7 +142,8 @@ jobs:
            /logs/**/*.log*

  complement:
-    if: "${{ !failure() && !cancelled() }}"
+    needs: check_repo
+    if: "!failure() && !cancelled() && needs.check_repo.outputs.should_run_workflow == 'true'"
    runs-on: ubuntu-latest

    strategy:
@@ -166,7 +190,7 @@ jobs:

  # open an issue if the build fails, so we know about it.
  open-issue:
-    if: failure()
+    if: failure() && needs.check_repo.outputs.should_run_workflow == 'true'
    needs:
      - mypy
      - trial
@@ -34,6 +34,7 @@ __pycache__/
 /logs
 /media_store/
 /uploads
+/homeserver-config-overrides.d

 # For direnv users
 /.envrc
@@ -4,9 +4,9 @@ version = 3

 [[package]]
 name = "aho-corasick"
-version = "0.7.19"
+version = "1.0.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b4f55bd91a0978cbfd91c457a164bab8b4001c833b7f323132c0a4e1922dd44e"
+checksum = "43f6cb1bf222025340178f382c426f13757b2960e89779dfcb319c32542a5a41"
 dependencies = [
 "memchr",
 ]
@@ -132,12 +132,9 @@ dependencies = [

 [[package]]
 name = "log"
-version = "0.4.17"
+version = "0.4.19"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "abb12e687cfb44aa40f41fc3978ef76448f9b6038cad6aef4259d3c095a2382e"
-dependencies = [
- "cfg-if",
-]
+checksum = "b06a4cde4c0f271a446782e3eff8de789548ce57dbc8eca9292c27f4a42004b4"

 [[package]]
 name = "memchr"
@@ -185,9 +182,9 @@ dependencies = [

 [[package]]
 name = "proc-macro2"
-version = "1.0.52"
+version = "1.0.64"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1d0e1ae9e836cc3beddd63db0df682593d7e2d3d891ae8c9083d2113e1744224"
+checksum = "78803b62cbf1f46fde80d7c0e803111524b9877184cfe7c3033659490ac7a7da"
 dependencies = [
 "unicode-ident",
 ]
@@ -232,9 +229,9 @@ dependencies = [

 [[package]]
 name = "pyo3-log"
-version = "0.8.1"
+version = "0.8.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f9c8b57fe71fb5dcf38970ebedc2b1531cf1c14b1b9b4c560a182a57e115575c"
+checksum = "c94ff6535a6bae58d7d0b85e60d4c53f7f84d0d0aa35d6a28c3f3e70bfe51444"
 dependencies = [
 "arc-swap",
 "log",
@@ -276,9 +273,9 @@ dependencies = [

 [[package]]
 name = "quote"
-version = "1.0.26"
+version = "1.0.29"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4424af4bf778aae2051a77b60283332f386554255d722233d09fbfc7e30da2fc"
+checksum = "573015e8ab27661678357f27dc26460738fd2b6c86e46f386fde94cb5d913105"
 dependencies = [
 "proc-macro2",
 ]
@@ -294,9 +291,21 @@ dependencies = [

 [[package]]
 name = "regex"
-version = "1.7.3"
+version = "1.9.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8b1f693b24f6ac912f4893ef08244d70b6067480d2f1a46e950c9691e6749d1d"
+checksum = "b2eae68fc220f7cf2532e4494aded17545fce192d59cd996e0fe7887f4ceb575"
+dependencies = [
+ "aho-corasick",
+ "memchr",
+ "regex-automata",
+ "regex-syntax",
+]
+
+[[package]]
+name = "regex-automata"
+version = "0.3.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "83d3daa6976cffb758ec878f108ba0e062a45b2d6ca3a2cca965338855476caf"
 dependencies = [
 "aho-corasick",
 "memchr",
@@ -305,9 +314,9 @@ dependencies = [

 [[package]]
 name = "regex-syntax"
-version = "0.6.29"
+version = "0.7.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f162c6dd7b008981e4d40210aca20b4bd0f9b60ca9271061b07f78537722f2e1"
+checksum = "2ab07dc67230e4a4718e70fd5c20055a4334b121f1f9db8fe63ef39ce9b8c846"

 [[package]]
 name = "ryu"
@@ -323,29 +332,29 @@ checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd"

 [[package]]
 name = "serde"
-version = "1.0.163"
+version = "1.0.171"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2113ab51b87a539ae008b5c6c02dc020ffa39afd2d83cffcb3f4eb2722cebec2"
+checksum = "30e27d1e4fd7659406c492fd6cfaf2066ba8773de45ca75e855590f856dc34a9"
 dependencies = [
 "serde_derive",
 ]

 [[package]]
 name = "serde_derive"
-version = "1.0.163"
+version = "1.0.171"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8c805777e3930c8883389c602315a24224bcc738b63905ef87cd1420353ea93e"
+checksum = "389894603bd18c46fa56231694f8d827779c0951a667087194cf9de94ed24682"
 dependencies = [
 "proc-macro2",
 "quote",
- "syn 2.0.10",
+ "syn 2.0.25",
 ]

 [[package]]
 name = "serde_json"
-version = "1.0.96"
+version = "1.0.100"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "057d394a50403bcac12672b2b18fb387ab6d289d957dab67dd201875391e52f1"
+checksum = "0f1e14e89be7aa4c4b78bdbdc9eb5bf8517829a600ae8eaa39a6e1d960b5185c"
 dependencies = [
 "itoa",
 "ryu",
@@ -377,9 +386,9 @@ dependencies = [

 [[package]]
 name = "syn"
-version = "2.0.10"
+version = "2.0.25"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5aad1363ed6d37b84299588d62d3a7d95b5a5c2d9aad5c85609fda12afaa1f40"
+checksum = "15e3fc8c0c74267e2df136e5e5fb656a464158aa57624053375eb9c8c6e25ae2"
 dependencies = [
 "proc-macro2",
 "quote",
@@ -3,3 +3,4 @@

 [workspace]
 members = ["rust"]
+resolver = "2"
@@ -1 +0,0 @@
-Remove the old version of the R30 (30-day retained users) phone-home metric.
@@ -1 +0,0 @@
-Fix a long-standing bug where setting the read marker could fail when using message retention. Contributed by Nick @ Beeper (@fizzadar).
@@ -1 +0,0 @@
-Add not null constraint to column full_user_id of tables profiles and user_filters.
@@ -1 +0,0 @@
-Print full error and stack-trace of any exception that occurs during startup/initialization.
@@ -1 +0,0 @@
-Fix a long-standing bug where the `url_preview_url_blacklist` configuration setting was not applied to oEmbed or image URLs found while previewing a URL.
@@ -1 +0,0 @@
-Run mypy type checking with the minimum supported Python version to catch new usage that isn't backwards-compatible.
@@ -1 +0,0 @@
-Fix subscriptable type usage in Python <3.9.
@@ -1 +0,0 @@
-Update internal terminology.
@@ -1 +0,0 @@
-Instrument `state` and `state_group` storage-related operations to better picture what's happening when tracing.
@@ -1 +0,0 @@
-Add a new admin API to create a new device for a user.
@@ -1 +0,0 @@
-Warn users that at least 3.75GB of space is needed for the nix Synapse development environment.
@@ -1 +0,0 @@
-Fix a bug introduced in Synapse 1.82.0 where the error message displayed when validation of the `app_service_config_files` config option fails would be incorrectly formatted.
@@ -1 +0,0 @@
-Re-type config paths in `ConfigError`s to be `StrSequence`s instead of `Iterable[str]`s.
@@ -1 +0,0 @@
-Update internal terminology.
@@ -1 +0,0 @@
-Update Mutual Rooms (MSC2666) implementation to match new proposal text.
@@ -1 +0,0 @@
-Fix a long-standing bug where deactivated users were still able to login using the custom `org.matrix.login.jwt` login type (if enabled).
@@ -1 +0,0 @@
-Remove the unstable identifiers from faster joins ([MSC3706](https://github.com/matrix-org/matrix-spec-proposals/pull/3706).
@@ -1 +0,0 @@
-Fix the olddeps CI.
@@ -1 +0,0 @@
-Fix two memory leaks in `trial` test runs.
@@ -1 +0,0 @@
-Trace how many new events from the backfill response we need to process.
@@ -1 +0,0 @@
-Bump types-setuptools from 67.7.0.2 to 67.8.0.0.
@@ -1 +0,0 @@
-Bump types-pillow from 9.5.0.2 to 9.5.0.4.
@@ -1 +0,0 @@
-Bump sphinx from 6.1.3 to 6.2.1.
@@ -1 +0,0 @@
-Bump furo from 2023.3.27 to 2023.5.20.
@@ -1 +0,0 @@
-Limit the size of the `HomeServerConfig` cache in trial test runs.
@@ -0,0 +1 @@
+Add Unix Socket support for HTTP Replication Listeners. Document and provide usage instructions for utilizing Unix sockets in Synapse. Contributed by Jason Little.
@@ -0,0 +1 @@
+Fix long-standing bug where remote invites weren't correctly pushed.
@@ -0,0 +1 @@
+Mark `get_user_in_directory` private since it is only used in tests. Also remove the cache from it.
@@ -0,0 +1 @@
+Document which Python version runs on a given Linux distribution so we can more easily clean up later.
@@ -0,0 +1 @@
+Allow `+` in Matrix IDs, per [MSC4009](https://github.com/matrix-org/matrix-spec-proposals/pull/4009).
@@ -0,0 +1 @@
+Better clarify how to run a worker instance (pass both configs).
@@ -0,0 +1 @@
+Add details to warning in log when we fail to fetch an alias.
@@ -0,0 +1 @@
+Add Unix Socket support for HTTP Replication Listeners. Document and provide usage instructions for utilizing Unix sockets in Synapse. Contributed by Jason Little.
@@ -0,0 +1 @@
+Fix a bug introduced in 1.86.0 where Synapse starting with an empty `experimental_features` configuration setting.
@@ -29,7 +29,7 @@
        "level": "error"
      },
      {
-        "line": "my-matrix-server-federation-sender-1 | 2023-01-25 20:56:20,995 - synapse.http.matrixfederationclient - 709 - WARNING - federation_transaction_transmission_loop-3 - {PUT-O-3} [example.com] Request failed: PUT matrix://example.com/_matrix/federation/v1/send/1674680155797: HttpResponseException('403: Forbidden')",
+        "line": "my-matrix-server-federation-sender-1 | 2023-01-25 20:56:20,995 - synapse.http.matrixfederationclient - 709 - WARNING - federation_transaction_transmission_loop-3 - {PUT-O-3} [example.com] Request failed: PUT matrix-federation://example.com/_matrix/federation/v1/send/1674680155797: HttpResponseException('403: Forbidden')",
        "level": "warning"
      },
      {
@@ -1,3 +1,81 @@
+matrix-synapse-py3 (1.88.0~rc1) stable; urgency=medium
+
+  * New Synapse release 1.88.0rc1.
+
+ -- Synapse Packaging team <packages@matrix.org>  Tue, 11 Jul 2023 10:20:19 +0100
+
+matrix-synapse-py3 (1.87.0) stable; urgency=medium
+
+  * New Synapse release 1.87.0.
+
+ -- Synapse Packaging team <packages@matrix.org>  Tue, 04 Jul 2023 16:24:00 +0100
+
+matrix-synapse-py3 (1.87.0~rc1) stable; urgency=medium
+
+  * New synapse release 1.87.0rc1.
+
+ -- Synapse Packaging team <packages@matrix.org>  Tue, 27 Jun 2023 15:27:04 +0000
+
+matrix-synapse-py3 (1.86.0) stable; urgency=medium
+
+  * New Synapse release 1.86.0.
+
+ -- Synapse Packaging team <packages@matrix.org>  Tue, 20 Jun 2023 17:22:46 +0200
+
+matrix-synapse-py3 (1.86.0~rc2) stable; urgency=medium
+
+  * New Synapse release 1.86.0rc2.
+
+ -- Synapse Packaging team <packages@matrix.org>  Wed, 14 Jun 2023 12:16:27 +0200
+
+matrix-synapse-py3 (1.86.0~rc1) stable; urgency=medium
+
+  * New Synapse release 1.86.0rc1.
+
+ -- Synapse Packaging team <packages@matrix.org>  Tue, 13 Jun 2023 14:30:45 +0200
+
+matrix-synapse-py3 (1.85.2) stable; urgency=medium
+
+  * New Synapse release 1.85.2.
+
+ -- Synapse Packaging team <packages@matrix.org>  Thu, 08 Jun 2023 13:04:18 +0100
+
+matrix-synapse-py3 (1.85.1) stable; urgency=medium
+
+  * New Synapse release 1.85.1.
+
+ -- Synapse Packaging team <packages@matrix.org>  Wed, 07 Jun 2023 10:51:12 +0100
+
+matrix-synapse-py3 (1.85.0) stable; urgency=medium
+
+  * New Synapse release 1.85.0.
+
+ -- Synapse Packaging team <packages@matrix.org>  Tue, 06 Jun 2023 09:39:29 +0100
+
+matrix-synapse-py3 (1.85.0~rc2) stable; urgency=medium
+
+  * New Synapse release 1.85.0rc2.
+
+ -- Synapse Packaging team <packages@matrix.org>  Thu, 01 Jun 2023 09:16:18 -0700
+
+matrix-synapse-py3 (1.85.0~rc1) stable; urgency=medium
+
+  * New Synapse release 1.85.0rc1.
+
+ -- Synapse Packaging team <packages@matrix.org>  Tue, 30 May 2023 13:56:54 +0100
+
+matrix-synapse-py3 (1.84.1) stable; urgency=medium
+
+  * New Synapse release 1.84.1.
+
+ -- Synapse Packaging team <packages@matrix.org>  Fri, 26 May 2023 16:15:30 +0100
+
+matrix-synapse-py3 (1.84.0) stable; urgency=medium
+
+  * New Synapse release 1.84.0.
+
+ -- Synapse Packaging team <packages@matrix.org>  Tue, 23 May 2023 10:57:22 +0100
+
 matrix-synapse-py3 (1.84.0~rc1) stable; urgency=medium

  * New Synapse release 1.84.0rc1.
@@ -27,7 +27,7 @@ ARG PYTHON_VERSION=3.11
 ###
 # We hardcode the use of Debian bullseye here because this could change upstream
 # and other Dockerfiles used for testing are expecting bullseye.
-FROM docker.io/python:${PYTHON_VERSION}-slim-bullseye as requirements
+FROM docker.io/library/python:${PYTHON_VERSION}-slim-bullseye as requirements

 # RUN --mount is specific to buildkit and is documented at
 # https://github.com/moby/buildkit/blob/master/frontend/dockerfile/docs/syntax.md#build-mounts-run---mount.
@@ -87,7 +87,7 @@ RUN if [ -z "$TEST_ONLY_IGNORE_POETRY_LOCKFILE" ]; then \
 ###
 ### Stage 1: builder
 ###
-FROM docker.io/python:${PYTHON_VERSION}-slim-bullseye as builder
+FROM docker.io/library/python:${PYTHON_VERSION}-slim-bullseye as builder

 # install the OS build deps
 RUN \
@@ -158,7 +158,7 @@ RUN --mount=type=cache,target=/synapse/target,sharing=locked \
 ### Stage 2: runtime
 ###

-FROM docker.io/python:${PYTHON_VERSION}-slim-bullseye
+FROM docker.io/library/python:${PYTHON_VERSION}-slim-bullseye

 LABEL org.opencontainers.image.url='https://matrix.org/docs/projects/server/synapse'
 LABEL org.opencontainers.image.documentation='https://github.com/matrix-org/synapse/blob/master/docker/README.md'
@@ -24,16 +24,16 @@ ARG distro=""
 # https://launchpad.net/~jyrki-pulliainen/+archive/ubuntu/dh-virtualenv, but
 # it's not obviously easier to use that than to build our own.)

-FROM ${distro} as builder
+FROM docker.io/library/${distro} as builder

 RUN apt-get update -qq -o Acquire::Languages=none
 RUN env DEBIAN_FRONTEND=noninteractive apt-get install \
-        -yqq --no-install-recommends \
-        build-essential \
-        ca-certificates \
-        devscripts \
-        equivs \
-        wget
+    -yqq --no-install-recommends \
+    build-essential \
+    ca-certificates \
+    devscripts \
+    equivs \
+    wget

 # fetch and unpack the package
 # We are temporarily using a fork of dh-virtualenv due to an incompatibility with Python 3.11, which ships with
@@ -55,40 +55,36 @@ RUN cd /dh-virtualenv && DEB_BUILD_OPTIONS=nodoc dpkg-buildpackage -us -uc -b
 ###
 ### Stage 1
 ###
-FROM ${distro}
+FROM docker.io/library/${distro}

 # Get the distro we want to pull from as a dynamic build variable
 # (We need to define it in each build stage)
 ARG distro=""
 ENV distro ${distro}

-# Python < 3.7 assumes LANG="C" means ASCII-only and throws on printing unicode
-# http://bugs.python.org/issue19846
-ENV LANG C.UTF-8
-
 # Install the build dependencies
 #
 # NB: keep this list in sync with the list of build-deps in debian/control
 # TODO: it would be nice to do that automatically.
 RUN apt-get update -qq -o Acquire::Languages=none \
    && env DEBIAN_FRONTEND=noninteractive apt-get install \
-        -yqq --no-install-recommends -o Dpkg::Options::=--force-unsafe-io \
-        build-essential \
-        curl \
-        debhelper \
-        devscripts \
-        libsystemd-dev \
-        lsb-release \
-        pkg-config \
-        python3-dev \
-        python3-pip \
-        python3-setuptools \
-        python3-venv \
-        sqlite3 \
-        libpq-dev \
-        libicu-dev \
-        pkg-config \
-        xmlsec1
+    -yqq --no-install-recommends -o Dpkg::Options::=--force-unsafe-io \
+    build-essential \
+    curl \
+    debhelper \
+    devscripts \
+    libsystemd-dev \
+    lsb-release \
+    pkg-config \
+    python3-dev \
+    python3-pip \
+    python3-setuptools \
+    python3-venv \
+    sqlite3 \
+    libpq-dev \
+    libicu-dev \
+    pkg-config \
+    xmlsec1

 # Install rust and ensure it's in the PATH
 ENV RUSTUP_HOME=/rust
@@ -7,7 +7,7 @@ ARG FROM=matrixdotorg/synapse:$SYNAPSE_VERSION
 # target image. For repeated rebuilds, this is much faster than apt installing
 # each time.

-FROM debian:bullseye-slim AS deps_base
+FROM docker.io/library/debian:bullseye-slim AS deps_base
    RUN \
       --mount=type=cache,target=/var/cache/apt,sharing=locked \
       --mount=type=cache,target=/var/lib/apt,sharing=locked \
@@ -21,7 +21,7 @@ FROM debian:bullseye-slim AS deps_base
 # which makes it much easier to copy (but we need to make sure we use an image
 # based on the same debian version as the synapse image, to make sure we get
 # the expected version of libc.
-FROM redis:6-bullseye AS redis_base
+FROM docker.io/library/redis:7-bullseye AS redis_base

 # now build the final image, based on the the regular Synapse docker image
 FROM $FROM
@@ -73,7 +73,8 @@ The following environment variables are supported in `generate` mode:
  will log sensitive information such as access tokens.
  This should not be needed unless you are a developer attempting to debug something
  particularly tricky.
-
+* `SYNAPSE_LOG_TESTING`: if set, Synapse will log additional information useful
+  for testing.

 ## Postgres

@@ -7,6 +7,7 @@
 # https://github.com/matrix-org/synapse/blob/develop/docker/README-testing.md#testing-with-postgresql-and-single-or-multi-process-synapse

 ARG SYNAPSE_VERSION=latest
+# This is an intermediate image, to be built locally (not pulled from a registry).
 ARG FROM=matrixdotorg/synapse-workers:$SYNAPSE_VERSION

 FROM $FROM
@@ -19,8 +20,8 @@ FROM $FROM
    # the same debian version as Synapse's docker image (so the versions of the
    # shared libraries match).
    RUN adduser --system --uid 999 postgres --home /var/lib/postgresql
-    COPY --from=postgres:13-bullseye /usr/lib/postgresql /usr/lib/postgresql
-    COPY --from=postgres:13-bullseye /usr/share/postgresql /usr/share/postgresql
+    COPY --from=docker.io/library/postgres:13-bullseye /usr/lib/postgresql /usr/lib/postgresql
+    COPY --from=docker.io/library/postgres:13-bullseye /usr/share/postgresql /usr/share/postgresql
    RUN mkdir /var/run/postgresql && chown postgres /var/run/postgresql
    ENV PATH="${PATH}:/usr/lib/postgresql/13/bin"
    ENV PGDATA=/var/lib/postgresql/data
@@ -92,8 +92,6 @@ allow_device_name_lookup_over_federation: true
 ## Experimental Features ##

 experimental_features:
-  # Enable history backfilling support
-  msc2716_enabled: true
  # client-side support for partial state in /send_join responses
  faster_joins: true
  # Enable support for polls
@@ -35,7 +35,11 @@ server {

    # Send all other traffic to the main process
    location ~* ^(\\/_matrix|\\/_synapse) {
+{% if using_unix_sockets %}
+        proxy_pass http://unix:/run/main_public.sock;
+{% else %}
        proxy_pass http://localhost:8080;
+{% endif %}
        proxy_set_header X-Forwarded-For $remote_addr;
        proxy_set_header X-Forwarded-Proto $scheme;
        proxy_set_header Host $host;
@@ -6,6 +6,9 @@
 {% if enable_redis %}
 redis:
    enabled: true
+    {% if using_unix_sockets %}
+    path: /tmp/redis.sock
+    {% endif %}
 {% endif %}

 {% if appservice_registrations is not none %}
@@ -19,7 +19,11 @@ username=www-data
 autorestart=true

 [program:redis]
+{% if using_unix_sockets %}
+command=/usr/local/bin/prefix-log /usr/local/bin/redis-server --unixsocket /tmp/redis.sock
+{% else %}
 command=/usr/local/bin/prefix-log /usr/local/bin/redis-server
+{% endif %}
 priority=1
 stdout_logfile=/dev/stdout
 stdout_logfile_maxbytes=0
@@ -8,7 +8,11 @@ worker_name: "{{ name }}"

 worker_listeners:
  - type: http
+{% if using_unix_sockets %}
+    path: "/run/worker.{{ port }}"
+{% else %}
    port: {{ port }}
+{% endif %}
 {% if listener_resources %}
    resources:
      - names:
@@ -36,12 +36,17 @@ listeners:

  # Allow configuring in case we want to reverse proxy 8008
  # using another process in the same container
+{% if SYNAPSE_USE_UNIX_SOCKET %}
+  # Unix sockets don't care about TLS or IP addresses or ports
+  - path: '/run/main_public.sock'
+    type: http
+{% else %}
  - port: {{ SYNAPSE_HTTP_PORT or 8008 }}
    tls: false
    bind_addresses: ['::']
    type: http
    x_forwarded: false
-
+{% endif %}
    resources:
      - names: [client]
        compress: true
@@ -57,8 +62,11 @@ database:
    user: "{{ POSTGRES_USER or "synapse" }}"
    password: "{{ POSTGRES_PASSWORD }}"
    database: "{{ POSTGRES_DB or "synapse" }}"
+{% if not SYNAPSE_USE_UNIX_SOCKET %}
+{# Synapse will use a default unix socket for Postgres when host/port is not specified (behavior from `psycopg2`). #}
    host: "{{ POSTGRES_HOST or "db" }}"
    port: "{{ POSTGRES_PORT or "5432" }}"
+{% endif %}
    cp_min: 5
    cp_max: 10
 {% else %}
@@ -49,17 +49,35 @@ handlers:
    class: logging.StreamHandler
    formatter: precise

-{% if not SYNAPSE_LOG_SENSITIVE %}
-{#
-  If SYNAPSE_LOG_SENSITIVE is unset, then override synapse.storage.SQL to INFO
-  so that DEBUG entries (containing sensitive information) are not emitted.
-#}
 loggers:
+    # This is just here so we can leave `loggers` in the config regardless of whether
+    # we configure other loggers below (avoid empty yaml dict error).
+    _placeholder:
+        level: "INFO"
+
+    {% if not SYNAPSE_LOG_SENSITIVE %}
+    {#
+      If SYNAPSE_LOG_SENSITIVE is unset, then override synapse.storage.SQL to INFO
+      so that DEBUG entries (containing sensitive information) are not emitted.
+    #}
    synapse.storage.SQL:
        # beware: increasing this to DEBUG will make synapse log sensitive
        # information such as access tokens.
        level: INFO
-{% endif %}
+    {% endif %}
+
+    {% if SYNAPSE_LOG_TESTING %}
+    {#
+      If Synapse is under test, log a few more useful things for a developer
+      attempting to debug something particularly tricky.
+
+      With `synapse.visibility.filtered_event_debug`, it logs when events are (maybe
+      unexpectedly) filtered out of responses in tests. It's just nice to be able to
+      look at the CI log and figure out why an event isn't being returned.
+    #}
+    synapse.visibility.filtered_event_debug:
+        level: DEBUG
+    {% endif %}

 root:
    level: {{ SYNAPSE_LOG_LEVEL or "INFO" }}
@@ -40,6 +40,8 @@
 #         log level. INFO is the default.
 #   * SYNAPSE_LOG_SENSITIVE: If unset, SQL and SQL values won't be logged,
 #         regardless of the SYNAPSE_LOG_LEVEL setting.
+#   * SYNAPSE_LOG_TESTING: if set, Synapse will log additional information useful
+#     for testing.
 #
 # NOTE: According to Complement's ENTRYPOINT expectations for a homeserver image (as defined
 # in the project's README), this script may be run multiple times, and functionality should
@@ -72,6 +74,9 @@ MAIN_PROCESS_HTTP_LISTENER_PORT = 8080
 MAIN_PROCESS_INSTANCE_NAME = "main"
 MAIN_PROCESS_LOCALHOST_ADDRESS = "127.0.0.1"
 MAIN_PROCESS_REPLICATION_PORT = 9093
+# Obviously, these would only be used with the UNIX socket option
+MAIN_PROCESS_UNIX_SOCKET_PUBLIC_PATH = "/run/main_public.sock"
+MAIN_PROCESS_UNIX_SOCKET_PRIVATE_PATH = "/run/main_private.sock"

 # A simple name used as a placeholder in the WORKERS_CONFIG below. This will be replaced
 # during processing with the name of the worker.
@@ -242,7 +247,6 @@ WORKERS_CONFIG: Dict[str, Dict[str, Any]] = {
            "^/_matrix/client/(api/v1|r0|v3|unstable)/join/",
            "^/_matrix/client/(api/v1|r0|v3|unstable)/knock/",
            "^/_matrix/client/(api/v1|r0|v3|unstable)/profile/",
-            "^/_matrix/client/(v1|unstable/org.matrix.msc2716)/rooms/.*/batch_send",
        ],
        "shared_extra_conf": {},
        "worker_extra_conf": "",
@@ -406,11 +410,15 @@ def add_worker_roles_to_shared_config(
        )

        # Map of stream writer instance names to host/ports combos
-        instance_map[worker_name] = {
-            "host": "localhost",
-            "port": worker_port,
-        }
-
+        if os.environ.get("SYNAPSE_USE_UNIX_SOCKET", False):
+            instance_map[worker_name] = {
+                "path": f"/run/worker.{worker_port}",
+            }
+        else:
+            instance_map[worker_name] = {
+                "host": "localhost",
+                "port": worker_port,
+            }
    # Update the list of stream writers. It's convenient that the name of the worker
    # type is the same as the stream to write. Iterate over the whole list in case there
    # is more than one.
@@ -422,10 +430,15 @@ def add_worker_roles_to_shared_config(

            # Map of stream writer instance names to host/ports combos
            # For now, all stream writers need http replication ports
-            instance_map[worker_name] = {
-                "host": "localhost",
-                "port": worker_port,
-            }
+            if os.environ.get("SYNAPSE_USE_UNIX_SOCKET", False):
+                instance_map[worker_name] = {
+                    "path": f"/run/worker.{worker_port}",
+                }
+            else:
+                instance_map[worker_name] = {
+                    "host": "localhost",
+                    "port": worker_port,
+                }


 def merge_worker_template_configs(
@@ -717,17 +730,29 @@ def generate_worker_files(
    # Note that yaml cares about indentation, so care should be taken to insert lines
    # into files at the correct indentation below.

+    # Convenience helper for if using unix sockets instead of host:port
+    using_unix_sockets = environ.get("SYNAPSE_USE_UNIX_SOCKET", False)
    # First read the original config file and extract the listeners block. Then we'll
    # add another listener for replication. Later we'll write out the result to the
    # shared config file.
-    listeners = [
-        {
-            "port": MAIN_PROCESS_REPLICATION_PORT,
-            "bind_address": MAIN_PROCESS_LOCALHOST_ADDRESS,
-            "type": "http",
-            "resources": [{"names": ["replication"]}],
-        }
-    ]
+    listeners: List[Any]
+    if using_unix_sockets:
+        listeners = [
+            {
+                "path": MAIN_PROCESS_UNIX_SOCKET_PRIVATE_PATH,
+                "type": "http",
+                "resources": [{"names": ["replication"]}],
+            }
+        ]
+    else:
+        listeners = [
+            {
+                "port": MAIN_PROCESS_REPLICATION_PORT,
+                "bind_address": MAIN_PROCESS_LOCALHOST_ADDRESS,
+                "type": "http",
+                "resources": [{"names": ["replication"]}],
+            }
+        ]
    with open(config_path) as file_stream:
        original_config = yaml.safe_load(file_stream)
        original_listeners = original_config.get("listeners")
@@ -768,7 +793,17 @@ def generate_worker_files(

    # A list of internal endpoints to healthcheck, starting with the main process
    # which exists even if no workers do.
-    healthcheck_urls = ["http://localhost:8080/health"]
+    # This list ends up being part of the command line to curl, (curl added support for
+    # Unix sockets in version 7.40).
+    if using_unix_sockets:
+        healthcheck_urls = [
+            f"--unix-socket {MAIN_PROCESS_UNIX_SOCKET_PUBLIC_PATH} "
+            # The scheme and hostname from the following URL are ignored.
+            # The only thing that matters is the path `/health`
+            "http://localhost/health"
+        ]
+    else:
+        healthcheck_urls = ["http://localhost:8080/health"]

    # Get the set of all worker types that we have configured
    all_worker_types_in_use = set(chain(*requested_worker_types.values()))
@@ -805,8 +840,12 @@ def generate_worker_files(
        # given worker_type needs to stay assigned and not be replaced.
        worker_config["shared_extra_conf"].update(shared_config)
        shared_config = worker_config["shared_extra_conf"]
-
-        healthcheck_urls.append("http://localhost:%d/health" % (worker_port,))
+        if using_unix_sockets:
+            healthcheck_urls.append(
+                f"--unix-socket /run/worker.{worker_port} http://localhost/health"
+            )
+        else:
+            healthcheck_urls.append("http://localhost:%d/health" % (worker_port,))

        # Update the shared config with sharding-related options if necessary
        add_worker_roles_to_shared_config(
@@ -825,6 +864,7 @@ def generate_worker_files(
            "/conf/workers/{name}.yaml".format(name=worker_name),
            **worker_config,
            worker_log_config_filepath=log_config_filepath,
+            using_unix_sockets=using_unix_sockets,
        )

        # Save this worker's port number to the correct nginx upstreams
@@ -845,8 +885,13 @@ def generate_worker_files(
    nginx_upstream_config = ""
    for upstream_worker_base_name, upstream_worker_ports in nginx_upstreams.items():
        body = ""
-        for port in upstream_worker_ports:
-            body += f"    server localhost:{port};\n"
+        if using_unix_sockets:
+            for port in upstream_worker_ports:
+                body += f"    server unix:/run/worker.{port};\n"
+
+        else:
+            for port in upstream_worker_ports:
+                body += f"    server localhost:{port};\n"

        # Add to the list of configured upstreams
        nginx_upstream_config += NGINX_UPSTREAM_CONFIG_BLOCK.format(
@@ -876,10 +921,15 @@ def generate_worker_files(
    # If there are workers, add the main process to the instance_map too.
    if workers_in_use:
        instance_map = shared_config.setdefault("instance_map", {})
-        instance_map[MAIN_PROCESS_INSTANCE_NAME] = {
-            "host": MAIN_PROCESS_LOCALHOST_ADDRESS,
-            "port": MAIN_PROCESS_REPLICATION_PORT,
-        }
+        if using_unix_sockets:
+            instance_map[MAIN_PROCESS_INSTANCE_NAME] = {
+                "path": MAIN_PROCESS_UNIX_SOCKET_PRIVATE_PATH,
+            }
+        else:
+            instance_map[MAIN_PROCESS_INSTANCE_NAME] = {
+                "host": MAIN_PROCESS_LOCALHOST_ADDRESS,
+                "port": MAIN_PROCESS_REPLICATION_PORT,
+            }

    # Shared homeserver config
    convert(
@@ -889,6 +939,7 @@ def generate_worker_files(
        appservice_registrations=appservice_registrations,
        enable_redis=workers_in_use,
        workers_in_use=workers_in_use,
+        using_unix_sockets=using_unix_sockets,
    )

    # Nginx config
@@ -899,6 +950,7 @@ def generate_worker_files(
        upstream_directives=nginx_upstream_config,
        tls_cert_path=os.environ.get("SYNAPSE_TLS_CERT"),
        tls_key_path=os.environ.get("SYNAPSE_TLS_KEY"),
+        using_unix_sockets=using_unix_sockets,
    )

    # Supervisord config
@@ -908,6 +960,7 @@ def generate_worker_files(
        "/etc/supervisor/supervisord.conf",
        main_config_path=config_path,
        enable_redis=workers_in_use,
+        using_unix_sockets=using_unix_sockets,
    )

    convert(
@@ -947,6 +1000,7 @@ def generate_worker_log_config(
    extra_log_template_args["SYNAPSE_LOG_SENSITIVE"] = environ.get(
        "SYNAPSE_LOG_SENSITIVE"
    )
+    extra_log_template_args["SYNAPSE_LOG_TESTING"] = environ.get("SYNAPSE_LOG_TESTING")

    # Render and write the file
    log_config_filepath = f"/conf/workers/{worker_name}.log.config"
@@ -10,7 +10,7 @@ ARG PYTHON_VERSION=3.9
 ###
 # We hardcode the use of Debian bullseye here because this could change upstream
 # and other Dockerfiles used for testing are expecting bullseye.
-FROM docker.io/python:${PYTHON_VERSION}-slim-bullseye
+FROM docker.io/library/python:${PYTHON_VERSION}-slim-bullseye

 # Install Rust and other dependencies (stolen from normal Dockerfile)
 # install the OS build deps
@@ -419,7 +419,7 @@ The following query parameters are available:

 * `from` (required) - The token to start returning events from. This token can be obtained from a prev_batch
  or next_batch token returned by the /sync endpoint, or from an end token returned by a previous request to this endpoint.
-* `to` - The token to spot returning events at.
+* `to` - The token to stop returning events at.
 * `limit` - The maximum number of events to return. Defaults to `10`.
 * `filter` - A JSON RoomEventFilter to filter returned events with.
 * `dir` - The direction to return events from. Either `f` for forwards or `b` for backwards. Setting
@@ -242,6 +242,9 @@ The following parameters should be set in the URL:

 - `dir` - Direction of media order. Either `f` for forwards or `b` for backwards.
  Setting this value to `b` will reverse the above sort order. Defaults to `f`.
+- `not_user_type` - Exclude certain user types, such as bot users, from the request.
+   Can be provided multiple times. Possible values are `bot`, `support` or "empty string".
+   "empty string" here means to exclude users without a type.

 Caution. The database only has indexes on the columns `name` and `creation_ts`.
 This means that if a different sort order is used (`is_guest`, `admin`,
@@ -1180,7 +1183,7 @@ The following parameters should be set in the URL:
 - `user_id` - The fully qualified MXID: for example, `@user:server.com`. The user must
  be local.

-### Check username availability
+## Check username availability

 Checks to see if a username is available, and valid, for the server. See [the client-server 
 API](https://matrix.org/docs/spec/client_server/r0.6.0#get-matrix-client-r0-register-available)
@@ -1198,7 +1201,7 @@ GET /_synapse/admin/v1/username_available?username=$localpart
 The request and response format is the same as the
 [/_matrix/client/r0/register/available](https://matrix.org/docs/spec/client_server/r0.6.0#get-matrix-client-r0-register-available) API.

-### Find a user based on their ID in an auth provider
+## Find a user based on their ID in an auth provider

 The API is:

@@ -1237,7 +1240,7 @@ Returns a `404` HTTP status code if no user was found, with a response body like
 _Added in Synapse 1.68.0._


-### Find a user based on their Third Party ID (ThreePID or 3PID)
+## Find a user based on their Third Party ID (ThreePID or 3PID)

 The API is:

@@ -23,7 +23,7 @@ people building from source should ensure they can fetch recent versions of Rust
 (e.g. by using [rustup](https://rustup.rs/)).

 The oldest supported version of SQLite is the version
-[provided](https://packages.debian.org/buster/libsqlite3-0) by
+[provided](https://packages.debian.org/bullseye/libsqlite3-0) by
 [Debian oldstable](https://wiki.debian.org/DebianOldStable).

 Context
@@ -22,6 +22,9 @@ on Windows is not officially supported.

 The code of Synapse is written in Python 3. To do pretty much anything, you'll need [a recent version of Python 3](https://www.python.org/downloads/). Your Python also needs support for [virtual environments](https://docs.python.org/3/library/venv.html). This is usually built-in, but some Linux distributions like Debian and Ubuntu split it out into its own package. Running `sudo apt install python3-venv` should be enough.

+A recent version of the Rust compiler is needed to build the native modules. The
+easiest way of installing the latest version is to use [rustup](https://rustup.rs/).
+
 Synapse can connect to PostgreSQL via the [psycopg2](https://pypi.org/project/psycopg2/) Python library. Building this library from source requires access to PostgreSQL's C header files. On Debian or Ubuntu Linux, these can be installed with `sudo apt install libpq-dev`.

 Synapse has an optional, improved user search with better Unicode support. For that you need the development package of `libicu`. On Debian or Ubuntu Linux, this can be installed with `sudo apt install libicu-dev`.
@@ -30,9 +33,6 @@ The source code of Synapse is hosted on GitHub. You will also need [a recent ver

 For some tests, you will need [a recent version of Docker](https://docs.docker.com/get-docker/).

-A recent version of the Rust compiler is needed to build the native modules. The
-easiest way of installing the latest version is to use [rustup](https://rustup.rs/).
-

 # 3. Get the source.

@@ -53,6 +53,11 @@ can find many good git tutorials on the web.

 # 4. Install the dependencies

+
+Before installing the Python dependencies, make sure you have installed a recent version
+of Rust (see the "What do I need?" section above). The easiest way of installing the
+latest version is to use [rustup](https://rustup.rs/).
+
 Synapse uses the [poetry](https://python-poetry.org/) project to manage its dependencies
 and development environment. Once you have installed Python 3 and added the
 source, you should install `poetry`.
@@ -76,7 +81,8 @@ cd path/where/you/have/cloned/the/repository
 poetry install --extras all
 ```

-This will install the runtime and developer dependencies for the project.
+This will install the runtime and developer dependencies for the project.  Be sure to check
+that the `poetry install` step completed cleanly.

 ## Running Synapse via poetry

@@ -84,14 +90,31 @@ To start a local instance of Synapse in the locked poetry environment, create a

 ```sh
 cp docs/sample_config.yaml homeserver.yaml
+cp docs/sample_log_config.yaml log_config.yaml
 ```

-Now edit homeserver.yaml, and run Synapse with:
+Now edit `homeserver.yaml`, things you might want to change include:
+
+- Set a `server_name`
+- Adjusting paths to be correct for your system like the `log_config` to point to the log config you just copied
+- Using a [PostgreSQL database instead of SQLite](https://matrix-org.github.io/synapse/latest/usage/configuration/config_documentation.html#database)
+- Adding a [`registration_shared_secret`](https://matrix-org.github.io/synapse/latest/usage/configuration/config_documentation.html#registration_shared_secret) so you can use [`register_new_matrix_user` command](https://matrix-org.github.io/synapse/latest/setup/installation.html#registering-a-user).
+
+And then run Synapse with the following command:

 ```sh
 poetry run python -m synapse.app.homeserver -c homeserver.yaml
 ```

+If you get an error like the following:
+
+```
+importlib.metadata.PackageNotFoundError: matrix-synapse
+```
+
+this probably indicates that the `poetry install` step did not complete cleanly - go back and
+resolve any issues and re-run until successful.
+
 # 5. Get in touch.

 Join our developer community on Matrix: [#synapse-dev:matrix.org](https://matrix.to/#/#synapse-dev:matrix.org)!
@@ -299,7 +322,7 @@ The following command will let you run the integration test with the most common
 configuration:

 ```sh
-$ docker run --rm -it -v /path/where/you/have/cloned/the/repository\:/src:ro -v /path/to/where/you/want/logs\:/logs matrixdotorg/sytest-synapse:buster
+$ docker run --rm -it -v /path/where/you/have/cloned/the/repository\:/src:ro -v /path/to/where/you/want/logs\:/logs matrixdotorg/sytest-synapse:focal
 ```
 (Note that the paths must be full paths! You could also write `$(realpath relative/path)` if needed.)

@@ -347,6 +370,7 @@ The above will run a monolithic (single-process) Synapse with SQLite as the data
    See the [worker documentation](../workers.md) for additional information on workers.
 - Passing `ASYNCIO_REACTOR=1` as an environment variable to use the Twisted asyncio reactor instead of the default one.
 - Passing `PODMAN=1` will use the [podman](https://podman.io/) container runtime, instead of docker.
+- Passing `UNIX_SOCKETS=1` will utilise Unix socket functionality for Synapse, Redis, and Postgres(when applicable).

 To increase the log level for the tests, set `SYNAPSE_TEST_LOG_LEVEL`, e.g:
 ```sh
@@ -260,15 +260,17 @@ doesn't require poetry. (It's what we use in CI too). However, you could try

 ## ...handle a Dependabot pull request?

-Synapse uses Dependabot to keep the `poetry.lock` file up-to-date. When it
-creates a pull request a GitHub Action will run to automatically create a changelog
-file. Ensure that:
+Synapse uses Dependabot to keep the `poetry.lock` and `Cargo.lock` file 
+up-to-date with the latest releases of our dependencies. The changelog check is
+omitted for Dependabot PRs; the release script will include them in the 
+changelog.
+
+When reviewing a dependabot PR, ensure that:

 * the lockfile changes look reasonable;
 * the upstream changelog file (linked in the description) doesn't include any
  breaking changes;
-* continuous integration passes (due to permissions, the GitHub Actions run on
-  the changelog commit will fail, look at the initial commit of the pull request);
+* continuous integration passes.

 In particular, any updates to the type hints (usually packages which start with `types-`)
 should be safe to merge if linting passes.
@@ -6,7 +6,7 @@ This is a work-in-progress set of notes with two goals:

 See also [MSC3902](https://github.com/matrix-org/matrix-spec-proposals/pull/3902).

-The key idea is described by [MSC706](https://github.com/matrix-org/matrix-spec-proposals/pull/3902). This allows servers to
+The key idea is described by [MSC3706](https://github.com/matrix-org/matrix-spec-proposals/pull/3706). This allows servers to
 request a lightweight response to the federation `/send_join` endpoint.
 This is called a **faster join**, also known as a **partial join**. In these
 notes we'll usually use the word "partial" as it matches the database schema.
@@ -46,6 +46,9 @@ instead.

 If the authentication is unsuccessful, the module must return `None`.

+Note that the user is not automatically registered, the `register_user(..)` method of
+the [module API](writing_a_module.html) can be used to lazily create users.
+
 If multiple modules register an auth checker for the same login type but with different
 fields, Synapse will refuse to start.

@@ -348,6 +348,42 @@ callback returns `False`, Synapse falls through to the next one. The value of th
 callback that does not return `False` will be used. If this happens, Synapse will not call
 any of the subsequent implementations of this callback.

+
+### `check_login_for_spam`
+
+_First introduced in Synapse v1.87.0_
+
+```python
+async def check_login_for_spam(
+    user_id: str,
+    device_id: Optional[str],
+    initial_display_name: Optional[str],
+    request_info: Collection[Tuple[Optional[str], str]],
+    auth_provider_id: Optional[str] = None,
+) -> Union["synapse.module_api.NOT_SPAM", "synapse.module_api.errors.Codes"]
+```
+
+Called when a user logs in.
+
+The arguments passed to this callback are:
+
+* `user_id`: The user ID the user is logging in with
+* `device_id`: The device ID the user is re-logging into.
+* `initial_display_name`: The device display name, if any.
+* `request_info`: A collection of tuples, which first item is a user agent, and which
+  second item is an IP address. These user agents and IP addresses are the ones that were
+  used during the login process.
+* `auth_provider_id`: The identifier of the SSO authentication provider, if any.
+
+If multiple modules implement this callback, they will be considered in order. If a
+callback returns `synapse.module_api.NOT_SPAM`, Synapse falls through to the next one.
+The value of the first callback that does not return `synapse.module_api.NOT_SPAM` will
+be used. If this happens, Synapse will not call any of the subsequent implementations of
+this callback.
+
+*Note:* This will not be called when a user registers.
+
+
 ## Example

 The example below is a module that implements the spam checker callback
@@ -68,9 +68,7 @@ root:
    # Write logs to the `buffer` handler, which will buffer them together in memory,
    # then write them to a file.
    #
-    # Replace "buffer" with "console" to log to stderr instead. (Note that you'll
-    # also need to update the configuration for the `twisted` logger above, in
-    # this case.)
+    # Replace "buffer" with "console" to log to stderr instead.
    #
    handlers: [buffer]

@@ -200,7 +200,7 @@ When following this route please make sure that the [Platform-specific prerequis
 System requirements:

 - POSIX-compliant system (tested on Linux & OS X)
- Python 3.7 or later, up to Python 3.11.
+- Python 3.8 or later, up to Python 3.11.
 - At least 1GB of free RAM if you want to join large public rooms like #matrix:matrix.org

 If building on an uncommon architecture for which pre-built wheels are
@@ -1,8 +1,4 @@
 worker_app: synapse.app.generic_worker
 worker_name: background_worker

-# The replication listener on the main synapse process.
-worker_replication_host: 127.0.0.1
-worker_replication_http_port: 9093
-
 worker_log_config: /etc/matrix-synapse/background-worker-log.yaml
@@ -1,9 +1,5 @@
 worker_app: synapse.app.generic_worker
-worker_name: event_persister1 
-
-# The replication listener on the main synapse process.
-worker_replication_host: 127.0.0.1
-worker_replication_http_port: 9093
+worker_name: event_persister1

 worker_listeners:
  - type: http
@@ -1,8 +1,4 @@
 worker_app: synapse.app.federation_sender
 worker_name: federation_sender1

-# The replication listener on the main synapse process.
-worker_replication_host: 127.0.0.1
-worker_replication_http_port: 9093
-
 worker_log_config: /etc/matrix-synapse/federation-sender-log.yaml
@@ -1,10 +1,6 @@
 worker_app: synapse.app.media_repository
 worker_name: media_worker

-# The replication listener on the main synapse process.
-worker_replication_host: 127.0.0.1
-worker_replication_http_port: 9093
-
 worker_listeners:
  - type: http
    port: 8085
@@ -1,8 +1,4 @@
 worker_app: synapse.app.pusher
 worker_name: pusher_worker1

-# The replication listener on the main synapse process.
-worker_replication_host: 127.0.0.1
-worker_replication_http_port: 9093
-
 worker_log_config: /etc/matrix-synapse/pusher-worker-log.yaml
@@ -88,19 +88,72 @@ process, for example:
    dpkg -i matrix-synapse-py3_1.3.0+stretch1_amd64.deb
    ```

+# Upgrading to v1.88.0
+
+## Minimum supported Python version
+
+The minimum supported Python version has been increased from v3.7 to v3.8.
+You will need Python 3.8 to run Synapse v1.88.0 (due out July 18th, 2023).
+
+If you use current versions of the Matrix.org-distributed Debian
+packages or Docker images, no action is required.
+
+## Removal of `worker_replication_*` settings
+
+As mentioned previously in [Upgrading to v1.84.0](#upgrading-to-v1840), the following deprecated settings
+are being removed in this release of Synapse:
+
+* [`worker_replication_host`](https://matrix-org.github.io/synapse/v1.86/usage/configuration/config_documentation.html#worker_replication_host)
+* [`worker_replication_http_port`](https://matrix-org.github.io/synapse/v1.86/usage/configuration/config_documentation.html#worker_replication_http_port)
+* [`worker_replication_http_tls`](https://matrix-org.github.io/synapse/v1.86/usage/configuration/config_documentation.html#worker_replication_http_tls)
+
+Please ensure that you have migrated to using `main` on your shared configuration's `instance_map`
+(or create one if necessary). This is required if you have ***any*** workers at all;
+administrators of single-process (monolith) installations don't need to do anything.
+
+For an illustrative example, please see [Upgrading to v1.84.0](#upgrading-to-v1840) below.
+
+
+# Upgrading to v1.86.0
+
+## Minimum supported Rust version
+
+The minimum supported Rust version has been increased from v1.58.1 to v1.60.0.
+Users building from source will need to ensure their `rustc` version is up to
+date.
+
+
+# Upgrading to v1.85.0
+
+## Application service registration with "user" property deprecation
+
+Application services should ensure they call the `/register` endpoint with a
+`username` property. The legacy `user` property is considered deprecated and
+should no longer be included.
+
+A future version of Synapse (v1.88.0 or later) will remove support for legacy
+application service login.
+
 # Upgrading to v1.84.0

 ## Deprecation of `worker_replication_*` configuration settings

-When using workers, 
+When using workers,
+
 * `worker_replication_host`
 * `worker_replication_http_port`
 * `worker_replication_http_tls`
 
-can now be removed from individual worker YAML configuration ***if*** you add the main process to the `instance_map` in the shared YAML configuration,
-using the name `main`.
+should now be removed from individual worker YAML configurations and the main process should instead be added to the `instance_map`
+in the shared YAML configuration, using the name `main`.
+
+The old `worker_replication_*` settings are now considered deprecated and are expected to be removed in Synapse v1.88.0.
+
+
+### Example change
+
+#### Before:

-### Before:
 Shared YAML
 ```yaml
 instance_map:
@@ -109,6 +162,7 @@ instance_map:
    port: 5678
    tls: false
 ```
+
 Worker YAML
 ```yaml
 worker_app: synapse.app.generic_worker
@@ -130,7 +184,10 @@ worker_listeners:

 worker_log_config: /etc/matrix-synapse/generic-worker-log.yaml
 ```
-### After:
+
+
+#### After:
+
 Shared YAML
 ```yaml
 instance_map:
@@ -143,6 +200,7 @@ instance_map:
    port: 5678
    tls: false
 ```
+
 Worker YAML
 ```yaml
 worker_app: synapse.app.generic_worker
@@ -165,7 +223,6 @@ Notes:
 * `tls` is optional but mirrors the functionality of `worker_replication_http_tls`


-
 # Upgrading to v1.81.0

 ## Application service path & authentication deprecations
@@ -27,9 +27,8 @@ What servers are currently participating in this room?
 Run this sql query on your db:
 ```sql
 SELECT DISTINCT split_part(state_key, ':', 2)
-    FROM current_state_events AS c
-    INNER JOIN room_memberships AS m USING (room_id, event_id)
-    WHERE room_id = '!cURbafjkfsMDVwdRDQ:matrix.org' AND membership = 'join';
+FROM current_state_events
+WHERE room_id = '!cURbafjkfsMDVwdRDQ:matrix.org' AND membership = 'join';
 ```

 What users are registered on my server?
@@ -462,6 +462,20 @@ See the docs [request log format](../administration/request_log.md).
 * `additional_resources`: Only valid for an 'http' listener. A map of
   additional endpoints which should be loaded via dynamic modules.

+Unix socket support (_Added in Synapse 1.89.0_):
+* `path`: A path and filename for a Unix socket. Make sure it is located in a
+  directory with read and write permissions, and that it already exists (the directory
+  will not be created). Defaults to `None`.
+  * **Note**: The use of both `path` and `port` options for the same `listener` is not
+    compatible.
+  * The `x_forwarded` option defaults to true  when using Unix sockets and can be omitted.
+  * Other options that would not make sense to use with a UNIX socket, such as 
+    `bind_addresses` and `tls` will be ignored and can be removed.
+* `mode`: The file permissions to set on the UNIX socket. Defaults to `666`
+* **Note:** Must be set as `type: http` (does not support `metrics` and `manhole`). 
+  Also make sure that `metrics` is not included in `resources` -> `names`
+
+
 Valid resource names are:

 * `client`: the client-server API (/_matrix/client), and the synapse admin API (/_synapse/admin). Also implies `media` and `static`.
@@ -474,7 +488,7 @@ Valid resource names are:

 * `media`: the media API (/_matrix/media).

-* `metrics`: the metrics interface. See [here](../../metrics-howto.md).
+* `metrics`: the metrics interface. See [here](../../metrics-howto.md). (Not compatible with Unix sockets)

 * `openid`: OpenID authentication. See [here](../../openid.md).

@@ -533,6 +547,22 @@ listeners:
    bind_addresses: ['::1', '127.0.0.1']
    type: manhole
 ```
+Example configuration #3:
+```yaml
+listeners:
+  # Unix socket listener: Ideal for Synapse deployments behind a reverse proxy, offering
+  # lightweight interprocess communication without TCP/IP overhead, avoid port
+  # conflicts, and providing enhanced security through system file permissions.
+  #
+  # Note that x_forwarded will default to true, when using a UNIX socket. Please see
+  # https://matrix-org.github.io/synapse/latest/reverse_proxy.html.
+  #
+  - path: /var/run/synapse/main_public.sock
+    type: http
+    resources:
+      - names: [client, federation]
+```
+
 ---
 ### `manhole_settings`

@@ -1196,6 +1226,32 @@ Example configuration:
 allow_device_name_lookup_over_federation: true
 ```
 ---
+### `federation`
+
+The federation section defines some sub-options related to federation.
+
+The following options are related to configuring timeout and retry logic for one request,
+independently of the others.
+Short retry algorithm is used when something or someone will wait for the request to have an
+answer, while long retry is used for requests that happen in the background,
+like sending a federation transaction.
+
+* `client_timeout`: timeout for the federation requests. Default to 60s.
+* `max_short_retry_delay`: maximum delay to be used for the short retry algo. Default to 2s.
+* `max_long_retry_delay`: maximum delay to be used for the short retry algo. Default to 60s.
+* `max_short_retries`: maximum number of retries for the short retry algo. Default to 3 attempts.
+* `max_long_retries`: maximum number of retries for the long retry algo. Default to 10 attempts.
+
+Example configuration:
+```yaml
+federation:
+  client_timeout: 180s
+  max_short_retry_delay: 7s
+  max_long_retry_delay: 100s
+  max_short_retries: 5
+  max_long_retries: 20
+```
+---
 ## Caching

 Options related to caching.
@@ -2570,7 +2626,50 @@ Example configuration:
 ```yaml
 nonrefreshable_access_token_lifetime: 24h
 ```
+---
+### `ui_auth`

+The amount of time to allow a user-interactive authentication session to be active.
+
+This defaults to 0, meaning the user is queried for their credentials
+before every action, but this can be overridden to allow a single
+validation to be re-used.  This weakens the protections afforded by
+the user-interactive authentication process, by allowing for multiple
+(and potentially different) operations to use the same validation session.
+
+This is ignored for potentially "dangerous" operations (including
+deactivating an account, modifying an account password, adding a 3PID,
+and minting additional login tokens).
+
+Use the `session_timeout` sub-option here to change the time allowed for credential validation.
+
+Example configuration:
+```yaml
+ui_auth:
+    session_timeout: "15s"
+```
+---
+### `login_via_existing_session`
+
+Matrix supports the ability of an existing session to mint a login token for
+another client.
+
+Synapse disables this by default as it has security ramifications -- a malicious
+client could use the mechanism to spawn more than one session.
+
+The duration of time the generated token is valid for can be configured with the
+`token_timeout` sub-option.
+
+User-interactive authentication is required when this is enabled unless the
+`require_ui_auth` sub-option is set to `False`.
+
+Example configuration:
+```yaml
+login_via_existing_session:
+    enabled: true
+    require_ui_auth: false
+    token_timeout: "5m"
+```
 ---
 ## Metrics
 Config options related to metrics.
@@ -3415,28 +3514,6 @@ password_config:
      require_uppercase: true
 ```
 ---
-### `ui_auth`
-
-The amount of time to allow a user-interactive authentication session to be active.
-
-This defaults to 0, meaning the user is queried for their credentials
-before every action, but this can be overridden to allow a single
-validation to be re-used.  This weakens the protections afforded by
-the user-interactive authentication process, by allowing for multiple
-(and potentially different) operations to use the same validation session.
-
-This is ignored for potentially "dangerous" operations (including
-deactivating an account, modifying an account password, and
-adding a 3PID).
-
-Use the `session_timeout` sub-option here to change the time allowed for credential validation.
-
-Example configuration:
-```yaml
-ui_auth:
-    session_timeout: "15s"
-```
---
 ## Push
 Configuration settings related to push notifications

@@ -3902,6 +3979,14 @@ instance_map:
    host: localhost
    port: 8034
 ```
+Example configuration(#2, for UNIX sockets):
+```yaml
+instance_map:
+  main:
+    path: /var/run/synapse/main_replication.sock
+  worker1:
+    path: /var/run/synapse/worker1_replication.sock
+```
 ---
 ### `stream_writers`

@@ -3979,6 +4064,8 @@ This setting has the following sub-options:
 * `enabled`: whether to use Redis support. Defaults to false.
 * `host` and `port`: Optional host and port to use to connect to redis. Defaults to
   localhost and 6379
+* `path`: The full path to a local Unix socket file. **If this is used, `host` and
+ `port` are ignored.** Defaults to `/tmp/redis.sock'
 * `password`: Optional password if configured on the Redis instance.
 * `dbid`: Optional redis dbid if needs to connect to specific redis logical db.
 * `use_tls`: Whether to use tls connection. Defaults to false.
@@ -3991,6 +4078,8 @@ This setting has the following sub-options:

  _Changed in Synapse 1.84.0: Added use\_tls, certificate\_file, private\_key\_file, ca\_file and ca\_path attributes_

+  _Changed in Synapse 1.85.0: Added path option to use a local Unix socket_
+
 Example configuration:
 ```yaml
 redis:
@@ -4039,51 +4128,6 @@ Example configuration:
 worker_name: generic_worker1
 ```
 ---
-### `worker_replication_host`
-*Deprecated as of version 1.84.0. Place `host` under `main` entry on the [`instance_map`](#instance_map) in your shared yaml configuration instead.*
-
-The HTTP replication endpoint that it should talk to on the main Synapse process.
-The main Synapse process defines this with a `replication` resource in
-[`listeners` option](#listeners).
-
-Example configuration:
-```yaml
-worker_replication_host: 127.0.0.1
-```
---
-### `worker_replication_http_port`
-*Deprecated as of version 1.84.0. Place `port` under `main` entry on the [`instance_map`](#instance_map) in your shared yaml configuration instead.*
-
-The HTTP replication port that it should talk to on the main Synapse process.
-The main Synapse process defines this with a `replication` resource in
-[`listeners` option](#listeners).
-
-Example configuration:
-```yaml
-worker_replication_http_port: 9093
-```
---
-### `worker_replication_http_tls`
-*Deprecated as of version 1.84.0. Place `tls` under `main` entry on the [`instance_map`](#instance_map) in your shared yaml configuration instead.*
-
-Whether TLS should be used for talking to the HTTP replication port on the main
-Synapse process.
-The main Synapse process defines this with the `tls` option on its [listener](#listeners) that
-has the `replication` resource enabled.
-
-**Please note:** by default, it is not safe to expose replication ports to the
-public Internet, even with TLS enabled.
-See [`worker_replication_secret`](#worker_replication_secret).
-
-Defaults to `false`.
-
-*Added in Synapse 1.72.0.*
-
-Example configuration:
-```yaml
-worker_replication_http_tls: true
-```
---
 ### `worker_listeners`

 A worker can handle HTTP requests. To do so, a `worker_listeners` option
@@ -4102,6 +4146,18 @@ worker_listeners:
    resources:
      - names: [client, federation]
 ```
+Example configuration(#2, using UNIX sockets with a `replication` listener):
+```yaml
+worker_listeners:
+  - type: http
+    path: /var/run/synapse/worker_public.sock
+    resources:
+      - names: [client, federation]
+  - type: http
+    path: /var/run/synapse/worker_replication.sock
+    resources:
+      - names: [replication]
+```
 ---
 ### `worker_manhole`

@@ -95,9 +95,12 @@ for the main process
 * Secondly, you need to enable
 [redis-based replication](usage/configuration/config_documentation.md#redis)
 * You will need to add an [`instance_map`](usage/configuration/config_documentation.md#instance_map) 
-with the `main` process defined, as well as the relevant connection information from 
-it's HTTP `replication` listener (defined in step 1 above). Note that the `host` defined 
-is the address the worker needs to look for the `main` process at, not necessarily the same address that is bound to.
+with the `main` process defined, as well as the relevant connection information from
+it's HTTP `replication` listener (defined in step 1 above).
+  * Note that the `host` defined is the address the worker needs to look for the `main`
+  process at, not necessarily the same address that is bound to.
+  * If you are using Unix sockets for the `replication` resource, make sure to
+  use a `path` to the socket file instead of a `port`.
 * Optionally, a [shared secret](usage/configuration/config_documentation.md#worker_replication_secret)
 can be used to authenticate HTTP traffic between workers. For example:

@@ -145,9 +148,6 @@ In the config file for each worker, you must specify:
   with an `http` listener.
 * **Synapse 1.72 and older:** if handling the `^/_matrix/client/v3/keys/upload` endpoint, the HTTP URI for
   the main process (`worker_main_http_uri`). This config option is no longer required and is ignored when running Synapse 1.73 and newer.
- * **Synapse 1.83 and older:** The HTTP replication endpoint that the worker should talk to on the main synapse process
-   ([`worker_replication_host`](usage/configuration/config_documentation.md#worker_replication_host) and
-   [`worker_replication_http_port`](usage/configuration/config_documentation.md#worker_replication_http_port)). If using Synapse 1.84 and newer, these are not needed if `main` is defined on the [shared configuration](#shared-configuration) `instance_map`

 For example:

@@ -177,11 +177,11 @@ The following applies to Synapse installations that have been installed from sou

 You can start the main Synapse process with Poetry by running the following command:
 ```console
-poetry run synapse_homeserver -c [your homeserver.yaml]
+poetry run synapse_homeserver --config-file [your homeserver.yaml]
 ```
 For worker setups, you can run the following command
 ```console
-poetry run synapse_worker -c [your worker.yaml]
+poetry run synapse_worker --config-file [your homeserver.yaml] --config-file [your worker.yaml]
 ```
 ## Available worker applications

@@ -232,7 +232,6 @@ information.
    ^/_matrix/client/v1/rooms/.*/hierarchy$
    ^/_matrix/client/(v1|unstable)/rooms/.*/relations/
    ^/_matrix/client/v1/rooms/.*/threads$
-    ^/_matrix/client/unstable/org.matrix.msc2716/rooms/.*/batch_send$
    ^/_matrix/client/unstable/im.nheko.summary/rooms/.*/summary$
    ^/_matrix/client/(r0|v3|unstable)/account/3pid$
    ^/_matrix/client/(r0|v3|unstable)/account/whoami$
@@ -95,6 +95,10 @@

                  # For building the Synapse documentation website.
                  mdbook
+
+                  # For releasing Synapse
+                  debian-devscripts # (`dch` for manipulating the Debian changelog)
+                  libnotify # (the release script uses `notify-send` to tell you when CI jobs are done)
                ];

                # Install Python and manage a virtualenv with Poetry.
@@ -174,7 +178,7 @@
                  EOF
                '';
                # Start synapse when `devenv up` is run.
-                processes.synapse.exec = "poetry run python -m synapse.app.homeserver -c homeserver.yaml --config-directory homeserver-config-overrides.d";
+                processes.synapse.exec = "poetry run python -m synapse.app.homeserver -c homeserver.yaml -c homeserver-config-overrides.d";

                # Define the perl modules we require to run SyTest.
                #
@@ -2,17 +2,29 @@
 namespace_packages = True
 plugins = pydantic.mypy, mypy_zope:plugin, scripts-dev/mypy_synapse_plugin.py
 follow_imports = normal
-check_untyped_defs = True
 show_error_codes = True
 show_traceback = True
 mypy_path = stubs
 warn_unreachable = True
-warn_unused_ignores = True
 local_partial_types = True
 no_implicit_optional = True
+
+# Strict checks, see mypy --help
+warn_unused_configs = True
+# disallow_any_generics = True
+disallow_subclassing_any = True
+# disallow_untyped_calls = True
 disallow_untyped_defs = True
-strict_equality = True
+disallow_incomplete_defs = True
+# check_untyped_defs = True
+# disallow_untyped_decorators = True
 warn_redundant_casts = True
+warn_unused_ignores = True
+# warn_return_any = True
+# no_implicit_reexport = True
+strict_equality = True
+strict_concatenate = True
+
 # Run mypy type checking with the minimum supported Python version to catch new usage
 # that isn't backwards-compatible (types, overloads, etc).
 python_version = 3.8
@@ -31,9 +43,7 @@ warn_unused_ignores = False

 [mypy-synapse.util.caches.treecache]
 disallow_untyped_defs = False
-
-[mypy-tests.util.caches.test_descriptors]
-disallow_untyped_defs = False
+disallow_incomplete_defs = False

 ;; Dependencies without annotations
 ;; Before ignoring a module, check to see if type stubs are available.
@@ -43,18 +53,18 @@ disallow_untyped_defs = False
 ;; which we can pull in as a dev dependency by adding to `pyproject.toml`'s
 ;; `[tool.poetry.dev-dependencies]` list.

+# https://github.com/lepture/authlib/issues/460
 [mypy-authlib.*]
 ignore_missing_imports = True

 [mypy-ijson.*]
 ignore_missing_imports = True

-[mypy-lxml]
-ignore_missing_imports = True
-
+# https://github.com/msgpack/msgpack-python/issues/448
 [mypy-msgpack]
 ignore_missing_imports = True

+# https://github.com/wolever/parameterized/issues/143
 [mypy-parameterized.*]
 ignore_missing_imports = True

@@ -76,6 +86,7 @@ ignore_missing_imports = True
 [mypy-srvlookup.*]
 ignore_missing_imports = True

+# https://github.com/twisted/treq/pull/366
 [mypy-treq.*]
 ignore_missing_imports = True

@@ -89,7 +89,7 @@ manifest-path = "rust/Cargo.toml"

 [tool.poetry]
 name = "matrix-synapse"
-version = "1.84.0rc1"
+version = "1.88.0rc1"
 description = "Homeserver for the Matrix decentralised comms protocol"
 authors = ["Matrix.org Team and Contributors <packages@matrix.org>"]
 license = "Apache-2.0"
@@ -147,7 +147,7 @@ synapse_review_recent_signups = "synapse._scripts.review_recent_signups:main"
 update_synapse_database = "synapse._scripts.update_synapse_database:main"

 [tool.poetry.dependencies]
-python = "^3.7.1"
+python = "^3.8.0"

 # Mandatory Dependencies
 # ----------------------
@@ -203,11 +203,9 @@ ijson = ">=3.1.4"
 matrix-common = "^1.3.0"
 # We need packaging.requirements.Requirement, added in 16.1.
 packaging = ">=16.1"
-# At the time of writing, we only use functions from the version `importlib.metadata`
-# which shipped in Python 3.8. This corresponds to version 1.4 of the backport.
-importlib_metadata = { version = ">=1.4", python = "<3.8" }
 # This is the most recent version of Pydantic with available on common distros.
-pydantic = ">=1.7.4"
+# We are currently incompatible with >=2.0.0: (https://github.com/matrix-org/synapse/issues/15858)
+pydantic = "^1.7.4"

 # This is for building the rust components during "poetry install", which
 # currently ignores the `build-system.requires` directive (c.f.
@@ -311,9 +309,10 @@ all = [
 # We pin black so that our tests don't start failing on new releases.
 isort = ">=5.10.1"
 black = ">=22.3.0"
-ruff = "0.0.265"
+ruff = "0.0.277"

 # Typechecking
+lxml-stubs = ">=0.4.0"
 mypy = "*"
 mypy-zope = "*"
 types-bleach = ">=4.1.0"
@@ -374,7 +373,15 @@ build-backend = "poetry.core.masonry.api"

 [tool.cibuildwheel]
 # Skip unsupported platforms (by us or by Rust).
-skip = "cp36* *-musllinux_i686 pp*aarch64 *-musllinux_aarch64"
+# See https://cibuildwheel.readthedocs.io/en/stable/options/#build-skip for the list of build targets.
+# We skip:
+#  - CPython 3.6 and 3.7: EOLed
+#  - PyPy 3.7: we only support Python 3.8+
+#  - musllinux i686: excluded to reduce number of wheels we build.
+#    c.f. https://github.com/matrix-org/synapse/pull/12595#discussion_r963107677
+#  - PyPy on Aarch64 and musllinux on aarch64: too slow to build.
+#    c.f. https://github.com/matrix-org/synapse/pull/14259
+skip = "cp36* cp37* pp37* *-musllinux_i686 pp*aarch64 *-musllinux_aarch64"

 # We need a rust compiler
 before-all =  "curl https://sh.rustup.rs -sSf | sh -s -- --default-toolchain stable -y --profile minimal"
@@ -7,7 +7,7 @@ name = "synapse"
 version = "0.1.0"

 edition = "2021"
-rust-version = "1.58.1"
+rust-version = "1.60.0"

 [lib]
 name = "synapse"
@@ -13,8 +13,6 @@
 // limitations under the License.

 #![feature(test)]
-use std::collections::BTreeSet;
-
 use synapse::push::{
    evaluator::PushRuleEvaluator, Condition, EventMatchCondition, FilteredPushRules, JsonValue,
    PushRules, SimpleJsonValue,
@@ -197,7 +195,6 @@ fn bench_eval_message(b: &mut Bencher) {
        false,
        false,
        false,
-        false,
    );

    b.iter(|| eval.run(&rules, Some("bob"), Some("person")));
@@ -142,11 +142,11 @@ pub const BASE_APPEND_OVERRIDE_RULES: &[PushRule] = &[
        default_enabled: true,
    },
    PushRule {
-        rule_id: Cow::Borrowed(".org.matrix.msc3952.is_user_mention"),
+        rule_id: Cow::Borrowed("global/override/.m.rule.is_user_mention"),
        priority_class: 5,
        conditions: Cow::Borrowed(&[Condition::Known(
            KnownCondition::ExactEventPropertyContainsType(EventPropertyIsTypeCondition {
-                key: Cow::Borrowed("content.org\\.matrix\\.msc3952\\.mentions.user_ids"),
+                key: Cow::Borrowed("content.m\\.mentions.user_ids"),
                value_type: Cow::Borrowed(&EventMatchPatternType::UserId),
            }),
        )]),
@@ -163,11 +163,11 @@ pub const BASE_APPEND_OVERRIDE_RULES: &[PushRule] = &[
        default_enabled: true,
    },
    PushRule {
-        rule_id: Cow::Borrowed(".org.matrix.msc3952.is_room_mention"),
+        rule_id: Cow::Borrowed("global/override/.m.rule.is_room_mention"),
        priority_class: 5,
        conditions: Cow::Borrowed(&[
            Condition::Known(KnownCondition::EventPropertyIs(EventPropertyIsCondition {
-                key: Cow::Borrowed("content.org\\.matrix\\.msc3952\\.mentions.room"),
+                key: Cow::Borrowed("content.m\\.mentions.room"),
                value: Cow::Borrowed(&SimpleJsonValue::Bool(true)),
            })),
            Condition::Known(KnownCondition::SenderNotificationPermission {
@@ -70,7 +70,9 @@ pub struct PushRuleEvaluator {
    /// The "content.body", if any.
    body: String,

-    /// True if the event has a mentions property and MSC3952 support is enabled.
+    /// True if the event has a m.mentions property. (Note that this is a separate
+    /// flag instead of checking flattened_keys since the m.mentions property
+    /// might be an empty map and not appear in flattened_keys.
    has_mentions: bool,

    /// The number of users in the room.
@@ -155,9 +157,7 @@ impl PushRuleEvaluator {
            let rule_id = &push_rule.rule_id().to_string();

            // For backwards-compatibility the legacy mention rules are disabled
-            // if the event contains the 'm.mentions' property (and if the
-            // experimental feature is enabled, both of these are represented
-            // by the has_mentions flag).
+            // if the event contains the 'm.mentions' property.
            if self.has_mentions
                && (rule_id == "global/override/.m.rule.contains_display_name"
                    || rule_id == "global/content/.m.rule.contains_user_name"
@@ -562,7 +562,7 @@ fn test_requires_room_version_supports_condition() {
    };
    let rules = PushRules::new(vec![custom_rule]);
    result = evaluator.run(
-        &FilteredPushRules::py_new(rules, BTreeMap::new(), true, false, true, false, false),
+        &FilteredPushRules::py_new(rules, BTreeMap::new(), true, false, true, false),
        None,
        None,
    );
@@ -527,7 +527,6 @@ pub struct FilteredPushRules {
    msc1767_enabled: bool,
    msc3381_polls_enabled: bool,
    msc3664_enabled: bool,
-    msc3952_intentional_mentions: bool,
    msc3958_suppress_edits_enabled: bool,
 }

@@ -540,7 +539,6 @@ impl FilteredPushRules {
        msc1767_enabled: bool,
        msc3381_polls_enabled: bool,
        msc3664_enabled: bool,
-        msc3952_intentional_mentions: bool,
        msc3958_suppress_edits_enabled: bool,
    ) -> Self {
        Self {
@@ -549,7 +547,6 @@ impl FilteredPushRules {
            msc1767_enabled,
            msc3381_polls_enabled,
            msc3664_enabled,
-            msc3952_intentional_mentions,
            msc3958_suppress_edits_enabled,
        }
    }
@@ -587,10 +584,6 @@ impl FilteredPushRules {
                    return false;
                }

-                if !self.msc3952_intentional_mentions && rule.rule_id.contains("org.matrix.msc3952")
-                {
-                    return false;
-                }
                if !self.msc3958_suppress_edits_enabled
                    && rule.rule_id == "global/override/.com.beeper.suppress_edits"
                {
@@ -20,15 +20,20 @@ from concurrent.futures import ThreadPoolExecutor
 from types import FrameType
 from typing import Collection, Optional, Sequence, Set

+# These are expanded inside the dockerfile to be a fully qualified image name.
+# e.g. docker.io/library/debian:bullseye
+#
+# If an EOL is forced by a Python version and we're dropping support for it, make sure
+# to remove references to the distibution across Synapse (search for "bullseye" for
+# example)
 DISTS = (
-    "debian:buster",  # oldstable: EOL 2022-08
-    "debian:bullseye",
-    "debian:bookworm",
-    "debian:sid",
-    "ubuntu:focal",  # 20.04 LTS (our EOL forced by Py38 on 2024-10-14)
-    "ubuntu:jammy",  # 22.04 LTS (EOL 2027-04)
-    "ubuntu:kinetic",  # 22.10 (EOL 2023-07-20)
-    "ubuntu:lunar",  # 23.04 (EOL 2024-01)
+    "debian:bullseye",  # (EOL ~2024-07) (our EOL forced by Python 3.9 is 2025-10-05)
+    "debian:bookworm",  # (EOL not specified yet) (our EOL forced by Python 3.11 is 2027-10-24)
+    "debian:sid",  # (EOL not specified yet) (our EOL forced by Python 3.11 is 2027-10-24)
+    "ubuntu:focal",  # 20.04 LTS (EOL 2025-04) (our EOL forced by Python 3.8 is 2024-10-14)
+    "ubuntu:jammy",  # 22.04 LTS (EOL 2027-04) (our EOL forced by Python 3.10 is 2026-10-04)
+    "ubuntu:kinetic",  # 22.10 (EOL 2023-07-20) (our EOL forced by Python 3.10 is 2026-10-04)
+    "ubuntu:lunar",  # 23.04 (EOL 2024-01) (our EOL forced by Python 3.11 is 2027-10-24)
 )

 DESC = """\
@@ -246,10 +246,6 @@ else
  else
    export PASS_SYNAPSE_COMPLEMENT_DATABASE=sqlite
  fi
-
-  # The tests for importing historical messages (MSC2716)
-  # only pass with monoliths, currently.
-  test_tags="$test_tags,msc2716"
 fi

 if [[ -n "$ASYNCIO_REACTOR" ]]; then
@@ -257,6 +253,10 @@ if [[ -n "$ASYNCIO_REACTOR" ]]; then
  export PASS_SYNAPSE_COMPLEMENT_USE_ASYNCIO_REACTOR=true
 fi

+if [[ -n "$UNIX_SOCKETS" ]]; then
+  # Enable full on Unix socket mode for Synapse, Redis and Postgresql
+  export PASS_SYNAPSE_USE_UNIX_SOCKET=1
+fi

 if [[ -n "$SYNAPSE_TEST_LOG_LEVEL" ]]; then
  # Set the log level to what is desired
@@ -269,6 +269,10 @@ if [[ -n "$SYNAPSE_TEST_LOG_LEVEL" ]]; then
  export PASS_SYNAPSE_LOG_SENSITIVE=1
 fi

+# Log a few more useful things for a developer attempting to debug something
+# particularly tricky.
+export PASS_SYNAPSE_LOG_TESTING=1
+
 # Run the tests!
 echo "Images built; running complement"
 cd "$COMPLEMENT_DIR"
@@ -136,11 +136,11 @@ def request(
        authorization_headers.append(header)
        print("Authorization: %s" % header, file=sys.stderr)

-    dest = "matrix://%s%s" % (destination, path)
+    dest = "matrix-federation://%s%s" % (destination, path)
    print("Requesting %s" % dest, file=sys.stderr)

    s = requests.Session()
-    s.mount("matrix://", MatrixConnectionAdapter())
+    s.mount("matrix-federation://", MatrixConnectionAdapter())

    headers: Dict[str, str] = {
        "Authorization": authorization_headers[0],
@@ -18,10 +18,11 @@ can crop up, e.g the cache descriptors.

 from typing import Callable, Optional, Type

+from mypy.erasetype import remove_instance_last_known_values
 from mypy.nodes import ARG_NAMED_OPT
 from mypy.plugin import MethodSigContext, Plugin
 from mypy.typeops import bind_self
-from mypy.types import CallableType, NoneType, UnionType
+from mypy.types import CallableType, Instance, NoneType, UnionType


 class SynapsePlugin(Plugin):
@@ -92,10 +93,41 @@ def cached_function_method_signature(ctx: MethodSigContext) -> CallableType:
    arg_names.append("on_invalidate")
    arg_kinds.append(ARG_NAMED_OPT)  # Arg is an optional kwarg.

+    # Finally we ensure the return type is a Deferred.
+    if (
+        isinstance(signature.ret_type, Instance)
+        and signature.ret_type.type.fullname == "twisted.internet.defer.Deferred"
+    ):
+        # If it is already a Deferred, nothing to do.
+        ret_type = signature.ret_type
+    else:
+        ret_arg = None
+        if isinstance(signature.ret_type, Instance):
+            # If a coroutine, wrap the coroutine's return type in a Deferred.
+            if signature.ret_type.type.fullname == "typing.Coroutine":
+                ret_arg = signature.ret_type.args[2]
+
+            # If an awaitable, wrap the awaitable's final value in a Deferred.
+            elif signature.ret_type.type.fullname == "typing.Awaitable":
+                ret_arg = signature.ret_type.args[0]
+
+        # Otherwise, wrap the return value in a Deferred.
+        if ret_arg is None:
+            ret_arg = signature.ret_type
+
+        # This should be able to use ctx.api.named_generic_type, but that doesn't seem
+        # to find the correct symbol for anything more than 1 module deep.
+        #
+        # modules is not part of CheckerPluginInterface. The following is a combination
+        # of TypeChecker.named_generic_type and TypeChecker.lookup_typeinfo.
+        sym = ctx.api.modules["twisted.internet.defer"].names.get("Deferred")  # type: ignore[attr-defined]
+        ret_type = Instance(sym.node, [remove_instance_last_known_values(ret_arg)])
+
    signature = signature.copy_modified(
        arg_types=arg_types,
        arg_names=arg_names,
        arg_kinds=arg_kinds,
+        ret_type=ret_type,
    )

    return signature
@@ -27,7 +27,7 @@ import time
 import urllib.request
 from os import path
 from tempfile import TemporaryDirectory
-from typing import Any, List, Optional
+from typing import Any, List, Match, Optional, Union

 import attr
 import click
@@ -233,7 +233,7 @@ def _prepare() -> None:
    subprocess.check_output(["poetry", "version", new_version])

    # Generate changelogs.
-    generate_and_write_changelog(current_version, new_version)
+    generate_and_write_changelog(synapse_repo, current_version, new_version)

    # Generate debian changelogs
    if parsed_new_version.pre is not None:
@@ -814,7 +814,7 @@ def get_changes_for_version(wanted_version: version.Version) -> str:


 def generate_and_write_changelog(
-    current_version: version.Version, new_version: str
+    repo: Repo, current_version: version.Version, new_version: str
 ) -> None:
    # We do this by getting a draft so that we can edit it before writing to the
    # changelog.
@@ -827,6 +827,10 @@ def generate_and_write_changelog(
    new_changes = new_changes.replace(
        "No significant changes.", f"No significant changes since {current_version}."
    )
+    new_changes += build_dependabot_changelog(
+        repo,
+        current_version,
+    )

    # Prepend changes to changelog
    with open("CHANGES.md", "r+") as f:
@@ -841,5 +845,49 @@ def generate_and_write_changelog(
        os.remove(filename)


+def build_dependabot_changelog(repo: Repo, current_version: version.Version) -> str:
+    """Summarise dependabot commits between `current_version` and `release_branch`.
+
+    Returns an empty string if there have been no such commits; otherwise outputs a
+    third-level markdown header followed by an unordered list."""
+    last_release_commit = repo.tag("v" + str(current_version)).commit
+    rev_spec = f"{last_release_commit.hexsha}.."
+    commits = list(git.objects.Commit.iter_items(repo, rev_spec))
+    messages = []
+    for commit in reversed(commits):
+        if commit.author.name == "dependabot[bot]":
+            message: Union[str, bytes] = commit.message
+            if isinstance(message, bytes):
+                message = message.decode("utf-8")
+            messages.append(message.split("\n", maxsplit=1)[0])
+
+    if not messages:
+        print(f"No dependabot commits in range {rev_spec}", file=sys.stderr)
+        return ""
+
+    messages.sort()
+
+    def replacer(match: Match[str]) -> str:
+        desc = match.group(1)
+        number = match.group(2)
+        return f"* {desc}. ([\\#{number}](https://github.com/matrix-org/synapse/issues/{number}))"
+
+    for i, message in enumerate(messages):
+        messages[i] = re.sub(r"(.*) \(#(\d+)\)$", replacer, message)
+    messages.insert(0, "### Updates to locked dependencies\n")
+    # Add an extra blank line to the bottom of the section
+    messages.append("")
+    return "\n".join(messages)
+
+
+@cli.command()
+@click.argument("since")
+def test_dependabot_changelog(since: str) -> None:
+    """Test building the dependabot changelog.
+
+    Summarises all dependabot commits between the SINCE tag and the current git HEAD."""
+    print(build_dependabot_changelog(git.Repo("."), version.Version(since)))
+
+
 if __name__ == "__main__":
    cli()
@@ -46,7 +46,6 @@ class FilteredPushRules:
        msc1767_enabled: bool,
        msc3381_polls_enabled: bool,
        msc3664_enabled: bool,
-        msc3952_intentional_mentions: bool,
        msc3958_suppress_edits_enabled: bool,
    ): ...
    def rules(self) -> Collection[Tuple[PushRule, bool]]: ...
@@ -61,6 +61,9 @@ def lazyConnection(
 # most methods to it via ConnectionHandler.__getattr__.
 class ConnectionHandler(RedisProtocol):
    def disconnect(self) -> "Deferred[None]": ...
+    def __repr__(self) -> str: ...
+
+class UnixConnectionHandler(ConnectionHandler): ...

 class RedisFactory(protocol.ReconnectingClientFactory):
    continueTrying: bool
@@ -25,8 +25,8 @@ from synapse.util.rust import check_rust_lib_up_to_date
 from synapse.util.stringutils import strtobool

 # Check that we're not running on an unsupported Python version.
-if sys.version_info < (3, 7):
-    print("Synapse requires Python 3.7 or above.")
+if sys.version_info < (3, 8):
+    print("Synapse requires Python 3.8 or above.")
    sys.exit(1)

 # Allow using the asyncio reactor via env var.
@@ -61,6 +61,7 @@ from synapse.storage.databases.main.deviceinbox import DeviceInboxBackgroundUpda
 from synapse.storage.databases.main.devices import DeviceBackgroundUpdateStore
 from synapse.storage.databases.main.e2e_room_keys import EndToEndRoomKeyBackgroundStore
 from synapse.storage.databases.main.end_to_end_keys import EndToEndKeyBackgroundStore
+from synapse.storage.databases.main.event_federation import EventFederationWorkerStore
 from synapse.storage.databases.main.event_push_actions import EventPushActionsStore
 from synapse.storage.databases.main.events_bg_updates import (
    EventsBackgroundUpdatesStore,
@@ -196,6 +197,11 @@ IGNORED_TABLES = {
    "ui_auth_sessions",
    "ui_auth_sessions_credentials",
    "ui_auth_sessions_ips",
+    # Ignore the worker locks table, as a) there shouldn't be any acquired locks
+    # after porting, and b) the circular foreign key constraints make it hard to
+    # port.
+    "worker_read_write_locks_mode",
+    "worker_read_write_locks",
 }


@@ -239,6 +245,7 @@ class Store(
    PresenceBackgroundUpdateStore,
    ReceiptsBackgroundUpdateStore,
    RelationsWorkerStore,
+    EventFederationWorkerStore,
 ):
    def execute(self, f: Callable[..., R], *args: Any, **kwargs: Any) -> Awaitable[R]:
        return self.db_pool.runInteraction(f.__name__, f, *args, **kwargs)
@@ -803,7 +810,9 @@ class Porter:
            )
            # Map from table name to args passed to `handle_table`, i.e. a tuple
            # of: `postgres_size`, `table_size`, `forward_chunk`, `backward_chunk`.
-            tables_to_port_info_map = {r[0]: r[1:] for r in setup_res}
+            tables_to_port_info_map = {
+                r[0]: r[1:] for r in setup_res if r[0] not in IGNORED_TABLES
+            }

            # Step 5. Do the copying.
            #
@@ -1369,6 +1378,9 @@ def main() -> None:
        sys.stderr.write("Database must use the 'psycopg2' connector.\n")
        sys.exit(3)

+    # Don't run the background tasks that get started by the data stores.
+    hs_config["run_background_tasks_on"] = "some_other_process"
+
    config = HomeServerConfig()
    config.parse_config_dict(hs_config, "", "")

@@ -0,0 +1,175 @@
+# Copyright 2023 The Matrix.org Foundation.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+from typing import Optional, Tuple
+
+from typing_extensions import Protocol
+
+from twisted.web.server import Request
+
+from synapse.appservice import ApplicationService
+from synapse.http.site import SynapseRequest
+from synapse.types import Requester
+
+# guests always get this device id.
+GUEST_DEVICE_ID = "guest_device"
+
+
+class Auth(Protocol):
+    """The interface that an auth provider must implement."""
+
+    async def check_user_in_room(
+        self,
+        room_id: str,
+        requester: Requester,
+        allow_departed_users: bool = False,
+    ) -> Tuple[str, Optional[str]]:
+        """Check if the user is in the room, or was at some point.
+        Args:
+            room_id: The room to check.
+
+            user_id: The user to check.
+
+            current_state: Optional map of the current state of the room.
+                If provided then that map is used to check whether they are a
+                member of the room. Otherwise the current membership is
+                loaded from the database.
+
+            allow_departed_users: if True, accept users that were previously
+                members but have now departed.
+
+        Raises:
+            AuthError if the user is/was not in the room.
+        Returns:
+            The current membership of the user in the room and the
+            membership event ID of the user.
+        """
+
+    async def get_user_by_req(
+        self,
+        request: SynapseRequest,
+        allow_guest: bool = False,
+        allow_expired: bool = False,
+    ) -> Requester:
+        """Get a registered user's ID.
+
+        Args:
+            request: An HTTP request with an access_token query parameter.
+            allow_guest: If False, will raise an AuthError if the user making the
+                request is a guest.
+            allow_expired: If True, allow the request through even if the account
+                is expired, or session token lifetime has ended. Note that
+                /login will deliver access tokens regardless of expiration.
+
+        Returns:
+            Resolves to the requester
+        Raises:
+            InvalidClientCredentialsError if no user by that token exists or the token
+                is invalid.
+            AuthError if access is denied for the user in the access token
+        """
+
+    async def validate_appservice_can_control_user_id(
+        self, app_service: ApplicationService, user_id: str
+    ) -> None:
+        """Validates that the app service is allowed to control
+        the given user.
+
+        Args:
+            app_service: The app service that controls the user
+            user_id: The author MXID that the app service is controlling
+
+        Raises:
+            AuthError: If the application service is not allowed to control the user
+                (user namespace regex does not match, wrong homeserver, etc)
+                or if the user has not been registered yet.
+        """
+
+    async def get_user_by_access_token(
+        self,
+        token: str,
+        allow_expired: bool = False,
+    ) -> Requester:
+        """Validate access token and get user_id from it
+
+        Args:
+            token: The access token to get the user by
+            allow_expired: If False, raises an InvalidClientTokenError
+                if the token is expired
+
+        Raises:
+            InvalidClientTokenError if a user by that token exists, but the token is
+                expired
+            InvalidClientCredentialsError if no user by that token exists or the token
+                is invalid
+        """
+
+    async def is_server_admin(self, requester: Requester) -> bool:
+        """Check if the given user is a local server admin.
+
+        Args:
+            requester: user to check
+
+        Returns:
+            True if the user is an admin
+        """
+
+    async def check_can_change_room_list(
+        self, room_id: str, requester: Requester
+    ) -> bool:
+        """Determine whether the user is allowed to edit the room's entry in the
+        published room list.
+
+        Args:
+            room_id
+            user
+        """
+
+    @staticmethod
+    def has_access_token(request: Request) -> bool:
+        """Checks if the request has an access_token.
+
+        Returns:
+            False if no access_token was given, True otherwise.
+        """
+
+    @staticmethod
+    def get_access_token_from_request(request: Request) -> str:
+        """Extracts the access_token from the request.
+
+        Args:
+            request: The http request.
+        Returns:
+            The access_token
+        Raises:
+            MissingClientTokenError: If there isn't a single access_token in the
+                request
+        """
+
+    async def check_user_in_room_or_world_readable(
+        self, room_id: str, requester: Requester, allow_departed_users: bool = False
+    ) -> Tuple[str, Optional[str]]:
+        """Checks that the user is or was in the room or the room is world
+        readable. If it isn't then an exception is raised.
+
+        Args:
+            room_id: room to check
+            user_id: user to check
+            allow_departed_users: if True, accept users that were previously
+                members but have now departed
+
+        Returns:
+            Resolves to the current membership of the user in the room and the
+            membership event ID of the user. If the user is not in the room and
+            never has been, then `(Membership.JOIN, None)` is returned.
+        """
--- a/Show More
+++ b/Show More
				`@@ -1 +0,0 @@`
				`Remove the old version of the R30 (30-day retained users) phone-home metric.`
				`@@ -1 +0,0 @@`
				`Fix a long-standing bug where setting the read marker could fail when using message retention. Contributed by Nick @ Beeper (@fizzadar).`
				`@@ -1 +0,0 @@`
				`Add not null constraint to column full_user_id of tables profiles and user_filters.`
				`@@ -1 +0,0 @@`
				`Print full error and stack-trace of any exception that occurs during startup/initialization.`
				`@@ -1 +0,0 @@`
				Fix a long-standing bug where the `url_preview_url_blacklist` configuration setting was not applied to oEmbed or image URLs found while previewing a URL.
				`@@ -1 +0,0 @@`
				`Run mypy type checking with the minimum supported Python version to catch new usage that isn't backwards-compatible.`
				`@@ -1 +0,0 @@`
				`Fix subscriptable type usage in Python <3.9.`
				`@@ -1 +0,0 @@`
				Instrument `state` and `state_group` storage-related operations to better picture what's happening when tracing.
				`@@ -1 +0,0 @@`
				`Add a new admin API to create a new device for a user.`
				`@@ -1 +0,0 @@`
				`Warn users that at least 3.75GB of space is needed for the nix Synapse development environment.`