Newsfile

Also delete old highlight notifications
Ensure we always use index
2023-05-02 16:00:13 +01:00 · 2023-05-02 15:56:19 +01:00 · 2023-05-02 15:52:30 +01:00 · 2023-05-02 09:29:40 -04:00 · 2023-05-02 07:59:55 -04:00 · 2023-05-01 11:34:01 -04:00
339 changed files with 9624 additions and 3708 deletions
@@ -35,9 +35,9 @@ sed -i \
 # compatible (as far the package metadata declares, anyway); pip's package resolver
 # is more lax.
 #
-# Rather than `poetry install --no-dev`, we drop all dev dependencies from the
-# toml file. This means we don't have to ensure compatibility between old deps and
-# dev tools.
+# Rather than `poetry install --no-dev`, we drop all dev dependencies and the dev-docs
+# group from the toml file. This means we don't have to ensure compatibility between
+# old deps and dev tools.

 pip install toml wheel

@@ -47,6 +47,7 @@ with open('pyproject.toml', 'r') as f:
    data = toml.loads(f.read())

 del data['tool']['poetry']['dev-dependencies']
+del data['tool']['poetry']['group']['dev-docs']

 with open('pyproject.toml', 'w') as f:
    toml.dump(data, f)
@@ -9,16 +9,6 @@ set -eu
 alias block='{ set +x; } 2>/dev/null; func() { echo "::group::$*"; set -x; }; func'
 alias endblock='{ set +x; } 2>/dev/null; func() { echo "::endgroup::"; set -x; }; func'

-block Set Go Version
-  # The path is set via a file given by $GITHUB_PATH. We need both Go 1.17 and GOPATH on the path to run Complement.
-  # See https://docs.github.com/en/actions/using-workflows/workflow-commands-for-github-actions#adding-a-system-path
-
-  # Add Go 1.17 to the PATH: see https://github.com/actions/virtual-environments/blob/main/images/linux/Ubuntu2004-Readme.md#environment-variables-2
-  echo "$GOROOT_1_17_X64/bin" >> $GITHUB_PATH
-  # Add the Go path to the PATH: We need this so we can call gotestfmt
-  echo "~/go/bin" >> $GITHUB_PATH
-endblock
-
 block Install Complement Dependencies
  sudo apt-get -qq update && sudo apt-get install -qqy libolm3 libolm-dev
  go install -v github.com/gotesttools/gotestfmt/v2/cmd/gotestfmt@latest
@@ -129,7 +129,7 @@ body:
    attributes:
      label: Relevant log output
      description: |
-        Please copy and paste any relevant log output, ideally at INFO or DEBUG log level.
+        Please copy and paste any relevant log output as text (not images), ideally at INFO or DEBUG log level.
        This will be automatically formatted into code, so there is no need for backticks (`\``).

        Please be careful to remove any personal or private data.
@@ -10,6 +10,7 @@ on:

 permissions:
  contents: read
+  packages: write

 jobs:
  build:
@@ -34,11 +35,20 @@ jobs:
          username: ${{ secrets.DOCKERHUB_USERNAME }}
          password: ${{ secrets.DOCKERHUB_TOKEN }}

+      - name: Log in to GHCR
+        uses: docker/login-action@v2
+        with:
+          registry: ghcr.io
+          username: ${{ github.repository_owner }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
      - name: Calculate docker image tag
        id: set-tag
        uses: docker/metadata-action@master
        with:
-          images: matrixdotorg/synapse
+          images: |
+            docker.io/matrixdotorg/synapse
+            ghcr.io/matrix-org/synapse
          flavor: |
            latest=false
          tags: |
@@ -14,7 +14,7 @@ jobs:
      # There's a 'download artifact' action, but it hasn't been updated for the workflow_run action
      # (https://github.com/actions/download-artifact/issues/60) so instead we get this mess:
      - name: 📥 Download artifact
-        uses: dawidd6/action-download-artifact@5e780fc7bbd0cac69fc73271ed86edf5dcb72d67 # v2.26.0
+        uses: dawidd6/action-download-artifact@246dbf436b23d7c49e21a7ab8204ca9ecd1fe615 # v2.27.0
        with:
          workflow: docs-pr.yaml
          run_id: ${{ github.event.workflow_run.id }}
@@ -13,25 +13,10 @@ on:
  workflow_dispatch:

 jobs:
-  pages:
-    name: GitHub Pages
+  pre:
+    name: Calculate variables for GitHub Pages deployment
    runs-on: ubuntu-latest
    steps:
-      - uses: actions/checkout@v3
-
-      - name: Setup mdbook
-        uses: peaceiris/actions-mdbook@adeb05db28a0c0004681db83893d56c0388ea9ea # v1.2.0
-        with:
-          mdbook-version: '0.4.17'
-
-      - name: Build the documentation
-        # mdbook will only create an index.html if we're including docs/README.md in SUMMARY.md.
-        # However, we're using docs/README.md for other purposes and need to pick a new page
-        # as the default. Let's opt for the welcome page instead.
-        run: |
-          mdbook build
-          cp book/welcome_and_overview.html book/index.html
-
      # Figure out the target directory.
      #
      # The target directory depends on the name of the branch
@@ -55,11 +40,65 @@ jobs:

          # finally, set the 'branch-version' var.
          echo "branch-version=$branch" >> "$GITHUB_OUTPUT"
-          
+    outputs:
+      branch-version: ${{ steps.vars.outputs.branch-version }}
+
+################################################################################
+  pages-docs:
+    name: GitHub Pages
+    runs-on: ubuntu-latest
+    needs:
+      - pre
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Setup mdbook
+        uses: peaceiris/actions-mdbook@adeb05db28a0c0004681db83893d56c0388ea9ea # v1.2.0
+        with:
+          mdbook-version: '0.4.17'
+
+      - name: Build the documentation
+        # mdbook will only create an index.html if we're including docs/README.md in SUMMARY.md.
+        # However, we're using docs/README.md for other purposes and need to pick a new page
+        # as the default. Let's opt for the welcome page instead.
+        run: |
+          mdbook build
+          cp book/welcome_and_overview.html book/index.html
+
      # Deploy to the target directory.
      - name: Deploy to gh pages
-        uses: peaceiris/actions-gh-pages@bd8c6b06eba6b3d25d72b7a1767993c0aeee42e7 # v3.9.2
+        uses: peaceiris/actions-gh-pages@373f7f263a76c20808c831209c920827a82a2847 # v3.9.3
        with:
          github_token: ${{ secrets.GITHUB_TOKEN }}
          publish_dir: ./book
-          destination_dir: ./${{ steps.vars.outputs.branch-version }}
+          destination_dir: ./${{ needs.pre.outputs.branch-version }}
+
+################################################################################
+  pages-devdocs:
+    name: GitHub Pages (developer docs)
+    runs-on: ubuntu-latest
+    needs:
+      - pre
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: "Set up Sphinx"
+        uses: matrix-org/setup-python-poetry@v1
+        with:
+          python-version: "3.x"
+          poetry-version: "1.3.2"
+          groups: "dev-docs"
+          extras: ""
+
+      - name: Build the documentation
+        run: |
+          cd dev-docs
+          poetry run make html
+
+      # Deploy to the target directory.
+      - name: Deploy to gh pages
+        uses: peaceiris/actions-gh-pages@373f7f263a76c20808c831209c920827a82a2847 # v3.9.3
+        with:
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+          publish_dir: ./dev-docs/_build/html
+          destination_dir: ./dev-docs/${{ needs.pre.outputs.branch-version }}
@@ -27,9 +27,7 @@ jobs:
    steps:
      - uses: actions/checkout@v3
      - name: Install Rust
-        uses: dtolnay/rust-toolchain@e12eda571dc9a5ee5d58eecf4738ec291c66f295
-        with:
-          toolchain: stable
+        uses: dtolnay/rust-toolchain@stable
      - uses: Swatinem/rust-cache@v2

      # The dev dependencies aren't exposed in the wheel metadata (at least with current
@@ -61,9 +59,7 @@ jobs:
      - uses: actions/checkout@v3

      - name: Install Rust
-        uses: dtolnay/rust-toolchain@e12eda571dc9a5ee5d58eecf4738ec291c66f295
-        with:
-          toolchain: stable
+        uses: dtolnay/rust-toolchain@stable
      - uses: Swatinem/rust-cache@v2

      - run: sudo apt-get -qq install xmlsec1
@@ -134,9 +130,7 @@ jobs:
      - uses: actions/checkout@v3

      - name: Install Rust
-        uses: dtolnay/rust-toolchain@e12eda571dc9a5ee5d58eecf4738ec291c66f295
-        with:
-          toolchain: stable
+        uses: dtolnay/rust-toolchain@stable
      - uses: Swatinem/rust-cache@v2

      - name: Ensure sytest runs `pip install`
@@ -184,6 +178,8 @@ jobs:
        with:
          path: synapse

+      - uses: actions/setup-go@v4
+
      - name: Prepare Complement's Prerequisites
        run: synapse/.ci/scripts/setup_complement_prerequisites.sh

@@ -34,6 +34,9 @@ jobs:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v3
+      - name: Install Rust
+        uses: dtolnay/rust-toolchain@1.58.1
+      - uses: Swatinem/rust-cache@v2
      - uses: matrix-org/setup-python-poetry@v1
        with:
          python-version: "3.x"
@@ -62,9 +65,59 @@ jobs:
      - run: .ci/scripts/check_lockfile.py

  lint:
-    uses: "matrix-org/backend-meta/.github/workflows/python-poetry-ci.yml@v2"
-    with:
-      typechecking-extras: "all"
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v3
+
+      - name: Setup Poetry
+        uses: matrix-org/setup-python-poetry@v1
+        with:
+          install-project: "false"
+
+      - name: Import order (isort)
+        run: poetry run isort --check --diff .
+
+      - name: Code style (black)
+        run: poetry run black --check --diff .
+
+      - name: Semantic checks (ruff)
+        # --quiet suppresses the update check.
+        run: poetry run ruff --quiet .
+
+  lint-mypy:
+    runs-on: ubuntu-latest
+    name: Typechecking
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v3
+
+      - name: Setup Poetry
+        uses: matrix-org/setup-python-poetry@v1
+        with:
+          # We want to make use of type hints in optional dependencies too.
+          extras: all
+          # We have seen odd mypy failures that were resolved when we started
+          # installing the project again:
+          # https://github.com/matrix-org/synapse/pull/15376#issuecomment-1498983775
+          # To make CI green, err towards caution and install the project.
+          install-project: "true"
+
+      - name: Install Rust
+        uses: dtolnay/rust-toolchain@1.58.1
+      - uses: Swatinem/rust-cache@v2
+
+      # NB: I have two concerns with this action:
+      # 1. We occasionally see odd mypy problems that aren't reproducible
+      #    locally with clean caches. I suspect some dodgy caching behaviour.
+      # 2. The action uses GHA machinery that's deprecated
+      #    (https://github.com/AustinScola/mypy-cache-github-action/issues/277)
+      # It may be simpler to use actions/cache ourselves to restore .mypy_cache.
+      - name: Restore/persist mypy's cache
+        uses: AustinScola/mypy-cache-github-action@df56268388422ee282636ee2c7a9cc55ec644a41
+
+      - name: Run mypy
+        run: poetry run mypy

  lint-crlf:
    runs-on: ubuntu-latest
@@ -95,6 +148,9 @@ jobs:
      - uses: actions/checkout@v3
        with:
          ref: ${{ github.event.pull_request.head.sha }}
+      - name: Install Rust
+        uses: dtolnay/rust-toolchain@1.58.1
+      - uses: Swatinem/rust-cache@v2
      - uses: matrix-org/setup-python-poetry@v1
        with:
          poetry-version: "1.3.2"
@@ -110,12 +166,8 @@ jobs:
      - uses: actions/checkout@v3

      - name: Install Rust
-        # There don't seem to be versioned releases of this action per se: for each rust
-        # version there is a branch which gets constantly rebased on top of master.
-        # We pin to a specific commit for paranoia's sake.
-        uses: dtolnay/rust-toolchain@e12eda571dc9a5ee5d58eecf4738ec291c66f295
+        uses: dtolnay/rust-toolchain@1.58.1
        with:
-            toolchain: 1.58.1
            components: clippy
      - uses: Swatinem/rust-cache@v2

@@ -132,10 +184,7 @@ jobs:
      - uses: actions/checkout@v3

      - name: Install Rust
-        # There don't seem to be versioned releases of this action per se: for each rust
-        # version there is a branch which gets constantly rebased on top of master.
-        # We pin to a specific commit for paranoia's sake.
-        uses: dtolnay/rust-toolchain@e12eda571dc9a5ee5d58eecf4738ec291c66f295
+        uses: dtolnay/rust-toolchain@master
        with:
            toolchain: nightly-2022-12-01
            components: clippy
@@ -152,10 +201,7 @@ jobs:
      - uses: actions/checkout@v3

      - name: Install Rust
-        # There don't seem to be versioned releases of this action per se: for each rust
-        # version there is a branch which gets constantly rebased on top of master.
-        # We pin to a specific commit for paranoia's sake.
-        uses: dtolnay/rust-toolchain@e12eda571dc9a5ee5d58eecf4738ec291c66f295
+        uses: dtolnay/rust-toolchain@master
        with:
          # We use nightly so that it correctly groups together imports
          toolchain: nightly-2022-12-01
@@ -169,6 +215,7 @@ jobs:
    if: ${{ !cancelled() }} # Run this even if prior jobs were skipped
    needs:
      - lint
+      - lint-mypy
      - lint-crlf
      - lint-newsfile
      - lint-pydantic
@@ -220,12 +267,7 @@ jobs:
            postgres:${{ matrix.job.postgres-version }}

      - name: Install Rust
-        # There don't seem to be versioned releases of this action per se: for each rust
-        # version there is a branch which gets constantly rebased on top of master.
-        # We pin to a specific commit for paranoia's sake.
-        uses: dtolnay/rust-toolchain@e12eda571dc9a5ee5d58eecf4738ec291c66f295
-        with:
-            toolchain: 1.58.1
+        uses: dtolnay/rust-toolchain@1.58.1
      - uses: Swatinem/rust-cache@v2

      - uses: matrix-org/setup-python-poetry@v1
@@ -265,12 +307,7 @@ jobs:
      - uses: actions/checkout@v3

      - name: Install Rust
-        # There don't seem to be versioned releases of this action per se: for each rust
-        # version there is a branch which gets constantly rebased on top of master.
-        # We pin to a specific commit for paranoia's sake.
-        uses: dtolnay/rust-toolchain@e12eda571dc9a5ee5d58eecf4738ec291c66f295
-        with:
-            toolchain: 1.58.1
+        uses: dtolnay/rust-toolchain@1.58.1
      - uses: Swatinem/rust-cache@v2

      # There aren't wheels for some of the older deps, so we need to install
@@ -386,12 +423,7 @@ jobs:
        run: cat sytest-blacklist .ci/worker-blacklist > synapse-blacklist-with-workers

      - name: Install Rust
-        # There don't seem to be versioned releases of this action per se: for each rust
-        # version there is a branch which gets constantly rebased on top of master.
-        # We pin to a specific commit for paranoia's sake.
-        uses: dtolnay/rust-toolchain@e12eda571dc9a5ee5d58eecf4738ec291c66f295
-        with:
-            toolchain: 1.58.1
+        uses: dtolnay/rust-toolchain@1.58.1
      - uses: Swatinem/rust-cache@v2

      - name: Run SyTest
@@ -531,14 +563,11 @@ jobs:
          path: synapse

      - name: Install Rust
-        # There don't seem to be versioned releases of this action per se: for each rust
-        # version there is a branch which gets constantly rebased on top of master.
-        # We pin to a specific commit for paranoia's sake.
-        uses: dtolnay/rust-toolchain@e12eda571dc9a5ee5d58eecf4738ec291c66f295
-        with:
-            toolchain: 1.58.1
+        uses: dtolnay/rust-toolchain@1.58.1
      - uses: Swatinem/rust-cache@v2

+      - uses: actions/setup-go@v4
+
      - name: Prepare Complement's Prerequisites
        run: synapse/.ci/scripts/setup_complement_prerequisites.sh

@@ -562,12 +591,7 @@ jobs:
      - uses: actions/checkout@v3

      - name: Install Rust
-        # There don't seem to be versioned releases of this action per se: for each rust
-        # version there is a branch which gets constantly rebased on top of master.
-        # We pin to a specific commit for paranoia's sake.
-        uses: dtolnay/rust-toolchain@e12eda571dc9a5ee5d58eecf4738ec291c66f295
-        with:
-            toolchain: 1.58.1
+        uses: dtolnay/rust-toolchain@1.58.1
      - uses: Swatinem/rust-cache@v2

      - run: cargo test
@@ -585,10 +609,7 @@ jobs:
      - uses: actions/checkout@v3

      - name: Install Rust
-        # There don't seem to be versioned releases of this action per se: for each rust
-        # version there is a branch which gets constantly rebased on top of master.
-        # We pin to a specific commit for paranoia's sake.
-        uses: dtolnay/rust-toolchain@e12eda571dc9a5ee5d58eecf4738ec291c66f295
+        uses: dtolnay/rust-toolchain@master
        with:
            toolchain: nightly-2022-12-01
      - uses: Swatinem/rust-cache@v2
@@ -5,6 +5,13 @@ on:
    - cron: 0 8 * * *

  workflow_dispatch:
+    inputs:
+      twisted_ref:
+        description: Commit, branch or tag to checkout from upstream Twisted.
+        required: false
+        default: 'trunk'
+        type: string
+

 concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}
@@ -18,9 +25,7 @@ jobs:
      - uses: actions/checkout@v3

      - name: Install Rust
-        uses: dtolnay/rust-toolchain@e12eda571dc9a5ee5d58eecf4738ec291c66f295
-        with:
-          toolchain: stable
+        uses: dtolnay/rust-toolchain@stable
      - uses: Swatinem/rust-cache@v2

      - uses: matrix-org/setup-python-poetry@v1
@@ -29,7 +34,7 @@ jobs:
          extras: "all"
      - run: |
          poetry remove twisted
-          poetry add --extras tls git+https://github.com/twisted/twisted.git#trunk
+          poetry add --extras tls git+https://github.com/twisted/twisted.git#${{ inputs.twisted_ref }}
          poetry install --no-interaction --extras "all test"
      - name: Remove warn_unused_ignores from mypy config
        run: sed '/warn_unused_ignores = True/d' -i mypy.ini
@@ -43,9 +48,7 @@ jobs:
      - run: sudo apt-get -qq install xmlsec1

      - name: Install Rust
-        uses: dtolnay/rust-toolchain@e12eda571dc9a5ee5d58eecf4738ec291c66f295
-        with:
-          toolchain: stable
+        uses: dtolnay/rust-toolchain@stable
      - uses: Swatinem/rust-cache@v2

      - uses: matrix-org/setup-python-poetry@v1
@@ -82,9 +85,7 @@ jobs:
      - uses: actions/checkout@v3

      - name: Install Rust
-        uses: dtolnay/rust-toolchain@e12eda571dc9a5ee5d58eecf4738ec291c66f295
-        with:
-          toolchain: stable
+        uses: dtolnay/rust-toolchain@stable
      - uses: Swatinem/rust-cache@v2

      - name: Patch dependencies
@@ -140,6 +141,8 @@ jobs:
        with:
          path: synapse

+      - uses: actions/setup-go@v4
+
      - name: Prepare Complement's Prerequisites
        run: synapse/.ci/scripts/setup_complement_prerequisites.sh

@@ -15,9 +15,10 @@ _trial_temp*/
 .DS_Store
 __pycache__/

-# We do want the poetry and cargo lockfile.
+# We do want poetry, cargo and flake lockfiles.
 !poetry.lock
 !Cargo.lock
+!flake.lock

 # stuff that is likely to exist when you run a server locally
 /*.db
@@ -38,6 +39,9 @@ __pycache__/
 /.envrc
 .direnv/

+# For nix/devenv users
+.devenv/
+
 # IDEs
 /.idea/
 /.ropeproject/
@@ -53,6 +57,7 @@ __pycache__/
 /coverage.*
 /dist/
 /docs/build/
+/dev-docs/_build/
 /htmlcov
 /pip-wheel-metadata/

@@ -61,7 +66,7 @@ book/

 # complement
 /complement-*
-/master.tar.gz
+/main.tar.gz

 # rust
 /target/
@@ -1,3 +1,265 @@
+Synapse 1.82.0 (2023-04-25)
+===========================
+
+No significant changes since 1.82.0rc1.
+
+
+Synapse 1.82.0rc1 (2023-04-18)
+==============================
+
+Features
+--------
+
+- Allow loading the `/directory/room/{roomAlias}` endpoint on workers. ([\#15333](https://github.com/matrix-org/synapse/issues/15333))
+- Add some validation to `instance_map` configuration loading. ([\#15431](https://github.com/matrix-org/synapse/issues/15431))
+- Allow loading the `/capabilities` endpoint on workers. ([\#15436](https://github.com/matrix-org/synapse/issues/15436))
+
+
+Bugfixes
+--------
+
+- Delete server-side backup keys when deactivating an account. ([\#15181](https://github.com/matrix-org/synapse/issues/15181))
+- Fix and document untold assumption that `on_logged_out` module hooks will be called before the deletion of pushers. ([\#15410](https://github.com/matrix-org/synapse/issues/15410))
+- Improve robustness when handling a perspective key response by deduplicating received server keys. ([\#15423](https://github.com/matrix-org/synapse/issues/15423))
+- Synapse now correctly fails to start if the config option `app_service_config_files` is not a list. ([\#15425](https://github.com/matrix-org/synapse/issues/15425))
+- Disable loading `RefreshTokenServlet` (`/_matrix/client/(r0|v3|unstable)/refresh`) on workers. ([\#15428](https://github.com/matrix-org/synapse/issues/15428))
+
+
+Improved Documentation
+----------------------
+
+- Note that the `delete_stale_devices_after` background job always runs on the main process. ([\#15452](https://github.com/matrix-org/synapse/issues/15452))
+
+
+Deprecations and Removals
+-------------------------
+
+- Remove the broken, unspecced registration fallback. Note that the *login* fallback is unaffected by this change. ([\#15405](https://github.com/matrix-org/synapse/issues/15405))
+
+
+Internal Changes
+----------------
+
+- Bump black from 23.1.0 to 23.3.0. ([\#15372](https://github.com/matrix-org/synapse/issues/15372))
+- Bump pyopenssl from 23.1.0 to 23.1.1. ([\#15373](https://github.com/matrix-org/synapse/issues/15373))
+- Bump types-psycopg2 from 2.9.21.8 to 2.9.21.9. ([\#15374](https://github.com/matrix-org/synapse/issues/15374))
+- Bump types-netaddr from 0.8.0.6 to 0.8.0.7. ([\#15375](https://github.com/matrix-org/synapse/issues/15375))
+- Bump types-opentracing from 2.4.10.3 to 2.4.10.4. ([\#15376](https://github.com/matrix-org/synapse/issues/15376))
+- Bump dawidd6/action-download-artifact from 2.26.0 to 2.26.1. ([\#15404](https://github.com/matrix-org/synapse/issues/15404))
+- Bump parameterized from 0.8.1 to 0.9.0. ([\#15412](https://github.com/matrix-org/synapse/issues/15412))
+- Bump types-pillow from 9.4.0.17 to 9.4.0.19. ([\#15413](https://github.com/matrix-org/synapse/issues/15413))
+- Bump sentry-sdk from 1.17.0 to 1.19.1. ([\#15414](https://github.com/matrix-org/synapse/issues/15414))
+- Bump immutabledict from 2.2.3 to 2.2.4. ([\#15415](https://github.com/matrix-org/synapse/issues/15415))
+- Bump dawidd6/action-download-artifact from 2.26.1 to 2.27.0. ([\#15441](https://github.com/matrix-org/synapse/issues/15441))
+- Bump serde_json from 1.0.95 to 1.0.96. ([\#15442](https://github.com/matrix-org/synapse/issues/15442))
+- Bump serde from 1.0.159 to 1.0.160. ([\#15443](https://github.com/matrix-org/synapse/issues/15443))
+- Bump pillow from 9.4.0 to 9.5.0. ([\#15444](https://github.com/matrix-org/synapse/issues/15444))
+- Bump furo from 2023.3.23 to 2023.3.27. ([\#15445](https://github.com/matrix-org/synapse/issues/15445))
+- Bump types-pyopenssl from 23.1.0.0 to 23.1.0.2. ([\#15446](https://github.com/matrix-org/synapse/issues/15446))
+- Bump mypy from 1.0.0 to 1.0.1. ([\#15447](https://github.com/matrix-org/synapse/issues/15447))
+- Bump psycopg2 from 2.9.5 to 2.9.6. ([\#15448](https://github.com/matrix-org/synapse/issues/15448))
+- Improve DB performance of clearing out old data from `stream_ordering_to_exterm`. ([\#15382](https://github.com/matrix-org/synapse/issues/15382), [\#15429](https://github.com/matrix-org/synapse/issues/15429))
+- Implement [MSC3989](https://github.com/matrix-org/matrix-spec-proposals/pull/3989) redaction algorithm. ([\#15393](https://github.com/matrix-org/synapse/issues/15393))
+- Implement [MSC2175](https://github.com/matrix-org/matrix-doc/pull/2175) to stop adding `creator` to create events. ([\#15394](https://github.com/matrix-org/synapse/issues/15394))
+- Implement [MSC2174](https://github.com/matrix-org/matrix-spec-proposals/pull/2174) to move the `redacts` key to a `content` property. ([\#15395](https://github.com/matrix-org/synapse/issues/15395))
+- Trust dtonlay/rust-toolchain in CI. ([\#15406](https://github.com/matrix-org/synapse/issues/15406))
+- Explicitly install Synapse during typechecking in CI. ([\#15409](https://github.com/matrix-org/synapse/issues/15409))
+- Only load the SSO redirect servlet if SSO is enabled. ([\#15421](https://github.com/matrix-org/synapse/issues/15421))
+- Refactor `SimpleHttpClient` to pull out a base class. ([\#15427](https://github.com/matrix-org/synapse/issues/15427))
+- Improve type hints. ([\#15432](https://github.com/matrix-org/synapse/issues/15432))
+- Convert async to normal tests in `TestSSOHandler`. ([\#15433](https://github.com/matrix-org/synapse/issues/15433))
+- Speed up the user directory background update. ([\#15435](https://github.com/matrix-org/synapse/issues/15435))
+- Disable directory listing for static resources in `/_matrix/static/`. ([\#15438](https://github.com/matrix-org/synapse/issues/15438))
+- Move various module API callback registration methods to a dedicated class. ([\#15453](https://github.com/matrix-org/synapse/issues/15453))
+
+
+Synapse 1.81.0 (2023-04-11)
+===========================
+
+Synapse now attempts the versioned appservice paths before falling back to the
+[legacy paths](https://spec.matrix.org/v1.6/application-service-api/#legacy-routes).
+Usage of the legacy routes should be considered deprecated.
+
+Additionally, Synapse has supported sending the application service access token
+via [the `Authorization` header](https://spec.matrix.org/v1.6/application-service-api/#authorization)
+since v1.70.0. For backwards compatibility it is *also* sent as the `access_token`
+query parameter. This is insecure and should be considered deprecated.
+
+A future version of Synapse (v1.88.0 or later) will remove support for legacy
+application service routes and query parameter authorization.
+
+
+No significant changes since 1.81.0rc2.
+
+
+Synapse 1.81.0rc2 (2023-04-06)
+==============================
+
+Bugfixes
+--------
+
+- Fix the `set_device_id_for_pushers_txn` background update crash. ([\#15391](https://github.com/matrix-org/synapse/issues/15391))
+
+
+Internal Changes
+----------------
+
+- Update CI to run complement under the latest stable go version. ([\#15403](https://github.com/matrix-org/synapse/issues/15403))
+
+
+Synapse 1.81.0rc1 (2023-04-04)
+==============================
+
+Features
+--------
+
+- Add the ability to enable/disable registrations when in the OIDC flow. ([\#14978](https://github.com/matrix-org/synapse/issues/14978))
+- Add a primitive helper script for listing worker endpoints. ([\#15243](https://github.com/matrix-org/synapse/issues/15243))
+- Experimental support for passing One Time Key and device key requests to application services ([MSC3983](https://github.com/matrix-org/matrix-spec-proposals/pull/3983) and [MSC3984](https://github.com/matrix-org/matrix-spec-proposals/pull/3984)). ([\#15314](https://github.com/matrix-org/synapse/issues/15314), [\#15321](https://github.com/matrix-org/synapse/issues/15321))
+- Allow loading `/password_policy` endpoint on workers. ([\#15331](https://github.com/matrix-org/synapse/issues/15331))
+- Add experimental support for Unix sockets. Contributed by Jason Little. ([\#15353](https://github.com/matrix-org/synapse/issues/15353))
+- Build Debian packages for Ubuntu 23.04 (Lunar Lobster). ([\#15381](https://github.com/matrix-org/synapse/issues/15381))
+
+
+Bugfixes
+--------
+
+- Fix a long-standing bug where edits of non-`m.room.message` events would not be correctly bundled. ([\#15295](https://github.com/matrix-org/synapse/issues/15295))
+- Fix a bug introduced in Synapse v1.55.0 which could delay remote homeservers being able to decrypt encrypted messages sent by local users. ([\#15297](https://github.com/matrix-org/synapse/issues/15297))
+- Add a check to [SQLite port_db script](https://matrix-org.github.io/synapse/latest/postgres.html#porting-from-sqlite)
+  to ensure that the sqlite database passed to the script exists before trying to port from it. ([\#15306](https://github.com/matrix-org/synapse/issues/15306))
+- Fix a bug introduced in Synapse 1.76.0 where responses from worker deployments could include an internal `_INT_STREAM_POS` key. ([\#15309](https://github.com/matrix-org/synapse/issues/15309))
+- Fix a long-standing bug that Synpase only used the [legacy appservice routes](https://spec.matrix.org/v1.6/application-service-api/#legacy-routes). ([\#15317](https://github.com/matrix-org/synapse/issues/15317))
+- Fix a long-standing bug preventing users from rejoining rooms after being banned and unbanned over federation. Contributed by Nico. ([\#15323](https://github.com/matrix-org/synapse/issues/15323))
+- Fix bug in worker mode where on a rolling restart of workers the "typing" worker would consume 100% CPU until it got restarted. ([\#15332](https://github.com/matrix-org/synapse/issues/15332))
+- Fix a long-standing bug where some to_device messages could be dropped when using workers. ([\#15349](https://github.com/matrix-org/synapse/issues/15349))
+- Fix a bug introduced in Synapse 1.70.0 where the background sync from a faster join could spin for hours when one of the events involved had been marked for backoff. ([\#15351](https://github.com/matrix-org/synapse/issues/15351))
+- Fix missing app variable in mail subject for password resets. Contributed by Cyberes. ([\#15352](https://github.com/matrix-org/synapse/issues/15352))
+- Fix a rare bug introduced in Synapse 1.66.0 where initial syncs would fail when the user had been kicked from a faster joined room that had not finished syncing. ([\#15383](https://github.com/matrix-org/synapse/issues/15383))
+
+
+Improved Documentation
+----------------------
+
+- Fix a typo in login requests ratelimit defaults. ([\#15341](https://github.com/matrix-org/synapse/issues/15341))
+- Add some clarification to the doc/comments regarding TCP replication. ([\#15354](https://github.com/matrix-org/synapse/issues/15354))
+- Note that Synapse 1.74 queued a rebuild of the user directory tables. ([\#15386](https://github.com/matrix-org/synapse/issues/15386))
+
+
+Internal Changes
+----------------
+
+- Use `immutabledict` instead of `frozendict`. ([\#15113](https://github.com/matrix-org/synapse/issues/15113))
+- Add developer documentation for the Federation Sender and add a documentation mechanism using Sphinx. ([\#15265](https://github.com/matrix-org/synapse/issues/15265), [\#15336](https://github.com/matrix-org/synapse/issues/15336))
+- Make the pushers rely on the `device_id` instead of the `access_token_id` for various operations. ([\#15280](https://github.com/matrix-org/synapse/issues/15280))
+- Bump sentry-sdk from 1.15.0 to 1.17.0. ([\#15285](https://github.com/matrix-org/synapse/issues/15285))
+- Allow running the Twisted trunk job against other branches. ([\#15302](https://github.com/matrix-org/synapse/issues/15302))
+- Remind the releaser to ask for changelog feedback in [#synapse-dev](https://matrix.to/#/#synapse-dev:matrix.org). ([\#15303](https://github.com/matrix-org/synapse/issues/15303))
+- Bump dtolnay/rust-toolchain from e12eda571dc9a5ee5d58eecf4738ec291c66f295 to fc3253060d0c959bea12a59f10f8391454a0b02d. ([\#15304](https://github.com/matrix-org/synapse/issues/15304))
+- Reject events with an invalid "mentions" property per [MSC3952](https://github.com/matrix-org/matrix-spec-proposals/pull/3952). ([\#15311](https://github.com/matrix-org/synapse/issues/15311))
+- As an optimisation, use `TRUNCATE` on Postgres when clearing the user directory tables. ([\#15316](https://github.com/matrix-org/synapse/issues/15316))
+- Fix `.gitignore` rule for the Complement source tarball downloaded automatically by `complement.sh`. ([\#15319](https://github.com/matrix-org/synapse/issues/15319))
+- Bump serde from 1.0.157 to 1.0.158. ([\#15324](https://github.com/matrix-org/synapse/issues/15324))
+- Bump regex from 1.7.1 to 1.7.3. ([\#15325](https://github.com/matrix-org/synapse/issues/15325))
+- Bump types-pyopenssl from 23.0.0.4 to 23.1.0.0. ([\#15326](https://github.com/matrix-org/synapse/issues/15326))
+- Bump furo from 2022.12.7 to 2023.3.23. ([\#15327](https://github.com/matrix-org/synapse/issues/15327))
+- Bump ruff from 0.0.252 to 0.0.259. ([\#15328](https://github.com/matrix-org/synapse/issues/15328))
+- Bump cryptography from 40.0.0 to 40.0.1. ([\#15329](https://github.com/matrix-org/synapse/issues/15329))
+- Bump mypy-zope from 0.9.0 to 0.9.1. ([\#15330](https://github.com/matrix-org/synapse/issues/15330))
+- Speed up unit tests when using SQLite3. ([\#15334](https://github.com/matrix-org/synapse/issues/15334))
+- Speed up pydantic CI job. ([\#15339](https://github.com/matrix-org/synapse/issues/15339))
+- Speed up sample config CI job. ([\#15340](https://github.com/matrix-org/synapse/issues/15340))
+- Fix copyright year in SSO footer template. ([\#15358](https://github.com/matrix-org/synapse/issues/15358))
+- Bump peaceiris/actions-gh-pages from 3.9.2 to 3.9.3. ([\#15369](https://github.com/matrix-org/synapse/issues/15369))
+- Bump serde from 1.0.158 to 1.0.159. ([\#15370](https://github.com/matrix-org/synapse/issues/15370))
+- Bump serde_json from 1.0.94 to 1.0.95. ([\#15371](https://github.com/matrix-org/synapse/issues/15371))
+- Speed up membership queries for users with forgotten rooms. ([\#15385](https://github.com/matrix-org/synapse/issues/15385))
+
+
+Synapse 1.80.0 (2023-03-28)
+===========================
+
+No significant changes since 1.80.0rc2.
+
+
+Synapse 1.80.0rc2 (2023-03-22)
+==============================
+
+Bugfixes
+--------
+
+- Fix a bug in which the [`POST /_matrix/client/v3/rooms/{roomId}/report/{eventId}`](https://spec.matrix.org/v1.6/client-server-api/#post_matrixclientv3roomsroomidreporteventid) endpoint would return the wrong error if the user did not have permission to view the event. This aligns Synapse's implementation with [MSC2249](https://github.com/matrix-org/matrix-spec-proposals/pull/2249). ([\#15298](https://github.com/matrix-org/synapse/issues/15298), [\#15300](https://github.com/matrix-org/synapse/issues/15300))
+- Fix a bug introduced in Synapse 1.75.0rc1 where the [SQLite port_db script](https://matrix-org.github.io/synapse/latest/postgres.html#porting-from-sqlite)
+  would fail to open the SQLite database. ([\#15301](https://github.com/matrix-org/synapse/issues/15301))
+
+
+Synapse 1.80.0rc1 (2023-03-21)
+==============================
+
+Features
+--------
+
+- Stabilise support for [MSC3966](https://github.com/matrix-org/matrix-spec-proposals/pull/3966): `event_property_contains` push condition. ([\#15187](https://github.com/matrix-org/synapse/issues/15187))
+- Implement [MSC2659](https://github.com/matrix-org/matrix-spec-proposals/pull/2659): application service ping endpoint. Contributed by Tulir @ Beeper. ([\#15249](https://github.com/matrix-org/synapse/issues/15249))
+- Allow loading `/register/available` endpoint on workers. ([\#15268](https://github.com/matrix-org/synapse/issues/15268))
+- Improve performance of creating and authenticating events. ([\#15195](https://github.com/matrix-org/synapse/issues/15195))
+- Add topic and name events to group of events that are batch persisted when creating a room. ([\#15229](https://github.com/matrix-org/synapse/issues/15229))
+
+
+Bugfixes
+--------
+
+- Fix a long-standing bug in which the user directory would assume any remote membership state events represent a profile change. ([\#14755](https://github.com/matrix-org/synapse/issues/14755), [\#14756](https://github.com/matrix-org/synapse/issues/14756))
+- Implement [MSC3873](https://github.com/matrix-org/matrix-spec-proposals/pull/3873) to fix a long-standing bug where properties with dots were handled ambiguously in push rules. ([\#15190](https://github.com/matrix-org/synapse/issues/15190))
+- Faster joins: Fix a bug introduced in Synapse 1.66 where spurious "Failed to find memberships ..." errors would be logged. ([\#15232](https://github.com/matrix-org/synapse/issues/15232))
+- Fix a long-standing error when sending message into deleted room. ([\#15235](https://github.com/matrix-org/synapse/issues/15235))
+
+
+Updates to the Docker image
+---------------------------
+
+- Ensure the Dockerfile builds on platforms that don't have a `cryptography` wheel. ([\#15239](https://github.com/matrix-org/synapse/issues/15239))
+- Mirror images to the GitHub Container Registry (`ghcr.io/matrix-org/synapse`). ([\#15281](https://github.com/matrix-org/synapse/issues/15281), [\#15282](https://github.com/matrix-org/synapse/issues/15282))
+
+
+Improved Documentation
+----------------------
+
+- Add a missing endpoint to the workers documentation. ([\#15223](https://github.com/matrix-org/synapse/issues/15223))
+
+
+Internal Changes
+----------------
+
+- Add additional functionality to declaring worker types when starting Complement in worker mode. ([\#14921](https://github.com/matrix-org/synapse/issues/14921))
+- Add `Synapse-Trace-Id` to `access-control-expose-headers` header. ([\#14974](https://github.com/matrix-org/synapse/issues/14974))
+- Make the `HttpTransactionCache` use the `Requester` in addition of the just the `Request` to build the transaction key. ([\#15200](https://github.com/matrix-org/synapse/issues/15200))
+- Improve log lines when purging rooms. ([\#15222](https://github.com/matrix-org/synapse/issues/15222))
+- Improve type hints. ([\#15230](https://github.com/matrix-org/synapse/issues/15230), [\#15231](https://github.com/matrix-org/synapse/issues/15231), [\#15238](https://github.com/matrix-org/synapse/issues/15238))
+- Move various module API callback registration methods to a dedicated class. ([\#15237](https://github.com/matrix-org/synapse/issues/15237))
+- Configure GitHub Actions for merge queues. ([\#15244](https://github.com/matrix-org/synapse/issues/15244))
+- Add schema comments about the `destinations` and `destination_rooms` tables. ([\#15247](https://github.com/matrix-org/synapse/issues/15247))
+- Skip processing of auto-join room behaviour if there are no auto-join rooms configured. ([\#15262](https://github.com/matrix-org/synapse/issues/15262))
+- Remove unused store method `_set_destination_retry_timings_emulated`. ([\#15266](https://github.com/matrix-org/synapse/issues/15266))
+- Reorganize URL preview code. ([\#15269](https://github.com/matrix-org/synapse/issues/15269))
+- Clean-up direct TCP replication code. ([\#15272](https://github.com/matrix-org/synapse/issues/15272), [\#15274](https://github.com/matrix-org/synapse/issues/15274))
+- Make `configure_workers_and_start` script used in Complement tests compatible with older versions of Python. ([\#15275](https://github.com/matrix-org/synapse/issues/15275))
+- Add a `/versions` flag for [MSC3952](https://github.com/matrix-org/matrix-spec-proposals/pull/3952). ([\#15293](https://github.com/matrix-org/synapse/issues/15293))
+- Bump hiredis from 2.2.1 to 2.2.2. ([\#15252](https://github.com/matrix-org/synapse/issues/15252))
+- Bump serde from 1.0.152 to 1.0.155. ([\#15253](https://github.com/matrix-org/synapse/issues/15253))
+- Bump pysaml2 from 7.2.1 to 7.3.1. ([\#15254](https://github.com/matrix-org/synapse/issues/15254))
+- Bump msgpack from 1.0.4 to 1.0.5. ([\#15255](https://github.com/matrix-org/synapse/issues/15255))
+- Bump gitpython from 3.1.30 to 3.1.31. ([\#15256](https://github.com/matrix-org/synapse/issues/15256))
+- Bump cryptography from 39.0.1 to 39.0.2. ([\#15257](https://github.com/matrix-org/synapse/issues/15257))
+- Bump pydantic from 1.10.4 to 1.10.6. ([\#15286](https://github.com/matrix-org/synapse/issues/15286))
+- Bump serde from 1.0.155 to 1.0.157. ([\#15287](https://github.com/matrix-org/synapse/issues/15287))
+- Bump anyhow from 1.0.69 to 1.0.70. ([\#15288](https://github.com/matrix-org/synapse/issues/15288))
+- Bump txredisapi from 1.4.7 to 1.4.9. ([\#15289](https://github.com/matrix-org/synapse/issues/15289))
+- Bump pygithub from 1.57 to 1.58.1. ([\#15290](https://github.com/matrix-org/synapse/issues/15290))
+- Bump types-requests from 2.28.11.12 to 2.28.11.15. ([\#15291](https://github.com/matrix-org/synapse/issues/15291))
+
+
+
 Synapse 1.79.0 (2023-03-14)
 ===========================

@@ -310,7 +572,7 @@ Those who are `poetry install`ing from source using our lockfile should ensure t
 Notes on faster joins
 ---------------------

-The faster joins project sees the most benefit when joining a room with a large number of members (joined or historical). We expect it to be particularly useful for joining large public rooms like the [Matrix HQ](https://matrix.to/#/#matrix:matrix.org) or [Synapse Admins](https://matrix.to/#/#synapse:matrix.org) rooms. 
+The faster joins project sees the most benefit when joining a room with a large number of members (joined or historical). We expect it to be particularly useful for joining large public rooms like the [Matrix HQ](https://matrix.to/#/#matrix:matrix.org) or [Synapse Admins](https://matrix.to/#/#synapse:matrix.org) rooms.

 After a faster join, Synapse considers that room "partially joined". In this state, you should be able to

@@ -13,9 +13,9 @@ dependencies = [

 [[package]]
 name = "anyhow"
-version = "1.0.69"
+version = "1.0.71"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "224afbd727c3d6e4b90103ece64b8d1b67fbb1973b1046c2281eed3f3803f800"
+checksum = "9c7d0618f0e0b7e8ff11427422b64564d5fb0be1940354bfe2e0529b18a9d9b8"

 [[package]]
 name = "arc-swap"
@@ -185,9 +185,9 @@ dependencies = [

 [[package]]
 name = "proc-macro2"
-version = "1.0.46"
+version = "1.0.52"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "94e2ef8dbfc347b10c094890f778ee2e36ca9bb4262e86dc99cd217e35f3470b"
+checksum = "1d0e1ae9e836cc3beddd63db0df682593d7e2d3d891ae8c9083d2113e1744224"
 dependencies = [
 "unicode-ident",
 ]
@@ -250,7 +250,7 @@ dependencies = [
 "proc-macro2",
 "pyo3-macros-backend",
 "quote",
- "syn",
+ "syn 1.0.104",
 ]

 [[package]]
@@ -261,7 +261,7 @@ checksum = "c8df9be978a2d2f0cdebabb03206ed73b11314701a5bfe71b0d753b81997777f"
 dependencies = [
 "proc-macro2",
 "quote",
- "syn",
+ "syn 1.0.104",
 ]

 [[package]]
@@ -276,9 +276,9 @@ dependencies = [

 [[package]]
 name = "quote"
-version = "1.0.21"
+version = "1.0.26"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bbe448f377a7d6961e30f5955f9b8d106c3f5e449d493ee1b125c1d43c2b5179"
+checksum = "4424af4bf778aae2051a77b60283332f386554255d722233d09fbfc7e30da2fc"
 dependencies = [
 "proc-macro2",
 ]
@@ -294,9 +294,9 @@ dependencies = [

 [[package]]
 name = "regex"
-version = "1.7.1"
+version = "1.7.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "48aaa5748ba571fb95cd2c85c09f629215d3a6ece942baa100950af03a34f733"
+checksum = "8b1f693b24f6ac912f4893ef08244d70b6067480d2f1a46e950c9691e6749d1d"
 dependencies = [
 "aho-corasick",
 "memchr",
@@ -305,9 +305,9 @@ dependencies = [

 [[package]]
 name = "regex-syntax"
-version = "0.6.27"
+version = "0.6.29"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a3f87b73ce11b1619a3c6332f45341e0047173771e8b8b73f87bfeefb7b56244"
+checksum = "f162c6dd7b008981e4d40210aca20b4bd0f9b60ca9271061b07f78537722f2e1"

 [[package]]
 name = "ryu"
@@ -323,29 +323,29 @@ checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd"

 [[package]]
 name = "serde"
-version = "1.0.155"
+version = "1.0.160"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "71f2b4817415c6d4210bfe1c7bfcf4801b2d904cb4d0e1a8fdb651013c9e86b8"
+checksum = "bb2f3770c8bce3bcda7e149193a069a0f4365bda1fa5cd88e03bca26afc1216c"
 dependencies = [
 "serde_derive",
 ]

 [[package]]
 name = "serde_derive"
-version = "1.0.155"
+version = "1.0.160"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d071a94a3fac4aff69d023a7f411e33f40f3483f8c5190b1953822b6b76d7630"
+checksum = "291a097c63d8497e00160b166a967a4a79c64f3facdd01cbd7502231688d77df"
 dependencies = [
 "proc-macro2",
 "quote",
- "syn",
+ "syn 2.0.10",
 ]

 [[package]]
 name = "serde_json"
-version = "1.0.94"
+version = "1.0.96"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1c533a59c9d8a93a09c6ab31f0fd5e5f4dd1b8fc9434804029839884765d04ea"
+checksum = "057d394a50403bcac12672b2b18fb387ab6d289d957dab67dd201875391e52f1"
 dependencies = [
 "itoa",
 "ryu",
@@ -375,6 +375,17 @@ dependencies = [
 "unicode-ident",
 ]

+[[package]]
+name = "syn"
+version = "2.0.10"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5aad1363ed6d37b84299588d62d3a7d95b5a5c2d9aad5c85609fda12afaa1f40"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "unicode-ident",
+]
+
 [[package]]
 name = "synapse"
 version = "0.1.0"
@@ -1 +0,0 @@
-Fix a long-standing bug in which the user directory would assume any remote membership state events represent a profile change.
@@ -1 +0,0 @@
-Fix a long-standing bug in which the user directory would assume any remote membership state events represent a profile change.
@@ -1 +0,0 @@
-Add additional functionality to declaring worker types when starting Complement in worker mode.
@@ -1 +0,0 @@
-Stabilise support for [MSC3966](https://github.com/matrix-org/matrix-spec-proposals/pull/3966): `event_property_contains` push condition.
@@ -1 +0,0 @@
-Implement [MSC3873](https://github.com/matrix-org/matrix-spec-proposals/pull/3873) to fix a long-standing bug where properties with dots were handled ambiguously in push rules.
@@ -1 +0,0 @@
-Improve performance of creating and authenticating events.
@@ -1 +0,0 @@
-Make the `HttpTransactionCache` use the `Requester` in addition of the just the `Request` to build the transaction key.
@@ -1 +0,0 @@
-Improve log lines when purging rooms.
@@ -1 +0,0 @@
-Add a missing endpoint to the workers documentation.
@@ -1 +0,0 @@
-Add topic and name events to group of events that are batch persisted when creating a room.
@@ -1 +0,0 @@
-Faster joins: Fix a bug introduced in Synapse 1.66 where spurious "Failed to find memberships ..." errors would be logged.
@@ -1 +0,0 @@
-Move various module API callback registration methods to a dedicated class.
@@ -1 +0,0 @@
-Ensure the Dockerfile builds on platforms that don't have a `cryptography` wheel.
@@ -1 +0,0 @@
-Configure GitHub Actions for merge queues.
@@ -1 +0,0 @@
-Add schema comments about the `destinations` and `destination_rooms` tables.
@@ -1 +0,0 @@
-Bump hiredis from 2.2.1 to 2.2.2.
@@ -1 +0,0 @@
-Bump serde from 1.0.152 to 1.0.155.
@@ -1 +0,0 @@
-Bump pysaml2 from 7.2.1 to 7.3.1.
@@ -1 +0,0 @@
-Bump msgpack from 1.0.4 to 1.0.5.
@@ -1 +0,0 @@
-Bump gitpython from 3.1.30 to 3.1.31.
@@ -1 +0,0 @@
-Bump cryptography from 39.0.1 to 39.0.2.
@@ -1 +0,0 @@
-Skip processing of auto-join room behaviour if there are not auto-join rooms configured.
@@ -1 +0,0 @@
-Remove unused store method `_set_destination_retry_timings_emulated`.
@@ -1 +0,0 @@
-Remove unused class `DirectTcpReplicationClientFactory`.
@@ -0,0 +1 @@
+Speedup tests by caching HomeServerConfig instances.
@@ -0,0 +1 @@
+Experimental support to recursively provide relations per [MSC3981](https://github.com/matrix-org/matrix-spec-proposals/pull/3981).
@@ -0,0 +1 @@
+Experimental support for MSC3970: Scope transaction IDs to devices.
@@ -0,0 +1 @@
+Add an admin API endpoint to support per-user feature flags.
@@ -0,0 +1 @@
+Add denormalised event stream ordering column to membership state tables for future use. Contributed by Nick @ Beeper (@fizzadar).
@@ -0,0 +1 @@
+Disable push rule evaluation for rooms excluded from sync.
@@ -0,0 +1 @@
+Add a module API to send an HTTP push notification.
@@ -0,0 +1 @@
+Docs: Add Nginx loadbalancing example with sticky mxid for workers.
@@ -0,0 +1 @@
+Fix a long-standing bug where cached key results which were directly fetched would not be properly re-used.
@@ -0,0 +1 @@
+Always use multi-user device resync replication endpoints.
@@ -0,0 +1 @@
+Add column `full_user_id` to tables `profiles` and `user_filters`.
@@ -0,0 +1 @@
+Update support for [MSC3983](https://github.com/matrix-org/matrix-spec-proposals/pull/3983) to allow always returning fallback-keys in a `/keys/claim` request.
@@ -0,0 +1 @@
+Update the check_schema_delta script to account for when the schema version has been bumped locally.
@@ -0,0 +1 @@
+Support claiming more than one OTK at a time.
@@ -0,0 +1 @@
+Bump types-pyyaml from 6.0.12.8 to 6.0.12.9.
@@ -0,0 +1 @@
+Bump pyasn1-modules from 0.2.8 to 0.3.0.
@@ -0,0 +1 @@
+Bump cryptography from 40.0.1 to 40.0.2.
@@ -0,0 +1 @@
+Bump types-netaddr from 0.8.0.7 to 0.8.0.8.
@@ -0,0 +1 @@
+Bump types-jsonschema from 4.17.0.6 to 4.17.0.7.
@@ -0,0 +1 @@
+Ask bug reporters to provide logs as text.
@@ -0,0 +1 @@
+Add admin endpoint to query the largest rooms by disk space used in the database.
@@ -0,0 +1 @@
+Fix a bug introduced in Synapse 1.73.0 where some experimental push rules were returned by default.
@@ -0,0 +1 @@
+Add a Nix flake for use as a development environment.
@@ -0,0 +1 @@
+Update outdated development docs that mention restrictions in versions of SQLite that we no longer support.
@@ -0,0 +1 @@
+Bump anyhow from 1.0.70 to 1.0.71.
@@ -0,0 +1 @@
+Bump types-pillow from 9.4.0.19 to 9.5.0.2.
@@ -0,0 +1 @@
+Bump packaging from 23.0 to 23.1.
@@ -0,0 +1 @@
+Bump types-requests from 2.28.11.16 to 2.29.0.0.
@@ -0,0 +1 @@
+Bump setuptools-rust from 1.5.2 to 1.6.0.
@@ -0,0 +1 @@
+Reduce the size of the HTTP connection pool for non-pushers.
@@ -0,0 +1 @@
+Fix bug where we didn't clean out old highlight push notifications from the database. Introduced in v1.62.0.
@@ -1,3 +1,51 @@
+matrix-synapse-py3 (1.82.0) stable; urgency=medium
+
+  * New Synapse release 1.82.0.
+
+ -- Synapse Packaging team <packages@matrix.org>  Tue, 25 Apr 2023 11:56:06 +0100
+
+matrix-synapse-py3 (1.82.0~rc1) stable; urgency=medium
+
+  * New Synapse release 1.82.0rc1.
+
+ -- Synapse Packaging team <packages@matrix.org>  Tue, 18 Apr 2023 09:47:30 +0100
+
+matrix-synapse-py3 (1.81.0) stable; urgency=medium
+
+  * New Synapse release 1.81.0.
+
+ -- Synapse Packaging team <packages@matrix.org>  Tue, 11 Apr 2023 14:18:35 +0100
+
+matrix-synapse-py3 (1.81.0~rc2) stable; urgency=medium
+
+  * New Synapse release 1.81.0rc2.
+
+ -- Synapse Packaging team <packages@matrix.org>  Thu, 06 Apr 2023 16:07:54 +0100
+
+matrix-synapse-py3 (1.81.0~rc1) stable; urgency=medium
+
+  * New Synapse release 1.81.0rc1.
+
+ -- Synapse Packaging team <packages@matrix.org>  Tue, 04 Apr 2023 14:29:03 +0100
+
+matrix-synapse-py3 (1.80.0) stable; urgency=medium
+
+  * New Synapse release 1.80.0.
+
+ -- Synapse Packaging team <packages@matrix.org>  Tue, 28 Mar 2023 11:10:33 +0100
+
+matrix-synapse-py3 (1.80.0~rc2) stable; urgency=medium
+
+  * New Synapse release 1.80.0rc2.
+
+ -- Synapse Packaging team <packages@matrix.org>  Wed, 22 Mar 2023 08:30:16 -0700
+
+matrix-synapse-py3 (1.80.0~rc1) stable; urgency=medium
+
+  * New Synapse release 1.80.0rc1.
+
+ -- Synapse Packaging team <packages@matrix.org>  Tue, 21 Mar 2023 10:56:08 -0700
+
 matrix-synapse-py3 (1.79.0) stable; urgency=medium

  * New Synapse release 1.79.0.
@@ -0,0 +1,20 @@
+# Minimal makefile for Sphinx documentation
+#
+
+# You can set these variables from the command line, and also
+# from the environment for the first two.
+SPHINXOPTS    ?=
+SPHINXBUILD   ?= sphinx-build
+SOURCEDIR     = .
+BUILDDIR      = _build
+
+# Put it first so that "make" without argument is like "make help".
+help:
+	@$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
+
+.PHONY: help Makefile
+
+# Catch-all target: route all unknown targets to Sphinx using the new
+# "make mode" option.  $(O) is meant as a shortcut for $(SPHINXOPTS).
+%: Makefile
+	@$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
@@ -0,0 +1,50 @@
+# Configuration file for the Sphinx documentation builder.
+#
+# For the full list of built-in configuration values, see the documentation:
+# https://www.sphinx-doc.org/en/master/usage/configuration.html
+
+# -- Project information -----------------------------------------------------
+# https://www.sphinx-doc.org/en/master/usage/configuration.html#project-information
+
+project = "Synapse development"
+copyright = "2023, The Matrix.org Foundation C.I.C."
+author = "The Synapse Maintainers and Community"
+
+# -- General configuration ---------------------------------------------------
+# https://www.sphinx-doc.org/en/master/usage/configuration.html#general-configuration
+
+extensions = [
+    "autodoc2",
+    "myst_parser",
+]
+
+templates_path = ["_templates"]
+exclude_patterns = ["_build", "Thumbs.db", ".DS_Store"]
+
+
+# -- Options for Autodoc2 ----------------------------------------------------
+
+autodoc2_docstring_parser_regexes = [
+    # this will render all docstrings as 'MyST' Markdown
+    (r".*", "myst"),
+]
+
+autodoc2_packages = [
+    {
+        "path": "../synapse",
+        # Don't render documentation for everything as a matter of course
+        "auto_mode": False,
+    },
+]
+
+
+# -- Options for MyST (Markdown) ---------------------------------------------
+
+# myst_heading_anchors = 2
+
+
+# -- Options for HTML output -------------------------------------------------
+# https://www.sphinx-doc.org/en/master/usage/configuration.html#options-for-html-output
+
+html_theme = "furo"
+html_static_path = ["_static"]
@@ -0,0 +1,22 @@
+.. Synapse Developer Documentation documentation master file, created by
+   sphinx-quickstart on Mon Mar 13 08:59:51 2023.
+   You can adapt this file completely to your liking, but it should at least
+   contain the root `toctree` directive.
+
+Welcome to the Synapse Developer Documentation!
+===========================================================
+
+.. toctree::
+   :maxdepth: 2
+   :caption: Contents:
+
+   modules/federation_sender
+
+
+
+Indices and tables
+==================
+
+* :ref:`genindex`
+* :ref:`modindex`
+* :ref:`search`
@@ -0,0 +1,5 @@
+Federation Sender
+=================
+
+```{autodoc2-docstring} synapse.federation.sender
+```
@@ -163,6 +163,7 @@ WORKERS_CONFIG: Dict[str, Dict[str, Any]] = {
            "^/_matrix/client/versions$",
            "^/_matrix/client/(api/v1|r0|v3|unstable)/voip/turnServer$",
            "^/_matrix/client/(r0|v3|unstable)/register$",
+            "^/_matrix/client/(r0|v3|unstable)/register/available$",
            "^/_matrix/client/(r0|v3|unstable)/auth/.*/fallback/web$",
            "^/_matrix/client/(api/v1|r0|v3|unstable)/rooms/.*/messages$",
            "^/_matrix/client/(api/v1|r0|v3|unstable)/rooms/.*/event",
@@ -171,6 +172,9 @@ WORKERS_CONFIG: Dict[str, Dict[str, Any]] = {
            "^/_matrix/client/v1/rooms/.*/timestamp_to_event$",
            "^/_matrix/client/(api/v1|r0|v3|unstable)/search",
            "^/_matrix/client/(r0|v3|unstable)/user/.*/filter(/|$)",
+            "^/_matrix/client/(r0|v3|unstable)/password_policy$",
+            "^/_matrix/client/(api/v1|r0|v3|unstable)/directory/room/.*$",
+            "^/_matrix/client/(r0|v3|unstable)/capabilities$",
        ],
        "shared_extra_conf": {},
        "worker_extra_conf": "",
@@ -422,7 +426,7 @@ def add_worker_roles_to_shared_config(


 def merge_worker_template_configs(
-    existing_dict: Dict[str, Any] | None,
+    existing_dict: Optional[Dict[str, Any]],
    to_be_merged_dict: Dict[str, Any],
 ) -> Dict[str, Any]:
    """When given an existing dict of worker template configuration consisting with both
@@ -0,0 +1,54 @@
+# Experimental Features API
+
+This API allows a server administrator to enable or disable some experimental features on a per-user
+basis. Currently supported features are [msc3026](https://github.com/matrix-org/matrix-spec-proposals/pull/3026): busy 
+presence state enabled, [msc2654](https://github.com/matrix-org/matrix-spec-proposals/pull/2654): enable unread counts,
+[msc3881](https://github.com/matrix-org/matrix-spec-proposals/pull/3881): enable remotely toggling push notifications 
+for another client, and [msc3967](https://github.com/matrix-org/matrix-spec-proposals/pull/3967): do not require
+UIA when first uploading cross-signing keys. 
+
+
+To use it, you will need to authenticate by providing an `access_token`
+for a server admin: see [Admin API](../usage/administration/admin_api/).
+
+## Enabling/Disabling Features
+
+This API allows a server administrator to enable experimental features for a given user. The request must 
+provide a body containing the user id and listing the features to enable/disable in the following format:
+```json
+{
+   "features": {
+      "msc3026":true,
+      "msc2654":true
+   }
+}
+```
+where true is  used to enable the feature, and false is used to disable the feature.
+
+
+The API is:
+
+```
+PUT /_synapse/admin/v1/experimental_features/<user_id>
+```
+
+## Listing Enabled Features
+ 
+To list which features are enabled/disabled for a given user send a request to the following API:
+
+```
+GET /_synapse/admin/v1/experimental_features/<user_id>
+```
+
+It will return a list of possible features and indicate whether they are enabled or disabled for the
+user like so:
+```json
+{
+   "features": {
+      "msc3026": true,
+      "msc2654": true,
+      "msc3881": false,
+      "msc3967": false
+   }
+}
+```
@@ -81,3 +81,52 @@ The following fields are returned in the JSON response body:
  - `user_id` - string - Fully-qualified user ID (ex. `@user:server.com`).
 * `next_token` - integer - Opaque value used for pagination. See above.
 * `total` - integer - Total number of users after filtering.
+
+
+# Get largest rooms by size in database
+
+Returns the 10 largest rooms and an estimate of how much space in the database
+they are taking.
+
+This does not include the size of any associated media associated with the room.
+
+Returns an error on SQLite.
+
+*Note:* This uses the planner statistics from PostgreSQL to do the estimates,
+which means that the returned information can vary widely from reality. However,
+it should be enough to get a rough idea of where database disk space is going.
+
+
+The API is:
+
+```
+GET /_synapse/admin/v1/statistics/statistics/database/rooms
+```
+
+A response body like the following is returned:
+
+```json
+{
+  "rooms": [
+    {
+      "room_id": "!OGEhHVWSdvArJzumhm:matrix.org",
+      "estimated_size": 47325417353
+    }
+  ],
+}
+```
+
+
+
+**Response**
+
+The following fields are returned in the JSON response body:
+
+* `rooms` - An array of objects, sorted by largest room first. Objects contain
+  the following fields:
+  - `room_id` - string - The room ID.
+  - `estimated_size` - integer - Estimated disk space used in bytes by the room
+    in the database.
+
+
+*Added in Synapse 1.83.0*
@@ -155,43 +155,11 @@ def run_upgrade(
 Boolean columns require special treatment, since SQLite treats booleans the
 same as integers.

-There are three separate aspects to this:
-
- * Any new boolean column must be added to the `BOOLEAN_COLUMNS` list in
+Any new boolean column must be added to the `BOOLEAN_COLUMNS` list in
   `synapse/_scripts/synapse_port_db.py`. This tells the port script to cast
   the integer value from SQLite to a boolean before writing the value to the
   postgres database.

- * Before SQLite 3.23, `TRUE` and `FALSE` were not recognised as constants by
-   SQLite, and the `IS [NOT] TRUE`/`IS [NOT] FALSE` operators were not
-   supported. This makes it necessary to avoid using `TRUE` and `FALSE`
-   constants in SQL commands.
-
-   For example, to insert a `TRUE` value into the database, write:
-
-   ```python
-   txn.execute("INSERT INTO tbl(col) VALUES (?)", (True, ))
-   ```
-
- * Default values for new boolean columns present a particular
-   difficulty. Generally it is best to create separate schema files for
-   Postgres and SQLite. For example:
-
-   ```sql
-   # in 00delta.sql.postgres:
-   ALTER TABLE tbl ADD COLUMN col BOOLEAN DEFAULT FALSE;
-   ```
-
-   ```sql
-   # in 00delta.sql.sqlite:
-   ALTER TABLE tbl ADD COLUMN col BOOLEAN DEFAULT 0;
-   ```
-
-   Note that there is a particularly insidious failure mode here: the Postgres
-   flavour will be accepted by SQLite 3.22, but will give a column whose
-   default value is the **string** `"FALSE"` - which, when cast back to a boolean
-   in Python, evaluates to `True`.
-

 ## `event_id` global uniqueness

@@ -103,6 +103,9 @@ Called during a logout request for a user. It is passed the qualified user ID, t
 deactivated device (if any: access tokens are occasionally created without an associated
 device ID), and the (now deactivated) access token.

+Deleting the related pushers is done after calling `on_logged_out`, so you can rely on them
+to still be present.
+
 If multiple modules implement this callback, Synapse runs them all in order.

 ### `get_username_for_registration`
@@ -26,8 +26,8 @@ for most users.
 #### Docker images and Ansible playbooks

 There is an official synapse image available at
-<https://hub.docker.com/r/matrixdotorg/synapse> which can be used with
-the docker-compose file available at
+<https://hub.docker.com/r/matrixdotorg/synapse> or at [`ghcr.io/matrix-org/synapse`](https://ghcr.io/matrix-org/synapse)
+which can be used with the docker-compose file available at
 [contrib/docker](https://github.com/matrix-org/synapse/tree/develop/contrib/docker).
 Further information on this including configuration options is available in the README
 on hub.docker.com.
@@ -25,7 +25,7 @@ position of all streams. The server then periodically sends `RDATA` commands
 which have the format `RDATA <stream_name> <instance_name> <token> <row>`, where
 the format of `<row>` is defined by the individual streams. The
 `<instance_name>` is the name of the Synapse process that generated the data
-(usually "master").
+(usually "master"). We expect an RDATA for every row in the DB.

 Error reporting happens by either the client or server sending an ERROR
 command, and usually the connection will be closed.
@@ -107,7 +107,7 @@ reconnect, following the steps above.
 If the server sends messages faster than the client can consume them the
 server will first buffer a (fairly large) number of commands and then
 disconnect the client. This ensures that we don't queue up an unbounded
-number of commands in memory and gives us a potential oppurtunity to
+number of commands in memory and gives us a potential opportunity to
 squawk loudly. When/if the client recovers it can reconnect to the
 server and ask for missed messages.

@@ -122,7 +122,7 @@ since these include tokens which can be used to restart the stream on
 connection errors.

 The client should keep track of the token in the last RDATA command
-received for each stream so that on reconneciton it can start streaming
+received for each stream so that on reconnection it can start streaming
 from the correct place. Note: not all RDATA have valid tokens due to
 batching. See `RdataCommand` for more details.

@@ -188,7 +188,8 @@ client (C):
   Two positions are included, the "new" position and the last position sent respectively.
   This allows servers to tell instances that the positions have advanced but no
   data has been written, without clients needlessly checking to see if they
-   have missed any updates.
+   have missed any updates. Instances will only fetch stuff if there is a gap between
+   their current position and the given last position.

 #### ERROR (S, C)

@@ -88,6 +88,34 @@ process, for example:
    dpkg -i matrix-synapse-py3_1.3.0+stretch1_amd64.deb
    ```

+# Upgrading to v1.81.0
+
+## Application service path & authentication deprecations
+
+Synapse now attempts the versioned appservice paths before falling back to the
+[legacy paths](https://spec.matrix.org/v1.6/application-service-api/#legacy-routes).
+Usage of the legacy routes should be considered deprecated.
+
+Additionally, Synapse has supported sending the application service access token
+via [the `Authorization` header](https://spec.matrix.org/v1.6/application-service-api/#authorization)
+since v1.70.0. For backwards compatibility it is *also* sent as the `access_token`
+query parameter. This is insecure and should be considered deprecated.
+
+A future version of Synapse (v1.88.0 or later) will remove support for legacy
+application service routes and query parameter authorization.
+
+# Upgrading to v1.80.0
+
+## Reporting events error code change
+
+Before this update, the
+[`POST /_matrix/client/v3/rooms/{roomId}/report/{eventId}`](https://spec.matrix.org/v1.6/client-server-api/#post_matrixclientv3roomsroomidreporteventid)
+endpoint would return a `403` if a user attempted to report an event that they did not have access to.
+This endpoint will now return a `404` in this case instead.
+
+Clients that implement event reporting should check that their error handling code will handle this
+change.
+
 # Upgrading to v1.79.0

 ## The `on_threepid_bind` module callback method has been deprecated
@@ -171,6 +199,17 @@ Docker images and Debian packages need nothing specific as they already
 include or specify ICU as an explicit dependency.


+## User directory rebuild
+
+Synapse 1.74 queues a background update
+[to rebuild the user directory](https://github.com/matrix-org/synapse/pull/14643),
+in order to fix missing or erroneous entries.
+
+When this update begins, the user directory will be cleared out and rebuilt from
+scratch. User directory lookups will be incomplete until the rebuild completes.
+Admins can monitor the rebuild's progress by using the
+[Background update Admin API](usage/administration/admin_api/background_updates.md#status).
+
 # Upgrading to v1.73.0

 ## Legacy Prometheus metric names have now been removed
@@ -577,6 +577,10 @@ delete any device that hasn't been accessed for more than the specified amount o

 Defaults to no duration, which means devices are never pruned.

+**Note:** This task will always run on the main process, regardless of the value of
+`run_background_tasks_on`. This is due to workers currently not having the ability to
+delete devices.
+
 Example configuration:
 ```yaml
 delete_stale_devices_after: 1y
@@ -1521,7 +1525,7 @@ This option specifies several limits for login:
      address. Defaults to `per_second: 0.003`, `burst_count: 5`.

 * `account` ratelimits login requests based on the account the
-  client is attempting to log into. Defaults to `per_second: 0.03`,
+  client is attempting to log into. Defaults to `per_second: 0.003`,
  `burst_count: 5`.

 * `failed_attempts` ratelimits login requests based on the account the
@@ -3100,6 +3104,11 @@ Options for each entry include:
   match a pre-existing account instead of failing. This could be used if
   switching from password logins to OIDC. Defaults to false.

+* `enable_registration`: set to 'false' to disable automatic registration of new
+   users. This allows the OIDC SSO flow to be limited to sign in only, rather than
+   automatically registering users that have a valid SSO login but do not have
+   a pre-registered account. Defaults to true.
+
 * `user_mapping_provider`: Configuration for how attributes returned from a OIDC
   provider are mapped onto a matrix user. This setting has the following
   sub-properties:
@@ -3216,6 +3225,7 @@ oidc_providers:
    userinfo_endpoint: "https://accounts.example.com/userinfo"
    jwks_uri: "https://accounts.example.com/.well-known/jwks.json"
    skip_verification: true
+    enable_registration: true
    user_mapping_provider:
      config:
        subject_claim: "id"
@@ -234,6 +234,8 @@ information.
    ^/_matrix/client/(api/v1|r0|v3|unstable/.*)/rooms/.*/aliases
    ^/_matrix/client/(api/v1|r0|v3|unstable)/search$
    ^/_matrix/client/(r0|v3|unstable)/user/.*/filter(/|$)
+    ^/_matrix/client/(api/v1|r0|v3|unstable)/directory/room/.*$
+    ^/_matrix/client/(r0|v3|unstable)/capabilities$

    # Encryption requests
    ^/_matrix/client/(r0|v3|unstable)/keys/query$
@@ -245,7 +247,9 @@ information.
    # Registration/login requests
    ^/_matrix/client/(api/v1|r0|v3|unstable)/login$
    ^/_matrix/client/(r0|v3|unstable)/register$
+    ^/_matrix/client/(r0|v3|unstable)/register/available$
    ^/_matrix/client/v1/register/m.login.registration_token/validity$
+    ^/_matrix/client/(r0|v3|unstable)/password_policy$

    # Event sending requests
    ^/_matrix/client/(api/v1|r0|v3|unstable)/rooms/.*/redact
@@ -321,8 +325,7 @@ load balancing can be done in different ways.

 For `/sync` and `/initialSync` requests it will be more efficient if all
 requests from a particular user are routed to a single instance. This can
-be done e.g. in nginx via IP `hash $http_x_forwarded_for;` or via
-`hash $http_authorization consistent;` which contains the users access token.
+be done in reverse proxy by extracting username part from the users access token.

 Admins may additionally wish to separate out `/sync`
 requests that have a `since` query parameter from those that don't (and
@@ -331,6 +334,69 @@ when a user logs in on a new device and can be *very* resource intensive, so
 isolating these requests will stop them from interfering with other users ongoing
 syncs.

+Example `nginx` configuration snippet that handles the cases above. This is just an
+example and probably requires some changes according to your particular setup:
+
+```nginx
+# Choose sync worker based on the existence of "since" query parameter
+map $arg_since $sync {
+    default synapse_sync;
+    '' synapse_initial_sync;
+}
+
+# Extract username from access token passed as URL parameter
+map $arg_access_token $accesstoken_from_urlparam {
+    # Defaults to just passing back the whole accesstoken
+    default   $arg_access_token;
+    # Try to extract username part from accesstoken URL parameter
+    "~syt_(?<username>.*?)_.*"           $username;
+}
+
+# Extract username from access token passed as authorization header
+map $http_authorization $mxid_localpart {
+    # Defaults to just passing back the whole accesstoken
+    default                              $http_authorization;
+    # Try to extract username part from accesstoken header
+    "~Bearer syt_(?<username>.*?)_.*"    $username;
+    # if no authorization-header exist, try mapper for URL parameter "access_token"
+    ""                                   $accesstoken_from_urlparam;
+}
+
+upstream synapse_initial_sync {
+    # Use the username mapper result for hash key
+    hash $mxid_localpart consistent;
+    server 127.0.0.1:8016;
+    server 127.0.0.1:8036;
+}
+
+upstream synapse_sync {
+    # Use the username mapper result for hash key
+    hash $mxid_localpart consistent;
+    server 127.0.0.1:8013;
+    server 127.0.0.1:8037;
+    server 127.0.0.1:8038;
+    server 127.0.0.1:8039;
+}
+
+# Sync initial/normal
+location ~ ^/_matrix/client/(r0|v3)/sync$ {
+	proxy_pass http://$sync;
+}
+
+# Normal sync
+location ~ ^/_matrix/client/(api/v1|r0|v3)/events$ {
+	proxy_pass http://synapse_sync;
+}
+
+# Initial_sync
+location ~ ^/_matrix/client/(api/v1|r0|v3)/initialSync$ {
+	proxy_pass http://synapse_initial_sync;
+}
+location ~ ^/_matrix/client/(api/v1|r0|v3)/rooms/[^/]+/initialSync$ {
+	proxy_pass http://synapse_initial_sync;
+}
+```
+
 Federation and client requests can be balanced via simple round robin.

 The inbound federation transaction request `^/_matrix/federation/v1/send/`
@@ -0,0 +1,274 @@
+{
+  "nodes": {
+    "devenv": {
+      "inputs": {
+        "flake-compat": "flake-compat",
+        "nix": "nix",
+        "nixpkgs": "nixpkgs",
+        "pre-commit-hooks": "pre-commit-hooks"
+      },
+      "locked": {
+        "lastModified": 1682534083,
+        "narHash": "sha256-lBgFaLNHRQtD3InZbBXzIS8HgZUgcPJ6jiqGa4FJPrk=",
+        "owner": "anoadragon453",
+        "repo": "devenv",
+        "rev": "9694bd0a845dd184d4468cc3d3461089aace787a",
+        "type": "github"
+      },
+      "original": {
+        "owner": "anoadragon453",
+        "ref": "anoa/fix_languages_python",
+        "repo": "devenv",
+        "type": "github"
+      }
+    },
+    "fenix": {
+      "inputs": {
+        "nixpkgs": [
+          "nixpkgs"
+        ],
+        "rust-analyzer-src": "rust-analyzer-src"
+      },
+      "locked": {
+        "lastModified": 1682490133,
+        "narHash": "sha256-tR2Qx0uuk97WySpSSk4rGS/oH7xb5LykbjATcw1vw1I=",
+        "owner": "nix-community",
+        "repo": "fenix",
+        "rev": "4e9412753ab75ef0e038a5fe54a062fb44c27c6a",
+        "type": "github"
+      },
+      "original": {
+        "owner": "nix-community",
+        "repo": "fenix",
+        "type": "github"
+      }
+    },
+    "flake-compat": {
+      "flake": false,
+      "locked": {
+        "lastModified": 1673956053,
+        "narHash": "sha256-4gtG9iQuiKITOjNQQeQIpoIB6b16fm+504Ch3sNKLd8=",
+        "owner": "edolstra",
+        "repo": "flake-compat",
+        "rev": "35bb57c0c8d8b62bbfd284272c928ceb64ddbde9",
+        "type": "github"
+      },
+      "original": {
+        "owner": "edolstra",
+        "repo": "flake-compat",
+        "type": "github"
+      }
+    },
+    "flake-utils": {
+      "locked": {
+        "lastModified": 1667395993,
+        "narHash": "sha256-nuEHfE/LcWyuSWnS8t12N1wc105Qtau+/OdUAjtQ0rA=",
+        "owner": "numtide",
+        "repo": "flake-utils",
+        "rev": "5aed5285a952e0b949eb3ba02c12fa4fcfef535f",
+        "type": "github"
+      },
+      "original": {
+        "owner": "numtide",
+        "repo": "flake-utils",
+        "type": "github"
+      }
+    },
+    "gitignore": {
+      "inputs": {
+        "nixpkgs": [
+          "devenv",
+          "pre-commit-hooks",
+          "nixpkgs"
+        ]
+      },
+      "locked": {
+        "lastModified": 1660459072,
+        "narHash": "sha256-8DFJjXG8zqoONA1vXtgeKXy68KdJL5UaXR8NtVMUbx8=",
+        "owner": "hercules-ci",
+        "repo": "gitignore.nix",
+        "rev": "a20de23b925fd8264fd7fad6454652e142fd7f73",
+        "type": "github"
+      },
+      "original": {
+        "owner": "hercules-ci",
+        "repo": "gitignore.nix",
+        "type": "github"
+      }
+    },
+    "lowdown-src": {
+      "flake": false,
+      "locked": {
+        "lastModified": 1633514407,
+        "narHash": "sha256-Dw32tiMjdK9t3ETl5fzGrutQTzh2rufgZV4A/BbxuD4=",
+        "owner": "kristapsdz",
+        "repo": "lowdown",
+        "rev": "d2c2b44ff6c27b936ec27358a2653caaef8f73b8",
+        "type": "github"
+      },
+      "original": {
+        "owner": "kristapsdz",
+        "repo": "lowdown",
+        "type": "github"
+      }
+    },
+    "nix": {
+      "inputs": {
+        "lowdown-src": "lowdown-src",
+        "nixpkgs": [
+          "devenv",
+          "nixpkgs"
+        ],
+        "nixpkgs-regression": "nixpkgs-regression"
+      },
+      "locked": {
+        "lastModified": 1676545802,
+        "narHash": "sha256-EK4rZ+Hd5hsvXnzSzk2ikhStJnD63odF7SzsQ8CuSPU=",
+        "owner": "domenkozar",
+        "repo": "nix",
+        "rev": "7c91803598ffbcfe4a55c44ac6d49b2cf07a527f",
+        "type": "github"
+      },
+      "original": {
+        "owner": "domenkozar",
+        "ref": "relaxed-flakes",
+        "repo": "nix",
+        "type": "github"
+      }
+    },
+    "nixpkgs": {
+      "locked": {
+        "lastModified": 1678875422,
+        "narHash": "sha256-T3o6NcQPwXjxJMn2shz86Chch4ljXgZn746c2caGxd8=",
+        "owner": "NixOS",
+        "repo": "nixpkgs",
+        "rev": "126f49a01de5b7e35a43fd43f891ecf6d3a51459",
+        "type": "github"
+      },
+      "original": {
+        "owner": "NixOS",
+        "ref": "nixpkgs-unstable",
+        "repo": "nixpkgs",
+        "type": "github"
+      }
+    },
+    "nixpkgs-regression": {
+      "locked": {
+        "lastModified": 1643052045,
+        "narHash": "sha256-uGJ0VXIhWKGXxkeNnq4TvV3CIOkUJ3PAoLZ3HMzNVMw=",
+        "owner": "NixOS",
+        "repo": "nixpkgs",
+        "rev": "215d4d0fd80ca5163643b03a33fde804a29cc1e2",
+        "type": "github"
+      },
+      "original": {
+        "owner": "NixOS",
+        "repo": "nixpkgs",
+        "rev": "215d4d0fd80ca5163643b03a33fde804a29cc1e2",
+        "type": "github"
+      }
+    },
+    "nixpkgs-stable": {
+      "locked": {
+        "lastModified": 1673800717,
+        "narHash": "sha256-SFHraUqLSu5cC6IxTprex/nTsI81ZQAtDvlBvGDWfnA=",
+        "owner": "NixOS",
+        "repo": "nixpkgs",
+        "rev": "2f9fd351ec37f5d479556cd48be4ca340da59b8f",
+        "type": "github"
+      },
+      "original": {
+        "owner": "NixOS",
+        "ref": "nixos-22.11",
+        "repo": "nixpkgs",
+        "type": "github"
+      }
+    },
+    "nixpkgs_2": {
+      "locked": {
+        "lastModified": 1682519441,
+        "narHash": "sha256-Vsq/8NOtvW1AoC6shCBxRxZyMQ+LhvPuJT6ltbzuv+Y=",
+        "owner": "NixOS",
+        "repo": "nixpkgs",
+        "rev": "7a32a141db568abde9bc389845949dc2a454dfd3",
+        "type": "github"
+      },
+      "original": {
+        "owner": "NixOS",
+        "ref": "master",
+        "repo": "nixpkgs",
+        "type": "github"
+      }
+    },
+    "pre-commit-hooks": {
+      "inputs": {
+        "flake-compat": [
+          "devenv",
+          "flake-compat"
+        ],
+        "flake-utils": "flake-utils",
+        "gitignore": "gitignore",
+        "nixpkgs": [
+          "devenv",
+          "nixpkgs"
+        ],
+        "nixpkgs-stable": "nixpkgs-stable"
+      },
+      "locked": {
+        "lastModified": 1678376203,
+        "narHash": "sha256-3tyYGyC8h7fBwncLZy5nCUjTJPrHbmNwp47LlNLOHSM=",
+        "owner": "cachix",
+        "repo": "pre-commit-hooks.nix",
+        "rev": "1a20b9708962096ec2481eeb2ddca29ed747770a",
+        "type": "github"
+      },
+      "original": {
+        "owner": "cachix",
+        "repo": "pre-commit-hooks.nix",
+        "type": "github"
+      }
+    },
+    "root": {
+      "inputs": {
+        "devenv": "devenv",
+        "fenix": "fenix",
+        "nixpkgs": "nixpkgs_2",
+        "systems": "systems"
+      }
+    },
+    "rust-analyzer-src": {
+      "flake": false,
+      "locked": {
+        "lastModified": 1682426789,
+        "narHash": "sha256-UqnLmJESRZE0tTEaGbRAw05Hm19TWIPA+R3meqi5I4w=",
+        "owner": "rust-lang",
+        "repo": "rust-analyzer",
+        "rev": "943d2a8a1ca15e8b28a1f51f5a5c135e3728da04",
+        "type": "github"
+      },
+      "original": {
+        "owner": "rust-lang",
+        "ref": "nightly",
+        "repo": "rust-analyzer",
+        "type": "github"
+      }
+    },
+    "systems": {
+      "locked": {
+        "lastModified": 1681028828,
+        "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=",
+        "owner": "nix-systems",
+        "repo": "default",
+        "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e",
+        "type": "github"
+      },
+      "original": {
+        "owner": "nix-systems",
+        "repo": "default",
+        "type": "github"
+      }
+    }
+  },
+  "root": "root",
+  "version": 7
+}
@@ -0,0 +1,204 @@
+# A nix flake that sets up a complete Synapse development environment. Dependencies
+# for the SyTest (https://github.com/matrix-org/sytest) and Complement
+# (https://github.com/matrix-org/complement) Matrix homeserver test suites are also
+# installed automatically.
+#
+# You must have already installed nix (https://nixos.org) on your system to use this.
+# nix can be installed on Linux or MacOS; NixOS is not required. Windows is not
+# directly supported, but nix can be installed inside of WSL2 or even Docker
+# containers. Please refer to https://nixos.org/download for details.
+#
+# You must also enable support for flakes in Nix. See the following for how to
+# do so permanently: https://nixos.wiki/wiki/Flakes#Enable_flakes
+#
+# Usage:
+#
+# With nix installed, navigate to the directory containing this flake and run
+# `nix develop --impure`. The `--impure` is necessary in order to store state
+# locally from "services", such as PostgreSQL and Redis.
+#
+# You should now be dropped into a new shell with all programs and dependencies
+# availabile to you!
+#
+# You can start up pre-configured, local PostgreSQL and Redis instances by
+# running: `devenv up`. To stop them, use Ctrl-C.
+#
+# A PostgreSQL database called 'synapse' will be set up for you, along with
+# a PostgreSQL user named 'synapse_user'.
+# The 'host' can be found by running `echo $PGHOST` with the development
+# shell activated. Use these values to configure your Synapse to connect
+# to the local PostgreSQL database. You do not need to specify a password.
+# https://matrix-org.github.io/synapse/latest/postgres
+#
+# All state (the venv, postgres and redis data and config) are stored in
+# .devenv/state. Deleting a file from here and then re-entering the shell
+# will recreate these files from scratch.
+#
+# You can exit the development shell by typing `exit`, or using Ctrl-D.
+#
+# If you would like this development environment to activate automatically
+# upon entering this directory in your terminal, first install `direnv`
+# (https://direnv.net/). Then run `echo 'use flake . --impure' >> .envrc` at
+# the root of the Synapse repo. Finally, run `direnv allow .` to allow the
+# contents of '.envrc' to run every time you enter this directory. Voilà!
+
+{
+  inputs = {
+    # Use the master/unstable branch of nixpkgs. The latest stable, 22.11,
+    # does not contain 'perl536Packages.NetAsyncHTTP', needed by Sytest.
+    nixpkgs.url = "github:NixOS/nixpkgs/master";
+    # Output a development shell for x86_64/aarch64 Linux/Darwin (MacOS).
+    systems.url = "github:nix-systems/default";
+    # A development environment manager built on Nix. See https://devenv.sh.
+    # This is temporarily overridden to a fork that fixes a quirk between
+    # devenv's service and python language features. This can be removed
+    # when https://github.com/cachix/devenv/pull/559 is merged upstream.
+    devenv.url = "github:anoadragon453/devenv/anoa/fix_languages_python";
+    #devenv.url = "github:cachix/devenv/main";
+    # Rust toolchains and rust-analyzer nightly.
+    fenix = {
+      url = "github:nix-community/fenix";
+      inputs.nixpkgs.follows = "nixpkgs";
+    };
+  };
+
+  outputs = { self, nixpkgs, devenv, systems, ... } @ inputs:
+    let
+      forEachSystem = nixpkgs.lib.genAttrs (import systems);
+    in {
+      devShells = forEachSystem (system:
+        let
+          pkgs = nixpkgs.legacyPackages.${system};
+        in {
+          # Everything is configured via devenv - a nix module for creating declarative
+          # developer environments. See https://devenv.sh/reference/options/ for a list
+          # of all possible options.
+          default = devenv.lib.mkShell {
+            inherit inputs pkgs;
+            modules = [
+              {
+                # Make use of the Starship command prompt when this development environment
+                # is manually activated (via `nix develop --impure`).
+                # See https://starship.rs/ for details on the prompt itself.
+                starship.enable = true;
+
+                # Configure packages to install.
+                # Search for package names at https://search.nixos.org/packages?channel=unstable
+                packages = with pkgs; [
+                  # Native dependencies for running Synapse.
+                  icu
+                  libffi
+                  libjpeg
+                  libpqxx
+                  libwebp
+                  libxml2
+                  libxslt
+                  sqlite
+
+                  # Native dependencies for unit tests (SyTest also requires OpenSSL).
+                  openssl
+
+                  # Native dependencies for running Complement.
+                  olm
+                ];
+
+                # Install Python and manage a virtualenv with Poetry.
+                languages.python.enable = true;
+                languages.python.poetry.enable = true;
+                # Automatically activate the poetry virtualenv upon entering the shell.
+                languages.python.poetry.activate.enable = true;
+                # Install all extra Python dependencies; this is needed to run the unit
+                # tests and utilitise all Synapse features.
+                languages.python.poetry.install.arguments = ["--extras all"];
+                # Install the 'matrix-synapse' package from the local checkout.
+                languages.python.poetry.install.installRootPackage = true;
+
+                # This is a work-around for NixOS systems. NixOS is special in
+                # that you can have multiple versions of packages installed at
+                # once, including your libc linker!
+                #
+                # Some binaries built for Linux expect those to be in a certain
+                # filepath, but that is not the case on NixOS. In that case, we
+                # force compiling those binaries locally instead.
+                env.POETRY_INSTALLER_NO_BINARY = "ruff";
+
+                # Install dependencies for the additional programming languages
+                # involved with Synapse development.
+                #
+                # * Rust is used for developing and running Synapse.
+                # * Golang is needed to run the Complement test suite.
+                # * Perl is needed to run the SyTest test suite.
+                languages.go.enable = true;
+                languages.rust.enable = true;
+                languages.rust.version = "stable";
+                languages.perl.enable = true;
+
+                # Postgres is needed to run Synapse with postgres support and
+                # to run certain unit tests that require postgres.
+                services.postgres.enable = true;
+
+                # On the first invocation of `devenv up`, create a database for
+                # Synapse to store data in.
+                services.postgres.initdbArgs = ["--locale=C" "--encoding=UTF8"];
+                services.postgres.initialDatabases = [
+                  { name = "synapse"; }
+                ];
+                # Create a postgres user called 'synapse_user' which has ownership
+                # over the 'synapse' database.
+                services.postgres.initialScript = ''
+                CREATE USER synapse_user;
+                ALTER DATABASE synapse OWNER TO synapse_user;
+                '';
+
+                # Redis is needed in order to run Synapse in worker mode.
+                services.redis.enable = true;
+
+                # Define the perl modules we require to run SyTest.
+                #
+                # This list was compiled by cross-referencing https://metacpan.org/
+                # with the modules defined in './cpanfile' and then finding the
+                # corresponding nix packages on https://search.nixos.org/packages.
+                #
+                # This was done until `./install-deps.pl --dryrun` produced no output.
+                env.PERL5LIB = "${with pkgs.perl536Packages; makePerlPath [
+                  DBI
+                  ClassMethodModifiers
+                  CryptEd25519
+                  DataDump
+                  DBDPg
+                  DigestHMAC
+                  DigestSHA1
+                  EmailAddressXS
+                  EmailMIME
+                  EmailSimple  # required by Email::Mime
+                  EmailMessageID  # required by Email::Mime
+                  EmailMIMEContentType  # required by Email::Mime
+                  TextUnidecode  # required by Email::Mime
+                  ModuleRuntime  # required by Email::Mime
+                  EmailMIMEEncodings  # required by Email::Mime
+                  FilePath
+                  FileSlurper
+                  Future
+                  GetoptLong
+                  HTTPMessage
+                  IOAsync
+                  IOAsyncSSL
+                  IOSocketSSL
+                  NetSSLeay
+                  JSON
+                  ListUtilsBy
+                  ScalarListUtils
+                  ModulePluggable
+                  NetAsyncHTTP
+                  MetricsAny  # required by Net::Async::HTTP
+                  NetAsyncHTTPServer
+                  StructDumb
+                  URI
+                  YAMLLibYAML
+                ]}";
+              }
+            ];
+          };
+        });
+    };
+}
@@ -21,26 +21,7 @@ files =
  tests/,
  build_rust.py

-# Note: Better exclusion syntax coming in mypy > 0.910
-# https://github.com/python/mypy/pull/11329
-#
-# For now, set the (?x) flag enable "verbose" regexes
-# https://docs.python.org/3/library/re.html#re.X
-exclude = (?x)
-  ^(
-   |synapse/storage/databases/__init__.py
-   |synapse/storage/databases/main/cache.py
-   |synapse/storage/schema/
-   )$
-
-[mypy-synapse.federation.transport.client]
-disallow_untyped_defs = False
-
-[mypy-synapse.http.matrixfederationclient]
-disallow_untyped_defs = False
-
 [mypy-synapse.metrics._reactor_metrics]
-disallow_untyped_defs = False
 # This module imports select.epoll. That exists on Linux, but doesn't on macOS.
 # See https://github.com/matrix-org/synapse/pull/11771.
 warn_unused_ignores = False
@@ -89,7 +89,7 @@ manifest-path = "rust/Cargo.toml"

 [tool.poetry]
 name = "matrix-synapse"
-version = "1.79.0"
+version = "1.82.0"
 description = "Homeserver for the Matrix decentralised comms protocol"
 authors = ["Matrix.org Team and Contributors <packages@matrix.org>"]
 license = "Apache-2.0"
@@ -153,15 +153,13 @@ python = "^3.7.1"
 # ----------------------
 # we use the TYPE_CHECKER.redefine method added in jsonschema 3.0.0
 jsonschema = ">=3.0.0"
-# frozendict 2.1.2 is broken on Debian 10: https://github.com/Marco-Sulla/python-frozendict/issues/41
-# We cannot test our wheels against the 2.3.5 release in CI. Putting in an upper bound for this
-# because frozendict has been more trouble than it's worth; we would like to move to immutabledict.
-frozendict = ">=1,!=2.1.2,<2.3.5"
+# We choose 2.0 as a lower bound: the most recent backwards incompatible release.
+# It seems generally available, judging by https://pkgs.org/search/?q=immutabledict
+immutabledict = ">=2.0"
 # We require 2.1.0 or higher for type hints. Previous guard was >= 1.1.0
 unpaddedbase64 = ">=2.1.0"
-# We require 1.5.0 to work around an issue when running against the C implementation of
-# frozendict: https://github.com/matrix-org/python-canonicaljson/issues/36
-canonicaljson = "^1.5.0"
+# We require 2.0.0 for immutabledict support.
+canonicaljson = "^2.0.0"
 # we use the type definitions added in signedjson 1.1.
 signedjson = "^1.1.0"
 # validating SSL certs for IP addresses requires service_identity 18.1.
@@ -313,7 +311,7 @@ all = [
 # We pin black so that our tests don't start failing on new releases.
 isort = ">=5.10.1"
 black = ">=22.3.0"
-ruff = "0.0.252"
+ruff = "0.0.259"

 # Typechecking
 mypy = "*"
@@ -352,6 +350,18 @@ towncrier = ">=18.6.0rc1"
 # Used for checking the Poetry lockfile
 tomli = ">=1.2.3"

+
+# Dependencies for building the development documentation
+[tool.poetry.group.dev-docs]
+optional = true
+
+[tool.poetry.group.dev-docs.dependencies]
+sphinx = {version = "^6.1", python = "^3.8"}
+sphinx-autodoc2 = {version = "^0.4.2", python = "^3.8"}
+myst-parser = {version = "^1.0.0", python = "^3.8"}
+furo = ">=2022.12.7,<2024.0.0"
+
+
 [build-system]
 # The upper bounds here are defensive, intended to prevent situations like
 # #13849 and #14079 where we see buildtime or runtime errors caused by build
@@ -568,7 +568,10 @@ impl FilteredPushRules {
            .filter(|rule| {
                // Ignore disabled experimental push rules

-                if !self.msc1767_enabled && rule.rule_id.contains("org.matrix.msc1767") {
+                if !self.msc1767_enabled
+                    && (rule.rule_id.contains("org.matrix.msc1767")
+                        || rule.rule_id.contains("org.matrix.msc3933"))
+                {
                    return false;
                }

@@ -28,6 +28,7 @@ DISTS = (
    "ubuntu:focal",  # 20.04 LTS (our EOL forced by Py38 on 2024-10-14)
    "ubuntu:jammy",  # 22.04 LTS (EOL 2027-04)
    "ubuntu:kinetic",  # 22.10 (EOL 2023-07-20)
+    "ubuntu:lunar",  # 23.04 (EOL 2024-01)
 )

 DESC = """\
@@ -40,10 +40,32 @@ def main(force_colors: bool) -> None:
    exec(r, locals)
    current_schema_version = locals["SCHEMA_VERSION"]

-    click.secho(f"Current schema version: {current_schema_version}")
-
    diffs: List[git.Diff] = repo.remote().refs.develop.commit.diff(None)

+    # Get the schema version of the local file to check against current schema on develop
+    with open("synapse/storage/schema/__init__.py", "r") as file:
+        local_schema = file.read()
+    new_locals: Dict[str, Any] = {}
+    exec(local_schema, new_locals)
+    local_schema_version = new_locals["SCHEMA_VERSION"]
+
+    if local_schema_version != current_schema_version:
+        # local schema version must be +/-1 the current schema version on develop
+        if abs(local_schema_version - current_schema_version) != 1:
+            click.secho(
+                "The proposed schema version has diverged more than one version from develop, please fix!",
+                fg="red",
+                bold=True,
+                color=force_colors,
+            )
+            click.get_current_context().exit(1)
+
+        # right, we've changed the schema version within the allowable tolerance so
+        # let's now use the local version as the canonical version
+        current_schema_version = local_schema_version
+
+    click.secho(f"Current schema version: {current_schema_version}")
+
    seen_deltas = False
    bad_files = []
    for diff in diffs:
@@ -91,6 +91,7 @@ else
          "synapse" "docker" "tests"
          "scripts-dev"
          "contrib" "synmark" "stubs" ".ci"
+          "dev-docs"
      )
  fi
 fi
@@ -280,7 +280,7 @@ def _prepare() -> None:
    )

    print("Opening the changelog in your browser...")
-    print("Please ask others to give it a check.")
+    print("Please ask #synapse-dev to give it a check.")
    click.launch(
        f"https://github.com/matrix-org/synapse/blob/{synapse_repo.active_branch.name}/CHANGES.md"
    )
@@ -1,39 +0,0 @@
-# Copyright 2020 The Matrix.org Foundation C.I.C.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# Stub for frozendict.
-
-from __future__ import annotations
-
-from typing import Any, Hashable, Iterable, Iterator, Mapping, Tuple, TypeVar, overload
-
-_KT = TypeVar("_KT", bound=Hashable)  # Key type.
-_VT = TypeVar("_VT")  # Value type.
-
-class frozendict(Mapping[_KT, _VT]):
-    @overload
-    def __init__(self, **kwargs: _VT) -> None: ...
-    @overload
-    def __init__(self, __map: Mapping[_KT, _VT], **kwargs: _VT) -> None: ...
-    @overload
-    def __init__(
-        self, __iterable: Iterable[Tuple[_KT, _VT]], **kwargs: _VT
-    ) -> None: ...
-    def __getitem__(self, key: _KT) -> _VT: ...
-    def __contains__(self, key: Any) -> bool: ...
-    def copy(self, **add_or_replace: Any) -> frozendict: ...
-    def __iter__(self) -> Iterator[_KT]: ...
-    def __len__(self) -> int: ...
-    def __repr__(self) -> str: ...
-    def __hash__(self) -> int: ...
@@ -17,9 +17,9 @@
 """ This is an implementation of a Matrix homeserver.
 """

-import json
 import os
 import sys
+from typing import Any, Dict

 from synapse.util.rust import check_rust_lib_up_to_date
 from synapse.util.stringutils import strtobool
@@ -61,11 +61,20 @@ try:
 except ImportError:
    pass

-# Use the standard library json implementation instead of simplejson.
+# Teach canonicaljson how to serialise immutabledicts.
 try:
-    from canonicaljson import set_json_library
+    from canonicaljson import register_preserialisation_callback
+    from immutabledict import immutabledict

-    set_json_library(json)
+    def _immutabledict_cb(d: immutabledict) -> Dict[str, Any]:
+        try:
+            return d._dict
+        except Exception:
+            # Paranoia: fall back to a `dict()` call, in case a future version of
+            # immutabledict removes `_dict` from the implementation.
+            return dict(d)
+
+    register_preserialisation_callback(immutabledict, _immutabledict_cb)
 except ImportError:
    pass

@@ -0,0 +1,302 @@
+#!/usr/bin/env python
+# Copyright 2022-2023 The Matrix.org Foundation C.I.C.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import argparse
+import logging
+import re
+from collections import defaultdict
+from dataclasses import dataclass
+from typing import Dict, Iterable, Optional, Pattern, Set, Tuple
+
+import yaml
+
+from synapse.config.homeserver import HomeServerConfig
+from synapse.federation.transport.server import (
+    TransportLayerServer,
+    register_servlets as register_federation_servlets,
+)
+from synapse.http.server import HttpServer, ServletCallback
+from synapse.rest import ClientRestResource
+from synapse.rest.key.v2 import RemoteKey
+from synapse.server import HomeServer
+from synapse.storage import DataStore
+
+logger = logging.getLogger("generate_workers_map")
+
+
+class MockHomeserver(HomeServer):
+    DATASTORE_CLASS = DataStore  # type: ignore
+
+    def __init__(self, config: HomeServerConfig, worker_app: Optional[str]) -> None:
+        super().__init__(config.server.server_name, config=config)
+        self.config.worker.worker_app = worker_app
+
+
+GROUP_PATTERN = re.compile(r"\(\?P<[^>]+?>(.+?)\)")
+
+
+@dataclass
+class EndpointDescription:
+    """
+    Describes an endpoint and how it should be routed.
+    """
+
+    # The servlet class that handles this endpoint
+    servlet_class: object
+
+    # The category of this endpoint. Is read from the `CATEGORY` constant in the servlet
+    # class.
+    category: Optional[str]
+
+    # TODO:
+    #  - does it need to be routed based on a stream writer config?
+    #  - does it benefit from any optimised, but optional, routing?
+    #  - what 'opinionated synapse worker class' (event_creator, synchrotron, etc) does
+    #    it go in?
+
+
+class EnumerationResource(HttpServer):
+    """
+    Accepts servlet registrations for the purposes of building up a description of
+    all endpoints.
+    """
+
+    def __init__(self, is_worker: bool) -> None:
+        self.registrations: Dict[Tuple[str, str], EndpointDescription] = {}
+        self._is_worker = is_worker
+
+    def register_paths(
+        self,
+        method: str,
+        path_patterns: Iterable[Pattern],
+        callback: ServletCallback,
+        servlet_classname: str,
+    ) -> None:
+        # federation servlet callbacks are wrapped, so unwrap them.
+        callback = getattr(callback, "__wrapped__", callback)
+
+        # fish out the servlet class
+        servlet_class = callback.__self__.__class__  # type: ignore
+
+        if self._is_worker and method in getattr(
+            servlet_class, "WORKERS_DENIED_METHODS", ()
+        ):
+            # This endpoint would cause an error if called on a worker, so pretend it
+            # was never registered!
+            return
+
+        sd = EndpointDescription(
+            servlet_class=servlet_class,
+            category=getattr(servlet_class, "CATEGORY", None),
+        )
+
+        for pat in path_patterns:
+            self.registrations[(method, pat.pattern)] = sd
+
+
+def get_registered_paths_for_hs(
+    hs: HomeServer,
+) -> Dict[Tuple[str, str], EndpointDescription]:
+    """
+    Given a homeserver, get all registered endpoints and their descriptions.
+    """
+
+    enumerator = EnumerationResource(is_worker=hs.config.worker.worker_app is not None)
+    ClientRestResource.register_servlets(enumerator, hs)
+    federation_server = TransportLayerServer(hs)
+
+    # we can't use `federation_server.register_servlets` but this line does the
+    # same thing, only it uses this enumerator
+    register_federation_servlets(
+        federation_server.hs,
+        resource=enumerator,
+        ratelimiter=federation_server.ratelimiter,
+        authenticator=federation_server.authenticator,
+        servlet_groups=federation_server.servlet_groups,
+    )
+
+    # the key server endpoints are separate again
+    RemoteKey(hs).register(enumerator)
+
+    return enumerator.registrations
+
+
+def get_registered_paths_for_default(
+    worker_app: Optional[str], base_config: HomeServerConfig
+) -> Dict[Tuple[str, str], EndpointDescription]:
+    """
+    Given the name of a worker application and a base homeserver configuration,
+    returns:
+
+        Dict from (method, path) to EndpointDescription
+
+    TODO Don't require passing in a config
+    """
+
+    hs = MockHomeserver(base_config, worker_app)
+    # TODO We only do this to avoid an error, but don't need the database etc
+    hs.setup()
+    return get_registered_paths_for_hs(hs)
+
+
+def elide_http_methods_if_unconflicting(
+    registrations: Dict[Tuple[str, str], EndpointDescription],
+    all_possible_registrations: Dict[Tuple[str, str], EndpointDescription],
+) -> Dict[Tuple[str, str], EndpointDescription]:
+    """
+    Elides HTTP methods (by replacing them with `*`) if all possible registered methods
+    can be handled by the worker whose registration map is `registrations`.
+
+    i.e. the only endpoints left with methods (other than `*`) should be the ones where
+    the worker can't handle all possible methods for that path.
+    """
+
+    def paths_to_methods_dict(
+        methods_and_paths: Iterable[Tuple[str, str]]
+    ) -> Dict[str, Set[str]]:
+        """
+        Given (method, path) pairs, produces a dict from path to set of methods
+        available at that path.
+        """
+        result: Dict[str, Set[str]] = {}
+        for method, path in methods_and_paths:
+            result.setdefault(path, set()).add(method)
+        return result
+
+    all_possible_reg_methods = paths_to_methods_dict(all_possible_registrations)
+    reg_methods = paths_to_methods_dict(registrations)
+
+    output = {}
+
+    for path, handleable_methods in reg_methods.items():
+        if handleable_methods == all_possible_reg_methods[path]:
+            any_method = next(iter(handleable_methods))
+            # TODO This assumes that all methods have the same servlet.
+            #      I suppose that's possibly dubious?
+            output[("*", path)] = registrations[(any_method, path)]
+        else:
+            for method in handleable_methods:
+                output[(method, path)] = registrations[(method, path)]
+
+    return output
+
+
+def simplify_path_regexes(
+    registrations: Dict[Tuple[str, str], EndpointDescription]
+) -> Dict[Tuple[str, str], EndpointDescription]:
+    """
+    Simplify all the path regexes for the dict of endpoint descriptions,
+    so that we don't use the Python-specific regex extensions
+    (and also to remove needlessly specific detail).
+    """
+
+    def simplify_path_regex(path: str) -> str:
+        """
+        Given a regex pattern, replaces all named capturing groups (e.g. `(?P<blah>xyz)`)
+        with a simpler version available in more common regex dialects (e.g. `.*`).
+        """
+
+        # TODO it's hard to choose between these two;
+        #      `.*` is a vague simplification
+        # return GROUP_PATTERN.sub(r"\1", path)
+        return GROUP_PATTERN.sub(r".*", path)
+
+    return {(m, simplify_path_regex(p)): v for (m, p), v in registrations.items()}
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(
+        description=(
+            "Updates a synapse database to the latest schema and optionally runs background updates"
+            " on it."
+        )
+    )
+    parser.add_argument("-v", action="store_true")
+    parser.add_argument(
+        "--config-path",
+        type=argparse.FileType("r"),
+        required=True,
+        help="Synapse configuration file",
+    )
+
+    args = parser.parse_args()
+
+    # TODO
+    # logging.basicConfig(**logging_config)
+
+    # Load, process and sanity-check the config.
+    hs_config = yaml.safe_load(args.config_path)
+
+    config = HomeServerConfig()
+    config.parse_config_dict(hs_config, "", "")
+
+    master_paths = get_registered_paths_for_default(None, config)
+    worker_paths = get_registered_paths_for_default(
+        "synapse.app.generic_worker", config
+    )
+
+    all_paths = {**master_paths, **worker_paths}
+
+    elided_worker_paths = elide_http_methods_if_unconflicting(worker_paths, all_paths)
+    elide_http_methods_if_unconflicting(master_paths, all_paths)
+
+    # TODO SSO endpoints (pick_idp etc) NOT REGISTERED BY THIS SCRIPT
+
+    categories_to_methods_and_paths: Dict[
+        Optional[str], Dict[Tuple[str, str], EndpointDescription]
+    ] = defaultdict(dict)
+
+    for (method, path), desc in elided_worker_paths.items():
+        categories_to_methods_and_paths[desc.category][method, path] = desc
+
+    for category, contents in categories_to_methods_and_paths.items():
+        print_category(category, contents)
+
+
+def print_category(
+    category_name: Optional[str],
+    elided_worker_paths: Dict[Tuple[str, str], EndpointDescription],
+) -> None:
+    """
+    Prints out a category, in documentation page style.
+
+    Example:
+    ```
+    # Category name
+    /path/xyz
+
+    GET /path/abc
+    ```
+    """
+
+    if category_name:
+        print(f"# {category_name}")
+    else:
+        print("# (Uncategorised requests)")
+
+    for ln in sorted(
+        p for m, p in simplify_path_regexes(elided_worker_paths) if m == "*"
+    ):
+        print(ln)
+    print()
+    for ln in sorted(
+        f"{m:6} {p}" for m, p in simplify_path_regexes(elided_worker_paths) if m != "*"
+    ):
+        print(ln)
+    print()
+
+
+if __name__ == "__main__":
+    main()
@@ -18,6 +18,7 @@
 import argparse
 import curses
 import logging
+import os
 import sys
 import time
 import traceback
@@ -53,11 +54,12 @@ from synapse.logging.context import (
 )
 from synapse.notifier import ReplicationNotifier
 from synapse.storage.database import DatabasePool, LoggingTransaction, make_conn
-from synapse.storage.databases.main import PushRuleStore
+from synapse.storage.databases.main import FilteringWorkerStore, PushRuleStore
 from synapse.storage.databases.main.account_data import AccountDataWorkerStore
 from synapse.storage.databases.main.client_ips import ClientIpBackgroundUpdateStore
 from synapse.storage.databases.main.deviceinbox import DeviceInboxBackgroundUpdateStore
 from synapse.storage.databases.main.devices import DeviceBackgroundUpdateStore
+from synapse.storage.databases.main.e2e_room_keys import EndToEndRoomKeyBackgroundStore
 from synapse.storage.databases.main.end_to_end_keys import EndToEndKeyBackgroundStore
 from synapse.storage.databases.main.event_push_actions import EventPushActionsStore
 from synapse.storage.databases.main.events_bg_updates import (
@@ -67,7 +69,11 @@ from synapse.storage.databases.main.media_repository import (
    MediaRepositoryBackgroundUpdateStore,
 )
 from synapse.storage.databases.main.presence import PresenceBackgroundUpdateStore
-from synapse.storage.databases.main.pusher import PusherWorkerStore
+from synapse.storage.databases.main.profile import ProfileWorkerStore
+from synapse.storage.databases.main.pusher import (
+    PusherBackgroundUpdatesStore,
+    PusherWorkerStore,
+)
 from synapse.storage.databases.main.receipts import ReceiptsBackgroundUpdateStore
 from synapse.storage.databases.main.registration import (
    RegistrationBackgroundUpdateStore,
@@ -119,6 +125,7 @@ BOOLEAN_COLUMNS = {
    "users": ["shadow_banned", "approved"],
    "un_partial_stated_event_stream": ["rejection_status_changed"],
    "users_who_share_rooms": ["share_private"],
+    "per_user_experimental_features": ["enabled"],
 }


@@ -221,10 +228,14 @@ class Store(
    MainStateBackgroundUpdateStore,
    UserDirectoryBackgroundUpdateStore,
    EndToEndKeyBackgroundStore,
+    EndToEndRoomKeyBackgroundStore,
    StatsStore,
    AccountDataWorkerStore,
+    FilteringWorkerStore,
+    ProfileWorkerStore,
    PushRuleStore,
    PusherWorkerStore,
+    PusherBackgroundUpdatesStore,
    PresenceBackgroundUpdateStore,
    ReceiptsBackgroundUpdateStore,
    RelationsWorkerStore,
@@ -1326,10 +1337,17 @@ def main() -> None:
        filename="port-synapse.log" if args.curses else None,
    )

+    if not os.path.isfile(args.sqlite_database):
+        sys.stderr.write(
+            "The sqlite database you specified does not exist, please check that you have the"
+            "correct path."
+        )
+        sys.exit(1)
+
    sqlite_config = {
        "name": "sqlite3",
        "args": {
-            "database": "file:{}?mode=rw".format(args.sqlite_database),
+            "database": args.sqlite_database,
            "cp_min": 1,
            "cp_max": 1,
            "check_same_thread": False,
@@ -215,6 +215,8 @@ class EventContentFields:
    FEDERATE: Final = "m.federate"

    # The creator of the room, as used in `m.room.create` events.
+    #
+    # This is deprecated in MSC2175.
    ROOM_CREATOR: Final = "creator"

    # Used in m.room.guest_access events.
--- a/Show More
+++ b/Show More
				`@@ -1 +0,0 @@`
				`Fix a long-standing bug in which the user directory would assume any remote membership state events represent a profile change.`
				`@@ -1 +0,0 @@`
				`Add additional functionality to declaring worker types when starting Complement in worker mode.`
				`@@ -1 +0,0 @@`
				Stabilise support for [MSC3966](https://github.com/matrix-org/matrix-spec-proposals/pull/3966): `event_property_contains` push condition.
				`@@ -1 +0,0 @@`
				`Implement [MSC3873](https://github.com/matrix-org/matrix-spec-proposals/pull/3873) to fix a long-standing bug where properties with dots were handled ambiguously in push rules.`
				`@@ -1 +0,0 @@`
				`Improve performance of creating and authenticating events.`
				`@@ -1 +0,0 @@`
				Make the `HttpTransactionCache` use the `Requester` in addition of the just the `Request` to build the transaction key.
				`@@ -1 +0,0 @@`
				`Add a missing endpoint to the workers documentation.`
				`@@ -1 +0,0 @@`
				`Add topic and name events to group of events that are batch persisted when creating a room.`
				`@@ -1 +0,0 @@`
				`Faster joins: Fix a bug introduced in Synapse 1.66 where spurious "Failed to find memberships ..." errors would be logged.`
				`@@ -1 +0,0 @@`
				`Move various module API callback registration methods to a dedicated class.`