Merge branch 'develop' into devon/remote-invite-reject-sss-fix

This PR fixes #18154 to avoid de-deltaing state groups which resulted in
DB size temporarily increasing until the DB was `VACUUM`'ed. As a
result, less state groups will get deleted now.
It also attempts to improve performance by not duplicating work when
processing state groups it has already processed in previous iterations.

### Pull Request Checklist

<!-- Please read
https://element-hq.github.io/synapse/latest/development/contributing_guide.html
before submitting your pull request -->

* [X] Pull request is based on the develop branch
* [X] Pull request includes a [changelog
file](https://element-hq.github.io/synapse/latest/development/contributing_guide.html#changelog).
The entry should:
- Be a short description of your change which makes sense to users.
"Fixed a bug that prevented receiving messages from other servers."
instead of "Moved X method from `EventStore` to `EventWorkerStore`.".
  - Use markdown where necessary, mostly for `code blocks`.
  - End with either a period (.) or an exclamation mark (!).
  - Start with a capital letter.
- Feel free to credit yourself, by adding a sentence "Contributed by
@github_username." or "Contributed by [Your Name]." to the end of the
entry.
* [X] [Code
style](https://element-hq.github.io/synapse/latest/code_style.html) is
correct
(run the
[linters](https://element-hq.github.io/synapse/latest/development/contributing_guide.html#run-the-linters))

---------

Co-authored-by: Erik Johnston <erikj@element.io>

.ci/scripts/check_lockfile.py

@@ -11,12 +11,12 @@ with open("poetry.lock", "rb") as f:
 
 try:
     lock_version = lockfile["metadata"]["lock-version"]
-    assert lock_version == "2.0"
+    assert lock_version == "2.1"
 except Exception:
     print(
         """\
-    Lockfile is not version 2.0. You probably need to upgrade poetry on your local box
-    and re-run `poetry lock --no-update`. See the Poetry cheat sheet at
+    Lockfile is not version 2.1. You probably need to upgrade poetry on your local box
+    and re-run `poetry lock`. See the Poetry cheat sheet at
     https://element-hq.github.io/synapse/develop/development/dependencies.html
     """
     )

.github/workflows/docker.yml

@@ -18,22 +18,22 @@ jobs:
     steps:
       - name: Set up QEMU
         id: qemu
-        uses: docker/setup-qemu-action@v3
+        uses: docker/setup-qemu-action@29109295f81e9208d7d86ff1c6c12d2833863392 # v3.6.0
         with:
           platforms: arm64
 
       - name: Set up Docker Buildx
         id: buildx
-        uses: docker/setup-buildx-action@v3
+        uses: docker/setup-buildx-action@b5ca514318bd6ebac0fb2aedd5d36ec1b5c232a2 # v3.10.0
 
       - name: Inspect builder
         run: docker buildx inspect
 
       - name: Install Cosign
-        uses: sigstore/cosign-installer@v3.8.1
+        uses: sigstore/cosign-installer@d7d6bc7722e3daa8354c50bcb52f4837da5e9b6a # v3.8.1
 
       - name: Checkout repository
-        uses: actions/checkout@v4
+        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
 
       - name: Extract version from pyproject.toml
         # Note: explicitly requesting bash will mean bash is invoked with `-eo pipefail`, see
@@ -43,13 +43,13 @@ jobs:
           echo "SYNAPSE_VERSION=$(grep "^version" pyproject.toml | sed -E 's/version\s*=\s*["]([^"]*)["]/\1/')" >> $GITHUB_ENV
 
       - name: Log in to DockerHub
-        uses: docker/login-action@v3
+        uses: docker/login-action@74a5d142397b4f367a81961eba4e8cd7edddf772 # v3.4.0
         with:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_TOKEN }}
 
       - name: Log in to GHCR
-        uses: docker/login-action@v3
+        uses: docker/login-action@74a5d142397b4f367a81961eba4e8cd7edddf772 # v3.4.0
         with:
           registry: ghcr.io
           username: ${{ github.repository_owner }}
@@ -57,7 +57,7 @@ jobs:
 
       - name: Calculate docker image tag
         id: set-tag
-        uses: docker/metadata-action@master
+        uses: docker/metadata-action@902fa8ec7d6ecbf8d84d538b9b233a880e428804 # v5.7.0
         with:
           images: |
             docker.io/matrixdotorg/synapse
@@ -72,7 +72,7 @@ jobs:
 
       - name: Build and push all platforms
         id: build-and-push
-        uses: docker/build-push-action@v6
+        uses: docker/build-push-action@471d1dc4e07e5cdedd4c2171150001c434f0b7a4 # v6.15.0
         with:
           push: true
           labels: |

.github/workflows/docs-pr-netlify.yaml

@@ -22,7 +22,7 @@ jobs:
           path: book
 
       - name: 📤 Deploy to Netlify
-        uses: matrix-org/netlify-pr-preview@v3
+        uses: matrix-org/netlify-pr-preview@9805cd123fc9a7e421e35340a05e1ebc5dee46b5 # v3
         with:
           path: book
           owner: ${{ github.event.workflow_run.head_repository.owner.login }}

.github/workflows/docs-pr.yaml

@@ -13,7 +13,7 @@ jobs:
     name: GitHub Pages
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
         with:
           # Fetch all history so that the schema_versions script works.
           fetch-depth: 0
@@ -24,7 +24,7 @@ jobs:
           mdbook-version: '0.4.17'
 
       - name: Setup python
-        uses: actions/setup-python@v5
+        uses: actions/setup-python@42375524e23c412d93fb67b49958b491fce71c38 # v5.4.0
         with:
           python-version: "3.x"
 
@@ -39,7 +39,7 @@ jobs:
           cp book/welcome_and_overview.html book/index.html
 
       - name: Upload Artifact
-        uses: actions/upload-artifact@v4
+        uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 # v4.6.1
         with:
           name: book
           path: book
@@ -50,7 +50,7 @@ jobs:
     name: Check links in documentation
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
 
       - name: Setup mdbook
         uses: peaceiris/actions-mdbook@ee69d230fe19748b7abf22df32acaa93833fad08 # v2.0.0

.github/workflows/docs.yaml

@@ -50,7 +50,7 @@ jobs:
     needs:
       - pre
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
         with:
           # Fetch all history so that the schema_versions script works.
           fetch-depth: 0
@@ -64,7 +64,7 @@ jobs:
         run: echo 'window.SYNAPSE_VERSION = "${{ needs.pre.outputs.branch-version }}";' > ./docs/website_files/version.js
 
       - name: Setup python
-        uses: actions/setup-python@v5
+        uses: actions/setup-python@42375524e23c412d93fb67b49958b491fce71c38 # v5.4.0
         with:
           python-version: "3.x"
 

.github/workflows/fix_lint.yaml

@@ -13,21 +13,22 @@ jobs:
 
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v4
+        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
 
       - name: Install Rust
-        uses: dtolnay/rust-toolchain@master
+        uses: dtolnay/rust-toolchain@56f84321dbccf38fb67ce29ab63e4754056677e0 # master (rust 1.85.1)
         with:
           # We use nightly so that `fmt` correctly groups together imports, and
           # clippy correctly fixes up the benchmarks.
           toolchain: nightly-2022-12-01
           components: clippy, rustfmt
-      - uses: Swatinem/rust-cache@v2
+      - uses: Swatinem/rust-cache@9d47c6ad4b02e050fd481d890b2ea34778fd09d6 # v2.7.8
 
       - name: Setup Poetry
-        uses: matrix-org/setup-python-poetry@v1
+        uses: matrix-org/setup-python-poetry@5bbf6603c5c930615ec8a29f1b5d7d258d905aa4 # v2.0.0
         with:
           install-project: "false"
+          poetry-version: "2.1.1"
 
       - name: Run ruff check
         continue-on-error: true
@@ -43,6 +44,6 @@ jobs:
       - run: cargo fmt
         continue-on-error: true
 
-      - uses: stefanzweifel/git-auto-commit-action@v5
+      - uses: stefanzweifel/git-auto-commit-action@e348103e9026cc0eee72ae06630dbe30c8bf7a79 # v5.1.0
         with:
           commit_message: "Attempt to fix linting"

.github/workflows/latest_deps.yml

@@ -39,17 +39,17 @@ jobs:
     if: needs.check_repo.outputs.should_run_workflow == 'true'
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
       - name: Install Rust
-        uses: dtolnay/rust-toolchain@stable
-      - uses: Swatinem/rust-cache@v2
+        uses: dtolnay/rust-toolchain@fcf085fcb4b4b8f63f96906cd713eb52181b5ea4 # stable (rust 1.85.1)
+      - uses: Swatinem/rust-cache@9d47c6ad4b02e050fd481d890b2ea34778fd09d6 # v2.7.8
 
       # The dev dependencies aren't exposed in the wheel metadata (at least with current
       # poetry-core versions), so we install with poetry.
-      - uses: matrix-org/setup-python-poetry@v1
+      - uses: matrix-org/setup-python-poetry@5bbf6603c5c930615ec8a29f1b5d7d258d905aa4 # v2.0.0
         with:
           python-version: "3.x"
-          poetry-version: "1.3.2"
+          poetry-version: "2.1.1"
           extras: "all"
       # Dump installed versions for debugging.
       - run: poetry run pip list > before.txt
@@ -72,11 +72,11 @@ jobs:
             postgres-version: "14"
 
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
 
       - name: Install Rust
-        uses: dtolnay/rust-toolchain@stable
-      - uses: Swatinem/rust-cache@v2
+        uses: dtolnay/rust-toolchain@fcf085fcb4b4b8f63f96906cd713eb52181b5ea4 # stable (rust 1.85.1)
+      - uses: Swatinem/rust-cache@9d47c6ad4b02e050fd481d890b2ea34778fd09d6 # v2.7.8
 
       - run: sudo apt-get -qq install xmlsec1
       - name: Set up PostgreSQL ${{ matrix.postgres-version }}
@@ -86,7 +86,7 @@ jobs:
             -e POSTGRES_PASSWORD=postgres \
             -e POSTGRES_INITDB_ARGS="--lc-collate C --lc-ctype C --encoding UTF8" \
             postgres:${{ matrix.postgres-version }}
-      - uses: actions/setup-python@v5
+      - uses: actions/setup-python@42375524e23c412d93fb67b49958b491fce71c38 # v5.4.0
         with:
           python-version: "3.x"
       - run: pip install .[all,test]
@@ -145,11 +145,11 @@ jobs:
       BLACKLIST: ${{ matrix.workers && 'synapse-blacklist-with-workers' }}
 
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
 
       - name: Install Rust
-        uses: dtolnay/rust-toolchain@stable
-      - uses: Swatinem/rust-cache@v2
+        uses: dtolnay/rust-toolchain@fcf085fcb4b4b8f63f96906cd713eb52181b5ea4 # stable (rust 1.85.1)
+      - uses: Swatinem/rust-cache@9d47c6ad4b02e050fd481d890b2ea34778fd09d6 # v2.7.8
 
       - name: Ensure sytest runs `pip install`
         # Delete the lockfile so sytest will `pip install` rather than `poetry install`
@@ -164,7 +164,7 @@ jobs:
         if: ${{ always() }}
         run: /sytest/scripts/tap_to_gha.pl /logs/results.tap
       - name: Upload SyTest logs
-        uses: actions/upload-artifact@v4
+        uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 # v4.6.1
         if: ${{ always() }}
         with:
           name: Sytest Logs - ${{ job.status }} - (${{ join(matrix.*, ', ') }})
@@ -192,15 +192,15 @@ jobs:
             database: Postgres
 
     steps:
-      - name: Run actions/checkout@v4 for synapse
-        uses: actions/checkout@v4
+      - name: Check out synapse codebase
+        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
         with:
           path: synapse
 
       - name: Prepare Complement's Prerequisites
         run: synapse/.ci/scripts/setup_complement_prerequisites.sh
 
-      - uses: actions/setup-go@v5
+      - uses: actions/setup-go@0aaccfd150d50ccaeb58ebd88d36e91967a5f35b # v5.4.0
         with:
           cache-dependency-path: complement/go.sum
           go-version-file: complement/go.mod
@@ -225,7 +225,7 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
       - uses: JasonEtco/create-an-issue@1b14a70e4d8dc185e5cc76d3bec9eab20257b2c5 # v2.9.2
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/poetry_lockfile.yaml

@@ -16,8 +16,8 @@ jobs:
     name: "Check locked dependencies have sdists"
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v4
-      - uses: actions/setup-python@v5
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+      - uses: actions/setup-python@42375524e23c412d93fb67b49958b491fce71c38 # v5.4.0
         with:
           python-version: '3.x'
       - run: pip install tomli

.github/workflows/push_complement_image.yml

@@ -33,29 +33,29 @@ jobs:
       packages: write
     steps:
       - name: Checkout specific branch (debug build)
-        uses: actions/checkout@v4
+        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
         if: github.event_name == 'workflow_dispatch'
         with:
           ref: ${{ inputs.branch }}
       - name: Checkout clean copy of develop (scheduled build)
-        uses: actions/checkout@v4
+        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
         if: github.event_name == 'schedule'
         with:
           ref: develop
       - name: Checkout clean copy of master (on-push)
-        uses: actions/checkout@v4
+        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
         if: github.event_name == 'push'
         with:
           ref: master
       - name: Login to registry
-        uses: docker/login-action@v3
+        uses: docker/login-action@74a5d142397b4f367a81961eba4e8cd7edddf772 # v3.4.0
         with:
           registry: ghcr.io
           username: ${{ github.actor }}
           password: ${{ secrets.GITHUB_TOKEN }}
       - name: Work out labels for complement image
         id: meta
-        uses: docker/metadata-action@v5
+        uses: docker/metadata-action@902fa8ec7d6ecbf8d84d538b9b233a880e428804 # v5.7.0
         with:
           images: ghcr.io/${{ github.repository }}/complement-synapse
           tags: |

.github/workflows/release-artifacts.yml

@@ -27,8 +27,8 @@ jobs:
     name: "Calculate list of debian distros"
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v4
-      - uses: actions/setup-python@v5
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+      - uses: actions/setup-python@42375524e23c412d93fb67b49958b491fce71c38 # v5.4.0
         with:
           python-version: '3.x'
       - id: set-distros
@@ -55,18 +55,18 @@ jobs:
 
     steps:
       - name: Checkout
-        uses: actions/checkout@v4
+        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
         with:
           path: src
 
       - name: Set up Docker Buildx
         id: buildx
-        uses: docker/setup-buildx-action@v3
+        uses: docker/setup-buildx-action@b5ca514318bd6ebac0fb2aedd5d36ec1b5c232a2 # v3.10.0
         with:
           install: true
 
       - name: Set up docker layer caching
-        uses: actions/cache@v4
+        uses: actions/cache@d4323d4df104b026a6aa633fdb11d772146be0bf # v4.2.2
         with:
           path: /tmp/.buildx-cache
           key: ${{ runner.os }}-buildx-${{ github.sha }}
@@ -74,7 +74,7 @@ jobs:
             ${{ runner.os }}-buildx-
 
       - name: Set up python
-        uses: actions/setup-python@v5
+        uses: actions/setup-python@42375524e23c412d93fb67b49958b491fce71c38 # v5.4.0
         with:
           python-version: '3.x'
 
@@ -101,7 +101,7 @@ jobs:
           echo "ARTIFACT_NAME=${DISTRO#*:}" >> "$GITHUB_OUTPUT"
 
       - name: Upload debs as artifacts
-        uses: actions/upload-artifact@v4
+        uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 # v4.6.1
         with:
           name: debs-${{ steps.artifact-name.outputs.ARTIFACT_NAME }}
           path: debs/*
@@ -130,9 +130,9 @@ jobs:
             arch: aarch64
 
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
 
-      - uses: actions/setup-python@v5
+      - uses: actions/setup-python@42375524e23c412d93fb67b49958b491fce71c38 # v5.4.0
         with:
           # setup-python@v4 doesn't impose a default python version. Need to use 3.x
           # here, because `python` on osx points to Python 2.7.
@@ -143,7 +143,7 @@ jobs:
 
       - name: Set up QEMU to emulate aarch64
         if: matrix.arch == 'aarch64'
-        uses: docker/setup-qemu-action@v3
+        uses: docker/setup-qemu-action@29109295f81e9208d7d86ff1c6c12d2833863392 # v3.6.0
         with:
           platforms: arm64
 
@@ -165,7 +165,7 @@ jobs:
           CARGO_NET_GIT_FETCH_WITH_CLI: true
           CIBW_ENVIRONMENT_PASS_LINUX: CARGO_NET_GIT_FETCH_WITH_CLI
 
-      - uses: actions/upload-artifact@v4
+      - uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 # v4.6.1
         with:
           name: Wheel-${{ matrix.os }}-${{ matrix.arch }}
           path: ./wheelhouse/*.whl
@@ -176,8 +176,8 @@ jobs:
     if: ${{ !startsWith(github.ref, 'refs/pull/') }}
 
     steps:
-      - uses: actions/checkout@v4
-      - uses: actions/setup-python@v5
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+      - uses: actions/setup-python@42375524e23c412d93fb67b49958b491fce71c38 # v5.4.0
         with:
           python-version: '3.10'
 
@@ -186,7 +186,7 @@ jobs:
       - name: Build sdist
         run: python -m build --sdist
 
-      - uses: actions/upload-artifact@v4
+      - uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 # v4.6.1
         with:
           name: Sdist
           path: dist/*.tar.gz
@@ -203,7 +203,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Download all workflow run artifacts
-        uses: actions/download-artifact@v4
+        uses: actions/download-artifact@b14cf4c92620c250e1c074ab0a5800e37df86765 # v4.2.0
       - name: Build a tarball for the debs
         # We need to merge all the debs uploads into one folder, then compress
         # that.
@@ -213,7 +213,7 @@ jobs:
           tar -cvJf debs.tar.xz debs
       - name: Attach to release
         # Pinned to work around https://github.com/softprops/action-gh-release/issues/445
-        uses: softprops/action-gh-release@v0.1.15
+        uses: softprops/action-gh-release@de2c0eb89ae2a093876385947365aca7b0e5f844 # v0.1.15
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         with:

.github/workflows/tests.yml

@@ -23,7 +23,7 @@ jobs:
       linting: ${{ !startsWith(github.ref, 'refs/pull/') || steps.filter.outputs.linting }}
       linting_readme: ${{ !startsWith(github.ref, 'refs/pull/') || steps.filter.outputs.linting_readme }}
     steps:
-    - uses: dorny/paths-filter@v3
+    - uses: dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36 # v3.0.2
       id: filter
       # We only check on PRs
       if: startsWith(github.ref, 'refs/pull/')
@@ -83,14 +83,14 @@ jobs:
     if: ${{ needs.changes.outputs.linting == 'true' }}
 
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
       - name: Install Rust
-        uses: dtolnay/rust-toolchain@1.66.0
-      - uses: Swatinem/rust-cache@v2
-      - uses: matrix-org/setup-python-poetry@v1
+        uses: dtolnay/rust-toolchain@e05ebb0e73db581a4877c6ce762e29fe1e0b5073 # 1.66.0
+      - uses: Swatinem/rust-cache@9d47c6ad4b02e050fd481d890b2ea34778fd09d6 # v2.7.8
+      - uses: matrix-org/setup-python-poetry@5bbf6603c5c930615ec8a29f1b5d7d258d905aa4 # v2.0.0
         with:
           python-version: "3.x"
-          poetry-version: "1.3.2"
+          poetry-version: "2.1.1"
           extras: "all"
       - run: poetry run scripts-dev/generate_sample_config.sh --check
       - run: poetry run scripts-dev/config-lint.sh
@@ -101,8 +101,8 @@ jobs:
     if: ${{ needs.changes.outputs.linting == 'true' }}
 
     steps:
-      - uses: actions/checkout@v4
-      - uses: actions/setup-python@v5
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+      - uses: actions/setup-python@42375524e23c412d93fb67b49958b491fce71c38 # v5.4.0
         with:
           python-version: "3.x"
       - run: "pip install 'click==8.1.1' 'GitPython>=3.1.20'"
@@ -111,8 +111,8 @@ jobs:
   check-lockfile:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v4
-      - uses: actions/setup-python@v5
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+      - uses: actions/setup-python@42375524e23c412d93fb67b49958b491fce71c38 # v5.4.0
         with:
           python-version: "3.x"
       - run: .ci/scripts/check_lockfile.py
@@ -124,11 +124,12 @@ jobs:
 
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v4
+        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
 
       - name: Setup Poetry
-        uses: matrix-org/setup-python-poetry@v1
+        uses: matrix-org/setup-python-poetry@5bbf6603c5c930615ec8a29f1b5d7d258d905aa4 # v2.0.0
         with:
+          poetry-version: "2.1.1"
           install-project: "false"
 
       - name: Run ruff check
@@ -145,14 +146,14 @@ jobs:
 
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v4
+        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
 
       - name: Install Rust
-        uses: dtolnay/rust-toolchain@1.66.0
-      - uses: Swatinem/rust-cache@v2
+        uses: dtolnay/rust-toolchain@e05ebb0e73db581a4877c6ce762e29fe1e0b5073 # 1.66.0
+      - uses: Swatinem/rust-cache@9d47c6ad4b02e050fd481d890b2ea34778fd09d6 # v2.7.8
 
       - name: Setup Poetry
-        uses: matrix-org/setup-python-poetry@v1
+        uses: matrix-org/setup-python-poetry@5bbf6603c5c930615ec8a29f1b5d7d258d905aa4 # v2.0.0
         with:
           # We want to make use of type hints in optional dependencies too.
           extras: all
@@ -161,11 +162,12 @@ jobs:
           # https://github.com/matrix-org/synapse/pull/15376#issuecomment-1498983775
           # To make CI green, err towards caution and install the project.
           install-project: "true"
+          poetry-version: "2.1.1"
 
       # Cribbed from
       # https://github.com/AustinScola/mypy-cache-github-action/blob/85ea4f2972abed39b33bd02c36e341b28ca59213/src/restore.ts#L10-L17
       - name: Restore/persist mypy's cache
-        uses: actions/cache@v4
+        uses: actions/cache@d4323d4df104b026a6aa633fdb11d772146be0bf # v4.2.2
         with:
           path: |
             .mypy_cache
@@ -178,7 +180,7 @@ jobs:
   lint-crlf:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
       - name: Check line endings
         run: scripts-dev/check_line_terminators.sh
 
@@ -186,11 +188,11 @@ jobs:
     if: ${{ (github.base_ref == 'develop'  || contains(github.base_ref, 'release-')) && github.actor != 'dependabot[bot]' }}
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
         with:
           ref: ${{ github.event.pull_request.head.sha }}
           fetch-depth: 0
-      - uses: actions/setup-python@v5
+      - uses: actions/setup-python@42375524e23c412d93fb67b49958b491fce71c38 # v5.4.0
         with:
           python-version: "3.x"
       - run: "pip install 'towncrier>=18.6.0rc1'"
@@ -204,15 +206,15 @@ jobs:
     if: ${{ needs.changes.outputs.linting == 'true' }}
 
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
         with:
           ref: ${{ github.event.pull_request.head.sha }}
       - name: Install Rust
-        uses: dtolnay/rust-toolchain@1.66.0
-      - uses: Swatinem/rust-cache@v2
-      - uses: matrix-org/setup-python-poetry@v1
+        uses: dtolnay/rust-toolchain@e05ebb0e73db581a4877c6ce762e29fe1e0b5073 # 1.66.0
+      - uses: Swatinem/rust-cache@9d47c6ad4b02e050fd481d890b2ea34778fd09d6 # v2.7.8
+      - uses: matrix-org/setup-python-poetry@5bbf6603c5c930615ec8a29f1b5d7d258d905aa4 # v2.0.0
         with:
-          poetry-version: "1.3.2"
+          poetry-version: "2.1.1"
           extras: "all"
       - run: poetry run scripts-dev/check_pydantic_models.py
 
@@ -222,13 +224,13 @@ jobs:
     if: ${{ needs.changes.outputs.rust == 'true' }}
 
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
 
       - name: Install Rust
-        uses: dtolnay/rust-toolchain@1.66.0
+        uses: dtolnay/rust-toolchain@e05ebb0e73db581a4877c6ce762e29fe1e0b5073 # 1.66.0
         with:
             components: clippy
-      - uses: Swatinem/rust-cache@v2
+      - uses: Swatinem/rust-cache@9d47c6ad4b02e050fd481d890b2ea34778fd09d6 # v2.7.8
 
       - run: cargo clippy -- -D warnings
 
@@ -240,14 +242,14 @@ jobs:
     if: ${{ needs.changes.outputs.rust == 'true' }}
 
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
 
       - name: Install Rust
-        uses: dtolnay/rust-toolchain@master
+        uses: dtolnay/rust-toolchain@56f84321dbccf38fb67ce29ab63e4754056677e0 # master (rust 1.85.1)
         with:
             toolchain: nightly-2022-12-01
             components: clippy
-      - uses: Swatinem/rust-cache@v2
+      - uses: Swatinem/rust-cache@9d47c6ad4b02e050fd481d890b2ea34778fd09d6 # v2.7.8
 
       - run: cargo clippy --all-features -- -D warnings
 
@@ -257,15 +259,15 @@ jobs:
     if: ${{ needs.changes.outputs.rust == 'true' }}
 
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
 
       - name: Install Rust
-        uses: dtolnay/rust-toolchain@master
+        uses: dtolnay/rust-toolchain@56f84321dbccf38fb67ce29ab63e4754056677e0 # master (rust 1.85.1)
         with:
           # We use nightly so that it correctly groups together imports
           toolchain: nightly-2022-12-01
           components: rustfmt
-      - uses: Swatinem/rust-cache@v2
+      - uses: Swatinem/rust-cache@9d47c6ad4b02e050fd481d890b2ea34778fd09d6 # v2.7.8
 
       - run: cargo fmt --check
 
@@ -276,8 +278,8 @@ jobs:
     needs: changes
     if: ${{ needs.changes.outputs.linting_readme == 'true' }}
     steps:
-      - uses: actions/checkout@v4
-      - uses: actions/setup-python@v5
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+      - uses: actions/setup-python@42375524e23c412d93fb67b49958b491fce71c38 # v5.4.0
         with:
           python-version: "3.x"
       - run: "pip install rstcheck"
@@ -301,7 +303,7 @@ jobs:
       - lint-readme
     runs-on: ubuntu-latest
     steps:
-      - uses: matrix-org/done-action@v3
+      - uses: matrix-org/done-action@3409aa904e8a2aaf2220f09bc954d3d0b0a2ee67 # v3
         with:
           needs: ${{ toJSON(needs) }}
 
@@ -324,8 +326,8 @@ jobs:
     needs: linting-done
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v4
-      - uses: actions/setup-python@v5
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+      - uses: actions/setup-python@42375524e23c412d93fb67b49958b491fce71c38 # v5.4.0
         with:
           python-version: "3.x"
       - id: get-matrix
@@ -345,7 +347,7 @@ jobs:
         job:  ${{ fromJson(needs.calculate-test-jobs.outputs.trial_test_matrix) }}
 
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
       - run: sudo apt-get -qq install xmlsec1
       - name: Set up PostgreSQL ${{ matrix.job.postgres-version }}
         if: ${{ matrix.job.postgres-version }}
@@ -360,13 +362,13 @@ jobs:
             postgres:${{ matrix.job.postgres-version }}
 
       - name: Install Rust
-        uses: dtolnay/rust-toolchain@1.66.0
-      - uses: Swatinem/rust-cache@v2
+        uses: dtolnay/rust-toolchain@e05ebb0e73db581a4877c6ce762e29fe1e0b5073 # 1.66.0
+      - uses: Swatinem/rust-cache@9d47c6ad4b02e050fd481d890b2ea34778fd09d6 # v2.7.8
 
-      - uses: matrix-org/setup-python-poetry@v1
+      - uses: matrix-org/setup-python-poetry@5bbf6603c5c930615ec8a29f1b5d7d258d905aa4 # v2.0.0
         with:
           python-version: ${{ matrix.job.python-version }}
-          poetry-version: "1.3.2"
+          poetry-version: "2.1.1"
           extras: ${{ matrix.job.extras }}
       - name: Await PostgreSQL
         if: ${{ matrix.job.postgres-version }}
@@ -399,11 +401,11 @@ jobs:
       - changes
     runs-on: ubuntu-22.04
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
 
       - name: Install Rust
-        uses: dtolnay/rust-toolchain@1.66.0
-      - uses: Swatinem/rust-cache@v2
+        uses: dtolnay/rust-toolchain@e05ebb0e73db581a4877c6ce762e29fe1e0b5073 # 1.66.0
+      - uses: Swatinem/rust-cache@9d47c6ad4b02e050fd481d890b2ea34778fd09d6 # v2.7.8
 
       # There aren't wheels for some of the older deps, so we need to install
       # their build dependencies
@@ -412,7 +414,7 @@ jobs:
           sudo apt-get -qq install build-essential libffi-dev python3-dev \
             libxml2-dev libxslt-dev xmlsec1 zlib1g-dev libjpeg-dev libwebp-dev
 
-      - uses: actions/setup-python@v5
+      - uses: actions/setup-python@42375524e23c412d93fb67b49958b491fce71c38 # v5.4.0
         with:
           python-version: '3.9'
 
@@ -462,13 +464,13 @@ jobs:
         extras: ["all"]
 
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
       # Install libs necessary for PyPy to build binary wheels for dependencies
       - run: sudo apt-get -qq install xmlsec1 libxml2-dev libxslt-dev
-      - uses: matrix-org/setup-python-poetry@v1
+      - uses: matrix-org/setup-python-poetry@5bbf6603c5c930615ec8a29f1b5d7d258d905aa4 # v2.0.0
         with:
           python-version: ${{ matrix.python-version }}
-          poetry-version: "1.3.2"
+          poetry-version: "2.1.1"
           extras: ${{ matrix.extras }}
       - run: poetry run trial --jobs=2 tests
       - name: Dump logs
@@ -512,13 +514,13 @@ jobs:
         job: ${{ fromJson(needs.calculate-test-jobs.outputs.sytest_test_matrix) }}
 
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
       - name: Prepare test blacklist
         run: cat sytest-blacklist .ci/worker-blacklist > synapse-blacklist-with-workers
 
       - name: Install Rust
-        uses: dtolnay/rust-toolchain@1.66.0
-      - uses: Swatinem/rust-cache@v2
+        uses: dtolnay/rust-toolchain@e05ebb0e73db581a4877c6ce762e29fe1e0b5073 # 1.66.0
+      - uses: Swatinem/rust-cache@9d47c6ad4b02e050fd481d890b2ea34778fd09d6 # v2.7.8
 
       - name: Run SyTest
         run: /bootstrap.sh synapse
@@ -527,7 +529,7 @@ jobs:
         if: ${{ always() }}
         run: /sytest/scripts/tap_to_gha.pl /logs/results.tap
       - name: Upload SyTest logs
-        uses: actions/upload-artifact@v4
+        uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 # v4.6.1
         if: ${{ always() }}
         with:
           name: Sytest Logs - ${{ job.status }} - (${{ join(matrix.job.*, ', ') }})
@@ -557,11 +559,11 @@ jobs:
           --health-retries 5
 
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
       - run: sudo apt-get -qq install xmlsec1 postgresql-client
-      - uses: matrix-org/setup-python-poetry@v1
+      - uses: matrix-org/setup-python-poetry@5bbf6603c5c930615ec8a29f1b5d7d258d905aa4 # v2.0.0
         with:
-          poetry-version: "1.3.2"
+          poetry-version: "2.1.1"
           extras: "postgres"
       - run: .ci/scripts/test_export_data_command.sh
         env:
@@ -601,7 +603,7 @@ jobs:
           --health-retries 5
 
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
       - name: Add PostgreSQL apt repository
         # We need a version of pg_dump that can handle the version of
         # PostgreSQL being tested against. The Ubuntu package repository lags
@@ -612,10 +614,10 @@ jobs:
           wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | sudo apt-key add -
           sudo apt-get update
       - run: sudo apt-get -qq install xmlsec1 postgresql-client
-      - uses: matrix-org/setup-python-poetry@v1
+      - uses: matrix-org/setup-python-poetry@5bbf6603c5c930615ec8a29f1b5d7d258d905aa4 # v2.0.0
         with:
           python-version: ${{ matrix.python-version }}
-          poetry-version: "1.3.2"
+          poetry-version: "2.1.1"
           extras: "postgres"
       - run: .ci/scripts/test_synapse_port_db.sh
         id: run_tester_script
@@ -625,7 +627,7 @@ jobs:
           PGPASSWORD: postgres
           PGDATABASE: postgres
       - name: "Upload schema differences"
-        uses: actions/upload-artifact@v4
+        uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 # v4.6.1
         if: ${{ failure() && !cancelled() && steps.run_tester_script.outcome == 'failure' }}
         with:
           name: Schema dumps
@@ -655,19 +657,19 @@ jobs:
             database: Postgres
 
     steps:
-      - name: Run actions/checkout@v4 for synapse
-        uses: actions/checkout@v4
+      - name: Checkout synapse codebase
+        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
         with:
           path: synapse
 
       - name: Install Rust
-        uses: dtolnay/rust-toolchain@1.66.0
-      - uses: Swatinem/rust-cache@v2
+        uses: dtolnay/rust-toolchain@e05ebb0e73db581a4877c6ce762e29fe1e0b5073 # 1.66.0
+      - uses: Swatinem/rust-cache@9d47c6ad4b02e050fd481d890b2ea34778fd09d6 # v2.7.8
 
       - name: Prepare Complement's Prerequisites
         run: synapse/.ci/scripts/setup_complement_prerequisites.sh
 
-      - uses: actions/setup-go@v5
+      - uses: actions/setup-go@0aaccfd150d50ccaeb58ebd88d36e91967a5f35b # v5.4.0
         with:
           cache-dependency-path: complement/go.sum
           go-version-file: complement/go.mod
@@ -690,11 +692,11 @@ jobs:
       - changes
 
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
 
       - name: Install Rust
-        uses: dtolnay/rust-toolchain@1.66.0
-      - uses: Swatinem/rust-cache@v2
+        uses: dtolnay/rust-toolchain@e05ebb0e73db581a4877c6ce762e29fe1e0b5073 # 1.66.0
+      - uses: Swatinem/rust-cache@9d47c6ad4b02e050fd481d890b2ea34778fd09d6 # v2.7.8
 
       - run: cargo test
 
@@ -708,13 +710,13 @@ jobs:
       - changes
 
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
 
       - name: Install Rust
-        uses: dtolnay/rust-toolchain@master
+        uses: dtolnay/rust-toolchain@56f84321dbccf38fb67ce29ab63e4754056677e0 # master (rust 1.85.1)
         with:
             toolchain: nightly-2022-12-01
-      - uses: Swatinem/rust-cache@v2
+      - uses: Swatinem/rust-cache@9d47c6ad4b02e050fd481d890b2ea34778fd09d6 # v2.7.8
 
       - run: cargo bench --no-run
 
@@ -733,7 +735,7 @@ jobs:
       - linting-done
     runs-on: ubuntu-latest
     steps:
-      - uses: matrix-org/done-action@v3
+      - uses: matrix-org/done-action@3409aa904e8a2aaf2220f09bc954d3d0b0a2ee67 # v3
         with:
           needs: ${{ toJSON(needs) }}
 

.github/workflows/triage-incoming.yml

@@ -6,7 +6,7 @@ on:
 
 jobs:
   triage:
-    uses: matrix-org/backend-meta/.github/workflows/triage-incoming.yml@v2
+    uses: matrix-org/backend-meta/.github/workflows/triage-incoming.yml@18beaf3c8e536108bd04d18e6c3dc40ba3931e28 # v2.0.3
     with:
       project_id: 'PVT_kwDOAIB0Bs4AFDdZ'
       content_id: ${{ github.event.issue.node_id }}

.github/workflows/triage_labelled.yml

@@ -11,7 +11,7 @@ jobs:
     if: >
       contains(github.event.issue.labels.*.name, 'X-Needs-Info')
     steps:
-      - uses: actions/add-to-project@main
+      - uses: actions/add-to-project@f5473ace9aeee8b97717b281e26980aa5097023f # main (v1.0.2 + 10 commits)
         id: add_project
         with:
           project-url: "https://github.com/orgs/matrix-org/projects/67"

.github/workflows/twisted_trunk.yml

@@ -40,16 +40,17 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
 
       - name: Install Rust
-        uses: dtolnay/rust-toolchain@stable
-      - uses: Swatinem/rust-cache@v2
+        uses: dtolnay/rust-toolchain@fcf085fcb4b4b8f63f96906cd713eb52181b5ea4 # stable (rust 1.85.1)
+      - uses: Swatinem/rust-cache@9d47c6ad4b02e050fd481d890b2ea34778fd09d6 # v2.7.8
 
-      - uses: matrix-org/setup-python-poetry@v1
+      - uses: matrix-org/setup-python-poetry@5bbf6603c5c930615ec8a29f1b5d7d258d905aa4 # v2.0.0
         with:
           python-version: "3.x"
           extras: "all"
+          poetry-version: "2.1.1"
       - run: |
           poetry remove twisted
           poetry add --extras tls git+https://github.com/twisted/twisted.git#${{ inputs.twisted_ref || 'trunk' }}
@@ -64,17 +65,18 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
       - run: sudo apt-get -qq install xmlsec1
 
       - name: Install Rust
-        uses: dtolnay/rust-toolchain@stable
-      - uses: Swatinem/rust-cache@v2
+        uses: dtolnay/rust-toolchain@fcf085fcb4b4b8f63f96906cd713eb52181b5ea4 # stable (rust 1.85.1)
+      - uses: Swatinem/rust-cache@9d47c6ad4b02e050fd481d890b2ea34778fd09d6 # v2.7.8
 
-      - uses: matrix-org/setup-python-poetry@v1
+      - uses: matrix-org/setup-python-poetry@5bbf6603c5c930615ec8a29f1b5d7d258d905aa4 # v2.0.0
         with:
           python-version: "3.x"
           extras: "all test"
+          poetry-version: "2.1.1"
       - run: |
           poetry remove twisted
           poetry add --extras tls git+https://github.com/twisted/twisted.git#trunk
@@ -108,11 +110,11 @@ jobs:
         - ${{ github.workspace }}:/src
 
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
 
       - name: Install Rust
-        uses: dtolnay/rust-toolchain@stable
-      - uses: Swatinem/rust-cache@v2
+        uses: dtolnay/rust-toolchain@fcf085fcb4b4b8f63f96906cd713eb52181b5ea4 # stable (rust 1.85.1)
+      - uses: Swatinem/rust-cache@9d47c6ad4b02e050fd481d890b2ea34778fd09d6 # v2.7.8
 
       - name: Patch dependencies
         # Note: The poetry commands want to create a virtualenv in /src/.venv/,
@@ -136,7 +138,7 @@ jobs:
         if: ${{ always() }}
         run: /sytest/scripts/tap_to_gha.pl /logs/results.tap
       - name: Upload SyTest logs
-        uses: actions/upload-artifact@v4
+        uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 # v4.6.1
         if: ${{ always() }}
         with:
           name: Sytest Logs - ${{ job.status }} - (${{ join(matrix.*, ', ') }})
@@ -164,14 +166,14 @@ jobs:
 
     steps:
       - name: Run actions/checkout@v4 for synapse
-        uses: actions/checkout@v4
+        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
         with:
           path: synapse
 
       - name: Prepare Complement's Prerequisites
         run: synapse/.ci/scripts/setup_complement_prerequisites.sh
 
-      - uses: actions/setup-go@v5
+      - uses: actions/setup-go@0aaccfd150d50ccaeb58ebd88d36e91967a5f35b # v5.4.0
         with:
           cache-dependency-path: complement/go.sum
           go-version-file: complement/go.mod
@@ -181,11 +183,11 @@ jobs:
         run: |
           set -x
           DEBIAN_FRONTEND=noninteractive sudo apt-get install -yqq python3 pipx
-          pipx install poetry==1.3.2
+          pipx install poetry==2.1.1
 
           poetry remove -n twisted
           poetry add -n --extras tls git+https://github.com/twisted/twisted.git#trunk
-          poetry lock --no-update
+          poetry lock
         working-directory: synapse
 
       - run: |
@@ -206,7 +208,7 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
       - uses: JasonEtco/create-an-issue@1b14a70e4d8dc185e5cc76d3bec9eab20257b2c5 # v2.9.2
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

changelog.d/18068.misc

@@ -0,0 +1 @@
+Add a column `participant` to `room_memberships` table.

changelog.d/18074.bugfix

@@ -0,0 +1 @@
+Add index to sliding sync membership snapshot table, to fix a performance issue.

changelog.d/18251.misc

@@ -0,0 +1 @@
+Update Poetry to 2.1.1, including updating the lock file version.

changelog.d/18254.feature

@@ -0,0 +1 @@
+Add background job to clear unreferenced state groups.

changelog.d/18255.misc

@@ -0,0 +1 @@
+Pin GitHub Actions dependencies by commit hash.

debian/build_virtualenv

@@ -35,7 +35,7 @@ TEMP_VENV="$(mktemp -d)"
 python3 -m venv "$TEMP_VENV"
 source "$TEMP_VENV/bin/activate"
 pip install -U pip
-pip install poetry==1.3.2
+pip install poetry==2.1.1 poetry-plugin-export==1.9.0
 poetry export \
     --extras all \
     --extras test \

debian/changelog

@@ -1,3 +1,9 @@
+matrix-synapse-py3 (1.127.0~rc1+nmu1) UNRELEASED; urgency=medium
+
+  * Update Poetry to 2.1.1.
+
+ -- Synapse Packaging team <packages@matrix.org>  Wed, 19 Mar 2025 17:38:49 +0000
+
 matrix-synapse-py3 (1.127.0~rc1) stable; urgency=medium
 
   * New Synapse release 1.127.0rc1.

docker/Dockerfile

@@ -22,7 +22,7 @@
 
 ARG DEBIAN_VERSION=bookworm
 ARG PYTHON_VERSION=3.12
-ARG POETRY_VERSION=1.8.3
+ARG POETRY_VERSION=2.1.1
 
 ###
 ### Stage 0: generate requirements.txt
@@ -56,7 +56,7 @@ ENV UV_LINK_MODE=copy
 ARG POETRY_VERSION
 RUN --mount=type=cache,target=/root/.cache/uv \
   if [ -z "$TEST_ONLY_IGNORE_POETRY_LOCKFILE" ]; then \
-    uvx --with poetry-plugin-export==1.8.0 \
+    uvx --with poetry-plugin-export==1.9.0 \
         poetry@${POETRY_VERSION} export --extras all -o /synapse/requirements.txt ${TEST_ONLY_SKIP_DEP_HASH_VERIFICATION:+--without-hashes}; \
   else \
     touch /synapse/requirements.txt; \

docs/development/database_schema.md

@@ -162,7 +162,7 @@ by a unique name, the current status (stored in JSON), and some dependency infor
 * Whether the update requires a previous update to be complete.
 * A rough ordering for which to complete updates.
 
-A new background updates needs to be added to the `background_updates` table:
+A new background update needs to be added to the `background_updates` table:
 
 ```sql
 INSERT INTO background_updates (ordering, update_name, depends_on, progress_json) VALUES

docs/development/dependencies.md

@@ -187,7 +187,7 @@ useful.
 ## ...add a new dependency?
 
 Either:
-- manually update `pyproject.toml`; then `poetry lock --no-update`; or else
+- manually update `pyproject.toml`; then `poetry lock`; or else
 - `poetry add packagename`. See `poetry add --help`; note the `--dev`,
   `--extras` and `--optional` flags in particular.
 
@@ -202,12 +202,12 @@ poetry remove packagename
 ```
 
 ought to do the trick. Alternatively, manually update `pyproject.toml` and
-`poetry lock --no-update`. Include the updated `pyproject.toml` and `poetry.lock`
+`poetry lock`. Include the updated `pyproject.toml` and `poetry.lock`
 files in your commit.
 
 ## ...update the version range for an existing dependency?
 
-Best done by manually editing `pyproject.toml`, then `poetry lock --no-update`.
+Best done by manually editing `pyproject.toml`, then `poetry lock`.
 Include the updated `pyproject.toml` and `poetry.lock` in your commit.
 
 ## ...update a dependency in the locked environment?
@@ -233,7 +233,7 @@ poetry add packagename==1.2.3
 
 # Get poetry to recompute the content-hash of pyproject.toml without changing
 # the locked package versions.
-poetry lock --no-update
+poetry lock
 ```
 
 Either way, include the updated `poetry.lock` file in your commit.

synapse/_scripts/synapse_port_db.py

@@ -128,6 +128,7 @@ BOOLEAN_COLUMNS = {
     "pushers": ["enabled"],
     "redactions": ["have_censored"],
     "remote_media_cache": ["authenticated"],
+    "room_memberships": ["participant"],
     "room_stats_state": ["is_federatable"],
     "rooms": ["is_public", "has_auth_chain_index"],
     "sliding_sync_joined_rooms": ["is_encrypted"],
@@ -191,6 +192,11 @@ APPEND_ONLY_TABLES = [
 
 
 IGNORED_TABLES = {
+    # Porting the auto generated sequence in this table is non-trivial.
+    # None of the entries in this list are mandatory for Synapse to keep working.
+    # If state group disk space is an issue after the port, the
+    # `mark_unreferenced_state_groups_for_deletion_bg_update` background task can be run again.
+    "state_groups_pending_deletion",
     # We don't port these tables, as they're a faff and we can regenerate
     # them anyway.
     "user_directory",
@@ -216,6 +222,15 @@ IGNORED_TABLES = {
 }
 
 
+# These background updates will not be applied upon creation of the postgres database.
+IGNORED_BACKGROUND_UPDATES = {
+    # Reapplying this background update to the postgres database is unnecessary after
+    # already having waited for the SQLite database to complete all running background
+    # updates.
+    "mark_unreferenced_state_groups_for_deletion_bg_update",
+}
+
+
 # Error returned by the run function. Used at the top-level part of the script to
 # handle errors and return codes.
 end_error: Optional[str] = None
@@ -687,6 +702,20 @@ class Porter:
         # 0 means off. 1 means full. 2 means incremental.
         return autovacuum_setting != 0
 
+    async def remove_ignored_background_updates_from_database(self) -> None:
+        def _remove_delete_unreferenced_state_groups_bg_updates(
+            txn: LoggingTransaction,
+        ) -> None:
+            txn.execute(
+                "DELETE FROM background_updates WHERE update_name = ANY(?)",
+                (list(IGNORED_BACKGROUND_UPDATES),),
+            )
+
+        await self.postgres_store.db_pool.runInteraction(
+            "remove_delete_unreferenced_state_groups_bg_updates",
+            _remove_delete_unreferenced_state_groups_bg_updates,
+        )
+
     async def run(self) -> None:
         """Ports the SQLite database to a PostgreSQL database.
 
@@ -732,6 +761,8 @@ class Porter:
                 self.hs_config.database.get_single_database()
             )
 
+            await self.remove_ignored_background_updates_from_database()
+
             await self.run_background_updates_on_postgres()
 
             self.progress.set_state("Creating port tables")

synapse/handlers/message.py

@@ -1462,6 +1462,12 @@ class EventCreationHandler:
                     )
                     return prev_event
 
+            if not event.is_state() and event.type in [
+                EventTypes.Message,
+                EventTypes.Encrypted,
+            ]:
+                await self.store.set_room_participation(event.user_id, event.room_id)
+
             if event.internal_metadata.is_out_of_band_membership():
                 # the only sort of out-of-band-membership events we expect to see here are
                 # invite rejections and rescinded knocks that we have generated ourselves.

synapse/storage/controllers/persist_events.py

@@ -509,9 +509,13 @@ class EventsPersistenceStorageController:
             )
         )
 
-        await self.persist_events_store.update_current_state(
-            room_id, delta, sliding_sync_table_changes
-        )
+        logger.error("Devon: persist update_current_state: %s %s %s", state, delta, sliding_sync_table_changes)
+
+        # FIXME: Devon removed this to stop deleting the leave event from the sliding sync
+        # tables
+        # await self.persist_events_store.update_current_state(
+        #     room_id, delta, sliding_sync_table_changes
+        # )
 
     async def _calculate_current_state(self, room_id: str) -> StateMap[str]:
         """Calculate the current state of a room, based on the forward extremities
@@ -638,6 +642,21 @@ class EventsPersistenceStorageController:
                     room_id, [e for e, _ in chunk]
                 )
 
+            # FIXME: Temporary hack to add remote invite to state delta
+            if not new_forward_extremities:
+                state_map: MutableStateMap[str] = {}
+                for (a, _) in chunk:
+                    if a.type == EventTypes.Member and a.membership == Membership.LEAVE:
+                        state_key = a.get_state_key()
+                        if state_key:
+                            logger.error("Devon: injecting hack into state map")
+                            state_map[(a.type, state_key)] = a.event_id
+            
+                if state_map:
+                    state_delta_for_room = DeltaState([], state_map, False)
+
+            logger.error("Devon: state delta for room: %s", state_delta_for_room)
+            
             # Stop the state groups from being deleted while we're persisting
             # them.
             async with self._state_deletion_store.persisting_state_group_references(

synapse/storage/controllers/purge_events.py

@@ -21,11 +21,19 @@
 
 import itertools
 import logging
-from typing import TYPE_CHECKING, Collection, Mapping, Set
+from typing import (
+    TYPE_CHECKING,
+    Collection,
+    Mapping,
+    Optional,
+    Set,
+)
 
 from synapse.logging.context import nested_logging_context
 from synapse.metrics.background_process_metrics import wrap_as_background_process
+from synapse.storage.database import LoggingTransaction
 from synapse.storage.databases import Databases
+from synapse.types.storage import _BackgroundUpdates
 
 if TYPE_CHECKING:
     from synapse.server import HomeServer
@@ -44,6 +52,11 @@ class PurgeEventsStorageController:
                 self._delete_state_groups_loop, 60 * 1000
             )
 
+        self.stores.state.db_pool.updates.register_background_update_handler(
+            _BackgroundUpdates.MARK_UNREFERENCED_STATE_GROUPS_FOR_DELETION_BG_UPDATE,
+            self._background_delete_unrefereneced_state_groups,
+        )
+
     async def purge_room(self, room_id: str) -> None:
         """Deletes all record of a room"""
 
@@ -81,7 +94,8 @@ class PurgeEventsStorageController:
             )
 
     async def _find_unreferenced_groups(
-        self, state_groups: Collection[int]
+        self,
+        state_groups: Collection[int],
     ) -> Set[int]:
         """Used when purging history to figure out which state groups can be
         deleted.
@@ -203,3 +217,232 @@ class PurgeEventsStorageController:
             room_id,
             groups_to_sequences,
         )
+
+    async def _background_delete_unrefereneced_state_groups(
+        self, progress: dict, batch_size: int
+    ) -> int:
+        """This background update will slowly delete any unreferenced state groups"""
+
+        last_checked_state_group = progress.get("last_checked_state_group")
+
+        if last_checked_state_group is None:
+            # This is the first run.
+            last_checked_state_group = (
+                await self.stores.state.db_pool.simple_select_one_onecol(
+                    table="state_groups",
+                    keyvalues={},
+                    retcol="MAX(id)",
+                    allow_none=True,
+                    desc="get_max_state_group",
+                )
+            )
+            if last_checked_state_group is None:
+                # There are no state groups so the background process is finished.
+                await self.stores.state.db_pool.updates._end_background_update(
+                    _BackgroundUpdates.MARK_UNREFERENCED_STATE_GROUPS_FOR_DELETION_BG_UPDATE
+                )
+                return batch_size
+            last_checked_state_group += 1
+
+        (
+            last_checked_state_group,
+            final_batch,
+        ) = await self._delete_unreferenced_state_groups_batch(
+            last_checked_state_group,
+            batch_size,
+        )
+
+        if not final_batch:
+            # There are more state groups to check.
+            progress = {
+                "last_checked_state_group": last_checked_state_group,
+            }
+            await self.stores.state.db_pool.updates._background_update_progress(
+                _BackgroundUpdates.MARK_UNREFERENCED_STATE_GROUPS_FOR_DELETION_BG_UPDATE,
+                progress,
+            )
+        else:
+            # This background process is finished.
+            await self.stores.state.db_pool.updates._end_background_update(
+                _BackgroundUpdates.MARK_UNREFERENCED_STATE_GROUPS_FOR_DELETION_BG_UPDATE
+            )
+
+        return batch_size
+
+    async def _delete_unreferenced_state_groups_batch(
+        self,
+        last_checked_state_group: int,
+        batch_size: int,
+    ) -> tuple[int, bool]:
+        """Looks for unreferenced state groups starting from the last state group
+        checked and marks them for deletion.
+
+        Args:
+            last_checked_state_group: The last state group that was checked.
+            batch_size: How many state groups to process in this iteration.
+
+        Returns:
+            (last_checked_state_group, final_batch)
+        """
+
+        # Find all state groups that can be deleted if any of the original set are deleted.
+        (
+            to_delete,
+            last_checked_state_group,
+            final_batch,
+        ) = await self._find_unreferenced_groups_for_background_deletion(
+            last_checked_state_group, batch_size
+        )
+
+        if len(to_delete) == 0:
+            return last_checked_state_group, final_batch
+
+        await self.stores.state_deletion.mark_state_groups_as_pending_deletion(
+            to_delete
+        )
+
+        return last_checked_state_group, final_batch
+
+    async def _find_unreferenced_groups_for_background_deletion(
+        self,
+        last_checked_state_group: int,
+        batch_size: int,
+    ) -> tuple[Set[int], int, bool]:
+        """Used when deleting unreferenced state groups in the background to figure out
+        which state groups can be deleted.
+        To avoid increased DB usage due to de-deltaing state groups, this returns only
+        state groups which are free standing (ie. no shared edges with referenced groups) or
+        state groups which do not share edges which result in a future referenced group.
+
+        The following scenarios outline the possibilities based on state group data in
+        the DB.
+
+        ie. Free standing -> state groups 1-N would be returned:
+            SG_1
+            |
+            ...
+            |
+            SG_N
+
+        ie. Previous reference -> state groups 2-N would be returned:
+            SG_1 <- referenced by event
+            |
+            SG_2
+            |
+            ...
+            |
+            SG_N
+
+        ie. Future reference -> none of the following state groups would be returned:
+            SG_1
+            |
+            SG_2
+            |
+            ...
+            |
+            SG_N <- referenced by event
+
+        Args:
+            last_checked_state_group: The last state group that was checked.
+            batch_size: How many state groups to process in this iteration.
+
+        Returns:
+            (to_delete, last_checked_state_group, final_batch)
+        """
+
+        # If a state group's next edge is not pending deletion then we don't delete the state group.
+        # If there is no next edge or the next edges are all marked for deletion, then delete
+        # the state group.
+        # This holds since we walk backwards from the latest state groups, ensuring that
+        # we've already checked newer state groups for event references along the way.
+        def get_next_state_groups_marked_for_deletion_txn(
+            txn: LoggingTransaction,
+        ) -> tuple[dict[int, bool], dict[int, int]]:
+            state_group_sql = """
+                SELECT s.id, e.state_group, d.state_group
+                FROM (
+                    SELECT id FROM state_groups
+                    WHERE id < ? ORDER BY id DESC LIMIT ?
+                ) as s
+                LEFT JOIN state_group_edges AS e ON (s.id = e.prev_state_group)
+                LEFT JOIN state_groups_pending_deletion AS d ON (e.state_group = d.state_group)
+            """
+            txn.execute(state_group_sql, (last_checked_state_group, batch_size))
+
+            # Mapping from state group to whether we should delete it.
+            state_groups_to_deletion: dict[int, bool] = {}
+
+            # Mapping from state group to prev state group.
+            state_groups_to_prev: dict[int, int] = {}
+
+            for row in txn:
+                state_group = row[0]
+                next_edge = row[1]
+                pending_deletion = row[2]
+
+                if next_edge is not None:
+                    state_groups_to_prev[next_edge] = state_group
+
+                if next_edge is not None and not pending_deletion:
+                    # We have found an edge not marked for deletion.
+                    # Check previous results to see if this group is part of a chain
+                    # within this batch that qualifies for deletion.
+                    # ie. batch contains:
+                    # SG_1 -> SG_2 -> SG_3
+                    # If SG_3 is a candidate for deletion, then SG_2 & SG_1 should also
+                    # be, even though they have edges which may not be marked for
+                    # deletion.
+                    # This relies on SQL results being sorted in DESC order to work.
+                    next_is_deletion_candidate = state_groups_to_deletion.get(next_edge)
+                    if (
+                        next_is_deletion_candidate is None
+                        or not next_is_deletion_candidate
+                    ):
+                        state_groups_to_deletion[state_group] = False
+                    else:
+                        state_groups_to_deletion.setdefault(state_group, True)
+                else:
+                    # This state group may be a candidate for deletion
+                    state_groups_to_deletion.setdefault(state_group, True)
+
+            return state_groups_to_deletion, state_groups_to_prev
+
+        (
+            state_groups_to_deletion,
+            state_group_edges,
+        ) = await self.stores.state.db_pool.runInteraction(
+            "get_next_state_groups_marked_for_deletion",
+            get_next_state_groups_marked_for_deletion_txn,
+        )
+        deletion_candidates = {
+            state_group
+            for state_group, deletion in state_groups_to_deletion.items()
+            if deletion
+        }
+
+        final_batch = False
+        state_groups = state_groups_to_deletion.keys()
+        if len(state_groups) < batch_size:
+            final_batch = True
+        else:
+            last_checked_state_group = min(state_groups)
+
+        if len(state_groups) == 0:
+            return set(), last_checked_state_group, final_batch
+
+        # Determine if any of the remaining state groups are directly referenced.
+        referenced = await self.stores.main.get_referenced_state_groups(
+            deletion_candidates
+        )
+
+        # Remove state groups from deletion_candidates which are directly referenced or share a
+        # future edge with a referenced state group within this batch.
+        def filter_reference_chains(group: Optional[int]) -> None:
+            while group is not None:
+                deletion_candidates.discard(group)
+                group = state_group_edges.get(group)
+
+        for referenced_group in referenced:
+            filter_reference_chains(referenced_group)
+
+        return deletion_candidates, last_checked_state_group, final_batch

synapse/storage/databases/main/events.py

@@ -562,7 +562,8 @@ class PersistEventsStore:
                     #
                     # Ideally, we could additionally assert that we're only here for
                     # valid non-join membership transitions.
-                    assert delta_state.no_longer_in_room
+                    #assert delta_state.no_longer_in_room
+                    pass
 
         # Handle gathering info for the `sliding_sync_joined_rooms` table
         #

synapse/storage/databases/main/roommember.py

@@ -79,6 +79,7 @@ logger = logging.getLogger(__name__)
 
 _MEMBERSHIP_PROFILE_UPDATE_NAME = "room_membership_profile_update"
 _CURRENT_STATE_MEMBERSHIP_UPDATE_NAME = "current_state_events_membership"
+_POPULATE_PARTICIPANT_BG_UPDATE_BATCH_SIZE = 1000
 
 
 @attr.s(frozen=True, slots=True, auto_attribs=True)
@@ -1606,6 +1607,66 @@ class RoomMemberWorkerStore(EventsWorkerStore, CacheInvalidationWorkerStore):
             from_ts,
         )
 
+    async def set_room_participation(self, user_id: str, room_id: str) -> None:
+        """
+        Record the provided user as participating in the given room
+
+        Args:
+            user_id: the user ID of the user
+            room_id: ID of the room to set the participant in
+        """
+
+        def _set_room_participation_txn(
+            txn: LoggingTransaction, user_id: str, room_id: str
+        ) -> None:
+            sql = """
+                UPDATE room_memberships
+                SET participant = true
+                WHERE (user_id, room_id) IN (
+                    SELECT user_id, room_id
+                    FROM room_memberships
+                    WHERE user_id = ?
+                    AND room_id = ?
+                    ORDER BY event_stream_ordering DESC
+                    LIMIT 1
+                )
+            """
+            txn.execute(sql, (user_id, room_id))
+
+        await self.db_pool.runInteraction(
+            "_set_room_participation_txn", _set_room_participation_txn, user_id, room_id
+        )
+
+    async def get_room_participation(self, user_id: str, room_id: str) -> bool:
+        """
+        Check whether a user is listed as a participant in a room
+
+        Args:
+            user_id: user ID of the user
+            room_id: ID of the room to check in
+        """
+
+        def _get_room_participation_txn(
+            txn: LoggingTransaction, user_id: str, room_id: str
+        ) -> bool:
+            sql = """
+                SELECT participant
+                FROM room_memberships
+                WHERE user_id = ?
+                AND room_id = ?
+                ORDER BY event_stream_ordering DESC
+                LIMIT 1
+            """
+            txn.execute(sql, (user_id, room_id))
+            res = txn.fetchone()
+            if res:
+                return res[0]
+            return False
+
+        return await self.db_pool.runInteraction(
+            "_get_room_participation_txn", _get_room_participation_txn, user_id, room_id
+        )
+
 
 class RoomMemberBackgroundUpdateStore(SQLBaseStore):
     def __init__(
@@ -1636,6 +1697,93 @@ class RoomMemberBackgroundUpdateStore(SQLBaseStore):
             columns=["user_id", "room_id"],
         )
 
+        self.db_pool.updates.register_background_update_handler(
+            "populate_participant_bg_update", self._populate_participant
+        )
+
+    async def _populate_participant(self, progress: JsonDict, batch_size: int) -> int:
+        """
+        Background update to populate column `participant` on `room_memberships` table
+
+        A 'participant' is someone who is currently joined to a room and has sent at least
+        one `m.room.message` or `m.room.encrypted` event.
+
+        This background update will set the `participant` column across all rows in
+        `room_memberships` based on the user's *current* join status, and if
+        they've *ever* sent a message or encrypted event. Therefore one should
+        never assume the `participant` column's value is based solely on whether
+        the user participated in a previous "session" (where a "session" is defined
+        as a period between the user joining and leaving). See
+        https://github.com/element-hq/synapse/pull/18068#discussion_r1931070291
+        for further detail.
+        """
+        stream_token = progress.get("last_stream_token", None)
+
+        def _get_max_stream_token_txn(txn: LoggingTransaction) -> int:
+            sql = """
+                SELECT event_stream_ordering from room_memberships
+                ORDER BY event_stream_ordering DESC
+                LIMIT 1;
+            """
+            txn.execute(sql)
+            res = txn.fetchone()
+            if not res or not res[0]:
+                return 0
+            return res[0]
+
+        def _background_populate_participant_txn(
+            txn: LoggingTransaction, stream_token: str
+        ) -> None:
+            sql = """
+                UPDATE room_memberships
+                SET participant = True
+                FROM (
+                    SELECT DISTINCT c.state_key, e.room_id
+                    FROM current_state_events AS c
+                    INNER JOIN events AS e ON c.room_id = e.room_id
+                    WHERE c.membership = 'join'
+                        AND c.state_key = e.sender
+                        AND (
+                            e.type = 'm.room.message'
+                            OR e.type = 'm.room.encrypted'
+                        )
+                ) AS subquery
+                WHERE room_memberships.user_id = subquery.state_key
+                    AND room_memberships.room_id = subquery.room_id
+                    AND room_memberships.event_stream_ordering <= ?
+                    AND room_memberships.event_stream_ordering > ?;
+            """
+            batch = int(stream_token) - _POPULATE_PARTICIPANT_BG_UPDATE_BATCH_SIZE
+            txn.execute(sql, (stream_token, batch))
+
+        if stream_token is None:
+            stream_token = await self.db_pool.runInteraction(
+                "_get_max_stream_token", _get_max_stream_token_txn
+            )
+
+        if stream_token < 0:
+            await self.db_pool.updates._end_background_update(
+                "populate_participant_bg_update"
+            )
+            return _POPULATE_PARTICIPANT_BG_UPDATE_BATCH_SIZE
+
+        await self.db_pool.runInteraction(
+            "_background_populate_participant_txn",
+            _background_populate_participant_txn,
+            stream_token,
+        )
+
+        progress["last_stream_token"] = (
+            stream_token - _POPULATE_PARTICIPANT_BG_UPDATE_BATCH_SIZE
+        )
+        await self.db_pool.runInteraction(
+            "populate_participant_bg_update",
+            self.db_pool.updates._background_update_progress_txn,
+            "populate_participant_bg_update",
+            progress,
+        )
+        return _POPULATE_PARTICIPANT_BG_UPDATE_BATCH_SIZE
+
     async def _background_add_membership_profile(
         self, progress: JsonDict, batch_size: int
     ) -> int:

synapse/storage/databases/main/sliding_sync.py

@@ -1,7 +1,7 @@
 #
 # This file is licensed under the Affero General Public License (AGPL) version 3.
 #
-# Copyright (C) 2023 New Vector, Ltd
+# Copyright (C) 2023, 2025 New Vector, Ltd
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as
@@ -61,6 +61,13 @@ class SlidingSyncStore(SQLBaseStore):
             columns=("required_state_id",),
         )
 
+        self.db_pool.updates.register_background_index_update(
+            update_name="sliding_sync_membership_snapshots_membership_event_id_idx",
+            index_name="sliding_sync_membership_snapshots_membership_event_id_idx",
+            table="sliding_sync_membership_snapshots",
+            columns=("membership_event_id",),
+        )
+
     async def get_latest_bump_stamp_for_room(
         self,
         room_id: str,

synapse/storage/databases/state/bg_updates.py

@@ -20,7 +20,15 @@
 #
 
 import logging
-from typing import TYPE_CHECKING, Dict, List, Mapping, Optional, Tuple, Union
+from typing import (
+    TYPE_CHECKING,
+    Dict,
+    List,
+    Mapping,
+    Optional,
+    Tuple,
+    Union,
+)
 
 from synapse.logging.opentracing import tag_args, trace
 from synapse.storage._base import SQLBaseStore

synapse/storage/databases/state/deletion.py

@@ -321,18 +321,42 @@ class StateDeletionDataStore:
     async def mark_state_groups_as_pending_deletion(
         self, state_groups: Collection[int]
     ) -> None:
-        """Mark the given state groups as pending deletion"""
+        """Mark the given state groups as pending deletion.
+
+        If any of the state groups are already pending deletion, then those records are
+        left as is.
+        """
+
+        await self.db_pool.runInteraction(
+            "mark_state_groups_as_pending_deletion",
+            self._mark_state_groups_as_pending_deletion_txn,
+            state_groups,
+        )
+
+    def _mark_state_groups_as_pending_deletion_txn(
+        self,
+        txn: LoggingTransaction,
+        state_groups: Collection[int],
+    ) -> None:
+        sql = """
+        INSERT INTO state_groups_pending_deletion (state_group, insertion_ts)
+        VALUES %s
+        ON CONFLICT (state_group)
+        DO NOTHING
+        """
 
         now = self._clock.time_msec()
-
-        await self.db_pool.simple_upsert_many(
-            table="state_groups_pending_deletion",
-            key_names=("state_group",),
-            key_values=[(state_group,) for state_group in state_groups],
-            value_names=("insertion_ts",),
-            value_values=[(now,) for _ in state_groups],
-            desc="mark_state_groups_as_pending_deletion",
-        )
+        rows = [
+            (
+                state_group,
+                now,
+            )
+            for state_group in state_groups
+        ]
+        if isinstance(txn.database_engine, PostgresEngine):
+            txn.execute_values(sql % ("?",), rows, fetch=False)
+        else:
+            txn.execute_batch(sql % ("(?, ?)",), rows)
 
     async def mark_state_groups_as_used(self, state_groups: Collection[int]) -> None:
         """Mark the given state groups as now being referenced"""

synapse/storage/schema/__init__.py

@@ -19,7 +19,7 @@
 #
 #
 
-SCHEMA_VERSION = 89  # remember to update the list below when updating
+SCHEMA_VERSION = 90  # remember to update the list below when updating
 """Represents the expectations made by the codebase about the database schema
 
 This should be incremented whenever the codebase changes its requirements on the
@@ -158,6 +158,10 @@ Changes in SCHEMA_VERSION = 88
 
 Changes in SCHEMA_VERSION = 89
     - Add `state_groups_pending_deletion` and `state_groups_persisting` tables.
+
+Changes in SCHEMA_VERSION = 90
+    - Add a column `participant` to `room_memberships` table
+    - Add background update to delete unreferenced state groups.
 """
 
 

synapse/storage/schema/main/delta/89/01_sliding_sync_membership_snapshot_index.sql

@@ -0,0 +1,15 @@
+--
+-- This file is licensed under the Affero General Public License (AGPL) version 3.
+--
+-- Copyright (C) 2025 New Vector, Ltd
+--
+-- This program is free software: you can redistribute it and/or modify
+-- it under the terms of the GNU Affero General Public License as
+-- published by the Free Software Foundation, either version 3 of the
+-- License, or (at your option) any later version.
+--
+-- See the GNU Affero General Public License for more details:
+-- <https://www.gnu.org/licenses/agpl-3.0.html>.
+
+INSERT INTO background_updates (ordering, update_name, progress_json) VALUES
+    (8901, 'sliding_sync_membership_snapshots_membership_event_id_idx', '{}');

synapse/storage/schema/main/delta/90/01_add_column_participant_room_memberships_table.sql

@@ -0,0 +1,20 @@
+--
+-- This file is licensed under the Affero General Public License (AGPL) version 3.
+--
+-- Copyright (C) 2025 New Vector, Ltd
+--
+-- This program is free software: you can redistribute it and/or modify
+-- it under the terms of the GNU Affero General Public License as
+-- published by the Free Software Foundation, either version 3 of the
+-- License, or (at your option) any later version.
+--
+-- See the GNU Affero General Public License for more details:
+-- <https://www.gnu.org/licenses/agpl-3.0.html>.
+
+-- Add a column `participant` to `room_memberships` table to track whether a room member has sent
+-- a `m.room.message` or `m.room.encrypted` event into a room they are a member of
+ALTER TABLE room_memberships ADD COLUMN participant BOOLEAN DEFAULT FALSE;
+
+-- Add a background update to populate `participant` column
+INSERT INTO background_updates (ordering, update_name, progress_json) VALUES
+  (9001, 'populate_participant_bg_update', '{}');

synapse/storage/schema/state/delta/90/02_delete_unreferenced_state_groups.sql

@@ -0,0 +1,16 @@
+--
+-- This file is licensed under the Affero General Public License (AGPL) version 3.
+--
+-- Copyright (C) 2025 New Vector, Ltd
+--
+-- This program is free software: you can redistribute it and/or modify
+-- it under the terms of the GNU Affero General Public License as
+-- published by the Free Software Foundation, either version 3 of the
+-- License, or (at your option) any later version.
+--
+-- See the GNU Affero General Public License for more details:
+-- <https://www.gnu.org/licenses/agpl-3.0.html>.
+
+-- Add a background update to delete any unreferenced state groups
+INSERT INTO background_updates (ordering, update_name, progress_json) VALUES
+  (9002, 'mark_unreferenced_state_groups_for_deletion_bg_update', '{}');

synapse/types/storage/__init__.py

@@ -48,3 +48,7 @@ class _BackgroundUpdates:
     SLIDING_SYNC_MEMBERSHIP_SNAPSHOTS_FIX_FORGOTTEN_COLUMN_BG_UPDATE = (
         "sliding_sync_membership_snapshots_fix_forgotten_column_bg_update"
     )
+
+    MARK_UNREFERENCED_STATE_GROUPS_FOR_DELETION_BG_UPDATE = (
+        "mark_unreferenced_state_groups_for_deletion_bg_update"
+    )

tests/rest/client/test_rooms.py

@@ -4208,3 +4208,196 @@ class UserSuspensionTests(unittest.HomeserverTestCase):
             shorthand=False,
         )
         self.assertEqual(channel.code, 200)
+
+
+class RoomParticipantTestCase(unittest.HomeserverTestCase):
+    servlets = [
+        login.register_servlets,
+        room.register_servlets,
+        profile.register_servlets,
+        admin.register_servlets,
+    ]
+
+    def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None:
+        self.user1 = self.register_user("thomas", "hackme")
+        self.tok1 = self.login("thomas", "hackme")
+
+        self.user2 = self.register_user("teresa", "hackme")
+        self.tok2 = self.login("teresa", "hackme")
+
+        self.room1 = self.helper.create_room_as(
+            room_creator=self.user1,
+            tok=self.tok1,
+            # Allow user2 to send state events into the room.
+            extra_content={
+                "power_level_content_override": {
+                    "state_default": 0,
+                },
+            },
+        )
+        self.store = hs.get_datastores().main
+
+    @parameterized.expand(
+        [
+            # Should record participation.
+            param(
+                is_state=False,
+                event_type="m.room.message",
+                event_content={
+                    "msgtype": "m.text",
+                    "body": "I am engaging in this room",
+                },
+                record_participation=True,
+            ),
+            param(
+                is_state=False,
+                event_type="m.room.encrypted",
+                event_content={
+                    "algorithm": "m.megolm.v1.aes-sha2",
+                    "ciphertext": "AwgAEnACgAkLmt6qF84IK++J7UDH2Za1YVchHyprqTqsg...",
+                    "device_id": "RJYKSTBOIE",
+                    "sender_key": "IlRMeOPX2e0MurIyfWEucYBRVOEEUMrOHqn/8mLqMjA",
+                    "session_id": "X3lUlvLELLYxeTx4yOVu6UDpasGEVO0Jbu+QFnm0cKQ",
+                },
+                record_participation=True,
+            ),
+            # Should not record participation.
+            param(
+                is_state=False,
+                event_type="m.sticker",
+                event_content={
+                    "body": "My great sticker",
+                    "info": {},
+                    "url": "mxc://unused/mxcurl",
+                },
+                record_participation=False,
+            ),
+            # An invalid **state event** with type `m.room.message`
+            param(
+                is_state=True,
+                event_type="m.room.message",
+                event_content={
+                    "msgtype": "m.text",
+                    "body": "I am engaging in this room",
+                },
+                record_participation=False,
+            ),
+            # An invalid **state event** with type `m.room.encrypted`
+            # Note: this may become valid in the future with encrypted state, though we
+            # still may not want to consider it grounds for marking a user as participating.
+            param(
+                is_state=True,
+                event_type="m.room.encrypted",
+                event_content={
+                    "algorithm": "m.megolm.v1.aes-sha2",
+                    "ciphertext": "AwgAEnACgAkLmt6qF84IK++J7UDH2Za1YVchHyprqTqsg...",
+                    "device_id": "RJYKSTBOIE",
+                    "sender_key": "IlRMeOPX2e0MurIyfWEucYBRVOEEUMrOHqn/8mLqMjA",
+                    "session_id": "X3lUlvLELLYxeTx4yOVu6UDpasGEVO0Jbu+QFnm0cKQ",
+                },
+                record_participation=False,
+            ),
+        ]
+    )
+    def test_sending_message_records_participation(
+        self,
+        is_state: bool,
+        event_type: str,
+        event_content: JsonDict,
+        record_participation: bool,
+    ) -> None:
+        """
+        Test that sending an various events into a room causes the user to
+        appropriately marked or not marked as a participant in that room.
+        """
+        self.helper.join(self.room1, self.user2, tok=self.tok2)
+
+        # user has not sent any messages, so should not be a participant
+        participant = self.get_success(
+            self.store.get_room_participation(self.user2, self.room1)
+        )
+        self.assertFalse(participant)
+
+        # send an event into the room
+        if is_state:
+            # send a state event
+            self.helper.send_state(
+                self.room1,
+                event_type,
+                body=event_content,
+                tok=self.tok2,
+            )
+        else:
+            # send a non-state event
+            self.helper.send_event(
+                self.room1,
+                event_type,
+                content=event_content,
+                tok=self.tok2,
+            )
+
+        # check whether the user has been marked as a participant
+        participant = self.get_success(
+            self.store.get_room_participation(self.user2, self.room1)
+        )
+        self.assertEqual(participant, record_participation)
+
+    @parameterized.expand(
+        [
+            param(
+                event_type="m.room.message",
+                event_content={
+                    "msgtype": "m.text",
+                    "body": "I am engaging in this room",
+                },
+            ),
+            param(
+                event_type="m.room.encrypted",
+                event_content={
+                    "algorithm": "m.megolm.v1.aes-sha2",
+                    "ciphertext": "AwgAEnACgAkLmt6qF84IK++J7UDH2Za1YVchHyprqTqsg...",
+                    "device_id": "RJYKSTBOIE",
+                    "sender_key": "IlRMeOPX2e0MurIyfWEucYBRVOEEUMrOHqn/8mLqMjA",
+                    "session_id": "X3lUlvLELLYxeTx4yOVu6UDpasGEVO0Jbu+QFnm0cKQ",
+                },
+            ),
+        ]
+    )
+    def test_sending_event_and_leaving_does_not_record_participation(
+        self,
+        event_type: str,
+        event_content: JsonDict,
+    ) -> None:
+        """
+        Test that sending an event into a room that should mark a user as a
+        participant, but then leaving the room, results in the user no longer
+        be marked as a participant in that room.
+        """
+        self.helper.join(self.room1, self.user2, tok=self.tok2)
+
+        # user has not sent any messages, so should not be a participant
+        participant = self.get_success(
+            self.store.get_room_participation(self.user2, self.room1)
+        )
+        self.assertFalse(participant)
+
+        # sending a message should now mark user as participant
+        self.helper.send_event(
+            self.room1,
+            event_type,
+            content=event_content,
+            tok=self.tok2,
+        )
+        participant = self.get_success(
+            self.store.get_room_participation(self.user2, self.room1)
+        )
+        self.assertTrue(participant)
+
+        # leave the room
+        self.helper.leave(self.room1, self.user2, tok=self.tok2)
+
+        # user should no longer be considered a participant
+        participant = self.get_success(
+            self.store.get_room_participation(self.user2, self.room1)
+        )
+        self.assertFalse(participant)

tests/storage/test_purge.py

@@ -24,6 +24,7 @@ from synapse.api.errors import NotFoundError, SynapseError
 from synapse.rest.client import room
 from synapse.server import HomeServer
 from synapse.types.state import StateFilter
+from synapse.types.storage import _BackgroundUpdates
 from synapse.util import Clock
 
 from tests.unittest import HomeserverTestCase
@@ -303,3 +304,156 @@ class PurgeTests(HomeserverTestCase):
             )
         )
         self.assertEqual(len(state_groups), 1)
+
+    def test_clear_unreferenced_state_groups(self) -> None:
+        """Test that any unreferenced state groups are automatically cleaned up."""
+
+        self.helper.send(self.room_id, body="test1")
+        state1 = self.helper.send_state(
+            self.room_id, "org.matrix.test", body={"number": 2}
+        )
+        # Create enough state events to require multiple batches of
+        # mark_unreferenced_state_groups_for_deletion_bg_update to be run.
+        for i in range(200):
+            self.helper.send_state(self.room_id, "org.matrix.test", body={"number": i})
+        self.helper.send(self.room_id, body="test4")
+        last = self.helper.send(self.room_id, body="test5")
+
+        # Create an unreferenced state group that has no prev group.
+        unreferenced_free_state_group = self.get_success(
+            self.state_store.store_state_group(
+                event_id=last["event_id"],
+                room_id=self.room_id,
+                prev_group=None,
+                delta_ids={("org.matrix.test", ""): state1["event_id"]},
+                current_state_ids={("org.matrix.test", ""): ""},
+            )
+        )
+
+        # Create some unreferenced state groups that have a prev group of one of the
+        # existing state groups.
+        prev_group = self.get_success(
+            self.store._get_state_group_for_event(state1["event_id"])
+        )
+        unreferenced_end_state_group = self.get_success(
+            self.state_store.store_state_group(
+                event_id=last["event_id"],
+                room_id=self.room_id,
+                prev_group=prev_group,
+                delta_ids={("org.matrix.test", ""): state1["event_id"]},
+                current_state_ids=None,
+            )
+        )
+        another_unreferenced_end_state_group = self.get_success(
+            self.state_store.store_state_group(
+                event_id=last["event_id"],
+                room_id=self.room_id,
+                prev_group=unreferenced_end_state_group,
+                delta_ids={("org.matrix.test", ""): state1["event_id"]},
+                current_state_ids=None,
+            )
+        )
+
+        # Add some other unreferenced state groups which lead to a referenced state
+        # group.
+        # These state groups should not get deleted.
+        chain_state_group = self.get_success(
+            self.state_store.store_state_group(
+                event_id=last["event_id"],
+                room_id=self.room_id,
+                prev_group=None,
+                delta_ids={("org.matrix.test", ""): ""},
+                current_state_ids={("org.matrix.test", ""): ""},
+            )
+        )
+        chain_state_group_2 = self.get_success(
+            self.state_store.store_state_group(
+                event_id=last["event_id"],
+                room_id=self.room_id,
+                prev_group=chain_state_group,
+                delta_ids={("org.matrix.test", ""): ""},
+                current_state_ids=None,
+            )
+        )
+        referenced_chain_state_group = self.get_success(
+            self.state_store.store_state_group(
+                event_id=last["event_id"],
+                room_id=self.room_id,
+                prev_group=chain_state_group_2,
+                delta_ids={("org.matrix.test", ""): ""},
+                current_state_ids=None,
+            )
+        )
+        self.get_success(
+            self.store.db_pool.simple_insert(
+                "event_to_state_groups",
+                {
+                    "event_id": "$new_event",
+                    "state_group": referenced_chain_state_group,
+                },
+            )
+        )
+
+        # Insert and run the background update.
+        self.get_success(
+            self.store.db_pool.simple_insert(
+                "background_updates",
+                {
+                    "update_name": _BackgroundUpdates.MARK_UNREFERENCED_STATE_GROUPS_FOR_DELETION_BG_UPDATE,
+                    "progress_json": "{}",
+                },
+            )
+        )
+        self.store.db_pool.updates._all_done = False
+        self.wait_for_background_updates()
+
+        # Advance so that the background job to delete the state groups runs
+        self.reactor.advance(
+            1 + self.state_deletion_store.DELAY_BEFORE_DELETION_MS / 1000
+        )
+
+        # We expect that the unreferenced free state group has been deleted.
+        row = self.get_success(
+            self.state_store.db_pool.simple_select_one_onecol(
+                table="state_groups",
+                keyvalues={"id": unreferenced_free_state_group},
+                retcol="id",
+                allow_none=True,
+                desc="test_purge_unreferenced_state_group",
+            )
+        )
+        self.assertIsNone(row)
+
+        # We expect that both unreferenced end state groups have been deleted.
+        row = self.get_success(
+            self.state_store.db_pool.simple_select_one_onecol(
+                table="state_groups",
+                keyvalues={"id": unreferenced_end_state_group},
+                retcol="id",
+                allow_none=True,
+                desc="test_purge_unreferenced_state_group",
+            )
+        )
+        self.assertIsNone(row)
+        row = self.get_success(
+            self.state_store.db_pool.simple_select_one_onecol(
+                table="state_groups",
+                keyvalues={"id": another_unreferenced_end_state_group},
+                retcol="id",
+                allow_none=True,
+                desc="test_purge_unreferenced_state_group",
+            )
+        )
+        self.assertIsNone(row)
+
+        # We expect there to now only be one state group for the room, which is
+        # the state group of the last event (as the only outlier).
+        state_groups = self.get_success(
+            self.state_store.db_pool.simple_select_onecol(
+                table="state_groups",
+                keyvalues={"room_id": self.room_id},
+                retcol="id",
+                desc="test_purge_unreferenced_state_group",
+            )
+        )
+        self.assertEqual(len(state_groups), 210)