Merge branch 'rav/event_visibility' into active_branches_base

Merge branch 'rav/log_state_desync' into active_branches_base
Enable MSC4115 in complement tests
2024-04-23 17:10:20 +01:00 · 2024-04-23 17:10:20 +01:00 · 2024-04-23 12:11:08 +01:00 · 2024-04-22 20:51:40 +01:00 · 2024-04-22 20:51:40 +01:00 · 2024-04-22 20:51:40 +01:00
83 changed files with 1670 additions and 499 deletions
@@ -30,7 +30,7 @@ jobs:
        run: docker buildx inspect
      - name: Install Cosign
-        uses: sigstore/cosign-installer@v3.4.0
+        uses: sigstore/cosign-installer@v3.5.0
      - name: Checkout repository
        uses: actions/checkout@v4
@@ -19,7 +19,7 @@ jobs:
          fetch-depth: 0
      - name: Setup mdbook
-        uses: peaceiris/actions-mdbook@adeb05db28a0c0004681db83893d56c0388ea9ea # v1.2.0
+        uses: peaceiris/actions-mdbook@ee69d230fe19748b7abf22df32acaa93833fad08 # v2.0.0
        with:
          mdbook-version: '0.4.17'
@@ -53,7 +53,7 @@ jobs:
      - uses: actions/checkout@v4
      - name: Setup mdbook
-        uses: peaceiris/actions-mdbook@adeb05db28a0c0004681db83893d56c0388ea9ea # v1.2.0
+        uses: peaceiris/actions-mdbook@ee69d230fe19748b7abf22df32acaa93833fad08 # v2.0.0
        with:
          mdbook-version: '0.4.17'
@@ -56,7 +56,7 @@ jobs:
          fetch-depth: 0
      - name: Setup mdbook
-        uses: peaceiris/actions-mdbook@adeb05db28a0c0004681db83893d56c0388ea9ea # v1.2.0
+        uses: peaceiris/actions-mdbook@ee69d230fe19748b7abf22df32acaa93833fad08 # v2.0.0
        with:
          mdbook-version: '0.4.17'
@@ -80,7 +80,7 @@ jobs:
      # Deploy to the target directory.
      - name: Deploy to gh pages
-        uses: peaceiris/actions-gh-pages@373f7f263a76c20808c831209c920827a82a2847 # v3.9.3
+        uses: peaceiris/actions-gh-pages@4f9cc6602d3f66b9c108549d475ec49e8ef4d45e # v4.0.0
        with:
          github_token: ${{ secrets.GITHUB_TOKEN }}
          publish_dir: ./book
@@ -110,7 +110,7 @@ jobs:
      # Deploy to the target directory.
      - name: Deploy to gh pages
-        uses: peaceiris/actions-gh-pages@373f7f263a76c20808c831209c920827a82a2847 # v3.9.3
+        uses: peaceiris/actions-gh-pages@4f9cc6602d3f66b9c108549d475ec49e8ef4d45e # v4.0.0
        with:
          github_token: ${{ secrets.GITHUB_TOKEN }}
          publish_dir: ./dev-docs/_build/html
@@ -81,7 +81,7 @@ jobs:
    steps:
      - uses: actions/checkout@v4
      - name: Install Rust
-        uses: dtolnay/rust-toolchain@1.65.0
+        uses: dtolnay/rust-toolchain@1.66.0
      - uses: Swatinem/rust-cache@v2
      - uses: matrix-org/setup-python-poetry@v1
        with:
@@ -148,7 +148,7 @@ jobs:
        uses: actions/checkout@v4
      - name: Install Rust
-        uses: dtolnay/rust-toolchain@1.65.0
+        uses: dtolnay/rust-toolchain@1.66.0
      - uses: Swatinem/rust-cache@v2
      - name: Setup Poetry
@@ -208,7 +208,7 @@ jobs:
        with:
          ref: ${{ github.event.pull_request.head.sha }}
      - name: Install Rust
-        uses: dtolnay/rust-toolchain@1.65.0
+        uses: dtolnay/rust-toolchain@1.66.0
      - uses: Swatinem/rust-cache@v2
      - uses: matrix-org/setup-python-poetry@v1
        with:
@@ -225,7 +225,7 @@ jobs:
      - uses: actions/checkout@v4
      - name: Install Rust
-        uses: dtolnay/rust-toolchain@1.65.0
+        uses: dtolnay/rust-toolchain@1.66.0
        with:
            components: clippy
      - uses: Swatinem/rust-cache@v2
@@ -344,7 +344,7 @@ jobs:
            postgres:${{ matrix.job.postgres-version }}
      - name: Install Rust
-        uses: dtolnay/rust-toolchain@1.65.0
+        uses: dtolnay/rust-toolchain@1.66.0
      - uses: Swatinem/rust-cache@v2
      - uses: matrix-org/setup-python-poetry@v1
@@ -386,7 +386,7 @@ jobs:
      - uses: actions/checkout@v4
      - name: Install Rust
-        uses: dtolnay/rust-toolchain@1.65.0
+        uses: dtolnay/rust-toolchain@1.66.0
      - uses: Swatinem/rust-cache@v2
      # There aren't wheels for some of the older deps, so we need to install
@@ -498,7 +498,7 @@ jobs:
        run: cat sytest-blacklist .ci/worker-blacklist > synapse-blacklist-with-workers
      - name: Install Rust
-        uses: dtolnay/rust-toolchain@1.65.0
+        uses: dtolnay/rust-toolchain@1.66.0
      - uses: Swatinem/rust-cache@v2
      - name: Run SyTest
@@ -642,7 +642,7 @@ jobs:
          path: synapse
      - name: Install Rust
-        uses: dtolnay/rust-toolchain@1.65.0
+        uses: dtolnay/rust-toolchain@1.66.0
      - uses: Swatinem/rust-cache@v2
      - name: Prepare Complement's Prerequisites
@@ -674,7 +674,7 @@ jobs:
      - uses: actions/checkout@v4
      - name: Install Rust
-        uses: dtolnay/rust-toolchain@1.65.0
+        uses: dtolnay/rust-toolchain@1.66.0
      - uses: Swatinem/rust-cache@v2
      - run: cargo test
@@ -1,3 +1,42 @@
 # Synapse 1.105.0 (2024-04-16)
 No significant changes since 1.105.0rc1.
 # Synapse 1.105.0rc1 (2024-04-11)
 ### Features
 - Stabilize support for [MSC4010](https://github.com/matrix-org/matrix-spec-proposals/pull/4010) which clarifies the interaction of push rules and account data. Contributed by @clokep. ([\#17022](https://github.com/element-hq/synapse/issues/17022))
 - Stabilize support for [MSC3981](https://github.com/matrix-org/matrix-spec-proposals/pull/3981): `/relations` recursion. Contributed by @clokep. ([\#17023](https://github.com/element-hq/synapse/issues/17023))
 - Add support for moving `/pushrules` off of main process. ([\#17037](https://github.com/element-hq/synapse/issues/17037), [\#17038](https://github.com/element-hq/synapse/issues/17038))
 ### Bugfixes
 - Fix various long-standing bugs which could cause incorrect state to be returned from `/sync` in certain situations. ([\#16930](https://github.com/element-hq/synapse/issues/16930), [\#16932](https://github.com/element-hq/synapse/issues/16932), [\#16942](https://github.com/element-hq/synapse/issues/16942), [\#17064](https://github.com/element-hq/synapse/issues/17064), [\#17065](https://github.com/element-hq/synapse/issues/17065), [\#17066](https://github.com/element-hq/synapse/issues/17066))
 - Fix server notice rooms not always being created as unencrypted rooms, even when `encryption_enabled_by_default_for_room_type` is in use (server notices are always unencrypted). ([\#17033](https://github.com/element-hq/synapse/issues/17033))
 - Fix the `.m.rule.encrypted_room_one_to_one` and `.m.rule.room_one_to_one` default underride push rules being in the wrong order. Contributed by @Sumpy1. ([\#17043](https://github.com/element-hq/synapse/issues/17043))
 ### Internal Changes
 - Refactor auth chain fetching to reduce duplication. ([\#17044](https://github.com/element-hq/synapse/issues/17044))
 - Improve database performance by adding a missing index to `access_tokens.refresh_token_id`. ([\#17045](https://github.com/element-hq/synapse/issues/17045), [\#17054](https://github.com/element-hq/synapse/issues/17054))
 - Improve database performance by reducing number of receipts fetched when sending push notifications. ([\#17049](https://github.com/element-hq/synapse/issues/17049))
 ### Updates to locked dependencies
 * Bump packaging from 23.2 to 24.0. ([\#17027](https://github.com/element-hq/synapse/issues/17027))
 * Bump regex from 1.10.3 to 1.10.4. ([\#17028](https://github.com/element-hq/synapse/issues/17028))
 * Bump ruff from 0.3.2 to 0.3.5. ([\#17060](https://github.com/element-hq/synapse/issues/17060))
 * Bump serde_json from 1.0.114 to 1.0.115. ([\#17041](https://github.com/element-hq/synapse/issues/17041))
 * Bump types-pillow from 10.2.0.20240125 to 10.2.0.20240406. ([\#17061](https://github.com/element-hq/synapse/issues/17061))
 * Bump types-requests from 2.31.0.20240125 to 2.31.0.20240406. ([\#17063](https://github.com/element-hq/synapse/issues/17063))
 * Bump typing-extensions from 4.9.0 to 4.11.0. ([\#17062](https://github.com/element-hq/synapse/issues/17062))
 # Synapse 1.104.0 (2024-04-02)
 ### Bugfixes
@@ -13,9 +13,9 @@ dependencies = [
 [[package]]
 name = "anyhow"
-version = "1.0.81"
+version = "1.0.82"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0952808a6c2afd1aa8947271f3a60f1a6763c7b912d210184c5149b5cf147247"
+checksum = "f538837af36e6f6a9be0faa67f9a314f8119e4e4b5867c6ab40ed60360142519"
 [[package]]
 name = "arc-swap"
@@ -29,6 +29,12 @@ version = "1.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa"
 [[package]]
 name = "base64"
 version = "0.21.7"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "9d297deb1925b89f2ccc13d7635fa0714f12c87adce1c75356b39ca9b7178567"
 [[package]]
 name = "bitflags"
 version = "1.3.2"
@@ -53,12 +59,27 @@ dependencies = [
 "generic-array",
 ]
 [[package]]
 name = "bytes"
 version = "1.6.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "514de17de45fdb8dc022b1a7975556c53c86f9f0aa5f534b98977b171857c2c9"
 [[package]]
 name = "cfg-if"
 version = "1.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
 [[package]]
 name = "cpufeatures"
 version = "0.2.12"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "53fe5e26ff1b7aef8bca9c6080520cfb8d9333c7568e1829cef191a9723e5504"
 dependencies = [
 "libc",
 ]
 [[package]]
 name = "crypto-common"
 version = "0.1.6"
@@ -80,6 +101,12 @@ dependencies = [
 "subtle",
 ]
 [[package]]
 name = "fnv"
 version = "1.0.7"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1"
 [[package]]
 name = "generic-array"
 version = "0.14.6"
@@ -90,6 +117,30 @@ dependencies = [
 "version_check",
 ]
 [[package]]
 name = "headers"
 version = "0.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "322106e6bd0cba2d5ead589ddb8150a13d7c4217cf80d7c4f682ca994ccc6aa9"
 dependencies = [
 "base64",
 "bytes",
 "headers-core",
 "http",
 "httpdate",
 "mime",
 "sha1",
 ]
 [[package]]
 name = "headers-core"
 version = "0.3.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "54b4a22553d4242c49fddb9ba998a99962b5cc6f22cb5a3482bec22522403ce4"
 dependencies = [
 "http",
 ]
 [[package]]
 name = "heck"
 version = "0.4.1"
@@ -102,6 +153,23 @@ version = "0.4.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70"
 [[package]]
 name = "http"
 version = "1.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "21b9ddb458710bc376481b842f5da65cdf31522de232c1ca8146abce2a358258"
 dependencies = [
 "bytes",
 "fnv",
 "itoa",
 ]
 [[package]]
 name = "httpdate"
 version = "1.0.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9"
 [[package]]
 name = "indoc"
 version = "2.0.4"
@@ -122,9 +190,9 @@ checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646"
 [[package]]
 name = "libc"
-version = "0.2.135"
+version = "0.2.153"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "68783febc7782c6c5cb401fbda4de5a9898be1762314da0bb2c10ced61f18b0c"
+checksum = "9c198f91728a82281a64e1f4f9eeb25d82cb32a5de251c6bd1b5154d63a8e7bd"
 [[package]]
 name = "lock_api"
@@ -157,6 +225,12 @@ dependencies = [
 "autocfg",
 ]
 [[package]]
 name = "mime"
 version = "0.3.17"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a"
 [[package]]
 name = "once_cell"
 version = "1.15.0"
@@ -306,9 +380,9 @@ dependencies = [
 [[package]]
 name = "regex"
-version = "1.10.3"
+version = "1.10.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b62dbe01f0b06f9d8dc7d49e05a0785f153b00b2c227856282f671e0318c9b15"
+checksum = "c117dbdfde9c8308975b6a18d71f3f385c89461f7b3fb054288ecf2a2058ba4c"
 dependencies = [
 "aho-corasick",
 "memchr",
@@ -367,15 +441,26 @@ dependencies = [
 [[package]]
 name = "serde_json"
-version = "1.0.114"
+version = "1.0.115"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c5f09b1bd632ef549eaa9f60a1f8de742bdbc698e6cee2095fc84dde5f549ae0"
+checksum = "12dc5c46daa8e9fdf4f5e71b6cf9a53f2487da0e86e55808e2d35539666497dd"
 dependencies = [
 "itoa",
 "ryu",
 "serde",
 ]
 [[package]]
 name = "sha1"
 version = "0.10.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "f04293dc80c3993519f2d7f6f511707ee7094fe0c6d3406feb330cdb3540eba3"
 dependencies = [
 "cfg-if",
 "cpufeatures",
 "digest",
 ]
 [[package]]
 name = "smallvec"
 version = "1.10.0"
@@ -405,7 +490,10 @@ version = "0.1.0"
 dependencies = [
 "anyhow",
 "blake2",
 "bytes",
 "headers",
 "hex",
 "http",
 "lazy_static",
 "log",
 "pyo3",
@@ -0,0 +1 @@
 Adds validation to ensure that the `limit` parameter on `/publicRooms` is non-negative.
@@ -0,0 +1 @@
 Return `400 M_NOT_JSON` upon receiving invalid JSON in query parameters across various client and admin endpoints, rather than an internal server error.
@@ -1 +0,0 @@
 Fix various long-standing bugs which could cause incorrect state to be returned from `/sync` in certain situations.
@@ -1 +0,0 @@
 Fix various long-standing bugs which could cause incorrect state to be returned from `/sync` in certain situations.
@@ -1 +0,0 @@
 Fix various long-standing bugs which could cause incorrect state to be returned from `/sync` in certain situations.
@@ -0,0 +1 @@
 Make the CSAPI endpoint `/keys/device_signing/upload` idempotent.
@@ -0,0 +1 @@
 Use new receipts column to optimise receipt and push action SQL queries. Contributed by Nick @ Beeper (@fizzadar).
@@ -0,0 +1 @@
 Fix mypy with latest Twisted release.
@@ -1 +0,0 @@
 Add support for moving `/pushrules` off of main process.
@@ -1 +0,0 @@
 Add support for moving `/pushrules` off of main process.
@@ -1 +0,0 @@
 Refactor auth chain fetching to reduce duplication.
@@ -1 +0,0 @@
 Improve database performance by adding a missing index to `access_tokens.refresh_token_id`.
@@ -1 +0,0 @@
 Improve database performance by reducing number of receipts fetched when sending push notifications.
@@ -0,0 +1 @@
 Add a prompt in the contributing guide to manually configure icu4c.
@@ -0,0 +1 @@
 Bump minimum supported Rust version to 1.66.0.
@@ -0,0 +1 @@
 Add helpers to transform Twisted requests to Rust http Requests/Responses.
@@ -0,0 +1 @@
 Support delegating the rendezvous mechanism described MSC4108 to an external implementation.
@@ -0,0 +1 @@
 Use new receipts column to optimise receipt and push action SQL queries. Contributed by Nick @ Beeper (@fizzadar).
@@ -0,0 +1 @@
 Clarify what part of message retention is still experimental.
@@ -0,0 +1 @@
 Add support for MSC4115 (membership metadata on events).
@@ -1,3 +1,15 @@
 matrix-synapse-py3 (1.105.0) stable; urgency=medium
  * New Synapse release 1.105.0.
 -- Synapse Packaging team <packages@matrix.org>  Tue, 16 Apr 2024 15:53:23 +0100
 matrix-synapse-py3 (1.105.0~rc1) stable; urgency=medium
  * New Synapse release 1.105.0rc1.
 -- Synapse Packaging team <packages@matrix.org>  Thu, 11 Apr 2024 12:15:49 +0100
 matrix-synapse-py3 (1.104.0) stable; urgency=medium
  * New Synapse release 1.104.0.
@@ -92,8 +92,6 @@ allow_device_name_lookup_over_federation: true
 ## Experimental Features ##
 experimental_features:
  # client-side support for partial state in /send_join responses
  faster_joins: true
  # Enable support for polls
  msc3381_polls_enabled: true
  # Enable deleting device-specific notification settings stored in account data
@@ -102,6 +100,10 @@ experimental_features:
  msc3391_enabled: true
  # Filtering /messages by relation type.
  msc3874_enabled: true
  # no UIA for x-signing upload for the first time
  msc3967_enabled: true
  msc4115_membership_on_events: true
 server_notices:
  system_mxid_localpart: _server
@@ -86,6 +86,8 @@ poetry install --extras all
 This will install the runtime and developer dependencies for the project.  Be sure to check
 that the `poetry install` step completed cleanly.
 For OSX users, be sure to set `PKG_CONFIG_PATH` to support `icu4c`. Run `brew info icu4c` for more details.
 ## Running Synapse via poetry
 To start a local instance of Synapse in the locked poetry environment, create a config file:
@@ -7,8 +7,10 @@ follow the semantics described in
 and allow server and room admins to configure how long messages should
 be kept in a homeserver's database before being purged from it.
 **Please note that, as this feature isn't part of the Matrix
-specification yet, this implementation is to be considered as
+specification yet, the use of `m.room.retention` events for per-room
-experimental.**
+retention policies is to be considered as experimental. However, the use
 of a default message retention policy is considered a stable feature
 in Synapse.**
 A message retention policy is mainly defined by its `max_lifetime`
 parameter, which defines how long a message can be kept around after
@@ -1,4 +1,4 @@
-# This file is automatically @generated by Poetry 1.7.1 and should not be changed by hand.
+# This file is automatically @generated by Poetry 1.8.2 and should not be changed by hand.
 [[package]]
 name = "alabaster"
@@ -1602,13 +1602,13 @@ tests = ["Sphinx", "doubles", "flake8", "flake8-quotes", "gevent", "mock", "pyte
 [[package]]
 name = "packaging"
-version = "23.2"
+version = "24.0"
 description = "Core utilities for Python packages"
 optional = false
 python-versions = ">=3.7"
 files = [
-    {file = "packaging-23.2-py3-none-any.whl", hash = "sha256:8c491190033a9af7e1d931d0b5dacc2ef47509b34dd0de67ed209b5203fc88c7"},
+    {file = "packaging-24.0-py3-none-any.whl", hash = "sha256:2ddfb553fdf02fb784c234c7ba6ccc288296ceabec964ad2eae3777778130bc5"},
-    {file = "packaging-23.2.tar.gz", hash = "sha256:048fb0e9405036518eaaf48a55953c750c11e1a1b68e0dd1a9d62ed0c092cfc5"},
+    {file = "packaging-24.0.tar.gz", hash = "sha256:eb82c5e3e56209074766e6885bb04b8c38a0c015d0a30036ebe7ece34c9989e9"},
 ]
 [[package]]
@@ -1848,17 +1848,17 @@ files = [
 [[package]]
 name = "pyasn1-modules"
-version = "0.3.0"
+version = "0.4.0"
 description = "A collection of ASN.1-based protocols modules"
 optional = false
-python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,>=2.7"
+python-versions = ">=3.8"
 files = [
-    {file = "pyasn1_modules-0.3.0-py2.py3-none-any.whl", hash = "sha256:d3ccd6ed470d9ffbc716be08bd90efbd44d0734bc9303818f7336070984a162d"},
+    {file = "pyasn1_modules-0.4.0-py3-none-any.whl", hash = "sha256:be04f15b66c206eed667e0bb5ab27e2b1855ea54a842e5037738099e8ca4ae0b"},
-    {file = "pyasn1_modules-0.3.0.tar.gz", hash = "sha256:5bd01446b736eb9d31512a30d46c1ac3395d676c6f3cafa4c03eb54b9925631c"},
+    {file = "pyasn1_modules-0.4.0.tar.gz", hash = "sha256:831dbcea1b177b28c9baddf4c6d1013c24c3accd14a1873fffaa6a2e905f17b6"},
 ]
 [package.dependencies]
-pyasn1 = ">=0.4.6,<0.6.0"
+pyasn1 = ">=0.4.6,<0.7.0"
 [[package]]
 name = "pycparser"
@@ -1983,13 +1983,13 @@ typing-extensions = ">=4.6.0,<4.7.0 || >4.7.0"
 [[package]]
 name = "pygithub"
-version = "2.2.0"
+version = "2.3.0"
 description = "Use the full Github API v3"
 optional = false
 python-versions = ">=3.7"
 files = [
-    {file = "PyGithub-2.2.0-py3-none-any.whl", hash = "sha256:41042ea53e4c372219db708c38d2ca1fd4fadab75475bac27d89d339596cfad1"},
+    {file = "PyGithub-2.3.0-py3-none-any.whl", hash = "sha256:65b499728be3ce7b0cd2cd760da3b32f0f4d7bc55e5e0677617f90f6564e793e"},
-    {file = "PyGithub-2.2.0.tar.gz", hash = "sha256:e39be7c4dc39418bdd6e3ecab5931c636170b8b21b4d26f9ecf7e6102a3b51c3"},
+    {file = "PyGithub-2.3.0.tar.gz", hash = "sha256:0148d7347a1cdeed99af905077010aef81a4dad988b0ba51d4108bf66b443f7e"},
 ]
 [package.dependencies]
@@ -2444,28 +2444,28 @@ files = [
 [[package]]
 name = "ruff"
-version = "0.3.2"
+version = "0.3.7"
 description = "An extremely fast Python linter and code formatter, written in Rust."
 optional = false
 python-versions = ">=3.7"
 files = [
-    {file = "ruff-0.3.2-py3-none-macosx_10_12_x86_64.macosx_11_0_arm64.macosx_10_12_universal2.whl", hash = "sha256:77f2612752e25f730da7421ca5e3147b213dca4f9a0f7e0b534e9562c5441f01"},
+    {file = "ruff-0.3.7-py3-none-macosx_10_12_x86_64.macosx_11_0_arm64.macosx_10_12_universal2.whl", hash = "sha256:0e8377cccb2f07abd25e84fc5b2cbe48eeb0fea9f1719cad7caedb061d70e5ce"},
-    {file = "ruff-0.3.2-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:9966b964b2dd1107797be9ca7195002b874424d1d5472097701ae8f43eadef5d"},
+    {file = "ruff-0.3.7-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:15a4d1cc1e64e556fa0d67bfd388fed416b7f3b26d5d1c3e7d192c897e39ba4b"},
-    {file = "ruff-0.3.2-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b83d17ff166aa0659d1e1deaf9f2f14cbe387293a906de09bc4860717eb2e2da"},
+    {file = "ruff-0.3.7-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d28bdf3d7dc71dd46929fafeec98ba89b7c3550c3f0978e36389b5631b793663"},
-    {file = "ruff-0.3.2-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:bb875c6cc87b3703aeda85f01c9aebdce3d217aeaca3c2e52e38077383f7268a"},
+    {file = "ruff-0.3.7-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:379b67d4f49774ba679593b232dcd90d9e10f04d96e3c8ce4a28037ae473f7bb"},
-    {file = "ruff-0.3.2-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:be75e468a6a86426430373d81c041b7605137a28f7014a72d2fc749e47f572aa"},
+    {file = "ruff-0.3.7-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c060aea8ad5ef21cdfbbe05475ab5104ce7827b639a78dd55383a6e9895b7c51"},
-    {file = "ruff-0.3.2-py3-none-manylinux_2_17_ppc64.manylinux2014_ppc64.whl", hash = "sha256:967978ac2d4506255e2f52afe70dda023fc602b283e97685c8447d036863a302"},
+    {file = "ruff-0.3.7-py3-none-manylinux_2_17_ppc64.manylinux2014_ppc64.whl", hash = "sha256:ebf8f615dde968272d70502c083ebf963b6781aacd3079081e03b32adfe4d58a"},
-    {file = "ruff-0.3.2-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1231eacd4510f73222940727ac927bc5d07667a86b0cbe822024dd00343e77e9"},
+    {file = "ruff-0.3.7-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d48098bd8f5c38897b03604f5428901b65e3c97d40b3952e38637b5404b739a2"},
-    {file = "ruff-0.3.2-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2c6d613b19e9a8021be2ee1d0e27710208d1603b56f47203d0abbde906929a9b"},
+    {file = "ruff-0.3.7-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:da8a4fda219bf9024692b1bc68c9cff4b80507879ada8769dc7e985755d662ea"},
-    {file = "ruff-0.3.2-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c8439338a6303585d27b66b4626cbde89bb3e50fa3cae86ce52c1db7449330a7"},
+    {file = "ruff-0.3.7-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6c44e0149f1d8b48c4d5c33d88c677a4aa22fd09b1683d6a7ff55b816b5d074f"},
-    {file = "ruff-0.3.2-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:de8b480d8379620cbb5ea466a9e53bb467d2fb07c7eca54a4aa8576483c35d36"},
+    {file = "ruff-0.3.7-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:3050ec0af72b709a62ecc2aca941b9cd479a7bf2b36cc4562f0033d688e44fa1"},
-    {file = "ruff-0.3.2-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:b74c3de9103bd35df2bb05d8b2899bf2dbe4efda6474ea9681280648ec4d237d"},
+    {file = "ruff-0.3.7-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:a29cc38e4c1ab00da18a3f6777f8b50099d73326981bb7d182e54a9a21bb4ff7"},
-    {file = "ruff-0.3.2-py3-none-musllinux_1_2_i686.whl", hash = "sha256:f380be9fc15a99765c9cf316b40b9da1f6ad2ab9639e551703e581a5e6da6745"},
+    {file = "ruff-0.3.7-py3-none-musllinux_1_2_i686.whl", hash = "sha256:5b15cc59c19edca917f51b1956637db47e200b0fc5e6e1878233d3a938384b0b"},
-    {file = "ruff-0.3.2-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:0ac06a3759c3ab9ef86bbeca665d31ad3aa9a4b1c17684aadb7e61c10baa0df4"},
+    {file = "ruff-0.3.7-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:e491045781b1e38b72c91247cf4634f040f8d0cb3e6d3d64d38dcf43616650b4"},
-    {file = "ruff-0.3.2-py3-none-win32.whl", hash = "sha256:9bd640a8f7dd07a0b6901fcebccedadeb1a705a50350fb86b4003b805c81385a"},
+    {file = "ruff-0.3.7-py3-none-win32.whl", hash = "sha256:bc931de87593d64fad3a22e201e55ad76271f1d5bfc44e1a1887edd0903c7d9f"},
-    {file = "ruff-0.3.2-py3-none-win_amd64.whl", hash = "sha256:0c1bdd9920cab5707c26c8b3bf33a064a4ca7842d91a99ec0634fec68f9f4037"},
+    {file = "ruff-0.3.7-py3-none-win_amd64.whl", hash = "sha256:5ef0e501e1e39f35e03c2acb1d1238c595b8bb36cf7a170e7c1df1b73da00e74"},
-    {file = "ruff-0.3.2-py3-none-win_arm64.whl", hash = "sha256:5f65103b1d76e0d600cabd577b04179ff592064eaa451a70a81085930e907d0b"},
+    {file = "ruff-0.3.7-py3-none-win_arm64.whl", hash = "sha256:789e144f6dc7019d1f92a812891c645274ed08af6037d11fc65fcbc183b7d59f"},
-    {file = "ruff-0.3.2.tar.gz", hash = "sha256:fa78ec9418eb1ca3db392811df3376b46471ae93792a81af2d1cbb0e5dcb5142"},
+    {file = "ruff-0.3.7.tar.gz", hash = "sha256:d5c1aebee5162c2226784800ae031f660c350e7a3402c4d1f8ea4e97e232e3ba"},
 ]
 [[package]]
@@ -2954,13 +2954,13 @@ docs = ["sphinx (<7.0.0)"]
 [[package]]
 name = "twine"
-version = "4.0.2"
+version = "5.0.0"
 description = "Collection of utilities for publishing packages on PyPI"
 optional = false
-python-versions = ">=3.7"
+python-versions = ">=3.8"
 files = [
-    {file = "twine-4.0.2-py3-none-any.whl", hash = "sha256:929bc3c280033347a00f847236564d1c52a3e61b1ac2516c97c48f3ceab756d8"},
+    {file = "twine-5.0.0-py3-none-any.whl", hash = "sha256:a262933de0b484c53408f9edae2e7821c1c45a3314ff2df9bdd343aa7ab8edc0"},
-    {file = "twine-4.0.2.tar.gz", hash = "sha256:9e102ef5fdd5a20661eb88fad46338806c3bd32cf1db729603fe3697b1bc83c8"},
+    {file = "twine-5.0.0.tar.gz", hash = "sha256:89b0cc7d370a4b66421cc6102f269aa910fe0f1861c124f573cf2ddedbc10cf4"},
 ]
 [package.dependencies]
@@ -3109,13 +3109,13 @@ files = [
 [[package]]
 name = "types-pillow"
-version = "10.2.0.20240125"
+version = "10.2.0.20240415"
 description = "Typing stubs for Pillow"
 optional = false
 python-versions = ">=3.8"
 files = [
-    {file = "types-Pillow-10.2.0.20240125.tar.gz", hash = "sha256:c449b2c43b9fdbe0494a7b950e6b39a4e50516091213fec24ef3f33c1d017717"},
+    {file = "types-Pillow-10.2.0.20240415.tar.gz", hash = "sha256:dd6058027639bcdc66ba78b228cc25fdae42524c2150c78c804da427e7e76e70"},
-    {file = "types_Pillow-10.2.0.20240125-py3-none-any.whl", hash = "sha256:322dbae32b4b7918da5e8a47c50ac0f24b0aa72a804a23857620f2722b03c858"},
+    {file = "types_Pillow-10.2.0.20240415-py3-none-any.whl", hash = "sha256:f933332b7e96010bae9b9cf82a4c9979ff0c270d63f5c5bbffb2d789b85cd00b"},
 ]
 [[package]]
@@ -3156,13 +3156,13 @@ files = [
 [[package]]
 name = "types-requests"
-version = "2.31.0.20240125"
+version = "2.31.0.20240406"
 description = "Typing stubs for requests"
 optional = false
 python-versions = ">=3.8"
 files = [
-    {file = "types-requests-2.31.0.20240125.tar.gz", hash = "sha256:03a28ce1d7cd54199148e043b2079cdded22d6795d19a2c2a6791a4b2b5e2eb5"},
+    {file = "types-requests-2.31.0.20240406.tar.gz", hash = "sha256:4428df33c5503945c74b3f42e82b181e86ec7b724620419a2966e2de604ce1a1"},
-    {file = "types_requests-2.31.0.20240125-py3-none-any.whl", hash = "sha256:9592a9a4cb92d6d75d9b491a41477272b710e021011a2a3061157e2fb1f1a5d1"},
+    {file = "types_requests-2.31.0.20240406-py3-none-any.whl", hash = "sha256:6216cdac377c6b9a040ac1c0404f7284bd13199c0e1bb235f4324627e8898cf5"},
 ]
 [package.dependencies]
@@ -3181,13 +3181,13 @@ files = [
 [[package]]
 name = "typing-extensions"
-version = "4.9.0"
+version = "4.11.0"
 description = "Backported and Experimental Type Hints for Python 3.8+"
 optional = false
 python-versions = ">=3.8"
 files = [
-    {file = "typing_extensions-4.9.0-py3-none-any.whl", hash = "sha256:af72aea155e91adfc61c3ae9e0e342dbc0cba726d6cba4b6c72c1f34e47291cd"},
+    {file = "typing_extensions-4.11.0-py3-none-any.whl", hash = "sha256:c1f94d72897edaf4ce775bb7558d5b79d8126906a14ea5ed1635921406c0387a"},
-    {file = "typing_extensions-4.9.0.tar.gz", hash = "sha256:23478f88c37f27d76ac8aee6c905017a143b0b1b886c3c9f66bc2fd94f9f5783"},
+    {file = "typing_extensions-4.11.0.tar.gz", hash = "sha256:83f085bd5ca59c80295fc2a82ab5dac679cbe02b9f33f7d83af68e241bea51b0"},
 ]
 [[package]]
@@ -3451,4 +3451,4 @@ user-search = ["pyicu"]
 [metadata]
 lock-version = "2.0"
 python-versions = "^3.8.0"
-content-hash = "b510fa05f4ea33194bec079f5d04ebb3f9ffbb5c1ea96a0341d57ba770ef81e6"
+content-hash = "1951f2b4623138d47db08a405edd970e67599d05804bb459af21a085e1665f69"
@@ -96,7 +96,7 @@ module-name = "synapse.synapse_rust"
 [tool.poetry]
 name = "matrix-synapse"
-version = "1.104.0"
+version = "1.105.0"
 description = "Homeserver for the Matrix decentralised comms protocol"
 authors = ["Matrix.org Team and Contributors <packages@matrix.org>"]
 license = "AGPL-3.0-or-later"
@@ -321,7 +321,7 @@ all = [
 # This helps prevents merge conflicts when running a batch of dependabot updates.
 isort = ">=5.10.1"
 black = ">=22.7.0"
-ruff = "0.3.2"
+ruff = "0.3.7"
 # Type checking only works with the pydantic.v1 compat module from pydantic v2
 pydantic = "^2"
@@ -7,7 +7,7 @@ name = "synapse"
 version = "0.1.0"
 edition = "2021"
-rust-version = "1.65.0"
+rust-version = "1.66.0"
 [lib]
 name = "synapse"
@@ -23,6 +23,9 @@ name = "synapse.synapse_rust"
 [dependencies]
 anyhow = "1.0.63"
 bytes = "1.6.0"
 headers = "0.4.0"
 http = "1.1.0"
 lazy_static = "1.4.0"
 log = "0.4.17"
 pyo3 = { version = "0.20.0", features = [
@@ -0,0 +1,60 @@
 /*
 * This file is licensed under the Affero General Public License (AGPL) version 3.
 *
 * Copyright (C) 2024 New Vector, Ltd
 *
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU Affero General Public License as
 * published by the Free Software Foundation, either version 3 of the
 * License, or (at your option) any later version.
 *
 * See the GNU Affero General Public License for more details:
 * <https://www.gnu.org/licenses/agpl-3.0.html>.
 */
 #![allow(clippy::new_ret_no_self)]
 use std::collections::HashMap;
 use http::{HeaderMap, StatusCode};
 use pyo3::{exceptions::PyValueError, import_exception};
 import_exception!(synapse.api.errors, SynapseError);
 impl SynapseError {
    pub fn new(
        code: StatusCode,
        message: String,
        errcode: &'static str,
        additional_fields: Option<HashMap<String, String>>,
        headers: Option<HeaderMap>,
    ) -> pyo3::PyErr {
        // Transform the HeaderMap into a HashMap<String, String>
        let headers = if let Some(headers) = headers {
            let mut map = HashMap::with_capacity(headers.len());
            for (key, value) in headers.iter() {
                let Ok(value) = value.to_str() else {
                    // This should never happen, but we don't want to panic in case it does
                    return PyValueError::new_err(
                        "Could not construct SynapseError: header value is not valid ASCII",
                    );
                };
                map.insert(key.as_str().to_owned(), value.to_owned());
            }
            Some(map)
        } else {
            None
        };
        SynapseError::new_err((code.as_u16(), message, errcode, additional_fields, headers))
    }
 }
 import_exception!(synapse.api.errors, NotFoundError);
 impl NotFoundError {
    pub fn new() -> pyo3::PyErr {
        NotFoundError::new_err(())
    }
 }
@@ -0,0 +1,165 @@
 /*
 * This file is licensed under the Affero General Public License (AGPL) version 3.
 *
 * Copyright (C) 2024 New Vector, Ltd
 *
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU Affero General Public License as
 * published by the Free Software Foundation, either version 3 of the
 * License, or (at your option) any later version.
 *
 * See the GNU Affero General Public License for more details:
 * <https://www.gnu.org/licenses/agpl-3.0.html>.
 */
 use bytes::{Buf, BufMut, Bytes, BytesMut};
 use headers::{Header, HeaderMapExt};
 use http::{HeaderName, HeaderValue, Method, Request, Response, StatusCode, Uri};
 use pyo3::{
    exceptions::PyValueError,
    types::{PyBytes, PySequence, PyTuple},
    PyAny, PyResult,
 };
 use crate::errors::SynapseError;
 /// Read a file-like Python object by chunks
 ///
 /// # Errors
 ///
 /// Returns an error if calling the `read` on the Python object failed
 fn read_io_body(body: &PyAny, chunk_size: usize) -> PyResult<Bytes> {
    let mut buf = BytesMut::new();
    loop {
        let bytes: &PyBytes = body.call_method1("read", (chunk_size,))?.downcast()?;
        if bytes.as_bytes().is_empty() {
            return Ok(buf.into());
        }
        buf.put(bytes.as_bytes());
    }
 }
 /// Transform a Twisted `IRequest` to an [`http::Request`]
 ///
 /// It uses the following members of `IRequest`:
 ///   - `content`, which is expected to be a file-like object with a `read` method
 ///   - `uri`, which is expected to be a valid URI as `bytes`
 ///   - `method`, which is expected to be a valid HTTP method as `bytes`
 ///   - `requestHeaders`, which is expected to have a `getAllRawHeaders` method
 ///
 /// # Errors
 ///
 /// Returns an error if the Python object doesn't properly implement `IRequest`
 pub fn http_request_from_twisted(request: &PyAny) -> PyResult<Request<Bytes>> {
    let content = request.getattr("content")?;
    let body = read_io_body(content, 4096)?;
    let mut req = Request::new(body);
    let uri: &PyBytes = request.getattr("uri")?.downcast()?;
    *req.uri_mut() =
        Uri::try_from(uri.as_bytes()).map_err(|_| PyValueError::new_err("invalid uri"))?;
    let method: &PyBytes = request.getattr("method")?.downcast()?;
    *req.method_mut() = Method::from_bytes(method.as_bytes())
        .map_err(|_| PyValueError::new_err("invalid method"))?;
    let headers_iter = request
        .getattr("requestHeaders")?
        .call_method0("getAllRawHeaders")?
        .iter()?;
    for header in headers_iter {
        let header = header?;
        let header: &PyTuple = header.downcast()?;
        let name: &PyBytes = header.get_item(0)?.downcast()?;
        let name = HeaderName::from_bytes(name.as_bytes())
            .map_err(|_| PyValueError::new_err("invalid header name"))?;
        let values: &PySequence = header.get_item(1)?.downcast()?;
        for index in 0..values.len()? {
            let value: &PyBytes = values.get_item(index)?.downcast()?;
            let value = HeaderValue::from_bytes(value.as_bytes())
                .map_err(|_| PyValueError::new_err("invalid header value"))?;
            req.headers_mut().append(name.clone(), value);
        }
    }
    Ok(req)
 }
 /// Send an [`http::Response`] through a Twisted `IRequest`
 ///
 /// It uses the following members of `IRequest`:
 ///
 ///  - `responseHeaders`, which is expected to have a `addRawHeader(bytes, bytes)` method
 ///  - `setResponseCode(int)` method
 ///  - `write(bytes)` method
 ///  - `finish()` method
 ///
 ///  # Errors
 ///
 /// Returns an error if the Python object doesn't properly implement `IRequest`
 pub fn http_response_to_twisted<B>(request: &PyAny, response: Response<B>) -> PyResult<()>
 where
    B: Buf,
 {
    let (parts, mut body) = response.into_parts();
    request.call_method1("setResponseCode", (parts.status.as_u16(),))?;
    let response_headers = request.getattr("responseHeaders")?;
    for (name, value) in parts.headers.iter() {
        response_headers.call_method1("addRawHeader", (name.as_str(), value.as_bytes()))?;
    }
    while body.remaining() != 0 {
        let chunk = body.chunk();
        request.call_method1("write", (chunk,))?;
        body.advance(chunk.len());
    }
    request.call_method0("finish")?;
    Ok(())
 }
 /// An extension trait for [`HeaderMap`] that provides typed access to headers, and throws the
 /// right python exceptions when the header is missing or fails to parse.
 ///
 /// [`HeaderMap`]: headers::HeaderMap
 pub trait HeaderMapPyExt: HeaderMapExt {
    /// Get a header from the map, returning an error if it is missing or invalid.
    fn typed_get_required<H>(&self) -> PyResult<H>
    where
        H: Header,
    {
        self.typed_get_optional::<H>()?.ok_or_else(|| {
            SynapseError::new(
                StatusCode::BAD_REQUEST,
                format!("Missing required header: {}", H::name()),
                "M_MISSING_PARAM",
                None,
                None,
            )
        })
    }
    /// Get a header from the map, returning `None` if it is missing and an error if it is invalid.
    fn typed_get_optional<H>(&self) -> PyResult<Option<H>>
    where
        H: Header,
    {
        self.typed_try_get::<H>().map_err(|_| {
            SynapseError::new(
                StatusCode::BAD_REQUEST,
                format!("Invalid header: {}", H::name()),
                "M_INVALID_PARAM",
                None,
                None,
            )
        })
    }
 }
 impl<T: HeaderMapExt> HeaderMapPyExt for T {}
@@ -3,7 +3,9 @@ use pyo3::prelude::*;
 use pyo3_log::ResetHandle;
 pub mod acl;
 pub mod errors;
 pub mod events;
 pub mod http;
 pub mod push;
 lazy_static! {
@@ -304,12 +304,12 @@ pub const BASE_APPEND_UNDERRIDE_RULES: &[PushRule] = &[
        default_enabled: true,
    },
    PushRule {
-        rule_id: Cow::Borrowed("global/underride/.m.rule.room_one_to_one"),
+        rule_id: Cow::Borrowed("global/underride/.m.rule.encrypted_room_one_to_one"),
        priority_class: 1,
        conditions: Cow::Borrowed(&[
            Condition::Known(KnownCondition::EventMatch(EventMatchCondition {
                key: Cow::Borrowed("type"),
-                pattern: Cow::Borrowed("m.room.message"),
+                pattern: Cow::Borrowed("m.room.encrypted"),
            })),
            Condition::Known(KnownCondition::RoomMemberCount {
                is: Some(Cow::Borrowed("2")),
@@ -320,12 +320,12 @@ pub const BASE_APPEND_UNDERRIDE_RULES: &[PushRule] = &[
        default_enabled: true,
    },
    PushRule {
-        rule_id: Cow::Borrowed("global/underride/.m.rule.encrypted_room_one_to_one"),
+        rule_id: Cow::Borrowed("global/underride/.m.rule.room_one_to_one"),
        priority_class: 1,
        conditions: Cow::Borrowed(&[
            Condition::Known(KnownCondition::EventMatch(EventMatchCondition {
                key: Cow::Borrowed("type"),
-                pattern: Cow::Borrowed("m.room.encrypted"),
+                pattern: Cow::Borrowed("m.room.message"),
            })),
            Condition::Known(KnownCondition::RoomMemberCount {
                is: Some(Cow::Borrowed("2")),
@@ -214,7 +214,7 @@ fi
 extra_test_args=()
-test_packages="./tests/csapi ./tests ./tests/msc3874 ./tests/msc3890 ./tests/msc3391 ./tests/msc3930 ./tests/msc3902"
+test_packages="./tests/csapi ./tests ./tests/msc3874 ./tests/msc3890 ./tests/msc3391 ./tests/msc3930 ./tests/msc3902 ./tests/msc3967"
 # Enable dirty runs, so tests will reuse the same container where possible.
 # This significantly speeds up tests, but increases the possibility of test pollution.
@@ -234,6 +234,13 @@ class EventContentFields:
    TO_DEVICE_MSGID: Final = "org.matrix.msgid"
 class EventUnsignedContentFields:
    """Fields found inside the 'unsigned' data on events"""
    # Requesting user's membership, per MSC4115
    MSC4115_MEMBERSHIP: Final = "io.element.msc4115.membership"
 class RoomTypes:
    """Understood values of the room_type field of m.room.create events."""
@@ -393,11 +393,6 @@ class ExperimentalConfig(Config):
        # MSC3967: Do not require UIA when first uploading cross signing keys
        self.msc3967_enabled = experimental.get("msc3967_enabled", False)
        # MSC3981: Recurse relations
        self.msc3981_recurse_relations = experimental.get(
            "msc3981_recurse_relations", False
        )
        # MSC3861: Matrix architecture change to delegate authentication via OIDC
        try:
            self.msc3861 = MSC3861(**experimental.get("msc3861", {}))
@@ -409,11 +404,6 @@ class ExperimentalConfig(Config):
        # Check that none of the other config options conflict with MSC3861 when enabled
        self.msc3861.check_config_conflicts(self.root)
        # MSC4010: Do not allow setting m.push_rules account data.
        self.msc4010_push_rules_account_data = experimental.get(
            "msc4010_push_rules_account_data", False
        )
        self.msc4028_push_encrypted_events = experimental.get(
            "msc4028_push_encrypted_events", False
        )
@@ -421,3 +411,18 @@ class ExperimentalConfig(Config):
        self.msc4069_profile_inhibit_propagation = experimental.get(
            "msc4069_profile_inhibit_propagation", False
        )
        # MSC4108: Mechanism to allow OIDC sign in and E2EE set up via QR code
        self.msc4108_delegation_endpoint: Optional[str] = experimental.get(
            "msc4108_delegation_endpoint", None
        )
        if self.msc4108_delegation_endpoint is not None and not self.msc3861.enabled:
            raise ConfigError(
                "MSC4108 requires MSC3861 to be enabled",
                ("experimental", "msc4108_delegation_endpoint"),
            )
        self.msc4115_membership_on_events = experimental.get(
            "msc4115_membership_on_events", False
        )
@@ -49,7 +49,7 @@ from synapse.api.errors import Codes, SynapseError
 from synapse.api.room_versions import RoomVersion
 from synapse.types import JsonDict, Requester
-from . import EventBase
+from . import EventBase, make_event_from_dict
 if TYPE_CHECKING:
    from synapse.handlers.relations import BundledAggregations
@@ -82,8 +82,6 @@ def prune_event(event: EventBase) -> EventBase:
    """
    pruned_event_dict = prune_event_dict(event.room_version, event.get_dict())
    from . import make_event_from_dict
    pruned_event = make_event_from_dict(
        pruned_event_dict, event.room_version, event.internal_metadata.get_dict()
    )
@@ -101,6 +99,25 @@ def prune_event(event: EventBase) -> EventBase:
    return pruned_event
 def clone_event(event: EventBase) -> EventBase:
    """Take a copy of the event.
    This is mostly useful because it does a *shallow* copy of the `unsigned` data,
    which means it can then be updated without corrupting the in-memory cache.
    """
    new_event = make_event_from_dict(
        event.get_dict(), event.room_version, event.internal_metadata.get_dict()
    )
    # copy the internal fields
    new_event.internal_metadata.stream_ordering = (
        event.internal_metadata.stream_ordering
    )
    new_event.internal_metadata.outlier = event.internal_metadata.outlier
    return new_event
 def prune_event_dict(room_version: RoomVersion, event_dict: JsonDict) -> JsonDict:
    """Redacts the event_dict in the same way as `prune_event`, except it
    operates on dicts rather than event objects
@@ -42,6 +42,7 @@ class AdminHandler:
        self._device_handler = hs.get_device_handler()
        self._storage_controllers = hs.get_storage_controllers()
        self._state_storage_controller = self._storage_controllers.state
        self._hs_config = hs.config
        self._msc3866_enabled = hs.config.experimental.msc3866.enabled
    async def get_whois(self, user: UserID) -> JsonMapping:
@@ -217,7 +218,10 @@ class AdminHandler:
                )
                events = await filter_events_for_client(
-                    self._storage_controllers, user_id, events
+                    self._storage_controllers,
                    user_id,
                    events,
                    msc4115_membership_on_events=self._hs_config.experimental.msc4115_membership_on_events,
                )
                writer.write_events(room_id, events)
@@ -1476,6 +1476,42 @@ class E2eKeysHandler:
        else:
            return exists, self.clock.time_msec() < ts_replacable_without_uia_before
    async def has_different_keys(self, user_id: str, body: JsonDict) -> bool:
        """
        Check if a key provided in `body` differs from the same key stored in the DB. Returns
        true on the first difference. If a key exists in `body` but does not exist in the DB,
        returns True. If `body` has no keys, this always returns False.
        Note by 'key' we mean Matrix key rather than JSON key.
        The purpose of this function is to detect whether or not we need to apply UIA checks.
        We must apply UIA checks if any key in the database is being overwritten. If a key is
        being inserted for the first time, or if the key exactly matches what is in the database,
        then no UIA check needs to be performed.
        Args:
            user_id: The user who sent the `body`.
            body: The JSON request body from POST /keys/device_signing/upload
        Returns:
            True if any key in `body` has a different value in the database.
        """
        # Ensure that each key provided in the request body exactly matches the one we have stored.
        # The first time we see the DB having a different key to the matching request key, bail.
        # Note: we do not care if the DB has a key which the request does not specify, as we only
        # care about *replacements* or *insertions* (i.e UPSERT)
        req_body_key_to_db_key = {
            "master_key": "master",
            "self_signing_key": "self_signing",
            "user_signing_key": "user_signing",
        }
        for req_body_key, db_key in req_body_key_to_db_key.items():
            if req_body_key in body:
                existing_key = await self.store.get_e2e_cross_signing_key(
                    user_id, db_key
                )
                if existing_key != body[req_body_key]:
                    return True
        return False
 def _check_cross_signing_key(
    key: JsonDict, user_id: str, key_type: str, signing_key: Optional[VerifyKey] = None
@@ -148,6 +148,7 @@ class EventHandler:
    def __init__(self, hs: "HomeServer"):
        self.store = hs.get_datastores().main
        self._storage_controllers = hs.get_storage_controllers()
        self._config = hs.config
    async def get_event(
        self,
@@ -189,7 +190,11 @@ class EventHandler:
        is_peeking = not is_user_in_room
        filtered = await filter_events_for_client(
-            self._storage_controllers, user.to_string(), [event], is_peeking=is_peeking
+            self._storage_controllers,
            user.to_string(),
            [event],
            is_peeking=is_peeking,
            msc4115_membership_on_events=self._config.experimental.msc4115_membership_on_events,
        )
        if not filtered:
@@ -221,7 +221,10 @@ class InitialSyncHandler:
                ).addErrback(unwrapFirstError)
                messages = await filter_events_for_client(
-                    self._storage_controllers, user_id, messages
+                    self._storage_controllers,
                    user_id,
                    messages,
                    msc4115_membership_on_events=self.hs.config.experimental.msc4115_membership_on_events,
                )
                start_token = now_token.copy_and_replace(StreamKeyType.ROOM, token)
@@ -380,6 +383,7 @@ class InitialSyncHandler:
            requester.user.to_string(),
            messages,
            is_peeking=is_peeking,
            msc4115_membership_on_events=self.hs.config.experimental.msc4115_membership_on_events,
        )
        start_token = StreamToken.START.copy_and_replace(StreamKeyType.ROOM, token)
@@ -494,6 +498,7 @@ class InitialSyncHandler:
            requester.user.to_string(),
            messages,
            is_peeking=is_peeking,
            msc4115_membership_on_events=self.hs.config.experimental.msc4115_membership_on_events,
        )
        start_token = now_token.copy_and_replace(StreamKeyType.ROOM, token)
@@ -623,6 +623,7 @@ class PaginationHandler:
                user_id,
                events,
                is_peeking=(member_event_id is None),
                msc4115_membership_on_events=self.hs.config.experimental.msc4115_membership_on_events,
            )
        # if after the filter applied there are no more events
@@ -95,6 +95,7 @@ class RelationsHandler:
        self._event_handler = hs.get_event_handler()
        self._event_serializer = hs.get_event_client_serializer()
        self._event_creation_handler = hs.get_event_creation_handler()
        self._config = hs.config
    async def get_relations(
        self,
@@ -163,6 +164,7 @@ class RelationsHandler:
            user_id,
            events,
            is_peeking=(member_event_id is None),
            msc4115_membership_on_events=self._config.experimental.msc4115_membership_on_events,
        )
        # The relations returned for the requested event do include their
@@ -608,6 +610,7 @@ class RelationsHandler:
            user_id,
            events,
            is_peeking=(member_event_id is None),
            msc4115_membership_on_events=self._config.experimental.msc4115_membership_on_events,
        )
        aggregations = await self.get_bundled_aggregations(
@@ -956,6 +956,7 @@ class RoomCreationHandler:
            room_alias=room_alias,
            power_level_content_override=power_level_content_override,
            creator_join_profile=creator_join_profile,
            ignore_forced_encryption=ignore_forced_encryption,
        )
        # we avoid dropping the lock between invites, as otherwise joins can
@@ -1475,6 +1476,7 @@ class RoomContextHandler:
                user.to_string(),
                events,
                is_peeking=is_peeking,
                msc4115_membership_on_events=self.hs.config.experimental.msc4115_membership_on_events,
            )
        event = await self.store.get_event(
@@ -480,7 +480,10 @@ class SearchHandler:
        filtered_events = await search_filter.filter([r["event"] for r in results])
        events = await filter_events_for_client(
-            self._storage_controllers, user.to_string(), filtered_events
+            self._storage_controllers,
            user.to_string(),
            filtered_events,
            msc4115_membership_on_events=self.hs.config.experimental.msc4115_membership_on_events,
        )
        events.sort(key=lambda e: -rank_map[e.event_id])
@@ -579,7 +582,10 @@ class SearchHandler:
            filtered_events = await search_filter.filter([r["event"] for r in results])
            events = await filter_events_for_client(
-                self._storage_controllers, user.to_string(), filtered_events
+                self._storage_controllers,
                user.to_string(),
                filtered_events,
                msc4115_membership_on_events=self.hs.config.experimental.msc4115_membership_on_events,
            )
            room_events.extend(events)
@@ -664,11 +670,17 @@ class SearchHandler:
            )
            events_before = await filter_events_for_client(
-                self._storage_controllers, user.to_string(), res.events_before
+                self._storage_controllers,
                user.to_string(),
                res.events_before,
                msc4115_membership_on_events=self.hs.config.experimental.msc4115_membership_on_events,
            )
            events_after = await filter_events_for_client(
-                self._storage_controllers, user.to_string(), res.events_after
+                self._storage_controllers,
                user.to_string(),
                res.events_after,
                msc4115_membership_on_events=self.hs.config.experimental.msc4115_membership_on_events,
            )
            context: JsonDict = {
@@ -88,6 +88,10 @@ if TYPE_CHECKING:
 logger = logging.getLogger(__name__)
 # Logging for https://github.com/matrix-org/matrix-spec/issues/1209 and
 # https://github.com/element-hq/synapse/issues/16940
 client_state_desync_logger = logging.getLogger("synapse.client_state_desync_debug")
 # Counts the number of times we returned a non-empty sync. `type` is one of
 # "initial_sync", "full_state_sync" or "incremental_sync", `lazy_loaded` is
 # "true" or "false" depending on if the request asked for lazy loaded members or
@@ -596,6 +600,7 @@ class SyncHandler:
                    sync_config.user.to_string(),
                    recents,
                    always_include_ids=current_state_ids,
                    msc4115_membership_on_events=self.hs_config.experimental.msc4115_membership_on_events,
                )
                log_kv({"recents_after_visibility_filtering": len(recents)})
            else:
@@ -681,6 +686,7 @@ class SyncHandler:
                    sync_config.user.to_string(),
                    loaded_recents,
                    always_include_ids=current_state_ids,
                    msc4115_membership_on_events=self.hs_config.experimental.msc4115_membership_on_events,
                )
                loaded_recents = []
@@ -1214,6 +1220,12 @@ class SyncHandler:
            previous_timeline_end={},
            lazy_load_members=lazy_load_members,
        )
        if client_state_desync_logger.isEnabledFor(logging.DEBUG):
            await self._log_client_state_desync(
                room_id, None, state_ids, timeline_state, lazy_load_members
            )
        return state_ids
    async def _compute_state_delta_for_incremental_sync(
@@ -1259,6 +1271,51 @@ class SyncHandler:
            await_full_state = True
            lazy_load_members = False
        # For a non-gappy sync if the events in the timeline are simply a linear
        # chain (i.e. no merging/branching of the graph), then we know the state
        # delta between the end of the previous sync and start of the new one is
        # empty.
        #
        # c.f. #16941 for an example of why we can't do this for all non-gappy
        # syncs.
        is_linear_timeline = True
        if batch.events:
            # We need to make sure the first event in our batch points to the
            # last event in the previous batch.
            last_event_id_prev_batch = (
                await self.store.get_last_event_in_room_before_stream_ordering(
                    room_id,
                    end_token=since_token.room_key,
                )
            )
            prev_event_id = last_event_id_prev_batch
            for e in batch.events:
                if e.prev_event_ids() != [prev_event_id]:
                    is_linear_timeline = False
                    break
                prev_event_id = e.event_id
        if is_linear_timeline and not batch.limited:
            state_ids: StateMap[str] = {}
            if lazy_load_members:
                if members_to_fetch and batch.events:
                    # We're lazy-loading, so the client might need some more
                    # member events to understand the events in this timeline.
                    # So we fish out all the member events corresponding to the
                    # timeline here. The caller will then dedupe any redundant
                    # ones.
                    state_ids = await self._state_storage_controller.get_state_ids_for_event(
                        batch.events[0].event_id,
                        # we only want members!
                        state_filter=StateFilter.from_types(
                            (EventTypes.Member, member) for member in members_to_fetch
                        ),
                        await_full_state=False,
                    )
            return state_ids
        if batch:
            state_at_timeline_start = (
                await self._state_storage_controller.get_state_ids_for_event(
@@ -1314,6 +1371,15 @@ class SyncHandler:
            lazy_load_members=lazy_load_members,
        )
        if client_state_desync_logger.isEnabledFor(logging.DEBUG):
            await self._log_client_state_desync(
                room_id,
                since_token,
                state_ids,
                timeline_state,
                lazy_load_members,
            )
        return state_ids
    async def _find_missing_partial_state_memberships(
@@ -1430,6 +1496,125 @@ class SyncHandler:
        return additional_state_ids
    async def _log_client_state_desync(
        self,
        room_id: str,
        since_token: Optional[StreamToken],
        sync_response_state_state: StateMap[str],
        sync_response_timeline_state: StateMap[str],
        lazy_load_members: bool,
    ) -> None:
        """
        Logging to see how often the client's state gets out of sync with the
        actual current state of the room.
        There are few different potential failure modes here:
         * State resolution can cause changes in the state of the room that don't
           directly correspond to events with the corresponding (type, state_key).
           https://github.com/matrix-org/matrix-spec/issues/1209 discusses this in
           more detail.
         * Even where there is an event that causes a given state change, Synapse
           may not serve it to the client, since it works on state at specific points
           in the DAG, rather than "current state".
           See https://github.com/element-hq/synapse/issues/16940.
         * Lazy-loading adds more complexity, as it means that events that would
           normally be served via the `state` part of an incremental sync are filtered
           out.
        To try to get a handle on this, let's put ourselves in the shoes of a client,
        and compare the state they will calculate against the actual current state.
        """
        # We only care about membership events.
        state_filter = StateFilter.from_types(types=(("m.room.member", None),))
        if since_token is None:
            if lazy_load_members:
                # For initial syncs with lazy-loading enabled, there's not too much
                # concern here. We know the client will do a `/members` query before
                # doing any encryption, so what sync returns isn't too important.
                #
                # (Of course, then `/members` might also return an incomplete list, but
                # that's a separate problem.)
                return
            # For regular initial syncs, compare the returned response with the actual
            # current state.
            client_calculated_state = {}
            client_calculated_state.update(sync_response_state_state)
            client_calculated_state.update(sync_response_timeline_state)
        else:
            # For an incremental (gappy or otherwise) sync, let's assume the client has
            # a complete membership list as of the last sync (or rather, at
            # `since_token`, which is the closest approximation we have to it
            # right now), and see what they would calculate as the current state given
            # this sync update.
            client_calculated_state = dict(
                await self.get_state_at(
                    room_id,
                    stream_position=since_token,
                    state_filter=state_filter,
                    await_full_state=False,
                )
            )
            client_calculated_state.update(sync_response_state_state)
            client_calculated_state.update(sync_response_timeline_state)
        current_state = await self._state_storage_controller.get_current_state_ids(
            room_id, state_filter=state_filter, await_full_state=False
        )
        missing_users = await self._calculate_missing_members(
            current_state, client_calculated_state
        )
        if missing_users:
            client_state_desync_logger.debug(
                "client state discrepancy in incremental sync in room %s: missing users %s",
                room_id,
                missing_users,
            )
    async def _calculate_missing_members(
        self,
        actual_state: StateMap[str],
        client_calculated_state: StateMap[str],
    ) -> List[str]:
        """Helper for `_log_client_state_desync`: calculates the difference in
        joined members between two state maps.
        Returns:
            A list of user IDs
        """
        missing_users = []
        async def event_id_to_membership(event_id: Optional[str]) -> Optional[str]:
            if event_id is None:
                return None
            event = await self.store.get_event(event_id, allow_none=True)
            if event is None:
                return "MISSING_EVENT"
            return event.membership
        # Check for joined members in the actual state that are missing or have a
        # different membership in the actual state.
        for (event_type, state_key), actual_event_id in actual_state.items():
            if event_type != EventTypes.Member:
                continue
            calculated_event_id = client_calculated_state.get((event_type, state_key))
            if calculated_event_id != actual_event_id:
                actual_membership = event_id_to_membership(actual_event_id)
                calculated_membership = event_id_to_membership(calculated_event_id)
                if (
                    actual_membership == Membership.JOIN
                    and calculated_membership != Membership.JOIN
                ):
                    missing_users.append(state_key)
        return missing_users
    async def unread_notifs_for_room_id(
        self, room_id: str, sync_config: SyncConfig
    ) -> RoomNotifCounts:
@@ -262,7 +262,8 @@ class _ProxyResponseBody(protocol.Protocol):
            self._request.finish()
        else:
            # Abort the underlying request since our remote request also failed.
-            self._request.transport.abortConnection()
+            if self._request.channel:
                self._request.channel.forceAbortClient()
 class ProxySite(Site):
@@ -153,9 +153,9 @@ def return_json_error(
    # Only respond with an error response if we haven't already started writing,
    # otherwise lets just kill the connection
    if request.startedWriting:
-        if request.transport:
+        if request.channel:
            try:
-                request.transport.abortConnection()
+                request.channel.forceAbortClient()
            except Exception:
                # abortConnection throws if the connection is already closed
                pass
@@ -909,7 +909,18 @@ def set_cors_headers(request: "SynapseRequest") -> None:
    request.setHeader(
        b"Access-Control-Allow-Methods", b"GET, HEAD, POST, PUT, DELETE, OPTIONS"
    )
-    if request.experimental_cors_msc3886:
+    if request.path is not None and request.path.startswith(
        b"/_matrix/client/unstable/org.matrix.msc4108/rendezvous"
    ):
        request.setHeader(
            b"Access-Control-Allow-Headers",
            b"Content-Type, If-Match, If-None-Match",
        )
        request.setHeader(
            b"Access-Control-Expose-Headers",
            b"Synapse-Trace-Id, Server, ETag",
        )
    elif request.experimental_cors_msc3886:
        request.setHeader(
            b"Access-Control-Allow-Headers",
            b"X-Requested-With, Content-Type, Authorization, Date, If-Match, If-None-Match",
@@ -19,9 +19,11 @@
 #
 #
-""" This module contains base REST classes for constructing REST servlets. """
+"""This module contains base REST classes for constructing REST servlets."""
 import enum
 import logging
 import urllib.parse as urlparse
 from http import HTTPStatus
 from typing import (
    TYPE_CHECKING,
@@ -65,17 +67,49 @@ def parse_integer(request: Request, name: str, default: int) -> int: ...
@overload
-def parse_integer(request: Request, name: str, *, required: Literal[True]) -> int: ...
+def parse_integer(
    request: Request, name: str, *, default: int, negative: bool
 ) -> int: ...
@overload
 def parse_integer(
-    request: Request, name: str, default: Optional[int] = None, required: bool = False
+    request: Request, name: str, *, default: int, negative: bool = False
 ) -> int: ...
@overload
 def parse_integer(
    request: Request, name: str, *, required: Literal[True], negative: bool = False
 ) -> int: ...
@overload
 def parse_integer(
    request: Request, name: str, *, default: Literal[None], negative: bool = False
 ) -> None: ...
@overload
 def parse_integer(request: Request, name: str, *, negative: bool) -> Optional[int]: ...
@overload
 def parse_integer(
    request: Request,
    name: str,
    default: Optional[int] = None,
    required: bool = False,
    negative: bool = False,
 ) -> Optional[int]: ...
 def parse_integer(
-    request: Request, name: str, default: Optional[int] = None, required: bool = False
+    request: Request,
    name: str,
    default: Optional[int] = None,
    required: bool = False,
    negative: bool = False,
 ) -> Optional[int]:
    """Parse an integer parameter from the request string
@@ -85,16 +119,17 @@ def parse_integer(
        default: value to use if the parameter is absent, defaults to None.
        required: whether to raise a 400 SynapseError if the parameter is absent,
            defaults to False.
-
+        negative: whether to allow negative integers, defaults to True.
    Returns:
        An int value or the default.
    Raises:
-        SynapseError: if the parameter is absent and required, or if the
+        SynapseError: if the parameter is absent and required, if the
-            parameter is present and not an integer.
+            parameter is present and not an integer, or if the
            parameter is illegitimate negative.
    """
    args: Mapping[bytes, Sequence[bytes]] = request.args  # type: ignore
-    return parse_integer_from_args(args, name, default, required)
+    return parse_integer_from_args(args, name, default, required, negative)
@overload
@@ -120,6 +155,7 @@ def parse_integer_from_args(
    name: str,
    default: Optional[int] = None,
    required: bool = False,
    negative: bool = False,
 ) -> Optional[int]: ...
@@ -128,6 +164,7 @@ def parse_integer_from_args(
    name: str,
    default: Optional[int] = None,
    required: bool = False,
    negative: bool = True,
 ) -> Optional[int]:
    """Parse an integer parameter from the request string
@@ -137,33 +174,37 @@ def parse_integer_from_args(
        default: value to use if the parameter is absent, defaults to None.
        required: whether to raise a 400 SynapseError if the parameter is absent,
            defaults to False.
        negative: whether to allow negative integers, defaults to True.
    Returns:
        An int value or the default.
    Raises:
-        SynapseError: if the parameter is absent and required, or if the
+        SynapseError: if the parameter is absent and required, if the
-            parameter is present and not an integer.
+            parameter is present and not an integer, or if the
            parameter is illegitimate negative.
    """
    name_bytes = name.encode("ascii")
-    if name_bytes in args:
+    if name_bytes not in args:
-        try:
+        if not required:
            return int(args[name_bytes][0])
        except Exception:
            message = "Query parameter %r must be an integer" % (name,)
            raise SynapseError(
                HTTPStatus.BAD_REQUEST, message, errcode=Codes.INVALID_PARAM
            )
    else:
        if required:
            message = "Missing integer query parameter %r" % (name,)
            raise SynapseError(
                HTTPStatus.BAD_REQUEST, message, errcode=Codes.MISSING_PARAM
            )
        else:
            return default
        message = f"Missing required integer query parameter {name}"
        raise SynapseError(HTTPStatus.BAD_REQUEST, message, errcode=Codes.MISSING_PARAM)
    try:
        integer = int(args[name_bytes][0])
    except Exception:
        message = f"Query parameter {name} must be an integer"
        raise SynapseError(HTTPStatus.BAD_REQUEST, message, errcode=Codes.INVALID_PARAM)
    if not negative and integer < 0:
        message = f"Query parameter {name} must be a positive integer."
        raise SynapseError(HTTPStatus.BAD_REQUEST, message, errcode=Codes.INVALID_PARAM)
    return integer
@overload
 def parse_boolean(request: Request, name: str, default: bool) -> bool: ...
@@ -410,6 +451,87 @@ def parse_string(
    )
 def parse_json(
    request: Request,
    name: str,
    default: Optional[dict] = None,
    required: bool = False,
    encoding: str = "ascii",
 ) -> Optional[JsonDict]:
    """
    Parse a JSON parameter from the request query string.
    Args:
        request: the twisted HTTP request.
        name: the name of the query parameter.
        default: value to use if the parameter is absent,
            defaults to None.
        required: whether to raise a 400 SynapseError if the
           parameter is absent, defaults to False.
        encoding: The encoding to decode the string content with.
    Returns:
        A JSON value, or `default` if the named query parameter was not found
        and `required` was False.
    Raises:
        SynapseError if the parameter is absent and required, or if the
            parameter is present and not a JSON object.
    """
    args: Mapping[bytes, Sequence[bytes]] = request.args  # type: ignore
    return parse_json_from_args(
        args,
        name,
        default,
        required=required,
        encoding=encoding,
    )
 def parse_json_from_args(
    args: Mapping[bytes, Sequence[bytes]],
    name: str,
    default: Optional[dict] = None,
    required: bool = False,
    encoding: str = "ascii",
 ) -> Optional[JsonDict]:
    """
    Parse a JSON parameter from the request query string.
    Args:
        args: a mapping of request args as bytes to a list of bytes (e.g. request.args).
        name: the name of the query parameter.
        default: value to use if the parameter is absent,
            defaults to None.
        required: whether to raise a 400 SynapseError if the
            parameter is absent, defaults to False.
        encoding: the encoding to decode the string content with.
        A JSON value, or `default` if the named query parameter was not found
        and `required` was False.
    Raises:
        SynapseError if the parameter is absent and required, or if the
            parameter is present and not a JSON object.
    """
    name_bytes = name.encode("ascii")
    if name_bytes not in args:
        if not required:
            return default
        message = f"Missing required integer query parameter {name}"
        raise SynapseError(HTTPStatus.BAD_REQUEST, message, errcode=Codes.MISSING_PARAM)
    json_str = parse_string_from_args(args, name, required=True, encoding=encoding)
    try:
        return json_decoder.decode(urlparse.unquote(json_str))
    except Exception:
        message = f"Query parameter {name} must be a valid JSON object"
        raise SynapseError(HTTPStatus.BAD_REQUEST, message, errcode=Codes.NOT_JSON)
 EnumT = TypeVar("EnumT", bound=enum.Enum)
@@ -150,7 +150,8 @@ class SynapseRequest(Request):
                self.get_method(),
                self.get_redacted_uri(),
            )
-            self.transport.abortConnection()
+            if self.channel:
                self.channel.forceAbortClient()
            return
        super().handleContentChunk(data)
@@ -721,6 +721,7 @@ class Notifier:
                        user.to_string(),
                        new_events,
                        is_peeking=is_peeking,
                        msc4115_membership_on_events=self.hs.config.experimental.msc4115_membership_on_events,
                    )
                elif keyname == StreamKeyType.PRESENCE:
                    now = self.clock.time_msec()
@@ -513,7 +513,10 @@ class Mailer:
        }
        the_events = await filter_events_for_client(
-            self._storage_controllers, user_id, results.events_before
+            self._storage_controllers,
            user_id,
            results.events_before,
            msc4115_membership_on_events=self.hs.config.experimental.msc4115_membership_on_events,
        )
        the_events.append(notif_event)
@@ -23,7 +23,7 @@ from http import HTTPStatus
 from typing import TYPE_CHECKING, Tuple
 from synapse.api.constants import Direction
-from synapse.api.errors import Codes, NotFoundError, SynapseError
+from synapse.api.errors import NotFoundError, SynapseError
 from synapse.federation.transport.server import Authenticator
 from synapse.http.servlet import RestServlet, parse_enum, parse_integer, parse_string
 from synapse.http.site import SynapseRequest
@@ -61,22 +61,8 @@ class ListDestinationsRestServlet(RestServlet):
    async def on_GET(self, request: SynapseRequest) -> Tuple[int, JsonDict]:
        await assert_requester_is_admin(self._auth, request)
-        start = parse_integer(request, "from", default=0)
+        start = parse_integer(request, "from", default=0, negative=False)
-        limit = parse_integer(request, "limit", default=100)
+        limit = parse_integer(request, "limit", default=100, negative=False)
        if start < 0:
            raise SynapseError(
                HTTPStatus.BAD_REQUEST,
                "Query parameter from must be a string representing a positive integer.",
                errcode=Codes.INVALID_PARAM,
            )
        if limit < 0:
            raise SynapseError(
                HTTPStatus.BAD_REQUEST,
                "Query parameter limit must be a string representing a positive integer.",
                errcode=Codes.INVALID_PARAM,
            )
        destination = parse_string(request, "destination")
@@ -195,22 +181,8 @@ class DestinationMembershipRestServlet(RestServlet):
        if not await self._store.is_destination_known(destination):
            raise NotFoundError("Unknown destination")
-        start = parse_integer(request, "from", default=0)
+        start = parse_integer(request, "from", default=0, negative=False)
-        limit = parse_integer(request, "limit", default=100)
+        limit = parse_integer(request, "limit", default=100, negative=False)
        if start < 0:
            raise SynapseError(
                HTTPStatus.BAD_REQUEST,
                "Query parameter from must be a string representing a positive integer.",
                errcode=Codes.INVALID_PARAM,
            )
        if limit < 0:
            raise SynapseError(
                HTTPStatus.BAD_REQUEST,
                "Query parameter limit must be a string representing a positive integer.",
                errcode=Codes.INVALID_PARAM,
            )
        direction = parse_enum(request, "dir", Direction, default=Direction.FORWARDS)
@@ -311,29 +311,17 @@ class DeleteMediaByDateSize(RestServlet):
    ) -> Tuple[int, JsonDict]:
        await assert_requester_is_admin(self.auth, request)
-        before_ts = parse_integer(request, "before_ts", required=True)
+        before_ts = parse_integer(request, "before_ts", required=True, negative=False)
-        size_gt = parse_integer(request, "size_gt", default=0)
+        size_gt = parse_integer(request, "size_gt", default=0, negative=False)
        keep_profiles = parse_boolean(request, "keep_profiles", default=True)
-        if before_ts < 0:
+        if before_ts < 30000000000:  # Dec 1970 in milliseconds, Aug 2920 in seconds
            raise SynapseError(
                HTTPStatus.BAD_REQUEST,
                "Query parameter before_ts must be a positive integer.",
                errcode=Codes.INVALID_PARAM,
            )
        elif before_ts < 30000000000:  # Dec 1970 in milliseconds, Aug 2920 in seconds
            raise SynapseError(
                HTTPStatus.BAD_REQUEST,
                "Query parameter before_ts you provided is from the year 1970. "
                + "Double check that you are providing a timestamp in milliseconds.",
                errcode=Codes.INVALID_PARAM,
            )
        if size_gt < 0:
            raise SynapseError(
                HTTPStatus.BAD_REQUEST,
                "Query parameter size_gt must be a string representing a positive integer.",
                errcode=Codes.INVALID_PARAM,
            )
        # This check is useless, we keep it for the legacy endpoint only.
        if server_name is not None and self.server_name != server_name:
@@ -389,22 +377,8 @@ class UserMediaRestServlet(RestServlet):
        if user is None:
            raise NotFoundError("Unknown user")
-        start = parse_integer(request, "from", default=0)
+        start = parse_integer(request, "from", default=0, negative=False)
-        limit = parse_integer(request, "limit", default=100)
+        limit = parse_integer(request, "limit", default=100, negative=False)
        if start < 0:
            raise SynapseError(
                HTTPStatus.BAD_REQUEST,
                "Query parameter from must be a string representing a positive integer.",
                errcode=Codes.INVALID_PARAM,
            )
        if limit < 0:
            raise SynapseError(
                HTTPStatus.BAD_REQUEST,
                "Query parameter limit must be a string representing a positive integer.",
                errcode=Codes.INVALID_PARAM,
            )
        # If neither `order_by` nor `dir` is set, set the default order
        # to newest media is on top for backward compatibility.
@@ -447,22 +421,8 @@ class UserMediaRestServlet(RestServlet):
        if user is None:
            raise NotFoundError("Unknown user")
-        start = parse_integer(request, "from", default=0)
+        start = parse_integer(request, "from", default=0, negative=False)
-        limit = parse_integer(request, "limit", default=100)
+        limit = parse_integer(request, "limit", default=100, negative=False)
        if start < 0:
            raise SynapseError(
                HTTPStatus.BAD_REQUEST,
                "Query parameter from must be a string representing a positive integer.",
                errcode=Codes.INVALID_PARAM,
            )
        if limit < 0:
            raise SynapseError(
                HTTPStatus.BAD_REQUEST,
                "Query parameter limit must be a string representing a positive integer.",
                errcode=Codes.INVALID_PARAM,
            )
        # If neither `order_by` nor `dir` is set, set the default order
        # to newest media is on top for backward compatibility.
@@ -21,7 +21,6 @@
 import logging
 from http import HTTPStatus
 from typing import TYPE_CHECKING, List, Optional, Tuple, cast
 from urllib import parse as urlparse
 import attr
@@ -38,6 +37,7 @@ from synapse.http.servlet import (
    assert_params_in_dict,
    parse_enum,
    parse_integer,
    parse_json,
    parse_json_object_from_request,
    parse_string,
 )
@@ -51,7 +51,6 @@ from synapse.storage.databases.main.room import RoomSortOrder
 from synapse.streams.config import PaginationConfig
 from synapse.types import JsonDict, RoomID, ScheduledTask, UserID, create_requester
 from synapse.types.state import StateFilter
 from synapse.util import json_decoder
 if TYPE_CHECKING:
    from synapse.api.auth import Auth
@@ -776,14 +775,8 @@ class RoomEventContextServlet(RestServlet):
        limit = parse_integer(request, "limit", default=10)
        # picking the API shape for symmetry with /messages
-        filter_str = parse_string(request, "filter", encoding="utf-8")
+        filter_json = parse_json(request, "filter", encoding="utf-8")
-        if filter_str:
+        event_filter = Filter(self._hs, filter_json) if filter_json else None
            filter_json = urlparse.unquote(filter_str)
            event_filter: Optional[Filter] = Filter(
                self._hs, json_decoder.decode(filter_json)
            )
        else:
            event_filter = None
        event_context = await self.room_context_handler.get_event_context(
            requester,
@@ -914,21 +907,16 @@ class RoomMessagesRestServlet(RestServlet):
        )
        # Twisted will have processed the args by now.
        assert request.args is not None
        filter_json = parse_json(request, "filter", encoding="utf-8")
        event_filter = Filter(self._hs, filter_json) if filter_json else None
        as_client_event = b"raw" not in request.args
-        filter_str = parse_string(request, "filter", encoding="utf-8")
+        if (
-        if filter_str:
+            event_filter
-            filter_json = urlparse.unquote(filter_str)
+            and event_filter.filter_json.get("event_format", "client") == "federation"
-            event_filter: Optional[Filter] = Filter(
+        ):
-                self._hs, json_decoder.decode(filter_json)
+            as_client_event = False
            )
            if (
                event_filter
                and event_filter.filter_json.get("event_format", "client")
                == "federation"
            ):
                as_client_event = False
        else:
            event_filter = None
        msgs = await self._pagination_handler.get_messages(
            room_id=room_id,
@@ -63,38 +63,12 @@ class UserMediaStatisticsRestServlet(RestServlet):
            ),
        )
-        start = parse_integer(request, "from", default=0)
+        start = parse_integer(request, "from", default=0, negative=False)
-        if start < 0:
+        limit = parse_integer(request, "limit", default=100, negative=False)
-            raise SynapseError(
+        from_ts = parse_integer(request, "from_ts", default=0, negative=False)
-                HTTPStatus.BAD_REQUEST,
+        until_ts = parse_integer(request, "until_ts", negative=False)
                "Query parameter from must be a string representing a positive integer.",
                errcode=Codes.INVALID_PARAM,
            )
        limit = parse_integer(request, "limit", default=100)
        if limit < 0:
            raise SynapseError(
                HTTPStatus.BAD_REQUEST,
                "Query parameter limit must be a string representing a positive integer.",
                errcode=Codes.INVALID_PARAM,
            )
        from_ts = parse_integer(request, "from_ts", default=0)
        if from_ts < 0:
            raise SynapseError(
                HTTPStatus.BAD_REQUEST,
                "Query parameter from_ts must be a string representing a positive integer.",
                errcode=Codes.INVALID_PARAM,
            )
        until_ts = parse_integer(request, "until_ts")
        if until_ts is not None:
            if until_ts < 0:
                raise SynapseError(
                    HTTPStatus.BAD_REQUEST,
                    "Query parameter until_ts must be a string representing a positive integer.",
                    errcode=Codes.INVALID_PARAM,
                )
            if until_ts <= from_ts:
                raise SynapseError(
                    HTTPStatus.BAD_REQUEST,
@@ -90,22 +90,8 @@ class UsersRestServletV2(RestServlet):
    async def on_GET(self, request: SynapseRequest) -> Tuple[int, JsonDict]:
        await assert_requester_is_admin(self.auth, request)
-        start = parse_integer(request, "from", default=0)
+        start = parse_integer(request, "from", default=0, negative=False)
-        limit = parse_integer(request, "limit", default=100)
+        limit = parse_integer(request, "limit", default=100, negative=False)
        if start < 0:
            raise SynapseError(
                HTTPStatus.BAD_REQUEST,
                "Query parameter from must be a string representing a positive integer.",
                errcode=Codes.INVALID_PARAM,
            )
        if limit < 0:
            raise SynapseError(
                HTTPStatus.BAD_REQUEST,
                "Query parameter limit must be a string representing a positive integer.",
                errcode=Codes.INVALID_PARAM,
            )
        user_id = parse_string(request, "user_id")
        name = parse_string(request, "name", encoding="utf-8")
@@ -81,8 +81,7 @@ class AccountDataServlet(RestServlet):
            raise AuthError(403, "Cannot add account data for other users.")
        # Raise an error if the account data type cannot be set directly.
-        if self._hs.config.experimental.msc4010_push_rules_account_data:
+        _check_can_set_account_data_type(account_data_type)
            _check_can_set_account_data_type(account_data_type)
        body = parse_json_object_from_request(request)
@@ -108,10 +107,7 @@ class AccountDataServlet(RestServlet):
            raise AuthError(403, "Cannot get account data for other users.")
        # Push rules are stored in a separate table and must be queried separately.
-        if (
+        if account_data_type == AccountDataTypes.PUSH_RULES:
            self._hs.config.experimental.msc4010_push_rules_account_data
            and account_data_type == AccountDataTypes.PUSH_RULES
        ):
            account_data: Optional[JsonMapping] = (
                await self._push_rules_handler.push_rules_for_user(requester.user)
            )
@@ -162,8 +158,7 @@ class UnstableAccountDataServlet(RestServlet):
            raise AuthError(403, "Cannot delete account data for other users.")
        # Raise an error if the account data type cannot be set directly.
-        if self._hs.config.experimental.msc4010_push_rules_account_data:
+        _check_can_set_account_data_type(account_data_type)
            _check_can_set_account_data_type(account_data_type)
        await self.handler.remove_account_data_for_user(user_id, account_data_type)
@@ -209,15 +204,7 @@ class RoomAccountDataServlet(RestServlet):
            )
        # Raise an error if the account data type cannot be set directly.
-        if self._hs.config.experimental.msc4010_push_rules_account_data:
+        _check_can_set_account_data_type(account_data_type)
            _check_can_set_account_data_type(account_data_type)
        elif account_data_type == ReceiptTypes.FULLY_READ:
            raise SynapseError(
                405,
                "Cannot set m.fully_read through this API."
                " Use /rooms/!roomId:server.name/read_markers",
                Codes.BAD_JSON,
            )
        body = parse_json_object_from_request(request)
@@ -256,10 +243,7 @@ class RoomAccountDataServlet(RestServlet):
            )
        # Room-specific push rules are not currently supported.
-        if (
+        if account_data_type == AccountDataTypes.PUSH_RULES:
            self._hs.config.experimental.msc4010_push_rules_account_data
            and account_data_type == AccountDataTypes.PUSH_RULES
        ):
            account_data: Optional[JsonMapping] = {}
        else:
            account_data = await self.store.get_account_data_for_room_and_type(
@@ -317,8 +301,7 @@ class UnstableRoomAccountDataServlet(RestServlet):
            )
        # Raise an error if the account data type cannot be set directly.
-        if self._hs.config.experimental.msc4010_push_rules_account_data:
+        _check_can_set_account_data_type(account_data_type)
            _check_can_set_account_data_type(account_data_type)
        await self.handler.remove_account_data_for_room(
            user_id, room_id, account_data_type
@@ -409,7 +409,18 @@ class SigningKeyUploadServlet(RestServlet):
            # But first-time setup is fine
        elif self.hs.config.experimental.msc3967_enabled:
-            # If we already have a master key then cross signing is set up and we require UIA to reset
+            # MSC3967 allows this endpoint to 200 OK for idempotency. Resending exactly the same
            # keys should just 200 OK without doing a UIA prompt.
            keys_are_different = await self.e2e_keys_handler.has_different_keys(
                user_id, body
            )
            if not keys_are_different:
                # FIXME: we do not fallthrough to upload_signing_keys_for_user because confusingly
                # if we do, we 500 as it looks like it tries to INSERT the same key twice, causing a
                # unique key constraint violation. This sounds like a bug?
                return 200, {}
            # the keys are different, is x-signing set up? If no, then the keys don't exist which is
            # why they are different. If yes, then we need to UIA to change them.
            if is_cross_signing_setup:
                await self.auth_handler.validate_user_via_ui_auth(
                    requester,
@@ -420,7 +431,6 @@ class SigningKeyUploadServlet(RestServlet):
                    can_skip_ui_auth=False,
                )
            # Otherwise we don't require UIA since we are setting up cross signing for first time
        else:
            # Previous behaviour is to always require UIA but allow it to be skipped
            await self.auth_handler.validate_user_via_ui_auth(
@@ -55,7 +55,6 @@ class RelationPaginationServlet(RestServlet):
        self.auth = hs.get_auth()
        self._store = hs.get_datastores().main
        self._relations_handler = hs.get_relations_handler()
        self._support_recurse = hs.config.experimental.msc3981_recurse_relations
    async def on_GET(
        self,
@@ -70,12 +69,9 @@ class RelationPaginationServlet(RestServlet):
        pagination_config = await PaginationConfig.from_request(
            self._store, request, default_limit=5, default_dir=Direction.BACKWARDS
        )
-        if self._support_recurse:
+        recurse = parse_boolean(request, "recurse", default=False) or parse_boolean(
-            recurse = parse_boolean(request, "recurse", default=False) or parse_boolean(
+            request, "org.matrix.msc3981.recurse", default=False
-                request, "org.matrix.msc3981.recurse", default=False
+        )
            )
        else:
            recurse = False
        # The unstable version of this API returns an extra field for client
        # compatibility, see https://github.com/matrix-org/synapse/issues/12930.
@@ -2,7 +2,7 @@
 # This file is licensed under the Affero General Public License (AGPL) version 3.
 #
 # Copyright 2022 The Matrix.org Foundation C.I.C.
-# Copyright (C) 2023 New Vector, Ltd
+# Copyright (C) 2023-2024 New Vector, Ltd
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as
@@ -34,7 +34,7 @@ if TYPE_CHECKING:
 logger = logging.getLogger(__name__)
-class RendezvousServlet(RestServlet):
+class MSC3886RendezvousServlet(RestServlet):
    """
    This is a placeholder implementation of [MSC3886](https://github.com/matrix-org/matrix-spec-proposals/pull/3886)
    simple client rendezvous capability that is used by the "Sign in with QR" functionality.
@@ -76,6 +76,30 @@ class RendezvousServlet(RestServlet):
    # PUT, GET and DELETE are not implemented as they should be fulfilled by the redirect target.
 class MSC4108DelegationRendezvousServlet(RestServlet):
    PATTERNS = client_patterns(
        "/org.matrix.msc4108/rendezvous$", releases=[], v1=False, unstable=True
    )
    def __init__(self, hs: "HomeServer"):
        super().__init__()
        redirection_target: Optional[str] = (
            hs.config.experimental.msc4108_delegation_endpoint
        )
        assert (
            redirection_target is not None
        ), "Servlet is only registered if there is a delegation target"
        self.endpoint = redirection_target.encode("utf-8")
    async def on_POST(self, request: SynapseRequest) -> None:
        respond_with_redirect(
            request, self.endpoint, statusCode=TEMPORARY_REDIRECT, cors=True
        )
 def register_servlets(hs: "HomeServer", http_server: HttpServer) -> None:
    if hs.config.experimental.msc3886_endpoint is not None:
-        RendezvousServlet(hs).register(http_server)
+        MSC3886RendezvousServlet(hs).register(http_server)
    if hs.config.experimental.msc4108_delegation_endpoint is not None:
        MSC4108DelegationRendezvousServlet(hs).register(http_server)
@@ -52,6 +52,7 @@ from synapse.http.servlet import (
    parse_boolean,
    parse_enum,
    parse_integer,
    parse_json,
    parse_json_object_from_request,
    parse_string,
    parse_strings_from_args,
@@ -65,7 +66,6 @@ from synapse.rest.client.transactions import HttpTransactionCache
 from synapse.streams.config import PaginationConfig
 from synapse.types import JsonDict, Requester, StreamToken, ThirdPartyInstanceID, UserID
 from synapse.types.state import StateFilter
 from synapse.util import json_decoder
 from synapse.util.cancellation import cancellable
 from synapse.util.stringutils import parse_and_validate_server_name, random_string
@@ -499,7 +499,7 @@ class PublicRoomListRestServlet(RestServlet):
            if server:
                raise e
-        limit: Optional[int] = parse_integer(request, "limit", 0)
+        limit: Optional[int] = parse_integer(request, "limit", 0, negative=False)
        since_token = parse_string(request, "since")
        if limit == 0:
@@ -703,21 +703,16 @@ class RoomMessageListRestServlet(RestServlet):
        )
        # Twisted will have processed the args by now.
        assert request.args is not None
        filter_json = parse_json(request, "filter", encoding="utf-8")
        event_filter = Filter(self._hs, filter_json) if filter_json else None
        as_client_event = b"raw" not in request.args
-        filter_str = parse_string(request, "filter", encoding="utf-8")
+        if (
-        if filter_str:
+            event_filter
-            filter_json = urlparse.unquote(filter_str)
+            and event_filter.filter_json.get("event_format", "client") == "federation"
-            event_filter: Optional[Filter] = Filter(
+        ):
-                self._hs, json_decoder.decode(filter_json)
+            as_client_event = False
            )
            if (
                event_filter
                and event_filter.filter_json.get("event_format", "client")
                == "federation"
            ):
                as_client_event = False
        else:
            event_filter = None
        msgs = await self.pagination_handler.get_messages(
            room_id=room_id,
@@ -898,14 +893,8 @@ class RoomEventContextServlet(RestServlet):
        limit = parse_integer(request, "limit", default=10)
        # picking the API shape for symmetry with /messages
-        filter_str = parse_string(request, "filter", encoding="utf-8")
+        filter_json = parse_json(request, "filter", encoding="utf-8")
-        if filter_str:
+        event_filter = Filter(self._hs, filter_json) if filter_json else None
            filter_json = urlparse.unquote(filter_str)
            event_filter: Optional[Filter] = Filter(
                self._hs, json_decoder.decode(filter_json)
            )
        else:
            event_filter = None
        event_context = await self.room_context_handler.get_event_context(
            requester, room_id, event_id, limit, event_filter
@@ -132,13 +132,17 @@ class VersionsRestServlet(RestServlet):
                    # Adds support for relation-based redactions as per MSC3912.
                    "org.matrix.msc3912": self.config.experimental.msc3912_enabled,
                    # Whether recursively provide relations is supported.
-                    "org.matrix.msc3981": self.config.experimental.msc3981_recurse_relations,
+                    # TODO This is no longer needed once unstable MSC3981 does not need to be supported.
                    "org.matrix.msc3981": True,
                    # Adds support for deleting account data.
                    "org.matrix.msc3391": self.config.experimental.msc3391_enabled,
                    # Allows clients to inhibit profile update propagation.
                    "org.matrix.msc4069": self.config.experimental.msc4069_profile_inhibit_propagation,
                    # Allows clients to handle push for encrypted events.
                    "org.matrix.msc4028": self.config.experimental.msc4028_push_encrypted_events,
                    # MSC4108: Mechanism to allow OIDC sign in and E2EE set up via QR code
                    "org.matrix.msc4108": self.config.experimental.msc4108_delegation_endpoint
                    is not None,
                },
            },
        )
@@ -72,9 +72,6 @@ class PreviewUrlResource(RestServlet):
        # XXX: if get_user_by_req fails, what should we do in an async render?
        requester = await self.auth.get_user_by_req(request)
        url = parse_string(request, "url", required=True)
-        ts = parse_integer(request, "ts")
+        ts = parse_integer(request, "ts", default=self.clock.time_msec())
        if ts is None:
            ts = self.clock.time_msec()
        og = await self.url_previewer.preview(url, requester.user, ts)
        respond_with_json_bytes(request, 200, og, send_cors=True)
@@ -385,7 +385,6 @@ class EventPushActionsWorkerStore(ReceiptsWorkerStore, StreamWorkerStore, SQLBas
            WITH all_receipts AS (
                SELECT room_id, thread_id, MAX(event_stream_ordering) AS max_receipt_stream_ordering
                FROM receipts_linearized
                LEFT JOIN events USING (room_id, event_id)
                WHERE
                    {receipt_types_clause}
                    AND user_id = ?
@@ -621,13 +620,12 @@ class EventPushActionsWorkerStore(ReceiptsWorkerStore, StreamWorkerStore, SQLBas
                SELECT notif_count, COALESCE(unread_count, 0), thread_id
                FROM event_push_summary
                LEFT JOIN (
-                    SELECT thread_id, MAX(stream_ordering) AS threaded_receipt_stream_ordering
+                    SELECT thread_id, MAX(event_stream_ordering) AS threaded_receipt_stream_ordering
                    FROM receipts_linearized
                    LEFT JOIN events USING (room_id, event_id)
                    WHERE
                        user_id = ?
                        AND room_id = ?
-                        AND stream_ordering > ?
+                        AND event_stream_ordering > ?
                        AND {receipt_types_clause}
                    GROUP BY thread_id
                ) AS receipts USING (thread_id)
@@ -659,13 +657,12 @@ class EventPushActionsWorkerStore(ReceiptsWorkerStore, StreamWorkerStore, SQLBas
        sql = f"""
            SELECT COUNT(*), thread_id FROM event_push_actions
            LEFT JOIN (
-                SELECT thread_id, MAX(stream_ordering) AS threaded_receipt_stream_ordering
+                SELECT thread_id, MAX(event_stream_ordering) AS threaded_receipt_stream_ordering
                FROM receipts_linearized
                LEFT JOIN events USING (room_id, event_id)
                WHERE
                    user_id = ?
                    AND room_id = ?
-                    AND stream_ordering > ?
+                    AND event_stream_ordering > ?
                    AND {receipt_types_clause}
                GROUP BY thread_id
            ) AS receipts USING (thread_id)
@@ -738,13 +735,12 @@ class EventPushActionsWorkerStore(ReceiptsWorkerStore, StreamWorkerStore, SQLBas
                thread_id
            FROM event_push_actions
            LEFT JOIN (
-                SELECT thread_id, MAX(stream_ordering) AS threaded_receipt_stream_ordering
+                SELECT thread_id, MAX(event_stream_ordering) AS threaded_receipt_stream_ordering
                FROM receipts_linearized
                LEFT JOIN events USING (room_id, event_id)
                WHERE
                    user_id = ?
                    AND room_id = ?
-                    AND stream_ordering > ?
+                    AND event_stream_ordering > ?
                    AND {receipt_types_clause}
                GROUP BY thread_id
            ) AS receipts USING (thread_id)
@@ -910,9 +906,8 @@ class EventPushActionsWorkerStore(ReceiptsWorkerStore, StreamWorkerStore, SQLBas
        # given this function generally gets called with only one room and
        # thread ID.
        sql = f"""
-            SELECT room_id, thread_id, MAX(stream_ordering)
+            SELECT room_id, thread_id, MAX(event_stream_ordering)
            FROM receipts_linearized
            INNER JOIN events USING (room_id, event_id)
            WHERE {receipt_types_clause}
                AND {thread_ids_clause}
                AND {room_ids_clause}
@@ -1442,9 +1437,8 @@ class EventPushActionsWorkerStore(ReceiptsWorkerStore, StreamWorkerStore, SQLBas
        )
        sql = """
-            SELECT r.stream_id, r.room_id, r.user_id, r.thread_id, e.stream_ordering
+            SELECT r.stream_id, r.room_id, r.user_id, r.thread_id, r.event_stream_ordering
            FROM receipts_linearized AS r
            INNER JOIN events AS e USING (event_id)
            WHERE ? < r.stream_id AND r.stream_id <= ? AND user_id LIKE ?
            ORDER BY r.stream_id ASC
            LIMIT ?
@@ -178,14 +178,13 @@ class ReceiptsWorkerStore(SQLBaseStore):
        )
        sql = f"""
-            SELECT event_id, stream_ordering
+            SELECT event_id, event_stream_ordering
            FROM receipts_linearized
            INNER JOIN events USING (room_id, event_id)
            WHERE {clause}
            AND user_id = ?
            AND room_id = ?
            AND thread_id IS NULL
-            ORDER BY stream_ordering DESC
+            ORDER BY event_stream_ordering DESC
            LIMIT 1
        """
@@ -735,10 +734,13 @@ class ReceiptsWorkerStore(SQLBaseStore):
                thread_clause = "r.thread_id = ?"
                thread_args = (thread_id,)
            # If the receipt doesn't have a stream ordering it is because we
            # don't have the associated event, and so must be a remote receipt.
            # Hence it's safe to just allow new receipts to clobber it.
            sql = f"""
-            SELECT stream_ordering, event_id FROM events
+            SELECT r.event_stream_ordering, r.event_id FROM receipts_linearized AS r
-            INNER JOIN receipts_linearized AS r USING (event_id, room_id)
+            WHERE r.room_id = ? AND r.receipt_type = ? AND r.user_id = ?
-            WHERE r.room_id = ? AND r.receipt_type = ? AND r.user_id = ? AND {thread_clause}
+            AND r.event_stream_ordering IS NOT NULL AND {thread_clause}
            """
            txn.execute(
                sql,
@@ -2108,6 +2108,13 @@ class RegistrationBackgroundUpdateStore(RegistrationWorkerStore):
            unique=False,
        )
        self.db_pool.updates.register_background_index_update(
            update_name="access_tokens_refresh_token_id_idx",
            index_name="access_tokens_refresh_token_id_idx",
            table="access_tokens",
            columns=("refresh_token_id",),
        )
    async def _background_update_set_deactivated_flag(
        self, progress: JsonDict, batch_size: int
    ) -> int:
@@ -2266,13 +2273,6 @@ class RegistrationStore(StatsStore, RegistrationBackgroundUpdateStore):
    ):
        super().__init__(database, db_conn, hs)
        self.db_pool.updates.register_background_index_update(
            update_name="access_tokens_refresh_token_id_idx",
            index_name="access_tokens_refresh_token_id_idx",
            table="access_tokens",
            columns=("refresh_token_id",),
        )
        self._ignore_unknown_session_error = (
            hs.config.server.request_token_inhibit_3pid_errors
        )
@@ -0,0 +1,15 @@
 --
 -- This file is licensed under the Affero General Public License (AGPL) version 3.
 --
 -- Copyright (C) 2023 New Vector, Ltd
 --
 -- This program is free software: you can redistribute it and/or modify
 -- it under the terms of the GNU Affero General Public License as
 -- published by the Free Software Foundation, either version 3 of the
 -- License, or (at your option) any later version.
 --
 -- See the GNU Affero General Public License for more details:
 -- <https://www.gnu.org/licenses/agpl-3.0.html>.
 INSERT INTO background_updates (ordering, update_name, progress_json) VALUES
    (8404, 'access_tokens_refresh_token_id_idx', '{}');
@@ -36,10 +36,15 @@ from typing import (
 import attr
-from synapse.api.constants import EventTypes, HistoryVisibility, Membership
+from synapse.api.constants import (
    EventTypes,
    EventUnsignedContentFields,
    HistoryVisibility,
    Membership,
 )
 from synapse.events import EventBase
 from synapse.events.snapshot import EventContext
-from synapse.events.utils import prune_event
+from synapse.events.utils import clone_event, prune_event
 from synapse.logging.opentracing import trace
 from synapse.storage.controllers import StorageControllers
 from synapse.storage.databases.main import DataStore
@@ -77,6 +82,7 @@ async def filter_events_for_client(
    is_peeking: bool = False,
    always_include_ids: FrozenSet[str] = frozenset(),
    filter_send_to_client: bool = True,
    msc4115_membership_on_events: bool = False,
 ) -> List[EventBase]:
    """
    Check which events a user is allowed to see. If the user can see the event but its
@@ -95,9 +101,12 @@ async def filter_events_for_client(
        filter_send_to_client: Whether we're checking an event that's going to be
            sent to a client. This might not always be the case since this function can
            also be called to check whether a user can see the state at a given point.
        msc4115_membership_on_events: Whether to include the requesting user's
            membership in the "unsigned" data, per MSC4115.
    Returns:
-        The filtered events.
+        The filtered events. If `msc4115_membership_on_events` is true, the `unsigned`
        data is annotated with the membership state of `user_id` at each event.
    """
    # Filter out events that have been soft failed so that we don't relay them
    # to clients.
@@ -134,7 +143,8 @@ async def filter_events_for_client(
            )
    def allowed(event: EventBase) -> Optional[EventBase]:
-        return _check_client_allowed_to_see_event(
+        state_after_event = event_id_to_state.get(event.event_id)
        filtered = _check_client_allowed_to_see_event(
            user_id=user_id,
            event=event,
            clock=storage.main.clock,
@@ -142,13 +152,45 @@ async def filter_events_for_client(
            sender_ignored=event.sender in ignore_list,
            always_include_ids=always_include_ids,
            retention_policy=retention_policies[room_id],
-            state=event_id_to_state.get(event.event_id),
+            state=state_after_event,
            is_peeking=is_peeking,
            sender_erased=erased_senders.get(event.sender, False),
        )
        if filtered is None:
            return None
-    # Check each event: gives an iterable of None or (a potentially modified)
+        if not msc4115_membership_on_events:
-    # EventBase.
+            return filtered
        # Annotate the event with the user's membership after the event.
        #
        # Normally we just look in `state_after_event`, but if the event is an outlier
        # we won't have such a state. The only outliers that are returned here are the
        # user's own membership event, so we can just inspect that.
        user_membership_event: Optional[EventBase]
        if event.type == EventTypes.Member and event.state_key == user_id:
            user_membership_event = event
        elif state_after_event is not None:
            user_membership_event = state_after_event.get((EventTypes.Member, user_id))
        else:
            # unreachable!
            raise Exception("Missing state for event that is not user's own membership")
        user_membership = (
            user_membership_event.membership
            if user_membership_event
            else Membership.LEAVE
        )
        # Copy the event before updating the unsigned data: this shouldn't be persisted
        # to the cache!
        cloned = clone_event(filtered)
        cloned.unsigned[EventUnsignedContentFields.MSC4115_MEMBERSHIP] = user_membership
        return cloned
    # Check each event: gives an iterable of None or (a modified) EventBase.
    filtered_events = map(allowed, events)
    # Turn it into a list and remove None entries before returning.
@@ -396,9 +438,13 @@ def _check_client_allowed_to_see_event(
@attr.s(frozen=True, slots=True, auto_attribs=True)
 class _CheckMembershipReturn:
-    "Return value of _check_membership"
+    """Return value of _check_membership"""
    allowed: bool
    """Whether the user should be allowed to see the event"""
    joined: bool
    """Whether the user was joined at the event"""
 def _check_membership(
@@ -408,12 +454,7 @@ def _check_membership(
    state: StateMap[EventBase],
    is_peeking: bool,
 ) -> _CheckMembershipReturn:
-    """Check whether the user can see the event due to their membership
+    """Check whether the user can see the event due to their membership"""
    Returns:
        True if they can, False if they can't, plus the membership of the user
        at the event.
    """
    # If the event is the user's own membership event, use the 'most joined'
    # membership
    membership = None
@@ -435,7 +476,7 @@ def _check_membership(
        if membership == "leave" and (
            prev_membership == "join" or prev_membership == "invite"
        ):
-            return _CheckMembershipReturn(True, membership == Membership.JOIN)
+            return _CheckMembershipReturn(True, False)
        new_priority = MEMBERSHIP_PRIORITY.index(membership)
        old_priority = MEMBERSHIP_PRIORITY.index(prev_membership)
@@ -32,6 +32,7 @@ from synapse.events.utils import (
    PowerLevelsContent,
    SerializeEventConfig,
    _split_field,
    clone_event,
    copy_and_fixup_power_levels_contents,
    maybe_upsert_event_field,
    prune_event,
@@ -611,6 +612,18 @@ class PruneEventTestCase(stdlib_unittest.TestCase):
        )
 class CloneEventTestCase(stdlib_unittest.TestCase):
    def test_unsigned_is_copied(self) -> None:
        original = make_event_from_dict(
            {"type": "A", "event_id": "$test:domain", "unsigned": {"a": 1, "b": 2}}
        )
        cloned = clone_event(original)
        cloned.unsigned["b"] = 3
        self.assertEqual(original.unsigned, {"a": 1, "b": 2})
        self.assertEqual(cloned.unsigned, {"a": 1, "b": 3})
 class SerializeEventTestCase(stdlib_unittest.TestCase):
    def serialize(self, ev: EventBase, fields: Optional[List[str]]) -> JsonDict:
        return serialize_event(
@@ -1101,6 +1101,56 @@ class E2eKeysHandlerTestCase(unittest.HomeserverTestCase):
            },
        )
    def test_has_different_keys(self) -> None:
        """check that has_different_keys returns True when the keys provided are different to what
        is in the database."""
        local_user = "@boris:" + self.hs.hostname
        keys1 = {
            "master_key": {
                # private key: 2lonYOM6xYKdEsO+6KrC766xBcHnYnim1x/4LFGF8B0
                "user_id": local_user,
                "usage": ["master"],
                "keys": {
                    "ed25519:nqOvzeuGWT/sRx3h7+MHoInYj3Uk2LD/unI9kDYcHwk": "nqOvzeuGWT/sRx3h7+MHoInYj3Uk2LD/unI9kDYcHwk"
                },
            }
        }
        self.get_success(self.handler.upload_signing_keys_for_user(local_user, keys1))
        is_different = self.get_success(
            self.handler.has_different_keys(
                local_user,
                {
                    "master_key": keys1["master_key"],
                },
            )
        )
        self.assertEqual(is_different, False)
        # change the usage => different keys
        keys1["master_key"]["usage"] = ["develop"]
        is_different = self.get_success(
            self.handler.has_different_keys(
                local_user,
                {
                    "master_key": keys1["master_key"],
                },
            )
        )
        self.assertEqual(is_different, True)
        keys1["master_key"]["usage"] = ["master"]  # reset
        # change the key => different keys
        keys1["master_key"]["keys"] = {
            "ed25519:nqOvzeuGWT/sRx3h7+MHoInYj3Uk2LD/unIc0rncs": "nqOvzeuGWT/sRx3h7+MHoInYj3Uk2LD/unIc0rncs"
        }
        is_different = self.get_success(
            self.handler.has_different_keys(
                local_user,
                {
                    "master_key": keys1["master_key"],
                },
            )
        )
        self.assertEqual(is_different, True)
    def test_query_devices_remote_sync(self) -> None:
        """Tests that querying keys for a remote user that we share a room with,
        but haven't yet fetched the keys for, returns the cross signing keys
@@ -435,6 +435,101 @@ class SyncTestCase(tests.unittest.HomeserverTestCase):
            [s2_event],
        )
    def test_state_includes_changes_on_long_lived_forks(self) -> None:
        """State changes that happen on a fork of the DAG must be included in `state`
        Given the following DAG:
             E1
           ↗    ↖
          |      S2
          |      ↑
        --|------|----
          E3     |
        --|------|----
          |      E4
          |      |
        ... and a filter that means we only return 1 event, represented by the dashed
        horizontal lines: `S2` must be included in the `state` section on the second sync.
        """
        alice = self.register_user("alice", "password")
        alice_tok = self.login(alice, "password")
        alice_requester = create_requester(alice)
        room_id = self.helper.create_room_as(alice, is_public=True, tok=alice_tok)
        # Do an initial sync as Alice to get a known starting point.
        initial_sync_result = self.get_success(
            self.sync_handler.wait_for_sync_for_user(
                alice_requester, generate_sync_config(alice)
            )
        )
        last_room_creation_event_id = (
            initial_sync_result.joined[0].timeline.events[-1].event_id
        )
        # Send a state event, and a regular event, both using the same prev ID
        with self._patch_get_latest_events([last_room_creation_event_id]):
            s2_event = self.helper.send_state(room_id, "s2", {}, tok=alice_tok)[
                "event_id"
            ]
            e3_event = self.helper.send(room_id, "e3", tok=alice_tok)["event_id"]
        # Do an incremental sync, this will return E3 but *not* S2 at this
        # point.
        incremental_sync = self.get_success(
            self.sync_handler.wait_for_sync_for_user(
                alice_requester,
                generate_sync_config(
                    alice,
                    filter_collection=FilterCollection(
                        self.hs, {"room": {"timeline": {"limit": 1}}}
                    ),
                ),
                since_token=initial_sync_result.next_batch,
            )
        )
        room_sync = incremental_sync.joined[0]
        self.assertEqual(room_sync.room_id, room_id)
        self.assertTrue(room_sync.timeline.limited)
        self.assertEqual(
            [e.event_id for e in room_sync.timeline.events],
            [e3_event],
        )
        self.assertEqual(
            [e.event_id for e in room_sync.state.values()],
            [],
        )
        # Now send another event that points to S2, but not E3.
        with self._patch_get_latest_events([s2_event]):
            e4_event = self.helper.send(room_id, "e4", tok=alice_tok)["event_id"]
        # Doing an incremental sync should return S2 in state.
        incremental_sync = self.get_success(
            self.sync_handler.wait_for_sync_for_user(
                alice_requester,
                generate_sync_config(
                    alice,
                    filter_collection=FilterCollection(
                        self.hs, {"room": {"timeline": {"limit": 1}}}
                    ),
                ),
                since_token=incremental_sync.next_batch,
            )
        )
        room_sync = incremental_sync.joined[0]
        self.assertEqual(room_sync.room_id, room_id)
        self.assertFalse(room_sync.timeline.limited)
        self.assertEqual(
            [e.event_id for e in room_sync.timeline.events],
            [e4_event],
        )
        self.assertEqual(
            [e.event_id for e in room_sync.state.values()],
            [s2_event],
        )
    def test_state_includes_changes_on_ungappy_syncs(self) -> None:
        """Test `state` where the sync is not gappy.
@@ -277,7 +277,8 @@ class DeleteMediaByDateSizeTestCase(_AdminMediaTests):
        self.assertEqual(400, channel.code, msg=channel.json_body)
        self.assertEqual(Codes.MISSING_PARAM, channel.json_body["errcode"])
        self.assertEqual(
-            "Missing integer query parameter 'before_ts'", channel.json_body["error"]
+            "Missing required integer query parameter before_ts",
            channel.json_body["error"],
        )
    def test_invalid_parameter(self) -> None:
@@ -320,7 +321,7 @@ class DeleteMediaByDateSizeTestCase(_AdminMediaTests):
        self.assertEqual(400, channel.code, msg=channel.json_body)
        self.assertEqual(Codes.INVALID_PARAM, channel.json_body["errcode"])
        self.assertEqual(
-            "Query parameter size_gt must be a string representing a positive integer.",
+            "Query parameter size_gt must be a positive integer.",
            channel.json_body["error"],
        )
@@ -21,6 +21,7 @@
 import json
 import time
 import urllib.parse
 from http import HTTPStatus
 from typing import List, Optional
 from unittest.mock import AsyncMock, Mock
@@ -2190,6 +2191,33 @@ class RoomMessagesTestCase(unittest.HomeserverTestCase):
        chunk = channel.json_body["chunk"]
        self.assertEqual(len(chunk), 0, [event["content"] for event in chunk])
    def test_room_message_filter_query_validation(self) -> None:
        # Test json validation in (filter) query parameter.
        # Does not test the validity of the filter, only the json validation.
        # Check Get with valid json filter parameter, expect 200.
        valid_filter_str = '{"types": ["m.room.message"]}'
        channel = self.make_request(
            "GET",
            f"/_synapse/admin/v1/rooms/{self.room_id}/messages?dir=b&filter={valid_filter_str}",
            access_token=self.admin_user_tok,
        )
        self.assertEqual(channel.code, HTTPStatus.OK, channel.json_body)
        # Check Get with invalid json filter parameter, expect 400 NOT_JSON.
        invalid_filter_str = "}}}{}"
        channel = self.make_request(
            "GET",
            f"/_synapse/admin/v1/rooms/{self.room_id}/messages?dir=b&filter={invalid_filter_str}",
            access_token=self.admin_user_tok,
        )
        self.assertEqual(channel.code, HTTPStatus.BAD_REQUEST, channel.json_body)
        self.assertEqual(
            channel.json_body["errcode"], Codes.NOT_JSON, channel.json_body
        )
 class JoinAliasRoomTestCase(unittest.HomeserverTestCase):
    servlets = [
@@ -2522,6 +2550,39 @@ class JoinAliasRoomTestCase(unittest.HomeserverTestCase):
            else:
                self.fail("Event %s from events_after not found" % j)
    def test_room_event_context_filter_query_validation(self) -> None:
        # Test json validation in (filter) query parameter.
        # Does not test the validity of the filter, only the json validation.
        # Create a user with room and event_id.
        user_id = self.register_user("test", "test")
        user_tok = self.login("test", "test")
        room_id = self.helper.create_room_as(user_id, tok=user_tok)
        event_id = self.helper.send(room_id, "message 1", tok=user_tok)["event_id"]
        # Check Get with valid json filter parameter, expect 200.
        valid_filter_str = '{"types": ["m.room.message"]}'
        channel = self.make_request(
            "GET",
            f"/_synapse/admin/v1/rooms/{room_id}/context/{event_id}?filter={valid_filter_str}",
            access_token=self.admin_user_tok,
        )
        self.assertEqual(channel.code, HTTPStatus.OK, channel.json_body)
        # Check Get with invalid json filter parameter, expect 400 NOT_JSON.
        invalid_filter_str = "}}}{}"
        channel = self.make_request(
            "GET",
            f"/_synapse/admin/v1/rooms/{room_id}/context/{event_id}?filter={invalid_filter_str}",
            access_token=self.admin_user_tok,
        )
        self.assertEqual(channel.code, HTTPStatus.BAD_REQUEST, channel.json_body)
        self.assertEqual(
            channel.json_body["errcode"], Codes.NOT_JSON, channel.json_body
        )
 class MakeRoomAdminTestCase(unittest.HomeserverTestCase):
    servlets = [
@@ -35,7 +35,6 @@ from synapse.util import Clock
 from tests import unittest
 from tests.server import FakeChannel
 from tests.test_utils.event_injection import inject_event
 from tests.unittest import override_config
 class BaseRelationsTestCase(unittest.HomeserverTestCase):
@@ -957,7 +956,6 @@ class RelationPaginationTestCase(BaseRelationsTestCase):
 class RecursiveRelationTestCase(BaseRelationsTestCase):
    @override_config({"experimental_features": {"msc3981_recurse_relations": True}})
    def test_recursive_relations(self) -> None:
        """Generate a complex, multi-level relationship tree and query it."""
        # Create a thread with a few messages in it.
@@ -1003,7 +1001,7 @@ class RecursiveRelationTestCase(BaseRelationsTestCase):
        channel = self.make_request(
            "GET",
            f"/_matrix/client/v1/rooms/{self.room}/relations/{self.parent_id}"
-            "?dir=f&limit=20&org.matrix.msc3981.recurse=true",
+            "?dir=f&limit=20&recurse=true",
            access_token=self.user_token,
        )
        self.assertEqual(200, channel.code, channel.json_body)
@@ -1024,7 +1022,6 @@ class RecursiveRelationTestCase(BaseRelationsTestCase):
            ],
        )
    @override_config({"experimental_features": {"msc3981_recurse_relations": True}})
    def test_recursive_relations_with_filter(self) -> None:
        """The event_type and rel_type still apply."""
        # Create a thread with a few messages in it.
@@ -1052,7 +1049,7 @@ class RecursiveRelationTestCase(BaseRelationsTestCase):
        channel = self.make_request(
            "GET",
            f"/_matrix/client/v1/rooms/{self.room}/relations/{self.parent_id}/{RelationTypes.ANNOTATION}"
-            "?dir=f&limit=20&org.matrix.msc3981.recurse=true",
+            "?dir=f&limit=20&recurse=true",
            access_token=self.user_token,
        )
        self.assertEqual(200, channel.code, channel.json_body)
@@ -1065,7 +1062,7 @@ class RecursiveRelationTestCase(BaseRelationsTestCase):
        channel = self.make_request(
            "GET",
            f"/_matrix/client/v1/rooms/{self.room}/relations/{self.parent_id}/{RelationTypes.ANNOTATION}/m.reaction"
-            "?dir=f&limit=20&org.matrix.msc3981.recurse=true",
+            "?dir=f&limit=20&recurse=true",
            access_token=self.user_token,
        )
        self.assertEqual(200, channel.code, channel.json_body)
@@ -27,8 +27,10 @@ from synapse.util import Clock
 from tests import unittest
 from tests.unittest import override_config
 from tests.utils import HAS_AUTHLIB
-endpoint = "/_matrix/client/unstable/org.matrix.msc3886/rendezvous"
+msc3886_endpoint = "/_matrix/client/unstable/org.matrix.msc3886/rendezvous"
 msc4108_endpoint = "/_matrix/client/unstable/org.matrix.msc4108/rendezvous"
 class RendezvousServletTestCase(unittest.HomeserverTestCase):
@@ -41,11 +43,35 @@ class RendezvousServletTestCase(unittest.HomeserverTestCase):
        return self.hs
    def test_disabled(self) -> None:
-        channel = self.make_request("POST", endpoint, {}, access_token=None)
+        channel = self.make_request("POST", msc3886_endpoint, {}, access_token=None)
        self.assertEqual(channel.code, 404)
        channel = self.make_request("POST", msc4108_endpoint, {}, access_token=None)
        self.assertEqual(channel.code, 404)
    @override_config({"experimental_features": {"msc3886_endpoint": "/asd"}})
-    def test_redirect(self) -> None:
+    def test_msc3886_redirect(self) -> None:
-        channel = self.make_request("POST", endpoint, {}, access_token=None)
+        channel = self.make_request("POST", msc3886_endpoint, {}, access_token=None)
        self.assertEqual(channel.code, 307)
        self.assertEqual(channel.headers.getRawHeaders("Location"), ["/asd"])
    @unittest.skip_unless(HAS_AUTHLIB, "requires authlib")
    @override_config(
        {
            "disable_registration": True,
            "experimental_features": {
                "msc4108_delegation_endpoint": "https://asd",
                "msc3861": {
                    "enabled": True,
                    "issuer": "https://issuer",
                    "client_id": "client_id",
                    "client_auth_method": "client_secret_post",
                    "client_secret": "client_secret",
                    "admin_token": "admin_token_value",
                },
            },
        }
    )
    def test_msc4108_delegation(self) -> None:
        channel = self.make_request("POST", msc4108_endpoint, {}, access_token=None)
        self.assertEqual(channel.code, 307)
        self.assertEqual(channel.headers.getRawHeaders("Location"), ["https://asd"])
@@ -163,7 +163,12 @@ class RetentionTestCase(unittest.HomeserverTestCase):
        )
        self.assertEqual(2, len(events), "events retrieved from database")
        filtered_events = self.get_success(
-            filter_events_for_client(storage_controllers, self.user_id, events)
+            filter_events_for_client(
                storage_controllers,
                self.user_id,
                events,
                msc4115_membership_on_events=True,
            )
        )
        # We should only get one event back.
@@ -2175,6 +2175,31 @@ class RoomMessageListTestCase(RoomBase):
        chunk = channel.json_body["chunk"]
        self.assertEqual(len(chunk), 0, [event["content"] for event in chunk])
    def test_room_message_filter_query_validation(self) -> None:
        # Test json validation in (filter) query parameter.
        # Does not test the validity of the filter, only the json validation.
        # Check Get with valid json filter parameter, expect 200.
        valid_filter_str = '{"types": ["m.room.message"]}'
        channel = self.make_request(
            "GET",
            f"/rooms/{self.room_id}/messages?access_token=x&dir=b&filter={valid_filter_str}",
        )
        self.assertEqual(channel.code, HTTPStatus.OK, channel.json_body)
        # Check Get with invalid json filter parameter, expect 400 NOT_JSON.
        invalid_filter_str = "}}}{}"
        channel = self.make_request(
            "GET",
            f"/rooms/{self.room_id}/messages?access_token=x&dir=b&filter={invalid_filter_str}",
        )
        self.assertEqual(channel.code, HTTPStatus.BAD_REQUEST, channel.json_body)
        self.assertEqual(
            channel.json_body["errcode"], Codes.NOT_JSON, channel.json_body
        )
 class RoomMessageFilterTestCase(RoomBase):
    """Tests /rooms/$room_id/messages REST events."""
@@ -3213,6 +3238,33 @@ class ContextTestCase(unittest.HomeserverTestCase):
        self.assertDictEqual(events_after[0].get("content"), {}, events_after[0])
        self.assertEqual(events_after[1].get("content"), {}, events_after[1])
    def test_room_event_context_filter_query_validation(self) -> None:
        # Test json validation in (filter) query parameter.
        # Does not test the validity of the filter, only the json validation.
        event_id = self.helper.send(self.room_id, "message 7", tok=self.tok)["event_id"]
        # Check Get with valid json filter parameter, expect 200.
        valid_filter_str = '{"types": ["m.room.message"]}'
        channel = self.make_request(
            "GET",
            f"/rooms/{self.room_id}/context/{event_id}?filter={valid_filter_str}",
            access_token=self.tok,
        )
        self.assertEqual(channel.code, HTTPStatus.OK, channel.json_body)
        # Check Get with invalid json filter parameter, expect 400 NOT_JSON.
        invalid_filter_str = "}}}{}"
        channel = self.make_request(
            "GET",
            f"/rooms/{self.room_id}/context/{event_id}?filter={invalid_filter_str}",
            access_token=self.tok,
        )
        self.assertEqual(channel.code, HTTPStatus.BAD_REQUEST, channel.json_body)
        self.assertEqual(
            channel.json_body["errcode"], Codes.NOT_JSON, channel.json_body
        )
 class RoomAliasListTestCase(unittest.HomeserverTestCase):
    servlets = [
@@ -21,13 +21,19 @@ import logging
 from typing import Optional
 from unittest.mock import patch
 from synapse.api.constants import EventUnsignedContentFields
 from synapse.api.room_versions import RoomVersions
 from synapse.events import EventBase, make_event_from_dict
 from synapse.events.snapshot import EventContext
-from synapse.types import JsonDict, create_requester
+from synapse.rest import admin
 from synapse.rest.client import login, room
 from synapse.server import HomeServer
 from synapse.types import create_requester
 from synapse.visibility import filter_events_for_client, filter_events_for_server
 from tests import unittest
 from tests.test_utils.event_injection import inject_event, inject_member_event
 from tests.unittest import HomeserverTestCase
 from tests.utils import create_room
 logger = logging.getLogger(__name__)
@@ -56,15 +62,31 @@ class FilterEventsForServerTestCase(unittest.HomeserverTestCase):
        #
        # before we do that, we persist some other events to act as state.
-        self._inject_visibility("@admin:hs", "joined")
+        self.get_success(
            inject_visibility_event(self.hs, TEST_ROOM_ID, "@admin:hs", "joined")
        )
        for i in range(10):
-            self._inject_room_member("@resident%i:hs" % i)
+            self.get_success(
                inject_member_event(
                    self.hs,
                    TEST_ROOM_ID,
                    "@resident%i:hs" % i,
                    "join",
                )
            )
        events_to_filter = []
        for i in range(10):
-            user = "@user%i:%s" % (i, "test_server" if i == 5 else "other_server")
+            evt = self.get_success(
-            evt = self._inject_room_member(user, extra_content={"a": "b"})
+                inject_member_event(
                    self.hs,
                    TEST_ROOM_ID,
                    "@user%i:%s" % (i, "test_server" if i == 5 else "other_server"),
                    "join",
                    extra_content={"a": "b"},
                )
            )
            events_to_filter.append(evt)
        filtered = self.get_success(
@@ -90,8 +112,19 @@ class FilterEventsForServerTestCase(unittest.HomeserverTestCase):
    def test_filter_outlier(self) -> None:
        # outlier events must be returned, for the good of the collective federation
-        self._inject_room_member("@resident:remote_hs")
+        self.get_success(
-        self._inject_visibility("@resident:remote_hs", "joined")
+            inject_member_event(
                self.hs,
                TEST_ROOM_ID,
                "@resident:remote_hs",
                "join",
            )
        )
        self.get_success(
            inject_visibility_event(
                self.hs, TEST_ROOM_ID, "@resident:remote_hs", "joined"
            )
        )
        outlier = self._inject_outlier()
        self.assertEqual(
@@ -110,7 +143,9 @@ class FilterEventsForServerTestCase(unittest.HomeserverTestCase):
        )
        # it should also work when there are other events in the list
-        evt = self._inject_message("@unerased:local_hs")
+        evt = self.get_success(
            inject_message_event(self.hs, TEST_ROOM_ID, "@unerased:local_hs")
        )
        filtered = self.get_success(
            filter_events_for_server(
@@ -150,19 +185,34 @@ class FilterEventsForServerTestCase(unittest.HomeserverTestCase):
        # change in the middle of them.
        events_to_filter = []
-        evt = self._inject_message("@unerased:local_hs")
+        evt = self.get_success(
            inject_message_event(self.hs, TEST_ROOM_ID, "@unerased:local_hs")
        )
        events_to_filter.append(evt)
-        evt = self._inject_message("@erased:local_hs")
+        evt = self.get_success(
            inject_message_event(self.hs, TEST_ROOM_ID, "@erased:local_hs")
        )
        events_to_filter.append(evt)
-        evt = self._inject_room_member("@joiner:remote_hs")
+        evt = self.get_success(
            inject_member_event(
                self.hs,
                TEST_ROOM_ID,
                "@joiner:remote_hs",
                "join",
            )
        )
        events_to_filter.append(evt)
-        evt = self._inject_message("@unerased:local_hs")
+        evt = self.get_success(
            inject_message_event(self.hs, TEST_ROOM_ID, "@unerased:local_hs")
        )
        events_to_filter.append(evt)
-        evt = self._inject_message("@erased:local_hs")
+        evt = self.get_success(
            inject_message_event(self.hs, TEST_ROOM_ID, "@erased:local_hs")
        )
        events_to_filter.append(evt)
        # the erasey user gets erased
@@ -200,76 +250,6 @@ class FilterEventsForServerTestCase(unittest.HomeserverTestCase):
        for i in (1, 4):
            self.assertNotIn("body", filtered[i].content)
    def _inject_visibility(self, user_id: str, visibility: str) -> EventBase:
        content = {"history_visibility": visibility}
        builder = self.event_builder_factory.for_room_version(
            RoomVersions.V1,
            {
                "type": "m.room.history_visibility",
                "sender": user_id,
                "state_key": "",
                "room_id": TEST_ROOM_ID,
                "content": content,
            },
        )
        event, unpersisted_context = self.get_success(
            self.event_creation_handler.create_new_client_event(builder)
        )
        context = self.get_success(unpersisted_context.persist(event))
        self.get_success(self._persistence.persist_event(event, context))
        return event
    def _inject_room_member(
        self,
        user_id: str,
        membership: str = "join",
        extra_content: Optional[JsonDict] = None,
    ) -> EventBase:
        content = {"membership": membership}
        content.update(extra_content or {})
        builder = self.event_builder_factory.for_room_version(
            RoomVersions.V1,
            {
                "type": "m.room.member",
                "sender": user_id,
                "state_key": user_id,
                "room_id": TEST_ROOM_ID,
                "content": content,
            },
        )
        event, unpersisted_context = self.get_success(
            self.event_creation_handler.create_new_client_event(builder)
        )
        context = self.get_success(unpersisted_context.persist(event))
        self.get_success(self._persistence.persist_event(event, context))
        return event
    def _inject_message(
        self, user_id: str, content: Optional[JsonDict] = None
    ) -> EventBase:
        if content is None:
            content = {"body": "testytest", "msgtype": "m.text"}
        builder = self.event_builder_factory.for_room_version(
            RoomVersions.V1,
            {
                "type": "m.room.message",
                "sender": user_id,
                "room_id": TEST_ROOM_ID,
                "content": content,
            },
        )
        event, unpersisted_context = self.get_success(
            self.event_creation_handler.create_new_client_event(builder)
        )
        context = self.get_success(unpersisted_context.persist(event))
        self.get_success(self._persistence.persist_event(event, context))
        return event
    def _inject_outlier(self) -> EventBase:
        builder = self.event_builder_factory.for_room_version(
            RoomVersions.V1,
@@ -292,7 +272,122 @@ class FilterEventsForServerTestCase(unittest.HomeserverTestCase):
        return event
-class FilterEventsForClientTestCase(unittest.FederatingHomeserverTestCase):
+class FilterEventsForClientTestCase(HomeserverTestCase):
    servlets = [
        admin.register_servlets,
        login.register_servlets,
        room.register_servlets,
    ]
    def test_joined_history_visibility(self) -> None:
        # User joins and leaves room. Should be able to seem the join and leave,
        # and messages sent between the two, but not before or after.
        self.register_user("resident", "p1")
        resident_token = self.login("resident", "p1")
        room_id = self.helper.create_room_as("resident", tok=resident_token)
        self.get_success(
            inject_visibility_event(self.hs, room_id, "@resident:test", "joined")
        )
        before_event = self.get_success(
            inject_message_event(self.hs, room_id, "@resident:test", body="before")
        )
        join_event = self.get_success(
            inject_member_event(self.hs, room_id, "@joiner:test", "join")
        )
        during_event = self.get_success(
            inject_message_event(self.hs, room_id, "@resident:test", body="during")
        )
        leave_event = self.get_success(
            inject_member_event(self.hs, room_id, "@joiner:test", "leave")
        )
        after_event = self.get_success(
            inject_message_event(self.hs, room_id, "@resident:test", body="after")
        )
        # We have to reload the events from the db, to ensure that prev_content is
        # populated.
        events_to_filter = [
            self.get_success(
                self.hs.get_storage_controllers().main.get_event(
                    e.event_id,
                    get_prev_content=True,
                )
            )
            for e in [
                before_event,
                join_event,
                during_event,
                leave_event,
                after_event,
            ]
        ]
        # Now run the events through the filter, and check that we can see the events
        # we expect, and that the membership prop is as expected.
        #
        # We deliberately do the queries for both users upfront; this simulates
        # concurrent queries on the server, and helps ensure that we aren't
        # accidentally serving the same event object (with the same unsigned.membership
        # property) to both users.
        joiner_filtered_events = self.get_success(
            filter_events_for_client(
                self.hs.get_storage_controllers(),
                "@joiner:test",
                events_to_filter,
                msc4115_membership_on_events=True,
            )
        )
        resident_filtered_events = self.get_success(
            filter_events_for_client(
                self.hs.get_storage_controllers(),
                "@resident:test",
                events_to_filter,
                msc4115_membership_on_events=True,
            )
        )
        # The joiner should be able to seem the join and leave,
        # and messages sent between the two, but not before or after.
        self.assertEqual(
            [e.event_id for e in [join_event, during_event, leave_event]],
            [e.event_id for e in joiner_filtered_events],
        )
        self.assertEqual(
            ["join", "join", "leave"],
            [
                e.unsigned[EventUnsignedContentFields.MSC4115_MEMBERSHIP]
                for e in joiner_filtered_events
            ],
        )
        # The resident user should see all the events.
        self.assertEqual(
            [
                e.event_id
                for e in [
                    before_event,
                    join_event,
                    during_event,
                    leave_event,
                    after_event,
                ]
            ],
            [e.event_id for e in resident_filtered_events],
        )
        self.assertEqual(
            ["join", "join", "join", "join", "join"],
            [
                e.unsigned[EventUnsignedContentFields.MSC4115_MEMBERSHIP]
                for e in resident_filtered_events
            ],
        )
 class FilterEventsOutOfBandEventsForClientTestCase(
    unittest.FederatingHomeserverTestCase
 ):
    def test_out_of_band_invite_rejection(self) -> None:
        # this is where we have received an invite event over federation, and then
        # rejected it.
@@ -341,15 +436,24 @@ class FilterEventsForClientTestCase(unittest.FederatingHomeserverTestCase):
        )
        # the invited user should be able to see both the invite and the rejection
        filtered_events = self.get_success(
            filter_events_for_client(
                self.hs.get_storage_controllers(),
                "@user:test",
                [invite_event, reject_event],
                msc4115_membership_on_events=True,
            )
        )
        self.assertEqual(
-            self.get_success(
+            [e.event_id for e in filtered_events],
-                filter_events_for_client(
+            [e.event_id for e in [invite_event, reject_event]],
-                    self.hs.get_storage_controllers(),
+        )
-                    "@user:test",
+        self.assertEqual(
-                    [invite_event, reject_event],
+            ["invite", "leave"],
-                )
+            [
-            ),
+                e.unsigned[EventUnsignedContentFields.MSC4115_MEMBERSHIP]
-            [invite_event, reject_event],
+                for e in filtered_events
            ],
        )
        # other users should see neither
@@ -359,7 +463,39 @@ class FilterEventsForClientTestCase(unittest.FederatingHomeserverTestCase):
                    self.hs.get_storage_controllers(),
                    "@other:test",
                    [invite_event, reject_event],
                    msc4115_membership_on_events=True,
                )
            ),
            [],
        )
 async def inject_visibility_event(
    hs: HomeServer,
    room_id: str,
    sender: str,
    visibility: str,
 ) -> EventBase:
    return await inject_event(
        hs,
        type="m.room.history_visibility",
        sender=sender,
        state_key="",
        room_id=room_id,
        content={"history_visibility": visibility},
    )
 async def inject_message_event(
    hs: HomeServer,
    room_id: str,
    sender: str,
    body: Optional[str] = "testytest",
 ) -> EventBase:
    return await inject_event(
        hs,
        type="m.room.message",
        sender=sender,
        room_id=room_id,
        content={"body": body, "msgtype": "m.text"},
    )
		`@@ -0,0 +1 @@`
							Adds validation to ensure that the `limit` parameter on `/publicRooms` is non-negative.
		`@@ -0,0 +1 @@`
							Return `400 M_NOT_JSON` upon receiving invalid JSON in query parameters across various client and admin endpoints, rather than an internal server error.
		`@@ -1 +0,0 @@`
			Fix various long-standing bugs which could cause incorrect state to be returned from `/sync` in certain situations.
		`@@ -0,0 +1 @@`
							Make the CSAPI endpoint `/keys/device_signing/upload` idempotent.
		`@@ -0,0 +1 @@`
							`Use new receipts column to optimise receipt and push action SQL queries. Contributed by Nick @ Beeper (@fizzadar).`
		`@@ -1 +0,0 @@`
			Add support for moving `/pushrules` off of main process.
		`@@ -1 +0,0 @@`
			`Refactor auth chain fetching to reduce duplication.`
		`@@ -1 +0,0 @@`
			Improve database performance by adding a missing index to `access_tokens.refresh_token_id`.
		`@@ -1 +0,0 @@`
			`Improve database performance by reducing number of receipts fetched when sending push notifications.`
		`@@ -0,0 +1 @@`
							`Add a prompt in the contributing guide to manually configure icu4c.`