wip

To make it more obvious that this method will also
attempt to unbind threepids that are deleted through it.

.git-blame-ignore-revs

@@ -21,8 +21,4 @@ aff1eb7c671b0a3813407321d2702ec46c71fa56
 0a00b7ff14890987f09112a2ae696c61001e6cf1
 
 # Convert tests/rest/admin/test_room.py to unix file endings (#7953).
-c4268e3da64f1abb5b31deaeb5769adb6510c0a7
-
-# Update black to 23.1.0 (#15103)
-9bb2eac71962970d02842bca441f4bcdbbf93a11
-
+c4268e3da64f1abb5b31deaeb5769adb6510c0a7

.github/workflows/docs-pr-netlify.yaml

@@ -14,7 +14,7 @@ jobs:
       # There's a 'download artifact' action, but it hasn't been updated for the workflow_run action
       # (https://github.com/actions/download-artifact/issues/60) so instead we get this mess:
       - name: 📥 Download artifact
-        uses: dawidd6/action-download-artifact@b59d8c6a6c5c6c6437954f470d963c0b20ea7415 # v2.25.0
+        uses: dawidd6/action-download-artifact@bd10f381a96414ce2b13a11bfa89902ba7cea07f # v2.24.3
         with:
           workflow: docs-pr.yaml
           run_id: ${{ github.event.workflow_run.id }}

.github/workflows/latest_deps.yml

@@ -27,7 +27,7 @@ jobs:
     steps:
       - uses: actions/checkout@v3
       - name: Install Rust
-        uses: dtolnay/rust-toolchain@e12eda571dc9a5ee5d58eecf4738ec291c66f295
+        uses: dtolnay/rust-toolchain@9cd00a88a73addc8617065438eff914dd08d0955
         with:
           toolchain: stable
       - uses: Swatinem/rust-cache@v2
@@ -61,7 +61,7 @@ jobs:
       - uses: actions/checkout@v3
 
       - name: Install Rust
-        uses: dtolnay/rust-toolchain@e12eda571dc9a5ee5d58eecf4738ec291c66f295
+        uses: dtolnay/rust-toolchain@9cd00a88a73addc8617065438eff914dd08d0955
         with:
           toolchain: stable
       - uses: Swatinem/rust-cache@v2
@@ -134,7 +134,7 @@ jobs:
       - uses: actions/checkout@v3
 
       - name: Install Rust
-        uses: dtolnay/rust-toolchain@e12eda571dc9a5ee5d58eecf4738ec291c66f295
+        uses: dtolnay/rust-toolchain@9cd00a88a73addc8617065438eff914dd08d0955
         with:
           toolchain: stable
       - uses: Swatinem/rust-cache@v2

.github/workflows/poetry_lockfile.yaml

@@ -1,24 +0,0 @@
-on:
-  push:
-    branches: ["develop", "release-*"]
-    paths:
-      - poetry.lock
-  pull_request:
-    paths:
-      - poetry.lock
-
-concurrency:
-  group: ${{ github.workflow }}-${{ github.ref }}
-  cancel-in-progress: true
-
-jobs:
-  check-sdists:
-    name: "Check locked dependencies have sdists"
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v3
-      - uses: actions/setup-python@v4
-        with:
-          python-version: '3.x'
-      - run: pip install tomli
-      - run: ./scripts-dev/check_locked_deps_have_sdists.py

.github/workflows/tests.yml

@@ -112,7 +112,7 @@ jobs:
         # There don't seem to be versioned releases of this action per se: for each rust
         # version there is a branch which gets constantly rebased on top of master.
         # We pin to a specific commit for paranoia's sake.
-        uses: dtolnay/rust-toolchain@e12eda571dc9a5ee5d58eecf4738ec291c66f295
+        uses: dtolnay/rust-toolchain@9cd00a88a73addc8617065438eff914dd08d0955
         with:
             toolchain: 1.58.1
             components: clippy
@@ -134,7 +134,7 @@ jobs:
         # There don't seem to be versioned releases of this action per se: for each rust
         # version there is a branch which gets constantly rebased on top of master.
         # We pin to a specific commit for paranoia's sake.
-        uses: dtolnay/rust-toolchain@e12eda571dc9a5ee5d58eecf4738ec291c66f295
+        uses: dtolnay/rust-toolchain@9cd00a88a73addc8617065438eff914dd08d0955
         with:
             toolchain: nightly-2022-12-01
             components: clippy
@@ -154,7 +154,7 @@ jobs:
         # There don't seem to be versioned releases of this action per se: for each rust
         # version there is a branch which gets constantly rebased on top of master.
         # We pin to a specific commit for paranoia's sake.
-        uses: dtolnay/rust-toolchain@e12eda571dc9a5ee5d58eecf4738ec291c66f295
+        uses: dtolnay/rust-toolchain@9cd00a88a73addc8617065438eff914dd08d0955
         with:
           toolchain: 1.58.1
           components: rustfmt
@@ -221,7 +221,7 @@ jobs:
         # There don't seem to be versioned releases of this action per se: for each rust
         # version there is a branch which gets constantly rebased on top of master.
         # We pin to a specific commit for paranoia's sake.
-        uses: dtolnay/rust-toolchain@e12eda571dc9a5ee5d58eecf4738ec291c66f295
+        uses: dtolnay/rust-toolchain@9cd00a88a73addc8617065438eff914dd08d0955
         with:
             toolchain: 1.58.1
       - uses: Swatinem/rust-cache@v2
@@ -266,7 +266,7 @@ jobs:
         # There don't seem to be versioned releases of this action per se: for each rust
         # version there is a branch which gets constantly rebased on top of master.
         # We pin to a specific commit for paranoia's sake.
-        uses: dtolnay/rust-toolchain@e12eda571dc9a5ee5d58eecf4738ec291c66f295
+        uses: dtolnay/rust-toolchain@9cd00a88a73addc8617065438eff914dd08d0955
         with:
             toolchain: 1.58.1
       - uses: Swatinem/rust-cache@v2
@@ -386,7 +386,7 @@ jobs:
         # There don't seem to be versioned releases of this action per se: for each rust
         # version there is a branch which gets constantly rebased on top of master.
         # We pin to a specific commit for paranoia's sake.
-        uses: dtolnay/rust-toolchain@e12eda571dc9a5ee5d58eecf4738ec291c66f295
+        uses: dtolnay/rust-toolchain@9cd00a88a73addc8617065438eff914dd08d0955
         with:
             toolchain: 1.58.1
       - uses: Swatinem/rust-cache@v2
@@ -531,7 +531,7 @@ jobs:
         # There don't seem to be versioned releases of this action per se: for each rust
         # version there is a branch which gets constantly rebased on top of master.
         # We pin to a specific commit for paranoia's sake.
-        uses: dtolnay/rust-toolchain@e12eda571dc9a5ee5d58eecf4738ec291c66f295
+        uses: dtolnay/rust-toolchain@9cd00a88a73addc8617065438eff914dd08d0955
         with:
             toolchain: 1.58.1
       - uses: Swatinem/rust-cache@v2
@@ -562,7 +562,7 @@ jobs:
         # There don't seem to be versioned releases of this action per se: for each rust
         # version there is a branch which gets constantly rebased on top of master.
         # We pin to a specific commit for paranoia's sake.
-        uses: dtolnay/rust-toolchain@e12eda571dc9a5ee5d58eecf4738ec291c66f295
+        uses: dtolnay/rust-toolchain@9cd00a88a73addc8617065438eff914dd08d0955
         with:
             toolchain: 1.58.1
       - uses: Swatinem/rust-cache@v2
@@ -585,7 +585,7 @@ jobs:
         # There don't seem to be versioned releases of this action per se: for each rust
         # version there is a branch which gets constantly rebased on top of master.
         # We pin to a specific commit for paranoia's sake.
-        uses: dtolnay/rust-toolchain@e12eda571dc9a5ee5d58eecf4738ec291c66f295
+        uses: dtolnay/rust-toolchain@9cd00a88a73addc8617065438eff914dd08d0955
         with:
             toolchain: nightly-2022-12-01
       - uses: Swatinem/rust-cache@v2

.github/workflows/twisted_trunk.yml

@@ -18,7 +18,7 @@ jobs:
       - uses: actions/checkout@v3
 
       - name: Install Rust
-        uses: dtolnay/rust-toolchain@e12eda571dc9a5ee5d58eecf4738ec291c66f295
+        uses: dtolnay/rust-toolchain@9cd00a88a73addc8617065438eff914dd08d0955
         with:
           toolchain: stable
       - uses: Swatinem/rust-cache@v2
@@ -43,7 +43,7 @@ jobs:
       - run: sudo apt-get -qq install xmlsec1
 
       - name: Install Rust
-        uses: dtolnay/rust-toolchain@e12eda571dc9a5ee5d58eecf4738ec291c66f295
+        uses: dtolnay/rust-toolchain@9cd00a88a73addc8617065438eff914dd08d0955
         with:
           toolchain: stable
       - uses: Swatinem/rust-cache@v2
@@ -82,7 +82,7 @@ jobs:
       - uses: actions/checkout@v3
 
       - name: Install Rust
-        uses: dtolnay/rust-toolchain@e12eda571dc9a5ee5d58eecf4738ec291c66f295
+        uses: dtolnay/rust-toolchain@9cd00a88a73addc8617065438eff914dd08d0955
         with:
           toolchain: stable
       - uses: Swatinem/rust-cache@v2

.gitignore

@@ -72,3 +72,8 @@ book/
 
 # Don't include users' poetry configs
 /poetry.toml
+
+# Devenv
+.devenv*
+devenv.local.nix
+

CHANGES.md

@@ -1,112 +1,3 @@
-Synapse 1.78.0rc1 (2023-02-21)
-==============================
-
-Features
---------
-
-- Implement the experimental `exact_event_match` push rule condition from [MSC3758](https://github.com/matrix-org/matrix-spec-proposals/pull/3758). ([\#14964](https://github.com/matrix-org/synapse/issues/14964))
-- Add account data to the command line [user data export tool](https://matrix-org.github.io/synapse/v1.78/usage/administration/admin_faq.html#how-can-i-export-user-data). ([\#14969](https://github.com/matrix-org/synapse/issues/14969))
-- Implement [MSC3873](https://github.com/matrix-org/matrix-spec-proposals/pull/3873) to disambiguate push rule keys with dots in them. ([\#15004](https://github.com/matrix-org/synapse/issues/15004))
-- Allow Synapse to use a specific Redis [logical database](https://redis.io/commands/select/) in worker-mode deployments. ([\#15034](https://github.com/matrix-org/synapse/issues/15034))
-- Tag opentracing spans for federation requests with the name of the worker serving the request. ([\#15042](https://github.com/matrix-org/synapse/issues/15042))
-- Implement the experimental `exact_event_property_contains` push rule condition from [MSC3966](https://github.com/matrix-org/matrix-spec-proposals/pull/3966). ([\#15045](https://github.com/matrix-org/synapse/issues/15045))
-- Remove spurious `dont_notify` action from the defaults for the `.m.rule.reaction` pushrule. ([\#15073](https://github.com/matrix-org/synapse/issues/15073))
-- Update the error code returned when user sends a duplicate annotation. ([\#15075](https://github.com/matrix-org/synapse/issues/15075))
-
-
-Bugfixes
---------
-
-- Prevent clients from reporting nonexistent events. ([\#13779](https://github.com/matrix-org/synapse/issues/13779))
-- Return spec-compliant JSON errors when unknown endpoints are requested. ([\#14605](https://github.com/matrix-org/synapse/issues/14605))
-- Fix a long-standing bug where the room aliases returned could be corrupted. ([\#15038](https://github.com/matrix-org/synapse/issues/15038))
-- Fix a bug introduced in Synapse 1.76.0 where partially-joined rooms could not be deleted using the [purge room API](https://matrix-org.github.io/synapse/latest/admin_api/rooms.html#delete-room-api). ([\#15068](https://github.com/matrix-org/synapse/issues/15068))
-- Fix a long-standing bug where federated joins would fail if the first server in the list of servers to try is not in the room. ([\#15074](https://github.com/matrix-org/synapse/issues/15074))
-- Fix a bug introduced in Synapse v1.74.0 where searching with colons when using ICU for search term tokenisation would fail with an error. ([\#15079](https://github.com/matrix-org/synapse/issues/15079))
-- Reduce the likelihood of a rare race condition where rejoining a restricted room over federation would fail. ([\#15080](https://github.com/matrix-org/synapse/issues/15080))
-- Fix a bug introduced in Synapse 1.76 where workers would fail to start if the `health` listener was configured. ([\#15096](https://github.com/matrix-org/synapse/issues/15096))
-- Fix a bug introduced in Synapse 1.75 where the [portdb script](https://matrix-org.github.io/synapse/release-v1.78/postgres.html#porting-from-sqlite) would fail to run after a room had been faster-joined. ([\#15108](https://github.com/matrix-org/synapse/issues/15108))
-
-
-Improved Documentation
-----------------------
-
-- Document how to start Synapse with Poetry. Contributed by @thezaidbintariq. ([\#14892](https://github.com/matrix-org/synapse/issues/14892), [\#15022](https://github.com/matrix-org/synapse/issues/15022))
-- Update delegation documentation to clarify that SRV DNS delegation does not eliminate all needs to serve files from .well-known locations. Contributed by @williamkray. ([\#14959](https://github.com/matrix-org/synapse/issues/14959))
-- Fix a mistake in registration_shared_secret_path docs. ([\#15078](https://github.com/matrix-org/synapse/issues/15078))
-- Refer to a more recent blog post on the [Database Maintenance Tools](https://matrix-org.github.io/synapse/latest/usage/administration/database_maintenance_tools.html) page. Contributed by @jahway603. ([\#15083](https://github.com/matrix-org/synapse/issues/15083))
-
-
-Internal Changes
-----------------
-
-- Re-type hint some collections as read-only. ([\#13755](https://github.com/matrix-org/synapse/issues/13755))
-- Faster joins: don't stall when another user joins during a partial-state room resync. ([\#14606](https://github.com/matrix-org/synapse/issues/14606))
-- Add a class `UnpersistedEventContext` to allow for the batching up of storing state groups. ([\#14675](https://github.com/matrix-org/synapse/issues/14675))
-- Add a check to ensure that locked dependencies have source distributions available. ([\#14742](https://github.com/matrix-org/synapse/issues/14742))
-- Tweak comment on `_is_local_room_accessible` as part of room visibility in `/hierarchy` to clarify the condition for a room being visible. ([\#14834](https://github.com/matrix-org/synapse/issues/14834))
-- Prevent `WARNING: there is already a transaction in progress` lines appearing in PostgreSQL's logs on some occasions. ([\#14840](https://github.com/matrix-org/synapse/issues/14840))
-- Use `StrCollection` to avoid potential bugs with `Collection[str]`. ([\#14929](https://github.com/matrix-org/synapse/issues/14929))
-- Improve performance of `/sync` in a few situations. ([\#14973](https://github.com/matrix-org/synapse/issues/14973))
-- Limit concurrent event creation for a room to avoid state resolution when sending bursts of events to a local room. ([\#14977](https://github.com/matrix-org/synapse/issues/14977))
-- Skip calculating unread push actions in /sync when enable_push is false. ([\#14980](https://github.com/matrix-org/synapse/issues/14980))
-- Add a schema dump symlinks inside `contrib`, to make it easier for IDEs to interrogate Synapse's database schema. ([\#14982](https://github.com/matrix-org/synapse/issues/14982))
-- Improve type hints. ([\#15008](https://github.com/matrix-org/synapse/issues/15008), [\#15026](https://github.com/matrix-org/synapse/issues/15026), [\#15027](https://github.com/matrix-org/synapse/issues/15027), [\#15028](https://github.com/matrix-org/synapse/issues/15028), [\#15031](https://github.com/matrix-org/synapse/issues/15031), [\#15035](https://github.com/matrix-org/synapse/issues/15035), [\#15052](https://github.com/matrix-org/synapse/issues/15052), [\#15072](https://github.com/matrix-org/synapse/issues/15072), [\#15084](https://github.com/matrix-org/synapse/issues/15084))
-- Update [MSC3952](https://github.com/matrix-org/matrix-spec-proposals/pull/3952) support based on changes to the MSC. ([\#15037](https://github.com/matrix-org/synapse/issues/15037))
-- Avoid mutating a cached value in `get_user_devices_from_cache`. ([\#15040](https://github.com/matrix-org/synapse/issues/15040))
-- Fix a rare exception in logs on start up. ([\#15041](https://github.com/matrix-org/synapse/issues/15041))
-- Update pyo3-log to v0.8.1. ([\#15043](https://github.com/matrix-org/synapse/issues/15043))
-- Avoid mutating cached values in `_generate_sync_entry_for_account_data`. ([\#15047](https://github.com/matrix-org/synapse/issues/15047))
-- Refactor arguments of `try_unbind_threepid` and `_try_unbind_threepid_with_id_server` to not use dictionaries. ([\#15053](https://github.com/matrix-org/synapse/issues/15053))
-- Merge debug logging from the hotfixes branch. ([\#15054](https://github.com/matrix-org/synapse/issues/15054))
-- Faster joins: omit device list updates originating from partial state rooms in /sync responses without lazy loading of members enabled. ([\#15069](https://github.com/matrix-org/synapse/issues/15069))
-- Fix clashing database transaction name. ([\#15070](https://github.com/matrix-org/synapse/issues/15070))
-- Upper-bound frozendict dependency. This works around us being unable to test installing our wheels against Python 3.11 in CI. ([\#15114](https://github.com/matrix-org/synapse/issues/15114))
-- Tweak logging for when a worker waits for its view of a replication stream to catch up. ([\#15120](https://github.com/matrix-org/synapse/issues/15120))
-
-<details><summary>Locked dependency updates</summary>
-
-- Bump bleach from 5.0.1 to 6.0.0. ([\#15059](https://github.com/matrix-org/synapse/issues/15059))
-- Bump cryptography from 38.0.4 to 39.0.1. ([\#15020](https://github.com/matrix-org/synapse/issues/15020))
-- Bump ruff version from 0.0.230 to 0.0.237. ([\#15033](https://github.com/matrix-org/synapse/issues/15033))
-- Bump dtolnay/rust-toolchain from 9cd00a88a73addc8617065438eff914dd08d0955 to 25dc93b901a87e864900a8aec6c12e9aa794c0c3. ([\#15060](https://github.com/matrix-org/synapse/issues/15060))
-- Bump systemd-python from 234 to 235. ([\#15061](https://github.com/matrix-org/synapse/issues/15061))
-- Bump serde_json from 1.0.92 to 1.0.93. ([\#15062](https://github.com/matrix-org/synapse/issues/15062))
-- Bump types-requests from 2.28.11.8 to 2.28.11.12. ([\#15063](https://github.com/matrix-org/synapse/issues/15063))
-- Bump types-pillow from 9.4.0.5 to 9.4.0.10. ([\#15064](https://github.com/matrix-org/synapse/issues/15064))
-- Bump sentry-sdk from 1.13.0 to 1.15.0. ([\#15065](https://github.com/matrix-org/synapse/issues/15065))
-- Bump types-jsonschema from 4.17.0.3 to 4.17.0.5. ([\#15099](https://github.com/matrix-org/synapse/issues/15099))
-- Bump types-bleach from 5.0.3.1 to 6.0.0.0. ([\#15100](https://github.com/matrix-org/synapse/issues/15100))
-- Bump dtolnay/rust-toolchain from 25dc93b901a87e864900a8aec6c12e9aa794c0c3 to e12eda571dc9a5ee5d58eecf4738ec291c66f295. ([\#15101](https://github.com/matrix-org/synapse/issues/15101))
-- Bump dawidd6/action-download-artifact from 2.24.3 to 2.25.0. ([\#15102](https://github.com/matrix-org/synapse/issues/15102))
-- Bump types-pillow from 9.4.0.10 to 9.4.0.13. ([\#15104](https://github.com/matrix-org/synapse/issues/15104))
-- Bump types-setuptools from 67.1.0.0 to 67.3.0.1. ([\#15105](https://github.com/matrix-org/synapse/issues/15105))
-
-
-</details>
-
-
-Synapse 1.77.0 (2023-02-14)
-===========================
-
-No significant changes since 1.77.0rc2.
-
-
-Synapse 1.77.0rc2 (2023-02-10)
-==============================
-
-Bugfixes
---------
-
-- Fix bug where retried replication requests would return a failure. Introduced in v1.76.0. ([\#15024](https://github.com/matrix-org/synapse/issues/15024))
-
-
-Internal Changes
-----------------
-
-- Prepare for future database schema changes. ([\#15036](https://github.com/matrix-org/synapse/issues/15036))
-
-
 Synapse 1.77.0rc1 (2023-02-07)
 ==============================
 
@@ -151,7 +42,7 @@ Internal Changes
 - Preparatory work for adding a denormalised event stream ordering column in the future. Contributed by Nick @ Beeper (@fizzadar). ([\#14979](https://github.com/matrix-org/synapse/issues/14979), [9cd7610](https://github.com/matrix-org/synapse/commit/9cd7610f86ab5051c9365dd38d1eec405a5f8ca6), [f10caa7](https://github.com/matrix-org/synapse/commit/f10caa73eee0caa91cf373966104d1ededae2aee); see [\#15014](https://github.com/matrix-org/synapse/issues/15014))
 - Add tests for `_flatten_dict`. ([\#14981](https://github.com/matrix-org/synapse/issues/14981), [\#15002](https://github.com/matrix-org/synapse/issues/15002))
 
-<details><summary>Locked dependency updates</summary>
+<details><summary>Dependabot updates</summary>
 
 - Bump dtolnay/rust-toolchain from e645b0cf01249a964ec099494d38d2da0f0b349f to 9cd00a88a73addc8617065438eff914dd08d0955. ([\#14968](https://github.com/matrix-org/synapse/issues/14968))
 - Bump docker/build-push-action from 3 to 4. ([\#14952](https://github.com/matrix-org/synapse/issues/14952))

Cargo.lock

@@ -232,9 +232,9 @@ dependencies = [
 
 [[package]]
 name = "pyo3-log"
-version = "0.8.1"
+version = "0.7.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f9c8b57fe71fb5dcf38970ebedc2b1531cf1c14b1b9b4c560a182a57e115575c"
+checksum = "e5695ccff5060c13ca1751cf8c857a12da9b0bf0378cb071c5e0326f7c7e4c1b"
 dependencies = [
  "arc-swap",
  "log",
@@ -343,9 +343,9 @@ dependencies = [
 
 [[package]]
 name = "serde_json"
-version = "1.0.93"
+version = "1.0.92"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "cad406b69c91885b5107daf2c29572f6c8cdb3c66826821e286c533490c0bc76"
+checksum = "7434af0dc1cbd59268aa98b4c22c131c0584d2232f6fb166efb993e2832e896a"
 dependencies = [
  "itoa",
  "ryu",

changelog.d/14605.bugfix

@@ -0,0 +1 @@
+Return spec-compliant JSON errors when unknown endpoints are requested.

changelog.d/14840.misc

@@ -0,0 +1 @@
+Prevent "WARNING:  there is already a transaction in progress" lines appearing in PostgreSQL's logs on some occasions.

changelog.d/14959.doc

@@ -0,0 +1 @@
+Update delegation documentation to clarify that SRV DNS delegation does not eliminate all needs to serve files from .well-known locations. Contributed by @williamkray.

changelog.d/14977.misc

@@ -0,0 +1 @@
+Limit concurrent event creation for a room to avoid state resolution when sending bursts of events to a local room.

changelog.d/15004.feature

@@ -0,0 +1 @@
+Implement [MSC3873](https://github.com/matrix-org/matrix-spec-proposals/pull/3873) to unambiguate push rule keys with dots in them.

changelog.d/15008.misc

@@ -0,0 +1 @@
+Improve type hints.

changelog.d/15020.misc

@@ -0,0 +1 @@
+Bump cryptography from 38.0.4 to 39.0.1.

changelog.d/15022.doc

@@ -0,0 +1 @@
+Document how to start Synapse in the contributing guide.

changelog.d/15026.misc

@@ -0,0 +1 @@
+Improve type hints.

changelog.d/15027.misc

@@ -0,0 +1 @@
+Improve type hints.

changelog.d/15028.misc

@@ -0,0 +1 @@
+Improve type hints.

changelog.d/15031.misc

@@ -0,0 +1 @@
+Improve type hints.

changelog.d/15033.misc

@@ -0,0 +1 @@
+Bump ruff version from 0.0.230 to 0.0.237.

changelog.d/15035.misc

@@ -0,0 +1 @@
+Improve type hints.

changelog.d/15038.bugfix

@@ -0,0 +1 @@
+Fix a long-standing bug where the room aliases returned could be corrupted.

changelog.d/15044.feature

@@ -0,0 +1 @@
+Add two new Third Party Rules module API callbacks: [`on_add_user_third_party_identifier`](https://matrix-org.github.io/synapse/v1.78/modules/third_party_rules_callbacks.html#on_add_user_third_party_identifier) and [`on_remove_user_third_party_identifier`](https://matrix-org.github.io/synapse/v1.78/modules/third_party_rules_callbacks.html#on_remove_user_third_party_identifier).

changelog.d/15071.doc

@@ -1 +0,0 @@
-Clarify which worker processes the ThirdPartyRules' [`on_new_event`](https://matrix-org.github.io/synapse/v1.78/modules/third_party_rules_callbacks.html#on_new_event) module API callback runs on.

changelog.d/15093.bugfix

@@ -1 +0,0 @@
-Remove the unspecced `room_alias` field from the [`/createRoom`](https://spec.matrix.org/v1.6/client-server-api/#post_matrixclientv3createroom) response.

changelog.d/15095.misc

@@ -1 +0,0 @@
-Refactor writing json data in `FileExfiltrationWriter`.

changelog.d/15103.misc

@@ -1 +0,0 @@
-Bump black from 22.12.0 to 23.1.0.

changelog.d/15107.feature

@@ -1 +0,0 @@
-Add media information to the command line [user data export tool](https://matrix-org.github.io/synapse/v1.79/usage/administration/admin_faq.html#how-can-i-export-user-data).

changelog.d/15112.doc

@@ -1 +0,0 @@
-Document using [Shibboleth](https://www.shibboleth.net/) as an OpenID Provider.

changelog.d/15135.misc

@@ -1 +0,0 @@
-Tighten the login ratelimit defaults.

changelog.d/15139.doc

@@ -1 +0,0 @@
-Correct reference to `federation_verify_certificates` in configuration documentation.

changelog.d/15149.feature

@@ -1 +0,0 @@
-Add a new `send_federation_http_request` method to the Module API to allow Synapse modules to make matrix federation requests over HTTP.

contrib/datagrip/README.md

@@ -1,28 +0,0 @@
-# Schema symlinks
-
-This directory contains symlinks to the latest dump of the postgres full schema. This is useful to have, as it allows IDEs to understand our schema and provide autocomplete, linters, inspections, etc.
-
-In particular, the DataGrip functionality in IntelliJ's products seems to only consider files called `*.sql` when defining a schema from DDL; `*.sql.postgres` will be ignored. To get around this we symlink those files to ones ending in `.sql`. We've chosen to ignore the `.sql.sqlite` schema dumps here, as they're not intended for production use (and are much quicker to test against).
-
-## Example
-![](datagrip-aware-of-schema.png)
-
-## Caveats
-
-- Doesn't include temporary tables created ad-hoc by Synapse.
-- Postgres only. IDEs will likely be confused by SQLite-specific queries.
-- Will not include migrations created after the latest schema dump.
-- Symlinks might confuse checkouts on Windows systems.
-
-## Instructions
-
-### Jetbrains IDEs with DataGrip plugin
-
-- View -> Tool Windows -> Database
-- `+` Icon -> DDL Data Source
-- Pick a name, e.g. `Synapse schema dump`
-- Under sources, click `+`.
-- Add an entry with Path pointing to this directory, and dialect set to PostgreSQL.
-- OK, and OK.
-- IDE should now be aware of the schema.
-- Try control-clicking on a table name in a bit of SQL e.g. in `_get_forgotten_rooms_for_user_txn`.

contrib/datagrip/common.sql

@@ -1 +0,0 @@
-../../synapse/storage/schema/common/full_schemas/72/full.sql.postgres

contrib/datagrip/main.sql

@@ -1 +0,0 @@
-../../synapse/storage/schema/main/full_schemas/72/full.sql.postgres

contrib/datagrip/schema_version.sql

@@ -1 +0,0 @@
-../../synapse/storage/schema/common/schema_version.sql

contrib/datagrip/state.sql

@@ -1 +0,0 @@
-../../synapse/storage/schema/state/full_schemas/72/full.sql.postgres

contrib/docker_compose_workers/README.md

@@ -68,7 +68,6 @@ redis:
   enabled: true
   host: redis
   port: 6379
-  # dbid:  <redis_logical_db_id>
   # password: <secret_password>  
 ```
 

debian/changelog

@@ -1,22 +1,3 @@
-matrix-synapse-py3 (1.78.0~rc1) stable; urgency=medium
-
-  * Add `matrix-org-archive-keyring` package as recommended.
-  * New Synapse release 1.78.0rc1.
-
- -- Synapse Packaging team <packages@matrix.org>  Tue, 21 Feb 2023 14:29:19 +0000
-
-matrix-synapse-py3 (1.77.0) stable; urgency=medium
-
-  * New Synapse release 1.77.0.
-
- -- Synapse Packaging team <packages@matrix.org>  Tue, 14 Feb 2023 12:59:02 +0100
-
-matrix-synapse-py3 (1.77.0~rc2) stable; urgency=medium
-
-  * New Synapse release 1.77.0rc2.
-
- -- Synapse Packaging team <packages@matrix.org>  Fri, 10 Feb 2023 12:44:21 +0000
-
 matrix-synapse-py3 (1.77.0~rc1) stable; urgency=medium
 
   * New Synapse release 1.77.0rc1.

debian/control

@@ -37,7 +37,6 @@ Depends:
 # so we put perl:Depends in Suggests rather than Depends.
 Recommends:
  ${shlibs1:Recommends},
- matrix-org-archive-keyring,
 Suggests:
  sqlite3,
  ${perl:Depends},

devenv.lock

@@ -0,0 +1,177 @@
+{
+  "nodes": {
+    "devenv": {
+      "locked": {
+        "dir": "src/modules",
+        "lastModified": 1673960114,
+        "narHash": "sha256-YNCok1a8cy71nP0idJds2Dwn2B1T6zGw9+2H1A0lNa0=",
+        "owner": "cachix",
+        "repo": "devenv",
+        "rev": "0960585a7221e6ede718cc9a2c2eade7ce75c229",
+        "type": "github"
+      },
+      "original": {
+        "dir": "src/modules",
+        "owner": "cachix",
+        "repo": "devenv",
+        "type": "github"
+      }
+    },
+    "fenix": {
+      "inputs": {
+        "nixpkgs": [
+          "nixpkgs"
+        ],
+        "rust-analyzer-src": "rust-analyzer-src"
+      },
+      "locked": {
+        "lastModified": 1674023045,
+        "narHash": "sha256-btQC+gTeVLmX9cYl/6Kig7BuDltVWJEh7TrITAc6QjA=",
+        "owner": "nix-community",
+        "repo": "fenix",
+        "rev": "75dbe699bc57323cdef636b82bcfef6028bd1530",
+        "type": "github"
+      },
+      "original": {
+        "owner": "nix-community",
+        "repo": "fenix",
+        "type": "github"
+      }
+    },
+    "flake-compat": {
+      "flake": false,
+      "locked": {
+        "lastModified": 1668681692,
+        "narHash": "sha256-Ht91NGdewz8IQLtWZ9LCeNXMSXHUss+9COoqu6JLmXU=",
+        "owner": "edolstra",
+        "repo": "flake-compat",
+        "rev": "009399224d5e398d03b22badca40a37ac85412a1",
+        "type": "github"
+      },
+      "original": {
+        "owner": "edolstra",
+        "repo": "flake-compat",
+        "type": "github"
+      }
+    },
+    "flake-utils": {
+      "locked": {
+        "lastModified": 1667395993,
+        "narHash": "sha256-nuEHfE/LcWyuSWnS8t12N1wc105Qtau+/OdUAjtQ0rA=",
+        "owner": "numtide",
+        "repo": "flake-utils",
+        "rev": "5aed5285a952e0b949eb3ba02c12fa4fcfef535f",
+        "type": "github"
+      },
+      "original": {
+        "owner": "numtide",
+        "repo": "flake-utils",
+        "type": "github"
+      }
+    },
+    "gitignore": {
+      "inputs": {
+        "nixpkgs": [
+          "pre-commit-hooks",
+          "nixpkgs"
+        ]
+      },
+      "locked": {
+        "lastModified": 1660459072,
+        "narHash": "sha256-8DFJjXG8zqoONA1vXtgeKXy68KdJL5UaXR8NtVMUbx8=",
+        "owner": "hercules-ci",
+        "repo": "gitignore.nix",
+        "rev": "a20de23b925fd8264fd7fad6454652e142fd7f73",
+        "type": "github"
+      },
+      "original": {
+        "owner": "hercules-ci",
+        "repo": "gitignore.nix",
+        "type": "github"
+      }
+    },
+    "nixpkgs": {
+      "locked": {
+        "lastModified": 1673947312,
+        "narHash": "sha256-xx/2nRwRy3bXrtry6TtydKpJpqHahjuDB5sFkQ/XNDE=",
+        "owner": "NixOS",
+        "repo": "nixpkgs",
+        "rev": "2d38b664b4400335086a713a0036aafaa002c003",
+        "type": "github"
+      },
+      "original": {
+        "owner": "NixOS",
+        "ref": "nixpkgs-unstable",
+        "repo": "nixpkgs",
+        "type": "github"
+      }
+    },
+    "nixpkgs-stable": {
+      "locked": {
+        "lastModified": 1671271954,
+        "narHash": "sha256-cSvu+bnvN08sOlTBWbBrKaBHQZq8mvk8bgpt0ZJ2Snc=",
+        "owner": "NixOS",
+        "repo": "nixpkgs",
+        "rev": "d513b448cc2a6da2c8803e3c197c9fc7e67b19e3",
+        "type": "github"
+      },
+      "original": {
+        "owner": "NixOS",
+        "ref": "nixos-22.05",
+        "repo": "nixpkgs",
+        "type": "github"
+      }
+    },
+    "pre-commit-hooks": {
+      "inputs": {
+        "flake-compat": "flake-compat",
+        "flake-utils": "flake-utils",
+        "gitignore": "gitignore",
+        "nixpkgs": [
+          "nixpkgs"
+        ],
+        "nixpkgs-stable": "nixpkgs-stable"
+      },
+      "locked": {
+        "lastModified": 1674046351,
+        "narHash": "sha256-vNErPj4gfO/G1vHuOh5/IbjLaydwePcRlD0fXlnUbmI=",
+        "owner": "cachix",
+        "repo": "pre-commit-hooks.nix",
+        "rev": "756cc26afb75b0f8bfec48bbc54a8836a04953fb",
+        "type": "github"
+      },
+      "original": {
+        "owner": "cachix",
+        "repo": "pre-commit-hooks.nix",
+        "type": "github"
+      }
+    },
+    "root": {
+      "inputs": {
+        "devenv": "devenv",
+        "fenix": "fenix",
+        "nixpkgs": "nixpkgs",
+        "pre-commit-hooks": "pre-commit-hooks"
+      }
+    },
+    "rust-analyzer-src": {
+      "flake": false,
+      "locked": {
+        "lastModified": 1673979364,
+        "narHash": "sha256-ecgQENol9XhhcYF+M9B8FMrsWYQ/ZvRsvgEWi8HI6D0=",
+        "owner": "rust-lang",
+        "repo": "rust-analyzer",
+        "rev": "3a7271336536f2cd558498755254ae8c0e73baa7",
+        "type": "github"
+      },
+      "original": {
+        "owner": "rust-lang",
+        "ref": "nightly",
+        "repo": "rust-analyzer",
+        "type": "github"
+      }
+    }
+  },
+  "root": "root",
+  "version": 7
+}

devenv.nix

@@ -0,0 +1,122 @@
+{ inputs, pkgs, ... }:
+
+{
+  # Configure packages to install.
+  # Search for package names at https://search.nixos.org/packages?channel=unstable
+  packages = with pkgs; [
+    # Native dependencies for running Synapse.
+    icu
+    libffi
+    libjpeg
+    libpqxx
+    libwebp
+    libxml2
+    libxslt
+    sqlite
+
+    # Native dependencies for unit tests (SyTest also requires OpenSSL).
+    openssl
+
+    # Native dependencies for running Complement.
+    olm
+
+    # Development tools.
+    poetry
+  ];
+
+  # Activate (and create if necessary) a poetry virtualenv on startup.
+  enterShell = ''
+    . "$(dirname $(poetry run which python))/activate"
+  '';
+
+  # Install dependencies for the additional programming languages
+  # involved with Synapse development. Python is already available
+  # from poetry's virtual environment.
+  #
+  # * Rust is used for developing and running Synapse.
+  # * Golang is needed to run the Complement test suite.
+  # * Perl is needed to run the SyTest test suite.
+  languages.go.enable = true;
+  languages.rust.enable = true;
+  languages.rust.version = "latest";
+  languages.perl.enable = true;
+
+  # Postgres is needed to run Synapse with postgres support and
+  # to run certain unit tests that require postgres.
+  services.postgres.enable = true;
+
+  # On the first invocation of `devenv up`, create a database for
+  # Syanpse to store data in.
+  services.postgres.initdbArgs = ["--locale=C" "--encoding=UTF8"];
+  services.postgres.initialDatabases = [
+    { name = "synapse"; }
+  ];
+
+  # Redis is needed in order to run Synapse in worker mode.
+  services.redis.enable = true;
+
+  # We wrap `poetry` with a bash script that disables the download
+  # of binary wheels for certain packages if the user is running
+  # NixOS. NixOS is special in that you can have multiple versions
+  # of packages installed at once, including your libc linker!
+  #
+  # Some binaries built for Linux expect those to be in a certain
+  # filepath, but that is not the case on NixOS. In that case, we
+  # force compiling those binaries locally instead.
+  scripts.poetry.exec = ''
+  if [ -z "$__NIXOS_SET_ENVIRONMENT_DONE" ]; then
+    # We are running on NixOS.
+    #
+    # Prevent poetry from downloading known problematic,
+    # dynamically-linked binaries for python dependencies.
+    POETRY_INSTALLER_NO_BINARY=ruff ${pkgs.poetry}/bin/poetry $@
+  else
+    ${pkgs.poetry}/bin/poetry $@
+  fi
+  '';
+
+  # Define the perl modules we require to run SyTest.
+  #
+  # This list was compiled by cross-referencing https://metacpan.org/
+  # with the modules defined in './cpanfile' and then finding the
+  # corresponding nix packages on https://search.nixos.org/packages.
+  #
+  # This was done until `./install-deps.pl --dryrun` produced no output.
+  env.PERL5LIB = "${with pkgs.perl536Packages; makePerlPath [
+    DBI
+    ClassMethodModifiers
+    CryptEd25519
+    DataDump
+    DBDPg
+    DigestHMAC
+    DigestSHA1
+    EmailAddressXS
+    EmailMIME
+    EmailSimple  # required by Email::Mime
+    EmailMessageID  # required by Email::Mime
+    EmailMIMEContentType  # required by Email::Mime
+    TextUnidecode  # required by Email::Mime
+    ModuleRuntime  # required by Email::Mime
+    EmailMIMEEncodings  # required by Email::Mime
+    FilePath
+    FileSlurper
+    Future
+    GetoptLong
+    HTTPMessage
+    IOAsync
+    IOAsyncSSL
+    IOSocketSSL
+    NetSSLeay
+    JSON
+    ListUtilsBy
+    ScalarListUtils
+    ModulePluggable
+    NetAsyncHTTP
+    MetricsAny  # required by Net::Async::HTTP
+    NetAsyncHTTPServer
+    StructDumb
+    URI
+    YAMLLibYAML
+  ]}";
+
+}

devenv.yaml

@@ -0,0 +1,8 @@
+inputs:
+  nixpkgs:
+    url: github:NixOS/nixpkgs/nixpkgs-unstable
+  fenix:
+    url: github:nix-community/fenix
+    inputs:
+      nixpkgs:
+        follows: nixpkgs

docs/modules/third_party_rules_callbacks.md

@@ -146,9 +146,6 @@ Note that this callback is called when the event has already been processed and
 into the room, which means this callback cannot be used to deny persisting the event. To
 deny an incoming event, see [`check_event_for_spam`](spam_checker_callbacks.md#check_event_for_spam) instead.
 
-For any given event, this callback will be called on every worker process, even if that worker will not end up
-acting on that event. This callback will not be called for events that are marked as rejected.
-
 If multiple modules implement this callback, Synapse runs them all in order.
 
 ### `check_can_shutdown_room`
@@ -254,6 +251,13 @@ If multiple modules implement this callback, Synapse runs them all in order.
 
 _First introduced in Synapse v1.56.0_
 
+**<span style="color:red">
+This callback is deprecated in favour of the `on_add_user_third_party_identifier` callback, which
+features nearly the same functionality. The only difference is that while `on_threepid_bind`
+was called *after* a third-party ID was associated with a user's account on the homeserver,
+`on_add_user_third_party_identifier` is called just *before* a third-party ID is associated.
+</span>**
+
 ```python
 async def on_threepid_bind(user_id: str, medium: str, address: str) -> None:
 ```
@@ -268,6 +272,44 @@ server_.
 
 If multiple modules implement this callback, Synapse runs them all in order.
 
+### `on_add_user_third_party_identifier`
+
+_First introduced in Synapse v1.78.0_
+
+```python
+async def on_add_user_third_party_identifier(user_id: str, medium: str, address: str) -> None:
+```
+
+Called just before creating an association between a user and a third-party identifier
+(email address, phone number). The module is given the Matrix ID of the user the
+association is for, as well as the medium (`email` or `msisdn`) and address of the
+third-party identifier (i.e. an email address).
+
+Note that this callback is _not_ called if a user attempts to bind their third-party identifier
+to an identity server (via a call to [`POST
+/_matrix/client/v3/account/3pid/bind`](https://spec.matrix.org/v1.5/client-server-api/#post_matrixclientv3account3pidbind)).
+
+If multiple modules implement this callback, Synapse runs them all in order.
+
+### `on_remove_user_third_party_identifier`
+
+_First introduced in Synapse v1.78.0_
+
+```python
+async def on_remove_user_third_party_identifier(user_id: str, medium: str, address: str) -> None:
+```
+
+Called just before removing an association between a user and a third-party identifier
+(email address, phone number). The module is given the Matrix ID of the user the
+association is for, as well as the medium (`email` or `msisdn`) and address of the
+third-party identifier (i.e. an email address).
+
+Note that this callback is _not_ called if a user attempts to unbind their third-party
+identifier from an identity server (via a call to [`POST
+/_matrix/client/v3/account/3pid/unbind`](https://spec.matrix.org/v1.5/client-server-api/#post_matrixclientv3account3pidunbind)).
+
+If multiple modules implement this callback, Synapse runs them all in order.
+
 ## Example
 
 The example below is a module that implements the third-party rules callback
@@ -300,4 +342,4 @@ class EventCensorer:
         )
         event_dict["content"] = new_event_content
         return event_dict
-```
+```

docs/openid.md

@@ -590,47 +590,6 @@ oidc_providers:
 
 Note that the fields `client_id` and `client_secret` are taken from the CURL response above.
 
-### Shibboleth with OIDC Plugin
-
-[Shibboleth](https://www.shibboleth.net/) is an open Standard IdP solution widely used by Universities.
-
-1. Shibboleth needs the [OIDC Plugin](https://shibboleth.atlassian.net/wiki/spaces/IDPPLUGINS/pages/1376878976/OIDC+OP) installed and working correctly.
-2. Create a new config on the IdP Side, ensure that the `client_id` and `client_secret`
-   are randomly generated data.
-```json
-{
-    "client_id": "SOME-CLIENT-ID",
-    "client_secret": "SOME-SUPER-SECRET-SECRET",
-    "response_types": ["code"],
-    "grant_types": ["authorization_code"],
-    "scope": "openid profile email",
-    "redirect_uris": ["https://[synapse public baseurl]/_synapse/client/oidc/callback"]
-}
-```
-
-Synapse config:
-
-```yaml
-oidc_providers:
-  # Shibboleth IDP
-  #
-  - idp_id: shibboleth
-    idp_name: "Shibboleth Login"
-    discover: true
-    issuer: "https://YOUR-IDP-URL.TLD"
-    client_id: "YOUR_CLIENT_ID"
-    client_secret: "YOUR-CLIENT-SECRECT-FROM-YOUR-IDP"
-    scopes: ["openid", "profile", "email"]
-    allow_existing_users: true
-    user_profile_method: "userinfo_endpoint"
-    user_mapping_provider:
-      config:
-        subject_claim: "sub"
-        localpart_template: "{{ user.sub.split('@')[0] }}"
-        display_name_template: "{{ user.name }}"
-        email_template: "{{ user.email }}"
-```
-
 ### Twitch
 
 1. Setup a developer account on [Twitch](https://dev.twitch.tv/)

docs/upgrade.md

@@ -97,6 +97,32 @@ admin API with an identical endpoint at `/_synapse/admin/v1/media/delete`. Pleas
 update your tooling to use the new endpoint. The deprecated version will be removed
 in a future release.
 
+## The `on_threepid_bind` module callback method has been deprecated
+
+The [`on_threepid_bind`](modules/third_party_rules_callbacks.md#on_threepid_bind) "third-party rules"
+Synapse module callback method has been deprecated
+in favour of a new module method,
+[`on_add_user_third_party_identifier`](modules/third_party_rules_callbacks.md#on_add_user_third_party_identifier).
+`on_threepid_bind` will be removed in a future version of Synapse. You should check whether any Synapse
+modules in use in your deployment are making use of `on_threepid_bind`, and update them where possible.
+
+The arguments and functionality of the new method are nearly the same; `on_threepid_bind` was called
+directly *after* a user adds a third-party identifier (email, phone number) to their account, while
+`on_add_user_third_party_identifier` is called just *before* a user adds a third-party identifier.
+
+The justification behind the swap in logical ordering is that `on_add_user_third_party_identifier`
+may be extended in the future to allow blocking a user from adding a third-party identifier to their
+account. The reason behind the name change is that the old method's name, `on_threepid_bind`, was
+misleading. A user is considered to "bind" their third-party ID to their Matrix ID only if they
+do so via an [identity server](https://spec.matrix.org/latest/identity-service-api/)
+(so that users on other homeservers may find them). But this method was not called in that case -
+it was only called when a user added a third-party identifier on the local homeserver.
+
+Module developers may also be interested in the related
+[`on_remove_user_third_party_identifier`](modules/third_party_rules_callbacks.md#on_remove_user_third_party_identifier)
+module callback method that was also added in Synapse v1.77.0. This new method is called when a
+user removes a third-party identifier from their account.
+
 # Upgrading to v1.76.0
 
 ## Faster joins are enabled by default

docs/usage/administration/admin_faq.md

@@ -70,55 +70,10 @@ output-directory
 │       ├───state
 │       ├───invite_state
 │       └───knock_state
-├───user_data
-│   ├───account_data
-│   │   ├───global
-│   │   └───<room_id>
-│   ├───connections
-│   ├───devices
-│   └───profile
-└───media_ids
-    └───<media_id>
-```
-
-The `media_ids` folder contains only the metadata of the media uploaded by the user.
-It does not contain the media itself.
-Furthermore, only the `media_ids` that Synapse manages itself are exported.
-If another media repository (e.g. [matrix-media-repo](https://github.com/turt2live/matrix-media-repo))
-is used, the data must be exported separately.
-
-With the `media_ids` the media files can be downloaded.
-Media that have been sent in encrypted rooms are only retrieved in encrypted form.
-The following script can help with download the media files:
-
-```bash
-#!/usr/bin/env bash
-
-# Parameters
-#
-#   source_directory: Directory which contains the export with the media_ids.
-#   target_directory: Directory into which all files are to be downloaded.
-#   repository_url: Address of the media repository resp. media worker.
-#   serverName: Name of the server (`server_name` from homeserver.yaml).
-#
-#   Example:
-#       ./download_media.sh /tmp/export_data/media_ids/ /tmp/export_data/media_files/ http://localhost:8008 matrix.example.com
-
-source_directory=$1
-target_directory=$2
-repository_url=$3
-serverName=$4
-
-mkdir -p $target_directory
-
-for file in $source_directory/*; do
-    filename=$(basename ${file})
-    url=$repository_url/_matrix/media/v3/download/$serverName/$filename
-    echo "Downloading $filename - $url"
-    if ! wget -o /dev/null -P $target_directory $url; then
-        echo "Could not download $filename"
-    fi
-done
+└───user_data
+    ├───connections
+    ├───devices
+    └───profile
 ```
 
 Manually resetting passwords
@@ -129,7 +84,7 @@ can reset a user's password using the [admin API](../../admin_api/user_admin_api
 
 I have a problem with my server. Can I just delete my database and start again?
 ---
-Deleting your database is unlikely to make anything better.
+Deleting your database is unlikely to make anything better. 
 
 It's easy to make the mistake of thinking that you can start again from a clean
 slate by dropping your database, but things don't work like that in a federated
@@ -144,7 +99,7 @@ Come and seek help in https://matrix.to/#/#synapse:matrix.org.
 
 There are two exceptions when it might be sensible to delete your database and start again:
 * You have *never* joined any rooms which are federated with other servers. For
-instance, a local deployment which the outside world can't talk to.
+instance, a local deployment which the outside world can't talk to. 
 * You are changing the `server_name` in the homeserver configuration. In effect
 this makes your server a completely new one from the point of view of the network,
 so in this case it makes sense to start with a clean database.
@@ -157,7 +112,7 @@ Using the following curl command:
 curl -H 'Authorization: Bearer <access-token>' -X DELETE https://matrix.org/_matrix/client/r0/directory/room/<room-alias>
 ```
 `<access-token>` - can be obtained in riot by looking in the riot settings, down the bottom is:
-Access Token:\<click to reveal\>
+Access Token:\<click to reveal\> 
 
 `<room-alias>` - the room alias, eg. #my_room:matrix.org this possibly needs to be URL encoded also, for example  %23my_room%3Amatrix.org
 
@@ -194,13 +149,13 @@ What are the biggest rooms on my server?
 ---
 
 ```sql
-SELECT s.canonical_alias, g.room_id, count(*) AS num_rows
-FROM
-  state_groups_state AS g,
-  room_stats_state AS s
-WHERE g.room_id = s.room_id
+SELECT s.canonical_alias, g.room_id, count(*) AS num_rows 
+FROM 
+  state_groups_state AS g, 
+  room_stats_state AS s 
+WHERE g.room_id = s.room_id 
 GROUP BY s.canonical_alias, g.room_id
-ORDER BY num_rows desc
+ORDER BY num_rows desc 
 LIMIT 10;
 ```
 

docs/usage/administration/database_maintenance_tools.md

@@ -1,4 +1,4 @@
-_This [blog post by Jackson Chen](https://jacksonchen666.com/posts/2022-12-03/14-33-00/) (Dec 2022) explains how to use many of the tools listed on this page. There is also an [earlier blog by Victor Berger](https://levans.fr/shrink-synapse-database.html) (June 2020), though this may be outdated in places._
+This blog post by Victor Berger explains how to use many of the tools listed on this page: https://levans.fr/shrink-synapse-database.html
 
 # List of useful tools and scripts for maintenance Synapse database:
 
@@ -15,4 +15,4 @@ The purge history API allows server admins to purge historic events from their d
 Tool for compressing (deduplicating) `state_groups_state` table.
 
 ## [SQL for analyzing Synapse PostgreSQL database stats](useful_sql_for_admins.md)
-Some easy SQL that reports useful stats about your Synapse database.
+Some easy SQL that reports useful stats about your Synapse database.

docs/usage/configuration/config_documentation.md

@@ -1105,7 +1105,7 @@ This setting should only be used in very specific cases, such as
 federation over Tor hidden services and similar. For private networks
 of homeservers, you likely want to use a private CA instead.
 
-Only effective if `federation_verify_certificates` is `true`.
+Only effective if `federation_verify_certicates` is `true`.
 
 Example configuration:
 ```yaml
@@ -1518,11 +1518,11 @@ rc_registration_token_validity:
 
 This option specifies several limits for login:
 * `address` ratelimits login requests based on the client's IP
-      address. Defaults to `per_second: 0.003`, `burst_count: 5`.
+      address. Defaults to `per_second: 0.17`, `burst_count: 3`.
 
 * `account` ratelimits login requests based on the account the
-  client is attempting to log into. Defaults to `per_second: 0.03`,
-  `burst_count: 5`.
+  client is attempting to log into. Defaults to `per_second: 0.17`,
+  `burst_count: 3`.
 
 * `failed_attempts` ratelimits login requests based on the account the
   client is attempting to log into, based on the amount of failed login
@@ -2232,7 +2232,7 @@ key on startup and store it in this file.
 
 Example configuration:
 ```yaml
-registration_shared_secret_path: /path/to/secrets/file
+registration_shared_secret_file: /path/to/secrets/file
 ```
 
 _Added in Synapse 1.67.0._
@@ -3927,9 +3927,6 @@ This setting has the following sub-options:
 * `host` and `port`: Optional host and port to use to connect to redis. Defaults to
    localhost and 6379
 * `password`: Optional password if configured on the Redis instance.
-* `dbid`: Optional redis dbid if needs to connect to specific redis logical db.
-
-  _Added in Synapse 1.78.0._
 
 Example configuration:
 ```yaml
@@ -3938,7 +3935,6 @@ redis:
   host: localhost
   port: 6379
   password: <secret_password>
-  dbid: <dbid>
 ```
 ---
 ## Individual worker configuration

docs/workers.md

@@ -160,18 +160,7 @@ recommend the use of `systemd` where available: for information on setting up
 [Systemd with Workers](systemd-with-workers/). To use `synctl`, see
 [Using synctl with Workers](synctl_workers.md).
 
-## Start Synapse with Poetry
 
-The following applies to Synapse installations that have been installed from source using `poetry`.
-
-You can start the main Synapse process with Poetry by running the following command:
-```console
-poetry run synapse_homeserver -c [your homeserver.yaml]
-```
-For worker setups, you can run the following command
-```console
-poetry run synapse_worker -c [your worker.yaml]
-```
 ## Available worker applications
 
 ### `synapse.app.generic_worker`

mypy.ini

@@ -31,6 +31,8 @@ exclude = (?x)
    |synapse/storage/databases/__init__.py
    |synapse/storage/databases/main/cache.py
    |synapse/storage/schema/
+
+   |tests/server.py
    )$
 
 [mypy-synapse.federation.transport.client]
@@ -54,6 +56,9 @@ disallow_untyped_defs = False
 [mypy-synapse.storage.database]
 disallow_untyped_defs = False
 
+[mypy-tests.unittest]
+disallow_untyped_defs = False
+
 [mypy-tests.util.caches.test_descriptors]
 disallow_untyped_defs = False
 

poetry.lock

@@ -90,46 +90,32 @@ typecheck = ["mypy"]
 
 [[package]]
 name = "black"
-version = "23.1.0"
+version = "22.12.0"
 description = "The uncompromising code formatter."
 category = "dev"
 optional = false
 python-versions = ">=3.7"
 files = [
-    {file = "black-23.1.0-cp310-cp310-macosx_10_16_arm64.whl", hash = "sha256:b6a92a41ee34b883b359998f0c8e6eb8e99803aa8bf3123bf2b2e6fec505a221"},
-    {file = "black-23.1.0-cp310-cp310-macosx_10_16_universal2.whl", hash = "sha256:57c18c5165c1dbe291d5306e53fb3988122890e57bd9b3dcb75f967f13411a26"},
-    {file = "black-23.1.0-cp310-cp310-macosx_10_16_x86_64.whl", hash = "sha256:9880d7d419bb7e709b37e28deb5e68a49227713b623c72b2b931028ea65f619b"},
-    {file = "black-23.1.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e6663f91b6feca5d06f2ccd49a10f254f9298cc1f7f49c46e498a0771b507104"},
-    {file = "black-23.1.0-cp310-cp310-win_amd64.whl", hash = "sha256:9afd3f493666a0cd8f8df9a0200c6359ac53940cbde049dcb1a7eb6ee2dd7074"},
-    {file = "black-23.1.0-cp311-cp311-macosx_10_16_arm64.whl", hash = "sha256:bfffba28dc52a58f04492181392ee380e95262af14ee01d4bc7bb1b1c6ca8d27"},
-    {file = "black-23.1.0-cp311-cp311-macosx_10_16_universal2.whl", hash = "sha256:c1c476bc7b7d021321e7d93dc2cbd78ce103b84d5a4cf97ed535fbc0d6660648"},
-    {file = "black-23.1.0-cp311-cp311-macosx_10_16_x86_64.whl", hash = "sha256:382998821f58e5c8238d3166c492139573325287820963d2f7de4d518bd76958"},
-    {file = "black-23.1.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2bf649fda611c8550ca9d7592b69f0637218c2369b7744694c5e4902873b2f3a"},
-    {file = "black-23.1.0-cp311-cp311-win_amd64.whl", hash = "sha256:121ca7f10b4a01fd99951234abdbd97728e1240be89fde18480ffac16503d481"},
-    {file = "black-23.1.0-cp37-cp37m-macosx_10_16_x86_64.whl", hash = "sha256:a8471939da5e824b891b25751955be52ee7f8a30a916d570a5ba8e0f2eb2ecad"},
-    {file = "black-23.1.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8178318cb74f98bc571eef19068f6ab5613b3e59d4f47771582f04e175570ed8"},
-    {file = "black-23.1.0-cp37-cp37m-win_amd64.whl", hash = "sha256:a436e7881d33acaf2536c46a454bb964a50eff59b21b51c6ccf5a40601fbef24"},
-    {file = "black-23.1.0-cp38-cp38-macosx_10_16_arm64.whl", hash = "sha256:a59db0a2094d2259c554676403fa2fac3473ccf1354c1c63eccf7ae65aac8ab6"},
-    {file = "black-23.1.0-cp38-cp38-macosx_10_16_universal2.whl", hash = "sha256:0052dba51dec07ed029ed61b18183942043e00008ec65d5028814afaab9a22fd"},
-    {file = "black-23.1.0-cp38-cp38-macosx_10_16_x86_64.whl", hash = "sha256:49f7b39e30f326a34b5c9a4213213a6b221d7ae9d58ec70df1c4a307cf2a1580"},
-    {file = "black-23.1.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:162e37d49e93bd6eb6f1afc3e17a3d23a823042530c37c3c42eeeaf026f38468"},
-    {file = "black-23.1.0-cp38-cp38-win_amd64.whl", hash = "sha256:8b70eb40a78dfac24842458476135f9b99ab952dd3f2dab738c1881a9b38b753"},
-    {file = "black-23.1.0-cp39-cp39-macosx_10_16_arm64.whl", hash = "sha256:a29650759a6a0944e7cca036674655c2f0f63806ddecc45ed40b7b8aa314b651"},
-    {file = "black-23.1.0-cp39-cp39-macosx_10_16_universal2.whl", hash = "sha256:bb460c8561c8c1bec7824ecbc3ce085eb50005883a6203dcfb0122e95797ee06"},
-    {file = "black-23.1.0-cp39-cp39-macosx_10_16_x86_64.whl", hash = "sha256:c91dfc2c2a4e50df0026f88d2215e166616e0c80e86004d0003ece0488db2739"},
-    {file = "black-23.1.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2a951cc83ab535d248c89f300eccbd625e80ab880fbcfb5ac8afb5f01a258ac9"},
-    {file = "black-23.1.0-cp39-cp39-win_amd64.whl", hash = "sha256:0680d4380db3719ebcfb2613f34e86c8e6d15ffeabcf8ec59355c5e7b85bb555"},
-    {file = "black-23.1.0-py3-none-any.whl", hash = "sha256:7a0f701d314cfa0896b9001df70a530eb2472babb76086344e688829efd97d32"},
-    {file = "black-23.1.0.tar.gz", hash = "sha256:b0bd97bea8903f5a2ba7219257a44e3f1f9d00073d6cc1add68f0beec69692ac"},
+    {file = "black-22.12.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9eedd20838bd5d75b80c9f5487dbcb06836a43833a37846cf1d8c1cc01cef59d"},
+    {file = "black-22.12.0-cp310-cp310-win_amd64.whl", hash = "sha256:159a46a4947f73387b4d83e87ea006dbb2337eab6c879620a3ba52699b1f4351"},
+    {file = "black-22.12.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d30b212bffeb1e252b31dd269dfae69dd17e06d92b87ad26e23890f3efea366f"},
+    {file = "black-22.12.0-cp311-cp311-win_amd64.whl", hash = "sha256:7412e75863aa5c5411886804678b7d083c7c28421210180d67dfd8cf1221e1f4"},
+    {file = "black-22.12.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c116eed0efb9ff870ded8b62fe9f28dd61ef6e9ddd28d83d7d264a38417dcee2"},
+    {file = "black-22.12.0-cp37-cp37m-win_amd64.whl", hash = "sha256:1f58cbe16dfe8c12b7434e50ff889fa479072096d79f0a7f25e4ab8e94cd8350"},
+    {file = "black-22.12.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:77d86c9f3db9b1bf6761244bc0b3572a546f5fe37917a044e02f3166d5aafa7d"},
+    {file = "black-22.12.0-cp38-cp38-win_amd64.whl", hash = "sha256:82d9fe8fee3401e02e79767016b4907820a7dc28d70d137eb397b92ef3cc5bfc"},
+    {file = "black-22.12.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:101c69b23df9b44247bd88e1d7e90154336ac4992502d4197bdac35dd7ee3320"},
+    {file = "black-22.12.0-cp39-cp39-win_amd64.whl", hash = "sha256:559c7a1ba9a006226f09e4916060982fd27334ae1998e7a38b3f33a37f7a2148"},
+    {file = "black-22.12.0-py3-none-any.whl", hash = "sha256:436cc9167dd28040ad90d3b404aec22cedf24a6e4d7de221bec2730ec0c97bcf"},
+    {file = "black-22.12.0.tar.gz", hash = "sha256:229351e5a18ca30f447bf724d007f890f97e13af070bb6ad4c0a441cd7596a2f"},
 ]
 
 [package.dependencies]
 click = ">=8.0.0"
 mypy-extensions = ">=0.4.3"
-packaging = ">=22.0"
 pathspec = ">=0.9.0"
 platformdirs = ">=2"
-tomli = {version = ">=1.1.0", markers = "python_version < \"3.11\""}
+tomli = {version = ">=1.1.0", markers = "python_full_version < \"3.11.0a7\""}
 typed-ast = {version = ">=1.4.2", markers = "python_version < \"3.8\" and implementation_name == \"cpython\""}
 typing-extensions = {version = ">=3.10.0.0", markers = "python_version < \"3.10\""}
 
@@ -141,14 +127,14 @@ uvloop = ["uvloop (>=0.15.2)"]
 
 [[package]]
 name = "bleach"
-version = "6.0.0"
+version = "5.0.1"
 description = "An easy safelist-based HTML-sanitizing tool."
 category = "main"
 optional = false
 python-versions = ">=3.7"
 files = [
-    {file = "bleach-6.0.0-py3-none-any.whl", hash = "sha256:33c16e3353dbd13028ab4799a0f89a83f113405c766e9c122df8a06f5b85b3f4"},
-    {file = "bleach-6.0.0.tar.gz", hash = "sha256:1a1a85c1595e07d8db14c5f09f09e6433502c51c595970edc090551f0db99414"},
+    {file = "bleach-5.0.1-py3-none-any.whl", hash = "sha256:085f7f33c15bd408dd9b17a4ad77c577db66d76203e5984b1bd59baeee948b2a"},
+    {file = "bleach-5.0.1.tar.gz", hash = "sha256:0d03255c47eb9bd2f26aa9bb7f2107732e7e8fe195ca2f64709fcf3b0a4a085c"},
 ]
 
 [package.dependencies]
@@ -157,17 +143,18 @@ webencodings = "*"
 
 [package.extras]
 css = ["tinycss2 (>=1.1.0,<1.2)"]
+dev = ["Sphinx (==4.3.2)", "black (==22.3.0)", "build (==0.8.0)", "flake8 (==4.0.1)", "hashin (==0.17.0)", "mypy (==0.961)", "pip-tools (==6.6.2)", "pytest (==7.1.2)", "tox (==3.25.0)", "twine (==4.0.1)", "wheel (==0.37.1)"]
 
 [[package]]
 name = "canonicaljson"
-version = "1.6.5"
+version = "1.6.4"
 description = "Canonical JSON"
 category = "main"
 optional = false
 python-versions = ">=3.7"
 files = [
-    {file = "canonicaljson-1.6.5-py3-none-any.whl", hash = "sha256:806ea6f2cbb7405d20259e1c36dd1214ba5c242fa9165f5bd0bf2081f82c23fb"},
-    {file = "canonicaljson-1.6.5.tar.gz", hash = "sha256:68dfc157b011e07d94bf74b5d4ccc01958584ed942d9dfd5fdd706609e81cd4b"},
+    {file = "canonicaljson-1.6.4-py3-none-any.whl", hash = "sha256:55d282853b4245dbcd953fe54c39b91571813d7c44e1dbf66e3c4f97ff134a48"},
+    {file = "canonicaljson-1.6.4.tar.gz", hash = "sha256:6c09b2119511f30eb1126cfcd973a10824e20f1cfd25039cde3d1218dd9c8d8f"},
 ]
 
 [package.dependencies]
@@ -371,8 +358,6 @@ files = [
     {file = "cryptography-39.0.1-cp36-abi3-win32.whl", hash = "sha256:fe913f20024eb2cb2f323e42a64bdf2911bb9738a15dba7d3cce48151034e3a8"},
     {file = "cryptography-39.0.1-cp36-abi3-win_amd64.whl", hash = "sha256:ced4e447ae29ca194449a3f1ce132ded8fcab06971ef5f618605aacaa612beac"},
     {file = "cryptography-39.0.1-pp38-pypy38_pp73-macosx_10_12_x86_64.whl", hash = "sha256:807ce09d4434881ca3a7594733669bd834f5b2c6d5c7e36f8c00f691887042ad"},
-    {file = "cryptography-39.0.1-pp38-pypy38_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:c5caeb8188c24888c90b5108a441c106f7faa4c4c075a2bcae438c6e8ca73cef"},
-    {file = "cryptography-39.0.1-pp38-pypy38_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:4789d1e3e257965e960232345002262ede4d094d1a19f4d3b52e48d4d8f3b885"},
     {file = "cryptography-39.0.1-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:96f1157a7c08b5b189b16b47bc9db2332269d6680a196341bf30046330d15388"},
     {file = "cryptography-39.0.1-pp39-pypy39_pp73-macosx_10_12_x86_64.whl", hash = "sha256:e422abdec8b5fa8462aa016786680720d78bdce7a30c652b7fadf83a4ba35336"},
     {file = "cryptography-39.0.1-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.manylinux_2_24_aarch64.whl", hash = "sha256:b0afd054cd42f3d213bf82c629efb1ee5f22eba35bf0eec88ea9ea7304f511a2"},
@@ -1160,38 +1145,36 @@ files = [
 
 [[package]]
 name = "mypy"
-version = "1.0.0"
+version = "0.981"
 description = "Optional static typing for Python"
 category = "dev"
 optional = false
 python-versions = ">=3.7"
 files = [
-    {file = "mypy-1.0.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:e0626db16705ab9f7fa6c249c017c887baf20738ce7f9129da162bb3075fc1af"},
-    {file = "mypy-1.0.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:1ace23f6bb4aec4604b86c4843276e8fa548d667dbbd0cb83a3ae14b18b2db6c"},
-    {file = "mypy-1.0.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:87edfaf344c9401942883fad030909116aa77b0fa7e6e8e1c5407e14549afe9a"},
-    {file = "mypy-1.0.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:0ab090d9240d6b4e99e1fa998c2d0aa5b29fc0fb06bd30e7ad6183c95fa07593"},
-    {file = "mypy-1.0.0-cp310-cp310-win_amd64.whl", hash = "sha256:7cc2c01dfc5a3cbddfa6c13f530ef3b95292f926329929001d45e124342cd6b7"},
-    {file = "mypy-1.0.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:14d776869a3e6c89c17eb943100f7868f677703c8a4e00b3803918f86aafbc52"},
-    {file = "mypy-1.0.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:bb2782a036d9eb6b5a6efcdda0986774bf798beef86a62da86cb73e2a10b423d"},
-    {file = "mypy-1.0.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5cfca124f0ac6707747544c127880893ad72a656e136adc935c8600740b21ff5"},
-    {file = "mypy-1.0.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:8845125d0b7c57838a10fd8925b0f5f709d0e08568ce587cc862aacce453e3dd"},
-    {file = "mypy-1.0.0-cp311-cp311-win_amd64.whl", hash = "sha256:01b1b9e1ed40544ef486fa8ac022232ccc57109f379611633ede8e71630d07d2"},
-    {file = "mypy-1.0.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:c7cf862aef988b5fbaa17764ad1d21b4831436701c7d2b653156a9497d92c83c"},
-    {file = "mypy-1.0.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5cd187d92b6939617f1168a4fe68f68add749902c010e66fe574c165c742ed88"},
-    {file = "mypy-1.0.0-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:4e5175026618c178dfba6188228b845b64131034ab3ba52acaffa8f6c361f805"},
-    {file = "mypy-1.0.0-cp37-cp37m-win_amd64.whl", hash = "sha256:2f6ac8c87e046dc18c7d1d7f6653a66787a4555085b056fe2d599f1f1a2a2d21"},
-    {file = "mypy-1.0.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:7306edca1c6f1b5fa0bc9aa645e6ac8393014fa82d0fa180d0ebc990ebe15964"},
-    {file = "mypy-1.0.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:3cfad08f16a9c6611e6143485a93de0e1e13f48cfb90bcad7d5fde1c0cec3d36"},
-    {file = "mypy-1.0.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:67cced7f15654710386e5c10b96608f1ee3d5c94ca1da5a2aad5889793a824c1"},
-    {file = "mypy-1.0.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:a86b794e8a56ada65c573183756eac8ac5b8d3d59daf9d5ebd72ecdbb7867a43"},
-    {file = "mypy-1.0.0-cp38-cp38-win_amd64.whl", hash = "sha256:50979d5efff8d4135d9db293c6cb2c42260e70fb010cbc697b1311a4d7a39ddb"},
-    {file = "mypy-1.0.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:3ae4c7a99e5153496243146a3baf33b9beff714464ca386b5f62daad601d87af"},
-    {file = "mypy-1.0.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:5e398652d005a198a7f3c132426b33c6b85d98aa7dc852137a2a3be8890c4072"},
-    {file = "mypy-1.0.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:be78077064d016bc1b639c2cbcc5be945b47b4261a4f4b7d8923f6c69c5c9457"},
-    {file = "mypy-1.0.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:92024447a339400ea00ac228369cd242e988dd775640755fa4ac0c126e49bb74"},
-    {file = "mypy-1.0.0-cp39-cp39-win_amd64.whl", hash = "sha256:fe523fcbd52c05040c7bee370d66fee8373c5972171e4fbc323153433198592d"},
-    {file = "mypy-1.0.0-py3-none-any.whl", hash = "sha256:2efa963bdddb27cb4a0d42545cd137a8d2b883bd181bbc4525b568ef6eca258f"},
-    {file = "mypy-1.0.0.tar.gz", hash = "sha256:f34495079c8d9da05b183f9f7daec2878280c2ad7cc81da686ef0b484cea2ecf"},
+    {file = "mypy-0.981-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:4bc460e43b7785f78862dab78674e62ec3cd523485baecfdf81a555ed29ecfa0"},
+    {file = "mypy-0.981-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:756fad8b263b3ba39e4e204ee53042671b660c36c9017412b43af210ddee7b08"},
+    {file = "mypy-0.981-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:a16a0145d6d7d00fbede2da3a3096dcc9ecea091adfa8da48fa6a7b75d35562d"},
+    {file = "mypy-0.981-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ce65f70b14a21fdac84c294cde75e6dbdabbcff22975335e20827b3b94bdbf49"},
+    {file = "mypy-0.981-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:6e35d764784b42c3e256848fb8ed1d4292c9fc0098413adb28d84974c095b279"},
+    {file = "mypy-0.981-cp310-cp310-win_amd64.whl", hash = "sha256:e53773073c864d5f5cec7f3fc72fbbcef65410cde8cc18d4f7242dea60dac52e"},
+    {file = "mypy-0.981-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:6ee196b1d10b8b215e835f438e06965d7a480f6fe016eddbc285f13955cca659"},
+    {file = "mypy-0.981-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8ad21d4c9d3673726cf986ea1d0c9fb66905258709550ddf7944c8f885f208be"},
+    {file = "mypy-0.981-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:d1debb09043e1f5ee845fa1e96d180e89115b30e47c5d3ce53bc967bab53f62d"},
+    {file = "mypy-0.981-cp37-cp37m-win_amd64.whl", hash = "sha256:9f362470a3480165c4c6151786b5379351b790d56952005be18bdbdd4c7ce0ae"},
+    {file = "mypy-0.981-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:c9e0efb95ed6ca1654951bd5ec2f3fa91b295d78bf6527e026529d4aaa1e0c30"},
+    {file = "mypy-0.981-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:e178eaffc3c5cd211a87965c8c0df6da91ed7d258b5fc72b8e047c3771317ddb"},
+    {file = "mypy-0.981-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:06e1eac8d99bd404ed8dd34ca29673c4346e76dd8e612ea507763dccd7e13c7a"},
+    {file = "mypy-0.981-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fa38f82f53e1e7beb45557ff167c177802ba7b387ad017eab1663d567017c8ee"},
+    {file = "mypy-0.981-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:64e1f6af81c003f85f0dfed52db632817dabb51b65c0318ffbf5ff51995bbb08"},
+    {file = "mypy-0.981-cp38-cp38-win_amd64.whl", hash = "sha256:e1acf62a8c4f7c092462c738aa2c2489e275ed386320c10b2e9bff31f6f7e8d6"},
+    {file = "mypy-0.981-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:b6ede64e52257931315826fdbfc6ea878d89a965580d1a65638ef77cb551f56d"},
+    {file = "mypy-0.981-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:eb3978b191b9fa0488524bb4ffedf2c573340e8c2b4206fc191d44c7093abfb7"},
+    {file = "mypy-0.981-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:77f8fcf7b4b3cc0c74fb33ae54a4cd00bb854d65645c48beccf65fa10b17882c"},
+    {file = "mypy-0.981-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f64d2ce043a209a297df322eb4054dfbaa9de9e8738291706eaafda81ab2b362"},
+    {file = "mypy-0.981-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:2ee3dbc53d4df7e6e3b1c68ac6a971d3a4fb2852bf10a05fda228721dd44fae1"},
+    {file = "mypy-0.981-cp39-cp39-win_amd64.whl", hash = "sha256:8e8e49aa9cc23aa4c926dc200ce32959d3501c4905147a66ce032f05cb5ecb92"},
+    {file = "mypy-0.981-py3-none-any.whl", hash = "sha256:794f385653e2b749387a42afb1e14c2135e18daeb027e0d97162e4b7031210f8"},
+    {file = "mypy-0.981.tar.gz", hash = "sha256:ad77c13037d3402fbeffda07d51e3f228ba078d1c7096a73759c9419ea031bf4"},
 ]
 
 [package.dependencies]
@@ -1202,7 +1185,6 @@ typing-extensions = ">=3.10"
 
 [package.extras]
 dmypy = ["psutil (>=4.0)"]
-install-types = ["pip"]
 python2 = ["typed-ast (>=1.4.0,<2)"]
 reports = ["lxml"]
 
@@ -1220,18 +1202,18 @@ files = [
 
 [[package]]
 name = "mypy-zope"
-version = "0.9.0"
+version = "0.3.11"
 description = "Plugin for mypy to support zope interfaces"
 category = "dev"
 optional = false
 python-versions = "*"
 files = [
-    {file = "mypy-zope-0.9.0.tar.gz", hash = "sha256:88bf6cd056e38b338e6956055958a7805b4ff84404ccd99e29883a3647a1aeb3"},
-    {file = "mypy_zope-0.9.0-py3-none-any.whl", hash = "sha256:e1bb4b57084f76ff8a154a3e07880a1af2ac6536c491dad4b143d529f72c5d15"},
+    {file = "mypy-zope-0.3.11.tar.gz", hash = "sha256:d4255f9f04d48c79083bbd4e2fea06513a6ac7b8de06f8c4ce563fd85142ca05"},
+    {file = "mypy_zope-0.3.11-py3-none-any.whl", hash = "sha256:ec080a6508d1f7805c8d2054f9fdd13c849742ce96803519e1fdfa3d3cab7140"},
 ]
 
 [package.dependencies]
-mypy = "1.0.0"
+mypy = "0.981"
 "zope.interface" = "*"
 "zope.schema" = "*"
 
@@ -2043,14 +2025,14 @@ doc = ["Sphinx", "sphinx-rtd-theme"]
 
 [[package]]
 name = "sentry-sdk"
-version = "1.15.0"
+version = "1.13.0"
 description = "Python client for Sentry (https://sentry.io)"
 category = "main"
 optional = true
 python-versions = "*"
 files = [
-    {file = "sentry-sdk-1.15.0.tar.gz", hash = "sha256:69ecbb2e1ff4db02a06c4f20f6f69cb5dfe3ebfbc06d023e40d77cf78e9c37e7"},
-    {file = "sentry_sdk-1.15.0-py2.py3-none-any.whl", hash = "sha256:7ad4d37dd093f4a7cb5ad804c6efe9e8fab8873f7ffc06042dc3f3fd700a93ec"},
+    {file = "sentry-sdk-1.13.0.tar.gz", hash = "sha256:72da0766c3069a3941eadbdfa0996f83f5a33e55902a19ba399557cfee1dddcc"},
+    {file = "sentry_sdk-1.13.0-py2.py3-none-any.whl", hash = "sha256:b7ff6318183e551145b5c4766eb65b59ad5b63ff234dffddc5fb50340cad6729"},
 ]
 
 [package.dependencies]
@@ -2068,8 +2050,7 @@ falcon = ["falcon (>=1.4)"]
 fastapi = ["fastapi (>=0.79.0)"]
 flask = ["blinker (>=1.1)", "flask (>=0.11)"]
 httpx = ["httpx (>=0.16.0)"]
-huey = ["huey (>=2)"]
-opentelemetry = ["opentelemetry-distro (>=0.35b0)"]
+opentelemetry = ["opentelemetry-distro (>=0.350b0)"]
 pure-eval = ["asttokens", "executing", "pure-eval"]
 pymongo = ["pymongo (>=3.1)"]
 pyspark = ["pyspark (>=2.4.4)"]
@@ -2271,13 +2252,13 @@ files = [
 
 [[package]]
 name = "systemd-python"
-version = "235"
+version = "234"
 description = "Python interface for libsystemd"
 category = "main"
 optional = true
 python-versions = "*"
 files = [
-    {file = "systemd-python-235.tar.gz", hash = "sha256:4e57f39797fd5d9e2d22b8806a252d7c0106c936039d1e71c8c6b8008e695c0a"},
+    {file = "systemd-python-234.tar.gz", hash = "sha256:fd0e44bf70eadae45aadc292cb0a7eb5b0b6372cd1b391228047d33895db83e7"},
 ]
 
 [[package]]
@@ -2562,14 +2543,14 @@ files = [
 
 [[package]]
 name = "types-bleach"
-version = "6.0.0.0"
+version = "5.0.3.1"
 description = "Typing stubs for bleach"
 category = "dev"
 optional = false
 python-versions = "*"
 files = [
-    {file = "types-bleach-6.0.0.0.tar.gz", hash = "sha256:770ce9c7ea6173743ef1a4a70f2619bb1819bf53c7cd0336d939af93f488fbe2"},
-    {file = "types_bleach-6.0.0.0-py3-none-any.whl", hash = "sha256:75f55f035837c5fce2cd0bd5162a2a90057680a89c9275588a5c12f5f597a14a"},
+    {file = "types-bleach-5.0.3.1.tar.gz", hash = "sha256:ce8772ea5126dab1883851b41e3aeff229aa5213ced36096990344e632e92373"},
+    {file = "types_bleach-5.0.3.1-py3-none-any.whl", hash = "sha256:af5f1b3a54ff279f54c29eccb2e6988ebb6718bc4061469588a5fd4880a79287"},
 ]
 
 [[package]]
@@ -2638,14 +2619,14 @@ files = [
 
 [[package]]
 name = "types-jsonschema"
-version = "4.17.0.5"
+version = "4.17.0.3"
 description = "Typing stubs for jsonschema"
 category = "dev"
 optional = false
 python-versions = "*"
 files = [
-    {file = "types-jsonschema-4.17.0.5.tar.gz", hash = "sha256:7adc7bfca4afe291de0c93eca9367aa72a4fbe8ce87fe15642c600ad97d45dd6"},
-    {file = "types_jsonschema-4.17.0.5-py3-none-any.whl", hash = "sha256:79ac8a7763fe728947af90a24168b91621edf7e8425bf3670abd4ea0d4758fba"},
+    {file = "types-jsonschema-4.17.0.3.tar.gz", hash = "sha256:746aa466ffed9a1acc7bdbd0ac0b5e068f00be2ee008c1d1e14b0944a8c8b24b"},
+    {file = "types_jsonschema-4.17.0.3-py3-none-any.whl", hash = "sha256:c8d5b26b7c8da6a48d7fb1ce029b97e0ff6e74db3727efb968c69f39ad013685"},
 ]
 
 [[package]]
@@ -2662,14 +2643,14 @@ files = [
 
 [[package]]
 name = "types-pillow"
-version = "9.4.0.13"
+version = "9.4.0.5"
 description = "Typing stubs for Pillow"
 category = "dev"
 optional = false
 python-versions = "*"
 files = [
-    {file = "types-Pillow-9.4.0.13.tar.gz", hash = "sha256:4510aa98a28947bf63f2b29edebbd11b7cff8647d90b867cec9b3674c0a8c321"},
-    {file = "types_Pillow-9.4.0.13-py3-none-any.whl", hash = "sha256:14a8a19021b8fe569a9fef9edc64a8d8a4aef340e38669d4fb3dc05cfd941130"},
+    {file = "types-Pillow-9.4.0.5.tar.gz", hash = "sha256:941cefaac2f5297d7d2a9989633c95b4063112690dc21c965d46bd5a7fff3c76"},
+    {file = "types_Pillow-9.4.0.5-py3-none-any.whl", hash = "sha256:a1d2b3e070b4d852af04f76f018d12bd51abb4abca3b725d91b35e01cda7a2de"},
 ]
 
 [[package]]
@@ -2713,14 +2694,14 @@ files = [
 
 [[package]]
 name = "types-requests"
-version = "2.28.11.12"
+version = "2.28.11.8"
 description = "Typing stubs for requests"
 category = "dev"
 optional = false
 python-versions = "*"
 files = [
-    {file = "types-requests-2.28.11.12.tar.gz", hash = "sha256:fd530aab3fc4f05ee36406af168f0836e6f00f1ee51a0b96b7311f82cb675230"},
-    {file = "types_requests-2.28.11.12-py3-none-any.whl", hash = "sha256:dbc2933635860e553ffc59f5e264264981358baffe6342b925e3eb8261f866ee"},
+    {file = "types-requests-2.28.11.8.tar.gz", hash = "sha256:e67424525f84adfbeab7268a159d3c633862dafae15c5b19547ce1b55954f0a3"},
+    {file = "types_requests-2.28.11.8-py3-none-any.whl", hash = "sha256:61960554baca0008ae7e2db2bd3b322ca9a144d3e80ce270f5fb640817e40994"},
 ]
 
 [package.dependencies]
@@ -2728,14 +2709,14 @@ types-urllib3 = "<1.27"
 
 [[package]]
 name = "types-setuptools"
-version = "67.3.0.1"
+version = "67.1.0.0"
 description = "Typing stubs for setuptools"
 category = "dev"
 optional = false
 python-versions = "*"
 files = [
-    {file = "types-setuptools-67.3.0.1.tar.gz", hash = "sha256:1a26d373036c720e566823b6edd664a2db4d138b6eeba856721ec1254203474f"},
-    {file = "types_setuptools-67.3.0.1-py3-none-any.whl", hash = "sha256:a7e0f0816b5b449f5bcdc0efa43da91ff81dbe6941f293a6490d68a450e130a1"},
+    {file = "types-setuptools-67.1.0.0.tar.gz", hash = "sha256:162a39d22e3a5eb802197c84f16b19e798101bbd33d9437837fbb45627da5627"},
+    {file = "types_setuptools-67.1.0.0-py3-none-any.whl", hash = "sha256:5bd7a10d93e468bfcb10d24cb8ea5e12ac4f4ac91267293959001f1448cf0619"},
 ]
 
 [package.dependencies]
@@ -3044,4 +3025,4 @@ user-search = ["pyicu"]
 [metadata]
 lock-version = "2.0"
 python-versions = "^3.7.1"
-content-hash = "e12077711e5ff83f3c6038ea44c37bd49773799ec8245035b01094b7800c5c92"
+content-hash = "16528ddab686d1bc3180ff37b09de35b904f68516cfdcc3942844163a126255e"

pyproject.toml

@@ -89,7 +89,7 @@ manifest-path = "rust/Cargo.toml"
 
 [tool.poetry]
 name = "matrix-synapse"
-version = "1.78.0rc1"
+version = "1.77.0rc1"
 description = "Homeserver for the Matrix decentralised comms protocol"
 authors = ["Matrix.org Team and Contributors <packages@matrix.org>"]
 license = "Apache-2.0"
@@ -154,9 +154,7 @@ python = "^3.7.1"
 # we use the TYPE_CHECKER.redefine method added in jsonschema 3.0.0
 jsonschema = ">=3.0.0"
 # frozendict 2.1.2 is broken on Debian 10: https://github.com/Marco-Sulla/python-frozendict/issues/41
-# We cannot test our wheels against the 2.3.5 release in CI. Putting in an upper bound for this
-# because frozendict has been more trouble than it's worth; we would like to move to immutabledict.
-frozendict = ">=1,!=2.1.2,<2.3.5"
+frozendict = ">=1,!=2.1.2"
 # We require 2.1.0 or higher for type hints. Previous guard was >= 1.1.0
 unpaddedbase64 = ">=2.1.0"
 # We require 1.5.0 to work around an issue when running against the C implementation of
@@ -348,9 +346,6 @@ twine = "*"
 # Towncrier min version comes from #3425. Rationale unclear.
 towncrier = ">=18.6.0rc1"
 
-# Used for checking the Poetry lockfile
-tomli = ">=1.2.3"
-
 [build-system]
 # The upper bounds here are defensive, intended to prevent situations like
 # #13849 and #14079 where we see buildtime or runtime errors caused by build

rust/Cargo.toml

@@ -24,7 +24,7 @@ anyhow = "1.0.63"
 lazy_static = "1.4.0"
 log = "0.4.17"
 pyo3 = { version = "0.17.1", features = ["macros", "anyhow", "abi3", "abi3-py37"] }
-pyo3-log = "0.8.1"
+pyo3-log = "0.7.0"
 pythonize = "0.17.0"
 regex = "1.6.0"
 serde = { version = "1.0.144", features = ["derive"] }

rust/benches/evaluator.rs

@@ -14,9 +14,9 @@
 
 #![feature(test)]
 use std::collections::BTreeSet;
+
 use synapse::push::{
-    evaluator::PushRuleEvaluator, Condition, EventMatchCondition, FilteredPushRules, JsonValue,
-    PushRules, SimpleJsonValue,
+    evaluator::PushRuleEvaluator, Condition, EventMatchCondition, FilteredPushRules, PushRules,
 };
 use test::Bencher;
 
@@ -25,18 +25,9 @@ extern crate test;
 #[bench]
 fn bench_match_exact(b: &mut Bencher) {
     let flattened_keys = [
-        (
-            "type".to_string(),
-            JsonValue::Value(SimpleJsonValue::Str("m.text".to_string())),
-        ),
-        (
-            "room_id".to_string(),
-            JsonValue::Value(SimpleJsonValue::Str("!room:server".to_string())),
-        ),
-        (
-            "content.body".to_string(),
-            JsonValue::Value(SimpleJsonValue::Str("test message".to_string())),
-        ),
+        ("type".to_string(), "m.text".to_string()),
+        ("room_id".to_string(), "!room:server".to_string()),
+        ("content.body".to_string(), "test message".to_string()),
     ]
     .into_iter()
     .collect();
@@ -45,6 +36,7 @@ fn bench_match_exact(b: &mut Bencher) {
         flattened_keys,
         false,
         BTreeSet::new(),
+        false,
         10,
         Some(0),
         Default::default(),
@@ -52,8 +44,6 @@ fn bench_match_exact(b: &mut Bencher) {
         true,
         vec![],
         false,
-        false,
-        false,
     )
     .unwrap();
 
@@ -74,18 +64,9 @@ fn bench_match_exact(b: &mut Bencher) {
 #[bench]
 fn bench_match_word(b: &mut Bencher) {
     let flattened_keys = [
-        (
-            "type".to_string(),
-            JsonValue::Value(SimpleJsonValue::Str("m.text".to_string())),
-        ),
-        (
-            "room_id".to_string(),
-            JsonValue::Value(SimpleJsonValue::Str("!room:server".to_string())),
-        ),
-        (
-            "content.body".to_string(),
-            JsonValue::Value(SimpleJsonValue::Str("test message".to_string())),
-        ),
+        ("type".to_string(), "m.text".to_string()),
+        ("room_id".to_string(), "!room:server".to_string()),
+        ("content.body".to_string(), "test message".to_string()),
     ]
     .into_iter()
     .collect();
@@ -94,6 +75,7 @@ fn bench_match_word(b: &mut Bencher) {
         flattened_keys,
         false,
         BTreeSet::new(),
+        false,
         10,
         Some(0),
         Default::default(),
@@ -101,8 +83,6 @@ fn bench_match_word(b: &mut Bencher) {
         true,
         vec![],
         false,
-        false,
-        false,
     )
     .unwrap();
 
@@ -123,18 +103,9 @@ fn bench_match_word(b: &mut Bencher) {
 #[bench]
 fn bench_match_word_miss(b: &mut Bencher) {
     let flattened_keys = [
-        (
-            "type".to_string(),
-            JsonValue::Value(SimpleJsonValue::Str("m.text".to_string())),
-        ),
-        (
-            "room_id".to_string(),
-            JsonValue::Value(SimpleJsonValue::Str("!room:server".to_string())),
-        ),
-        (
-            "content.body".to_string(),
-            JsonValue::Value(SimpleJsonValue::Str("test message".to_string())),
-        ),
+        ("type".to_string(), "m.text".to_string()),
+        ("room_id".to_string(), "!room:server".to_string()),
+        ("content.body".to_string(), "test message".to_string()),
     ]
     .into_iter()
     .collect();
@@ -143,6 +114,7 @@ fn bench_match_word_miss(b: &mut Bencher) {
         flattened_keys,
         false,
         BTreeSet::new(),
+        false,
         10,
         Some(0),
         Default::default(),
@@ -150,8 +122,6 @@ fn bench_match_word_miss(b: &mut Bencher) {
         true,
         vec![],
         false,
-        false,
-        false,
     )
     .unwrap();
 
@@ -172,18 +142,9 @@ fn bench_match_word_miss(b: &mut Bencher) {
 #[bench]
 fn bench_eval_message(b: &mut Bencher) {
     let flattened_keys = [
-        (
-            "type".to_string(),
-            JsonValue::Value(SimpleJsonValue::Str("m.text".to_string())),
-        ),
-        (
-            "room_id".to_string(),
-            JsonValue::Value(SimpleJsonValue::Str("!room:server".to_string())),
-        ),
-        (
-            "content.body".to_string(),
-            JsonValue::Value(SimpleJsonValue::Str("test message".to_string())),
-        ),
+        ("type".to_string(), "m.text".to_string()),
+        ("room_id".to_string(), "!room:server".to_string()),
+        ("content.body".to_string(), "test message".to_string()),
     ]
     .into_iter()
     .collect();
@@ -192,6 +153,7 @@ fn bench_eval_message(b: &mut Bencher) {
         flattened_keys,
         false,
         BTreeSet::new(),
+        false,
         10,
         Some(0),
         Default::default(),
@@ -199,8 +161,6 @@ fn bench_eval_message(b: &mut Bencher) {
         true,
         vec![],
         false,
-        false,
-        false,
     )
     .unwrap();
 

rust/src/push/base_rules.rs

@@ -21,13 +21,13 @@ use lazy_static::lazy_static;
 use serde_json::Value;
 
 use super::KnownCondition;
+use crate::push::Action;
 use crate::push::Condition;
 use crate::push::EventMatchCondition;
 use crate::push::PushRule;
 use crate::push::RelatedEventMatchCondition;
 use crate::push::SetTweak;
 use crate::push::TweakValue;
-use crate::push::{Action, ExactEventMatchCondition, SimpleJsonValue};
 
 const HIGHLIGHT_ACTION: Action = Action::SetTweak(SetTweak {
     set_tweak: Cow::Borrowed("highlight"),
@@ -168,10 +168,7 @@ pub const BASE_APPEND_OVERRIDE_RULES: &[PushRule] = &[
         rule_id: Cow::Borrowed(".org.matrix.msc3952.is_room_mention"),
         priority_class: 5,
         conditions: Cow::Borrowed(&[
-            Condition::Known(KnownCondition::ExactEventMatch(ExactEventMatchCondition {
-                key: Cow::Borrowed("content.org.matrix.msc3952.mentions.room"),
-                value: Cow::Borrowed(&SimpleJsonValue::Bool(true)),
-            })),
+            Condition::Known(KnownCondition::IsRoomMention),
             Condition::Known(KnownCondition::SenderNotificationPermission {
                 key: Cow::Borrowed("room"),
             }),
@@ -226,7 +223,7 @@ pub const BASE_APPEND_OVERRIDE_RULES: &[PushRule] = &[
                 pattern_type: None,
             },
         ))]),
-        actions: Cow::Borrowed(&[]),
+        actions: Cow::Borrowed(&[Action::DontNotify]),
         default: true,
         default_enabled: true,
     },

rust/src/push/evaluator.rs

@@ -14,7 +14,6 @@
 
 use std::collections::{BTreeMap, BTreeSet};
 
-use crate::push::JsonValue;
 use anyhow::{Context, Error};
 use lazy_static::lazy_static;
 use log::warn;
@@ -23,8 +22,8 @@ use regex::Regex;
 
 use super::{
     utils::{get_glob_matcher, get_localpart_from_id, GlobMatchType},
-    Action, Condition, EventMatchCondition, ExactEventMatchCondition, FilteredPushRules,
-    KnownCondition, RelatedEventMatchCondition, SimpleJsonValue,
+    Action, Condition, EventMatchCondition, FilteredPushRules, KnownCondition,
+    RelatedEventMatchCondition,
 };
 
 lazy_static! {
@@ -62,9 +61,9 @@ impl RoomVersionFeatures {
 /// Allows running a set of push rules against a particular event.
 #[pyclass]
 pub struct PushRuleEvaluator {
-    /// A mapping of "flattened" keys to simple JSON values in the event, e.g.
+    /// A mapping of "flattened" keys to string values in the event, e.g.
     /// includes things like "type" and "content.msgtype".
-    flattened_keys: BTreeMap<String, JsonValue>,
+    flattened_keys: BTreeMap<String, String>,
 
     /// The "content.body", if any.
     body: String,
@@ -73,6 +72,8 @@ pub struct PushRuleEvaluator {
     has_mentions: bool,
     /// The user mentions that were part of the message.
     user_mentions: BTreeSet<String>,
+    /// True if the message is a room message.
+    room_mention: bool,
 
     /// The number of users in the room.
     room_member_count: u64,
@@ -86,7 +87,7 @@ pub struct PushRuleEvaluator {
 
     /// The related events, indexed by relation type. Flattened in the same manner as
     /// `flattened_keys`.
-    related_events_flattened: BTreeMap<String, BTreeMap<String, JsonValue>>,
+    related_events_flattened: BTreeMap<String, BTreeMap<String, String>>,
 
     /// If msc3664, push rules for related events, is enabled.
     related_event_match_enabled: bool,
@@ -97,12 +98,6 @@ pub struct PushRuleEvaluator {
     /// If MSC3931 (room version feature flags) is enabled. Usually controlled by the same
     /// flag as MSC1767 (extensible events core).
     msc3931_enabled: bool,
-
-    /// If MSC3758 (exact_event_match push rule condition) is enabled.
-    msc3758_exact_event_match: bool,
-
-    /// If MSC3966 (exact_event_property_contains push rule condition) is enabled.
-    msc3966_exact_event_property_contains: bool,
 }
 
 #[pymethods]
@@ -111,29 +106,29 @@ impl PushRuleEvaluator {
     #[allow(clippy::too_many_arguments)]
     #[new]
     pub fn py_new(
-        flattened_keys: BTreeMap<String, JsonValue>,
+        flattened_keys: BTreeMap<String, String>,
         has_mentions: bool,
         user_mentions: BTreeSet<String>,
+        room_mention: bool,
         room_member_count: u64,
         sender_power_level: Option<i64>,
         notification_power_levels: BTreeMap<String, i64>,
-        related_events_flattened: BTreeMap<String, BTreeMap<String, JsonValue>>,
+        related_events_flattened: BTreeMap<String, BTreeMap<String, String>>,
         related_event_match_enabled: bool,
         room_version_feature_flags: Vec<String>,
         msc3931_enabled: bool,
-        msc3758_exact_event_match: bool,
-        msc3966_exact_event_property_contains: bool,
     ) -> Result<Self, Error> {
-        let body = match flattened_keys.get("content.body") {
-            Some(JsonValue::Value(SimpleJsonValue::Str(s))) => s.clone(),
-            _ => String::new(),
-        };
+        let body = flattened_keys
+            .get("content.body")
+            .cloned()
+            .unwrap_or_default();
 
         Ok(PushRuleEvaluator {
             flattened_keys,
             body,
             has_mentions,
             user_mentions,
+            room_mention,
             room_member_count,
             notification_power_levels,
             sender_power_level,
@@ -141,8 +136,6 @@ impl PushRuleEvaluator {
             related_event_match_enabled,
             room_version_feature_flags,
             msc3931_enabled,
-            msc3758_exact_event_match,
-            msc3966_exact_event_property_contains,
         })
     }
 
@@ -259,15 +252,9 @@ impl PushRuleEvaluator {
             KnownCondition::EventMatch(event_match) => {
                 self.match_event_match(event_match, user_id)?
             }
-            KnownCondition::ExactEventMatch(exact_event_match) => {
-                self.match_exact_event_match(exact_event_match)?
-            }
             KnownCondition::RelatedEventMatch(event_match) => {
                 self.match_related_event_match(event_match, user_id)?
             }
-            KnownCondition::ExactEventPropertyContains(exact_event_match) => {
-                self.match_exact_event_property_contains(exact_event_match)?
-            }
             KnownCondition::IsUserMention => {
                 if let Some(uid) = user_id {
                     self.user_mentions.contains(uid)
@@ -275,6 +262,7 @@ impl PushRuleEvaluator {
                     false
                 }
             }
+            KnownCondition::IsRoomMention => self.room_mention,
             KnownCondition::ContainsDisplayName => {
                 if let Some(dn) = display_name {
                     if !dn.is_empty() {
@@ -349,9 +337,7 @@ impl PushRuleEvaluator {
             return Ok(false);
         };
 
-        let haystack = if let Some(JsonValue::Value(SimpleJsonValue::Str(haystack))) =
-            self.flattened_keys.get(&*event_match.key)
-        {
+        let haystack = if let Some(haystack) = self.flattened_keys.get(&*event_match.key) {
             haystack
         } else {
             return Ok(false);
@@ -369,29 +355,6 @@ impl PushRuleEvaluator {
         compiled_pattern.is_match(haystack)
     }
 
-    /// Evaluates a `exact_event_match` condition. (MSC3758)
-    fn match_exact_event_match(
-        &self,
-        exact_event_match: &ExactEventMatchCondition,
-    ) -> Result<bool, Error> {
-        // First check if the feature is enabled.
-        if !self.msc3758_exact_event_match {
-            return Ok(false);
-        }
-
-        let value = &exact_event_match.value;
-
-        let haystack = if let Some(JsonValue::Value(haystack)) =
-            self.flattened_keys.get(&*exact_event_match.key)
-        {
-            haystack
-        } else {
-            return Ok(false);
-        };
-
-        Ok(haystack == &**value)
-    }
-
     /// Evaluates a `related_event_match` condition. (MSC3664)
     fn match_related_event_match(
         &self,
@@ -447,12 +410,11 @@ impl PushRuleEvaluator {
             return Ok(false);
         };
 
-        let haystack =
-            if let Some(JsonValue::Value(SimpleJsonValue::Str(haystack))) = event.get(&**key) {
-                haystack
-            } else {
-                return Ok(false);
-            };
+        let haystack = if let Some(haystack) = event.get(&**key) {
+            haystack
+        } else {
+            return Ok(false);
+        };
 
         // For the content.body we match against "words", but for everything
         // else we match against the entire value.
@@ -466,29 +428,6 @@ impl PushRuleEvaluator {
         compiled_pattern.is_match(haystack)
     }
 
-    /// Evaluates a `exact_event_property_contains` condition. (MSC3758)
-    fn match_exact_event_property_contains(
-        &self,
-        exact_event_match: &ExactEventMatchCondition,
-    ) -> Result<bool, Error> {
-        // First check if the feature is enabled.
-        if !self.msc3966_exact_event_property_contains {
-            return Ok(false);
-        }
-
-        let value = &exact_event_match.value;
-
-        let haystack = if let Some(JsonValue::Array(haystack)) =
-            self.flattened_keys.get(&*exact_event_match.key)
-        {
-            haystack
-        } else {
-            return Ok(false);
-        };
-
-        Ok(haystack.contains(&**value))
-    }
-
     /// Match the member count against an 'is' condition
     /// The `is` condition can be things like '>2', '==3' or even just '4'.
     fn match_member_count(&self, is: &str) -> Result<bool, Error> {
@@ -516,14 +455,12 @@ impl PushRuleEvaluator {
 #[test]
 fn push_rule_evaluator() {
     let mut flattened_keys = BTreeMap::new();
-    flattened_keys.insert(
-        "content.body".to_string(),
-        JsonValue::Value(SimpleJsonValue::Str("foo bar bob hello".to_string())),
-    );
+    flattened_keys.insert("content.body".to_string(), "foo bar bob hello".to_string());
     let evaluator = PushRuleEvaluator::py_new(
         flattened_keys,
         false,
         BTreeSet::new(),
+        false,
         10,
         Some(0),
         BTreeMap::new(),
@@ -531,8 +468,6 @@ fn push_rule_evaluator() {
         true,
         vec![],
         true,
-        true,
-        true,
     )
     .unwrap();
 
@@ -547,15 +482,13 @@ fn test_requires_room_version_supports_condition() {
     use crate::push::{PushRule, PushRules};
 
     let mut flattened_keys = BTreeMap::new();
-    flattened_keys.insert(
-        "content.body".to_string(),
-        JsonValue::Value(SimpleJsonValue::Str("foo bar bob hello".to_string())),
-    );
+    flattened_keys.insert("content.body".to_string(), "foo bar bob hello".to_string());
     let flags = vec![RoomVersionFeatures::ExtensibleEvents.as_str().to_string()];
     let evaluator = PushRuleEvaluator::py_new(
         flattened_keys,
         false,
         BTreeSet::new(),
+        false,
         10,
         Some(0),
         BTreeMap::new(),
@@ -563,8 +496,6 @@ fn test_requires_room_version_supports_condition() {
         false,
         flags,
         true,
-        true,
-        true,
     )
     .unwrap();
 

rust/src/push/mod.rs

@@ -56,9 +56,7 @@ use std::collections::{BTreeMap, HashMap, HashSet};
 
 use anyhow::{Context, Error};
 use log::warn;
-use pyo3::exceptions::PyTypeError;
 use pyo3::prelude::*;
-use pyo3::types::{PyBool, PyList, PyLong, PyString};
 use pythonize::{depythonize, pythonize};
 use serde::de::Error as _;
 use serde::{Deserialize, Serialize};
@@ -250,65 +248,6 @@ impl<'de> Deserialize<'de> for Action {
     }
 }
 
-/// A simple JSON values (string, int, boolean, or null).
-#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq)]
-#[serde(untagged)]
-pub enum SimpleJsonValue {
-    Str(String),
-    Int(i64),
-    Bool(bool),
-    Null,
-}
-
-impl<'source> FromPyObject<'source> for SimpleJsonValue {
-    fn extract(ob: &'source PyAny) -> PyResult<Self> {
-        if let Ok(s) = <PyString as pyo3::PyTryFrom>::try_from(ob) {
-            Ok(SimpleJsonValue::Str(s.to_string()))
-        // A bool *is* an int, ensure we try bool first.
-        } else if let Ok(b) = <PyBool as pyo3::PyTryFrom>::try_from(ob) {
-            Ok(SimpleJsonValue::Bool(b.extract()?))
-        } else if let Ok(i) = <PyLong as pyo3::PyTryFrom>::try_from(ob) {
-            Ok(SimpleJsonValue::Int(i.extract()?))
-        } else if ob.is_none() {
-            Ok(SimpleJsonValue::Null)
-        } else {
-            Err(PyTypeError::new_err(format!(
-                "Can't convert from {} to SimpleJsonValue",
-                ob.get_type().name()?
-            )))
-        }
-    }
-}
-
-/// A JSON values (list, string, int, boolean, or null).
-#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq)]
-#[serde(untagged)]
-pub enum JsonValue {
-    Array(Vec<SimpleJsonValue>),
-    Value(SimpleJsonValue),
-}
-
-impl<'source> FromPyObject<'source> for JsonValue {
-    fn extract(ob: &'source PyAny) -> PyResult<Self> {
-        if let Ok(l) = <PyList as pyo3::PyTryFrom>::try_from(ob) {
-            match l.iter().map(SimpleJsonValue::extract).collect() {
-                Ok(a) => Ok(JsonValue::Array(a)),
-                Err(e) => Err(PyTypeError::new_err(format!(
-                    "Can't convert to JsonValue::Array: {}",
-                    e
-                ))),
-            }
-        } else if let Ok(v) = SimpleJsonValue::extract(ob) {
-            Ok(JsonValue::Value(v))
-        } else {
-            Err(PyTypeError::new_err(format!(
-                "Can't convert from {} to JsonValue",
-                ob.get_type().name()?
-            )))
-        }
-    }
-}
-
 /// A condition used in push rules to match against an event.
 ///
 /// We need this split as `serde` doesn't give us the ability to have a
@@ -328,14 +267,12 @@ pub enum Condition {
 #[serde(tag = "kind")]
 pub enum KnownCondition {
     EventMatch(EventMatchCondition),
-    #[serde(rename = "com.beeper.msc3758.exact_event_match")]
-    ExactEventMatch(ExactEventMatchCondition),
     #[serde(rename = "im.nheko.msc3664.related_event_match")]
     RelatedEventMatch(RelatedEventMatchCondition),
-    #[serde(rename = "org.matrix.msc3966.exact_event_property_contains")]
-    ExactEventPropertyContains(ExactEventMatchCondition),
     #[serde(rename = "org.matrix.msc3952.is_user_mention")]
     IsUserMention,
+    #[serde(rename = "org.matrix.msc3952.is_room_mention")]
+    IsRoomMention,
     ContainsDisplayName,
     RoomMemberCount {
         #[serde(skip_serializing_if = "Option::is_none")]
@@ -372,13 +309,6 @@ pub struct EventMatchCondition {
     pub pattern_type: Option<Cow<'static, str>>,
 }
 
-/// The body of a [`Condition::ExactEventMatch`]
-#[derive(Serialize, Deserialize, Debug, Clone)]
-pub struct ExactEventMatchCondition {
-    pub key: Cow<'static, str>,
-    pub value: Cow<'static, SimpleJsonValue>,
-}
-
 /// The body of a [`Condition::RelatedEventMatch`]
 #[derive(Serialize, Deserialize, Debug, Clone)]
 pub struct RelatedEventMatchCondition {
@@ -612,48 +542,6 @@ fn test_deserialize_unstable_msc3931_condition() {
     ));
 }
 
-#[test]
-fn test_deserialize_unstable_msc3758_condition() {
-    // A string condition should work.
-    let json =
-        r#"{"kind":"com.beeper.msc3758.exact_event_match","key":"content.value","value":"foo"}"#;
-
-    let condition: Condition = serde_json::from_str(json).unwrap();
-    assert!(matches!(
-        condition,
-        Condition::Known(KnownCondition::ExactEventMatch(_))
-    ));
-
-    // A boolean condition should work.
-    let json =
-        r#"{"kind":"com.beeper.msc3758.exact_event_match","key":"content.value","value":true}"#;
-
-    let condition: Condition = serde_json::from_str(json).unwrap();
-    assert!(matches!(
-        condition,
-        Condition::Known(KnownCondition::ExactEventMatch(_))
-    ));
-
-    // An integer condition should work.
-    let json = r#"{"kind":"com.beeper.msc3758.exact_event_match","key":"content.value","value":1}"#;
-
-    let condition: Condition = serde_json::from_str(json).unwrap();
-    assert!(matches!(
-        condition,
-        Condition::Known(KnownCondition::ExactEventMatch(_))
-    ));
-
-    // A null condition should work
-    let json =
-        r#"{"kind":"com.beeper.msc3758.exact_event_match","key":"content.value","value":null}"#;
-
-    let condition: Condition = serde_json::from_str(json).unwrap();
-    assert!(matches!(
-        condition,
-        Condition::Known(KnownCondition::ExactEventMatch(_))
-    ));
-}
-
 #[test]
 fn test_deserialize_unstable_msc3952_user_condition() {
     let json = r#"{"kind":"org.matrix.msc3952.is_user_mention"}"#;
@@ -665,6 +553,17 @@ fn test_deserialize_unstable_msc3952_user_condition() {
     ));
 }
 
+#[test]
+fn test_deserialize_unstable_msc3952_room_condition() {
+    let json = r#"{"kind":"org.matrix.msc3952.is_room_mention"}"#;
+
+    let condition: Condition = serde_json::from_str(json).unwrap();
+    assert!(matches!(
+        condition,
+        Condition::Known(KnownCondition::IsRoomMention)
+    ));
+}
+
 #[test]
 fn test_deserialize_custom_condition() {
     let json = r#"{"kind":"custom_tag"}"#;

scripts-dev/check_locked_deps_have_sdists.py

@@ -1,58 +0,0 @@
-#! /usr/bin/env python
-# Copyright 2022 The Matrix.org Foundation C.I.C.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-import sys
-from pathlib import Path
-from typing import Dict, List
-
-import tomli
-
-
-def main() -> None:
-    lockfile_path = Path(__file__).parent.parent.joinpath("poetry.lock")
-    with open(lockfile_path, "rb") as lockfile:
-        lockfile_content = tomli.load(lockfile)
-
-    # Poetry 1.3+ lockfile format:
-    # There's a `files` inline table in each [[package]]
-    packages_to_assets: Dict[str, List[Dict[str, str]]] = {
-        package["name"]: package["files"] for package in lockfile_content["package"]
-    }
-
-    success = True
-
-    for package_name, assets in packages_to_assets.items():
-        has_sdist = any(asset["file"].endswith(".tar.gz") for asset in assets)
-        if not has_sdist:
-            success = False
-            print(
-                f"Locked package {package_name!r} does not have a source distribution!",
-                file=sys.stderr,
-            )
-
-    if not success:
-        print(
-            "\nThere were some problems with the Poetry lockfile (poetry.lock).",
-            file=sys.stderr,
-        )
-        sys.exit(1)
-
-    print(
-        f"Poetry lockfile OK. {len(packages_to_assets)} locked packages checked.",
-        file=sys.stderr,
-    )
-
-
-if __name__ == "__main__":
-    main()

scripts-dev/make_full_schema.sh

@@ -19,8 +19,7 @@ usage() {
   echo "-c"
   echo "  CI mode. Prints every command that the script runs."
   echo "-o <path>"
-  echo "  Directory to output full schema files to. You probably want to use"
-  echo "  '-o synapse/storage/schema'"
+  echo "  Directory to output full schema files to."
   echo "-n <schema number>"
   echo "  Schema number for the new snapshot. Used to set the location of files within "
   echo "  the output directory, mimicking that of synapse/storage/schemas."
@@ -28,11 +27,6 @@ usage() {
   echo "-h"
   echo "  Display this help text."
   echo ""
-  echo ""
-  echo "You probably want to invoke this with something like"
-  echo "  docker run --rm -e POSTGRES_PASSWORD=postgres -e POSTGRES_USER=postgres -e POSTGRES_DB=synapse -p 5432:5432 postgres:11-alpine"
-  echo "  echo postgres | scripts-dev/make_full_schema.sh -p postgres -n MY_SCHEMA_NUMBER -o synapse/storage/schema"
-  echo ""
   echo "  NB: make sure to run this against the *oldest* supported version of postgres,"
   echo "  or else pg_dump might output non-backwards-compatible syntax."
 }
@@ -195,7 +189,7 @@ python -m synapse.app.homeserver --generate-keys -c "$SQLITE_CONFIG"
 
 # Make sure the SQLite3 database is using the latest schema and has no pending background update.
 echo "Running db background jobs..."
-poetry run python synapse/_scripts/update_synapse_database.py --database-config "$SQLITE_CONFIG" --run-background-updates
+synapse/_scripts/update_synapse_database.py --database-config "$SQLITE_CONFIG" --run-background-updates
 
 # Create the PostgreSQL database.
 echo "Creating postgres databases..."
@@ -204,7 +198,7 @@ createdb --lc-collate=C --lc-ctype=C --template=template0 "$POSTGRES_MAIN_DB_NAM
 createdb --lc-collate=C --lc-ctype=C --template=template0 "$POSTGRES_STATE_DB_NAME"
 
 echo "Running db background jobs..."
-poetry run python synapse/_scripts/update_synapse_database.py --database-config "$POSTGRES_CONFIG" --run-background-updates
+synapse/_scripts/update_synapse_database.py --database-config "$POSTGRES_CONFIG" --run-background-updates
 
 
 echo "Dropping unwanted db tables..."
@@ -299,12 +293,4 @@ pg_dump --format=plain --data-only --inserts --no-tablespaces --no-acl --no-owne
 pg_dump --format=plain --schema-only         --no-tablespaces --no-acl --no-owner "$POSTGRES_STATE_DB_NAME"  | cleanup_pg_schema  > "$OUTPUT_DIR/state/full_schemas/$SCHEMA_NUMBER/full.sql.postgres"
 pg_dump --format=plain --data-only --inserts --no-tablespaces --no-acl --no-owner "$POSTGRES_STATE_DB_NAME"  | cleanup_pg_schema >> "$OUTPUT_DIR/state/full_schemas/$SCHEMA_NUMBER/full.sql.postgres"
 
-if [[ "$OUTPUT_DIR" == *synapse/storage/schema ]]; then
-  echo "Updating contrib/datagrip symlinks..."
-  ln -sf "../../synapse/storage/schema/common/full_schemas/$SCHEMA_NUMBER/full.sql.postgres" "contrib/datagrip/common.sql"
-  ln -sf "../../synapse/storage/schema/main/full_schemas/$SCHEMA_NUMBER/full.sql.postgres"   "contrib/datagrip/main.sql"
-  ln -sf "../../synapse/storage/schema/state/full_schemas/$SCHEMA_NUMBER/full.sql.postgres"  "contrib/datagrip/state.sql"
-else
-  echo "Not updating contrib/datagrip symlinks (unknown output directory)"
-fi
 echo "Done! Files dumped to: $OUTPUT_DIR"

stubs/sortedcontainers/sortedlist.pyi

@@ -29,6 +29,7 @@ _Repr = Callable[[], str]
 def recursive_repr(fillvalue: str = ...) -> Callable[[_Repr], _Repr]: ...
 
 class SortedList(MutableSequence[_T]):
+
     DEFAULT_LOAD_FACTOR: int = ...
     def __init__(
         self,

stubs/synapse/synapse_rust/push.pyi

@@ -14,7 +14,7 @@
 
 from typing import Any, Collection, Dict, Mapping, Optional, Sequence, Set, Tuple, Union
 
-from synapse.types import JsonDict, JsonValue
+from synapse.types import JsonDict
 
 class PushRule:
     @property
@@ -56,18 +56,17 @@ def get_base_rule_ids() -> Collection[str]: ...
 class PushRuleEvaluator:
     def __init__(
         self,
-        flattened_keys: Mapping[str, JsonValue],
+        flattened_keys: Mapping[str, str],
         has_mentions: bool,
         user_mentions: Set[str],
+        room_mention: bool,
         room_member_count: int,
         sender_power_level: Optional[int],
         notification_power_levels: Mapping[str, int],
-        related_events_flattened: Mapping[str, Mapping[str, JsonValue]],
+        related_events_flattened: Mapping[str, Mapping[str, str]],
         related_event_match_enabled: bool,
         room_version_feature_flags: Tuple[str, ...],
         msc3931_enabled: bool,
-        msc3758_exact_event_match: bool,
-        msc3966_exact_event_property_contains: bool,
     ): ...
     def run(
         self,

synapse/_scripts/register_new_matrix_user.py

@@ -47,6 +47,7 @@ def request_registration(
     _print: Callable[[str], None] = print,
     exit: Callable[[int], None] = sys.exit,
 ) -> None:
+
     url = "%s/_synapse/admin/v1/register" % (server_location.rstrip("/"),)
 
     # Get the nonce
@@ -153,6 +154,7 @@ def register_new_user(
 
 
 def main() -> None:
+
     logging.captureWarnings(True)
 
     parser = argparse.ArgumentParser(

synapse/_scripts/synapse_port_db.py

@@ -94,80 +94,61 @@ reactor = cast(ISynapseReactor, reactor_)
 logger = logging.getLogger("synapse_port_db")
 
 
-# SQLite doesn't have a dedicated boolean type (it stores True/False as 1/0). This means
-# portdb will read sqlite bools as integers, then try to insert them into postgres
-# boolean columns---which fails. Lacking some Python-parseable metaschema, we must
-# specify which integer columns should be inserted as booleans into postgres.
 BOOLEAN_COLUMNS = {
-    "access_tokens": ["used"],
-    "account_validity": ["email_sent"],
-    "device_lists_changes_in_room": ["converted_to_destinations"],
-    "device_lists_outbound_pokes": ["sent"],
-    "devices": ["hidden"],
-    "e2e_fallback_keys_json": ["used"],
-    "e2e_room_keys": ["is_verified"],
-    "event_edges": ["is_state"],
     "events": ["processed", "outlier", "contains_url"],
-    "local_media_repository": ["safe_from_quarantine"],
+    "rooms": ["is_public", "has_auth_chain_index"],
+    "event_edges": ["is_state"],
     "presence_list": ["accepted"],
     "presence_stream": ["currently_active"],
     "public_room_list_stream": ["visibility"],
-    "pushers": ["enabled"],
+    "devices": ["hidden"],
+    "device_lists_outbound_pokes": ["sent"],
+    "users_who_share_rooms": ["share_private"],
+    "e2e_room_keys": ["is_verified"],
+    "account_validity": ["email_sent"],
     "redactions": ["have_censored"],
     "room_stats_state": ["is_federatable"],
-    "rooms": ["is_public", "has_auth_chain_index"],
+    "local_media_repository": ["safe_from_quarantine"],
     "users": ["shadow_banned", "approved"],
-    "un_partial_stated_event_stream": ["rejection_status_changed"],
-    "users_who_share_rooms": ["share_private"],
+    "e2e_fallback_keys_json": ["used"],
+    "access_tokens": ["used"],
+    "device_lists_changes_in_room": ["converted_to_destinations"],
+    "pushers": ["enabled"],
 }
 
 
-# These tables are never deleted from in normal operation [*], so we can resume porting
-# over rows from a previous attempt rather than starting from scratch.
-#
-# [*]: We do delete from many of these tables when purging a room, and
-#      presumably when purging old events. So we might e.g.
-#
-#      1. Run portdb and port half of some table.
-#      2. Stop portdb.
-#      3. Purge something, deleting some of the rows we've ported over.
-#      4. Restart portdb. The rows deleted from sqlite are still present in postgres.
-#
-#      But this isn't the end of the world: we should be able to repeat the purge
-#      on the postgres DB when porting completes.
 APPEND_ONLY_TABLES = [
-    "cache_invalidation_stream_by_instance",
-    "event_auth",
-    "event_edges",
-    "event_json",
     "event_reference_hashes",
-    "event_search",
-    "event_to_state_groups",
     "events",
-    "ex_outlier_stream",
-    "local_media_repository",
-    "local_media_repository_thumbnails",
-    "presence_stream",
-    "public_room_list_stream",
-    "push_rules_stream",
-    "received_transactions",
-    "redactions",
-    "rejections",
-    "remote_media_cache",
-    "remote_media_cache_thumbnails",
+    "event_json",
+    "state_events",
     "room_memberships",
+    "topics",
     "room_names",
     "rooms",
+    "local_media_repository",
+    "local_media_repository_thumbnails",
+    "remote_media_cache",
+    "remote_media_cache_thumbnails",
+    "redactions",
+    "event_edges",
+    "event_auth",
+    "received_transactions",
     "sent_transactions",
-    "state_events",
-    "state_group_edges",
+    "transaction_id_to_pdu",
+    "users",
     "state_groups",
     "state_groups_state",
+    "event_to_state_groups",
+    "rejections",
+    "event_search",
+    "presence_stream",
+    "push_rules_stream",
+    "ex_outlier_stream",
+    "cache_invalidation_stream_by_instance",
+    "public_room_list_stream",
+    "state_group_edges",
     "stream_ordering_to_exterm",
-    "topics",
-    "transaction_id_to_pdu",
-    "un_partial_stated_event_stream",
-    "users",
 ]
 
 
@@ -1205,6 +1186,7 @@ class CursesProgress(Progress):
         if self.finished:
             status = "Time spent: %s (Done!)" % (duration_str,)
         else:
+
             if self.total_processed > 0:
                 left = float(self.total_remaining) / self.total_processed
 

synapse/_scripts/synctl.py

@@ -167,6 +167,7 @@ Worker = collections.namedtuple(
 
 
 def main() -> None:
+
     parser = argparse.ArgumentParser()
 
     parser.add_argument(

synapse/api/auth.py

@@ -32,6 +32,7 @@ from synapse.appservice import ApplicationService
 from synapse.http import get_request_user_agent
 from synapse.http.site import SynapseRequest
 from synapse.logging.opentracing import (
+    SynapseTags,
     active_span,
     force_tracing,
     start_active_span,
@@ -161,6 +162,12 @@ class Auth:
                 parent_span.set_tag(
                     "authenticated_entity", requester.authenticated_entity
                 )
+                # We tag the Synapse instance name so that it's an easy jumping
+                # off point into the logs. Can also be used to filter for an
+                # instance that is under load.
+                parent_span.set_tag(
+                    SynapseTags.INSTANCE_NAME, self.hs.get_instance_name()
+                )
                 parent_span.set_tag("user_id", requester.user.to_string())
                 if requester.device_id is not None:
                     parent_span.set_tag("device_id", requester.device_id)

synapse/api/errors.py

@@ -108,10 +108,6 @@ class Codes(str, Enum):
 
     USER_AWAITING_APPROVAL = "ORG.MATRIX.MSC3866_USER_AWAITING_APPROVAL"
 
-    # Attempt to send a second annotation with the same event type & annotation key
-    # MSC2677
-    DUPLICATE_ANNOTATION = "M_DUPLICATE_ANNOTATION"
-
 
 class CodeMessageException(RuntimeError):
     """An exception with integer code and message string attributes.
@@ -755,25 +751,3 @@ class ModuleFailedException(Exception):
     Raised when a module API callback fails, for example because it raised an
     exception.
     """
-
-
-class PartialStateConflictError(SynapseError):
-    """An internal error raised when attempting to persist an event with partial state
-    after the room containing the event has been un-partial stated.
-
-    This error should be handled by recomputing the event context and trying again.
-
-    This error has an HTTP status code so that it can be transported over replication.
-    It should not be exposed to clients.
-    """
-
-    @staticmethod
-    def message() -> str:
-        return "Cannot persist partial state event in un-partial stated room"
-
-    def __init__(self) -> None:
-        super().__init__(
-            HTTPStatus.CONFLICT,
-            msg=PartialStateConflictError.message(),
-            errcode=Codes.UNKNOWN,
-        )

synapse/api/filtering.py

@@ -219,13 +219,9 @@ class FilterCollection:
         self._room_timeline_filter = Filter(hs, room_filter_json.get("timeline", {}))
         self._room_state_filter = Filter(hs, room_filter_json.get("state", {}))
         self._room_ephemeral_filter = Filter(hs, room_filter_json.get("ephemeral", {}))
-        self._room_account_data_filter = Filter(
-            hs, room_filter_json.get("account_data", {})
-        )
+        self._room_account_data = Filter(hs, room_filter_json.get("account_data", {}))
         self._presence_filter = Filter(hs, filter_json.get("presence", {}))
-        self._global_account_data_filter = Filter(
-            hs, filter_json.get("account_data", {})
-        )
+        self._account_data = Filter(hs, filter_json.get("account_data", {}))
 
         self.include_leave = filter_json.get("room", {}).get("include_leave", False)
         self.event_fields = filter_json.get("event_fields", [])
@@ -260,10 +256,8 @@ class FilterCollection:
     ) -> List[UserPresenceState]:
         return await self._presence_filter.filter(presence_states)
 
-    async def filter_global_account_data(
-        self, events: Iterable[JsonDict]
-    ) -> List[JsonDict]:
-        return await self._global_account_data_filter.filter(events)
+    async def filter_account_data(self, events: Iterable[JsonDict]) -> List[JsonDict]:
+        return await self._account_data.filter(events)
 
     async def filter_room_state(self, events: Iterable[EventBase]) -> List[EventBase]:
         return await self._room_state_filter.filter(
@@ -285,7 +279,7 @@ class FilterCollection:
     async def filter_room_account_data(
         self, events: Iterable[JsonDict]
     ) -> List[JsonDict]:
-        return await self._room_account_data_filter.filter(
+        return await self._room_account_data.filter(
             await self._room_filter.filter(events)
         )
 
@@ -298,13 +292,6 @@ class FilterCollection:
             or self._presence_filter.filters_all_senders()
         )
 
-    def blocks_all_global_account_data(self) -> bool:
-        """True if all global acount data will be filtered out."""
-        return (
-            self._global_account_data_filter.filters_all_types()
-            or self._global_account_data_filter.filters_all_senders()
-        )
-
     def blocks_all_room_ephemeral(self) -> bool:
         return (
             self._room_ephemeral_filter.filters_all_types()
@@ -312,13 +299,6 @@ class FilterCollection:
             or self._room_ephemeral_filter.filters_all_rooms()
         )
 
-    def blocks_all_room_account_data(self) -> bool:
-        return (
-            self._room_account_data_filter.filters_all_types()
-            or self._room_account_data_filter.filters_all_senders()
-            or self._room_account_data_filter.filters_all_rooms()
-        )
-
     def blocks_all_room_timeline(self) -> bool:
         return (
             self._room_timeline_filter.filters_all_types()

synapse/app/_base.py

@@ -213,7 +213,7 @@ def handle_startup_exception(e: Exception) -> NoReturn:
 def redirect_stdio_to_logs() -> None:
     streams = [("stdout", LogLevel.info), ("stderr", LogLevel.error)]
 
-    for stream, level in streams:
+    for (stream, level) in streams:
         oldStream = getattr(sys, stream)
         loggingFile = LoggingFile(
             logger=twisted.logger.Logger(namespace=stream),

synapse/app/admin_cmd.py

@@ -17,7 +17,7 @@ import logging
 import os
 import sys
 import tempfile
-from typing import List, Mapping, Optional
+from typing import List, Optional
 
 from twisted.internet import defer, task
 
@@ -44,7 +44,6 @@ from synapse.storage.databases.main.event_push_actions import (
 )
 from synapse.storage.databases.main.events_worker import EventsWorkerStore
 from synapse.storage.databases.main.filtering import FilteringWorkerStore
-from synapse.storage.databases.main.media_repository import MediaRepositoryStore
 from synapse.storage.databases.main.profile import ProfileWorkerStore
 from synapse.storage.databases.main.push_rule import PushRulesWorkerStore
 from synapse.storage.databases.main.receipts import ReceiptsWorkerStore
@@ -87,7 +86,6 @@ class AdminCmdSlavedStore(
     RegistrationWorkerStore,
     RoomWorkerStore,
     ProfileWorkerStore,
-    MediaRepositoryStore,
 ):
     def __init__(
         self,
@@ -151,7 +149,7 @@ class FileExfiltrationWriter(ExfiltrationWriter):
 
         with open(events_file, "a") as f:
             for event in events:
-                json.dump(event.get_pdu_json(), fp=f)
+                print(json.dumps(event.get_pdu_json()), file=f)
 
     def write_state(
         self, room_id: str, event_id: str, state: StateMap[EventBase]
@@ -164,7 +162,7 @@ class FileExfiltrationWriter(ExfiltrationWriter):
 
         with open(event_file, "a") as f:
             for event in state.values():
-                json.dump(event.get_pdu_json(), fp=f)
+                print(json.dumps(event.get_pdu_json()), file=f)
 
     def write_invite(
         self, room_id: str, event: EventBase, state: StateMap[EventBase]
@@ -180,7 +178,7 @@ class FileExfiltrationWriter(ExfiltrationWriter):
 
         with open(invite_state, "a") as f:
             for event in state.values():
-                json.dump(event, fp=f)
+                print(json.dumps(event), file=f)
 
     def write_knock(
         self, room_id: str, event: EventBase, state: StateMap[EventBase]
@@ -196,7 +194,7 @@ class FileExfiltrationWriter(ExfiltrationWriter):
 
         with open(knock_state, "a") as f:
             for event in state.values():
-                json.dump(event, fp=f)
+                print(json.dumps(event), file=f)
 
     def write_profile(self, profile: JsonDict) -> None:
         user_directory = os.path.join(self.base_directory, "user_data")
@@ -204,7 +202,7 @@ class FileExfiltrationWriter(ExfiltrationWriter):
         profile_file = os.path.join(user_directory, "profile")
 
         with open(profile_file, "a") as f:
-            json.dump(profile, fp=f)
+            print(json.dumps(profile), file=f)
 
     def write_devices(self, devices: List[JsonDict]) -> None:
         user_directory = os.path.join(self.base_directory, "user_data")
@@ -213,7 +211,7 @@ class FileExfiltrationWriter(ExfiltrationWriter):
 
         for device in devices:
             with open(device_file, "a") as f:
-                json.dump(device, fp=f)
+                print(json.dumps(device), file=f)
 
     def write_connections(self, connections: List[JsonDict]) -> None:
         user_directory = os.path.join(self.base_directory, "user_data")
@@ -222,28 +220,7 @@ class FileExfiltrationWriter(ExfiltrationWriter):
 
         for connection in connections:
             with open(connection_file, "a") as f:
-                json.dump(connection, fp=f)
-
-    def write_account_data(
-        self, file_name: str, account_data: Mapping[str, JsonDict]
-    ) -> None:
-        account_data_directory = os.path.join(
-            self.base_directory, "user_data", "account_data"
-        )
-        os.makedirs(account_data_directory, exist_ok=True)
-
-        account_data_file = os.path.join(account_data_directory, file_name)
-
-        with open(account_data_file, "a") as f:
-            json.dump(account_data, fp=f)
-
-    def write_media_id(self, media_id: str, media_metadata: JsonDict) -> None:
-        file_directory = os.path.join(self.base_directory, "media_ids")
-        os.makedirs(file_directory, exist_ok=True)
-        media_id_file = os.path.join(file_directory, media_id)
-
-        with open(media_id_file, "w") as f:
-            json.dump(media_metadata, fp=f)
+                print(json.dumps(connection), file=f)
 
     def finished(self) -> str:
         return self.base_directory

synapse/app/complement_fork_starter.py

@@ -219,7 +219,7 @@ def main() -> None:
     # memory space and don't need to repeat the work of loading the code!
     # Instead of using fork() directly, we use the multiprocessing library,
     # which uses fork() on Unix platforms.
-    for func, worker_args in zip(worker_functions, args_by_worker):
+    for (func, worker_args) in zip(worker_functions, args_by_worker):
         process = multiprocessing.Process(
             target=_worker_entrypoint, args=(func, proxy_reactor, worker_args)
         )

synapse/app/generic_worker.py

@@ -157,6 +157,7 @@ class GenericWorkerServer(HomeServer):
     DATASTORE_CLASS = GenericWorkerSlavedStore  # type: ignore
 
     def _listen_http(self, listener_config: ListenerConfig) -> None:
+
         assert listener_config.http_options is not None
 
         # We always include a health resource.

synapse/app/homeserver.py

@@ -321,6 +321,7 @@ def setup(config_options: List[str]) -> SynapseHomeServer:
             and not config.registration.registrations_require_3pid
             and not config.registration.registration_requires_token
         ):
+
             raise ConfigError(
                 "You have enabled open registration without any verification. This is a known vector for "
                 "spam and abuse. If you would like to allow public registration, please consider adding email, "

synapse/app/phone_stats_home.py

@@ -15,7 +15,7 @@ import logging
 import math
 import resource
 import sys
-from typing import TYPE_CHECKING, List, Mapping, Sized, Tuple
+from typing import TYPE_CHECKING, List, Sized, Tuple
 
 from prometheus_client import Gauge
 
@@ -194,7 +194,7 @@ def start_phone_stats_home(hs: "HomeServer") -> None:
     @wrap_as_background_process("generate_monthly_active_users")
     async def generate_monthly_active_users() -> None:
         current_mau_count = 0
-        current_mau_count_by_service: Mapping[str, int] = {}
+        current_mau_count_by_service = {}
         reserved_users: Sized = ()
         store = hs.get_datastores().main
         if hs.config.server.limit_usage_by_mau or hs.config.server.mau_stats_only:

synapse/config/consent.py

@@ -22,6 +22,7 @@ from ._base import Config
 
 
 class ConsentConfig(Config):
+
     section = "consent"
 
     def __init__(self, *args: Any):

synapse/config/database.py

@@ -154,6 +154,7 @@ class DatabaseConfig(Config):
             logger.warning(NON_SQLITE_DATABASE_PATH_WARNING)
 
     def set_databasepath(self, database_path: str) -> None:
+
         if database_path != ":memory:":
             database_path = self.abspath(database_path)
 

synapse/config/experimental.py

@@ -169,28 +169,17 @@ class ExperimentalConfig(Config):
         # MSC3925: do not replace events with their edits
         self.msc3925_inhibit_edit = experimental.get("msc3925_inhibit_edit", False)
 
-        # MSC3758: exact_event_match push rule condition
-        self.msc3758_exact_event_match = experimental.get(
-            "msc3758_exact_event_match", False
-        )
-
         # MSC3873: Disambiguate event_match keys.
         self.msc3783_escape_event_match_key = experimental.get(
             "msc3783_escape_event_match_key", False
         )
 
-        # MSC3952: Intentional mentions, this depends on MSC3758.
-        self.msc3952_intentional_mentions = (
-            experimental.get("msc3952_intentional_mentions", False)
-            and self.msc3758_exact_event_match
+        # MSC3952: Intentional mentions
+        self.msc3952_intentional_mentions = experimental.get(
+            "msc3952_intentional_mentions", False
         )
 
         # MSC3959: Do not generate notifications for edits.
         self.msc3958_supress_edit_notifs = experimental.get(
             "msc3958_supress_edit_notifs", False
         )
-
-        # MSC3966: exact_event_property_contains push rule condition.
-        self.msc3966_exact_event_property_contains = experimental.get(
-            "msc3966_exact_event_property_contains", False
-        )

synapse/config/homeserver.py

@@ -56,6 +56,7 @@ from .workers import WorkerConfig
 
 
 class HomeServerConfig(RootConfig):
+
     config_classes = [
         ModulesConfig,
         ServerConfig,

synapse/config/ratelimiting.py

@@ -46,6 +46,7 @@ class RatelimitConfig(Config):
     section = "ratelimiting"
 
     def read_config(self, config: JsonDict, **kwargs: Any) -> None:
+
         # Load the new-style messages config if it exists. Otherwise fall back
         # to the old method.
         if "rc_message" in config:
@@ -86,18 +87,9 @@ class RatelimitConfig(Config):
             defaults={"per_second": 0.1, "burst_count": 5},
         )
 
-        # It is reasonable to login with a bunch of devices at once (i.e. when
-        # setting up an account), but it is *not* valid to continually be
-        # logging into new devices.
         rc_login_config = config.get("rc_login", {})
-        self.rc_login_address = RatelimitSettings(
-            rc_login_config.get("address", {}),
-            defaults={"per_second": 0.003, "burst_count": 5},
-        )
-        self.rc_login_account = RatelimitSettings(
-            rc_login_config.get("account", {}),
-            defaults={"per_second": 0.003, "burst_count": 5},
-        )
+        self.rc_login_address = RatelimitSettings(rc_login_config.get("address", {}))
+        self.rc_login_account = RatelimitSettings(rc_login_config.get("account", {}))
         self.rc_login_failed_attempts = RatelimitSettings(
             rc_login_config.get("failed_attempts", {})
         )

synapse/config/redis.py

@@ -33,5 +33,4 @@ class RedisConfig(Config):
 
         self.redis_host = redis_config.get("host", "localhost")
         self.redis_port = redis_config.get("port", 6379)
-        self.redis_dbid = redis_config.get("dbid", None)
         self.redis_password = redis_config.get("password")

synapse/config/repository.py

@@ -116,6 +116,7 @@ class ContentRepositoryConfig(Config):
     section = "media"
 
     def read_config(self, config: JsonDict, **kwargs: Any) -> None:
+
         # Only enable the media repo if either the media repo is enabled or the
         # current worker app is the media repo.
         if (

synapse/config/room_directory.py

@@ -13,7 +13,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from typing import Any, Collection
+from typing import Any, List
 
 from matrix_common.regex import glob_to_regex
 
@@ -70,7 +70,7 @@ class RoomDirectoryConfig(Config):
         return False
 
     def is_publishing_room_allowed(
-        self, user_id: str, room_id: str, aliases: Collection[str]
+        self, user_id: str, room_id: str, aliases: List[str]
     ) -> bool:
         """Checks if the given user is allowed to publish the room
 
@@ -122,7 +122,7 @@ class _RoomDirectoryRule:
         except Exception as e:
             raise ConfigError("Failed to parse glob into regex") from e
 
-    def matches(self, user_id: str, room_id: str, aliases: Collection[str]) -> bool:
+    def matches(self, user_id: str, room_id: str, aliases: List[str]) -> bool:
         """Tests if this rule matches the given user_id, room_id and aliases.
 
         Args:

synapse/config/server.py

@@ -177,7 +177,6 @@ KNOWN_RESOURCES = {
     "client",
     "consent",
     "federation",
-    "health",
     "keys",
     "media",
     "metrics",
@@ -735,6 +734,7 @@ class ServerConfig(Config):
         listeners: Optional[List[dict]],
         **kwargs: Any,
     ) -> str:
+
         _, bind_port = parse_and_validate_server_name(server_name)
         if bind_port is not None:
             unsecure_port = bind_port - 400

synapse/config/tls.py

@@ -30,6 +30,7 @@ class TlsConfig(Config):
     section = "tls"
 
     def read_config(self, config: JsonDict, **kwargs: Any) -> None:
+
         self.tls_certificate_file = self.abspath(config.get("tls_certificate_path"))
         self.tls_private_key_file = self.abspath(config.get("tls_private_key_path"))
 

synapse/crypto/keyring.py

@@ -399,7 +399,7 @@ class Keyring:
         # We now convert the returned list of results into a map from server
         # name to key ID to FetchKeyResult, to return.
         to_return: Dict[str, Dict[str, FetchKeyResult]] = {}
-        for request, results in zip(deduped_requests, results_per_request):
+        for (request, results) in zip(deduped_requests, results_per_request):
             to_return_by_server = to_return.setdefault(request.server_name, {})
             for key_id, key_result in results.items():
                 existing = to_return_by_server.get(key_id)

synapse/event_auth.py

@@ -16,7 +16,18 @@
 import collections.abc
 import logging
 import typing
-from typing import Any, Dict, Iterable, List, Mapping, Optional, Set, Tuple, Union
+from typing import (
+    Any,
+    Collection,
+    Dict,
+    Iterable,
+    List,
+    Mapping,
+    Optional,
+    Set,
+    Tuple,
+    Union,
+)
 
 from canonicaljson import encode_canonical_json
 from signedjson.key import decode_verify_key_bytes
@@ -45,13 +56,7 @@ from synapse.api.room_versions import (
     RoomVersions,
 )
 from synapse.storage.databases.main.events_worker import EventRedactBehaviour
-from synapse.types import (
-    MutableStateMap,
-    StateMap,
-    StrCollection,
-    UserID,
-    get_domain_from_id,
-)
+from synapse.types import MutableStateMap, StateMap, UserID, get_domain_from_id
 
 if typing.TYPE_CHECKING:
     # conditional imports to avoid import cycle
@@ -64,7 +69,7 @@ logger = logging.getLogger(__name__)
 class _EventSourceStore(Protocol):
     async def get_events(
         self,
-        event_ids: StrCollection,
+        event_ids: Collection[str],
         redact_behaviour: EventRedactBehaviour,
         get_prev_content: bool = False,
         allow_rejected: bool = False,

synapse/events/__init__.py

@@ -39,7 +39,7 @@ from unpaddedbase64 import encode_base64
 
 from synapse.api.constants import RelationTypes
 from synapse.api.room_versions import EventFormatVersions, RoomVersion, RoomVersions
-from synapse.types import JsonDict, RoomStreamToken, StrCollection
+from synapse.types import JsonDict, RoomStreamToken
 from synapse.util.caches import intern_dict
 from synapse.util.frozenutils import freeze
 from synapse.util.stringutils import strtobool
@@ -413,7 +413,7 @@ class EventBase(metaclass=abc.ABCMeta):
         """
         return [e for e, _ in self._dict["prev_events"]]
 
-    def auth_event_ids(self) -> StrCollection:
+    def auth_event_ids(self) -> Sequence[str]:
         """Returns the list of auth event IDs. The order matches the order
         specified in the event, though there is no meaning to it.
 
@@ -558,7 +558,7 @@ class FrozenEventV2(EventBase):
         """
         return self._dict["prev_events"]
 
-    def auth_event_ids(self) -> StrCollection:
+    def auth_event_ids(self) -> Sequence[str]:
         """Returns the list of auth event IDs. The order matches the order
         specified in the event, though there is no meaning to it.
 

synapse/events/builder.py

@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 import logging
-from typing import TYPE_CHECKING, Any, Collection, Dict, List, Optional, Tuple, Union
+from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union
 
 import attr
 from signedjson.types import SigningKey
@@ -103,7 +103,7 @@ class EventBuilder:
 
     async def build(
         self,
-        prev_event_ids: Collection[str],
+        prev_event_ids: List[str],
         auth_event_ids: Optional[List[str]],
         depth: Optional[int] = None,
     ) -> EventBase:
@@ -136,7 +136,7 @@ class EventBuilder:
 
         format_version = self.room_version.event_format
         # The types of auth/prev events changes between event versions.
-        prev_events: Union[Collection[str], List[Tuple[str, Dict[str, str]]]]
+        prev_events: Union[List[str], List[Tuple[str, Dict[str, str]]]]
         auth_events: Union[List[str], List[Tuple[str, Dict[str, str]]]]
         if format_version == EventFormatVersions.ROOM_V1_V2:
             auth_events = await self._store.add_event_hashes(auth_event_ids)

synapse/events/snapshot.py

@@ -11,7 +11,6 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-from abc import ABC, abstractmethod
 from typing import TYPE_CHECKING, List, Optional, Tuple
 
 import attr
@@ -27,51 +26,8 @@ if TYPE_CHECKING:
     from synapse.types.state import StateFilter
 
 
-class UnpersistedEventContextBase(ABC):
-    """
-    This is a base class for EventContext and UnpersistedEventContext, objects which
-    hold information relevant to storing an associated event. Note that an
-    UnpersistedEventContexts must be converted into an EventContext before it is
-    suitable to send to the db with its associated event.
-
-    Attributes:
-        _storage: storage controllers for interfacing with the database
-        app_service: If the associated event is being sent by a (local) application service, that
-            app service.
-    """
-
-    def __init__(self, storage_controller: "StorageControllers"):
-        self._storage: "StorageControllers" = storage_controller
-        self.app_service: Optional[ApplicationService] = None
-
-    @abstractmethod
-    async def persist(
-        self,
-        event: EventBase,
-    ) -> "EventContext":
-        """
-        A method to convert an UnpersistedEventContext to an EventContext, suitable for
-        sending to the database with the associated event.
-        """
-        pass
-
-    @abstractmethod
-    async def get_prev_state_ids(
-        self, state_filter: Optional["StateFilter"] = None
-    ) -> StateMap[str]:
-        """
-        Gets the room state at the event (ie not including the event if the event is a
-        state event).
-
-        Args:
-            state_filter: specifies the type of state event to fetch from DB, example:
-            EventTypes.JoinRules
-        """
-        pass
-
-
 @attr.s(slots=True, auto_attribs=True)
-class EventContext(UnpersistedEventContextBase):
+class EventContext:
     """
     Holds information relevant to persisting an event
 
@@ -121,6 +77,9 @@ class EventContext(UnpersistedEventContextBase):
         delta_ids: If ``prev_group`` is not None, the state delta between ``prev_group``
             and ``state_group``.
 
+        app_service: If this event is being sent by a (local) application service, that
+            app service.
+
         partial_state: if True, we may be storing this event with a temporary,
             incomplete state.
     """
@@ -163,9 +122,6 @@ class EventContext(UnpersistedEventContextBase):
         """Return an EventContext instance suitable for persisting an outlier event"""
         return EventContext(storage=storage)
 
-    async def persist(self, event: EventBase) -> "EventContext":
-        return self
-
     async def serialize(self, event: EventBase, store: "DataStore") -> JsonDict:
         """Converts self to a type that can be serialized as JSON, and then
         deserialized by `deserialize`
@@ -298,128 +254,6 @@ class EventContext(UnpersistedEventContextBase):
         )
 
 
-@attr.s(slots=True, auto_attribs=True)
-class UnpersistedEventContext(UnpersistedEventContextBase):
-    """
-    The event context holds information about the state groups for an event. It is important
-    to remember that an event technically has two state groups: the state group before the
-    event, and the state group after the event. If the event is not a state event, the state
-    group will not change (ie the state group before the event will be the same as the state
-    group after the event), but if it is a state event the state group before the event
-    will differ from the state group after the event.
-    This is a version of an EventContext before the new state group (if any) has been
-    computed and stored. It contains information about the state before the event (which
-    also may be the information after the event, if the event is not a state event). The
-    UnpersistedEventContext must be converted into an EventContext by calling the method
-    'persist' on it before it is suitable to be sent to the DB for processing.
-
-        state_group_after_event:
-             The state group after the event. This will always be None until it is persisted.
-             If the event is not a state event, this will be the same as
-             state_group_before_event.
-
-        state_group_before_event:
-            The ID of the state group representing the state of the room before this event.
-
-        state_delta_due_to_event:
-            If the event is a state event, then this is the delta of the state between
-             `state_group` and `state_group_before_event`
-
-        prev_group_for_state_group_before_event:
-            If it is known, ``state_group_before_event``'s previous state group.
-
-        delta_ids_to_state_group_before_event:
-             If ``prev_group_for_state_group_before_event`` is not None, the state delta
-             between ``prev_group_for_state_group_before_event`` and ``state_group_before_event``.
-
-        partial_state:
-            Whether the event has partial state.
-
-        state_map_before_event:
-            A map of the state before the event, i.e. the state at `state_group_before_event`
-    """
-
-    _storage: "StorageControllers"
-    state_group_before_event: Optional[int]
-    state_group_after_event: Optional[int]
-    state_delta_due_to_event: Optional[dict]
-    prev_group_for_state_group_before_event: Optional[int]
-    delta_ids_to_state_group_before_event: Optional[StateMap[str]]
-    partial_state: bool
-    state_map_before_event: Optional[StateMap[str]] = None
-
-    async def get_prev_state_ids(
-        self, state_filter: Optional["StateFilter"] = None
-    ) -> StateMap[str]:
-        """
-        Gets the room state map, excluding this event.
-
-        Args:
-            state_filter: specifies the type of state event to fetch from DB
-
-        Returns:
-            Maps a (type, state_key) to the event ID of the state event matching
-            this tuple.
-        """
-        if self.state_map_before_event:
-            return self.state_map_before_event
-
-        assert self.state_group_before_event is not None
-        return await self._storage.state.get_state_ids_for_group(
-            self.state_group_before_event, state_filter
-        )
-
-    async def persist(self, event: EventBase) -> EventContext:
-        """
-        Creates a full `EventContext` for the event, persisting any referenced state that
-        has not yet been persisted.
-
-        Args:
-             event: event that the EventContext is associated with.
-
-        Returns: An EventContext suitable for sending to the database with the event
-        for persisting
-        """
-        assert self.partial_state is not None
-
-        # If we have a full set of state for before the event but don't have a state
-        # group for that state, we need to get one
-        if self.state_group_before_event is None:
-            assert self.state_map_before_event
-            state_group_before_event = await self._storage.state.store_state_group(
-                event.event_id,
-                event.room_id,
-                prev_group=self.prev_group_for_state_group_before_event,
-                delta_ids=self.delta_ids_to_state_group_before_event,
-                current_state_ids=self.state_map_before_event,
-            )
-            self.state_group_before_event = state_group_before_event
-
-        # if the event isn't a state event the state group doesn't change
-        if not self.state_delta_due_to_event:
-            state_group_after_event = self.state_group_before_event
-
-        # otherwise if it is a state event we need to get a state group for it
-        else:
-            state_group_after_event = await self._storage.state.store_state_group(
-                event.event_id,
-                event.room_id,
-                prev_group=self.state_group_before_event,
-                delta_ids=self.state_delta_due_to_event,
-                current_state_ids=None,
-            )
-
-        return EventContext.with_state(
-            storage=self._storage,
-            state_group=state_group_after_event,
-            state_group_before_event=self.state_group_before_event,
-            state_delta_due_to_event=self.state_delta_due_to_event,
-            partial_state=self.partial_state,
-            prev_group=self.state_group_before_event,
-            delta_ids=self.state_delta_due_to_event,
-        )
-
-
 def _encode_state_dict(
     state_dict: Optional[StateMap[str]],
 ) -> Optional[List[Tuple[str, str, str]]]:

synapse/events/third_party_rules.py

@@ -18,7 +18,7 @@ from twisted.internet.defer import CancelledError
 
 from synapse.api.errors import ModuleFailedException, SynapseError
 from synapse.events import EventBase
-from synapse.events.snapshot import UnpersistedEventContextBase
+from synapse.events.snapshot import EventContext
 from synapse.storage.roommember import ProfileInfo
 from synapse.types import Requester, StateMap
 from synapse.util.async_helpers import delay_cancellation, maybe_awaitable
@@ -45,6 +45,8 @@ CHECK_CAN_DEACTIVATE_USER_CALLBACK = Callable[[str, bool], Awaitable[bool]]
 ON_PROFILE_UPDATE_CALLBACK = Callable[[str, ProfileInfo, bool, bool], Awaitable]
 ON_USER_DEACTIVATION_STATUS_CHANGED_CALLBACK = Callable[[str, bool, bool], Awaitable]
 ON_THREEPID_BIND_CALLBACK = Callable[[str, str, str], Awaitable]
+ON_ADD_USER_THIRD_PARTY_IDENTIFIER_CALLBACK = Callable[[str, str, str], Awaitable]
+ON_REMOVE_USER_THIRD_PARTY_IDENTIFIER_CALLBACK = Callable[[str, str, str], Awaitable]
 
 
 def load_legacy_third_party_event_rules(hs: "HomeServer") -> None:
@@ -78,6 +80,7 @@ def load_legacy_third_party_event_rules(hs: "HomeServer") -> None:
         # correctly, we need to await its result. Therefore it doesn't make a lot of
         # sense to make it go through the run() wrapper.
         if f.__name__ == "check_event_allowed":
+
             # We need to wrap check_event_allowed because its old form would return either
             # a boolean or a dict, but now we want to return the dict separately from the
             # boolean.
@@ -99,6 +102,7 @@ def load_legacy_third_party_event_rules(hs: "HomeServer") -> None:
             return wrap_check_event_allowed
 
         if f.__name__ == "on_create_room":
+
             # We need to wrap on_create_room because its old form would return a boolean
             # if the room creation is denied, but now we just want it to raise an
             # exception.
@@ -172,6 +176,12 @@ class ThirdPartyEventRules:
             ON_USER_DEACTIVATION_STATUS_CHANGED_CALLBACK
         ] = []
         self._on_threepid_bind_callbacks: List[ON_THREEPID_BIND_CALLBACK] = []
+        self._on_add_user_third_party_identifier_callbacks: List[
+            ON_ADD_USER_THIRD_PARTY_IDENTIFIER_CALLBACK
+        ] = []
+        self._on_remove_user_third_party_identifier_callbacks: List[
+            ON_REMOVE_USER_THIRD_PARTY_IDENTIFIER_CALLBACK
+        ] = []
 
     def register_third_party_rules_callbacks(
         self,
@@ -191,6 +201,12 @@ class ThirdPartyEventRules:
             ON_USER_DEACTIVATION_STATUS_CHANGED_CALLBACK
         ] = None,
         on_threepid_bind: Optional[ON_THREEPID_BIND_CALLBACK] = None,
+        on_add_user_third_party_identifier: Optional[
+            ON_ADD_USER_THIRD_PARTY_IDENTIFIER_CALLBACK
+        ] = None,
+        on_remove_user_third_party_identifier: Optional[
+            ON_REMOVE_USER_THIRD_PARTY_IDENTIFIER_CALLBACK
+        ] = None,
     ) -> None:
         """Register callbacks from modules for each hook."""
         if check_event_allowed is not None:
@@ -228,10 +244,13 @@ class ThirdPartyEventRules:
         if on_threepid_bind is not None:
             self._on_threepid_bind_callbacks.append(on_threepid_bind)
 
+        if on_add_user_third_party_identifier is not None:
+            self._on_add_user_third_party_identifier_callbacks.append(
+                on_add_user_third_party_identifier
+            )
+
     async def check_event_allowed(
-        self,
-        event: EventBase,
-        context: UnpersistedEventContextBase,
+        self, event: EventBase, context: EventContext
     ) -> Tuple[bool, Optional[dict]]:
         """Check if a provided event should be allowed in the given context.
 
@@ -511,6 +530,9 @@ class ThirdPartyEventRules:
         local homeserver, not when it's created on an identity server (and then kept track
         of so that it can be unbound on the same IS later on).
 
+        THIS MODULE CALLBACK METHOD HAS BEEN DEPRECATED. Please use the
+        `on_add_user_third_party_identifier` callback method instead.
+
         Args:
             user_id: the user being associated with the threepid.
             medium: the threepid's medium.
@@ -523,3 +545,43 @@ class ThirdPartyEventRules:
                 logger.exception(
                     "Failed to run module API callback %s: %s", callback, e
                 )
+
+    async def on_add_user_third_party_identifier(
+        self, user_id: str, medium: str, address: str
+    ) -> None:
+        """Called when an association between a user's Matrix ID and a third-party ID
+        (email, phone number) is about to be registered on the homeserver.
+
+        Args:
+            user_id: The User ID to include in the association.
+            medium: The medium of the third-party ID (email, msisdn).
+            address: The address of the third-party ID (i.e. an email address).
+        """
+        for callback in self._on_add_user_third_party_identifier_callbacks:
+            try:
+                await callback(user_id, medium, address)
+            except Exception as e:
+                logger.exception(
+                    "Failed to run module API callback %s: %s", callback, e
+                )
+
+    async def on_remove_user_third_party_identifier(
+        self, user_id: str, medium: str, address: str
+    ) -> None:
+        """Called when an association between a user's Matrix ID and a third-party ID
+        (email, phone number) is about to be removed on the homeserver. This is called
+        *after* any known bindings on identity servers for this association have been
+        removed.
+
+        Args:
+            user_id: The User ID including in the association to remove.
+            medium: The medium of the third-party ID (email, msisdn).
+            address: The address of the third-party ID (i.e. an email address).
+        """
+        for callback in self._on_remove_user_third_party_identifier_callbacks:
+            try:
+                await callback(user_id, medium, address)
+            except Exception as e:
+                logger.exception(
+                    "Failed to run module API callback %s: %s", callback, e
+                )

synapse/federation/federation_client.py

@@ -884,7 +884,7 @@ class FederationClient(FederationBase):
                 if 500 <= e.code < 600:
                     failover = True
 
-                elif 400 <= e.code < 500 and synapse_error.errcode in failover_errcodes:
+                elif e.code == 400 and synapse_error.errcode in failover_errcodes:
                     failover = True
 
                 elif failover_on_unknown_endpoint and self._is_unknown_endpoint(
@@ -999,13 +999,14 @@ class FederationClient(FederationBase):
 
             return destination, ev, room_version
 
-        failover_errcodes = {Codes.NOT_FOUND}
         # MSC3083 defines additional error codes for room joins. Unfortunately
         # we do not yet know the room version, assume these will only be returned
         # by valid room versions.
-        if membership == Membership.JOIN:
-            failover_errcodes.add(Codes.UNABLE_AUTHORISE_JOIN)
-            failover_errcodes.add(Codes.UNABLE_TO_GRANT_JOIN)
+        failover_errcodes = (
+            (Codes.UNABLE_AUTHORISE_JOIN, Codes.UNABLE_TO_GRANT_JOIN)
+            if membership == Membership.JOIN
+            else None
+        )
 
         return await self._try_destination_list(
             "make_" + membership,

synapse/federation/federation_server.py

@@ -23,7 +23,6 @@ from typing import (
     Collection,
     Dict,
     List,
-    Mapping,
     Optional,
     Tuple,
     Union,
@@ -48,7 +47,6 @@ from synapse.api.errors import (
     FederationError,
     IncompatibleRoomVersionError,
     NotFoundError,
-    PartialStateConflictError,
     SynapseError,
     UnsupportedRoomVersionError,
 )
@@ -82,6 +80,7 @@ from synapse.replication.http.federation import (
     ReplicationFederationSendEduRestServlet,
     ReplicationGetQueryRestServlet,
 )
+from synapse.storage.databases.main.events import PartialStateConflictError
 from synapse.storage.databases.main.lock import Lock
 from synapse.storage.databases.main.roommember import extract_heroes_from_room_summary
 from synapse.storage.roommember import MemberSummary
@@ -1513,7 +1512,7 @@ class FederationHandlerRegistry:
 def _get_event_ids_for_partial_state_join(
     join_event: EventBase,
     prev_state_ids: StateMap[str],
-    summary: Mapping[str, MemberSummary],
+    summary: Dict[str, MemberSummary],
 ) -> Collection[str]:
     """Calculate state to be returned in a partial_state send_join
 

synapse/federation/send_queue.py

@@ -314,7 +314,7 @@ class FederationRemoteSendQueue(AbstractFederationSender):
         # stream position.
         keyed_edus = {v: k for k, v in self.keyed_edu_changed.items()[i:j]}
 
-        for (destination, edu_key), pos in keyed_edus.items():
+        for ((destination, edu_key), pos) in keyed_edus.items():
             rows.append(
                 (
                     pos,
@@ -329,7 +329,7 @@ class FederationRemoteSendQueue(AbstractFederationSender):
         j = self.edus.bisect_right(to_token) + 1
         edus = self.edus.items()[i:j]
 
-        for pos, edu in edus:
+        for (pos, edu) in edus:
             rows.append((pos, EduRow(edu)))
 
         # Sort rows based on pos

synapse/handlers/account_data.py

@@ -343,12 +343,10 @@ class AccountDataEventSource(EventSource[int, JsonDict]):
                 }
             )
 
-        account_data = await self.store.get_updated_global_account_data_for_user(
-            user_id, last_stream_id
-        )
-        room_account_data = await self.store.get_updated_room_account_data_for_user(
-            user_id, last_stream_id
-        )
+        (
+            account_data,
+            room_account_data,
+        ) = await self.store.get_updated_account_data_for_user(user_id, last_stream_id)
 
         for account_data_type, content in account_data.items():
             results.append({"type": account_data_type, "content": content})

synapse/handlers/admin.py

@@ -14,7 +14,7 @@
 
 import abc
 import logging
-from typing import TYPE_CHECKING, Any, Dict, List, Mapping, Optional, Set
+from typing import TYPE_CHECKING, Any, Dict, List, Optional, Set
 
 from synapse.api.constants import Direction, Membership
 from synapse.events import EventBase
@@ -29,7 +29,7 @@ logger = logging.getLogger(__name__)
 
 class AdminHandler:
     def __init__(self, hs: "HomeServer"):
-        self._store = hs.get_datastores().main
+        self.store = hs.get_datastores().main
         self._device_handler = hs.get_device_handler()
         self._storage_controllers = hs.get_storage_controllers()
         self._state_storage_controller = self._storage_controllers.state
@@ -38,7 +38,7 @@ class AdminHandler:
     async def get_whois(self, user: UserID) -> JsonDict:
         connections = []
 
-        sessions = await self._store.get_user_ip_and_agents(user)
+        sessions = await self.store.get_user_ip_and_agents(user)
         for session in sessions:
             connections.append(
                 {
@@ -57,7 +57,7 @@ class AdminHandler:
 
     async def get_user(self, user: UserID) -> Optional[JsonDict]:
         """Function to get user details"""
-        user_info_dict = await self._store.get_user_by_id(user.to_string())
+        user_info_dict = await self.store.get_user_by_id(user.to_string())
         if user_info_dict is None:
             return None
 
@@ -89,11 +89,11 @@ class AdminHandler:
         }
 
         # Add additional user metadata
-        profile = await self._store.get_profileinfo(user.localpart)
-        threepids = await self._store.user_get_threepids(user.to_string())
+        profile = await self.store.get_profileinfo(user.localpart)
+        threepids = await self.store.user_get_threepids(user.to_string())
         external_ids = [
             ({"auth_provider": auth_provider, "external_id": external_id})
-            for auth_provider, external_id in await self._store.get_external_ids_by_user(
+            for auth_provider, external_id in await self.store.get_external_ids_by_user(
                 user.to_string()
             )
         ]
@@ -101,7 +101,7 @@ class AdminHandler:
         user_info_dict["avatar_url"] = profile.avatar_url
         user_info_dict["threepids"] = threepids
         user_info_dict["external_ids"] = external_ids
-        user_info_dict["erased"] = await self._store.is_user_erased(user.to_string())
+        user_info_dict["erased"] = await self.store.is_user_erased(user.to_string())
 
         return user_info_dict
 
@@ -117,7 +117,7 @@ class AdminHandler:
             The returned value is that returned by `writer.finished()`.
         """
         # Get all rooms the user is in or has been in
-        rooms = await self._store.get_rooms_for_local_user_where_membership_is(
+        rooms = await self.store.get_rooms_for_local_user_where_membership_is(
             user_id,
             membership_list=(
                 Membership.JOIN,
@@ -131,7 +131,7 @@ class AdminHandler:
         # We only try and fetch events for rooms the user has been in. If
         # they've been e.g. invited to a room without joining then we handle
         # those separately.
-        rooms_user_has_been_in = await self._store.get_rooms_user_has_been_in(user_id)
+        rooms_user_has_been_in = await self.store.get_rooms_user_has_been_in(user_id)
 
         for index, room in enumerate(rooms):
             room_id = room.room_id
@@ -140,7 +140,7 @@ class AdminHandler:
                 "[%s] Handling room %s, %d/%d", user_id, room_id, index + 1, len(rooms)
             )
 
-            forgotten = await self._store.did_forget(user_id, room_id)
+            forgotten = await self.store.did_forget(user_id, room_id)
             if forgotten:
                 logger.info("[%s] User forgot room %d, ignoring", user_id, room_id)
                 continue
@@ -152,14 +152,14 @@ class AdminHandler:
 
                 if room.membership == Membership.INVITE:
                     event_id = room.event_id
-                    invite = await self._store.get_event(event_id, allow_none=True)
+                    invite = await self.store.get_event(event_id, allow_none=True)
                     if invite:
                         invited_state = invite.unsigned["invite_room_state"]
                         writer.write_invite(room_id, invite, invited_state)
 
                 if room.membership == Membership.KNOCK:
                     event_id = room.event_id
-                    knock = await self._store.get_event(event_id, allow_none=True)
+                    knock = await self.store.get_event(event_id, allow_none=True)
                     if knock:
                         knock_state = knock.unsigned["knock_room_state"]
                         writer.write_knock(room_id, knock, knock_state)
@@ -170,7 +170,7 @@ class AdminHandler:
             # were joined. We estimate that point by looking at the
             # stream_ordering of the last membership if it wasn't a join.
             if room.membership == Membership.JOIN:
-                stream_ordering = self._store.get_room_max_stream_ordering()
+                stream_ordering = self.store.get_room_max_stream_ordering()
             else:
                 stream_ordering = room.stream_ordering
 
@@ -197,7 +197,7 @@ class AdminHandler:
             # events that we have and then filtering, this isn't the most
             # efficient method perhaps but it does guarantee we get everything.
             while True:
-                events, _ = await self._store.paginate_room_events(
+                events, _ = await self.store.paginate_room_events(
                     room_id, from_key, to_key, limit=100, direction=Direction.FORWARDS
                 )
                 if not events:
@@ -252,49 +252,16 @@ class AdminHandler:
         profile = await self.get_user(UserID.from_string(user_id))
         if profile is not None:
             writer.write_profile(profile)
-            logger.info("[%s] Written profile", user_id)
 
         # Get all devices the user has
         devices = await self._device_handler.get_devices_by_user(user_id)
         writer.write_devices(devices)
-        logger.info("[%s] Written %s devices", user_id, len(devices))
 
         # Get all connections the user has
         connections = await self.get_whois(UserID.from_string(user_id))
         writer.write_connections(
             connections["devices"][""]["sessions"][0]["connections"]
         )
-        logger.info("[%s] Written %s connections", user_id, len(connections))
-
-        # Get all account data the user has global and in rooms
-        global_data = await self._store.get_global_account_data_for_user(user_id)
-        by_room_data = await self._store.get_room_account_data_for_user(user_id)
-        writer.write_account_data("global", global_data)
-        for room_id in by_room_data:
-            writer.write_account_data(room_id, by_room_data[room_id])
-        logger.info(
-            "[%s] Written account data for %s rooms", user_id, len(by_room_data)
-        )
-
-        # Get all media ids the user has
-        limit = 100
-        start = 0
-        while True:
-            media_ids, total = await self._store.get_local_media_by_user_paginate(
-                start, limit, user_id
-            )
-            for media in media_ids:
-                writer.write_media_id(media["media_id"], media)
-
-            logger.info(
-                "[%s] Written %d media_ids of %s",
-                user_id,
-                (start + len(media_ids)),
-                total,
-            )
-            if (start + limit) >= total:
-                break
-            start += limit
 
         return writer.finished()
 
@@ -373,30 +340,6 @@ class ExfiltrationWriter(metaclass=abc.ABCMeta):
         """
         raise NotImplementedError()
 
-    @abc.abstractmethod
-    def write_account_data(
-        self, file_name: str, account_data: Mapping[str, JsonDict]
-    ) -> None:
-        """Write the account data of a user.
-
-        Args:
-            file_name: file name to write data
-            account_data: mapping of global or room account_data
-        """
-        raise NotImplementedError()
-
-    @abc.abstractmethod
-    def write_media_id(self, media_id: str, media_metadata: JsonDict) -> None:
-        """Write the media's metadata of a user.
-        Exports only the metadata, as this can be fetched from the database via
-        read only. In order to access the files, a connection to the correct
-        media repository would be required.
-
-        Args:
-            media_id: ID of the media.
-            media_metadata: Metadata of one media file.
-        """
-
     @abc.abstractmethod
     def finished(self) -> Any:
         """Called when all data has successfully been exported and written.

synapse/handlers/appservice.py

@@ -737,7 +737,7 @@ class ApplicationServicesHandler:
         )
 
         ret = []
-        for success, result in results:
+        for (success, result) in results:
             if success:
                 ret.extend(result)
 

synapse/handlers/auth.py

@@ -201,7 +201,7 @@ class AuthHandler:
         for auth_checker_class in INTERACTIVE_AUTH_CHECKERS:
             inst = auth_checker_class(hs)
             if inst.is_enabled():
-                self.checkers[inst.AUTH_TYPE] = inst
+                self.checkers[inst.AUTH_TYPE] = inst  # type: ignore
 
         self.bcrypt_rounds = hs.config.registration.bcrypt_rounds
 
@@ -815,6 +815,7 @@ class AuthHandler:
         now_ms = self._clock.time_msec()
 
         if existing_token.expiry_ts is not None and existing_token.expiry_ts < now_ms:
+
             raise SynapseError(
                 HTTPStatus.FORBIDDEN,
                 "The supplied refresh token has expired",
@@ -1542,6 +1543,17 @@ class AuthHandler:
     async def add_threepid(
         self, user_id: str, medium: str, address: str, validated_at: int
     ) -> None:
+        """
+        Adds an association between a user's Matrix ID and a third-party ID (email,
+        phone number).
+
+        Args:
+            user_id: The ID of the user to associate.
+            medium: The medium of the third-party ID (email, msisdn).
+            address: The address of the third-party ID (i.e. an email address).
+            validated_at: The timestamp in ms of when the validation that the user owns
+                this third-party ID occurred.
+        """
         # check if medium has a valid value
         if medium not in ["email", "msisdn"]:
             raise SynapseError(
@@ -1562,15 +1574,30 @@ class AuthHandler:
         if medium == "email":
             address = canonicalise_email(address)
 
-        await self.store.user_add_threepid(
-            user_id, medium, address, validated_at, self.hs.get_clock().time_msec()
+        # Inform Synapse modules that a 3PID association is about to be created.
+        await self._third_party_rules.on_add_user_third_party_identifier(
+            user_id, medium, address
         )
 
+        try:
+            await self.store.user_add_threepid(
+                user_id, medium, address, validated_at, self.hs.get_clock().time_msec()
+            )
+        except Exception:
+            # We failed to store the association, but told Synapse modules otherwise.
+            # Tell them that the association was deleted.
+            await self._third_party_rules.on_remove_user_third_party_identifier(
+                user_id, medium, address
+            )
+            raise
+
+        # Deprecated method for informing Synapse modules that a 3PID association
+        # has successfully been created.
         await self._third_party_rules.on_threepid_bind(user_id, medium, address)
 
-    async def delete_threepid(
-        self, user_id: str, medium: str, address: str, id_server: Optional[str] = None
-    ) -> bool:
+    async def delete_local_threepid(
+        self, user_id: str, medium: str, address: str
+    ) -> None:
         """Attempts to unbind the 3pid on the identity servers and deletes it
         from the local database.
 
@@ -1578,30 +1605,30 @@ class AuthHandler:
             user_id: ID of user to remove the 3pid from.
             medium: The medium of the 3pid being removed: "email" or "msisdn".
             address: The 3pid address to remove.
-            id_server: Use the given identity server when unbinding
-                any threepids. If None then will attempt to unbind using the
-                identity server specified when binding (if known).
-
-        Returns:
-            Returns True if successfully unbound the 3pid on
-            the identity server, False if identity server doesn't support the
-            unbind API.
         """
-
         # 'Canonicalise' email addresses as per above
         if medium == "email":
             address = canonicalise_email(address)
 
-        result = await self.hs.get_identity_handler().try_unbind_threepid(
-            user_id, medium, address, id_server
+        # Inform Synapse modules that a 3PID association is about to be deleted.
+        await self._third_party_rules.on_remove_user_third_party_identifier(
+            user_id, medium, address
         )
 
-        await self.store.user_delete_threepid(user_id, medium, address)
+        try:
+            await self.store.user_delete_threepid(user_id, medium, address)
+        except Exception:
+            # We failed to store the association, but told Synapse modules otherwise.
+            # Tell them that the association has come back.
+            await self._third_party_rules.on_add_user_third_party_identifier(
+                user_id, medium, address
+            )
+            raise
+
         if medium == "email":
             await self.store.delete_pusher_by_app_id_pushkey_user_id(
                 app_id="m.email", pushkey=address, user_id=user_id
             )
-        return result
 
     async def hash(self, password: str) -> str:
         """Computes a secure hash of password.
@@ -2258,6 +2285,7 @@ class PasswordAuthProvider:
     async def on_logged_out(
         self, user_id: str, device_id: Optional[str], access_token: str
     ) -> None:
+
         # call all of the on_logged_out callbacks
         for callback in self.on_logged_out_callbacks:
             try: