Lint

Support running tests in CI.
Back out unneeded changes.
2023-11-09 15:01:30 -05:00 · 2023-11-09 14:58:51 -05:00 · 2023-11-09 14:46:48 -05:00 · 2023-11-09 14:45:50 -05:00 · 2023-11-09 14:40:45 -05:00 · 2023-11-09 14:40:30 -05:00
199 changed files with 5549 additions and 2435 deletions
@@ -50,23 +50,38 @@ if not IS_PR:
        for version in ("3.9", "3.10", "3.11", "3.12")
    )

+# Run with both psycopg2 and psycopg.
 trial_postgres_tests = [
    {
        "python-version": "3.8",
        "database": "postgres",
        "postgres-version": "11",
        "extras": "all",
-    }
+    },
+    {
+        "python-version": "3.8",
+        "database": "psycopg",
+        "postgres-version": "11",
+        "extras": "all",
+    },
 ]

 if not IS_PR:
-    trial_postgres_tests.append(
-        {
-            "python-version": "3.12",
-            "database": "postgres",
-            "postgres-version": "16",
-            "extras": "all",
-        }
+    trial_postgres_tests.extend(
+        [
+            {
+                "python-version": "3.12",
+                "database": "postgres",
+                "postgres-version": "16",
+                "extras": "all",
+            },
+            {
+                "python-version": "3.12",
+                "database": "psycopg",
+                "postgres-version": "16",
+                "extras": "all",
+            },
+        ]
    )

 trial_no_extra_tests = [
@@ -197,11 +197,14 @@ jobs:
        with:
          path: synapse

-      - uses: actions/setup-go@v4
-
      - name: Prepare Complement's Prerequisites
        run: synapse/.ci/scripts/setup_complement_prerequisites.sh

+      - uses: actions/setup-go@v4
+        with:
+          cache-dependency-path: complement/go.sum
+          go-version-file: complement/go.mod
+
      - run: |
          set -o pipefail
          TEST_ONLY_IGNORE_POETRY_LOCKFILE=1 POSTGRES=${{ (matrix.database == 'Postgres') && 1 || '' }} WORKERS=${{ (matrix.arrangement == 'workers') && 1 || '' }} COMPLEMENT_DIR=`pwd`/complement synapse/scripts-dev/complement.sh -json 2>&1 | synapse/.ci/scripts/gotestfmt
@@ -346,7 +346,9 @@ jobs:
        run: until pg_isready -h localhost; do sleep 1; done
      - run: poetry run trial --jobs=6 tests
        env:
-          SYNAPSE_POSTGRES: ${{ matrix.job.database == 'postgres' || '' }}
+          # If matrix.job.database is 'psycopg' set SYNAPSE_POSTGRES to that string;
+          # otherwise if it is 'postgres' set it to true. Otherwise, empty.
+          SYNAPSE_POSTGRES: ${{ matrix.job.database == 'psycopg' && 'psycopg' || matrix.job.database == 'postgres' || '' }}
          SYNAPSE_POSTGRES_HOST: /var/run/postgresql
          SYNAPSE_POSTGRES_USER: postgres
          SYNAPSE_POSTGRES_PASSWORD: postgres
@@ -633,14 +635,18 @@ jobs:
        uses: dtolnay/rust-toolchain@1.61.0
      - uses: Swatinem/rust-cache@v2

-      - uses: actions/setup-go@v4
-
      - name: Prepare Complement's Prerequisites
        run: synapse/.ci/scripts/setup_complement_prerequisites.sh

+      - uses: actions/setup-go@v4
+        with:
+          cache-dependency-path: complement/go.sum
+          go-version-file: complement/go.mod
+
+        # use p=1 concurrency as GHA boxes are underpowered and don't like running tons of synapses at once.
      - run: |
          set -o pipefail
-          COMPLEMENT_DIR=`pwd`/complement synapse/scripts-dev/complement.sh -json 2>&1 | synapse/.ci/scripts/gotestfmt
+          COMPLEMENT_DIR=`pwd`/complement synapse/scripts-dev/complement.sh -p 1 -json 2>&1 | synapse/.ci/scripts/gotestfmt
        shell: bash
        env:
          POSTGRES: ${{ (matrix.database == 'Postgres') && 1 || '' }}
@@ -168,11 +168,14 @@ jobs:
        with:
          path: synapse

-      - uses: actions/setup-go@v4
-
      - name: Prepare Complement's Prerequisites
        run: synapse/.ci/scripts/setup_complement_prerequisites.sh

+      - uses: actions/setup-go@v4
+        with:
+          cache-dependency-path: complement/go.sum
+          go-version-file: complement/go.mod
+
      # This step is specific to the 'Twisted trunk' test run:
      - name: Patch dependencies
        run: |
@@ -1,3 +1,78 @@
+# Synapse 1.96.0rc1 (2023-10-31)
+
+### Features
+
+- Add experimental support to allow multiple workers to write to receipts stream. ([\#16432](https://github.com/matrix-org/synapse/issues/16432))
+- Add a new module API for controller presence. ([\#16544](https://github.com/matrix-org/synapse/issues/16544))
+- Add a new module API callback that allows adding extra fields to events' unsigned section when sent down to clients. ([\#16549](https://github.com/matrix-org/synapse/issues/16549))
+- Improve the performance of claiming encryption keys. ([\#16565](https://github.com/matrix-org/synapse/issues/16565), [\#16570](https://github.com/matrix-org/synapse/issues/16570))
+
+### Bugfixes
+
+- Fixed a bug in the example Grafana dashboard that prevents it from finding the correct datasource. Contributed by @MichaelSasser. ([\#16471](https://github.com/matrix-org/synapse/issues/16471))
+- Fix a long-standing, exceedingly rare edge case where the first event persisted by a new event persister worker might not be sent down `/sync`. ([\#16473](https://github.com/matrix-org/synapse/issues/16473), [\#16557](https://github.com/matrix-org/synapse/issues/16557), [\#16561](https://github.com/matrix-org/synapse/issues/16561), [\#16578](https://github.com/matrix-org/synapse/issues/16578), [\#16580](https://github.com/matrix-org/synapse/issues/16580))
+- Fix long-standing bug where `/sync` incorrectly did not mark a room as `limited` in a sync requests when there were missing remote events. ([\#16485](https://github.com/matrix-org/synapse/issues/16485))
+- Fix a bug introduced in Synapse 1.41 where HTTP(S) forward proxy authorization would fail when using basic HTTP authentication with a long `username:password` string. ([\#16504](https://github.com/matrix-org/synapse/issues/16504))
+- Force TLS certificate verification in user registration script. ([\#16530](https://github.com/matrix-org/synapse/issues/16530))
+- Fix long-standing bug where `/sync` could tightloop after restart when using SQLite. ([\#16540](https://github.com/matrix-org/synapse/issues/16540))
+- Fix ratelimiting of message sending when using workers, where the ratelimit would only be applied after most of the work has been done. ([\#16558](https://github.com/matrix-org/synapse/issues/16558))
+- Fix a long-standing bug where invited/knocking users would not leave during a room purge. ([\#16559](https://github.com/matrix-org/synapse/issues/16559))
+
+### Improved Documentation
+
+- Improve documentation of presence router. ([\#16529](https://github.com/matrix-org/synapse/issues/16529))
+- Add a sentence to the [opentracing docs](https://matrix-org.github.io/synapse/latest/opentracing.html) on how you can have jaeger in a different place than synapse. ([\#16531](https://github.com/matrix-org/synapse/issues/16531))
+- Correctly describe the meaning of unspecified rule lists in the [`alias_creation_rules`](https://matrix-org.github.io/synapse/latest/usage/configuration/config_documentation.html#alias_creation_rules) and [`room_list_publication_rules`](https://matrix-org.github.io/synapse/latest/usage/configuration/config_documentation.html#room_list_publication_rules) config options and improve their descriptions more generally. ([\#16541](https://github.com/matrix-org/synapse/issues/16541))
+- Pin the recommended poetry version in [contributors' guide](https://matrix-org.github.io/synapse/latest/development/contributing_guide.html). ([\#16550](https://github.com/matrix-org/synapse/issues/16550))
+- Fix a broken link to the [client breakdown](https://matrix.org/ecosystem/clients/) in the README. ([\#16569](https://github.com/matrix-org/synapse/issues/16569))
+
+### Internal Changes
+
+- Improve performance of delete device messages query, cf issue [16479](https://github.com/matrix-org/synapse/issues/16479). ([\#16492](https://github.com/matrix-org/synapse/issues/16492))
+- Reduce memory allocations. ([\#16505](https://github.com/matrix-org/synapse/issues/16505))
+- Improve replication performance when purging rooms. ([\#16510](https://github.com/matrix-org/synapse/issues/16510))
+- Run tests against Python 3.12. ([\#16511](https://github.com/matrix-org/synapse/issues/16511))
+- Run trial & integration tests in continuous integration when `.ci` directory is modified. ([\#16512](https://github.com/matrix-org/synapse/issues/16512))
+- Remove duplicate call to mark remote server 'awake' when using a federation sending worker. ([\#16515](https://github.com/matrix-org/synapse/issues/16515))
+- Enable dirty runs on Complement CI, which is significantly faster. ([\#16520](https://github.com/matrix-org/synapse/issues/16520))
+- Stop deleting from an unused table. ([\#16521](https://github.com/matrix-org/synapse/issues/16521))
+- Improve type hints. ([\#16526](https://github.com/matrix-org/synapse/issues/16526), [\#16551](https://github.com/matrix-org/synapse/issues/16551))
+- Fix running unit tests on Twisted trunk. ([\#16528](https://github.com/matrix-org/synapse/issues/16528))
+- Reduce some spurious logging in worker mode. ([\#16555](https://github.com/matrix-org/synapse/issues/16555))
+- Stop porting a table in port db that we're going to nuke and rebuild anyway. ([\#16563](https://github.com/matrix-org/synapse/issues/16563))
+- Deal with warnings from running complement in CI. ([\#16567](https://github.com/matrix-org/synapse/issues/16567))
+- Allow building with `setuptools_rust` 1.8.0. ([\#16574](https://github.com/matrix-org/synapse/issues/16574))
+
+### Updates to locked dependencies
+
+* Bump black from 23.10.0 to 23.10.1. ([\#16575](https://github.com/matrix-org/synapse/issues/16575))
+* Bump black from 23.9.1 to 23.10.0. ([\#16538](https://github.com/matrix-org/synapse/issues/16538))
+* Bump cryptography from 41.0.4 to 41.0.5. ([\#16572](https://github.com/matrix-org/synapse/issues/16572))
+* Bump gitpython from 3.1.37 to 3.1.40. ([\#16534](https://github.com/matrix-org/synapse/issues/16534))
+* Bump phonenumbers from 8.13.22 to 8.13.23. ([\#16576](https://github.com/matrix-org/synapse/issues/16576))
+* Bump pygithub from 1.59.1 to 2.1.1. ([\#16535](https://github.com/matrix-org/synapse/issues/16535))
+- Bump matrix-synapse-ldap3 from 0.2.2 to 0.3.0. ([\#16539](https://github.com/matrix-org/synapse/issues/16539))
+* Bump serde from 1.0.189 to 1.0.190. ([\#16577](https://github.com/matrix-org/synapse/issues/16577))
+* Bump setuptools-rust from 1.7.0 to 1.8.0. ([\#16574](https://github.com/matrix-org/synapse/issues/16574))
+* Bump types-pillow from 10.0.0.3 to 10.1.0.0. ([\#16536](https://github.com/matrix-org/synapse/issues/16536))
+* Bump types-psycopg2 from 2.9.21.14 to 2.9.21.15. ([\#16573](https://github.com/matrix-org/synapse/issues/16573))
+* Bump types-requests from 2.31.0.2 to 2.31.0.10. ([\#16537](https://github.com/matrix-org/synapse/issues/16537))
+* Bump urllib3 from 1.26.17 to 1.26.18. ([\#16516](https://github.com/matrix-org/synapse/issues/16516))
+
+# Synapse 1.95.1 (2023-10-31)
+
+## Security advisory
+
+The following issue is fixed in 1.95.1.
+
+- [GHSA-mp92-3jfm-3575](https://github.com/matrix-org/synapse/security/advisories/GHSA-mp92-3jfm-3575) / [CVE-2023-43796](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2023-43796) — Moderate Severity
+
+  Cached device information of remote users can be queried from Synapse. This can be used to enumerate the remote users known to a homeserver.
+
+See the advisory for more details. If you have any questions, email security@matrix.org.
+
+
+
 # Synapse 1.95.0 (2023-10-24)

 ### Internal Changes
@@ -332,18 +332,18 @@ checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd"

 [[package]]
 name = "serde"
-version = "1.0.189"
+version = "1.0.190"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8e422a44e74ad4001bdc8eede9a4570ab52f71190e9c076d14369f38b9200537"
+checksum = "91d3c334ca1ee894a2c6f6ad698fe8c435b76d504b13d436f0685d648d6d96f7"
 dependencies = [
 "serde_derive",
 ]

 [[package]]
 name = "serde_derive"
-version = "1.0.189"
+version = "1.0.190"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1e48d1f918009ce3145511378cf68d613e3b3d9137d67272562080d68a2b32d5"
+checksum = "67c5609f394e5c2bd7fc51efda478004ea80ef42fee983d5c67a65e34f32c0e3"
 dependencies = [
 "proc-macro2",
 "quote",
@@ -352,9 +352,9 @@ dependencies = [

 [[package]]
 name = "serde_json"
-version = "1.0.107"
+version = "1.0.108"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6b420ce6e3d8bd882e9b243c6eed35dbc9a6110c9769e74b584e0d68d1f20c65"
+checksum = "3d1c7e3eac408d115102c4c24ad393e0821bb3a5df4d506a80f85f7a742a526b"
 dependencies = [
 "itoa",
 "ryu",
@@ -122,7 +122,7 @@ You will need to change the server you are logging into from ``matrix.org``
 and instead specify a Homeserver URL of ``https://<server_name>:8448``
 (or just ``https://<server_name>`` if you are using a reverse proxy).
 If you prefer to use another client, refer to our
-`client breakdown <https://matrix.org/docs/projects/clients-matrix>`_.
+`client breakdown <https://matrix.org/ecosystem/clients/>`_.

 If all goes well you should at least be able to log in, create a room, and
 start sending messages.
@@ -1 +0,0 @@
-Fixed a bug that prevents Grafana from finding the correct datasource. Contributed by @MichaelSasser.
@@ -1 +0,0 @@
-Fix a long-standing, exceedingly rare edge case where the first event persisted by a new event persister worker might not be sent down `/sync`.
@@ -1 +0,0 @@
-Fix long-standing bug where `/sync` incorrectly did not mark a room as `limited` in a sync requests when there were missing remote events.
@@ -1 +0,0 @@
-Improve performance of delete device messages query, cf issue [16479](https://github.com/matrix-org/synapse/issues/16479).
@@ -1 +0,0 @@
-Improve replication performance when purging rooms.
@@ -1 +0,0 @@
-Run tests against Python 3.12.
@@ -1 +0,0 @@
-Run trial & integration tests in continuous integration when `.ci` directory is modified.
@@ -1 +0,0 @@
-Remove duplicate call to mark remote server 'awake' when using a federation sending worker.
@@ -1 +0,0 @@
-Stop deleting from an unused table.
@@ -1 +0,0 @@
-Improve documentation of presence router.
@@ -1 +0,0 @@
-Force TLS certificate verification in user registration script.
@@ -1 +0,0 @@
-Add a sentence to the opentracing docs on how you can have jaeger in a different place than synapse.
@@ -0,0 +1 @@
+Support reactor tick timings on more types of event loops.
@@ -1 +0,0 @@
-Bump matrix-synapse-ldap3 from 0.2.2 to 0.3.0.
@@ -1 +0,0 @@
-Fix long-standing bug where `/sync` could tightloop after restart when using SQLite.
@@ -1 +0,0 @@
-Correctly describe the meaning of unspecified rule lists in the [`alias_creation_rules`](https://matrix-org.github.io/synapse/latest/usage/configuration/config_documentation.html#alias_creation_rules) and [`room_list_publication_rules`](https://matrix-org.github.io/synapse/latest/usage/configuration/config_documentation.html#room_list_publication_rules) config options and improve their descriptions more generally.
@@ -0,0 +1 @@
+Avoid executing no-op queries.
@@ -0,0 +1 @@
+Simplify persistance code to be per-room.
@@ -0,0 +1 @@
+Use standard SQL helpers in persistence code.
@@ -0,0 +1 @@
+Avoid updating the stream cache unnecessarily.
@@ -0,0 +1 @@
+Bump twisted from 23.8.0 to 23.10.0.
@@ -0,0 +1 @@
+Improve performance when using opentracing.
@@ -0,0 +1 @@
+Run push rule evaluator setup in parallel.
@@ -0,0 +1 @@
+Improve tests of the SQL generator.
@@ -0,0 +1 @@
+Bump setuptools-rust from 1.8.0 to 1.8.1.
@@ -0,0 +1 @@
+Fix a long-standing bug where some queries updated the same row twice. Introduced in Synapse 1.57.0.
@@ -0,0 +1 @@
+Improve type hints.
@@ -0,0 +1 @@
+Improve type hints.
@@ -0,0 +1 @@
+Improve the performance of some operations in multi-worker deployments.
@@ -0,0 +1 @@
+Improve the performance of some operations in multi-worker deployments.
@@ -0,0 +1 @@
+Use `dbname` instead of the deprecated `database` connection parameter for psycopg2.
@@ -1,3 +1,15 @@
+matrix-synapse-py3 (1.96.0~rc1) stable; urgency=medium
+
+  * New Synapse release 1.96.0rc1.
+
+ -- Synapse Packaging team <packages@matrix.org>  Tue, 31 Oct 2023 14:09:09 +0000
+
+matrix-synapse-py3 (1.95.1) stable; urgency=medium
+
+  * New Synapse release 1.95.1.
+
+ -- Synapse Packaging team <packages@matrix.org>  Tue, 31 Oct 2023 14:00:00 +0000
+
 matrix-synapse-py3 (1.95.0) stable; urgency=medium

  * New Synapse release 1.95.0.
@@ -68,6 +68,11 @@ if [[ -n "$SYNAPSE_COMPLEMENT_USE_WORKERS" ]]; then

  fi
  log "Workers requested: $SYNAPSE_WORKER_TYPES"
+  # adjust connection pool limits on worker mode as otherwise running lots of worker synapses
+  # can make docker unhappy (in GHA)
+  export POSTGRES_CP_MIN=1
+  export POSTGRES_CP_MAX=3
+  echo "using reduced connection pool limits for worker mode"
  # Improve startup times by using a launcher based on fork()
  export SYNAPSE_USE_EXPERIMENTAL_FORKING_LAUNCHER=1
 else
@@ -67,8 +67,8 @@ database:
    host: "{{ POSTGRES_HOST or "db" }}"
    port: "{{ POSTGRES_PORT or "5432" }}"
 {% endif %}
-    cp_min: 5
-    cp_max: 10
+    cp_min: {{ POSTGRES_CP_MIN or 5 }}
+    cp_max: {{ POSTGRES_CP_MAX or 10 }}
 {% else %}
 database:
  name: "sqlite3"
@@ -19,7 +19,7 @@
 # Usage
  - [Federation](federate.md)
  - [Configuration](usage/configuration/README.md)
-    - [Configuration Manual](usage/configuration/config_documentation.md) 
+    - [Configuration Manual](usage/configuration/config_documentation.md)
    - [Homeserver Sample Config File](usage/configuration/homeserver_sample_config.md)
    - [Logging Sample Config File](usage/configuration/logging_sample_config.md)
    - [Structured Logging](structured_logging.md)
@@ -48,6 +48,7 @@
        - [Password auth provider callbacks](modules/password_auth_provider_callbacks.md)
        - [Background update controller callbacks](modules/background_update_controller_callbacks.md)
        - [Account data callbacks](modules/account_data_callbacks.md)
+        - [Add extra fields to client events unsigned section callbacks](modules/add_extra_fields_to_client_events_unsigned.md)
        - [Porting a legacy module to the new interface](modules/porting_legacy_module.md)
    - [Workers](workers.md)
      - [Using `synctl` with Workers](synctl_workers.md)
@@ -66,7 +66,7 @@ Of their installation methods, we recommend

 ```shell
 pip install --user pipx
-pipx install poetry
+pipx install poetry==1.5.2  # Problems with Poetry 1.6, see https://github.com/matrix-org/synapse/issues/16147
 ```

 but see poetry's [installation instructions](https://python-poetry.org/docs/#installation)
@@ -0,0 +1,32 @@
+# Add extra fields to client events unsigned section callbacks
+
+_First introduced in Synapse v1.96.0_
+
+This callback allows modules to add extra fields to the unsigned section of
+events when they get sent down to clients.
+
+These get called *every* time an event is to be sent to clients, so care should
+be taken to ensure with respect to performance.
+
+### API
+
+To register the callback, use
+`register_add_extra_fields_to_unsigned_client_event_callbacks` on the
+`ModuleApi`.
+
+The callback should be of the form
+
+```python
+async def add_field_to_unsigned(
+    event: EventBase,
+) -> JsonDict:
+```
+
+where the extra fields to add to the event's unsigned section is returned.
+(Modules must not attempt to modify the `event` directly).
+
+This cannot be used to alter the "core" fields in the unsigned section emitted
+by Synapse itself.
+
+If multiple such callbacks try to add the same field to an event's unsigned
+section, the last-registered callback wins.
@@ -66,7 +66,7 @@ database:
  args:
    user: <user>
    password: <pass>
-    database: <db>
+    dbname: <db>
    host: <host>
    cp_min: 5
    cp_max: 10
@@ -230,6 +230,13 @@ Example configuration:
 presence:
  enabled: false
 ```
+
+`enabled` can also be set to a special value of "untracked" which ignores updates
+received via clients and federation, while still accepting updates from the
+[module API](../../modules/index.md).
+
+*The "untracked" option was added in Synapse 1.96.0.*
+
 ---
 ### `require_auth_for_profile_requests`

@@ -1440,7 +1447,7 @@ database:
  args:
    user: synapse_user
    password: secretpassword
-    database: synapse
+    dbname: synapse
    host: localhost
    port: 5432
    cp_min: 5
@@ -1519,7 +1526,7 @@ databases:
    args:
      user: synapse_user
      password: secretpassword
-      database: synapse_main
+      dbname: synapse_main
      host: localhost
      port: 5432
      cp_min: 5
@@ -1532,7 +1539,7 @@ databases:
    args:
      user: synapse_user
      password: secretpassword
-      database: synapse_state
+      dbname: synapse_state
      host: localhost
      port: 5432
      cp_min: 5
@@ -37,8 +37,8 @@ files =
  build_rust.py

 [mypy-synapse.metrics._reactor_metrics]
-# This module imports select.epoll. That exists on Linux, but doesn't on macOS.
-# See https://github.com/matrix-org/synapse/pull/11771.
+# This module  pokes at the internals of OS-specific classes, to appease mypy
+# on different systems we add additional ignores.
 warn_unused_ignores = False

 [mypy-synapse.util.caches.treecache]
@@ -1,4 +1,4 @@
-# This file is automatically @generated by Poetry 1.5.1 and should not be changed by hand.
+# This file is automatically @generated by Poetry 1.7.0 and should not be changed by hand.

 [[package]]
 name = "alabaster"
@@ -108,6 +108,34 @@ files = [
 [package.dependencies]
 pytz = {version = ">=2015.7", markers = "python_version < \"3.9\""}

+[[package]]
+name = "backports-zoneinfo"
+version = "0.2.1"
+description = "Backport of the standard library zoneinfo module"
+optional = true
+python-versions = ">=3.6"
+files = [
+    {file = "backports.zoneinfo-0.2.1-cp36-cp36m-macosx_10_14_x86_64.whl", hash = "sha256:da6013fd84a690242c310d77ddb8441a559e9cb3d3d59ebac9aca1a57b2e18bc"},
+    {file = "backports.zoneinfo-0.2.1-cp36-cp36m-manylinux1_i686.whl", hash = "sha256:89a48c0d158a3cc3f654da4c2de1ceba85263fafb861b98b59040a5086259722"},
+    {file = "backports.zoneinfo-0.2.1-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:1c5742112073a563c81f786e77514969acb58649bcdf6cdf0b4ed31a348d4546"},
+    {file = "backports.zoneinfo-0.2.1-cp36-cp36m-win32.whl", hash = "sha256:e8236383a20872c0cdf5a62b554b27538db7fa1bbec52429d8d106effbaeca08"},
+    {file = "backports.zoneinfo-0.2.1-cp36-cp36m-win_amd64.whl", hash = "sha256:8439c030a11780786a2002261569bdf362264f605dfa4d65090b64b05c9f79a7"},
+    {file = "backports.zoneinfo-0.2.1-cp37-cp37m-macosx_10_14_x86_64.whl", hash = "sha256:f04e857b59d9d1ccc39ce2da1021d196e47234873820cbeaad210724b1ee28ac"},
+    {file = "backports.zoneinfo-0.2.1-cp37-cp37m-manylinux1_i686.whl", hash = "sha256:17746bd546106fa389c51dbea67c8b7c8f0d14b5526a579ca6ccf5ed72c526cf"},
+    {file = "backports.zoneinfo-0.2.1-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:5c144945a7752ca544b4b78c8c41544cdfaf9786f25fe5ffb10e838e19a27570"},
+    {file = "backports.zoneinfo-0.2.1-cp37-cp37m-win32.whl", hash = "sha256:e55b384612d93be96506932a786bbcde5a2db7a9e6a4bb4bffe8b733f5b9036b"},
+    {file = "backports.zoneinfo-0.2.1-cp37-cp37m-win_amd64.whl", hash = "sha256:a76b38c52400b762e48131494ba26be363491ac4f9a04c1b7e92483d169f6582"},
+    {file = "backports.zoneinfo-0.2.1-cp38-cp38-macosx_10_14_x86_64.whl", hash = "sha256:8961c0f32cd0336fb8e8ead11a1f8cd99ec07145ec2931122faaac1c8f7fd987"},
+    {file = "backports.zoneinfo-0.2.1-cp38-cp38-manylinux1_i686.whl", hash = "sha256:e81b76cace8eda1fca50e345242ba977f9be6ae3945af8d46326d776b4cf78d1"},
+    {file = "backports.zoneinfo-0.2.1-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:7b0a64cda4145548fed9efc10322770f929b944ce5cee6c0dfe0c87bf4c0c8c9"},
+    {file = "backports.zoneinfo-0.2.1-cp38-cp38-win32.whl", hash = "sha256:1b13e654a55cd45672cb54ed12148cd33628f672548f373963b0bff67b217328"},
+    {file = "backports.zoneinfo-0.2.1-cp38-cp38-win_amd64.whl", hash = "sha256:4a0f800587060bf8880f954dbef70de6c11bbe59c673c3d818921f042f9954a6"},
+    {file = "backports.zoneinfo-0.2.1.tar.gz", hash = "sha256:fadbfe37f74051d024037f223b8e001611eac868b5c5b06144ef4d8b799862f2"},
+]
+
+[package.extras]
+tzdata = ["tzdata"]
+
 [[package]]
 name = "bcrypt"
 version = "4.0.1"
@@ -162,29 +190,29 @@ lxml = ["lxml"]

 [[package]]
 name = "black"
-version = "23.10.0"
+version = "23.10.1"
 description = "The uncompromising code formatter."
 optional = false
 python-versions = ">=3.8"
 files = [
-    {file = "black-23.10.0-cp310-cp310-macosx_10_16_arm64.whl", hash = "sha256:f8dc7d50d94063cdfd13c82368afd8588bac4ce360e4224ac399e769d6704e98"},
-    {file = "black-23.10.0-cp310-cp310-macosx_10_16_x86_64.whl", hash = "sha256:f20ff03f3fdd2fd4460b4f631663813e57dc277e37fb216463f3b907aa5a9bdd"},
-    {file = "black-23.10.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d3d9129ce05b0829730323bdcb00f928a448a124af5acf90aa94d9aba6969604"},
-    {file = "black-23.10.0-cp310-cp310-win_amd64.whl", hash = "sha256:960c21555be135c4b37b7018d63d6248bdae8514e5c55b71e994ad37407f45b8"},
-    {file = "black-23.10.0-cp311-cp311-macosx_10_16_arm64.whl", hash = "sha256:30b78ac9b54cf87bcb9910ee3d499d2bc893afd52495066c49d9ee6b21eee06e"},
-    {file = "black-23.10.0-cp311-cp311-macosx_10_16_x86_64.whl", hash = "sha256:0e232f24a337fed7a82c1185ae46c56c4a6167fb0fe37411b43e876892c76699"},
-    {file = "black-23.10.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:31946ec6f9c54ed7ba431c38bc81d758970dd734b96b8e8c2b17a367d7908171"},
-    {file = "black-23.10.0-cp311-cp311-win_amd64.whl", hash = "sha256:c870bee76ad5f7a5ea7bd01dc646028d05568d33b0b09b7ecfc8ec0da3f3f39c"},
-    {file = "black-23.10.0-cp38-cp38-macosx_10_16_arm64.whl", hash = "sha256:6901631b937acbee93c75537e74f69463adaf34379a04eef32425b88aca88a23"},
-    {file = "black-23.10.0-cp38-cp38-macosx_10_16_x86_64.whl", hash = "sha256:481167c60cd3e6b1cb8ef2aac0f76165843a374346aeeaa9d86765fe0dd0318b"},
-    {file = "black-23.10.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f74892b4b836e5162aa0452393112a574dac85e13902c57dfbaaf388e4eda37c"},
-    {file = "black-23.10.0-cp38-cp38-win_amd64.whl", hash = "sha256:47c4510f70ec2e8f9135ba490811c071419c115e46f143e4dce2ac45afdcf4c9"},
-    {file = "black-23.10.0-cp39-cp39-macosx_10_16_arm64.whl", hash = "sha256:76baba9281e5e5b230c9b7f83a96daf67a95e919c2dfc240d9e6295eab7b9204"},
-    {file = "black-23.10.0-cp39-cp39-macosx_10_16_x86_64.whl", hash = "sha256:a3c2ddb35f71976a4cfeca558848c2f2f89abc86b06e8dd89b5a65c1e6c0f22a"},
-    {file = "black-23.10.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:db451a3363b1e765c172c3fd86213a4ce63fb8524c938ebd82919bf2a6e28c6a"},
-    {file = "black-23.10.0-cp39-cp39-win_amd64.whl", hash = "sha256:7fb5fc36bb65160df21498d5a3dd330af8b6401be3f25af60c6ebfe23753f747"},
-    {file = "black-23.10.0-py3-none-any.whl", hash = "sha256:e223b731a0e025f8ef427dd79d8cd69c167da807f5710add30cdf131f13dd62e"},
-    {file = "black-23.10.0.tar.gz", hash = "sha256:31b9f87b277a68d0e99d2905edae08807c007973eaa609da5f0c62def6b7c0bd"},
+    {file = "black-23.10.1-cp310-cp310-macosx_10_16_arm64.whl", hash = "sha256:ec3f8e6234c4e46ff9e16d9ae96f4ef69fa328bb4ad08198c8cee45bb1f08c69"},
+    {file = "black-23.10.1-cp310-cp310-macosx_10_16_x86_64.whl", hash = "sha256:1b917a2aa020ca600483a7b340c165970b26e9029067f019e3755b56e8dd5916"},
+    {file = "black-23.10.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9c74de4c77b849e6359c6f01987e94873c707098322b91490d24296f66d067dc"},
+    {file = "black-23.10.1-cp310-cp310-win_amd64.whl", hash = "sha256:7b4d10b0f016616a0d93d24a448100adf1699712fb7a4efd0e2c32bbb219b173"},
+    {file = "black-23.10.1-cp311-cp311-macosx_10_16_arm64.whl", hash = "sha256:b15b75fc53a2fbcac8a87d3e20f69874d161beef13954747e053bca7a1ce53a0"},
+    {file = "black-23.10.1-cp311-cp311-macosx_10_16_x86_64.whl", hash = "sha256:e293e4c2f4a992b980032bbd62df07c1bcff82d6964d6c9496f2cd726e246ace"},
+    {file = "black-23.10.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7d56124b7a61d092cb52cce34182a5280e160e6aff3137172a68c2c2c4b76bcb"},
+    {file = "black-23.10.1-cp311-cp311-win_amd64.whl", hash = "sha256:3f157a8945a7b2d424da3335f7ace89c14a3b0625e6593d21139c2d8214d55ce"},
+    {file = "black-23.10.1-cp38-cp38-macosx_10_16_arm64.whl", hash = "sha256:cfcce6f0a384d0da692119f2d72d79ed07c7159879d0bb1bb32d2e443382bf3a"},
+    {file = "black-23.10.1-cp38-cp38-macosx_10_16_x86_64.whl", hash = "sha256:33d40f5b06be80c1bbce17b173cda17994fbad096ce60eb22054da021bf933d1"},
+    {file = "black-23.10.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:840015166dbdfbc47992871325799fd2dc0dcf9395e401ada6d88fe11498abad"},
+    {file = "black-23.10.1-cp38-cp38-win_amd64.whl", hash = "sha256:037e9b4664cafda5f025a1728c50a9e9aedb99a759c89f760bd83730e76ba884"},
+    {file = "black-23.10.1-cp39-cp39-macosx_10_16_arm64.whl", hash = "sha256:7cb5936e686e782fddb1c73f8aa6f459e1ad38a6a7b0e54b403f1f05a1507ee9"},
+    {file = "black-23.10.1-cp39-cp39-macosx_10_16_x86_64.whl", hash = "sha256:7670242e90dc129c539e9ca17665e39a146a761e681805c54fbd86015c7c84f7"},
+    {file = "black-23.10.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5ed45ac9a613fb52dad3b61c8dea2ec9510bf3108d4db88422bacc7d1ba1243d"},
+    {file = "black-23.10.1-cp39-cp39-win_amd64.whl", hash = "sha256:6d23d7822140e3fef190734216cefb262521789367fbdc0b3f22af6744058982"},
+    {file = "black-23.10.1-py3-none-any.whl", hash = "sha256:d431e6739f727bb2e0495df64a6c7a5310758e87505f5f8cde9ff6c0f2d7e4fe"},
+    {file = "black-23.10.1.tar.gz", hash = "sha256:1f8ce316753428ff68749c65a5f7844631aa18c8679dfd3ca9dc1a289979c258"},
 ]

 [package.dependencies]
@@ -467,34 +495,34 @@ files = [

 [[package]]
 name = "cryptography"
-version = "41.0.4"
+version = "41.0.5"
 description = "cryptography is a package which provides cryptographic recipes and primitives to Python developers."
 optional = false
 python-versions = ">=3.7"
 files = [
-    {file = "cryptography-41.0.4-cp37-abi3-macosx_10_12_universal2.whl", hash = "sha256:80907d3faa55dc5434a16579952ac6da800935cd98d14dbd62f6f042c7f5e839"},
-    {file = "cryptography-41.0.4-cp37-abi3-macosx_10_12_x86_64.whl", hash = "sha256:35c00f637cd0b9d5b6c6bd11b6c3359194a8eba9c46d4e875a3660e3b400005f"},
-    {file = "cryptography-41.0.4-cp37-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cecfefa17042941f94ab54f769c8ce0fe14beff2694e9ac684176a2535bf9714"},
-    {file = "cryptography-41.0.4-cp37-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e40211b4923ba5a6dc9769eab704bdb3fbb58d56c5b336d30996c24fcf12aadb"},
-    {file = "cryptography-41.0.4-cp37-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:23a25c09dfd0d9f28da2352503b23e086f8e78096b9fd585d1d14eca01613e13"},
-    {file = "cryptography-41.0.4-cp37-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:2ed09183922d66c4ec5fdaa59b4d14e105c084dd0febd27452de8f6f74704143"},
-    {file = "cryptography-41.0.4-cp37-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:5a0f09cefded00e648a127048119f77bc2b2ec61e736660b5789e638f43cc397"},
-    {file = "cryptography-41.0.4-cp37-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:9eeb77214afae972a00dee47382d2591abe77bdae166bda672fb1e24702a3860"},
-    {file = "cryptography-41.0.4-cp37-abi3-win32.whl", hash = "sha256:3b224890962a2d7b57cf5eeb16ccaafba6083f7b811829f00476309bce2fe0fd"},
-    {file = "cryptography-41.0.4-cp37-abi3-win_amd64.whl", hash = "sha256:c880eba5175f4307129784eca96f4e70b88e57aa3f680aeba3bab0e980b0f37d"},
-    {file = "cryptography-41.0.4-pp310-pypy310_pp73-macosx_10_12_x86_64.whl", hash = "sha256:004b6ccc95943f6a9ad3142cfabcc769d7ee38a3f60fb0dddbfb431f818c3a67"},
-    {file = "cryptography-41.0.4-pp310-pypy310_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:86defa8d248c3fa029da68ce61fe735432b047e32179883bdb1e79ed9bb8195e"},
-    {file = "cryptography-41.0.4-pp310-pypy310_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:37480760ae08065437e6573d14be973112c9e6dcaf5f11d00147ee74f37a3829"},
-    {file = "cryptography-41.0.4-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:b5f4dfe950ff0479f1f00eda09c18798d4f49b98f4e2006d644b3301682ebdca"},
-    {file = "cryptography-41.0.4-pp38-pypy38_pp73-macosx_10_12_x86_64.whl", hash = "sha256:7e53db173370dea832190870e975a1e09c86a879b613948f09eb49324218c14d"},
-    {file = "cryptography-41.0.4-pp38-pypy38_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:5b72205a360f3b6176485a333256b9bcd48700fc755fef51c8e7e67c4b63e3ac"},
-    {file = "cryptography-41.0.4-pp38-pypy38_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:93530900d14c37a46ce3d6c9e6fd35dbe5f5601bf6b3a5c325c7bffc030344d9"},
-    {file = "cryptography-41.0.4-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:efc8ad4e6fc4f1752ebfb58aefece8b4e3c4cae940b0994d43649bdfce8d0d4f"},
-    {file = "cryptography-41.0.4-pp39-pypy39_pp73-macosx_10_12_x86_64.whl", hash = "sha256:c3391bd8e6de35f6f1140e50aaeb3e2b3d6a9012536ca23ab0d9c35ec18c8a91"},
-    {file = "cryptography-41.0.4-pp39-pypy39_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:0d9409894f495d465fe6fda92cb70e8323e9648af912d5b9141d616df40a87b8"},
-    {file = "cryptography-41.0.4-pp39-pypy39_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:8ac4f9ead4bbd0bc8ab2d318f97d85147167a488be0e08814a37eb2f439d5cf6"},
-    {file = "cryptography-41.0.4-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:047c4603aeb4bbd8db2756e38f5b8bd7e94318c047cfe4efeb5d715e08b49311"},
-    {file = "cryptography-41.0.4.tar.gz", hash = "sha256:7febc3094125fc126a7f6fb1f420d0da639f3f32cb15c8ff0dc3997c4549f51a"},
+    {file = "cryptography-41.0.5-cp37-abi3-macosx_10_12_universal2.whl", hash = "sha256:da6a0ff8f1016ccc7477e6339e1d50ce5f59b88905585f77193ebd5068f1e797"},
+    {file = "cryptography-41.0.5-cp37-abi3-macosx_10_12_x86_64.whl", hash = "sha256:b948e09fe5fb18517d99994184854ebd50b57248736fd4c720ad540560174ec5"},
+    {file = "cryptography-41.0.5-cp37-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d38e6031e113b7421db1de0c1b1f7739564a88f1684c6b89234fbf6c11b75147"},
+    {file = "cryptography-41.0.5-cp37-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e270c04f4d9b5671ebcc792b3ba5d4488bf7c42c3c241a3748e2599776f29696"},
+    {file = "cryptography-41.0.5-cp37-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:ec3b055ff8f1dce8e6ef28f626e0972981475173d7973d63f271b29c8a2897da"},
+    {file = "cryptography-41.0.5-cp37-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:7d208c21e47940369accfc9e85f0de7693d9a5d843c2509b3846b2db170dfd20"},
+    {file = "cryptography-41.0.5-cp37-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:8254962e6ba1f4d2090c44daf50a547cd5f0bf446dc658a8e5f8156cae0d8548"},
+    {file = "cryptography-41.0.5-cp37-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:a48e74dad1fb349f3dc1d449ed88e0017d792997a7ad2ec9587ed17405667e6d"},
+    {file = "cryptography-41.0.5-cp37-abi3-win32.whl", hash = "sha256:d3977f0e276f6f5bf245c403156673db103283266601405376f075c849a0b936"},
+    {file = "cryptography-41.0.5-cp37-abi3-win_amd64.whl", hash = "sha256:73801ac9736741f220e20435f84ecec75ed70eda90f781a148f1bad546963d81"},
+    {file = "cryptography-41.0.5-pp310-pypy310_pp73-macosx_10_12_x86_64.whl", hash = "sha256:3be3ca726e1572517d2bef99a818378bbcf7d7799d5372a46c79c29eb8d166c1"},
+    {file = "cryptography-41.0.5-pp310-pypy310_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:e886098619d3815e0ad5790c973afeee2c0e6e04b4da90b88e6bd06e2a0b1b72"},
+    {file = "cryptography-41.0.5-pp310-pypy310_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:573eb7128cbca75f9157dcde974781209463ce56b5804983e11a1c462f0f4e88"},
+    {file = "cryptography-41.0.5-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:0c327cac00f082013c7c9fb6c46b7cc9fa3c288ca702c74773968173bda421bf"},
+    {file = "cryptography-41.0.5-pp38-pypy38_pp73-macosx_10_12_x86_64.whl", hash = "sha256:227ec057cd32a41c6651701abc0328135e472ed450f47c2766f23267b792a88e"},
+    {file = "cryptography-41.0.5-pp38-pypy38_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:22892cc830d8b2c89ea60148227631bb96a7da0c1b722f2aac8824b1b7c0b6b8"},
+    {file = "cryptography-41.0.5-pp38-pypy38_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:5a70187954ba7292c7876734183e810b728b4f3965fbe571421cb2434d279179"},
+    {file = "cryptography-41.0.5-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:88417bff20162f635f24f849ab182b092697922088b477a7abd6664ddd82291d"},
+    {file = "cryptography-41.0.5-pp39-pypy39_pp73-macosx_10_12_x86_64.whl", hash = "sha256:c707f7afd813478e2019ae32a7c49cd932dd60ab2d2a93e796f68236b7e1fbf1"},
+    {file = "cryptography-41.0.5-pp39-pypy39_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:580afc7b7216deeb87a098ef0674d6ee34ab55993140838b14c9b83312b37b86"},
+    {file = "cryptography-41.0.5-pp39-pypy39_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:fba1e91467c65fe64a82c689dc6cf58151158993b13eb7a7f3f4b7f395636723"},
+    {file = "cryptography-41.0.5-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:0d2a6a598847c46e3e321a7aef8af1436f11c27f1254933746304ff014664d84"},
+    {file = "cryptography-41.0.5.tar.gz", hash = "sha256:392cb88b597247177172e02da6b7a63deeff1937fa6fec3bbf902ebd75d97ec7"},
 ]

 [package.dependencies]
@@ -763,17 +791,6 @@ files = [
    {file = "ijson-3.2.3-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:4a3a6a2fbbe7550ffe52d151cf76065e6b89cfb3e9d0463e49a7e322a25d0426"},
    {file = "ijson-3.2.3-cp311-cp311-win32.whl", hash = "sha256:6a4db2f7fb9acfb855c9ae1aae602e4648dd1f88804a0d5cfb78c3639bcf156c"},
    {file = "ijson-3.2.3-cp311-cp311-win_amd64.whl", hash = "sha256:ccd6be56335cbb845f3d3021b1766299c056c70c4c9165fb2fbe2d62258bae3f"},
-    {file = "ijson-3.2.3-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:055b71bbc37af5c3c5861afe789e15211d2d3d06ac51ee5a647adf4def19c0ea"},
-    {file = "ijson-3.2.3-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:c075a547de32f265a5dd139ab2035900fef6653951628862e5cdce0d101af557"},
-    {file = "ijson-3.2.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:457f8a5fc559478ac6b06b6d37ebacb4811f8c5156e997f0d87d708b0d8ab2ae"},
-    {file = "ijson-3.2.3-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9788f0c915351f41f0e69ec2618b81ebfcf9f13d9d67c6d404c7f5afda3e4afb"},
-    {file = "ijson-3.2.3-cp312-cp312-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:fa234ab7a6a33ed51494d9d2197fb96296f9217ecae57f5551a55589091e7853"},
-    {file = "ijson-3.2.3-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bdd0dc5da4f9dc6d12ab6e8e0c57d8b41d3c8f9ceed31a99dae7b2baf9ea769a"},
-    {file = "ijson-3.2.3-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:c6beb80df19713e39e68dc5c337b5c76d36ccf69c30b79034634e5e4c14d6904"},
-    {file = "ijson-3.2.3-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:a2973ce57afb142d96f35a14e9cfec08308ef178a2c76b8b5e1e98f3960438bf"},
-    {file = "ijson-3.2.3-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:105c314fd624e81ed20f925271ec506523b8dd236589ab6c0208b8707d652a0e"},
-    {file = "ijson-3.2.3-cp312-cp312-win32.whl", hash = "sha256:ac44781de5e901ce8339352bb5594fcb3b94ced315a34dbe840b4cff3450e23b"},
-    {file = "ijson-3.2.3-cp312-cp312-win_amd64.whl", hash = "sha256:0567e8c833825b119e74e10a7c29761dc65fcd155f5d4cb10f9d3b8916ef9912"},
    {file = "ijson-3.2.3-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:eeb286639649fb6bed37997a5e30eefcacddac79476d24128348ec890b2a0ccb"},
    {file = "ijson-3.2.3-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:396338a655fb9af4ac59dd09c189885b51fa0eefc84d35408662031023c110d1"},
    {file = "ijson-3.2.3-cp36-cp36m-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0e0243d166d11a2a47c17c7e885debf3b19ed136be2af1f5d1c34212850236ac"},
@@ -1624,13 +1641,13 @@ files = [

 [[package]]
 name = "phonenumbers"
-version = "8.13.22"
+version = "8.13.23"
 description = "Python version of Google's common library for parsing, formatting, storing and validating international phone numbers."
 optional = false
 python-versions = "*"
 files = [
-    {file = "phonenumbers-8.13.22-py2.py3-none-any.whl", hash = "sha256:85ceeba9e67984ba98182c77e8e4c70093d38c0c6a0cb2bd392e0694ddaeb1f6"},
-    {file = "phonenumbers-8.13.22.tar.gz", hash = "sha256:001664c90f59b8954766c2db85adafc8dbc96177efeb49607ca4e64a7acaf569"},
+    {file = "phonenumbers-8.13.23-py2.py3-none-any.whl", hash = "sha256:34d6cb279dd4a64714e324c71350f96e5bda3237be28d11b4c555c44701544cd"},
+    {file = "phonenumbers-8.13.23.tar.gz", hash = "sha256:869e44fcaaf276eca6b953a401e2b27d57461f3a18a66cf5f13377e7bb0e228c"},
 ]

 [[package]]
@@ -1754,6 +1771,30 @@ files = [
 [package.extras]
 twisted = ["twisted"]

+[[package]]
+name = "psycopg"
+version = "3.1.12"
+description = "PostgreSQL database adapter for Python"
+optional = true
+python-versions = ">=3.7"
+files = [
+    {file = "psycopg-3.1.12-py3-none-any.whl", hash = "sha256:8ec5230d6a7eb654b4fb3cf2d3eda8871d68f24807b934790504467f1deee9f8"},
+    {file = "psycopg-3.1.12.tar.gz", hash = "sha256:cec7ad2bc6a8510e56c45746c631cf9394148bdc8a9a11fd8cf8554ce129ae78"},
+]
+
+[package.dependencies]
+"backports.zoneinfo" = {version = ">=0.2.0", markers = "python_version < \"3.9\""}
+typing-extensions = ">=4.1"
+tzdata = {version = "*", markers = "sys_platform == \"win32\""}
+
+[package.extras]
+binary = ["psycopg-binary (==3.1.12)"]
+c = ["psycopg-c (==3.1.12)"]
+dev = ["black (>=23.1.0)", "dnspython (>=2.1)", "flake8 (>=4.0)", "mypy (>=1.4.1)", "types-setuptools (>=57.4)", "wheel (>=0.37)"]
+docs = ["Sphinx (>=5.0)", "furo (==2022.6.21)", "sphinx-autobuild (>=2021.3.14)", "sphinx-autodoc-typehints (>=1.12)"]
+pool = ["psycopg-pool"]
+test = ["anyio (>=3.6.2,<4.0)", "mypy (>=1.4.1)", "pproxy (>=2.7)", "pytest (>=6.2.5)", "pytest-cov (>=3.0)", "pytest-randomly (>=3.5)"]
+
 [[package]]
 name = "psycopg2"
 version = "2.9.9"
@@ -2010,12 +2051,12 @@ plugins = ["importlib-metadata"]

 [[package]]
 name = "pyicu"
-version = "2.11"
+version = "2.12"
 description = "Python extension wrapping the ICU C++ API"
 optional = true
 python-versions = "*"
 files = [
-    {file = "PyICU-2.11.tar.gz", hash = "sha256:3ab531264cfe9132b3d2ac5d708da9a4649d25f6e6813730ac88cf040a08a844"},
+    {file = "PyICU-2.12.tar.gz", hash = "sha256:bd7ab5efa93ad692e6daa29cd249364e521218329221726a113ca3cb281c8611"},
 ]

 [[package]]
@@ -2180,7 +2221,6 @@ files = [
    {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:69b023b2b4daa7548bcfbd4aa3da05b3a74b772db9e23b982788168117739938"},
    {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:81e0b275a9ecc9c0c0c07b4b90ba548307583c125f54d5b6946cfee6360c733d"},
    {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ba336e390cd8e4d1739f42dfe9bb83a3cc2e80f567d8805e11b46f4a943f5515"},
-    {file = "PyYAML-6.0.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:326c013efe8048858a6d312ddd31d56e468118ad4cdeda36c719bf5bb6192290"},
    {file = "PyYAML-6.0.1-cp310-cp310-win32.whl", hash = "sha256:bd4af7373a854424dabd882decdc5579653d7868b8fb26dc7d0e99f823aa5924"},
    {file = "PyYAML-6.0.1-cp310-cp310-win_amd64.whl", hash = "sha256:fd1592b3fdf65fff2ad0004b5e363300ef59ced41c2e6b3a99d4089fa8c5435d"},
    {file = "PyYAML-6.0.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:6965a7bc3cf88e5a1c3bd2e0b5c22f8d677dc88a455344035f03399034eb3007"},
@@ -2188,15 +2228,8 @@ files = [
    {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:42f8152b8dbc4fe7d96729ec2b99c7097d656dc1213a3229ca5383f973a5ed6d"},
    {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:062582fca9fabdd2c8b54a3ef1c978d786e0f6b3a1510e0ac93ef59e0ddae2bc"},
    {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d2b04aac4d386b172d5b9692e2d2da8de7bfb6c387fa4f801fbf6fb2e6ba4673"},
-    {file = "PyYAML-6.0.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:e7d73685e87afe9f3b36c799222440d6cf362062f78be1013661b00c5c6f678b"},
    {file = "PyYAML-6.0.1-cp311-cp311-win32.whl", hash = "sha256:1635fd110e8d85d55237ab316b5b011de701ea0f29d07611174a1b42f1444741"},
    {file = "PyYAML-6.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:bf07ee2fef7014951eeb99f56f39c9bb4af143d8aa3c21b1677805985307da34"},
-    {file = "PyYAML-6.0.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:855fb52b0dc35af121542a76b9a84f8d1cd886ea97c84703eaa6d88e37a2ad28"},
-    {file = "PyYAML-6.0.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:40df9b996c2b73138957fe23a16a4f0ba614f4c0efce1e9406a184b6d07fa3a9"},
-    {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6c22bec3fbe2524cde73d7ada88f6566758a8f7227bfbf93a408a9d86bcc12a0"},
-    {file = "PyYAML-6.0.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8d4e9c88387b0f5c7d5f281e55304de64cf7f9c0021a3525bd3b1c542da3b0e4"},
-    {file = "PyYAML-6.0.1-cp312-cp312-win32.whl", hash = "sha256:d483d2cdf104e7c9fa60c544d92981f12ad66a457afae824d146093b8c294c54"},
-    {file = "PyYAML-6.0.1-cp312-cp312-win_amd64.whl", hash = "sha256:0d3304d8c0adc42be59c5f8a4d9e3d7379e6955ad754aa9d6ab7a398b59dd1df"},
    {file = "PyYAML-6.0.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:50550eb667afee136e9a77d6dc71ae76a44df8b3e51e41b77f6de2932bfe0f47"},
    {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1fe35611261b29bd1de0070f0b2f47cb6ff71fa6595c077e42bd0c419fa27b98"},
    {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:704219a11b772aea0d8ecd7058d0082713c3562b4e271b849ad7dc4a5c90c13c"},
@@ -2213,7 +2246,6 @@ files = [
    {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a0cd17c15d3bb3fa06978b4e8958dcdc6e0174ccea823003a106c7d4d7899ac5"},
    {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:28c119d996beec18c05208a8bd78cbe4007878c6dd15091efb73a30e90539696"},
    {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7e07cbde391ba96ab58e532ff4803f79c4129397514e1413a7dc761ccd755735"},
-    {file = "PyYAML-6.0.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:49a183be227561de579b4a36efbb21b3eab9651dd81b1858589f796549873dd6"},
    {file = "PyYAML-6.0.1-cp38-cp38-win32.whl", hash = "sha256:184c5108a2aca3c5b3d3bf9395d50893a7ab82a38004c8f61c258d4428e80206"},
    {file = "PyYAML-6.0.1-cp38-cp38-win_amd64.whl", hash = "sha256:1e2722cc9fbb45d9b87631ac70924c11d3a401b2d7f410cc0e3bbf249f2dca62"},
    {file = "PyYAML-6.0.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9eb6caa9a297fc2c2fb8862bc5370d0303ddba53ba97e71f08023b6cd73d16a8"},
@@ -2221,7 +2253,6 @@ files = [
    {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5773183b6446b2c99bb77e77595dd486303b4faab2b086e7b17bc6bef28865f6"},
    {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b786eecbdf8499b9ca1d697215862083bd6d2a99965554781d0d8d1ad31e13a0"},
    {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bc1bf2925a1ecd43da378f4db9e4f799775d6367bdb94671027b73b393a7c42c"},
-    {file = "PyYAML-6.0.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:04ac92ad1925b2cff1db0cfebffb6ffc43457495c9b3c39d3fcae417d7125dc5"},
    {file = "PyYAML-6.0.1-cp39-cp39-win32.whl", hash = "sha256:faca3bdcf85b2fc05d06ff3fbc1f83e1391b3e724afa3feba7d13eeab355484c"},
    {file = "PyYAML-6.0.1-cp39-cp39-win_amd64.whl", hash = "sha256:510c9deebc5c0225e8c96813043e62b680ba2f9c50a08d3724c7f28a747d1486"},
    {file = "PyYAML-6.0.1.tar.gz", hash = "sha256:bfdf460b1736c775f2ba9f6a92bca30bc2095067b8a9d77876d1fad6cc3b4a43"},
@@ -2437,28 +2468,28 @@ files = [

 [[package]]
 name = "ruff"
-version = "0.0.292"
-description = "An extremely fast Python linter, written in Rust."
+version = "0.1.4"
+description = "An extremely fast Python linter and code formatter, written in Rust."
 optional = false
 python-versions = ">=3.7"
 files = [
-    {file = "ruff-0.0.292-py3-none-macosx_10_7_x86_64.whl", hash = "sha256:02f29db018c9d474270c704e6c6b13b18ed0ecac82761e4fcf0faa3728430c96"},
-    {file = "ruff-0.0.292-py3-none-macosx_10_9_x86_64.macosx_11_0_arm64.macosx_10_9_universal2.whl", hash = "sha256:69654e564342f507edfa09ee6897883ca76e331d4bbc3676d8a8403838e9fade"},
-    {file = "ruff-0.0.292-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6c3c91859a9b845c33778f11902e7b26440d64b9d5110edd4e4fa1726c41e0a4"},
-    {file = "ruff-0.0.292-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:f4476f1243af2d8c29da5f235c13dca52177117935e1f9393f9d90f9833f69e4"},
-    {file = "ruff-0.0.292-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:be8eb50eaf8648070b8e58ece8e69c9322d34afe367eec4210fdee9a555e4ca7"},
-    {file = "ruff-0.0.292-py3-none-manylinux_2_17_ppc64.manylinux2014_ppc64.whl", hash = "sha256:9889bac18a0c07018aac75ef6c1e6511d8411724d67cb879103b01758e110a81"},
-    {file = "ruff-0.0.292-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:6bdfabd4334684a4418b99b3118793f2c13bb67bf1540a769d7816410402a205"},
-    {file = "ruff-0.0.292-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:aa7c77c53bfcd75dbcd4d1f42d6cabf2485d2e1ee0678da850f08e1ab13081a8"},
-    {file = "ruff-0.0.292-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8e087b24d0d849c5c81516ec740bf4fd48bf363cfb104545464e0fca749b6af9"},
-    {file = "ruff-0.0.292-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:f160b5ec26be32362d0774964e218f3fcf0a7da299f7e220ef45ae9e3e67101a"},
-    {file = "ruff-0.0.292-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:ac153eee6dd4444501c4bb92bff866491d4bfb01ce26dd2fff7ca472c8df9ad0"},
-    {file = "ruff-0.0.292-py3-none-musllinux_1_2_i686.whl", hash = "sha256:87616771e72820800b8faea82edd858324b29bb99a920d6aa3d3949dd3f88fb0"},
-    {file = "ruff-0.0.292-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:b76deb3bdbea2ef97db286cf953488745dd6424c122d275f05836c53f62d4016"},
-    {file = "ruff-0.0.292-py3-none-win32.whl", hash = "sha256:e854b05408f7a8033a027e4b1c7f9889563dd2aca545d13d06711e5c39c3d003"},
-    {file = "ruff-0.0.292-py3-none-win_amd64.whl", hash = "sha256:f27282bedfd04d4c3492e5c3398360c9d86a295be00eccc63914438b4ac8a83c"},
-    {file = "ruff-0.0.292-py3-none-win_arm64.whl", hash = "sha256:7f67a69c8f12fbc8daf6ae6d36705037bde315abf8b82b6e1f4c9e74eb750f68"},
-    {file = "ruff-0.0.292.tar.gz", hash = "sha256:1093449e37dd1e9b813798f6ad70932b57cf614e5c2b5c51005bf67d55db33ac"},
+    {file = "ruff-0.1.4-py3-none-macosx_10_7_x86_64.whl", hash = "sha256:864958706b669cce31d629902175138ad8a069d99ca53514611521f532d91495"},
+    {file = "ruff-0.1.4-py3-none-macosx_10_9_x86_64.macosx_11_0_arm64.macosx_10_9_universal2.whl", hash = "sha256:9fdd61883bb34317c788af87f4cd75dfee3a73f5ded714b77ba928e418d6e39e"},
+    {file = "ruff-0.1.4-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b4eaca8c9cc39aa7f0f0d7b8fe24ecb51232d1bb620fc4441a61161be4a17539"},
+    {file = "ruff-0.1.4-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:a9a1301dc43cbf633fb603242bccd0aaa34834750a14a4c1817e2e5c8d60de17"},
+    {file = "ruff-0.1.4-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:78e8db8ab6f100f02e28b3d713270c857d370b8d61871d5c7d1702ae411df683"},
+    {file = "ruff-0.1.4-py3-none-manylinux_2_17_ppc64.manylinux2014_ppc64.whl", hash = "sha256:80fea754eaae06335784b8ea053d6eb8e9aac75359ebddd6fee0858e87c8d510"},
+    {file = "ruff-0.1.4-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:6bc02a480d4bfffd163a723698da15d1a9aec2fced4c06f2a753f87f4ce6969c"},
+    {file = "ruff-0.1.4-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:9862811b403063765b03e716dac0fda8fdbe78b675cd947ed5873506448acea4"},
+    {file = "ruff-0.1.4-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:58826efb8b3efbb59bb306f4b19640b7e366967a31c049d49311d9eb3a4c60cb"},
+    {file = "ruff-0.1.4-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:fdfd453fc91d9d86d6aaa33b1bafa69d114cf7421057868f0b79104079d3e66e"},
+    {file = "ruff-0.1.4-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:e8791482d508bd0b36c76481ad3117987301b86072158bdb69d796503e1c84a8"},
+    {file = "ruff-0.1.4-py3-none-musllinux_1_2_i686.whl", hash = "sha256:01206e361021426e3c1b7fba06ddcb20dbc5037d64f6841e5f2b21084dc51800"},
+    {file = "ruff-0.1.4-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:645591a613a42cb7e5c2b667cbefd3877b21e0252b59272ba7212c3d35a5819f"},
+    {file = "ruff-0.1.4-py3-none-win32.whl", hash = "sha256:99908ca2b3b85bffe7e1414275d004917d1e0dfc99d497ccd2ecd19ad115fd0d"},
+    {file = "ruff-0.1.4-py3-none-win_amd64.whl", hash = "sha256:1dfd6bf8f6ad0a4ac99333f437e0ec168989adc5d837ecd38ddb2cc4a2e3db8a"},
+    {file = "ruff-0.1.4-py3-none-win_arm64.whl", hash = "sha256:d98ae9ebf56444e18a3e3652b3383204748f73e247dea6caaf8b52d37e6b32da"},
+    {file = "ruff-0.1.4.tar.gz", hash = "sha256:21520ecca4cc555162068d87c747b8f95e1e95f8ecfcbbe59e8dd00710586315"},
 ]

 [[package]]
@@ -2578,20 +2609,19 @@ testing-integration = ["build[virtualenv]", "filelock (>=3.4.0)", "jaraco.envs (

 [[package]]
 name = "setuptools-rust"
-version = "1.7.0"
+version = "1.8.1"
 description = "Setuptools Rust extension plugin"
 optional = false
-python-versions = ">=3.7"
+python-versions = ">=3.8"
 files = [
-    {file = "setuptools-rust-1.7.0.tar.gz", hash = "sha256:c7100999948235a38ae7e555fe199aa66c253dc384b125f5d85473bf81eae3a3"},
-    {file = "setuptools_rust-1.7.0-py3-none-any.whl", hash = "sha256:071099885949132a2180d16abf907b60837e74b4085047ba7e9c0f5b365310c1"},
+    {file = "setuptools-rust-1.8.1.tar.gz", hash = "sha256:94b1dd5d5308b3138d5b933c3a2b55e6d6927d1a22632e509fcea9ddd0f7e486"},
+    {file = "setuptools_rust-1.8.1-py3-none-any.whl", hash = "sha256:b5324493949ccd6aa0c03890c5f6b5f02de4512e3ac1697d02e9a6c02b18aa8e"},
 ]

 [package.dependencies]
 semantic-version = ">=2.8.2,<3"
 setuptools = ">=62.4"
 tomli = {version = ">=1.2.1", markers = "python_version < \"3.11\""}
-typing-extensions = ">=3.7.4.3"

 [[package]]
 name = "signedjson"
@@ -2971,13 +3001,13 @@ urllib3 = ">=1.26.0"

 [[package]]
 name = "twisted"
-version = "23.8.0"
+version = "23.10.0"
 description = "An asynchronous networking framework written in Python"
 optional = false
-python-versions = ">=3.7.1"
+python-versions = ">=3.8.0"
 files = [
-    {file = "twisted-23.8.0-py3-none-any.whl", hash = "sha256:b8bdba145de120ffb36c20e6e071cce984e89fba798611ed0704216fb7f884cd"},
-    {file = "twisted-23.8.0.tar.gz", hash = "sha256:3c73360add17336a622c0d811c2a2ce29866b6e59b1125fd6509b17252098a24"},
+    {file = "twisted-23.10.0-py3-none-any.whl", hash = "sha256:4ae8bce12999a35f7fe6443e7f1893e6fe09588c8d2bed9c35cdce8ff2d5b444"},
+    {file = "twisted-23.10.0.tar.gz", hash = "sha256:987847a0790a2c597197613686e2784fd54167df3a55d0fb17c8412305d76ce5"},
 ]

 [package.dependencies]
@@ -2990,19 +3020,18 @@ incremental = ">=22.10.0"
 pyopenssl = {version = ">=21.0.0", optional = true, markers = "extra == \"tls\""}
 service-identity = {version = ">=18.1.0", optional = true, markers = "extra == \"tls\""}
 twisted-iocpsupport = {version = ">=1.0.2,<2", markers = "platform_system == \"Windows\""}
-typing-extensions = ">=3.10.0"
+typing-extensions = ">=4.2.0"
 zope-interface = ">=5"

 [package.extras]
-all-non-platform = ["twisted[conch,contextvars,http2,serial,test,tls]", "twisted[conch,contextvars,http2,serial,test,tls]"]
+all-non-platform = ["twisted[conch,http2,serial,test,tls]", "twisted[conch,http2,serial,test,tls]"]
 conch = ["appdirs (>=1.4.0)", "bcrypt (>=3.1.3)", "cryptography (>=3.3)"]
-contextvars = ["contextvars (>=2.4,<3)"]
 dev = ["coverage (>=6b1,<7)", "pyflakes (>=2.2,<3.0)", "python-subunit (>=1.4,<2.0)", "twisted[dev-release]", "twistedchecker (>=0.7,<1.0)"]
-dev-release = ["pydoctor (>=23.4.0,<23.5.0)", "pydoctor (>=23.4.0,<23.5.0)", "readthedocs-sphinx-ext (>=2.2,<3.0)", "readthedocs-sphinx-ext (>=2.2,<3.0)", "sphinx (>=5,<7)", "sphinx (>=5,<7)", "sphinx-rtd-theme (>=1.2,<2.0)", "sphinx-rtd-theme (>=1.2,<2.0)", "towncrier (>=22.12,<23.0)", "towncrier (>=22.12,<23.0)", "urllib3 (<2)", "urllib3 (<2)"]
+dev-release = ["pydoctor (>=23.9.0,<23.10.0)", "pydoctor (>=23.9.0,<23.10.0)", "sphinx (>=6,<7)", "sphinx (>=6,<7)", "sphinx-rtd-theme (>=1.3,<2.0)", "sphinx-rtd-theme (>=1.3,<2.0)", "towncrier (>=23.6,<24.0)", "towncrier (>=23.6,<24.0)"]
 gtk-platform = ["pygobject", "pygobject", "twisted[all-non-platform]", "twisted[all-non-platform]"]
 http2 = ["h2 (>=3.0,<5.0)", "priority (>=1.1.0,<2.0)"]
 macos-platform = ["pyobjc-core", "pyobjc-core", "pyobjc-framework-cfnetwork", "pyobjc-framework-cfnetwork", "pyobjc-framework-cocoa", "pyobjc-framework-cocoa", "twisted[all-non-platform]", "twisted[all-non-platform]"]
-mypy = ["mypy (==0.981)", "mypy-extensions (==0.4.3)", "mypy-zope (==0.3.11)", "twisted[all-non-platform,dev]", "types-pyopenssl", "types-setuptools"]
+mypy = ["mypy (>=1.5.1,<1.6.0)", "mypy-zope (>=1.0.1,<1.1.0)", "twisted[all-non-platform,dev]", "types-pyopenssl", "types-setuptools"]
 osx-platform = ["twisted[macos-platform]", "twisted[macos-platform]"]
 serial = ["pyserial (>=3.0)", "pywin32 (!=226)"]
 test = ["cython-test-exception-raiser (>=1.0.2,<2)", "hypothesis (>=6.56)", "pyhamcrest (>=2)"]
@@ -3069,13 +3098,13 @@ files = [

 [[package]]
 name = "types-jsonschema"
-version = "4.19.0.3"
+version = "4.19.0.4"
 description = "Typing stubs for jsonschema"
 optional = false
 python-versions = ">=3.8"
 files = [
-    {file = "types-jsonschema-4.19.0.3.tar.gz", hash = "sha256:e0fc0f5d51fd0988bf193be42174a5376b0096820ff79505d9c1b66de23f0581"},
-    {file = "types_jsonschema-4.19.0.3-py3-none-any.whl", hash = "sha256:5cedbb661e5ca88d95b94b79902423e3f97a389c245e5fe0ab384122f27d56b9"},
+    {file = "types-jsonschema-4.19.0.4.tar.gz", hash = "sha256:994feb6632818259c4b5dbd733867824cb475029a6abc2c2b5201a2268b6e7d2"},
+    {file = "types_jsonschema-4.19.0.4-py3-none-any.whl", hash = "sha256:b73c3f4ba3cd8108602d1198a438e2698d5eb6b9db206ed89a33e24729b0abe7"},
 ]

 [package.dependencies]
@@ -3116,13 +3145,13 @@ files = [

 [[package]]
 name = "types-psycopg2"
-version = "2.9.21.14"
+version = "2.9.21.15"
 description = "Typing stubs for psycopg2"
 optional = false
-python-versions = "*"
+python-versions = ">=3.7"
 files = [
-    {file = "types-psycopg2-2.9.21.14.tar.gz", hash = "sha256:bf73a0ac4da4e278c89bf1b01fc596d5a5ac7a356cfe6ac0249f47b9e259f868"},
-    {file = "types_psycopg2-2.9.21.14-py3-none-any.whl", hash = "sha256:cd9c5350631f3bc6184ec8d48f2ed31d4ea660f89d0fffe78239450782f383c5"},
+    {file = "types-psycopg2-2.9.21.15.tar.gz", hash = "sha256:cf99b62ab32cd4ef412fc3c4da1c29ca5a130847dff06d709b84a523802406f0"},
+    {file = "types_psycopg2-2.9.21.15-py3-none-any.whl", hash = "sha256:cc80479def02e4dd1ef21649d82f04426c73bc0693bcc0a8b5223c7c168472af"},
 ]

 [[package]]
@@ -3141,13 +3170,13 @@ cryptography = ">=35.0.0"

 [[package]]
 name = "types-pyyaml"
-version = "6.0.12.11"
+version = "6.0.12.12"
 description = "Typing stubs for PyYAML"
 optional = false
 python-versions = "*"
 files = [
-    {file = "types-PyYAML-6.0.12.11.tar.gz", hash = "sha256:7d340b19ca28cddfdba438ee638cd4084bde213e501a3978738543e27094775b"},
-    {file = "types_PyYAML-6.0.12.11-py3-none-any.whl", hash = "sha256:a461508f3096d1d5810ec5ab95d7eeecb651f3a15b71959999988942063bf01d"},
+    {file = "types-PyYAML-6.0.12.12.tar.gz", hash = "sha256:334373d392fde0fdf95af5c3f1661885fa10c52167b14593eb856289e1855062"},
+    {file = "types_PyYAML-6.0.12.12-py3-none-any.whl", hash = "sha256:c05bc6c158facb0676674b7f11fe3960db4f389718e19e62bd2b84d6205cfd24"},
 ]

 [[package]]
@@ -3186,6 +3215,17 @@ files = [
    {file = "typing_extensions-4.8.0.tar.gz", hash = "sha256:df8e4339e9cb77357558cbdbceca33c303714cf861d1eef15e1070055ae8b7ef"},
 ]

+[[package]]
+name = "tzdata"
+version = "2023.3"
+description = "Provider of IANA time zone data"
+optional = true
+python-versions = ">=2"
+files = [
+    {file = "tzdata-2023.3-py2.py3-none-any.whl", hash = "sha256:7e65763eef3120314099b6939b5546db7adce1e7d6f2e179e3df563c70511eda"},
+    {file = "tzdata-2023.3.tar.gz", hash = "sha256:11ef1e08e54acb0d4f95bdb1be05da659673de4acbd21bf9c69e94cc5e907a3a"},
+]
+
 [[package]]
 name = "unpaddedbase64"
 version = "2.1.0"
@@ -3429,13 +3469,13 @@ docs = ["Sphinx", "repoze.sphinx.autointerface"]
 test = ["zope.i18nmessageid", "zope.testing", "zope.testrunner"]

 [extras]
-all = ["Pympler", "authlib", "hiredis", "jaeger-client", "lxml", "matrix-synapse-ldap3", "opentracing", "psycopg2", "psycopg2cffi", "psycopg2cffi-compat", "pyicu", "pysaml2", "sentry-sdk", "txredisapi"]
+all = ["Pympler", "authlib", "hiredis", "jaeger-client", "lxml", "matrix-synapse-ldap3", "opentracing", "psycopg", "psycopg2", "psycopg2cffi", "psycopg2cffi-compat", "pyicu", "pysaml2", "sentry-sdk", "txredisapi"]
 cache-memory = ["Pympler"]
 jwt = ["authlib"]
 matrix-synapse-ldap3 = ["matrix-synapse-ldap3"]
 oidc = ["authlib"]
 opentracing = ["jaeger-client", "opentracing"]
-postgres = ["psycopg2", "psycopg2cffi", "psycopg2cffi-compat"]
+postgres = ["psycopg", "psycopg2", "psycopg2cffi", "psycopg2cffi-compat"]
 redis = ["hiredis", "txredisapi"]
 saml2 = ["pysaml2"]
 sentry = ["sentry-sdk"]
@@ -3447,4 +3487,4 @@ user-search = ["pyicu"]
 [metadata]
 lock-version = "2.0"
 python-versions = "^3.8.0"
-content-hash = "a08543c65f18cc7e9dea648e89c18ab88fc1747aa2e029aa208f777fc3db06dd"
+content-hash = "73869fa198a8ddd7f724915dbea25b462a44fa418ce47a425b1adf16ecd1b82f"
@@ -96,7 +96,7 @@ module-name = "synapse.synapse_rust"

 [tool.poetry]
 name = "matrix-synapse"
-version = "1.95.0"
+version = "1.96.0rc1"
 description = "Homeserver for the Matrix decentralised comms protocol"
 authors = ["Matrix.org Team and Contributors <packages@matrix.org>"]
 license = "Apache-2.0"
@@ -233,6 +233,7 @@ matrix-synapse-ldap3 = { version = ">=0.1", optional = true }
 psycopg2 = { version = ">=2.8", markers = "platform_python_implementation != 'PyPy'", optional = true }
 psycopg2cffi = { version = ">=2.8", markers = "platform_python_implementation == 'PyPy'", optional = true }
 psycopg2cffi-compat = { version = "==1.1", markers = "platform_python_implementation == 'PyPy'", optional = true }
+psycopg = { version = "^3.1", optional = true }
 pysaml2 = { version = ">=4.5.0", optional = true }
 authlib = { version = ">=0.15.1", optional = true }
 # systemd-python is necessary for logging to the systemd journal via
@@ -255,7 +256,7 @@ pyicu = { version = ">=2.10.2", optional = true }
 # NB: Packages that should be part of `pip install matrix-synapse[all]` need to be specified
 # twice: once here, and once in the `all` extra.
 matrix-synapse-ldap3 = ["matrix-synapse-ldap3"]
-postgres = ["psycopg2", "psycopg2cffi", "psycopg2cffi-compat"]
+postgres = ["psycopg2", "psycopg2cffi", "psycopg2cffi-compat", "psycopg"]
 saml2 = ["pysaml2"]
 oidc = ["authlib"]
 # systemd-python is necessary for logging to the systemd journal via
@@ -292,7 +293,7 @@ all = [
    # matrix-synapse-ldap3
    "matrix-synapse-ldap3",
    # postgres
-    "psycopg2", "psycopg2cffi", "psycopg2cffi-compat",
+    "psycopg2", "psycopg2cffi", "psycopg2cffi-compat", "psycopg",
    # saml2
    "pysaml2",
    # oidc and jwt
@@ -321,7 +322,7 @@ all = [
 # This helps prevents merge conflicts when running a batch of dependabot updates.
 isort = ">=5.10.1"
 black = ">=22.7.0"
-ruff = "0.0.292"
+ruff = "0.1.4"
 # Type checking only works with the pydantic.v1 compat module from pydantic v2
 pydantic = "^2"

@@ -381,7 +382,7 @@ furo = ">=2022.12.7,<2024.0.0"
 # system changes.
 # We are happy to raise these upper bounds upon request,
 # provided we check that it's safe to do so (i.e. that CI passes).
-requires = ["poetry-core>=1.1.0,<=1.7.0", "setuptools_rust>=1.3,<=1.7.0"]
+requires = ["poetry-core>=1.1.0,<=1.7.0", "setuptools_rust>=1.3,<=1.8.1"]
 build-backend = "poetry.core.masonry.api"


@@ -216,6 +216,10 @@ extra_test_args=()

 test_packages="./tests/csapi ./tests ./tests/msc3874 ./tests/msc3890 ./tests/msc3391 ./tests/msc3930 ./tests/msc3902"

+# Enable dirty runs, so tests will reuse the same container where possible.
+# This significantly speeds up tests, but increases the possibility of test pollution.
+export COMPLEMENT_ENABLE_DIRTY_RUNS=1
+
 # All environment variables starting with PASS_ will be shared.
 # (The prefix is stripped off before reaching the container.)
 export COMPLEMENT_SHARE_ENV_PREFIX=PASS_
@@ -274,7 +278,7 @@ fi
 export PASS_SYNAPSE_LOG_TESTING=1

 # Run the tests!
-echo "Images built; running complement"
+echo "Images built; running complement with ${extra_test_args[@]} $@ $test_packages"
 cd "$COMPLEMENT_DIR"

 go test -v -tags "synapse_blacklist" -count=1 "${extra_test_args[@]}" "$@" $test_packages
@@ -191,7 +191,7 @@ IGNORED_TABLES = {
    "user_directory_search_stat",
    "user_directory_search_pos",
    "users_who_share_private_rooms",
-    "users_in_public_room",
+    "users_in_public_rooms",
    # UI auth sessions have foreign keys so additional care needs to be taken,
    # the sessions are transient anyway, so ignore them.
    "ui_auth_sessions",
@@ -348,8 +348,7 @@ class Porter:
                    backward_chunk = 0
                    already_ported = 0
            else:
-                forward_chunk = row["forward_rowid"]
-                backward_chunk = row["backward_rowid"]
+                forward_chunk, backward_chunk = row

            if total_to_port is None:
                already_ported, total_to_port = await self._get_total_count_to_port(
@@ -1391,7 +1390,7 @@ def main() -> None:
    if "name" not in postgres_config:
        sys.stderr.write("Malformed database config: no 'name'\n")
        sys.exit(2)
-    if postgres_config["name"] != "psycopg2":
+    if postgres_config["name"] not in ("psycopg", "psycopg2"):
        sys.stderr.write("Database must use the 'psycopg2' connector.\n")
        sys.exit(3)

@@ -50,7 +50,7 @@ class DatabaseConnectionConfig:
    def __init__(self, name: str, db_config: dict):
        db_engine = db_config.get("name", "sqlite3")

-        if db_engine not in ("sqlite3", "psycopg2"):
+        if db_engine not in ("sqlite3", "psycopg2", "psycopg"):
            raise ConfigError("Unsupported database type %r" % (db_engine,))

        if db_engine == "sqlite3":
@@ -368,9 +368,14 @@ class ServerConfig(Config):

        # Whether to enable user presence.
        presence_config = config.get("presence") or {}
-        self.use_presence = presence_config.get("enabled")
-        if self.use_presence is None:
-            self.use_presence = config.get("use_presence", True)
+        presence_enabled = presence_config.get("enabled")
+        if presence_enabled is None:
+            presence_enabled = config.get("use_presence", True)
+
+        # Whether presence is enabled *at all*.
+        self.presence_enabled = bool(presence_enabled)
+        # Whether to internally track presence, requires that presence is enabled,
+        self.track_presence = self.presence_enabled and presence_enabled != "untracked"

        # Custom presence router module
        # This is the legacy way of configuring it (the config should now be put in the modules section)
@@ -358,9 +358,9 @@ class WorkerConfig(Config):
                "Must only specify one instance to handle `account_data` messages."
            )

-        if len(self.writers.receipts) != 1:
+        if len(self.writers.receipts) == 0:
            raise ConfigError(
-                "Must only specify one instance to handle `receipts` messages."
+                "Must specify at least one instance to handle `receipts` messages."
            )

        if len(self.writers.events) == 0:
@@ -17,6 +17,7 @@ import re
 from typing import (
    TYPE_CHECKING,
    Any,
+    Awaitable,
    Callable,
    Dict,
    Iterable,
@@ -45,6 +46,7 @@ from . import EventBase

 if TYPE_CHECKING:
    from synapse.handlers.relations import BundledAggregations
+    from synapse.server import HomeServer


 # Split strings on "." but not "\." (or "\\\.").
@@ -56,6 +58,13 @@ CANONICALJSON_MAX_INT = (2**53) - 1
 CANONICALJSON_MIN_INT = -CANONICALJSON_MAX_INT


+# Module API callback that allows adding fields to the unsigned section of
+# events that are sent to clients.
+ADD_EXTRA_FIELDS_TO_UNSIGNED_CLIENT_EVENT_CALLBACK = Callable[
+    [EventBase], Awaitable[JsonDict]
+]
+
+
 def prune_event(event: EventBase) -> EventBase:
    """Returns a pruned version of the given event, which removes all keys we
    don't know about or think could potentially be dodgy.
@@ -509,7 +518,13 @@ class EventClientSerializer:
    clients.
    """

-    def serialize_event(
+    def __init__(self, hs: "HomeServer") -> None:
+        self._store = hs.get_datastores().main
+        self._add_extra_fields_to_unsigned_client_event_callbacks: List[
+            ADD_EXTRA_FIELDS_TO_UNSIGNED_CLIENT_EVENT_CALLBACK
+        ] = []
+
+    async def serialize_event(
        self,
        event: Union[JsonDict, EventBase],
        time_now: int,
@@ -535,10 +550,21 @@ class EventClientSerializer:

        serialized_event = serialize_event(event, time_now, config=config)

+        new_unsigned = {}
+        for callback in self._add_extra_fields_to_unsigned_client_event_callbacks:
+            u = await callback(event)
+            new_unsigned.update(u)
+
+        if new_unsigned:
+            # We do the `update` this way round so that modules can't clobber
+            # existing fields.
+            new_unsigned.update(serialized_event["unsigned"])
+            serialized_event["unsigned"] = new_unsigned
+
        # Check if there are any bundled aggregations to include with the event.
        if bundle_aggregations:
            if event.event_id in bundle_aggregations:
-                self._inject_bundled_aggregations(
+                await self._inject_bundled_aggregations(
                    event,
                    time_now,
                    config,
@@ -548,7 +574,7 @@ class EventClientSerializer:

        return serialized_event

-    def _inject_bundled_aggregations(
+    async def _inject_bundled_aggregations(
        self,
        event: EventBase,
        time_now: int,
@@ -590,7 +616,7 @@ class EventClientSerializer:
            # said that we should only include the `event_id`, `origin_server_ts` and
            # `sender` of the edit; however MSC3925 proposes extending it to the whole
            # of the edit, which is what we do here.
-            serialized_aggregations[RelationTypes.REPLACE] = self.serialize_event(
+            serialized_aggregations[RelationTypes.REPLACE] = await self.serialize_event(
                event_aggregations.replace,
                time_now,
                config=config,
@@ -600,7 +626,7 @@ class EventClientSerializer:
        if event_aggregations.thread:
            thread = event_aggregations.thread

-            serialized_latest_event = self.serialize_event(
+            serialized_latest_event = await self.serialize_event(
                thread.latest_event,
                time_now,
                config=config,
@@ -623,7 +649,7 @@ class EventClientSerializer:
                "m.relations", {}
            ).update(serialized_aggregations)

-    def serialize_events(
+    async def serialize_events(
        self,
        events: Iterable[Union[JsonDict, EventBase]],
        time_now: int,
@@ -645,7 +671,7 @@ class EventClientSerializer:
            The list of serialized events
        """
        return [
-            self.serialize_event(
+            await self.serialize_event(
                event,
                time_now,
                config=config,
@@ -654,6 +680,14 @@ class EventClientSerializer:
            for event in events
        ]

+    def register_add_extra_fields_to_unsigned_client_event_callback(
+        self, callback: ADD_EXTRA_FIELDS_TO_UNSIGNED_CLIENT_EVENT_CALLBACK
+    ) -> None:
+        """Register a callback that returns additions to the unsigned section of
+        serialized events.
+        """
+        self._add_extra_fields_to_unsigned_client_event_callbacks.append(callback)
+

 _PowerLevel = Union[str, int]
 PowerLevelsContent = Mapping[str, Union[_PowerLevel, Mapping[str, _PowerLevel]]]
@@ -84,7 +84,7 @@ from synapse.replication.http.federation import (
 from synapse.storage.databases.main.lock import Lock
 from synapse.storage.databases.main.roommember import extract_heroes_from_room_summary
 from synapse.storage.roommember import MemberSummary
-from synapse.types import JsonDict, StateMap, get_domain_from_id
+from synapse.types import JsonDict, StateMap, UserID, get_domain_from_id
 from synapse.util import unwrapFirstError
 from synapse.util.async_helpers import Linearizer, concurrently_execute, gather_results
 from synapse.util.caches.response_cache import ResponseCache
@@ -999,6 +999,12 @@ class FederationServer(FederationBase):
    async def on_claim_client_keys(
        self, query: List[Tuple[str, str, str, int]], always_include_fallback_keys: bool
    ) -> Dict[str, Any]:
+        if any(
+            not self.hs.is_mine(UserID.from_string(user_id))
+            for user_id, _, _, _ in query
+        ):
+            raise SynapseError(400, "User is not hosted on this homeserver")
+
        log_kv({"message": "Claiming one time keys.", "user, device pairs": query})
        results = await self._e2e_keys_handler.claim_local_one_time_keys(
            query, always_include_fallback_keys=always_include_fallback_keys
@@ -1395,7 +1401,7 @@ class FederationHandlerRegistry:
        self._edu_type_to_instance[edu_type] = instance_names

    async def on_edu(self, edu_type: str, origin: str, content: dict) -> None:
-        if not self.config.server.use_presence and edu_type == EduTypes.PRESENCE:
+        if not self.config.server.track_presence and edu_type == EduTypes.PRESENCE:
            return

        # Check if we have a handler on this instance
@@ -844,7 +844,7 @@ class FederationSender(AbstractFederationSender):
        destinations (list[str])
        """

-        if not states or not self.hs.config.server.use_presence:
+        if not states or not self.hs.config.server.track_presence:
            # No-op if presence is disabled.
            return

@@ -283,7 +283,7 @@ class AdminHandler:
                start, limit, user_id
            )
            for media in media_ids:
-                writer.write_media_id(media["media_id"], media)
+                writer.write_media_id(media.media_id, attr.asdict(media))

            logger.info(
                "[%s] Written %d media_ids of %s",
@@ -47,6 +47,7 @@ from synapse.types import (
    DeviceListUpdates,
    JsonDict,
    JsonMapping,
+    MultiWriterStreamToken,
    RoomAlias,
    RoomStreamToken,
    StreamKeyType,
@@ -217,7 +218,7 @@ class ApplicationServicesHandler:
    def notify_interested_services_ephemeral(
        self,
        stream_key: StreamKeyType,
-        new_token: Union[int, RoomStreamToken],
+        new_token: Union[int, RoomStreamToken, MultiWriterStreamToken],
        users: Collection[Union[str, UserID]],
    ) -> None:
        """
@@ -259,19 +260,6 @@ class ApplicationServicesHandler:
        ):
            return

-        # Assert that new_token is an integer (and not a RoomStreamToken).
-        # All of the supported streams that this function handles use an
-        # integer to track progress (rather than a RoomStreamToken - a
-        # vector clock implementation) as they don't support multiple
-        # stream writers.
-        #
-        # As a result, we simply assert that new_token is an integer.
-        # If we do end up needing to pass a RoomStreamToken down here
-        # in the future, using RoomStreamToken.stream (the minimum stream
-        # position) to convert to an ascending integer value should work.
-        # Additional context: https://github.com/matrix-org/synapse/pull/11137
-        assert isinstance(new_token, int)
-
        # Ignore to-device messages if the feature flag is not enabled
        if (
            stream_key == StreamKeyType.TO_DEVICE
@@ -286,6 +274,9 @@ class ApplicationServicesHandler:
        ):
            return

+        # We know we're not a `RoomStreamToken` at this point.
+        assert not isinstance(new_token, RoomStreamToken)
+
        # Check whether there are any appservices which have registered to receive
        # ephemeral events.
        #
@@ -327,7 +318,7 @@ class ApplicationServicesHandler:
        self,
        services: List[ApplicationService],
        stream_key: StreamKeyType,
-        new_token: int,
+        new_token: Union[int, MultiWriterStreamToken],
        users: Collection[Union[str, UserID]],
    ) -> None:
        logger.debug("Checking interested services for %s", stream_key)
@@ -340,6 +331,7 @@ class ApplicationServicesHandler:
                    #
                    # Instead we simply grab the latest typing updates in _handle_typing
                    # and, if they apply to this application service, send it off.
+                    assert isinstance(new_token, int)
                    events = await self._handle_typing(service, new_token)
                    if events:
                        self.scheduler.enqueue_for_appservice(service, ephemeral=events)
@@ -350,15 +342,23 @@ class ApplicationServicesHandler:
                    (service.id, stream_key)
                ):
                    if stream_key == StreamKeyType.RECEIPT:
+                        assert isinstance(new_token, MultiWriterStreamToken)
+
+                        # We store appservice tokens as integers, so we ignore
+                        # the `instance_map` components and instead simply
+                        # follow the base stream position.
+                        new_token = MultiWriterStreamToken(stream=new_token.stream)
+
                        events = await self._handle_receipts(service, new_token)
                        self.scheduler.enqueue_for_appservice(service, ephemeral=events)

                        # Persist the latest handled stream token for this appservice
                        await self.store.set_appservice_stream_type_pos(
-                            service, "read_receipt", new_token
+                            service, "read_receipt", new_token.stream
                        )

                    elif stream_key == StreamKeyType.PRESENCE:
+                        assert isinstance(new_token, int)
                        events = await self._handle_presence(service, users, new_token)
                        self.scheduler.enqueue_for_appservice(service, ephemeral=events)

@@ -368,6 +368,7 @@ class ApplicationServicesHandler:
                        )

                    elif stream_key == StreamKeyType.TO_DEVICE:
+                        assert isinstance(new_token, int)
                        # Retrieve a list of to-device message events, as well as the
                        # maximum stream token of the messages we were able to retrieve.
                        to_device_messages = await self._get_to_device_messages(
@@ -383,6 +384,7 @@ class ApplicationServicesHandler:
                        )

                    elif stream_key == StreamKeyType.DEVICE_LIST:
+                        assert isinstance(new_token, int)
                        device_list_summary = await self._get_device_list_summary(
                            service, new_token
                        )
@@ -432,7 +434,7 @@ class ApplicationServicesHandler:
        return typing

    async def _handle_receipts(
-        self, service: ApplicationService, new_token: int
+        self, service: ApplicationService, new_token: MultiWriterStreamToken
    ) -> List[JsonMapping]:
        """
        Return the latest read receipts that the given application service should receive.
@@ -455,15 +457,17 @@ class ApplicationServicesHandler:
        from_key = await self.store.get_type_stream_id_for_appservice(
            service, "read_receipt"
        )
-        if new_token is not None and new_token <= from_key:
+        if new_token is not None and new_token.stream <= from_key:
            logger.debug(
                "Rejecting token lower than or equal to stored: %s" % (new_token,)
            )
            return []

+        from_token = MultiWriterStreamToken(stream=from_key)
+
        receipts_source = self.event_sources.sources.receipt
        receipts, _ = await receipts_source.get_new_events_as(
-            service=service, from_key=from_key, to_key=new_token
+            service=service, from_key=from_token, to_key=new_token
        )
        return receipts

@@ -103,10 +103,10 @@ class DeactivateAccountHandler:
        # Attempt to unbind any known bound threepids to this account from identity
        # server(s).
        bound_threepids = await self.store.user_get_bound_threepids(user_id)
-        for threepid in bound_threepids:
+        for medium, address in bound_threepids:
            try:
                result = await self._identity_handler.try_unbind_threepid(
-                    user_id, threepid["medium"], threepid["address"], id_server
+                    user_id, medium, address, id_server
                )
            except Exception:
                # Do we want this to be a fatal error or should we carry on?
@@ -328,6 +328,9 @@ class DeviceWorkerHandler:
        return result

    async def on_federation_query_user_devices(self, user_id: str) -> JsonDict:
+        if not self.hs.is_mine(UserID.from_string(user_id)):
+            raise SynapseError(400, "User is not hosted on this homeserver")
+
        stream_id, devices = await self.store.get_e2e_device_keys_for_federation_query(
            user_id
        )
@@ -542,6 +542,12 @@ class E2eKeysHandler:
        device_keys_query: Dict[str, Optional[List[str]]] = query_body.get(
            "device_keys", {}
        )
+        if any(
+            not self.is_mine(UserID.from_string(user_id))
+            for user_id in device_keys_query
+        ):
+            raise SynapseError(400, "User is not hosted on this homeserver")
+
        res = await self.query_local_devices(
            device_keys_query,
            include_displaynames=(
@@ -659,6 +665,20 @@ class E2eKeysHandler:
        timeout: Optional[int],
        always_include_fallback_keys: bool,
    ) -> JsonDict:
+        """
+        Args:
+            query: A chain of maps from (user_id, device_id, algorithm) to the requested
+                number of keys to claim.
+            user: The user who is claiming these keys.
+            timeout: How long to wait for any federation key claim requests before
+                giving up.
+            always_include_fallback_keys: always include a fallback key for local users'
+                devices, even if we managed to claim a one-time-key.
+
+        Returns: a heterogeneous dict with two keys:
+            one_time_keys: chain of maps user ID -> device ID -> key ID -> key.
+            failures: map from remote destination to a JsonDict describing the error.
+        """
        local_query: List[Tuple[str, str, str, int]] = []
        remote_queries: Dict[str, Dict[str, Dict[str, Dict[str, int]]]] = {}

@@ -739,6 +759,16 @@ class E2eKeysHandler:
    async def upload_keys_for_user(
        self, user_id: str, device_id: str, keys: JsonDict
    ) -> JsonDict:
+        """
+        Args:
+            user_id: user whose keys are being uploaded.
+            device_id: device whose keys are being uploaded.
+            keys: the body of a /keys/upload request.
+
+        Returns a dictionary with one field:
+            "one_time_keys": A mapping from algorithm to number of keys for that
+                algorithm, including those previously persisted.
+        """
        # This can only be called from the main process.
        assert isinstance(self.device_handler, DeviceHandler)

@@ -120,7 +120,7 @@ class EventStreamHandler:

            events.extend(to_add)

-            chunks = self._event_serializer.serialize_events(
+            chunks = await self._event_serializer.serialize_events(
                events,
                time_now,
                config=SerializeEventConfig(
@@ -19,6 +19,8 @@ import logging
 import urllib.parse
 from typing import TYPE_CHECKING, Awaitable, Callable, Dict, List, Optional, Tuple

+import attr
+
 from synapse.api.errors import (
    CodeMessageException,
    Codes,
@@ -357,9 +359,9 @@ class IdentityHandler:

        # Check to see if a session already exists and that it is not yet
        # marked as validated
-        if session and session.get("validated_at") is None:
-            session_id = session["session_id"]
-            last_send_attempt = session["last_send_attempt"]
+        if session and session.validated_at is None:
+            session_id = session.session_id
+            last_send_attempt = session.last_send_attempt

            # Check that the send_attempt is higher than previous attempts
            if send_attempt <= last_send_attempt:
@@ -480,7 +482,6 @@ class IdentityHandler:

        # We don't actually know which medium this 3PID is. Thus we first assume it's email,
        # and if validation fails we try msisdn
-        validation_session = None

        # Try to validate as email
        if self.hs.config.email.can_verify_email:
@@ -488,19 +489,18 @@ class IdentityHandler:
            validation_session = await self.store.get_threepid_validation_session(
                "email", client_secret, sid=sid, validated=True
            )
-
-        if validation_session:
-            return validation_session
+            if validation_session:
+                return attr.asdict(validation_session)

        # Try to validate as msisdn
        if self.hs.config.registration.account_threepid_delegate_msisdn:
            # Ask our delegated msisdn identity server
-            validation_session = await self.threepid_from_creds(
+            return await self.threepid_from_creds(
                self.hs.config.registration.account_threepid_delegate_msisdn,
                threepid_creds,
            )

-        return validation_session
+        return None

    async def proxy_msisdn_submit_token(
        self, id_server: str, client_secret: str, sid: str, token: str
@@ -145,7 +145,7 @@ class InitialSyncHandler:
        joined_rooms = [r.room_id for r in room_list if r.membership == Membership.JOIN]
        receipt = await self.store.get_linearized_receipts_for_rooms(
            joined_rooms,
-            to_key=int(now_token.receipt_key),
+            to_key=now_token.receipt_key,
        )

        receipt = ReceiptEventSource.filter_out_private_receipts(receipt, user_id)
@@ -173,7 +173,7 @@ class InitialSyncHandler:
                d["inviter"] = event.sender

                invite_event = await self.store.get_event(event.event_id)
-                d["invite"] = self._event_serializer.serialize_event(
+                d["invite"] = await self._event_serializer.serialize_event(
                    invite_event,
                    time_now,
                    config=serializer_options,
@@ -225,7 +225,7 @@ class InitialSyncHandler:

                d["messages"] = {
                    "chunk": (
-                        self._event_serializer.serialize_events(
+                        await self._event_serializer.serialize_events(
                            messages,
                            time_now=time_now,
                            config=serializer_options,
@@ -235,7 +235,7 @@ class InitialSyncHandler:
                    "end": await end_token.to_string(self.store),
                }

-                d["state"] = self._event_serializer.serialize_events(
+                d["state"] = await self._event_serializer.serialize_events(
                    current_state.values(),
                    time_now=time_now,
                    config=serializer_options,
@@ -387,7 +387,7 @@ class InitialSyncHandler:
            "messages": {
                "chunk": (
                    # Don't bundle aggregations as this is a deprecated API.
-                    self._event_serializer.serialize_events(
+                    await self._event_serializer.serialize_events(
                        messages, time_now, config=serialize_options
                    )
                ),
@@ -396,7 +396,7 @@ class InitialSyncHandler:
            },
            "state": (
                # Don't bundle aggregations as this is a deprecated API.
-                self._event_serializer.serialize_events(
+                await self._event_serializer.serialize_events(
                    room_state.values(), time_now, config=serialize_options
                )
            ),
@@ -420,7 +420,7 @@ class InitialSyncHandler:
        time_now = self.clock.time_msec()
        serialize_options = SerializeEventConfig(requester=requester)
        # Don't bundle aggregations as this is a deprecated API.
-        state = self._event_serializer.serialize_events(
+        state = await self._event_serializer.serialize_events(
            current_state.values(),
            time_now,
            config=serialize_options,
@@ -439,7 +439,7 @@ class InitialSyncHandler:

        async def get_presence() -> List[JsonDict]:
            # If presence is disabled, return an empty list
-            if not self.hs.config.server.use_presence:
+            if not self.hs.config.server.presence_enabled:
                return []

            states = await presence_handler.get_states(
@@ -497,7 +497,7 @@ class InitialSyncHandler:
            "messages": {
                "chunk": (
                    # Don't bundle aggregations as this is a deprecated API.
-                    self._event_serializer.serialize_events(
+                    await self._event_serializer.serialize_events(
                        messages, time_now, config=serialize_options
                    )
                ),
@@ -244,7 +244,7 @@ class MessageHandler:
                )
                room_state = room_state_events[membership_event_id]

-        events = self._event_serializer.serialize_events(
+        events = await self._event_serializer.serialize_events(
            room_state.values(),
            self.clock.time_msec(),
            config=SerializeEventConfig(requester=requester),
@@ -999,7 +999,26 @@ class EventCreationHandler:
            raise ShadowBanError()

        if ratelimit:
-            await self.request_ratelimiter.ratelimit(requester, update=False)
+            room_id = event_dict["room_id"]
+            try:
+                room_version = await self.store.get_room_version(room_id)
+            except NotFoundError:
+                # The room doesn't exist.
+                raise AuthError(403, f"User {requester.user} not in room {room_id}")
+
+            if room_version.updated_redaction_rules:
+                redacts = event_dict["content"].get("redacts")
+            else:
+                redacts = event_dict.get("redacts")
+
+            is_admin_redaction = await self.is_admin_redaction(
+                event_type=event_dict["type"],
+                sender=event_dict["sender"],
+                redacts=redacts,
+            )
+            await self.request_ratelimiter.ratelimit(
+                requester, is_admin_redaction=is_admin_redaction, update=False
+            )

        # We limit the number of concurrent event sends in a room so that we
        # don't fork the DAG too much. If we don't limit then we can end up in
@@ -1508,6 +1527,18 @@ class EventCreationHandler:
                first_event.room_id
            )
            if writer_instance != self._instance_name:
+                # Ratelimit before sending to the other event persister, to
+                # ensure that we correctly have ratelimits on both the event
+                # creators and event persisters.
+                if ratelimit:
+                    for event, _ in events_and_context:
+                        is_admin_redaction = await self.is_admin_redaction(
+                            event.type, event.sender, event.redacts
+                        )
+                        await self.request_ratelimiter.ratelimit(
+                            requester, is_admin_redaction=is_admin_redaction
+                        )
+
                try:
                    result = await self.send_events(
                        instance_name=writer_instance,
@@ -1538,6 +1569,7 @@ class EventCreationHandler:
                    # stream_ordering entry manually (as it was persisted on
                    # another worker).
                    event.internal_metadata.stream_ordering = stream_id
+
                return event

            event = await self.persist_and_notify_client_events(
@@ -1696,21 +1728,9 @@ class EventCreationHandler:
                # can apply different ratelimiting. We do this by simply checking
                # it's not a self-redaction (to avoid having to look up whether the
                # user is actually admin or not).
-                is_admin_redaction = False
-                if event.type == EventTypes.Redaction:
-                    assert event.redacts is not None
-
-                    original_event = await self.store.get_event(
-                        event.redacts,
-                        redact_behaviour=EventRedactBehaviour.as_is,
-                        get_prev_content=False,
-                        allow_rejected=False,
-                        allow_none=True,
-                    )
-
-                    is_admin_redaction = bool(
-                        original_event and event.sender != original_event.sender
-                    )
+                is_admin_redaction = await self.is_admin_redaction(
+                    event.type, event.sender, event.redacts
+                )

                await self.request_ratelimiter.ratelimit(
                    requester, is_admin_redaction=is_admin_redaction
@@ -1930,6 +1950,27 @@ class EventCreationHandler:

        return persisted_events[-1]

+    async def is_admin_redaction(
+        self, event_type: str, sender: str, redacts: Optional[str]
+    ) -> bool:
+        """Return whether the event is a redaction made by an admin, and thus
+        should use a different ratelimiter.
+        """
+        if event_type != EventTypes.Redaction:
+            return False
+
+        assert redacts is not None
+
+        original_event = await self.store.get_event(
+            redacts,
+            redact_behaviour=EventRedactBehaviour.as_is,
+            get_prev_content=False,
+            allow_rejected=False,
+            allow_none=True,
+        )
+
+        return bool(original_event and sender != original_event.sender)
+
    async def _maybe_kick_guest_users(
        self, event: EventBase, context: EventContext
    ) -> None:
@@ -657,7 +657,7 @@ class PaginationHandler:

        chunk = {
            "chunk": (
-                self._event_serializer.serialize_events(
+                await self._event_serializer.serialize_events(
                    events,
                    time_now,
                    config=serialize_options,
@@ -669,7 +669,7 @@ class PaginationHandler:
        }

        if state:
-            chunk["state"] = self._event_serializer.serialize_events(
+            chunk["state"] = await self._event_serializer.serialize_events(
                state, time_now, config=serialize_options
            )

@@ -192,7 +192,8 @@ class BasePresenceHandler(abc.ABC):
        self.state = hs.get_state_handler()
        self.is_mine_id = hs.is_mine_id

-        self._presence_enabled = hs.config.server.use_presence
+        self._presence_enabled = hs.config.server.presence_enabled
+        self._track_presence = hs.config.server.track_presence

        self._federation = None
        if hs.should_send_federation():
@@ -512,7 +513,7 @@ class WorkerPresenceHandler(BasePresenceHandler):
        )

    async def _on_shutdown(self) -> None:
-        if self._presence_enabled:
+        if self._track_presence:
            self.hs.get_replication_command_handler().send_command(
                ClearUserSyncsCommand(self.instance_id)
            )
@@ -524,7 +525,7 @@ class WorkerPresenceHandler(BasePresenceHandler):
        is_syncing: bool,
        last_sync_ms: int,
    ) -> None:
-        if self._presence_enabled:
+        if self._track_presence:
            self.hs.get_replication_command_handler().send_user_sync(
                self.instance_id, user_id, device_id, is_syncing, last_sync_ms
            )
@@ -571,7 +572,7 @@ class WorkerPresenceHandler(BasePresenceHandler):
        Called by the sync and events servlets to record that a user has connected to
        this worker and is waiting for some events.
        """
-        if not affect_presence or not self._presence_enabled:
+        if not affect_presence or not self._track_presence:
            return _NullContextManager()

        # Note that this causes last_active_ts to be incremented which is not
@@ -702,8 +703,8 @@ class WorkerPresenceHandler(BasePresenceHandler):

        user_id = target_user.to_string()

-        # If presence is disabled, no-op
-        if not self._presence_enabled:
+        # If tracking of presence is disabled, no-op
+        if not self._track_presence:
            return

        # Proxy request to instance that writes presence
@@ -723,7 +724,7 @@ class WorkerPresenceHandler(BasePresenceHandler):
        with the app.
        """
        # If presence is disabled, no-op
-        if not self._presence_enabled:
+        if not self._track_presence:
            return

        # Proxy request to instance that writes presence
@@ -760,7 +761,7 @@ class PresenceHandler(BasePresenceHandler):
        ] = {}

        now = self.clock.time_msec()
-        if self._presence_enabled:
+        if self._track_presence:
            for state in self.user_to_current_state.values():
                # Create a psuedo-device to properly handle time outs. This will
                # be overridden by any "real" devices within SYNC_ONLINE_TIMEOUT.
@@ -831,7 +832,7 @@ class PresenceHandler(BasePresenceHandler):

        self.external_sync_linearizer = Linearizer(name="external_sync_linearizer")

-        if self._presence_enabled:
+        if self._track_presence:
            # Start a LoopingCall in 30s that fires every 5s.
            # The initial delay is to allow disconnected clients a chance to
            # reconnect before we treat them as offline.
@@ -839,6 +840,9 @@ class PresenceHandler(BasePresenceHandler):
                30, self.clock.looping_call, self._handle_timeouts, 5000
            )

+        # Presence information is persisted, whether or not it is being tracked
+        # internally.
+        if self._presence_enabled:
            self.clock.call_later(
                60,
                self.clock.looping_call,
@@ -854,7 +858,7 @@ class PresenceHandler(BasePresenceHandler):
        )

        # Used to handle sending of presence to newly joined users/servers
-        if self._presence_enabled:
+        if self._track_presence:
            self.notifier.add_replication_callback(self.notify_new_event)

        # Presence is best effort and quickly heals itself, so lets just always
@@ -905,7 +909,9 @@ class PresenceHandler(BasePresenceHandler):
            )

    async def _update_states(
-        self, new_states: Iterable[UserPresenceState], force_notify: bool = False
+        self,
+        new_states: Iterable[UserPresenceState],
+        force_notify: bool = False,
    ) -> None:
        """Updates presence of users. Sets the appropriate timeouts. Pokes
        the notifier and federation if and only if the changed presence state
@@ -943,7 +949,7 @@ class PresenceHandler(BasePresenceHandler):
            for new_state in new_states:
                user_id = new_state.user_id

-                # Its fine to not hit the database here, as the only thing not in
+                # It's fine to not hit the database here, as the only thing not in
                # the current state cache are OFFLINE states, where the only field
                # of interest is last_active which is safe enough to assume is 0
                # here.
@@ -957,6 +963,9 @@ class PresenceHandler(BasePresenceHandler):
                    is_mine=self.is_mine_id(user_id),
                    wheel_timer=self.wheel_timer,
                    now=now,
+                    # When overriding disabled presence, don't kick off all the
+                    # wheel timers.
+                    persist=not self._track_presence,
                )

                if force_notify:
@@ -1072,7 +1081,7 @@ class PresenceHandler(BasePresenceHandler):
        with the app.
        """
        # If presence is disabled, no-op
-        if not self._presence_enabled:
+        if not self._track_presence:
            return

        user_id = user.to_string()
@@ -1124,7 +1133,7 @@ class PresenceHandler(BasePresenceHandler):
                client that is being used by a user.
            presence_state: The presence state indicated in the sync request
        """
-        if not affect_presence or not self._presence_enabled:
+        if not affect_presence or not self._track_presence:
            return _NullContextManager()

        curr_sync = self._user_device_to_num_current_syncs.get((user_id, device_id), 0)
@@ -1284,7 +1293,7 @@ class PresenceHandler(BasePresenceHandler):

    async def incoming_presence(self, origin: str, content: JsonDict) -> None:
        """Called when we receive a `m.presence` EDU from a remote server."""
-        if not self._presence_enabled:
+        if not self._track_presence:
            return

        now = self.clock.time_msec()
@@ -1359,7 +1368,7 @@ class PresenceHandler(BasePresenceHandler):
            raise SynapseError(400, "Invalid presence state")

        # If presence is disabled, no-op
-        if not self._presence_enabled:
+        if not self._track_presence:
            return

        user_id = target_user.to_string()
@@ -2118,6 +2127,7 @@ def handle_update(
    is_mine: bool,
    wheel_timer: WheelTimer,
    now: int,
+    persist: bool,
 ) -> Tuple[UserPresenceState, bool, bool]:
    """Given a presence update:
        1. Add any appropriate timers.
@@ -2129,6 +2139,8 @@ def handle_update(
        is_mine: Whether the user is ours
        wheel_timer
        now: Time now in ms
+        persist: True if this state should persist until another update occurs.
+            Skips insertion into wheel timers.

    Returns:
        3-tuple: `(new_state, persist_and_notify, federation_ping)` where:
@@ -2146,14 +2158,15 @@ def handle_update(
    if is_mine:
        if new_state.state == PresenceState.ONLINE:
            # Idle timer
-            wheel_timer.insert(
-                now=now, obj=user_id, then=new_state.last_active_ts + IDLE_TIMER
-            )
+            if not persist:
+                wheel_timer.insert(
+                    now=now, obj=user_id, then=new_state.last_active_ts + IDLE_TIMER
+                )

            active = now - new_state.last_active_ts < LAST_ACTIVE_GRANULARITY
            new_state = new_state.copy_and_replace(currently_active=active)

-            if active:
+            if active and not persist:
                wheel_timer.insert(
                    now=now,
                    obj=user_id,
@@ -2162,11 +2175,12 @@ def handle_update(

        if new_state.state != PresenceState.OFFLINE:
            # User has stopped syncing
-            wheel_timer.insert(
-                now=now,
-                obj=user_id,
-                then=new_state.last_user_sync_ts + SYNC_ONLINE_TIMEOUT,
-            )
+            if not persist:
+                wheel_timer.insert(
+                    now=now,
+                    obj=user_id,
+                    then=new_state.last_user_sync_ts + SYNC_ONLINE_TIMEOUT,
+                )

            last_federate = new_state.last_federation_update_ts
            if now - last_federate > FEDERATION_PING_INTERVAL:
@@ -2174,7 +2188,7 @@ def handle_update(
                new_state = new_state.copy_and_replace(last_federation_update_ts=now)
                federation_ping = True

-        if new_state.state == PresenceState.BUSY:
+        if new_state.state == PresenceState.BUSY and not persist:
            wheel_timer.insert(
                now=now,
                obj=user_id,
@@ -2182,11 +2196,13 @@ def handle_update(
            )

    else:
-        wheel_timer.insert(
-            now=now,
-            obj=user_id,
-            then=new_state.last_federation_update_ts + FEDERATION_TIMEOUT,
-        )
+        # An update for a remote user was received.
+        if not persist:
+            wheel_timer.insert(
+                now=now,
+                obj=user_id,
+                then=new_state.last_federation_update_ts + FEDERATION_TIMEOUT,
+            )

    # Check whether the change was something worth notifying about
    if should_notify(prev_state, new_state, is_mine):
@@ -13,7 +13,7 @@
 # limitations under the License.
 import logging
 import random
-from typing import TYPE_CHECKING, Optional
+from typing import TYPE_CHECKING, Optional, Union

 from synapse.api.errors import (
    AuthError,
@@ -23,6 +23,7 @@ from synapse.api.errors import (
    StoreError,
    SynapseError,
 )
+from synapse.storage.databases.main.media_repository import LocalMedia, RemoteMedia
 from synapse.types import JsonDict, Requester, UserID, create_requester
 from synapse.util.caches.descriptors import cached
 from synapse.util.stringutils import parse_and_validate_mxc_uri
@@ -306,7 +307,9 @@ class ProfileHandler:
            server_name = host

        if self._is_mine_server_name(server_name):
-            media_info = await self.store.get_local_media(media_id)
+            media_info: Optional[
+                Union[LocalMedia, RemoteMedia]
+            ] = await self.store.get_local_media(media_id)
        else:
            media_info = await self.store.get_cached_remote_media(server_name, media_id)

@@ -322,12 +325,12 @@ class ProfileHandler:

        if self.max_avatar_size:
            # Ensure avatar does not exceed max allowed avatar size
-            if media_info["media_length"] > self.max_avatar_size:
+            if media_info.media_length > self.max_avatar_size:
                logger.warning(
                    "Forbidding avatar change to %s: %d bytes is above the allowed size "
                    "limit",
                    mxc,
-                    media_info["media_length"],
+                    media_info.media_length,
                )
                return False

@@ -335,12 +338,12 @@ class ProfileHandler:
            # Ensure the avatar's file type is allowed
            if (
                self.allowed_avatar_mimetypes
-                and media_info["media_type"] not in self.allowed_avatar_mimetypes
+                and media_info.media_type not in self.allowed_avatar_mimetypes
            ):
                logger.warning(
                    "Forbidding avatar change to %s: mimetype %s not allowed",
                    mxc,
-                    media_info["media_type"],
+                    media_info.media_type,
                )
                return False

@@ -20,6 +20,7 @@ from synapse.streams import EventSource
 from synapse.types import (
    JsonDict,
    JsonMapping,
+    MultiWriterStreamToken,
    ReadReceipt,
    StreamKeyType,
    UserID,
@@ -200,7 +201,7 @@ class ReceiptsHandler:
            await self.federation_sender.send_read_receipt(receipt)


-class ReceiptEventSource(EventSource[int, JsonMapping]):
+class ReceiptEventSource(EventSource[MultiWriterStreamToken, JsonMapping]):
    def __init__(self, hs: "HomeServer"):
        self.store = hs.get_datastores().main
        self.config = hs.config
@@ -273,13 +274,12 @@ class ReceiptEventSource(EventSource[int, JsonMapping]):
    async def get_new_events(
        self,
        user: UserID,
-        from_key: int,
+        from_key: MultiWriterStreamToken,
        limit: int,
        room_ids: Iterable[str],
        is_guest: bool,
        explicit_room_id: Optional[str] = None,
-    ) -> Tuple[List[JsonMapping], int]:
-        from_key = int(from_key)
+    ) -> Tuple[List[JsonMapping], MultiWriterStreamToken]:
        to_key = self.get_current_key()

        if from_key == to_key:
@@ -296,8 +296,11 @@ class ReceiptEventSource(EventSource[int, JsonMapping]):
        return events, to_key

    async def get_new_events_as(
-        self, from_key: int, to_key: int, service: ApplicationService
-    ) -> Tuple[List[JsonMapping], int]:
+        self,
+        from_key: MultiWriterStreamToken,
+        to_key: MultiWriterStreamToken,
+        service: ApplicationService,
+    ) -> Tuple[List[JsonMapping], MultiWriterStreamToken]:
        """Returns a set of new read receipt events that an appservice
        may be interested in.

@@ -312,8 +315,6 @@ class ReceiptEventSource(EventSource[int, JsonMapping]):
                  appservice may be interested in.
                * The current read receipt stream token.
        """
-        from_key = int(from_key)
-
        if from_key == to_key:
            return [], to_key

@@ -333,5 +334,5 @@ class ReceiptEventSource(EventSource[int, JsonMapping]):

        return events, to_key

-    def get_current_key(self) -> int:
+    def get_current_key(self) -> MultiWriterStreamToken:
        return self.store.get_max_receipt_stream_id()
@@ -167,7 +167,7 @@ class RelationsHandler:
        now = self._clock.time_msec()
        serialize_options = SerializeEventConfig(requester=requester)
        return_value: JsonDict = {
-            "chunk": self._event_serializer.serialize_events(
+            "chunk": await self._event_serializer.serialize_events(
                events,
                now,
                bundle_aggregations=aggregations,
@@ -177,7 +177,9 @@ class RelationsHandler:
        if include_original_event:
            # Do not bundle aggregations when retrieving the original event because
            # we want the content before relations are applied to it.
-            return_value["original_event"] = self._event_serializer.serialize_event(
+            return_value[
+                "original_event"
+            ] = await self._event_serializer.serialize_event(
                event,
                now,
                bundle_aggregations=None,
@@ -602,7 +604,7 @@ class RelationsHandler:
        )

        now = self._clock.time_msec()
-        serialized_events = self._event_serializer.serialize_events(
+        serialized_events = await self._event_serializer.serialize_events(
            events, now, bundle_aggregations=aggregations
        )

@@ -269,7 +269,7 @@ class RoomCreationHandler:
        self,
        requester: Requester,
        old_room_id: str,
-        old_room: Dict[str, Any],
+        old_room: Tuple[bool, str, bool],
        new_room_id: str,
        new_version: RoomVersion,
        tombstone_event: EventBase,
@@ -279,7 +279,7 @@ class RoomCreationHandler:
        Args:
            requester: the user requesting the upgrade
            old_room_id: the id of the room to be replaced
-            old_room: a dict containing room information for the room to be replaced,
+            old_room: a tuple containing room information for the room to be replaced,
                as returned by `RoomWorkerStore.get_room`.
            new_room_id: the id of the replacement room
            new_version: the version to upgrade the room to
@@ -299,7 +299,7 @@ class RoomCreationHandler:
        await self.store.store_room(
            room_id=new_room_id,
            room_creator_user_id=user_id,
-            is_public=old_room["is_public"],
+            is_public=old_room[0],
            room_version=new_version,
        )

@@ -1939,9 +1939,10 @@ class RoomShutdownHandler:
        else:
            logger.info("Shutting down room %r", room_id)

-        users = await self.store.get_users_in_room(room_id)
-        for user_id in users:
-            if not self.hs.is_mine_id(user_id):
+        users = await self.store.get_local_users_related_to_room(room_id)
+        for user_id, membership in users:
+            # If the user is not in the room (or is banned), nothing to do.
+            if membership not in (Membership.JOIN, Membership.INVITE, Membership.KNOCK):
                continue

            logger.info("Kicking %r from %r...", user_id, room_id)
@@ -33,6 +33,7 @@ from synapse.api.errors import (
    RequestSendFailed,
    SynapseError,
 )
+from synapse.storage.databases.main.room import LargestRoomStats
 from synapse.types import JsonDict, JsonMapping, ThirdPartyInstanceID
 from synapse.util.caches.descriptors import _CacheContext, cached
 from synapse.util.caches.response_cache import ResponseCache
@@ -170,26 +171,24 @@ class RoomListHandler:
            ignore_non_federatable=from_federation,
        )

-        def build_room_entry(room: JsonDict) -> JsonDict:
+        def build_room_entry(room: LargestRoomStats) -> JsonDict:
            entry = {
-                "room_id": room["room_id"],
-                "name": room["name"],
-                "topic": room["topic"],
-                "canonical_alias": room["canonical_alias"],
-                "num_joined_members": room["joined_members"],
-                "avatar_url": room["avatar"],
-                "world_readable": room["history_visibility"]
+                "room_id": room.room_id,
+                "name": room.name,
+                "topic": room.topic,
+                "canonical_alias": room.canonical_alias,
+                "num_joined_members": room.joined_members,
+                "avatar_url": room.avatar,
+                "world_readable": room.history_visibility
                == HistoryVisibility.WORLD_READABLE,
-                "guest_can_join": room["guest_access"] == "can_join",
-                "join_rule": room["join_rules"],
-                "room_type": room["room_type"],
+                "guest_can_join": room.guest_access == "can_join",
+                "join_rule": room.join_rules,
+                "room_type": room.room_type,
            }

            # Filter out Nones – rather omit the field altogether
            return {k: v for k, v in entry.items() if v is not None}

-        results = [build_room_entry(r) for r in results]
-
        response: JsonDict = {}
        num_results = len(results)
        if limit is not None:
@@ -212,33 +211,33 @@ class RoomListHandler:
                    # If there was a token given then we assume that there
                    # must be previous results.
                    response["prev_batch"] = RoomListNextBatch(
-                        last_joined_members=initial_entry["num_joined_members"],
-                        last_room_id=initial_entry["room_id"],
+                        last_joined_members=initial_entry.joined_members,
+                        last_room_id=initial_entry.room_id,
                        direction_is_forward=False,
                    ).to_token()

                if more_to_come:
                    response["next_batch"] = RoomListNextBatch(
-                        last_joined_members=final_entry["num_joined_members"],
-                        last_room_id=final_entry["room_id"],
+                        last_joined_members=final_entry.joined_members,
+                        last_room_id=final_entry.room_id,
                        direction_is_forward=True,
                    ).to_token()
            else:
                if has_batch_token:
                    response["next_batch"] = RoomListNextBatch(
-                        last_joined_members=final_entry["num_joined_members"],
-                        last_room_id=final_entry["room_id"],
+                        last_joined_members=final_entry.joined_members,
+                        last_room_id=final_entry.room_id,
                        direction_is_forward=True,
                    ).to_token()

                if more_to_come:
                    response["prev_batch"] = RoomListNextBatch(
-                        last_joined_members=initial_entry["num_joined_members"],
-                        last_room_id=initial_entry["room_id"],
+                        last_joined_members=initial_entry.joined_members,
+                        last_room_id=initial_entry.room_id,
                        direction_is_forward=False,
                    ).to_token()

-        response["chunk"] = results
+        response["chunk"] = [build_room_entry(r) for r in results]

        response["total_room_count_estimate"] = await self.store.count_public_rooms(
            network_tuple,
@@ -1260,7 +1260,8 @@ class RoomMemberHandler(metaclass=abc.ABCMeta):
        # Add new room to the room directory if the old room was there
        # Remove old room from the room directory
        old_room = await self.store.get_room(old_room_id)
-        if old_room is not None and old_room["is_public"]:
+        # If the old room exists and is public.
+        if old_room is not None and old_room[0]:
            await self.store.set_room_is_public(old_room_id, False)
            await self.store.set_room_is_public(room_id, True)

@@ -703,24 +703,24 @@ class RoomSummaryHandler:
        # there should always be an entry
        assert stats is not None, "unable to retrieve stats for %s" % (room_id,)

-        entry = {
-            "room_id": stats["room_id"],
-            "name": stats["name"],
-            "topic": stats["topic"],
-            "canonical_alias": stats["canonical_alias"],
-            "num_joined_members": stats["joined_members"],
-            "avatar_url": stats["avatar"],
-            "join_rule": stats["join_rules"],
+        entry: JsonDict = {
+            "room_id": stats.room_id,
+            "name": stats.name,
+            "topic": stats.topic,
+            "canonical_alias": stats.canonical_alias,
+            "num_joined_members": stats.joined_members,
+            "avatar_url": stats.avatar,
+            "join_rule": stats.join_rules,
            "world_readable": (
-                stats["history_visibility"] == HistoryVisibility.WORLD_READABLE
+                stats.history_visibility == HistoryVisibility.WORLD_READABLE
            ),
-            "guest_can_join": stats["guest_access"] == "can_join",
-            "room_type": stats["room_type"],
+            "guest_can_join": stats.guest_access == "can_join",
+            "room_type": stats.room_type,
        }

        if self._msc3266_enabled:
-            entry["im.nheko.summary.version"] = stats["version"]
-            entry["im.nheko.summary.encryption"] = stats["encryption"]
+            entry["im.nheko.summary.version"] = stats.version
+            entry["im.nheko.summary.encryption"] = stats.encryption

        # Federation requests need to provide additional information so the
        # requested server is able to filter the response appropriately.
@@ -374,13 +374,13 @@ class SearchHandler:
        serialize_options = SerializeEventConfig(requester=requester)

        for context in contexts.values():
-            context["events_before"] = self._event_serializer.serialize_events(
+            context["events_before"] = await self._event_serializer.serialize_events(
                context["events_before"],
                time_now,
                bundle_aggregations=aggregations,
                config=serialize_options,
            )
-            context["events_after"] = self._event_serializer.serialize_events(
+            context["events_after"] = await self._event_serializer.serialize_events(
                context["events_after"],
                time_now,
                bundle_aggregations=aggregations,
@@ -390,7 +390,7 @@ class SearchHandler:
        results = [
            {
                "rank": search_result.rank_map[e.event_id],
-                "result": self._event_serializer.serialize_event(
+                "result": await self._event_serializer.serialize_event(
                    e,
                    time_now,
                    bundle_aggregations=aggregations,
@@ -409,7 +409,7 @@ class SearchHandler:

        if state_results:
            rooms_cat_res["state"] = {
-                room_id: self._event_serializer.serialize_events(
+                room_id: await self._event_serializer.serialize_events(
                    state_events, time_now, config=serialize_options
                )
                for room_id, state_events in state_results.items()
@@ -806,7 +806,7 @@ class SsoHandler:
                media_id = profile["avatar_url"].split("/")[-1]
                if self._is_mine_server_name(server_name):
                    media = await self._media_repo.store.get_local_media(media_id)
-                    if media is not None and upload_name == media["upload_name"]:
+                    if media is not None and upload_name == media.upload_name:
                        logger.info("skipping saving the user avatar")
                        return True

@@ -1206,10 +1206,7 @@ class SsoHandler:
        # We have no guarantee that all the devices of that session are for the same
        # `user_id`. Hence, we have to iterate over the list of devices and log them out
        # one by one.
-        for device in devices:
-            user_id = device["user_id"]
-            device_id = device["device_id"]
-
+        for user_id, device_id in devices:
            # If the user_id associated with that device/session is not the one we got
            # out of the `sub` claim, skip that device and show log an error.
            if expected_user_id is not None and user_id != expected_user_id:
@@ -57,6 +57,7 @@ from synapse.types import (
    DeviceListUpdates,
    JsonDict,
    JsonMapping,
+    MultiWriterStreamToken,
    MutableStateMap,
    Requester,
    RoomStreamToken,
@@ -477,7 +478,11 @@ class SyncHandler:
                event_copy = {k: v for (k, v) in event.items() if k != "room_id"}
                ephemeral_by_room.setdefault(room_id, []).append(event_copy)

-            receipt_key = since_token.receipt_key if since_token else 0
+            receipt_key = (
+                since_token.receipt_key
+                if since_token
+                else MultiWriterStreamToken(stream=0)
+            )

            receipt_source = self.event_sources.sources.receipt
            receipts, receipt_key = await receipt_source.get_new_events(
@@ -1512,7 +1517,7 @@ class SyncHandler:

        # Presence data is included if the server has it enabled and not filtered out.
        include_presence_data = bool(
-            self.hs_config.server.use_presence
+            self.hs_config.server.presence_enabled
            and not sync_config.filter_collection.blocks_all_presence()
        )
        # Device list updates are sent if a since token is provided.
@@ -187,9 +187,9 @@ class _BaseThreepidAuthChecker:

                if row:
                    threepid = {
-                        "medium": row["medium"],
-                        "address": row["address"],
-                        "validated_at": row["validated_at"],
+                        "medium": row.medium,
+                        "address": row.address,
+                        "validated_at": row.validated_at,
                    }

                    # Valid threepid returned, delete from the db
@@ -59,7 +59,7 @@ class BasicProxyCredentials(ProxyCredentials):
            a Proxy-Authorization header.
        """
        # Encode as base64 and prepend the authorization type
-        return b"Basic " + base64.encodebytes(self.username_password)
+        return b"Basic " + base64.b64encode(self.username_password)


@attr.s(auto_attribs=True)
@@ -1019,11 +1019,14 @@ def tag_args(func: Callable[P, R]) -> Callable[P, R]:
    if not opentracing:
        return func

+    # getfullargspec is somewhat expensive, so ensure it is only called a single
+    # time (the function signature shouldn't change anyway).
+    argspec = inspect.getfullargspec(func)
+
    @contextlib.contextmanager
    def _wrapping_logic(
-        func: Callable[P, R], *args: P.args, **kwargs: P.kwargs
+        _func: Callable[P, R], *args: P.args, **kwargs: P.kwargs
    ) -> Generator[None, None, None]:
-        argspec = inspect.getfullargspec(func)
        # We use `[1:]` to skip the `self` object reference and `start=1` to
        # make the index line up with `argspec.args`.
        #
@@ -19,6 +19,7 @@ import shutil
 from io import BytesIO
 from typing import IO, TYPE_CHECKING, Dict, List, Optional, Set, Tuple

+import attr
 from matrix_common.types.mxc_uri import MXCUri

 import twisted.internet.error
@@ -50,6 +51,7 @@ from synapse.media.storage_provider import StorageProviderWrapper
 from synapse.media.thumbnailer import Thumbnailer, ThumbnailError
 from synapse.media.url_previewer import UrlPreviewer
 from synapse.metrics.background_process_metrics import run_as_background_process
+from synapse.storage.databases.main.media_repository import RemoteMedia
 from synapse.types import UserID
 from synapse.util.async_helpers import Linearizer
 from synapse.util.retryutils import NotRetryingDestination
@@ -245,18 +247,18 @@ class MediaRepository:
            Resolves once a response has successfully been written to request
        """
        media_info = await self.store.get_local_media(media_id)
-        if not media_info or media_info["quarantined_by"]:
+        if not media_info or media_info.quarantined_by:
            respond_404(request)
            return

        self.mark_recently_accessed(None, media_id)

-        media_type = media_info["media_type"]
+        media_type = media_info.media_type
        if not media_type:
            media_type = "application/octet-stream"
-        media_length = media_info["media_length"]
-        upload_name = name if name else media_info["upload_name"]
-        url_cache = media_info["url_cache"]
+        media_length = media_info.media_length
+        upload_name = name if name else media_info.upload_name
+        url_cache = media_info.url_cache

        file_info = FileInfo(None, media_id, url_cache=bool(url_cache))

@@ -310,16 +312,20 @@ class MediaRepository:

        # We deliberately stream the file outside the lock
        if responder:
-            media_type = media_info["media_type"]
-            media_length = media_info["media_length"]
-            upload_name = name if name else media_info["upload_name"]
+            upload_name = name if name else media_info.upload_name
            await respond_with_responder(
-                request, responder, media_type, media_length, upload_name
+                request,
+                responder,
+                media_info.media_type,
+                media_info.media_length,
+                upload_name,
            )
        else:
            respond_404(request)

-    async def get_remote_media_info(self, server_name: str, media_id: str) -> dict:
+    async def get_remote_media_info(
+        self, server_name: str, media_id: str
+    ) -> RemoteMedia:
        """Gets the media info associated with the remote file, downloading
        if necessary.

@@ -353,7 +359,7 @@ class MediaRepository:

    async def _get_remote_media_impl(
        self, server_name: str, media_id: str
-    ) -> Tuple[Optional[Responder], dict]:
+    ) -> Tuple[Optional[Responder], RemoteMedia]:
        """Looks for media in local cache, if not there then attempt to
        download from remote server.

@@ -373,15 +379,17 @@ class MediaRepository:

        # If we have an entry in the DB, try and look for it
        if media_info:
-            file_id = media_info["filesystem_id"]
+            file_id = media_info.filesystem_id
            file_info = FileInfo(server_name, file_id)

-            if media_info["quarantined_by"]:
+            if media_info.quarantined_by:
                logger.info("Media is quarantined")
                raise NotFoundError()

-            if not media_info["media_type"]:
-                media_info["media_type"] = "application/octet-stream"
+            if not media_info.media_type:
+                media_info = attr.evolve(
+                    media_info, media_type="application/octet-stream"
+                )

            responder = await self.media_storage.fetch_media(file_info)
            if responder:
@@ -403,9 +411,9 @@ class MediaRepository:
            if not media_info:
                raise e

-        file_id = media_info["filesystem_id"]
-        if not media_info["media_type"]:
-            media_info["media_type"] = "application/octet-stream"
+        file_id = media_info.filesystem_id
+        if not media_info.media_type:
+            media_info = attr.evolve(media_info, media_type="application/octet-stream")
        file_info = FileInfo(server_name, file_id)

        # We generate thumbnails even if another process downloaded the media
@@ -415,7 +423,7 @@ class MediaRepository:
        # otherwise they'll request thumbnails and get a 404 if they're not
        # ready yet.
        await self._generate_thumbnails(
-            server_name, media_id, file_id, media_info["media_type"]
+            server_name, media_id, file_id, media_info.media_type
        )

        responder = await self.media_storage.fetch_media(file_info)
@@ -425,7 +433,7 @@ class MediaRepository:
        self,
        server_name: str,
        media_id: str,
-    ) -> dict:
+    ) -> RemoteMedia:
        """Attempt to download the remote file from the given server name,
        using the given file_id as the local id.

@@ -518,7 +526,7 @@ class MediaRepository:
                origin=server_name,
                media_id=media_id,
                media_type=media_type,
-                time_now_ms=self.clock.time_msec(),
+                time_now_ms=time_now_ms,
                upload_name=upload_name,
                media_length=length,
                filesystem_id=file_id,
@@ -526,15 +534,17 @@ class MediaRepository:

        logger.info("Stored remote media in file %r", fname)

-        media_info = {
-            "media_type": media_type,
-            "media_length": length,
-            "upload_name": upload_name,
-            "created_ts": time_now_ms,
-            "filesystem_id": file_id,
-        }
-
-        return media_info
+        return RemoteMedia(
+            media_origin=server_name,
+            media_id=media_id,
+            media_type=media_type,
+            media_length=length,
+            upload_name=upload_name,
+            created_ts=time_now_ms,
+            filesystem_id=file_id,
+            last_access_ts=time_now_ms,
+            quarantined_by=None,
+        )

    def _get_thumbnail_requirements(
        self, media_type: str
@@ -949,10 +959,7 @@ class MediaRepository:

        deleted = 0

-        for media in old_media:
-            origin = media["media_origin"]
-            media_id = media["media_id"]
-            file_id = media["filesystem_id"]
+        for origin, media_id, file_id in old_media:
            key = (origin, media_id)

            logger.info("Deleting: %r", key)
@@ -240,15 +240,14 @@ class UrlPreviewer:
        cache_result = await self.store.get_url_cache(url, ts)
        if (
            cache_result
-            and cache_result["expires_ts"] > ts
-            and cache_result["response_code"] / 100 == 2
+            and cache_result.expires_ts > ts
+            and cache_result.response_code // 100 == 2
        ):
            # It may be stored as text in the database, not as bytes (such as
            # PostgreSQL). If so, encode it back before handing it on.
-            og = cache_result["og"]
-            if isinstance(og, str):
-                og = og.encode("utf8")
-            return og
+            if isinstance(cache_result.og, str):
+                return cache_result.og.encode("utf8")
+            return cache_result.og

        # If this URL can be accessed via an allowed oEmbed, use that instead.
        url_to_download = url
@@ -12,17 +12,45 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.

-import select
+import logging
 import time
-from typing import Any, Iterable, List, Tuple
+from selectors import SelectSelector, _PollLikeSelector  # type: ignore[attr-defined]
+from typing import Any, Callable, Iterable

 from prometheus_client import Histogram, Metric
 from prometheus_client.core import REGISTRY, GaugeMetricFamily

-from twisted.internet import reactor
+from twisted.internet import reactor, selectreactor
+from twisted.internet.asyncioreactor import AsyncioSelectorReactor

 from synapse.metrics._types import Collector

+try:
+    from selectors import KqueueSelector
+except ImportError:
+
+    class KqueueSelector:  # type: ignore[no-redef]
+        pass
+
+
+try:
+    from twisted.internet.epollreactor import EPollReactor
+except ImportError:
+
+    class EPollReactor:  # type: ignore[no-redef]
+        pass
+
+
+try:
+    from twisted.internet.pollreactor import PollReactor
+except ImportError:
+
+    class PollReactor:  # type: ignore[no-redef]
+        pass
+
+
+logger = logging.getLogger(__name__)
+
 #
 # Twisted reactor metrics
 #
@@ -34,52 +62,100 @@ tick_time = Histogram(
 )


-class EpollWrapper:
-    """a wrapper for an epoll object which records the time between polls"""
+class CallWrapper:
+    """A wrapper for a callable which records the time between calls"""

-    def __init__(self, poller: "select.epoll"):  # type: ignore[name-defined]
+    def __init__(self, wrapped: Callable[..., Any]):
        self.last_polled = time.time()
-        self._poller = poller
+        self._wrapped = wrapped

-    def poll(self, *args, **kwargs) -> List[Tuple[int, int]]:  # type: ignore[no-untyped-def]
-        # record the time since poll() was last called. This gives a good proxy for
+    def __call__(self, *args, **kwargs) -> Any:  # type: ignore[no-untyped-def]
+        # record the time since this was last called. This gives a good proxy for
        # how long it takes to run everything in the reactor - ie, how long anything
        # waiting for the next tick will have to wait.
        tick_time.observe(time.time() - self.last_polled)

-        ret = self._poller.poll(*args, **kwargs)
+        ret = self._wrapped(*args, **kwargs)

        self.last_polled = time.time()
        return ret

+
+class ObjWrapper:
+    """A wrapper for an object which wraps a specified method in CallWrapper.
+
+    Other methods/attributes are passed to the original object.
+
+    This is necessary when the wrapped object does not allow the attribute to be
+    overwritten.
+    """
+
+    def __init__(self, wrapped: Any, method_name: str):
+        self._wrapped = wrapped
+        self._method_name = method_name
+        self._wrapped_method = CallWrapper(getattr(wrapped, method_name))
+
    def __getattr__(self, item: str) -> Any:
-        return getattr(self._poller, item)
+        if item == self._method_name:
+            return self._wrapped_method
+
+        return getattr(self._wrapped, item)


 class ReactorLastSeenMetric(Collector):
-    def __init__(self, epoll_wrapper: EpollWrapper):
-        self._epoll_wrapper = epoll_wrapper
+    def __init__(self, call_wrapper: CallWrapper):
+        self._call_wrapper = call_wrapper

    def collect(self) -> Iterable[Metric]:
        cm = GaugeMetricFamily(
            "python_twisted_reactor_last_seen",
            "Seconds since the Twisted reactor was last seen",
        )
-        cm.add_metric([], time.time() - self._epoll_wrapper.last_polled)
+        cm.add_metric([], time.time() - self._call_wrapper.last_polled)
        yield cm


+# Twisted has already select a reasonable reactor for us, so assumptions can be
+# made about the shape.
+wrapper = None
 try:
-    # if the reactor has a `_poller` attribute, which is an `epoll` object
-    # (ie, it's an EPollReactor), we wrap the `epoll` with a thing that will
-    # measure the time between ticks
-    from select import epoll  # type: ignore[attr-defined]
+    if isinstance(reactor, (PollReactor, EPollReactor)):
+        reactor._poller = ObjWrapper(reactor._poller, "poll")  # type: ignore[attr-defined]
+        wrapper = reactor._poller._wrapped_method  # type: ignore[attr-defined]

-    poller = reactor._poller  # type: ignore[attr-defined]
-except (AttributeError, ImportError):
-    pass
-else:
-    if isinstance(poller, epoll):
-        poller = EpollWrapper(poller)
-        reactor._poller = poller  # type: ignore[attr-defined]
-        REGISTRY.register(ReactorLastSeenMetric(poller))
+    elif isinstance(reactor, selectreactor.SelectReactor):
+        # Twisted uses a module-level _select function.
+        wrapper = selectreactor._select = CallWrapper(selectreactor._select)
+
+    elif isinstance(reactor, AsyncioSelectorReactor):
+        # For asyncio look at the underlying asyncio event loop.
+        asyncio_loop = reactor._asyncioEventloop  # A sub-class of BaseEventLoop,
+
+        # A sub-class of BaseSelector.
+        selector = asyncio_loop._selector  # type: ignore[attr-defined]
+
+        if isinstance(selector, SelectSelector):
+            wrapper = selector._select = CallWrapper(selector._select)  # type: ignore[attr-defined]
+
+        # poll, epoll, and /dev/poll.
+        elif isinstance(selector, _PollLikeSelector):
+            selector._selector = ObjWrapper(selector._selector, "poll")  # type: ignore[attr-defined]
+            wrapper = selector._selector._wrapped_method  # type: ignore[attr-defined]
+
+        elif isinstance(selector, KqueueSelector):
+            selector._selector = ObjWrapper(selector._selector, "control")  # type: ignore[attr-defined]
+            wrapper = selector._selector._wrapped_method  # type: ignore[attr-defined]
+
+        else:
+            # E.g. this does not support the (Windows-only) ProactorEventLoop.
+            logger.warning(
+                "Skipping configuring ReactorLastSeenMetric: unexpected asyncio loop selector: %r via %r",
+                selector,
+                asyncio_loop,
+            )
+except Exception as e:
+    logger.warning("Configuring ReactorLastSeenMetric failed: %r", e)
+
+
+if wrapper:
+    REGISTRY.register(ReactorLastSeenMetric(wrapper))
@@ -23,6 +23,7 @@ from typing import (
    Generator,
    Iterable,
    List,
+    Mapping,
    Optional,
    Tuple,
    TypeVar,
@@ -39,6 +40,7 @@ from twisted.web.resource import Resource

 from synapse.api import errors
 from synapse.api.errors import SynapseError
+from synapse.api.presence import UserPresenceState
 from synapse.config import ConfigError
 from synapse.events import EventBase
 from synapse.events.presence_router import (
@@ -46,6 +48,7 @@ from synapse.events.presence_router import (
    GET_USERS_FOR_STATES_CALLBACK,
    PresenceRouter,
 )
+from synapse.events.utils import ADD_EXTRA_FIELDS_TO_UNSIGNED_CLIENT_EVENT_CALLBACK
 from synapse.handlers.account_data import ON_ACCOUNT_DATA_UPDATED_CALLBACK
 from synapse.handlers.auth import (
    CHECK_3PID_AUTH_CALLBACK,
@@ -257,6 +260,7 @@ class ModuleApi:
        self.custom_template_dir = hs.config.server.custom_template_directory
        self._callbacks = hs.get_module_api_callbacks()
        self.msc3861_oauth_delegation_enabled = hs.config.experimental.msc3861.enabled
+        self._event_serializer = hs.get_event_client_serializer()

        try:
            app_name = self._hs.config.email.email_app_name
@@ -488,6 +492,25 @@ class ModuleApi:
        """
        self._hs.register_module_web_resource(path, resource)

+    def register_add_extra_fields_to_unsigned_client_event_callbacks(
+        self,
+        *,
+        add_field_to_unsigned_callback: Optional[
+            ADD_EXTRA_FIELDS_TO_UNSIGNED_CLIENT_EVENT_CALLBACK
+        ] = None,
+    ) -> None:
+        """Registers a callback that can be used to add fields to the unsigned
+        section of events.
+
+        The callback is called every time an event is sent down to a client.
+
+        Added in Synapse 1.96.0
+        """
+        if add_field_to_unsigned_callback is not None:
+            self._event_serializer.register_add_extra_fields_to_unsigned_client_event_callback(
+                add_field_to_unsigned_callback
+            )
+
    #########################################################################
    # The following methods can be called by the module at any point in time.

@@ -1184,6 +1207,37 @@ class ModuleApi:
                presence_events, [destination]
            )

+    async def set_presence_for_users(
+        self, users: Mapping[str, Tuple[str, Optional[str]]]
+    ) -> None:
+        """
+        Update the internal presence state of users.
+
+        This can be used for either local or remote users.
+
+        Note that this method can only be run on the process that is configured to write to the
+        presence stream. By default, this is the main process.
+
+        Added in Synapse v1.96.0.
+        """
+
+        # We pull out the presence handler here to break a cyclic
+        # dependency between the presence router and module API.
+        presence_handler = self._hs.get_presence_handler()
+
+        from synapse.handlers.presence import PresenceHandler
+
+        assert isinstance(presence_handler, PresenceHandler)
+
+        states = await presence_handler.current_state_for_users(users.keys())
+        for user_id, (state, status_msg) in users.items():
+            prev_state = states.setdefault(user_id, UserPresenceState.default(user_id))
+            states[user_id] = prev_state.copy_and_replace(
+                state=state, status_msg=status_msg
+            )
+
+        await presence_handler._update_states(states.values(), force_notify=True)
+
    def looping_background_call(
        self,
        f: Callable,
@@ -1806,7 +1860,8 @@ class PublicRoomListManager:
        if not room:
            return False

-        return room.get("is_public", False)
+        # The first item is whether the room is public.
+        return room[0]

    async def add_room_to_public_room_list(self, room_id: str) -> None:
        """Publishes a room to the public room list.
@@ -21,11 +21,13 @@ from typing import (
    Dict,
    Iterable,
    List,
+    Literal,
    Optional,
    Set,
    Tuple,
    TypeVar,
    Union,
+    overload,
 )

 import attr
@@ -44,6 +46,7 @@ from synapse.metrics import LaterGauge
 from synapse.streams.config import PaginationConfig
 from synapse.types import (
    JsonDict,
+    MultiWriterStreamToken,
    PersistedEventPosition,
    RoomStreamToken,
    StrCollection,
@@ -127,7 +130,7 @@ class _NotifierUserStream:
    def notify(
        self,
        stream_key: StreamKeyType,
-        stream_id: Union[int, RoomStreamToken],
+        stream_id: Union[int, RoomStreamToken, MultiWriterStreamToken],
        time_now_ms: int,
    ) -> None:
        """Notify any listeners for this user of a new event from an
@@ -452,10 +455,48 @@ class Notifier:
        except Exception:
            logger.exception("Error pusher pool of event")

+    @overload
+    def on_new_event(
+        self,
+        stream_key: Literal[StreamKeyType.ROOM],
+        new_token: RoomStreamToken,
+        users: Optional[Collection[Union[str, UserID]]] = None,
+        rooms: Optional[StrCollection] = None,
+    ) -> None:
+        ...
+
+    @overload
+    def on_new_event(
+        self,
+        stream_key: Literal[StreamKeyType.RECEIPT],
+        new_token: MultiWriterStreamToken,
+        users: Optional[Collection[Union[str, UserID]]] = None,
+        rooms: Optional[StrCollection] = None,
+    ) -> None:
+        ...
+
+    @overload
+    def on_new_event(
+        self,
+        stream_key: Literal[
+            StreamKeyType.ACCOUNT_DATA,
+            StreamKeyType.DEVICE_LIST,
+            StreamKeyType.PRESENCE,
+            StreamKeyType.PUSH_RULES,
+            StreamKeyType.TO_DEVICE,
+            StreamKeyType.TYPING,
+            StreamKeyType.UN_PARTIAL_STATED_ROOMS,
+        ],
+        new_token: int,
+        users: Optional[Collection[Union[str, UserID]]] = None,
+        rooms: Optional[StrCollection] = None,
+    ) -> None:
+        ...
+
    def on_new_event(
        self,
        stream_key: StreamKeyType,
-        new_token: Union[int, RoomStreamToken],
+        new_token: Union[int, RoomStreamToken, MultiWriterStreamToken],
        users: Optional[Collection[Union[str, UserID]]] = None,
        rooms: Optional[StrCollection] = None,
    ) -> None:
@@ -25,10 +25,13 @@ from typing import (
    Sequence,
    Tuple,
    Union,
+    cast,
 )

 from prometheus_client import Counter

+from twisted.internet.defer import Deferred
+
 from synapse.api.constants import (
    MAIN_TIMELINE,
    EventContentFields,
@@ -40,11 +43,15 @@ from synapse.api.room_versions import PushRuleRoomFlag
 from synapse.event_auth import auth_types_for_event, get_user_power_level
 from synapse.events import EventBase, relation_from_event
 from synapse.events.snapshot import EventContext
+from synapse.logging.context import make_deferred_yieldable, run_in_background
 from synapse.state import POWER_KEY
 from synapse.storage.databases.main.roommember import EventIdMembership
+from synapse.storage.roommember import ProfileInfo
 from synapse.synapse_rust.push import FilteredPushRules, PushRuleEvaluator
 from synapse.types import JsonValue
 from synapse.types.state import StateFilter
+from synapse.util import unwrapFirstError
+from synapse.util.async_helpers import gather_results
 from synapse.util.caches import register_cache
 from synapse.util.metrics import measure_func
 from synapse.visibility import filter_event_for_clients_with_state
@@ -342,15 +349,41 @@ class BulkPushRuleEvaluator:
        rules_by_user = await self._get_rules_for_event(event)
        actions_by_user: Dict[str, Collection[Union[Mapping, str]]] = {}

-        room_member_count = await self.store.get_number_joined_users_in_room(
-            event.room_id
-        )
-
+        # Gather a bunch of info in parallel.
+        #
+        # This has a lot of ignored types and casting due to the use of @cached
+        # decorated functions passed into run_in_background.
+        #
+        # See https://github.com/matrix-org/synapse/issues/16606
        (
-            power_levels,
-            sender_power_level,
-        ) = await self._get_power_levels_and_sender_level(
-            event, context, event_id_to_event
+            room_member_count,
+            (power_levels, sender_power_level),
+            related_events,
+            profiles,
+        ) = await make_deferred_yieldable(
+            cast(
+                "Deferred[Tuple[int, Tuple[dict, Optional[int]], Dict[str, Dict[str, JsonValue]], Mapping[str, ProfileInfo]]]",
+                gather_results(
+                    (
+                        run_in_background(  # type: ignore[call-arg]
+                            self.store.get_number_joined_users_in_room, event.room_id  # type: ignore[arg-type]
+                        ),
+                        run_in_background(
+                            self._get_power_levels_and_sender_level,
+                            event,
+                            context,
+                            event_id_to_event,
+                        ),
+                        run_in_background(self._related_events, event),
+                        run_in_background(  # type: ignore[call-arg]
+                            self.store.get_subset_users_in_room_with_profiles,
+                            event.room_id,  # type: ignore[arg-type]
+                            rules_by_user.keys(),  # type: ignore[arg-type]
+                        ),
+                    ),
+                    consumeErrors=True,
+                ).addErrback(unwrapFirstError),
+            )
        )

        # Find the event's thread ID.
@@ -366,8 +399,6 @@ class BulkPushRuleEvaluator:
                # the parent is part of a thread.
                thread_id = await self.store.get_thread_id(relation.parent_id)

-        related_events = await self._related_events(event)
-
        # It's possible that old room versions have non-integer power levels (floats or
        # strings; even the occasional `null`). For old rooms, we interpret these as if
        # they were integers. Do this here for the `@room` power level threshold.
@@ -400,11 +431,6 @@ class BulkPushRuleEvaluator:
            self.hs.config.experimental.msc1767_enabled,  # MSC3931 flag
        )

-        users = rules_by_user.keys()
-        profiles = await self.store.get_subset_users_in_room_with_profiles(
-            event.room_id, users
-        )
-
        for uid, rules in rules_by_user.items():
            if event.sender == uid:
                continue
@@ -433,7 +433,7 @@ class ReplicationEndpoint(metaclass=abc.ABCMeta):

        if self.WAIT_FOR_STREAMS:
            response[_STREAM_POSITION_KEY] = {
-                stream.NAME: stream.current_token(self._instance_name)
+                stream.NAME: stream.minimal_local_current_token()
                for stream in self._streams
            }

@@ -126,8 +126,9 @@ class ReplicationDataHandler:
                StreamKeyType.ACCOUNT_DATA, token, users=[row.user_id for row in rows]
            )
        elif stream_name == ReceiptsStream.NAME:
+            new_token = self.store.get_max_receipt_stream_id()
            self.notifier.on_new_event(
-                StreamKeyType.RECEIPT, token, rooms=[row.room_id for row in rows]
+                StreamKeyType.RECEIPT, new_token, rooms=[row.room_id for row in rows]
            )
            await self._pusher_pool.on_new_receipts({row.user_id for row in rows})
        elif stream_name == ToDeviceStream.NAME:
@@ -611,10 +611,14 @@ class ReplicationCommandHandler:
        # Find where we previously streamed up to.
        current_token = stream.current_token(cmd.instance_name)

-        # If the position token matches our current token then we're up to
-        # date and there's nothing to do. Otherwise, fetch all updates
-        # between then and now.
-        missing_updates = cmd.prev_token != current_token
+        # If the incoming previous position is less than our current position
+        # then we're up to date and there's nothing to do. Otherwise, fetch
+        # all updates between then and now.
+        #
+        # Note: We also have to check that `current_token` is at most the
+        # new position, to handle the case where the stream gets "reset"
+        # (e.g. for `caches` and `typing` after the writer's restart).
+        missing_updates = not (cmd.prev_token <= current_token <= cmd.new_token)
        while missing_updates:
            # Note: There may very well not be any new updates, but we check to
            # make sure. This can particularly happen for the event stream where
@@ -644,7 +648,7 @@ class ReplicationCommandHandler:
                    [stream.parse_row(row) for row in rows],
                )

-        logger.info("Caught up with stream '%s' to %i", stream_name, cmd.new_token)
+            logger.info("Caught up with stream '%s' to %i", stream_name, current_token)

        # We've now caught up to position sent to us, notify handler.
        await self._replication_data_handler.on_position(
@@ -27,7 +27,7 @@ from synapse.metrics.background_process_metrics import run_as_background_process
 from synapse.replication.tcp.commands import PositionCommand
 from synapse.replication.tcp.protocol import ServerReplicationStreamProtocol
 from synapse.replication.tcp.streams import EventsStream
-from synapse.replication.tcp.streams._base import StreamRow, Token
+from synapse.replication.tcp.streams._base import CachesStream, StreamRow, Token
 from synapse.util.metrics import Measure

 if TYPE_CHECKING:
@@ -204,6 +204,23 @@ class ReplicationStreamer:
                            # The token has advanced but there is no data to
                            # send, so we send a `POSITION` to inform other
                            # workers of the updated position.
+                            #
+                            # There are two reasons for this: 1) this instance
+                            # requested a stream ID but didn't use it, or 2)
+                            # this instance advanced its own stream position due
+                            # to receiving notifications about other instances
+                            # advancing their stream position.
+
+                            # We skip sending `POSITION` for the `caches` stream
+                            # for the second case as a) it generates a lot of
+                            # traffic as every worker would echo each write, and
+                            # b) nothing cares if a given worker's caches stream
+                            # position lags.
+                            if stream.NAME == CachesStream.NAME:
+                                # If there haven't been any writes since the
+                                # `last_token` then we're in the second case.
+                                if stream.minimal_local_current_token() <= last_token:
+                                    continue

                            # Note: `last_token` may not *actually* be the
                            # last token we sent out in a RDATA or POSITION.
@@ -161,6 +161,14 @@ class Stream:
            and `limited` is whether there are more updates to fetch.
        """
        current_token = self.current_token(self.local_instance_name)
+
+        # If the minimum current token for the local instance is less than or
+        # equal to the last thing we published, we know that there are no
+        # updates.
+        if self.last_token >= self.minimal_local_current_token():
+            self.last_token = current_token
+            return [], current_token, False
+
        updates, current_token, limited = await self.get_updates_since(
            self.local_instance_name, self.last_token, current_token
        )
@@ -489,6 +497,8 @@ class CachesStream(Stream):
        return self.store.get_cache_stream_token_for_writer(instance_name)

    def minimal_local_current_token(self) -> Token:
+        if self.store._cache_id_gen:
+            return self.store._cache_id_gen.get_minimal_local_current_token()
        return self.current_token(self.local_instance_name)


@@ -157,6 +157,12 @@ class EventsStream(_StreamFromIdGen):
        current_token: Token,
        target_row_count: int,
    ) -> StreamUpdateResult:
+        # The events stream cannot be "reset", so its safe to return early if
+        # the from token is larger than the current token (the DB query will
+        # trivially return 0 rows anyway).
+        if from_token >= current_token:
+            return [], current_token, False
+
        # the events stream merges together three separate sources:
        #  * new events
        #  * current_state changes
@@ -85,7 +85,19 @@ class ListDestinationsRestServlet(RestServlet):
        destinations, total = await self._store.get_destinations_paginate(
            start, limit, destination, order_by, direction
        )
-        response = {"destinations": destinations, "total": total}
+        response = {
+            "destinations": [
+                {
+                    "destination": r[0],
+                    "retry_last_ts": r[1],
+                    "retry_interval": r[2],
+                    "failure_ts": r[3],
+                    "last_successful_stream_ordering": r[4],
+                }
+                for r in destinations
+            ],
+            "total": total,
+        }
        if (start + limit) < total:
            response["next_token"] = str(start + len(destinations))

@@ -17,6 +17,8 @@ import logging
 from http import HTTPStatus
 from typing import TYPE_CHECKING, Optional, Tuple

+import attr
+
 from synapse.api.constants import Direction
 from synapse.api.errors import Codes, NotFoundError, SynapseError
 from synapse.http.server import HttpServer
@@ -418,7 +420,7 @@ class UserMediaRestServlet(RestServlet):
            start, limit, user_id, order_by, direction
        )

-        ret = {"media": media, "total": total}
+        ret = {"media": [attr.asdict(m) for m in media], "total": total}
        if (start + limit) < total:
            ret["next_token"] = start + len(media)

@@ -477,7 +479,7 @@ class UserMediaRestServlet(RestServlet):
        )

        deleted_media, total = await self.media_repository.delete_local_media_ids(
-            [row["media_id"] for row in media]
+            [m.media_id for m in media]
        )

        return HTTPStatus.OK, {"deleted_media": deleted_media, "total": total}
@@ -77,7 +77,18 @@ class ListRegistrationTokensRestServlet(RestServlet):
        await assert_requester_is_admin(self.auth, request)
        valid = parse_boolean(request, "valid")
        token_list = await self.store.get_registration_tokens(valid)
-        return HTTPStatus.OK, {"registration_tokens": token_list}
+        return HTTPStatus.OK, {
+            "registration_tokens": [
+                {
+                    "token": t[0],
+                    "uses_allowed": t[1],
+                    "pending": t[2],
+                    "completed": t[3],
+                    "expiry_time": t[4],
+                }
+                for t in token_list
+            ]
+        }


 class NewRegistrationTokenRestServlet(RestServlet):
@@ -16,6 +16,8 @@ from http import HTTPStatus
 from typing import TYPE_CHECKING, List, Optional, Tuple, cast
 from urllib import parse as urlparse

+import attr
+
 from synapse.api.constants import Direction, EventTypes, JoinRules, Membership
 from synapse.api.errors import AuthError, Codes, NotFoundError, SynapseError
 from synapse.api.filtering import Filter
@@ -306,10 +308,13 @@ class RoomRestServlet(RestServlet):
            raise NotFoundError("Room not found")

        members = await self.store.get_users_in_room(room_id)
-        ret["joined_local_devices"] = await self.store.count_devices_by_users(members)
-        ret["forgotten"] = await self.store.is_locally_forgotten_room(room_id)
+        result = attr.asdict(ret)
+        result["joined_local_devices"] = await self.store.count_devices_by_users(
+            members
+        )
+        result["forgotten"] = await self.store.is_locally_forgotten_room(room_id)

-        return HTTPStatus.OK, ret
+        return HTTPStatus.OK, result

    async def on_DELETE(
        self, request: SynapseRequest, room_id: str
@@ -408,8 +413,8 @@ class RoomMembersRestServlet(RestServlet):
    ) -> Tuple[int, JsonDict]:
        await assert_requester_is_admin(self.auth, request)

-        ret = await self.store.get_room(room_id)
-        if not ret:
+        room = await self.store.get_room(room_id)
+        if not room:
            raise NotFoundError("Room not found")

        members = await self.store.get_users_in_room(room_id)
@@ -437,14 +442,14 @@ class RoomStateRestServlet(RestServlet):
    ) -> Tuple[int, JsonDict]:
        await assert_requester_is_admin(self.auth, request)

-        ret = await self.store.get_room(room_id)
-        if not ret:
+        room = await self.store.get_room(room_id)
+        if not room:
            raise NotFoundError("Room not found")

        event_ids = await self._storage_controllers.state.get_current_state_ids(room_id)
        events = await self.store.get_events(event_ids.values())
        now = self.clock.time_msec()
-        room_state = self._event_serializer.serialize_events(events.values(), now)
+        room_state = await self._event_serializer.serialize_events(events.values(), now)
        ret = {"state": room_state}

        return HTTPStatus.OK, ret
@@ -724,7 +729,17 @@ class ForwardExtremitiesRestServlet(ResolveRoomIdMixin, RestServlet):
        room_id, _ = await self.resolve_room_id(room_identifier)

        extremities = await self.store.get_forward_extremities_for_room(room_id)
-        return HTTPStatus.OK, {"count": len(extremities), "results": extremities}
+        result = [
+            {
+                "event_id": ex[0],
+                "state_group": ex[1],
+                "depth": ex[2],
+                "received_ts": ex[3],
+            }
+            for ex in extremities
+        ]
+
+        return HTTPStatus.OK, {"count": len(extremities), "results": result}


 class RoomEventContextServlet(RestServlet):
@@ -779,22 +794,22 @@ class RoomEventContextServlet(RestServlet):

        time_now = self.clock.time_msec()
        results = {
-            "events_before": self._event_serializer.serialize_events(
+            "events_before": await self._event_serializer.serialize_events(
                event_context.events_before,
                time_now,
                bundle_aggregations=event_context.aggregations,
            ),
-            "event": self._event_serializer.serialize_event(
+            "event": await self._event_serializer.serialize_event(
                event_context.event,
                time_now,
                bundle_aggregations=event_context.aggregations,
            ),
-            "events_after": self._event_serializer.serialize_events(
+            "events_after": await self._event_serializer.serialize_events(
                event_context.events_after,
                time_now,
                bundle_aggregations=event_context.aggregations,
            ),
-            "state": self._event_serializer.serialize_events(
+            "state": await self._event_serializer.serialize_events(
                event_context.state, time_now
            ),
            "start": event_context.start,
@@ -108,7 +108,18 @@ class UserMediaStatisticsRestServlet(RestServlet):
        users_media, total = await self.store.get_users_media_usage_paginate(
            start, limit, from_ts, until_ts, order_by, direction, search_term
        )
-        ret = {"users": users_media, "total": total}
+        ret = {
+            "users": [
+                {
+                    "user_id": r[0],
+                    "displayname": r[1],
+                    "media_count": r[2],
+                    "media_length": r[3],
+                }
+                for r in users_media
+            ],
+            "total": total,
+        }
        if (start + limit) < total:
            ret["next_token"] = start + len(users_media)

@@ -18,6 +18,8 @@ import secrets
 from http import HTTPStatus
 from typing import TYPE_CHECKING, Dict, List, Optional, Tuple

+import attr
+
 from synapse.api.constants import Direction, UserTypes
 from synapse.api.errors import Codes, NotFoundError, SynapseError
 from synapse.http.servlet import (
@@ -161,11 +163,13 @@ class UsersRestServletV2(RestServlet):
        )

        # If support for MSC3866 is not enabled, don't show the approval flag.
+        filter = None
        if not self._msc3866_enabled:
-            for user in users:
-                del user["approved"]

-        ret = {"users": users, "total": total}
+            def _filter(a: attr.Attribute) -> bool:
+                return a.name != "approved"
+
+        ret = {"users": [attr.asdict(u, filter=filter) for u in users], "total": total}
        if (start + limit) < total:
            ret["next_token"] = str(start + len(users))

--- a/Show More
+++ b/Show More
Author	SHA1	Message	Date
Patrick Cloke	f118aa6889	Lint	2023-11-09 15:01:30 -05:00
Patrick Cloke	7b82cba4bc	Support running tests in CI.	2023-11-09 14:58:51 -05:00
Patrick Cloke	ff40f22a23	Back out unneeded changes.	2023-11-09 14:46:48 -05:00
Patrick Cloke	55193f38e5	Merge remote-tracking branch 'origin/develop' into clokep/psycopg3	2023-11-09 14:45:50 -05:00
Patrick Cloke	bc4372ad81	Use dbname instead of database for Postgres config. (#16618 )	2023-11-09 14:40:45 -05:00
Patrick Cloke	9f514dd0fb	Use _invalidate_cache_and_stream_bulk in more places. (#16616 ) This takes advantage of the new bulk method in more places to invalidate caches for many keys at once (and then to stream that over replication).	2023-11-09 14:40:30 -05:00
Patrick Cloke	eb2b0dceb4	Another instance of psycopg2.	2023-11-09 14:38:15 -05:00
Patrick Cloke	626f468155	Backout unneeded changes.	2023-11-09 14:21:33 -05:00
Patrick Cloke	ab3f1b3b53	Convert simple_select_one_txn and simple_select_one to return tuples. (#16612 )	2023-11-09 11:13:31 -05:00
Patrick Cloke	ff716b483b	Return attrs for more media repo APIs. (#16611 )	2023-11-09 11:00:30 -05:00
David Robertson	91587d4cf9	Bulk-invalidate e2e cached queries after claiming keys (#16613 ) Co-authored-by: Patrick Cloke <patrickc@matrix.org>	2023-11-09 15:57:09 +00:00
Patrick Cloke	396fa974a1	Merge remote-tracking branch 'origin/develop' into clokep/psycopg3	2023-11-09 09:48:24 -05:00
dependabot[bot]	f6aa047aa2	Bump pyicu from 2.11 to 2.12 (#16603 ) Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>	2023-11-08 09:28:51 -05:00
dependabot[bot]	2a336cd2fc	Bump serde_json from 1.0.107 to 1.0.108 (#16604 ) Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>	2023-11-08 09:27:51 -05:00
Patrick Cloke	455ef04187	Avoid updating the same rows multiple times with simple_update_many_txn. (#16609 ) simple_update_many_txn had a bug in it which would cause each update to be applied twice.	2023-11-07 14:02:09 -05:00
Patrick Cloke	9738b1c497	Avoid executing no-op queries. (#16583 ) If simple_{insert,upsert,update}_many_txn is called without any data to modify then return instead of executing the query. This matches the behavior of simple_{select,delete}_many_txn.	2023-11-07 14:00:25 -05:00
Patrick Cloke	ec9ff389f4	More tests for the simple_* methods. (#16596 ) Expand tests for the simple_* database methods, additionally test against both PostgreSQL and SQLite variants.	2023-11-07 09:34:23 -05:00
Patrick Cloke	7e5d3b06fa	Collect information for PushRuleEvaluator in parallel. (#16590 ) Fetch information needed for push rule evaluation in parallel. Ideally this would use query pipelining, but this is not available in psycopg2. Due to the database thread pool this may result in little to no parallelization.	2023-11-06 15:41:57 -05:00
Patrick Cloke	1dd3074629	Bump setuptools_rust to match pinned version. (#16605 )	2023-11-06 09:13:53 -05:00
Patrick Cloke	cc4fe68adf	Support reactor timing metric on more reactors. (#16532 ) Previously only Twisted's EPollReactor was compatible with the reactor timing metric, notably not working when asyncio was used. After this change, the following configurations support the reactor timing metric: * poll, epoll, or select reactors * asyncio reactor with a poll, epoll, select, /dev/poll, or kqueue event loop.	2023-11-06 08:31:22 -05:00
dependabot[bot]	1a9b22a3d1	Bump setuptools-rust from 1.8.0 to 1.8.1 (#16601 ) Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>	2023-11-06 08:12:18 -05:00
dependabot[bot]	5cf2988694	Bump types-pyyaml from 6.0.12.11 to 6.0.12.12 (#16602 ) Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>	2023-11-06 08:12:01 -05:00
dependabot[bot]	a28339b867	Bump types-jsonschema from 4.19.0.3 to 4.19.0.4 (#16599 ) Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>	2023-11-06 08:11:31 -05:00
dependabot[bot]	2f689a6326	Bump ruff from 0.0.292 to 0.1.4 (#16600 ) Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>	2023-11-06 08:11:15 -05:00
Patrick Cloke	92828a7f95	Simplify event persistence code (#16584 ) The event persistence code used to handle multiple rooms at a time, but was simplified to only ever be called with a single room at a time (different rooms are now handled in parallel). The code is still generic to multiple rooms causing a lot of work that is unnecessary (e.g. unnecessary loops, and partitioning data by room). This strips out the ability to handle multiple rooms at once, greatly simplifying the code.	2023-11-03 07:30:31 -04:00
Patrick Cloke	0afbef30cf	Use simple_select_many_txn in event persistance code. (#16585 ) Just to standardize on the normal helpers, it might also have a slight perf improvement on PostgreSQL which will now use `ANY (?)` instead of `IN (?, ?, ...)`.	2023-11-02 09:41:00 -04:00
dependabot[bot]	c812f43bd7	Bump twisted from 23.8.0 to 23.10.0 (#16588 )	2023-11-01 10:23:13 +00:00
Patrick Cloke	ed1b879576	Do not call getfullargspec on every call. (#16589 ) getfullargspec is relatively expensive and the results will not change between calls, so precalculate it outside the wrapper.	2023-10-31 20:16:17 +00:00
Patrick Cloke	cfb6d38c47	Remove remaining usage of cursor_to_dict. (#16564 )	2023-10-31 13:13:28 -04:00
Erik Johnston	c0ba319b22	Merge branch 'release-v1.96' into develop	2023-10-31 16:30:16 +00:00
Patrick Cloke	70b503f144	Fix import ordering issue introduced in `7a3a55ac98`.	2023-10-31 10:32:35 -04:00
Erik Johnston	c5b543938b	Update changelog	2023-10-31 14:17:27 +00:00
Erik Johnston	4bb2b4aa9a	1.96.0rc1	2023-10-31 14:09:13 +00:00
Erik Johnston	1f033ad5e4	Merge remote-tracking branch 'origin/develop' into release-v1.96	2023-10-31 14:08:51 +00:00
Erik Johnston	60c5e8d79b	Revert "1.96.0rc1" This reverts commit `4724a6ded1`.	2023-10-31 14:08:45 +00:00
Erik Johnston	0cc6509cfe	Merge branch 'release-v1.95' into develop	2023-10-31 14:05:52 +00:00
Erik Johnston	a11511954a	1.95.1	2023-10-31 14:02:32 +00:00
Patrick Cloke	daec55e1fe	Merge pull request from GHSA-mp92-3jfm-3575	2023-10-31 13:59:09 +00:00
Patrick Cloke	7a3a55ac98	Merge pull request from GHSA-mp92-3jfm-3575	2023-10-31 13:58:30 +00:00
Erik Johnston	4724a6ded1	1.96.0rc1	2023-10-31 13:47:08 +00:00
David Robertson	de981ae567	Claim local one-time-keys in bulk (#16565 ) Co-authored-by: Patrick Cloke <clokep@users.noreply.github.com>	2023-10-30 21:25:21 +00:00
Niranjan Kurhade	91aa52c911	Clients link fixed in README (#16569 )	2023-10-30 16:05:34 +00:00
Erik Johnston	408c13801a	Add fast path for replication events stream fetch (#16580 ) We can bail early if the from token is greater than or equal to the current token.	2023-10-30 14:47:57 +00:00
David Robertson	fdce83ee60	Claim fallback keys in bulk (#16570 )	2023-10-30 14:34:37 +00:00
dependabot[bot]	a3f6200d65	Bump setuptools-rust from 1.7.0 to 1.8.0 (#16574 ) Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: David Robertson <davidr@element.io>	2023-10-30 13:40:17 +00:00
David Robertson	4e1a19d375	Run actions/setup-go after checking out complement (#16567 )	2023-10-30 13:07:08 +00:00
Erik Johnston	8c63e93286	Fix HTTP repl response to use minimum token (#16578 )	2023-10-30 12:27:14 +00:00
dependabot[bot]	ba55835000	Bump cryptography from 41.0.4 to 41.0.5 (#16572 ) Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>	2023-10-30 11:44:36 +00:00
dependabot[bot]	8f7cd4cd03	Bump serde from 1.0.189 to 1.0.190 (#16577 ) Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>	2023-10-30 11:32:36 +00:00
dependabot[bot]	425cb9c23c	Bump phonenumbers from 8.13.22 to 8.13.23 (#16576 ) Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>	2023-10-30 11:30:26 +00:00
dependabot[bot]	13f6467785	Bump black from 23.10.0 to 23.10.1 (#16575 ) Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>	2023-10-30 11:29:52 +00:00
dependabot[bot]	bcaaeab410	Bump types-psycopg2 from 2.9.21.14 to 2.9.21.15 (#16573 ) Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>	2023-10-30 11:28:50 +00:00
David Robertson	747416e94c	Portdb: don't copy a table that gets rebuilt (#16563 )	2023-10-27 20:14:02 +01:00
kegsay	11a8ae0632	complement: enable dirty runs (#16520 ) * complement: enable dirty runs * Add changelog * Set a low connpool limit when running in Complement Dirty runs can cause many containers to be running concurrently, which seems to easily exhaust resources on the host. The increased speedup from dirty runs also seems to use more db connections on workers, which are misconfigured currently to have `SUM(workers * cp_max) > max_connections`, causing ``` FATAL: sorry, too many clients already ``` which results in tests failing. * Try p=2 concurrency to restrict slowness of servers which causes partial state join tests to flake * Debug logging * Only run flakey tests * Only adjust connection pool limits in worker mode * Move cp vars to somewhere where they get executed in CI * Move cp values back to where they actually work * Debug logging * Try p=1 to see if this makes worker mode happier * Remove debug logging	2023-10-27 18:29:20 +01:00
Patrick Cloke	2bf9341406	Ensure local invited & knocking users leave before purge. (#16559 ) This is mostly useful for federated rooms where some users would get stuck in the invite or knock state when the room was purged from their homeserver.	2023-10-27 12:50:50 -04:00
Erik Johnston	5413cefe32	Reduce amount of caches POSITIONS we send (#16561 ) Follow on from / actually correctly does #16557	2023-10-27 16:07:11 +01:00
Erik Johnston	89dbbd68e1	Reduce spurious replication catchup (#16555 )	2023-10-27 13:27:20 +00:00
Erik Johnston	928e964857	Fix cross-worker ratelimiting (#16558 ) c.f. #16481	2023-10-27 12:52:40 +01:00
Erik Johnston	0680d76659	Reduce replication traffic due to reflected cache stream POSITION (#16557 )	2023-10-27 12:51:08 +01:00
Erik Johnston	c02406ac71	Add new module API for adding custom fields to events `unsigned` section (#16549 )	2023-10-27 09:04:08 +00:00
Patrick Cloke	679c691f6f	Remove more usages of cursor_to_dict. (#16551 ) Mostly to improve type safety.	2023-10-26 15:12:28 -04:00
Patrick Cloke	85e5f2dc25	Add a new module API to update user presence state. (#16544 ) This adds a module API which allows a module to update a user's presence state/status message. This is useful for controlling presence from an external system. To fully control presence from the module the presence.enabled config parameter gains a new state of "untracked" which disables internal tracking of presence changes via user actions, etc. Only updates from the module will be persisted and sent down sync properly).	2023-10-26 15:11:24 -04:00
Patrick Cloke	9407d5ba78	Convert simple_select_list and simple_select_list_txn to return lists of tuples (#16505 ) This should use fewer allocations and improves type hints.	2023-10-26 13:01:36 -04:00
David Robertson	c14a7de6af	Pin the recommended poetry version in contributors' guide (#16550 )	2023-10-25 16:31:15 +01:00
Erik Johnston	ba47fea528	Allow multiple workers to write to receipts stream. (#16432 ) Fixes #16417	2023-10-25 16:16:19 +01:00
Patrick Cloke	e182dbb5b9	Fix tests on Twisted trunk. (#16528 ) Twisted trunk makes a change to the `TLSMemoryBIOFactory` where the underlying protocol is changed from `TLSMemoryBIOProtocol` to `BufferingTLSTransport` to improve performance of TLS code (see https://github.com/twisted/twisted/issues/11989). In order to properly hook this code up in tests we need to pass the test reactor's clock into `TLSMemoryBIOFactory` to avoid the global (trial) reactor being used by default. Twisted does something similar internally for tests: https://github.com/twisted/twisted/blob/157cd8e659705940e895d321339d467e76ae9d0a/src/twisted/web/test/test_agent.py#L871-L874	2023-10-25 07:39:45 -04:00
Richard Brežák	95076f77c1	Fix http/s proxy authentication with long username/passwords (#16504 )	2023-10-24 13:45:21 +00:00
David Robertson	2f1065f81b	Revert "Add test case to detect dodgy b64 encoding" This reverts commit `5fe76b9434`. I think I had this accidentally commited on my local develop branch, and so it accidentally got merged into upstream develop. This should re-land with corrections in #16504.	2023-10-24 14:34:47 +01:00
Patrick Cloke	a280d117dc	Don't use separate copy_read method.	2023-09-29 14:24:54 -04:00
Patrick Cloke	a072285e9d	Use _do_execute for COPY TO/FROM.	2023-09-29 14:07:34 -04:00
Patrick Cloke	29492b7e85	Lint.	2023-09-29 13:44:49 -04:00
Patrick Cloke	208a5944a6	Tweaks	2023-09-29 06:33:26 -04:00
Patrick Cloke	856faa8fce	Add to poetry env.	2023-09-28 15:41:41 -04:00
Patrick Cloke	2b02f4a41c	Merge remote-tracking branch 'origin/develop' into clokep/psycopg3	2023-09-28 15:31:22 -04:00
Patrick Cloke	4a0dfb336f	temp	2023-09-22 15:58:55 -04:00
Patrick Cloke	edff9f7dca	Merge remote-tracking branch 'origin/develop' into clokep/psycopg3	2023-09-22 14:26:42 -04:00
Patrick Cloke	2d8cbbd53a	More fixes.	2023-09-22 14:26:40 -04:00
Patrick Cloke	be042ce2c6	Separate engines via subclassing.	2023-09-22 14:25:23 -04:00
Patrick Cloke	8bb700e9ac	Merge remote-tracking branch 'origin/develop' into clokep/psycopg3	2023-09-13 14:35:46 -04:00
Patrick Cloke	9f0ccbdbaf	Working version.	2022-11-30 08:20:48 -05:00
Patrick Cloke	f5ef7e13d7	Initial abstraction.	2022-11-30 08:20:48 -05:00
Patrick Cloke	61fc1cb1e7	Copy PostgresEngine.	2022-11-30 08:20:47 -05:00
				`@@ -1 +0,0 @@`
				`Fixed a bug that prevents Grafana from finding the correct datasource. Contributed by @MichaelSasser.`
				`@@ -1 +0,0 @@`
				Fix a long-standing, exceedingly rare edge case where the first event persisted by a new event persister worker might not be sent down `/sync`.
				`@@ -1 +0,0 @@`
				Fix long-standing bug where `/sync` incorrectly did not mark a room as `limited` in a sync requests when there were missing remote events.
				`@@ -1 +0,0 @@`
				`Improve performance of delete device messages query, cf issue [16479](https://github.com/matrix-org/synapse/issues/16479).`
				`@@ -1 +0,0 @@`
				`Improve replication performance when purging rooms.`
				`@@ -1 +0,0 @@`
				Run trial & integration tests in continuous integration when `.ci` directory is modified.
				`@@ -1 +0,0 @@`
				`Remove duplicate call to mark remote server 'awake' when using a federation sending worker.`
				`@@ -1 +0,0 @@`
				`Force TLS certificate verification in user registration script.`
				`@@ -1 +0,0 @@`
				`Add a sentence to the opentracing docs on how you can have jaeger in a different place than synapse.`