Fix running background tasks on BG worker

WORKING workers setup template
STASH
2022-03-18 13:53:55 +00:00 · 2022-03-17 10:42:28 +00:00 · 2022-03-16 14:02:58 +00:00 · 2022-03-11 11:36:47 +00:00 · 2022-02-24 17:10:23 +00:00 · 2022-02-23 11:04:02 +00:00
412 changed files with 7171 additions and 3096 deletions
@@ -8,7 +8,9 @@ export DEBIAN_FRONTEND=noninteractive
 set -ex

 apt-get update
-apt-get install -y python3 python3-dev python3-pip libxml2-dev libxslt-dev xmlsec1 zlib1g-dev tox libjpeg-dev libwebp-dev
+apt-get install -y \
+        python3 python3-dev python3-pip python3-venv \
+        libxml2-dev libxslt-dev xmlsec1 zlib1g-dev tox libjpeg-dev libwebp-dev

 export LANG="C.UTF-8"

@@ -0,0 +1,11 @@
+# TODO: incorporate this into pyproject.toml if flake8 supports it in the future.
+# See https://github.com/PyCQA/flake8/issues/234
+[flake8]
+# see https://pycodestyle.readthedocs.io/en/latest/intro.html#error-codes
+# for error codes. The ones we ignore are:
+#  W503: line break before binary operator
+#  W504: line break after binary operator
+#  E203: whitespace before ':' (which is contrary to pep8?)
+#  E731: do not assign a lambda expression, use a def
+#  E501: Line too long (black enforces this for us)
+ignore=W503,W504,E203,E731,E501
@@ -7,7 +7,7 @@ on:
  # of things breaking (but only build one set of debs)
  pull_request:
  push:
-    branches: ["develop"]
+    branches: ["develop", "release-*"]

    # we do the full build on tags.
    tags: ["v*"]
@@ -91,17 +91,7 @@ jobs:

  build-sdist:
    name: "Build pypi distribution files"
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
-      - uses: actions/setup-python@v2
-      - run: pip install wheel
-      - run: |
-          python setup.py sdist bdist_wheel
-      - uses: actions/upload-artifact@v2
-        with:
-          name: python-dist
-          path: dist/*
+    uses: "matrix-org/backend-meta/.github/workflows/packaging.yml@v1"

  # if it's a tag, create a release and attach the artifacts to it
  attach-assets:
@@ -48,24 +48,10 @@ jobs:
        env:
          PULL_REQUEST_NUMBER: ${{ github.event.number }}

-  lint-sdist:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
-      - uses: actions/setup-python@v2
-        with:
-          python-version: "3.x"
-      - run: pip install wheel
-      - run: python setup.py sdist bdist_wheel
-      - uses: actions/upload-artifact@v2
-        with:
-          name: Python Distributions
-          path: dist/*
-
  # Dummy step to gate other tests on without repeating the whole list
  linting-done:
    if: ${{ !cancelled() }} # Run this even if prior jobs were skipped
-    needs: [lint, lint-crlf, lint-newsfile, lint-sdist]
+    needs: [lint, lint-crlf, lint-newsfile]
    runs-on: ubuntu-latest
    steps:
      - run: "true"
@@ -345,7 +331,7 @@ jobs:
          path: synapse

      # Attempt to check out the same branch of Complement as the PR. If it
-      # doesn't exist, fallback to master.
+      # doesn't exist, fallback to HEAD.
      - name: Checkout complement
        shell: bash
        run: |
@@ -358,8 +344,8 @@ jobs:
          #    for pull requests, otherwise GITHUB_REF).
          # 2. Attempt to use the base branch, e.g. when merging into release-vX.Y
          #    (GITHUB_BASE_REF for pull requests).
-          # 3. Use the default complement branch ("master").
-          for BRANCH_NAME in "$GITHUB_HEAD_REF" "$GITHUB_BASE_REF" "${GITHUB_REF#refs/heads/}" "master"; do
+          # 3. Use the default complement branch ("HEAD").
+          for BRANCH_NAME in "$GITHUB_HEAD_REF" "$GITHUB_BASE_REF" "${GITHUB_REF#refs/heads/}" "HEAD"; do
            # Skip empty branch names and merge commits.
            if [[ -z "$BRANCH_NAME" || $BRANCH_NAME =~ ^refs/pull/.* ]]; then
              continue
@@ -383,7 +369,7 @@ jobs:
      # Run Complement
      - run: |
          set -o pipefail
-          go test -v -json -tags synapse_blacklist,msc2403 ./tests/... 2>&1 | gotestfmt
+          go test -v -json -p 1 -tags synapse_blacklist,msc2403 ./tests/... 2>&1 | gotestfmt
        shell: bash
        name: Run Complement Tests
        env:
@@ -397,7 +383,6 @@ jobs:
      - lint
      - lint-crlf
      - lint-newsfile
-      - lint-sdist
      - trial
      - trial-olddeps
      - sytest
@@ -1,3 +1,102 @@
+Synapse 1.53.0 (2022-02-22)
+===========================
+
+No significant changes since 1.53.0rc1.
+
+
+Synapse 1.53.0rc1 (2022-02-15)
+==============================
+
+Features
+--------
+
+- Add experimental support for sending to-device messages to application services, as specified by [MSC2409](https://github.com/matrix-org/matrix-doc/pull/2409). ([\#11215](https://github.com/matrix-org/synapse/issues/11215), [\#11966](https://github.com/matrix-org/synapse/issues/11966))
+- Add a background database update to purge account data for deactivated users. ([\#11655](https://github.com/matrix-org/synapse/issues/11655))
+- Experimental support for [MSC3666](https://github.com/matrix-org/matrix-doc/pull/3666): including bundled aggregations in server side search results. ([\#11837](https://github.com/matrix-org/synapse/issues/11837))
+- Enable cache time-based expiry by default. The `expiry_time` config flag has been superseded by `expire_caches` and `cache_entry_ttl`. ([\#11849](https://github.com/matrix-org/synapse/issues/11849))
+- Add a callback to allow modules to allow or forbid a 3PID (email address, phone number) from being associated to a local account. ([\#11854](https://github.com/matrix-org/synapse/issues/11854))
+- Stabilize support and remove unstable endpoints for [MSC3231](https://github.com/matrix-org/matrix-doc/pull/3231). Clients must switch to the stable identifier and endpoint. See the [upgrade notes](https://matrix-org.github.io/synapse/develop/upgrade#stablisation-of-msc3231) for more information. ([\#11867](https://github.com/matrix-org/synapse/issues/11867))
+- Allow modules to retrieve the current instance's server name and worker name. ([\#11868](https://github.com/matrix-org/synapse/issues/11868))
+- Use a dedicated configurable rate limiter for 3PID invites. ([\#11892](https://github.com/matrix-org/synapse/issues/11892))
+- Support the stable API endpoint for [MSC3283](https://github.com/matrix-org/matrix-doc/pull/3283): new settings in `/capabilities` endpoint. ([\#11933](https://github.com/matrix-org/synapse/issues/11933), [\#11989](https://github.com/matrix-org/synapse/issues/11989))
+- Support the `dir` parameter on the `/relations` endpoint, per [MSC3715](https://github.com/matrix-org/matrix-doc/pull/3715). ([\#11941](https://github.com/matrix-org/synapse/issues/11941))
+- Experimental implementation of [MSC3706](https://github.com/matrix-org/matrix-doc/pull/3706): extensions to `/send_join` to support reduced response size. ([\#11967](https://github.com/matrix-org/synapse/issues/11967))
+
+
+Bugfixes
+--------
+
+- Fix [MSC2716](https://github.com/matrix-org/matrix-doc/pull/2716) historical messages backfilling in random order on remote homeservers. ([\#11114](https://github.com/matrix-org/synapse/issues/11114))
+- Fix a bug introduced in Synapse 1.51.0 where incoming federation transactions containing at least one EDU would be dropped if debug logging was enabled for `synapse.8631_debug`. ([\#11890](https://github.com/matrix-org/synapse/issues/11890))
+- Fix a long-standing bug where some unknown endpoints would return HTML error pages instead of JSON `M_UNRECOGNIZED` errors. ([\#11930](https://github.com/matrix-org/synapse/issues/11930))
+- Implement an allow list of content types for which we will attempt to preview a URL. This prevents Synapse from making useless longer-lived connections to streaming media servers. ([\#11936](https://github.com/matrix-org/synapse/issues/11936))
+- Fix a long-standing bug where pagination tokens from `/sync` and `/messages` could not be provided to the `/relations` API. ([\#11952](https://github.com/matrix-org/synapse/issues/11952))
+- Require that modules register their callbacks using keyword arguments. ([\#11975](https://github.com/matrix-org/synapse/issues/11975))
+- Fix a long-standing bug where `M_WRONG_ROOM_KEYS_VERSION` errors would not include the specced `current_version` field. ([\#11988](https://github.com/matrix-org/synapse/issues/11988))
+
+
+Improved Documentation
+----------------------
+
+- Fix typo in User Admin API: unpind -> unbind. ([\#11859](https://github.com/matrix-org/synapse/issues/11859))
+- Document images returned by the User List Media Admin API can include those generated by URL previews. ([\#11862](https://github.com/matrix-org/synapse/issues/11862))
+- Remove outdated MSC1711 FAQ document. ([\#11907](https://github.com/matrix-org/synapse/issues/11907))
+- Correct the structured logging configuration example. Contributed by Brad Jones. ([\#11946](https://github.com/matrix-org/synapse/issues/11946))
+- Add information on the Synapse release cycle. ([\#11954](https://github.com/matrix-org/synapse/issues/11954))
+- Fix broken link in the README to the admin API for password reset. ([\#11955](https://github.com/matrix-org/synapse/issues/11955))
+
+
+Deprecations and Removals
+-------------------------
+
+- Drop support for `webclient` listeners and configuring `web_client_location` to a non-HTTP(S) URL. Deprecated configurations are a configuration error. ([\#11895](https://github.com/matrix-org/synapse/issues/11895))
+- Remove deprecated `user_may_create_room_with_invites` spam checker callback. See the [upgrade notes](https://matrix-org.github.io/synapse/latest/upgrade.html#removal-of-user_may_create_room_with_invites) for more information. ([\#11950](https://github.com/matrix-org/synapse/issues/11950))
+- No longer build `.deb` packages for Ubuntu 21.04 Hirsute Hippo, which has now EOLed. ([\#11961](https://github.com/matrix-org/synapse/issues/11961))
+
+
+Internal Changes
+----------------
+
+- Enhance user registration test helpers to make them more useful for tests involving application services and devices. ([\#11615](https://github.com/matrix-org/synapse/issues/11615), [\#11616](https://github.com/matrix-org/synapse/issues/11616))
+- Improve performance when fetching bundled aggregations for multiple events. ([\#11660](https://github.com/matrix-org/synapse/issues/11660), [\#11752](https://github.com/matrix-org/synapse/issues/11752))
+- Fix type errors introduced by new annotations in the Prometheus Client library. ([\#11832](https://github.com/matrix-org/synapse/issues/11832))
+- Add missing type hints to replication code. ([\#11856](https://github.com/matrix-org/synapse/issues/11856), [\#11938](https://github.com/matrix-org/synapse/issues/11938))
+- Ensure that `opentracing` scopes are activated and closed at the right time. ([\#11869](https://github.com/matrix-org/synapse/issues/11869))
+- Improve opentracing for incoming federation requests. ([\#11870](https://github.com/matrix-org/synapse/issues/11870))
+- Improve internal docstrings in `synapse.util.caches`. ([\#11876](https://github.com/matrix-org/synapse/issues/11876))
+- Do not needlessly clear the `get_users_in_room` and `get_users_in_room_with_profiles` caches when any room state changes. ([\#11878](https://github.com/matrix-org/synapse/issues/11878))
+- Convert `ApplicationServiceTestCase` to use `simple_async_mock`. ([\#11880](https://github.com/matrix-org/synapse/issues/11880))
+- Remove experimental changes to the default push rules which were introduced in Synapse 1.19.0 but never enabled. ([\#11884](https://github.com/matrix-org/synapse/issues/11884))
+- Disable coverage calculation for olddeps build. ([\#11888](https://github.com/matrix-org/synapse/issues/11888))
+- Preparation to support sending device list updates to application services. ([\#11905](https://github.com/matrix-org/synapse/issues/11905))
+- Add a test that checks users receive their own device list updates down `/sync`. ([\#11909](https://github.com/matrix-org/synapse/issues/11909))
+- Run Complement tests sequentially. ([\#11910](https://github.com/matrix-org/synapse/issues/11910))
+- Various refactors to the application service notifier code. ([\#11911](https://github.com/matrix-org/synapse/issues/11911), [\#11912](https://github.com/matrix-org/synapse/issues/11912))
+- Tests: replace mocked `Authenticator` with the real thing. ([\#11913](https://github.com/matrix-org/synapse/issues/11913))
+- Various refactors to the typing notifications code. ([\#11914](https://github.com/matrix-org/synapse/issues/11914))
+- Use the proper type for the `Content-Length` header in the `UploadResource`. ([\#11927](https://github.com/matrix-org/synapse/issues/11927))
+- Remove an unnecessary ignoring of type hints due to fixes in upstream packages. ([\#11939](https://github.com/matrix-org/synapse/issues/11939))
+- Add missing type hints. ([\#11953](https://github.com/matrix-org/synapse/issues/11953))
+- Fix an import cycle in `synapse.event_auth`. ([\#11965](https://github.com/matrix-org/synapse/issues/11965))
+- Unpin `frozendict` but exclude the known bad version 2.1.2. ([\#11969](https://github.com/matrix-org/synapse/issues/11969))
+- Prepare for rename of default Complement branch. ([\#11971](https://github.com/matrix-org/synapse/issues/11971))
+- Fetch Synapse's version using a helper from `matrix-common`. ([\#11979](https://github.com/matrix-org/synapse/issues/11979))
+
+
+Synapse 1.52.0 (2022-02-08)
+===========================
+
+No significant changes since 1.52.0rc1.
+
+Note that [Twisted 22.1.0](https://github.com/twisted/twisted/releases/tag/twisted-22.1.0)
+has recently been released, which fixes a [security issue](https://github.com/twisted/twisted/security/advisories/GHSA-92x2-jw7w-xvvx)
+within the Twisted library. We do not believe Synapse is affected by this vulnerability,
+though we advise server administrators who installed Synapse via pip to upgrade Twisted
+with `pip install --upgrade Twisted treq` as a matter of good practice. The Docker image
+`matrixdotorg/synapse` and the Debian packages from `packages.matrix.org` are using the
+updated library.
+
+
 Synapse 1.52.0rc1 (2022-02-01)
 ==============================

@@ -7,7 +106,7 @@ Features
 - Remove account data (including client config, push rules and ignored users) upon user deactivation. ([\#11621](https://github.com/matrix-org/synapse/issues/11621), [\#11788](https://github.com/matrix-org/synapse/issues/11788), [\#11789](https://github.com/matrix-org/synapse/issues/11789))
 - Add an admin API to reset connection timeouts for remote server. ([\#11639](https://github.com/matrix-org/synapse/issues/11639))
 - Add an admin API to get a list of rooms that federate with a given remote homeserver. ([\#11658](https://github.com/matrix-org/synapse/issues/11658))
- Add a config flag to inhibit M_USER_IN_USE during registration. ([\#11743](https://github.com/matrix-org/synapse/issues/11743))
+- Add a config flag to inhibit `M_USER_IN_USE` during registration. ([\#11743](https://github.com/matrix-org/synapse/issues/11743))
 - Add a module callback to set username at registration. ([\#11790](https://github.com/matrix-org/synapse/issues/11790))
 - Allow configuring a maximum file size as well as a list of allowed content types for avatars. ([\#11846](https://github.com/matrix-org/synapse/issues/11846))

@@ -174,7 +273,7 @@ Bugfixes
 Synapse 1.50.0 (2022-01-18)
 ===========================

-**This release contains a critical bug that may prevent clients from being able to connect. 
+**This release contains a critical bug that may prevent clients from being able to connect.
 As such, it is not recommended to upgrade to 1.50.0. Instead, please upgrade straight to
 to 1.50.1. Further details are available in [this issue](https://github.com/matrix-org/synapse/issues/11763).**

@@ -45,6 +45,7 @@ include book.toml
 include pyproject.toml
 recursive-include changelog.d *

+include .flake8
 prune .circleci
 prune .github
 prune .ci
@@ -246,7 +246,7 @@ Password reset
 ==============

 Users can reset their password through their client. Alternatively, a server admin
-can reset a users password using the `admin API <docs/admin_api/user_admin_api.rst#reset-password>`_
+can reset a users password using the `admin API <docs/admin_api/user_admin_api.md#reset-password>`_
 or by directly editing the database as shown below.

 First calculate the hash of the new password::
@@ -0,0 +1 @@
+Deduplicate in-flight requests in `_get_state_for_groups`.
@@ -1 +0,0 @@
-Add experimental support for sending to-device messages to application services, as specified by [MSC2409](https://github.com/matrix-org/matrix-doc/pull/2409). Disabled by default.
@@ -0,0 +1 @@
+Deduplicate in-flight requests in `_get_state_for_groups`.
@@ -1 +0,0 @@
-Enhance user registration test helpers to make them more useful for tests involving Application Services and devices.
@@ -1 +0,0 @@
-Enhance user registration test helpers to make them more useful for tests involving Application Services and devices.
@@ -1 +0,0 @@
-Remove account data (including client config, push rules and ignored users) upon user deactivation.
@@ -1 +0,0 @@
-Fix type errors introduced by new annotations in the Prometheus Client library.
@@ -0,0 +1 @@
+Make a `POST` to `/rooms/<room_id>/receipt/m.read/<event_id>` only trigger a push notification if the count of unread messages is different to the one in the last successfully sent push.
@@ -1 +0,0 @@
-Allow modules to retrieve the current instance's server name and worker name.
@@ -1 +0,0 @@
-Ensure that `opentracing` scopes are activated and closed at the right time.
@@ -1 +0,0 @@
-Improve opentracing for incoming federation requests.
@@ -1 +0,0 @@
-Improve internal docstrings in `synapse.util.caches`.
@@ -1 +0,0 @@
-Do not needlessly clear the `get_users_in_room` and `get_users_in_room_with_profiles` caches when any room state changes.
@@ -1 +0,0 @@
-Convert `ApplicationServiceTestCase` to use `simple_async_mock`.
@@ -1 +0,0 @@
-Remove experimental changes to the default push rules which were introduced in Synapse 1.19.0 but never enabled.
@@ -1 +0,0 @@
-Disable coverage calculation for olddeps build.
@@ -1 +0,0 @@
-Fix a bug introduced in Synapse 1.51.0rc1 where incoming federation transactions containing at least one EDU would be dropped if debug logging was enabled for `synapse.8631_debug`.
@@ -1 +0,0 @@
-Use a dedicated configurable rate limiter for 3PID invites.
@@ -0,0 +1 @@
+Remove unnecessary condition on knock->leave auth rule check.
@@ -0,0 +1 @@
+Add tests for device list changes between local users.
@@ -0,0 +1 @@
+Optimise calculating device_list changes in `/sync`.
@@ -0,0 +1 @@
+Add missing type hints to storage classes.
@@ -0,0 +1 @@
+Fetch images when previewing Twitter URLs. Contributed by @AndrewRyanChama.
@@ -0,0 +1 @@
+Refactor the search code for improved readability.
@@ -0,0 +1 @@
+Fix a bug introduced in Synapse v1.48.0 where an edit of the latest event in a thread would not be properly applied to the thread summary.
@@ -0,0 +1 @@
+Move common deduplication code down into `_auth_and_persist_outliers`.
@@ -0,0 +1 @@
+Limit concurrent joins from applications services.
@@ -0,0 +1 @@
+The docker image no longer automatically creates a temporary volume at `/data`. This is not expected to affect normal usage.
@@ -0,0 +1 @@
+Fix long standing bug where `get_rooms_for_user` was not correctly invalidated for remote users when the server left a room.
@@ -0,0 +1 @@
+Track cache invalidations in Prometheus metrics, as already happens for cache eviction based on size or time.
@@ -0,0 +1 @@
+Implement experimental support for [MSC3720](https://github.com/matrix-org/matrix-doc/pull/3720) (account status endpoints).
@@ -0,0 +1 @@
+Explain the meaning of spam checker callbacks' return values.
@@ -0,0 +1 @@
+Clarify information about external Identity Provider IDs.
@@ -0,0 +1 @@
+Preparation for faster-room-join work: when parsing the `send_join` response, get the `m.room.create` event from `state`, not `auth_chain`.
@@ -0,0 +1 @@
+Remove support for the legacy structured logging configuration (please see the the [upgrade notes](https://matrix-org.github.io/synapse/develop/upgrade#legacy-structured-logging-configuration-removal) if you are using `structured: true` in the Synapse configuration).
@@ -0,0 +1 @@
+Enable modules to set a custom display name when registering a user.
@@ -0,0 +1 @@
+Preparation for faster-room-join work: parse msc3706 fields in send_join response.
@@ -0,0 +1 @@
+Preparation for faster-room-join work: Support for calling `/federation/v1/state` on a remote server.
@@ -0,0 +1 @@
+Configure `tox` to use `venv` rather than `virtualenv`.
@@ -0,0 +1 @@
+Fix bug in `StateFilter.return_expanded()` and add some tests.
@@ -0,0 +1 @@
+Drop support for [MSC3283](https://github.com/matrix-org/matrix-doc/pull/3283) unstable flags now that the stable flags are supported.
@@ -0,0 +1 @@
+Use Matrix v1.1 endpoints (`/_matrix/client/v3/auth/...`) in fallback auth HTML forms.
@@ -0,0 +1 @@
+Advertise Matrix 1.1 support on `/_matrix/client/versions`.
@@ -0,0 +1 @@
+Support only the stable identifier for [MSC3069](https://github.com/matrix-org/matrix-doc/pull/3069)'s `is_guest` on `/_matrix/client/v3/account/whoami`.
@@ -0,0 +1 @@
+Advertise Matrix 1.2 support on `/_matrix/client/versions`.
@@ -0,0 +1 @@
+Fix 500 error with Postgres when looking backwards with the [MSC3030](https://github.com/matrix-org/matrix-doc/pull/3030) `/timestamp_to_event?dir=b` endpoint.
@@ -0,0 +1 @@
+Update the `olddeps` CI job to use an old version of `markupsafe`.
@@ -0,0 +1 @@
+Upgrade mypy to version 0.931.
@@ -0,0 +1 @@
+Remove legacy `HomeServer.get_datastore()`.
@@ -0,0 +1 @@
+Deduplicate in-flight requests in `_get_state_for_groups`.
@@ -0,0 +1 @@
+Minor typing fixes.
@@ -0,0 +1 @@
+Preparation for faster-room-join work: when parsing the `send_join` response, get the `m.room.create` event from `state`, not `auth_chain`.
@@ -0,0 +1 @@
+After joining a room, create a dedicated logcontext to process the queued events.
@@ -0,0 +1 @@
+Tidy up GitHub Actions config which builds distributions for PyPI.
@@ -0,0 +1 @@
+Move configuration out of `setup.cfg`.
@@ -0,0 +1 @@
+Fix slow performance of `/logout` in some cases where refresh tokens are in use. The slowness existed since the initial implementation of refresh tokens.
@@ -0,0 +1 @@
+Use room version 9 as the default room version (per [MSC3589](https://github.com/matrix-org/matrix-doc/pull/3589)).
@@ -0,0 +1 @@
+Move configuration out of `setup.cfg`.
@@ -0,0 +1 @@
+Fix error message when a worker process fails to talk to another worker process.
@@ -0,0 +1 @@
+Fix using the complement.sh script without specifying a dir or a branch. Contributed by Nico on behalf of Famedly.
@@ -1,3 +1,21 @@
+matrix-synapse-py3 (1.53.0) stable; urgency=medium
+
+  * New synapse release 1.53.0.
+
+ -- Synapse Packaging team <packages@matrix.org>  Tue, 22 Feb 2022 11:32:06 +0000
+
+matrix-synapse-py3 (1.53.0~rc1) stable; urgency=medium
+
+  * New synapse release 1.53.0~rc1.
+
+ -- Synapse Packaging team <packages@matrix.org>  Tue, 15 Feb 2022 10:40:50 +0000
+
+matrix-synapse-py3 (1.52.0) stable; urgency=medium
+
+  * New synapse release 1.52.0.
+
+ -- Synapse Packaging team <packages@matrix.org>  Tue, 08 Feb 2022 11:34:54 +0000
+
 matrix-synapse-py3 (1.52.0~rc1) stable; urgency=medium

  * New synapse release 1.52.0~rc1.
@@ -98,8 +98,6 @@ COPY --from=builder /install /usr/local
 COPY ./docker/start.py /start.py
 COPY ./docker/conf /conf

-VOLUME ["/data"]
-
 EXPOSE 8008/tcp 8009/tcp 8448/tcp

 ENTRYPOINT ["/start.py"]
@@ -1,314 +0,0 @@
-# MSC1711 Certificates FAQ
-
-## Historical Note
-This document was originally written to guide server admins through the upgrade
-path towards Synapse 1.0. Specifically,
-[MSC1711](https://github.com/matrix-org/matrix-doc/blob/main/proposals/1711-x509-for-federation.md)
-required that all servers present valid TLS certificates on their federation
-API. Admins were encouraged to achieve compliance from version 0.99.0 (released
-in February 2019) ahead of version 1.0 (released June 2019) enforcing the
-certificate checks.
-
-Much of what follows is now outdated since most admins will have already
-upgraded, however it may be of use to those with old installs returning to the
-project.
-
-If you are setting up a server from scratch you almost certainly should look at
-the [installation guide](setup/installation.md) instead.
-
-## Introduction
-The goal of Synapse 0.99.0 is to act as a stepping stone to Synapse 1.0.0. It
-supports the r0.1 release of the server to server specification, but is
-compatible with both the legacy Matrix federation behaviour (pre-r0.1) as well
-as post-r0.1 behaviour, in order to allow for a smooth upgrade across the
-federation.
-
-The most important thing to know is that Synapse 1.0.0 will require a valid TLS
-certificate on federation endpoints. Self signed certificates will not be
-sufficient.
-
-Synapse 0.99.0 makes it easy to configure TLS certificates and will
-interoperate with both >= 1.0.0 servers as well as existing servers yet to
-upgrade.
-
-**It is critical that all admins upgrade to 0.99.0 and configure a valid TLS
-certificate.** Admins will have 1 month to do so, after which 1.0.0 will be
-released and those servers without a valid certificate will not longer be able
-to federate with >= 1.0.0 servers.
-
-Full details on how to carry out this configuration change is given
-[below](#configuring-certificates-for-compatibility-with-synapse-100). A
-timeline and some frequently asked questions are also given below.
-
-For more details and context on the release of the r0.1 Server/Server API and
-imminent Matrix 1.0 release, you can also see our
-[main talk from FOSDEM 2019](https://matrix.org/blog/2019/02/04/matrix-at-fosdem-2019/).
-
-## Timeline
-
-**5th Feb 2019  - Synapse 0.99.0 is released.**
-
-All server admins are encouraged to upgrade.
-
-0.99.0:
-
-   provides support for ACME to make setting up Let's Encrypt certs easy, as
-    well as .well-known support.
-
-   does not enforce that a valid CA cert is present on the federation API, but
-    rather makes it easy to set one up.
-
-   provides support for .well-known
-
-Admins should upgrade and configure a valid CA cert. Homeservers that require a
-.well-known entry (see below), should retain their SRV record and use it
-alongside their .well-known record.
-
-**10th June 2019  - Synapse 1.0.0 is released**
-
-1.0.0 is scheduled for release on 10th June. In
-accordance with the the [S2S spec](https://matrix.org/docs/spec/server_server/r0.1.0.html)
-1.0.0 will enforce certificate validity. This means that any homeserver without a
-valid certificate after this point will no longer be able to federate with
-1.0.0 servers.
-
-## Configuring certificates for compatibility with Synapse 1.0.0
-
-### If you do not currently have an SRV record
-
-In this case, your `server_name` points to the host where your Synapse is
-running. There is no need to create a `.well-known` URI or an SRV record, but
-you will need to give Synapse a valid, signed, certificate.
-
-### If you do have an SRV record currently
-
-If you are using an SRV record, your matrix domain (`server_name`) may not
-point to the same host that your Synapse is running on (the 'target
-domain'). (If it does, you can follow the recommendation above; otherwise, read
-on.)
-
-Let's assume that your `server_name` is `example.com`, and your Synapse is
-hosted at a target domain of `customer.example.net`. Currently you should have
-an SRV record which looks like:
-
-```
-_matrix._tcp.example.com. IN SRV 10 5 8000 customer.example.net.
-```
-
-In this situation, you have three choices for how to proceed:
-
-#### Option 1: give Synapse a certificate for your matrix domain
-
-Synapse 1.0 will expect your server to present a TLS certificate for your
-`server_name` (`example.com` in the above example). You can achieve this by acquiring a
-certificate for the `server_name` yourself (for example, using `certbot`), and giving it
-and the key to Synapse via `tls_certificate_path` and `tls_private_key_path`.
-
-#### Option 2: run Synapse behind a reverse proxy
-
-If you have an existing reverse proxy set up with correct TLS certificates for
-your domain, you can simply route all traffic through the reverse proxy by
-updating the SRV record appropriately (or removing it, if the proxy listens on
-8448).
-
-See [the reverse proxy documentation](reverse_proxy.md) for information on setting up a
-reverse proxy.
-
-#### Option 3: add a .well-known file to delegate your matrix traffic
-
-This will allow you to keep Synapse on a separate domain, without having to
-give it a certificate for the matrix domain.
-
-You can do this with a `.well-known` file as follows:
-
- 1. Keep the SRV record in place - it is needed for backwards compatibility
-    with Synapse 0.34 and earlier.
-
-  2. Give Synapse a certificate corresponding to the target domain
-    (`customer.example.net` in the above example). You can do this by acquire a 
-    certificate for the target domain and giving it to Synapse via `tls_certificate_path`
-    and `tls_private_key_path`.
-
- 3. Restart Synapse to ensure the new certificate is loaded.
-
- 4. Arrange for a `.well-known` file at
-    `https://<server_name>/.well-known/matrix/server` with contents:
-
-    ```json
-    {"m.server": "<target server name>"}
-    ```
-
-    where the target server name is resolved as usual (i.e. SRV lookup, falling
-    back to talking to port 8448).
-
-    In the above example, where synapse is listening on port 8000,
-    `https://example.com/.well-known/matrix/server` should have `m.server` set to one of:
-
-    1. `customer.example.net` ─ with a SRV record on
-       `_matrix._tcp.customer.example.com` pointing to port 8000, or:
-
-    2. `customer.example.net` ─ updating synapse to listen on the default port
-       8448, or:
-
-    3. `customer.example.net:8000` ─ ensuring that if there is a reverse proxy
-       on `customer.example.net:8000` it correctly handles HTTP requests with
-       Host header set to `customer.example.net:8000`.
-
-## FAQ
-
-### Synapse 0.99.0 has just been released, what do I need to do right now?
-
-Upgrade as soon as you can in preparation for Synapse 1.0.0, and update your
-TLS certificates as [above](#configuring-certificates-for-compatibility-with-synapse-100).
-
-### What will happen if I do not set up a valid federation certificate immediately?
-
-Nothing initially, but once 1.0.0 is in the wild it will not be possible to
-federate with 1.0.0 servers.
-
-### What will happen if I do nothing at all?
-
-If the admin takes no action at all, and remains on a Synapse < 0.99.0 then the
-homeserver will be unable to federate with those who have implemented
-.well-known. Then, as above, once the month upgrade window has expired the
-homeserver will not be able to federate with any Synapse >= 1.0.0
-
-### When do I need a SRV record or .well-known URI?
-
-If your homeserver listens on the default federation port (8448), and your
-`server_name` points to the host that your homeserver runs on, you do not need an
-SRV record or `.well-known/matrix/server` URI.
-
-For instance, if you registered `example.com` and pointed its DNS A record at a
-fresh Upcloud VPS or similar, you could install Synapse 0.99 on that host,
-giving it a server_name of `example.com`, and it would automatically generate a
-valid TLS certificate for you via Let's Encrypt and no SRV record or
-`.well-known` URI would be needed.
-
-This is the common case, although you can add an SRV record or
-`.well-known/matrix/server` URI for completeness if you wish.
-
-**However**, if your server does not listen on port 8448, or if your `server_name`
-does not point to the host that your homeserver runs on, you will need to let
-other servers know how to find it.
-
-In this case, you should see ["If you do have an SRV record
-currently"](#if-you-do-have-an-srv-record-currently) above.
-
-### Can I still use an SRV record?
-
-Firstly, if you didn't need an SRV record before (because your server is
-listening on port 8448 of your server_name), you certainly don't need one now:
-the defaults are still the same.
-
-If you previously had an SRV record, you can keep using it provided you are
-able to give Synapse a TLS certificate corresponding to your server name. For
-example, suppose you had the following SRV record, which directs matrix traffic
-for example.com to matrix.example.com:443:
-
-```
-_matrix._tcp.example.com. IN SRV 10 5 443 matrix.example.com
-```
-
-In this case, Synapse must be given a certificate for example.com - or be
-configured to acquire one from Let's Encrypt.
-
-If you are unable to give Synapse a certificate for your server_name, you will
-also need to use a .well-known URI instead. However, see also "I have created a
-.well-known URI. Do I still need an SRV record?".
-
-### I have created a .well-known URI. Do I still need an SRV record?
-
-As of Synapse 0.99, Synapse will first check for the existence of a `.well-known`
-URI and follow any delegation it suggests. It will only then check for the
-existence of an SRV record.
-
-That means that the SRV record will often be redundant. However, you should
-remember that there may still be older versions of Synapse in the federation
-which do not understand `.well-known` URIs, so if you removed your SRV record you
-would no longer be able to federate with them.
-
-It is therefore best to leave the SRV record in place for now. Synapse 0.34 and
-earlier will follow the SRV record (and not care about the invalid
-certificate). Synapse 0.99 and later will follow the .well-known URI, with the
-correct certificate chain.
-
-### It used to work just fine, why are you breaking everything?
-
-We have always wanted Matrix servers to be as easy to set up as possible, and
-so back when we started federation in 2014 we didn't want admins to have to go
-through the cumbersome process of buying a valid TLS certificate to run a
-server. This was before Let's Encrypt came along and made getting a free and
-valid TLS certificate straightforward. So instead, we adopted a system based on
-[Perspectives](https://en.wikipedia.org/wiki/Convergence_(SSL)): an approach
-where you check a set of "notary servers" (in practice, homeservers) to vouch
-for the validity of a certificate rather than having it signed by a CA. As long
-as enough different notaries agree on the certificate's validity, then it is
-trusted.
-
-However, in practice this has never worked properly. Most people only use the
-default notary server (matrix.org), leading to inadvertent centralisation which
-we want to eliminate. Meanwhile, we never implemented the full consensus
-algorithm to query the servers participating in a room to determine consensus
-on whether a given certificate is valid. This is fiddly to get right
-(especially in face of sybil attacks), and we found ourselves questioning
-whether it was worth the effort to finish the work and commit to maintaining a
-secure certificate validation system as opposed to focusing on core Matrix
-development.
-
-Meanwhile, Let's Encrypt came along in 2016, and put the final nail in the
-coffin of the Perspectives project (which was already pretty dead). So, the
-Spec Core Team decided that a better approach would be to mandate valid TLS
-certificates for federation alongside the rest of the Web. More details can be
-found in
-[MSC1711](https://github.com/matrix-org/matrix-doc/blob/main/proposals/1711-x509-for-federation.md#background-the-failure-of-the-perspectives-approach).
-
-This results in a breaking change, which is disruptive, but absolutely critical
-for the security model. However, the existence of Let's Encrypt as a trivial
-way to replace the old self-signed certificates with valid CA-signed ones helps
-smooth things over massively, especially as Synapse can now automate Let's
-Encrypt certificate generation if needed.
-
-### Can I manage my own certificates rather than having Synapse renew certificates itself?
-
-Yes, you are welcome to manage your certificates yourself. Synapse will only
-attempt to obtain certificates from Let's Encrypt if you configure it to do
-so.The only requirement is that there is a valid TLS cert present for
-federation end points.
-
-### Do you still recommend against using a reverse proxy on the federation port?
-
-We no longer actively recommend against using a reverse proxy. Many admins will
-find it easier to direct federation traffic to a reverse proxy and manage their
-own TLS certificates, and this is a supported configuration.
-
-See [the reverse proxy documentation](reverse_proxy.md) for information on setting up a
-reverse proxy.
-
-### Do I still need to give my TLS certificates to Synapse if I am using a reverse proxy?
-
-Practically speaking, this is no longer necessary.
-
-If you are using a reverse proxy for all of your TLS traffic, then you can set
-`no_tls: True`. In that case, the only reason Synapse needs the certificate is
-to populate a legacy 'tls_fingerprints' field in the federation API. This is
-ignored by Synapse 0.99.0 and later, and the only time pre-0.99 Synapses will
-check it is when attempting to fetch the server keys - and generally this is
-delegated via `matrix.org`, which is on 0.99.0.
-
-However, there is a bug in Synapse 0.99.0
-[4554](<https://github.com/matrix-org/synapse/issues/4554>) which prevents
-Synapse from starting if you do not give it a TLS certificate. To work around
-this, you can give it any TLS certificate at all. This will be fixed soon.
-
-### Do I need the same certificate for the client and federation port?
-
-No. There is nothing stopping you from using different certificates,
-particularly if you are using a reverse proxy. However, Synapse will use the
-same certificate on any ports where TLS is configured.
-
-### How do I tell Synapse to reload my keys/certificates after I replace them?
-
-Synapse will reload the keys and certificates when it receives a SIGHUP - for
-example `kill -HUP $(cat homeserver.pid)`. Alternatively, simply restart
-Synapse, though this will result in downtime while it restarts.
@@ -13,7 +13,6 @@

 # Upgrading
  - [Upgrading between Synapse Versions](upgrade.md)
-  - [Upgrading from pre-Synapse 1.0](MSC1711_certificates_FAQ.md)

 # Usage
  - [Federation](federate.md)
@@ -72,7 +71,7 @@
    - [Understanding Synapse Through Grafana Graphs](usage/administration/understanding_synapse_through_grafana_graphs.md)
    - [Useful SQL for Admins](usage/administration/useful_sql_for_admins.md)
    - [Database Maintenance Tools](usage/administration/database_maintenance_tools.md)
-    - [State Groups](usage/administration/state_groups.md) 
+    - [State Groups](usage/administration/state_groups.md)
    - [Request log format](usage/administration/request_log.md)
    - [Admin FAQ](usage/administration/admin_faq.md)
    - [Scripts]()
@@ -80,6 +79,7 @@
 # Development
  - [Contributing Guide](development/contributing_guide.md)
  - [Code Style](code_style.md)
+  - [Release Cycle](development/releases.md)
  - [Git Usage](development/git.md)
  - [Testing]()
  - [OpenTracing](opentracing.md)
@@ -2,6 +2,9 @@

 These APIs allow extracting media information from the homeserver.

+Details about the format of the `media_id` and storage of the media in the file system
+are documented under [media repository](../media_repository.md).
+
 To use it, you will need to authenticate by providing an `access_token`
 for a server admin: see [Admin API](../usage/administration/admin_api).

@@ -126,7 +126,8 @@ Body parameters:
  [Sample Configuration File](../usage/configuration/homeserver_sample_config.html)
  section `sso` and `oidc_providers`.
  - `auth_provider` - string. ID of the external identity provider. Value of `idp_id`
-    in homeserver configuration.
+    in the homeserver configuration. Note that no error is raised if the provided
+    value is not in the homeserver configuration.
  - `external_id` - string, user ID in the external identity provider.
 - `avatar_url` - string, optional, must be a
  [MXC URI](https://matrix.org/docs/spec/client_server/r0.6.0#matrix-content-mxc-uris).
@@ -331,7 +332,7 @@ An empty body may be passed for backwards compatibility.

 The following actions are performed when deactivating an user:

- Try to unpind 3PIDs from the identity server
+- Try to unbind 3PIDs from the identity server
 - Remove all 3PIDs from the homeserver
 - Delete all devices and E2EE keys
 - Delete all access tokens
@@ -539,6 +540,11 @@ The following fields are returned in the JSON response body:

 ### List media uploaded by a user
 Gets a list of all local media that a specific `user_id` has created.
+These are media that the user has uploaded themselves
+([local media](../media_repository.md#local-media)), as well as
+[URL preview images](../media_repository.md#url-previews) requested by the user if the
+[feature is enabled](../development/url_previews.md).
+
 By default, the response is ordered by descending creation date and ascending media ID.
 The newest media is on top. You can change the order with parameters
 `order_by` and `dir`.
@@ -635,7 +641,9 @@ The following fields are returned in the JSON response body:
  Media objects contain the following fields:
  - `created_ts` - integer - Timestamp when the content was uploaded in ms.
  - `last_access_ts` - integer - Timestamp when the content was last accessed in ms.
-  - `media_id` - string - The id used to refer to the media.
+  - `media_id` - string - The id used to refer to the media. Details about the format
+    are documented under
+    [media repository](../media_repository.md).
  - `media_length` - integer - Length of the media in bytes.
  - `media_type` - string - The MIME-type of the media.
  - `quarantined_by` - string - The user ID that initiated the quarantine request
@@ -0,0 +1,37 @@
+# Synapse Release Cycle
+
+Releases of Synapse follow a two week release cycle with new releases usually
+occurring on Tuesdays:
+
+* Day 0: Synapse `N - 1` is released.
+* Day 7: Synapse `N` release candidate 1 is released.
+* Days 7 - 13: Synapse `N` release candidates 2+ are released, if bugs are found.
+* Day 14: Synapse `N` is released.
+
+Note that this schedule might be modified depending on the availability of the
+Synapse team, e.g. releases may be skipped to avoid holidays.
+
+Release announcements can be found in the
+[release category of the Matrix blog](https://matrix.org/blog/category/releases).
+
+## Bugfix releases
+
+If a bug is found after release that is deemed severe enough (by a combination
+of the impacted users and the impact on those users) then a bugfix release may
+be issued. This may be at any point in the release cycle.
+
+## Security releases
+
+Security will sometimes be backported to the previous version and released
+immediately before the next release candidate. An example of this might be:
+
+* Day 0: Synapse N - 1 is released.
+* Day 7: Synapse (N - 1).1 is released as Synapse N - 1 + the security fix.
+* Day 7: Synapse N release candidate 1 is released (including the security fix).
+
+Depending on the impact and complexity of security fixes, multiple fixes might
+be held to be released together.
+
+In some cases, a pre-disclosure of a security release will be issued as a notice
+to Synapse operators that there is an upcoming security release. These can be
+found in the [security category of the Matrix blog](https://matrix.org/blog/category/security).
@@ -94,6 +94,6 @@ As a simple example, retrieving an event from the database:

 ```pycon
 >>> from twisted.internet import defer
->>> defer.ensureDeferred(hs.get_datastore().get_event('$1416420717069yeQaw:matrix.org'))
+>>> defer.ensureDeferred(hs.get_datastores().main.get_event('$1416420717069yeQaw:matrix.org'))
 <Deferred at 0x7ff253fc6998 current result: <FrozenEvent event_id='$1416420717069yeQaw:matrix.org', type='m.room.create', state_key=''>>
 ```
@@ -85,7 +85,7 @@ If the authentication is unsuccessful, the module must return `None`.
 If multiple modules implement this callback, they will be considered in order. If a
 callback returns `None`, Synapse falls through to the next one. The value of the first
 callback that does not return `None` will be used. If this happens, Synapse will not call
-any of the subsequent implementations of this callback. If every callback return `None`,
+any of the subsequent implementations of this callback. If every callback returns `None`,
 the authentication is denied.

 ### `on_logged_out`
@@ -148,7 +148,7 @@ Here's an example featuring all currently supported keys:
        "address": "33123456789",
        "validated_at": 1642701357084,
    },
-    "org.matrix.msc3231.login.registration_token": "sometoken",  # User has registered through the flow described in MSC3231
+    "m.login.registration_token": "sometoken",  # User has registered through a registration token
 }
 ```

@@ -162,10 +162,57 @@ return `None`.
 If multiple modules implement this callback, they will be considered in order. If a
 callback returns `None`, Synapse falls through to the next one. The value of the first
 callback that does not return `None` will be used. If this happens, Synapse will not call
-any of the subsequent implementations of this callback. If every callback return `None`,
+any of the subsequent implementations of this callback. If every callback returns `None`,
 the username provided by the user is used, if any (otherwise one is automatically
 generated).

+### `get_displayname_for_registration`
+
+_First introduced in Synapse v1.54.0_
+
+```python
+async def get_displayname_for_registration(
+    uia_results: Dict[str, Any],
+    params: Dict[str, Any],
+) -> Optional[str]
+```
+
+Called when registering a new user. The module can return a display name to set for the
+user being registered by returning it as a string, or `None` if it doesn't wish to force a
+display name for this user.
+
+This callback is called once [User-Interactive Authentication](https://spec.matrix.org/latest/client-server-api/#user-interactive-authentication-api)
+has been completed by the user. It is not called when registering a user via SSO. It is
+passed two dictionaries, which include the information that the user has provided during
+the registration process. These dictionaries are identical to the ones passed to
+[`get_username_for_registration`](#get_username_for_registration), so refer to the
+documentation of this callback for more information about them.
+
+If multiple modules implement this callback, they will be considered in order. If a
+callback returns `None`, Synapse falls through to the next one. The value of the first
+callback that does not return `None` will be used. If this happens, Synapse will not call
+any of the subsequent implementations of this callback. If every callback returns `None`,
+the username will be used (e.g. `alice` if the user being registered is `@alice:example.com`).
+
+## `is_3pid_allowed`
+
+_First introduced in Synapse v1.53.0_
+
+```python
+async def is_3pid_allowed(self, medium: str, address: str, registration: bool) -> bool
+```
+
+Called when attempting to bind a third-party identifier (i.e. an email address or a phone
+number). The module is given the medium of the third-party identifier (which is `email` if
+the identifier is an email address, or `msisdn` if the identifier is a phone number) and
+its address, as well as a boolean indicating whether the attempt to bind is happening as
+part of registering a new user. The module must return a boolean indicating whether the
+identifier can be allowed to be bound to an account on the local homeserver.
+
+If multiple modules implement this callback, they will be considered in order. If a
+callback returns `True`, Synapse falls through to the next one. The value of the first
+callback that does not return `True` will be used. If this happens, Synapse will not call
+any of the subsequent implementations of this callback.

 ## Example

@@ -175,8 +222,7 @@ The example module below implements authentication checkers for two different lo
    - Is checked by the method: `self.check_my_login`
 - `m.login.password` (defined in [the spec](https://matrix.org/docs/spec/client_server/latest#password-based))
    - Expects a `password` field to be sent to `/login`
-    - Is checked by the method: `self.check_pass` 
-
+    - Is checked by the method: `self.check_pass`

 ```python
 from typing import Awaitable, Callable, Optional, Tuple
@@ -16,10 +16,12 @@ _First introduced in Synapse v1.37.0_
 async def check_event_for_spam(event: "synapse.events.EventBase") -> Union[bool, str]
 ```

-Called when receiving an event from a client or via federation. The module can return
-either a `bool` to indicate whether the event must be rejected because of spam, or a `str`
-to indicate the event must be rejected because of spam and to give a rejection reason to
-forward to clients.
+Called when receiving an event from a client or via federation. The callback must return
+either:
+- an error message string, to indicate the event must be rejected because of spam and 
+  give a rejection reason to forward to clients;
+- the boolean `True`, to indicate that the event is spammy, but not provide further details; or
+- the booelan `False`, to indicate that the event is not considered spammy.

 If multiple modules implement this callback, they will be considered in order. If a
 callback returns `False`, Synapse falls through to the next one. The value of the first
@@ -35,7 +37,10 @@ async def user_may_join_room(user: str, room: str, is_invited: bool) -> bool
 ```

 Called when a user is trying to join a room. The module must return a `bool` to indicate
-whether the user can join the room. The user is represented by their Matrix user ID (e.g.
+whether the user can join the room. Return `False` to prevent the user from joining the
+room; otherwise return `True` to permit the joining.
+
+The user is represented by their Matrix user ID (e.g.
 `@alice:example.com`) and the room is represented by its Matrix ID (e.g.
 `!room:example.com`). The module is also given a boolean to indicate whether the user
 currently has a pending invite in the room.
@@ -58,7 +63,8 @@ async def user_may_invite(inviter: str, invitee: str, room_id: str) -> bool

 Called when processing an invitation. The module must return a `bool` indicating whether
 the inviter can invite the invitee to the given room. Both inviter and invitee are
-represented by their Matrix user ID (e.g. `@alice:example.com`).
+represented by their Matrix user ID (e.g. `@alice:example.com`). Return `False` to prevent
+the invitation; otherwise return `True` to permit it.

 If multiple modules implement this callback, they will be considered in order. If a
 callback returns `True`, Synapse falls through to the next one. The value of the first
@@ -80,7 +86,8 @@ async def user_may_send_3pid_invite(

 Called when processing an invitation using a third-party identifier (also called a 3PID,
 e.g. an email address or a phone number). The module must return a `bool` indicating
-whether the inviter can invite the invitee to the given room.
+whether the inviter can invite the invitee to the given room. Return `False` to prevent
+the invitation; otherwise return `True` to permit it.

 The inviter is represented by their Matrix user ID (e.g. `@alice:example.com`), and the
 invitee is represented by its medium (e.g. "email") and its address
@@ -117,6 +124,7 @@ async def user_may_create_room(user: str) -> bool

 Called when processing a room creation request. The module must return a `bool` indicating
 whether the given user (represented by their Matrix user ID) is allowed to create a room.
+Return `False` to prevent room creation; otherwise return `True` to permit it.

 If multiple modules implement this callback, they will be considered in order. If a
 callback returns `True`, Synapse falls through to the next one. The value of the first
@@ -133,7 +141,8 @@ async def user_may_create_room_alias(user: str, room_alias: "synapse.types.RoomA

 Called when trying to associate an alias with an existing room. The module must return a
 `bool` indicating whether the given user (represented by their Matrix user ID) is allowed
-to set the given alias.
+to set the given alias. Return `False` to prevent the alias creation; otherwise return 
+`True` to permit it.

 If multiple modules implement this callback, they will be considered in order. If a
 callback returns `True`, Synapse falls through to the next one. The value of the first
@@ -150,7 +159,8 @@ async def user_may_publish_room(user: str, room_id: str) -> bool

 Called when trying to publish a room to the homeserver's public rooms directory. The
 module must return a `bool` indicating whether the given user (represented by their
-Matrix user ID) is allowed to publish the given room.
+Matrix user ID) is allowed to publish the given room. Return `False` to prevent the
+room from being published; otherwise return `True` to permit its publication.

 If multiple modules implement this callback, they will be considered in order. If a
 callback returns `True`, Synapse falls through to the next one. The value of the first
@@ -166,8 +176,11 @@ async def check_username_for_spam(user_profile: Dict[str, str]) -> bool
 ```

 Called when computing search results in the user directory. The module must return a
-`bool` indicating whether the given user profile can appear in search results. The profile
-is represented as a dictionary with the following keys:
+`bool` indicating whether the given user should be excluded from user directory 
+searches. Return `True` to indicate that the user is spammy and exclude them from 
+search results; otherwise return `False`.
+
+The profile is represented as a dictionary with the following keys:

 * `user_id`: The Matrix ID for this user.
 * `display_name`: The user's display name.
@@ -225,8 +238,9 @@ async def check_media_file_for_spam(
 ) -> bool
 ```

-Called when storing a local or remote file. The module must return a boolean indicating
-whether the given file can be stored in the homeserver's media store.
+Called when storing a local or remote file. The module must return a `bool` indicating
+whether the given file should be excluded from the homeserver's media store. Return
+`True` to prevent this file from being stored; otherwise return `False`.

 If multiple modules implement this callback, they will be considered in order. If a
 callback returns `False`, Synapse falls through to the next one. The value of the first
@@ -163,7 +163,7 @@ presence:
 # For example, for room version 1, default_room_version should be set
 # to "1".
 #
-#default_room_version: "6"
+#default_room_version: "9"

 # The GC threshold parameters to pass to `gc.set_threshold`, if defined
 #
@@ -751,11 +751,16 @@ caches:
  per_cache_factors:
    #get_users_who_share_room_with_user: 2.0

-  # Controls how long an entry can be in a cache without having been
-  # accessed before being evicted. Defaults to None, which means
-  # entries are never evicted based on time.
+  # Controls whether cache entries are evicted after a specified time
+  # period. Defaults to true. Uncomment to disable this feature.
  #
-  #expiry_time: 30m
+  #expire_caches: false
+
+  # If expire_caches is enabled, this flag controls how long an entry can
+  # be in a cache without having been accessed before being evicted.
+  # Defaults to 30m. Uncomment to set a different time to live for cache entries.
+  #
+  #cache_entry_ttl: 30m

  # Controls how long the results of a /sync request are cached for after
  # a successful response is returned. A higher duration can help clients with
@@ -81,14 +81,12 @@ remote endpoint at 10.1.2.3:9999.

 ## Upgrading from legacy structured logging configuration

-Versions of Synapse prior to v1.23.0 included a custom structured logging
-configuration which is deprecated. It used a `structured: true` flag and
-configured `drains` instead of ``handlers`` and `formatters`.
+Versions of Synapse prior to v1.54.0 automatically converted the legacy
+structured logging configuration, which was deprecated in v1.23.0, to the standard
+library logging configuration.

-Synapse currently automatically converts the old configuration to the new
-configuration, but this will be removed in a future version of Synapse. The
-following reference can be used to update your configuration. Based on the drain
-`type`, we can pick a new handler:
+The following reference can be used to update your configuration. Based on the
+drain `type`, we can pick a new handler:

 1. For a type of `console`, `console_json`, or `console_json_terse`: a handler
   with a class of `logging.StreamHandler` and a `stream` of `ext://sys.stdout`
@@ -141,7 +139,7 @@ formatters:
 handlers:
    console:
        class: logging.StreamHandler
-        location: ext://sys.stdout
+        stream: ext://sys.stdout
    file:
        class: logging.FileHandler
        formatter: json
@@ -85,6 +85,91 @@ process, for example:
    dpkg -i matrix-synapse-py3_1.3.0+stretch1_amd64.deb
    ```

+# Upgrading to v1.54.0
+
+## Legacy structured logging configuration removal
+
+This release removes support for the `structured: true` logging configuration
+which was deprecated in Synapse v1.23.0. If your logging configuration contains
+`structured: true` then it should be modified based on the
+[structured logging documentation](structured_logging.md).
+
+# Upgrading to v1.53.0
+
+## Dropping support for `webclient` listeners and non-HTTP(S) `web_client_location`
+
+Per the deprecation notice in Synapse v1.51.0, listeners of type  `webclient`
+are no longer supported and configuring them is a now a configuration error.
+
+Configuring a non-HTTP(S) `web_client_location` configuration is is now a
+configuration error. Since the `webclient` listener is no longer supported, this
+setting only applies to the root path `/` of Synapse's web server and no longer
+the `/_matrix/client/` path.
+
+## Stablisation of MSC3231
+
+The unstable validity-check endpoint for the 
+[Registration Tokens](https://spec.matrix.org/v1.2/client-server-api/#get_matrixclientv1registermloginregistration_tokenvalidity) 
+feature has been stabilised and moved from:
+
+`/_matrix/client/unstable/org.matrix.msc3231/register/org.matrix.msc3231.login.registration_token/validity`
+
+to:
+
+`/_matrix/client/v1/register/m.login.registration_token/validity`
+
+Please update any relevant reverse proxy or firewall configurations appropriately.
+
+## Time-based cache expiry is now enabled by default
+
+Formerly, entries in the cache were not evicted regardless of whether they were accessed after storing.
+This behavior has now changed. By default entries in the cache are now evicted after 30m of not being accessed. 
+To change the default behavior, go to the `caches` section of the config and change the `expire_caches` and 
+`cache_entry_ttl` flags as necessary. Please note that these flags replace the `expiry_time` flag in the config.  
+The `expiry_time` flag will still continue to work, but it has been deprecated and will be removed in the future.
+
+## Deprecation of `capability` `org.matrix.msc3283.*`
+
+The `capabilities` of MSC3283 from the REST API `/_matrix/client/r0/capabilities`
+becomes stable.
+
+The old `capabilities`
+- `org.matrix.msc3283.set_displayname`,
+- `org.matrix.msc3283.set_avatar_url` and
+- `org.matrix.msc3283.3pid_changes`
+
+are deprecated and scheduled to be removed in Synapse v1.54.0.
+
+The new `capabilities`
+- `m.set_displayname`,
+- `m.set_avatar_url` and
+- `m.3pid_changes`
+
+are now active by default.
+
+## Removal of `user_may_create_room_with_invites`
+
+As announced with the release of [Synapse 1.47.0](#deprecation-of-the-user_may_create_room_with_invites-module-callback),
+the deprecated `user_may_create_room_with_invites` module callback has been removed.
+
+Modules relying on it can instead implement [`user_may_invite`](https://matrix-org.github.io/synapse/latest/modules/spam_checker_callbacks.html#user_may_invite)
+and use the [`get_room_state`](https://github.com/matrix-org/synapse/blob/872f23b95fa980a61b0866c1475e84491991fa20/synapse/module_api/__init__.py#L869-L876)
+module API to infer whether the invite is happening while creating a room (see [this function](https://github.com/matrix-org/synapse-domain-rule-checker/blob/e7d092dd9f2a7f844928771dbfd9fd24c2332e48/synapse_domain_rule_checker/__init__.py#L56-L89)
+as an example). Alternately, modules can also implement [`on_create_room`](https://matrix-org.github.io/synapse/latest/modules/third_party_rules_callbacks.html#on_create_room).
+
+
+# Upgrading to v1.52.0
+
+## Twisted security release
+
+Note that [Twisted 22.1.0](https://github.com/twisted/twisted/releases/tag/twisted-22.1.0)
+has recently been released, which fixes a [security issue](https://github.com/twisted/twisted/security/advisories/GHSA-92x2-jw7w-xvvx)
+within the Twisted library. We do not believe Synapse is affected by this vulnerability,
+though we advise server administrators who installed Synapse via pip to upgrade Twisted
+with `pip install --upgrade Twisted treq` as a matter of good practice. The Docker image
+`matrixdotorg/synapse` and the Debian packages from `packages.matrix.org` are using the
+updated library.
+
 # Upgrading to v1.51.0

 ## Deprecation of `webclient` listeners and non-HTTP(S) `web_client_location`
@@ -1129,8 +1214,7 @@ more details on upgrading your database.

 Synapse v1.0 is the first release to enforce validation of TLS
 certificates for the federation API. It is therefore essential that your
-certificates are correctly configured. See the
-[FAQ](MSC1711_certificates_FAQ.md) for more information.
+certificates are correctly configured.

 Note, v1.0 installations will also no longer be able to federate with
 servers that have not correctly configured their certificates.
@@ -1195,9 +1279,6 @@ you will need to replace any self-signed certificates with those
 verified by a root CA. Information on how to do so can be found at the
 ACME docs.

-For more information on configuring TLS certificates see the
-[FAQ](MSC1711_certificates_FAQ.md).
-
 # Upgrading to v0.34.0

 1.  This release is the first to fully support Python 3. Synapse will
@@ -241,7 +241,7 @@ expressions:
    # Registration/login requests
    ^/_matrix/client/(api/v1|r0|v3|unstable)/login$
    ^/_matrix/client/(r0|v3|unstable)/register$
-    ^/_matrix/client/unstable/org.matrix.msc3231/register/org.matrix.msc3231.login.registration_token/validity$
+    ^/_matrix/client/v1/register/m.login.registration_token/validity$

    # Event sending requests
    ^/_matrix/client/(api/v1|r0|v3|unstable)/rooms/.*/redact
@@ -31,14 +31,11 @@ exclude = (?x)
   |synapse/storage/databases/main/group_server.py
   |synapse/storage/databases/main/metrics.py
   |synapse/storage/databases/main/monthly_active_users.py
-   |synapse/storage/databases/main/presence.py
-   |synapse/storage/databases/main/purge_events.py
   |synapse/storage/databases/main/push_rule.py
   |synapse/storage/databases/main/receipts.py
   |synapse/storage/databases/main/roommember.py
   |synapse/storage/databases/main/search.py
   |synapse/storage/databases/main/state.py
-   |synapse/storage/databases/main/user_directory.py
   |synapse/storage/schema/

   |tests/api/test_auth.py
@@ -142,6 +139,9 @@ disallow_untyped_defs = True
 [mypy-synapse.crypto.*]
 disallow_untyped_defs = True

+[mypy-synapse.event_auth]
+disallow_untyped_defs = True
+
 [mypy-synapse.events.*]
 disallow_untyped_defs = True

@@ -166,9 +166,15 @@ disallow_untyped_defs = True
 [mypy-synapse.module_api.*]
 disallow_untyped_defs = True

+[mypy-synapse.notifier]
+disallow_untyped_defs = True
+
 [mypy-synapse.push.*]
 disallow_untyped_defs = True

+[mypy-synapse.replication.*]
+disallow_untyped_defs = True
+
 [mypy-synapse.rest.*]
 disallow_untyped_defs = True

@@ -54,3 +54,15 @@ exclude = '''
  )/
 )
 '''
+
+[tool.isort]
+line_length = 88
+sections = ["FUTURE", "STDLIB", "THIRDPARTY", "TWISTED", "FIRSTPARTY", "TESTS", "LOCALFOLDER"]
+default_section = "THIRDPARTY"
+known_first_party = ["synapse"]
+known_tests = ["tests"]
+known_twisted = ["twisted", "OpenSSL"]
+multi_line_output = 3
+include_trailing_comma = true
+combine_as_imports = true
+
@@ -25,7 +25,6 @@ DISTS = (
    "debian:bookworm",
    "debian:sid",
    "ubuntu:focal",  # 20.04 LTS (our EOL forced by Py38 on 2024-10-14)
-    "ubuntu:hirsute",  # 21.04 (EOL 2022-01-05)
    "ubuntu:impish",  # 21.10  (EOL 2022-07)
 )

@@ -5,7 +5,7 @@
 # It makes a Synapse image which represents the current checkout,
 # builds a synapse-complement image on top, then runs tests with it.
 #
-# By default the script will fetch the latest Complement master branch and
+# By default the script will fetch the latest Complement main branch and
 # run tests with that. This can be overridden to use a custom Complement
 # checkout by setting the COMPLEMENT_DIR environment variable to the
 # filepath of a local Complement checkout or by setting the COMPLEMENT_REF
@@ -32,7 +32,7 @@ cd "$(dirname $0)/.."

 # Check for a user-specified Complement checkout
 if [[ -z "$COMPLEMENT_DIR" ]]; then
-  COMPLEMENT_REF=${COMPLEMENT_REF:-master}
+  COMPLEMENT_REF=${COMPLEMENT_REF:-main}
  echo "COMPLEMENT_DIR not set. Fetching Complement checkout from ${COMPLEMENT_REF}..."
  wget -Nq https://github.com/matrix-org/complement/archive/${COMPLEMENT_REF}.tar.gz
  tar -xzf ${COMPLEMENT_REF}.tar.gz
@@ -0,0 +1,61 @@
+# workers_setup
+
+This gives you a **development-grade** installation of workerised Synapse.
+
+DO NOT USE ME IN PRODUCTION.
+
+## Known defects
+
+* Non-generic workers aren't set up properly with their worker type.
+* I haven't checked the routes that well; they are probably wrong.
+
+
+## Requirements from you:
+
+* Redis on default port (unauthenticated)
+  ```
+  # You need Redis. On Ubuntu, this gets you what you need running on the right port:
+  apt install redis-server redis-tools
+  ```
+* Postgres on default port, using UNIX sockets for authentication.
+  This means you want your normal user account to have a corresponding Postgres account,
+  and let Postgres authenticate you automatically.
+  On Ubuntu, this just means you need to `createuser <your Linux account name>`.
+  You need a database with the same name as your server_name (I used `syn7`).
+  It should be owned by your user; see `createdb` to do that properly (and don't
+  forget to follow the Synapse instructions to use a C locale!)
+  Typing `psql syn7` should just work once your database is ready.
+  (If your UNIX socket is not numbered 5432, you might have to add `port: 5433`
+  to the config. Somehow I messed up my Postgres installation ages ago that it
+  chose port 5433 rather than the default 5432...)
+* Virtualenv with Synapse (don't forget: `[postgres,redis]`)
+* You'll need a bog standard Caddy binary (as the reverse proxy / router).
+  The website offers pre-built static binaries.
+* (Optional): If you want to federate, you can set up TLS yourself afterwards.
+  I haven't bothered so far.
+
+
+## Run the script
+
+```
+# python scripts-dev/workers_setup.py (path to server dir) (server name)
+python scripts-dev/workers_setup.py ../servers/syn7_auto syn7
+```
+
+
+## Launching the homeserver
+
+```
+cd syn7_auto
+/path/to/synapse/.venv/bin/synctl start homeserver.yaml -a workers
+/path/to/caddy run
+```
+
+
+## Stopping the homeserver
+
+```
+# ^C to stop Caddy
+/path/to/synapse/.venv/bin/synctl stop homeserver.yaml -a workers
+```
+
@@ -0,0 +1,305 @@
+#!/usr/bin/env python
+# Copyright 2022 The Matrix.org Foundation C.I.C.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import dataclasses
+import sys
+from os.path import dirname
+from pathlib import Path
+from typing import Collection, Dict, Iterable, List, Sequence, Tuple
+
+from jinja2 import Environment, FileSystemLoader
+from signedjson.key import generate_signing_key, write_signing_keys
+
+from synapse.util.stringutils import random_string
+
+DESIRED_WORKERS = (
+    ("main", 1),
+    ("synchrotron", 2),
+    ("federation_inbound", 2),
+    ("federation_reader", 2),
+    ("federation_sender", 2),
+    ("typing", 1),
+    ("appservice", 1),
+    ("client_reader", 2),
+    ("event_creator", 2),
+    ("event_persister", 2),
+    ("media_repository", 1),
+    ("pusher", 2),
+    ("user_dir", 1),
+    ("background_worker", 1),
+    # TODO ("encryption", 1),  # ??
+    ("receipts_account_data", 1)
+    # TODO frontend_proxy?
+)
+
+# TODO These are probably all wrong
+
+# ^/_matrix/client/(api/v1|r0|v3|unstable)/sendToDevice/ ?
+# ^/_matrix/client/(api/v1|r0|v3|unstable)/.*/tags
+# ^/_matrix/client/(api/v1|r0|v3|unstable)/.*/account_data ?
+# ^/_matrix/client/(api/v1|r0|v3|unstable)/rooms/.*/receipt
+# ^/_matrix/client/(api/v1|r0|v3|unstable)/rooms/.*/read_markers ?
+# ^/_matrix/client/(api/v1|r0|v3|unstable)/presence/ ?
+
+WORKER_ROUTES: Dict[str, Tuple[str, ...]] = {
+    "main": (),
+    "synchrotron": (
+        "^/_matrix/client/(v2_alpha|r0|v3)/sync$",
+        "^/_matrix/client/(api/v1|v2_alpha|r0|v3)/events$",
+        "^/_matrix/client/(api/v1|r0|v3)/initialSync$",
+        "^/_matrix/client/(api/v1|r0|v3)/rooms/[^/]+/initialSync$",
+    ),
+    "federation_inbound": ("^/_matrix/federation/v1/send/",),
+    "federation_reader": (
+        "^/_matrix/federation/v1/event/",
+        "^/_matrix/federation/v1/state/",
+        "^/_matrix/federation/v1/state_ids/",
+        "^/_matrix/federation/v1/backfill/",
+        "^/_matrix/federation/v1/get_missing_events/",
+        "^/_matrix/federation/v1/publicRooms",
+        "^/_matrix/federation/v1/query/",
+        "^/_matrix/federation/v1/make_join/",
+        "^/_matrix/federation/v1/make_leave/",
+        "^/_matrix/federation/v1/send_join/",
+        "^/_matrix/federation/v2/send_join/",
+        "^/_matrix/federation/v1/send_leave/",
+        "^/_matrix/federation/v2/send_leave/",
+        "^/_matrix/federation/v1/invite/",
+        "^/_matrix/federation/v2/invite/",
+        "^/_matrix/federation/v1/query_auth/",
+        "^/_matrix/federation/v1/event_auth/",
+        "^/_matrix/federation/v1/exchange_third_party_invite/",
+        "^/_matrix/federation/v1/user/devices/",
+        "^/_matrix/federation/v1/get_groups_publicised$",
+        "^/_matrix/key/v2/query",
+        "^/_matrix/federation/(v1|unstable/org.matrix.msc2946)/hierarchy/",
+    ),
+    "federation_sender": (),
+    "typing": ("^/_matrix/client/(api/v1|r0|v3|unstable)/rooms/.*/typing",),
+    "appservice": (),
+    "client_reader": (
+        "^/_matrix/client/(api/v1|r0|v3|unstable)/createRoom$",
+        "^/_matrix/client/(api/v1|r0|v3|unstable)/publicRooms$",
+        "^/_matrix/client/(api/v1|r0|v3|unstable)/rooms/.*/joined_members$",
+        "^/_matrix/client/(api/v1|r0|v3|unstable)/rooms/.*/context/.*$",
+        "^/_matrix/client/(api/v1|r0|v3|unstable)/rooms/.*/members$",
+        "^/_matrix/client/(api/v1|r0|v3|unstable)/rooms/.*/state$",
+        "^/_matrix/client/(v1|unstable/org.matrix.msc2946)/rooms/.*/hierarchy$",
+        "^/_matrix/client/unstable/im.nheko.summary/rooms/.*/summary$",
+        "^/_matrix/client/(r0|v3|unstable)/account/3pid$",
+        "^/_matrix/client/(r0|v3|unstable)/devices$",
+        "^/_matrix/client/versions$",
+        "^/_matrix/client/(api/v1|r0|v3|unstable)/voip/turnServer$",
+        "^/_matrix/client/(r0|v3|unstable)/joined_groups$",
+        "^/_matrix/client/(r0|v3|unstable)/publicised_groups$",
+        "^/_matrix/client/(r0|v3|unstable)/publicised_groups/",
+        "^/_matrix/client/(api/v1|r0|v3|unstable)/rooms/.*/event/",
+        "^/_matrix/client/(api/v1|r0|v3|unstable)/joined_rooms$",
+        "^/_matrix/client/(api/v1|r0|v3|unstable)/search$",
+        "^/_matrix/client/(r0|v3|unstable)/keys/query$",
+        "^/_matrix/client/(r0|v3|unstable)/keys/changes$",
+        "^/_matrix/client/(r0|v3|unstable)/keys/claim$",
+        "^/_matrix/client/(r0|v3|unstable)/room_keys/",
+        #
+        "^/_matrix/client/(api/v1|r0|v3|unstable)/login$",
+        "^/_matrix/client/(r0|v3|unstable)/register$",
+        "^/_matrix/client/v1/register/m.login.registration_token/validity$",
+        "^/_matrix/client/(api/v1|r0|v3|unstable)/rooms/.*/redact",
+        "^/_matrix/client/(api/v1|r0|v3|unstable)/rooms/.*/send",
+        "^/_matrix/client/(api/v1|r0|v3|unstable)/rooms/.*/state/",
+        "^/_matrix/client/(api/v1|r0|v3|unstable)/rooms/.*/(join|invite|leave|ban|unban|kick)$",
+        "^/_matrix/client/(api/v1|r0|v3|unstable)/join/",
+        "^/_matrix/client/(api/v1|r0|v3|unstable)/profile/",
+        "^/_matrix/client/(r0|v3|unstable)/sendToDevice/",
+        "^/_matrix/client/(r0|v3|unstable)/.*/tags",
+        "^/_matrix/client/(r0|v3|unstable)/.*/account_data",
+        "^/_matrix/client/(api/v1|r0|v3|unstable)/presence/",
+        "^/_matrix/client/(r0|v3|unstable)/rooms/.*/receipt",
+        "^/_matrix/client/(r0|v3|unstable)/rooms/.*/read_markers",
+        "^/_matrix/client/(r0|v3|unstable)/.*/tags",
+        "^/_matrix/client/(r0|v3|unstable)/.*/account_data",
+        "^/_matrix/client/(api/v1|r0|v3|unstable)/rooms/.*/messages$",
+        "^/_matrix/client/(api/v1|r0|v3|unstable)/login/sso/redirect",
+        "^/_synapse/client/pick_idp$",
+        "^/_synapse/client/pick_username",
+        "^/_synapse/client/new_user_consent$",
+        "^/_synapse/client/sso_register$",
+        "^/_synapse/client/oidc/callback$",
+        "^/_synapse/client/saml2/authn_response$",
+        "^/_matrix/client/(api/v1|r0|v3|unstable)/login/cas/ticket$",
+    ),
+    "event_creator": (),
+    "event_persister": (),
+    "media_repository": (
+        "^/_synapse/admin/v1/purge_media_cache$",
+        "^/_synapse/admin/v1/room/.*/media.*$",
+        "^/_synapse/admin/v1/user/.*/media.*$",
+        "^/_synapse/admin/v1/media/.*$",
+        "^/_synapse/admin/v1/quarantine_media/.*$",
+        "^/_synapse/admin/v1/users/.*/media$",
+    ),
+    "pusher": (),
+    "user_dir": ("^/_matrix/client/(api/v1|r0|v3|unstable)/user_directory/search$",),
+    "background_worker": (),
+    "receipts_account_data": (),
+}
+
+
+@dataclasses.dataclass
+class Worker:
+    name: str
+    kind: str
+    index: int
+    ip: str
+
+
+def worker_num_to_ip(num: int) -> str:
+    return f"127.0.57.{num}"
+
+
+def make_workers(workers: Iterable[Tuple[str, int]]) -> List[Worker]:
+    result = []
+    worker_overall_num = 0
+    for worker_type, worker_type_count in workers:
+        for worker_idx in range(1, worker_type_count + 1):
+            worker_overall_num += 1
+            if worker_type == "main":
+                worker_name = "main"
+            else:
+                worker_name = f"{worker_type}{worker_idx}"
+
+            result.append(
+                Worker(
+                    worker_name,
+                    worker_type,
+                    worker_idx,
+                    worker_num_to_ip(worker_overall_num),
+                )
+            )
+
+    return result
+
+
+def generate(
+    worker_counts: Tuple[Tuple[str, int], ...], target_path: Path, server_name: str
+) -> None:
+    if target_path.exists():
+        print("Target path already exists. Won't overwrite.")
+        return
+    target_path.mkdir()
+
+    # Generate a signing key
+    key_id = "a_" + random_string(4)
+    key = (generate_signing_key(key_id),)
+    with open(target_path.joinpath("signing.key"), "w") as fout:
+        write_signing_keys(fout, key)
+
+    macaroon_secret_key = random_string(32)
+
+    env = Environment(loader=FileSystemLoader(dirname(__file__) + "/workers_setup"))
+    hs_template = env.get_template("homeserver.yaml.j2")
+    worker_template = env.get_template("worker.yaml.j2")
+    logging_template = env.get_template("logging.yaml.j2")
+    rp_template = env.get_template("Caddyfile.j2")
+
+    worker_dir = target_path.joinpath("workers")
+    worker_dir.mkdir()
+    worker_logging_dir = target_path.joinpath("workers.logging")
+    worker_logging_dir.mkdir()
+    worker_dir = worker_dir.resolve()
+
+    logs_dir = target_path.joinpath("logs")
+    logs_dir.mkdir()
+    logs_dir = logs_dir.resolve()
+
+    all_workers = make_workers(worker_counts)
+    workers_by_name = {worker.name: worker for worker in all_workers}
+
+    for worker in all_workers:
+        log_config_path = worker_logging_dir.joinpath(f"{worker.name}.logging.yaml")
+        log_config = logging_template.render(
+            worker=worker,
+            worker_dir=worker_dir,
+            logs_dir=logs_dir,
+            all_workers=all_workers,
+            workers_by_name=workers_by_name,
+        )
+        with open(log_config_path, "w") as fout:
+            fout.write(log_config)
+
+        # if worker.name == "main":
+        # Main can't use a worker file.
+        # continue
+
+        worker_config_path = worker_dir.joinpath(f"{worker.name}.yaml")
+        worker_config = worker_template.render(
+            worker=worker,
+            worker_dir=worker_dir,
+            logs_dir=logs_dir,
+            all_workers=all_workers,
+            workers_by_name=workers_by_name,
+        )
+        with open(worker_config_path, "w") as fout:
+            fout.write(worker_config)
+
+    hs_config_path = target_path.joinpath("homeserver.yaml")
+    hs_config = hs_template.render(
+        all_workers=all_workers,
+        workers_by_name=workers_by_name,
+        worker_dir=worker_dir,
+        logs_dir=logs_dir,
+        server_name=server_name,
+        macaroon_secret_key=macaroon_secret_key,
+    )
+    with open(hs_config_path, "w") as fout:
+        fout.write(hs_config)
+
+    caddy_config_path = target_path.joinpath("Caddyfile")
+    caddy_config = rp_template.render(
+        server_name=server_name,
+        port=8447,
+        http_ip=worker_num_to_ip(1),
+        routing=build_routes_template_var(all_workers),
+        main_server=f"{worker_num_to_ip(1)}:8080",
+    )
+    with open(caddy_config_path, "w") as fout:
+        fout.write(caddy_config)
+
+
+def build_routes_template_var(
+    all_workers: List[Worker],
+) -> Sequence[Tuple[str, Collection[str], List[str]]]:
+    route_groups = {}
+    for worker in all_workers:
+        if worker.kind not in route_groups:
+            if WORKER_ROUTES[worker.kind]:
+                route_groups[worker.kind] = worker.kind, WORKER_ROUTES[worker.kind], []
+            else:
+                continue
+
+        _, _routes, server_endpoints = route_groups[worker.kind]
+
+        server_endpoints.append(f"{worker.ip}:8080")
+
+    return tuple(route_groups.values())
+
+
+def main(target_path: Path, server_name: str) -> None:
+    generate(DESIRED_WORKERS, target_path, server_name)
+
+
+if __name__ == "__main__":
+    target_path = Path(sys.argv[1])
+    server_name = sys.argv[2]
+    main(target_path, server_name)
@@ -0,0 +1,24 @@
+{
+  # Prevents Caddy from asking for sudo password to install a root cert that
+  # we don't even want to use here.
+  skip_install_trust
+}
+
+# If you want TLS, you can add https:// schemes and configure the TLS cert... somehow.
+http://{{ server_name }}:{{ port }}, http://{{ http_ip }}:{{ port }} {
+  {%- for route_group_name, routes, route_servers in routing %}
+    @{{ route_group_name }} {
+      {%- for route in routes %}
+      path_regexp {{ route }}
+      {%- endfor %}
+    }
+    route @{{ route_group_name }} {
+      reverse_proxy {% for server in route_servers %} {{ server }} {% endfor %}
+    }
+  {%- endfor %}
+
+  # fallback to main
+  route {
+    reverse_proxy {{ main_server }}
+  }
+}
@@ -0,0 +1,96 @@
+
+server_name: {{ server_name }}
+
+report_stats: false
+
+signing_key_path: "{{ worker_dir }}/../signing.key"
+
+macaroon_secret_key: "{{ macaroon_secret_key }}"
+
+enable_registration: true
+
+redis:
+  enabled: true
+  #host: localhost
+  #port: 6379
+
+trusted_key_servers: []
+
+listeners:
+  - port: 8080
+    bind_address: {{ workers_by_name.main.ip }}
+    type: http
+    resources:
+      - names: [client, federation]
+
+  # The HTTP replication port
+  - port: 9090
+    bind_address: {{ workers_by_name.main.ip }}
+    type: http
+    resources:
+      - names: [replication]
+
+database:
+  name: psycopg2
+  args:
+    # Comment out user, password and host to use UNIX socket auth.
+    # For testing, create a database owned by your Postgres user that is logged
+    # in with your UNIX user
+    #user: "synapse"
+    #password:
+    database: "{{ server_name }}"
+    #host: "localhost"
+    cp_min: 5
+    cp_max: 10
+
+instance_map:
+{%- for worker in all_workers %}
+  {{ worker.name }}:
+    host: {{ worker.ip }}
+    port: 9090
+{%- endfor %}
+
+stream_writers:
+  events:
+{%- for worker in all_workers %}
+{%- if worker.kind == "event_persister" %}
+    - {{ worker.name }}
+{%- endif %}
+{%- endfor %}
+  typing:
+{%- for worker in all_workers %}
+{%- if worker.kind == "typing" %}
+    - {{ worker.name }}
+{%- endif %}
+{%- endfor %}
+
+
+start_pushers: false
+
+pusher_instances:
+{% for worker in all_workers -%}
+{%- if worker.kind == "pusher" %}
+    - {{ worker.name }}
+{%- endif %}
+{%- endfor %}
+
+notify_appservices: False
+
+federation_sender_instances:
+{% for worker in all_workers -%}
+{%- if worker.kind == "federation_sender" %}
+    - {{ worker.name }}
+{% endif -%}
+{% endfor %}
+
+enable_media_repo: False
+
+media_instance_running_background_jobs: "media1"
+
+update_user_directory: False
+
+pid_file: "{{ logs_dir }}/main.pid"
+
+log_config: '{{ worker_dir }}.logging/main.logging.yaml'
+
+run_background_tasks_on: background_worker1
@@ -0,0 +1,32 @@
+
+version: 1
+
+formatters:
+    precise:
+        format: '%(asctime)s - %(name)s - %(lineno)d - %(levelname)s - %(request)s - %(message)s'
+
+filters:
+    context:
+        (): synapse.util.logcontext.LoggingContextFilter
+        request: ""
+
+handlers:
+    console:
+        class: logging.FileHandler
+        formatter: precise
+        filters: [context]
+        encoding: 'UTF-8'
+        filename: '{{ logs_dir }}/{{ worker.name }}.log'
+
+loggers:
+    synapse:
+        level: DEBUG
+
+    synapse.storage.SQL:
+        # beware: increasing this to DEBUG will make synapse log sensitive
+        # information such as access tokens.
+        level: INFO
+
+root:
+    level: WARNING
+    handlers: [console]
@@ -0,0 +1,36 @@
+{% set main_worker = workers_by_name.main %}
+
+{% if worker.kind == "main" %}
+
+worker_app: synapse.app.homeserver
+
+{% else %}
+
+worker_app: synapse.app.generic_worker
+worker_name: {{ worker.name }}
+# The replication listener on the main synapse process.
+worker_replication_host: {{ main_worker.ip }}
+worker_replication_http_port: 9090
+worker_listeners:
+  - type: http
+    bind_address: {{ worker.ip }}
+    port: 8080
+    resources:
+    - names:
+      - client
+      - federation
+{%- if worker.kind == "media" %}
+      - media
+{%- endif %}
+  - type: http
+    bind_address: {{ worker.ip }}
+    port: 9090
+    resources:
+    - names: [replication]
+
+worker_log_config: '{{ worker_dir }}.logging/{{ worker.name }}.logging.yaml'
+worker_pid_file: '{{ logs_dir }}/{{ worker.name }}.pid'
+
+worker_main_http_uri: http://{{ main_worker.ip }}:8080
+
+{% endif %}
@@ -24,10 +24,10 @@ import traceback
 from typing import Dict, Iterable, Optional, Set

 import yaml
+from matrix_common.versionstring import get_distribution_version_string

 from twisted.internet import defer, reactor

-import synapse
 from synapse.config.database import DatabaseConnectionConfig
 from synapse.config.homeserver import HomeServerConfig
 from synapse.logging.context import (
@@ -67,7 +67,6 @@ from synapse.storage.databases.state.bg_updates import StateBackgroundUpdateStor
 from synapse.storage.engines import create_engine
 from synapse.storage.prepare_database import prepare_database
 from synapse.util import Clock
-from synapse.util.versionstring import get_version_string

 logger = logging.getLogger("synapse_port_db")

@@ -222,7 +221,9 @@ class MockHomeserver:
        self.clock = Clock(reactor)
        self.config = config
        self.hostname = config.server.server_name
-        self.version_string = "Synapse/" + get_version_string(synapse)
+        self.version_string = "Synapse/" + get_distribution_version_string(
+            "matrix-synapse"
+        )

    def get_clock(self):
        return self.clock
@@ -18,15 +18,14 @@ import logging
 import sys

 import yaml
+from matrix_common.versionstring import get_distribution_version_string

 from twisted.internet import defer, reactor

-import synapse
 from synapse.config.homeserver import HomeServerConfig
 from synapse.metrics.background_process_metrics import run_as_background_process
 from synapse.server import HomeServer
 from synapse.storage import DataStore
-from synapse.util.versionstring import get_version_string

 logger = logging.getLogger("update_database")

@@ -39,11 +38,13 @@ class MockHomeserver(HomeServer):
            config.server.server_name, reactor=reactor, config=config, **kwargs
        )

-        self.version_string = "Synapse/" + get_version_string(synapse)
+        self.version_string = "Synapse/" + get_distribution_version_string(
+            "matrix-synapse"
+        )


 def run_background_updates(hs):
-    store = hs.get_datastore()
+    store = hs.get_datastores().main

    async def run_background_updates():
        await store.db_pool.updates.run_background_updates(sleep=False)
@@ -1,6 +1,3 @@
-[trial]
-test_suite = tests
-
 [check-manifest]
 ignore =
    .git-blame-ignore-revs
@@ -10,23 +7,3 @@ ignore =
    pylint.cfg
    tox.ini

-[flake8]
-# see https://pycodestyle.readthedocs.io/en/latest/intro.html#error-codes
-# for error codes. The ones we ignore are:
-#  W503: line break before binary operator
-#  W504: line break after binary operator
-#  E203: whitespace before ':' (which is contrary to pep8?)
-#  E731: do not assign a lambda expression, use a def
-#  E501: Line too long (black enforces this for us)
-ignore=W503,W504,E203,E731,E501
-
-[isort]
-line_length = 88
-sections=FUTURE,STDLIB,THIRDPARTY,TWISTED,FIRSTPARTY,TESTS,LOCALFOLDER
-default_section=THIRDPARTY
-known_first_party = synapse
-known_tests=tests
-known_twisted=twisted,OpenSSL
-multi_line_output=3
-include_trailing_comma=true
-combine_as_imports=true
@@ -103,8 +103,8 @@ CONDITIONAL_REQUIREMENTS["lint"] = [
 ]

 CONDITIONAL_REQUIREMENTS["mypy"] = [
-    "mypy==0.910",
-    "mypy-zope==0.3.2",
+    "mypy==0.931",
+    "mypy-zope==0.3.5",
    "types-bleach>=4.1.0",
    "types-jsonschema>=3.2.0",
    "types-opentracing>=2.4.2",
@@ -66,13 +66,18 @@ class SortedDict(Dict[_KT, _VT]):
    def __copy__(self: _SD) -> _SD: ...
    @classmethod
    @overload
-    def fromkeys(cls, seq: Iterable[_T_h]) -> SortedDict[_T_h, None]: ...
+    def fromkeys(
+        cls, seq: Iterable[_T_h], value: None = ...
+    ) -> SortedDict[_T_h, None]: ...
    @classmethod
    @overload
    def fromkeys(cls, seq: Iterable[_T_h], value: _S) -> SortedDict[_T_h, _S]: ...
-    def keys(self) -> SortedKeysView[_KT]: ...
-    def items(self) -> SortedItemsView[_KT, _VT]: ...
-    def values(self) -> SortedValuesView[_VT]: ...
+    # As of Python 3.10, `dict_{keys,items,values}` have an extra `mapping` attribute and so
+    # `Sorted{Keys,Items,Values}View` are no longer compatible with them.
+    # See https://github.com/python/typeshed/issues/6837
+    def keys(self) -> SortedKeysView[_KT]: ...  # type: ignore[override]
+    def items(self) -> SortedItemsView[_KT, _VT]: ...  # type: ignore[override]
+    def values(self) -> SortedValuesView[_VT]: ...  # type: ignore[override]
    @overload
    def pop(self, key: _KT) -> _VT: ...
    @overload
@@ -47,7 +47,7 @@ try:
 except ImportError:
    pass

-__version__ = "1.52.0rc1"
+__version__ = "1.53.0"

 if bool(os.environ.get("SYNAPSE_TEST_PATCH_LOG_CONTEXTS", False)):
    # We import here so that we don't have to install a bunch of deps when
@@ -60,7 +60,7 @@ class Auth:
    def __init__(self, hs: "HomeServer"):
        self.hs = hs
        self.clock = hs.get_clock()
-        self.store = hs.get_datastore()
+        self.store = hs.get_datastores().main
        self.state = hs.get_state_handler()
        self._account_validity_handler = hs.get_account_validity_handler()

@@ -28,7 +28,7 @@ logger = logging.getLogger(__name__)

 class AuthBlocking:
    def __init__(self, hs: "HomeServer"):
-        self.store = hs.get_datastore()
+        self.store = hs.get_datastores().main

        self._server_notices_mxid = hs.config.servernotices.server_notices_mxid
        self._hs_disabled = hs.config.server.hs_disabled
@@ -81,7 +81,7 @@ class LoginType:
    TERMS: Final = "m.login.terms"
    SSO: Final = "m.login.sso"
    DUMMY: Final = "m.login.dummy"
-    REGISTRATION_TOKEN: Final = "org.matrix.msc3231.login.registration_token"
+    REGISTRATION_TOKEN: Final = "m.login.registration_token"


 # This is used in the `type` parameter for /register when called by
--- a/Show More
+++ b/Show More
				`@@ -0,0 +1 @@`
				Deduplicate in-flight requests in `_get_state_for_groups`.
				`@@ -1 +0,0 @@`
				`Add experimental support for sending to-device messages to application services, as specified by [MSC2409](https://github.com/matrix-org/matrix-doc/pull/2409). Disabled by default.`
				`@@ -1 +0,0 @@`
				`Enhance user registration test helpers to make them more useful for tests involving Application Services and devices.`
				`@@ -1 +0,0 @@`
				`Remove account data (including client config, push rules and ignored users) upon user deactivation.`
				`@@ -1 +0,0 @@`
				`Fix type errors introduced by new annotations in the Prometheus Client library.`
				`@@ -0,0 +1 @@`
				Make a `POST` to `/rooms/<room_id>/receipt/m.read/<event_id>` only trigger a push notification if the count of unread messages is different to the one in the last successfully sent push.
				`@@ -1 +0,0 @@`
				`Allow modules to retrieve the current instance's server name and worker name.`
				`@@ -1 +0,0 @@`
				Ensure that `opentracing` scopes are activated and closed at the right time.
				`@@ -1 +0,0 @@`
				`Improve opentracing for incoming federation requests.`
				`@@ -1 +0,0 @@`
				Improve internal docstrings in `synapse.util.caches`.