I'm logging errors too. LoggingContextFilter.

Changelog
Log exceptions in tests to stderr
2021-08-19 13:35:38 +01:00 · 2021-08-19 13:09:18 +01:00 · 2021-08-19 13:06:52 +01:00 · 2021-08-18 09:22:07 -04:00 · 2021-08-18 08:13:35 -04:00 · 2021-08-18 12:36:22 +01:00
295 changed files with 9599 additions and 5898 deletions
@@ -1,13 +0,0 @@
-CI
-BUILDKITE
-BUILDKITE_BUILD_NUMBER
-BUILDKITE_BRANCH
-BUILDKITE_BUILD_NUMBER
-BUILDKITE_JOB_ID
-BUILDKITE_BUILD_URL
-BUILDKITE_PROJECT_SLUG
-BUILDKITE_COMMIT
-BUILDKITE_PULL_REQUEST
-BUILDKITE_TAG
-CODECOV_TOKEN
-TRIAL_FLAGS
@@ -1,35 +0,0 @@
-#!/usr/bin/env bash
-
-set -e
-
-if [[ "$BUILDKITE_BRANCH" =~ ^(develop|master|dinsic|shhs|release-.*)$ ]]; then
-    echo "Not merging forward, as this is a release branch"
-    exit 0
-fi
-
-if [[ -z $BUILDKITE_PULL_REQUEST_BASE_BRANCH ]]; then
-    echo "Not a pull request, or hasn't had a PR opened yet..."
-
-    # It probably hasn't had a PR opened yet. Since all PRs land on develop, we
-    # can probably assume it's based on it and will be merged into it.
-    GITBASE="develop"
-else
-    # Get the reference, using the GitHub API
-    GITBASE=$BUILDKITE_PULL_REQUEST_BASE_BRANCH
-fi
-
-echo "--- merge_base_branch $GITBASE"
-
-# Show what we are before
-git --no-pager show -s
-
-# Set up username so it can do a merge
-git config --global user.email bot@matrix.org
-git config --global user.name "A robot"
-
-# Fetch and merge. If it doesn't work, it will raise due to set -e.
-git fetch -u origin $GITBASE
-git merge --no-edit --no-commit origin/$GITBASE
-
-# Show what we are after.
-git --no-pager show -s
@@ -3,7 +3,7 @@
 # CI's Docker setup at the point where this file is considered.
 server_name: "localhost:8800"

-signing_key_path: ".buildkite/test.signing.key"
+signing_key_path: ".ci/test.signing.key"

 report_stats: false

@@ -11,7 +11,7 @@ database:
  name: "psycopg2"
  args:
    user: postgres
-    host: postgres
+    host: localhost
    password: postgres
    database: synapse

@@ -23,7 +23,7 @@ import psycopg2
 # We use "postgres" as a database because it's bound to exist and the "synapse" one
 # doesn't exist yet.
 db_conn = psycopg2.connect(
-    user="postgres", host="postgres", password="postgres", dbname="postgres"
+    user="postgres", host="localhost", password="postgres", dbname="postgres"
 )
 db_conn.autocommit = True
 cur = db_conn.cursor()
@@ -1,6 +1,6 @@
 #!/usr/bin/env bash

-# this script is run by buildkite in a plain `bionic` container; it installs the
+# this script is run by GitHub Actions in a plain `bionic` container; it installs the
 # minimal requirements for tox and hands over to the py3-old tox environment.

 set -ex
@@ -20,22 +20,22 @@ pip install -e .
 echo "--- Generate the signing key"

 # Generate the server's signing key.
-python -m synapse.app.homeserver --generate-keys -c .buildkite/sqlite-config.yaml
+python -m synapse.app.homeserver --generate-keys -c .ci/sqlite-config.yaml

 echo "--- Prepare test database"

 # Make sure the SQLite3 database is using the latest schema and has no pending background update.
-scripts-dev/update_database --database-config .buildkite/sqlite-config.yaml
+scripts-dev/update_database --database-config .ci/sqlite-config.yaml

 # Create the PostgreSQL database.
-./.buildkite/scripts/postgres_exec.py "CREATE DATABASE synapse"
+.ci/scripts/postgres_exec.py "CREATE DATABASE synapse"

 echo "+++ Run synapse_port_db against test database"
-coverage run scripts/synapse_port_db --sqlite-database .buildkite/test_db.db --postgres-config .buildkite/postgres-config.yaml
+coverage run scripts/synapse_port_db --sqlite-database .ci/test_db.db --postgres-config .ci/postgres-config.yaml

 # We should be able to run twice against the same database.
 echo "+++ Run synapse_port_db a second time"
-coverage run scripts/synapse_port_db --sqlite-database .buildkite/test_db.db --postgres-config .buildkite/postgres-config.yaml
+coverage run scripts/synapse_port_db --sqlite-database .ci/test_db.db --postgres-config .ci/postgres-config.yaml

 #####

@@ -44,14 +44,14 @@ coverage run scripts/synapse_port_db --sqlite-database .buildkite/test_db.db --p
 echo "--- Prepare empty SQLite database"

 # we do this by deleting the sqlite db, and then doing the same again.
-rm .buildkite/test_db.db
+rm .ci/test_db.db

-scripts-dev/update_database --database-config .buildkite/sqlite-config.yaml
+scripts-dev/update_database --database-config .ci/sqlite-config.yaml

 # re-create the PostgreSQL database.
-./.buildkite/scripts/postgres_exec.py \
+.ci/scripts/postgres_exec.py \
  "DROP DATABASE synapse" \
  "CREATE DATABASE synapse"

 echo "+++ Run synapse_port_db against empty database"
-coverage run scripts/synapse_port_db --sqlite-database .buildkite/test_db.db --postgres-config .buildkite/postgres-config.yaml
+coverage run scripts/synapse_port_db --sqlite-database .ci/test_db.db --postgres-config .ci/postgres-config.yaml
@@ -3,14 +3,14 @@
 # schema and run background updates on it.
 server_name: "localhost:8800"

-signing_key_path: ".buildkite/test.signing.key"
+signing_key_path: ".ci/test.signing.key"

 report_stats: false

 database:
  name: "sqlite3"
  args:
-    database: ".buildkite/test_db.db"
+    database: ".ci/test_db.db"

 # Suppress the key server warning.
 trusted_key_servers: []
@@ -8,7 +8,7 @@ on:
 concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: true
-  
+
 jobs:
  lint:
    runs-on: ubuntu-latest
@@ -38,20 +38,15 @@ jobs:
    if: ${{ github.base_ref == 'develop'  || contains(github.base_ref, 'release-') }}
    runs-on: ubuntu-latest
    steps:
-      # Note: This and the script can be simplified once we drop Buildkite. See:
-      #   https://github.com/actions/checkout/issues/266#issuecomment-638346893
-      #   https://github.com/actions/checkout/issues/416
      - uses: actions/checkout@v2
        with:
          ref: ${{ github.event.pull_request.head.sha }}
          fetch-depth: 0
      - uses: actions/setup-python@v2
      - run: pip install tox
-      - name: Patch Buildkite-specific test script
-        run: |
-          sed -i -e 's/\$BUILDKITE_PULL_REQUEST/${{ github.event.number }}/' \
-            scripts-dev/check-newsfragment
      - run: scripts-dev/check-newsfragment
+        env:
+          PULL_REQUEST_NUMBER: ${{ github.event.number }}

  lint-sdist:
    runs-on: ubuntu-latest
@@ -144,7 +139,7 @@ jobs:
        uses: docker://ubuntu:bionic # For old python and sqlite
        with:
          workdir: /github/workspace
-          entrypoint: .buildkite/scripts/test_old_deps.sh
+          entrypoint: .ci/scripts/test_old_deps.sh
        env:
          TRIAL_FLAGS: "--jobs=2"
      - name: Dump logs
@@ -197,12 +192,12 @@ jobs:
      volumes:
        - ${{ github.workspace }}:/src
      env:
-        BUILDKITE_BRANCH: ${{ github.head_ref }}
        POSTGRES: ${{ matrix.postgres && 1}}
        MULTI_POSTGRES: ${{ (matrix.postgres == 'multi-postgres') && 1}}
        WORKERS: ${{ matrix.workers && 1 }}
        REDIS: ${{ matrix.redis && 1 }}
        BLACKLIST: ${{ matrix.workers && 'synapse-blacklist-with-workers' }}
+        TOP: ${{ github.workspace }}

    strategy:
      fail-fast: false
@@ -232,7 +227,7 @@ jobs:
    steps:
      - uses: actions/checkout@v2
      - name: Prepare test blacklist
-        run: cat sytest-blacklist .buildkite/worker-blacklist > synapse-blacklist-with-workers
+        run: cat sytest-blacklist .ci/worker-blacklist > synapse-blacklist-with-workers
      - name: Run SyTest
        run: /bootstrap.sh synapse
        working-directory: /src
@@ -252,6 +247,8 @@ jobs:
    if: ${{ !failure() && !cancelled() }} # Allow previous steps to be skipped, but not fail
    needs: linting-done
    runs-on: ubuntu-latest
+    env:
+      TOP: ${{ github.workspace }}
    strategy:
      matrix:
        include:
@@ -281,13 +278,7 @@ jobs:
      - uses: actions/setup-python@v2
        with:
          python-version: ${{ matrix.python-version }}
-      - name: Patch Buildkite-specific test scripts
-        run: |
-          sed -i -e 's/host="postgres"/host="localhost"/' .buildkite/scripts/postgres_exec.py
-          sed -i -e 's/host: postgres/host: localhost/' .buildkite/postgres-config.yaml
-          sed -i -e 's|/src/||' .buildkite/{sqlite,postgres}-config.yaml
-          sed -i -e 's/\$TOP/\$GITHUB_WORKSPACE/' .coveragerc
-      - run: .buildkite/scripts/test_synapse_port_db.sh
+      - run: .ci/scripts/test_synapse_port_db.sh

  complement:
    if: ${{ !failure() && !cancelled() }}
@@ -374,6 +365,11 @@ jobs:
          rc=0
          results=$(jq -r 'to_entries[] | [.key,.value.result] | join(" ")' <<< $NEEDS_CONTEXT)
          while read job result ; do
+              # The newsfile lint may be skipped on non PR builds
+              if [ $result == "skipped" ] && [ $job == "lint-newsfile" ]; then
+                continue
+              fi
+
              if [ "$result" != "success" ]; then
                  echo "::set-failed ::Job $job returned $result"
                  rc=1
@@ -13,8 +13,9 @@ This document aims to get you started with contributing to this repo!
 - [7. Turn coffee and documentation into code and documentation!](#7-turn-coffee-and-documentation-into-code-and-documentation)
 - [8. Test, test, test!](#8-test-test-test)
  * [Run the linters.](#run-the-linters)
-  * [Run the unit tests.](#run-the-unit-tests)
-  * [Run the integration tests.](#run-the-integration-tests)
+  * [Run the unit tests.](#run-the-unit-tests-twisted-trial)
+  * [Run the integration tests (SyTest).](#run-the-integration-tests-sytest)
+  * [Run the integration tests (Complement).](#run-the-integration-tests-complement)
 - [9. Submit your patch.](#9-submit-your-patch)
  * [Changelog](#changelog)
    + [How do I know what to call the changelog file before I create the PR?](#how-do-i-know-what-to-call-the-changelog-file-before-i-create-the-pr)
@@ -197,7 +198,7 @@ The following command will let you run the integration test with the most common
 configuration:

 ```sh
-$ docker run --rm -it -v /path/where/you/have/cloned/the/repository\:/src:ro -v /path/to/where/you/want/logs\:/logs matrixdotorg/sytest-synapse:py37
+$ docker run --rm -it -v /path/where/you/have/cloned/the/repository\:/src:ro -v /path/to/where/you/want/logs\:/logs matrixdotorg/sytest-synapse:buster
 ```

 This configuration should generally cover  your needs. For more details about other configurations, see [documentation in the SyTest repo](https://github.com/matrix-org/sytest/blob/develop/docker/README.md).
@@ -252,6 +253,7 @@ To prepare a Pull Request, please:
 4. on GitHub, [create the Pull Request](https://docs.github.com/en/github/collaborating-with-issues-and-pull-requests/creating-a-pull-request);
 5. add a [changelog entry](#changelog) and push it to your Pull Request;
 6. for most contributors, that's all - however, if you are a member of the organization `matrix-org`, on GitHub, please request a review from `matrix.org / Synapse Core`.
+7. if you need to update your PR, please avoid rebasing and just add new commits to your branch.


 ## Changelog
@@ -44,9 +44,9 @@ include book.toml
 include pyproject.toml
 recursive-include changelog.d *

-prune .buildkite
 prune .circleci
 prune .github
+prune .ci
 prune contrib
 prune debian
 prune demo/etc
@@ -0,0 +1 @@
+Improve event caching mechanism to avoid having multiple copies of an event in memory at a time.
@@ -0,0 +1 @@
+Add some clarification to the sample config file. Contributed by @Kentokamoto.
@@ -0,0 +1 @@
+Initial local support for [MSC3266](https://github.com/matrix-org/synapse/pull/10394), Room Summary over the unstable `/rooms/{roomIdOrAlias}/summary` API.
@@ -0,0 +1 @@
+Experimental support for [MSC3288](https://github.com/matrix-org/matrix-doc/pull/3288), sending `room_type` to the identity server for 3pid invites over the `/store-invite` API.
@@ -0,0 +1 @@
+Add documentation for configuration a forward proxy.
@@ -0,0 +1 @@
+Add support for sending federation requests through a proxy. Contributed by @Bubu and @dklimpel.
@@ -0,0 +1 @@
+Add support for "marker" events which makes historical events discoverable for servers that already have all of the scrollback history (part of MSC2716).
@@ -0,0 +1 @@
+Reduce errors in PostgreSQL logs due to concurrent serialization errors.
@@ -0,0 +1 @@
+Include room ID in ignored EDU log messages. Contributed by @ilmari.
@@ -0,0 +1 @@
+Add a configuration setting for the time a `/sync` response is cached for.
@@ -0,0 +1 @@
+Port the PresenceRouter module interface to the new generic interface.
@@ -0,0 +1 @@
+Add pagination to the spaces summary based on updates to [MSC2946](https://github.com/matrix-org/matrix-doc/pull/2946).
@@ -0,0 +1 @@
+Fix CI to not break when run against branches rather than pull requests.
@@ -0,0 +1 @@
+Add pagination to the spaces summary based on updates to [MSC2946](https://github.com/matrix-org/matrix-doc/pull/2946).
@@ -0,0 +1 @@
+Fix a long-standing bug where protocols which are not implemented by any appservices were incorrectly returned via `GET /_matrix/client/r0/thirdparty/protocols`.
@@ -0,0 +1 @@
+Mark all events stemming from the MSC2716 `/batch_send` endpoint as historical.
@@ -0,0 +1 @@
+Add support for new redaction rules for historical events specified in [MSC2716](https://github.com/matrix-org/matrix-doc/pull/2716).
@@ -0,0 +1 @@
+Clean up some of the federation event authentication code for clarity.
@@ -0,0 +1 @@
+Fix exceptions in logs when failing to get remote room list.
@@ -0,0 +1 @@
+Convert `Transaction` and `Edu` objects to attrs.
@@ -0,0 +1 @@
+Add a setting to disable TLS when sending email.
@@ -0,0 +1 @@
+Add pagination to the spaces summary based on updates to [MSC2946](https://github.com/matrix-org/matrix-doc/pull/2946).
@@ -0,0 +1 @@
+Fix longstanding bug which caused the user "status" to be reset when the user went offline. Contributed by @dklimpel.
@@ -0,0 +1 @@
+Updated the reverse proxy documentation to highlight the homserver configuration that is needed to make Synapse aware that is is intentionally reverse proxied.
@@ -0,0 +1 @@
+Update `/batch_send` endpoint to only return `state_events` created by the `state_events_from_before` passed in.
@@ -0,0 +1 @@
+Admin API to delete several media for a specific user. Contributed by @dklimpel.
@@ -0,0 +1 @@
+Add pagination to the spaces summary based on updates to [MSC2946](https://github.com/matrix-org/matrix-doc/pull/2946).
@@ -0,0 +1 @@
+Display an error on User-Interactive Authentication fallback pages when authentication fails. Contributed by Callum Brown.
@@ -0,0 +1 @@
+Update contributing.md to warn against rebasing an open PR.
@@ -0,0 +1 @@
+Add pagination to the spaces summary based on updates to [MSC2946](https://github.com/matrix-org/matrix-doc/pull/2946).
@@ -0,0 +1 @@
+Update the Synapse Grafana dashboard.
@@ -0,0 +1 @@
+Clarify error message when failing to join a restricted room.
@@ -0,0 +1 @@
+Remove references to BuildKite in favour of GitHub Actions.
@@ -0,0 +1 @@
+Add pagination to the spaces summary based on updates to [MSC2946](https://github.com/matrix-org/matrix-doc/pull/2946).
@@ -0,0 +1 @@
+Add pagination to the spaces summary based on updates to [MSC2946](https://github.com/matrix-org/matrix-doc/pull/2946).
@@ -0,0 +1 @@
+Move `/batch_send` endpoint defined by [MSC2716](https://github.com/matrix-org/matrix-doc/pull/2716) to the `/v2_alpha` directory.
@@ -0,0 +1 @@
+Add an admin API (`GET /_synapse/admin/username_available`) to check if a username is available (regardless of registration settings).
@@ -0,0 +1 @@
+Add pagination to the spaces summary based on updates to [MSC2946](https://github.com/matrix-org/matrix-doc/pull/2946).
@@ -0,0 +1 @@
+Allow public rooms to be previewed in the spaces summary APIs from [MSC2946](https://github.com/matrix-org/matrix-doc/pull/2946).
@@ -0,0 +1 @@
+Allow multiple custom directories in `read_templates`.
@@ -0,0 +1 @@
+No longer build `.dev` packages for Ubuntu 20.10 LTS Groovy Gorilla, which has now EOLed.
@@ -0,0 +1 @@
+Re-organize the `synapse.federation.transport.server` module to create smaller files.
@@ -0,0 +1 @@
+Clean up some of the federation event authentication code for clarity.
@@ -0,0 +1 @@
+Fix a bug introduced in v1.37.1 where an error could occur in the asyncronous processing of PDUs when the queue was empty.
@@ -0,0 +1 @@
+Allow editing a user's `external_ids` via the "Edit User" admin API. Contributed by @dklimpel.
@@ -0,0 +1 @@
+Update CONTRIBUTING.md to fix index links and the instructions for SyTest in docker.
@@ -0,0 +1 @@
+Flatten the `synapse.rest.client` package by moving the contents of `v1` and `v2_alpha` into the parent.
@@ -0,0 +1 @@
+The Synapse manhole no longer needs coroutines to be wrapped in `defer.ensureDeferred`.
@@ -0,0 +1 @@
+Fix errors on /sync when read receipt data is a string. Only affects homeservers with the experimental flag for [MSC2285](https://github.com/matrix-org/matrix-doc/pull/2285) enabled. Contributed by @SimonBrandner.
@@ -0,0 +1 @@
+Additional validation for the spaces summary API to avoid errors like `ValueError: Stop argument for islice() must be None or an integer`. The missing validation has existed since v1.31.0.
@@ -0,0 +1 @@
+Clean up some of the federation event authentication code for clarity.
@@ -0,0 +1 @@
+Clean up some of the federation event authentication code for clarity.
@@ -0,0 +1 @@
+Fix up a couple of links to the database schema documentation.
@@ -0,0 +1 @@
+Revert behaviour introduced in v1.38.0 that strips `org.matrix.msc2732.device_unused_fallback_key_types` from `/sync` when its value is empty. This field should instead always be present according to [MSC2732](https://github.com/matrix-org/matrix-doc/blob/master/proposals/2732-olm-fallback-keys.md).
@@ -0,0 +1 @@
+Convert room member storage tuples to `attrs` classes.
@@ -0,0 +1 @@
+Use auto-attribs for the attrs classes used in sync.
@@ -0,0 +1 @@
+Log errors in tests to stderr.
@@ -0,0 +1 @@
+Remove deprecated Shutdown Room and Purge Room Admin API.
@@ -0,0 +1 @@
+Add `get_userinfo_by_id` method to ModuleApi.
@@ -54,7 +54,7 @@
  "gnetId": null,
  "graphTooltip": 0,
  "id": null,
-  "iteration": 1621258266004,
+  "iteration": 1628606819564,
  "links": [
    {
      "asDropdown": false,
@@ -307,7 +307,6 @@
      ],
      "thresholds": [
        {
-          "$$hashKey": "object:283",
          "colorMode": "warning",
          "fill": false,
          "line": true,
@@ -316,7 +315,6 @@
          "yaxis": "left"
        },
        {
-          "$$hashKey": "object:284",
          "colorMode": "critical",
          "fill": false,
          "line": true,
@@ -344,7 +342,6 @@
      },
      "yaxes": [
        {
-          "$$hashKey": "object:255",
          "decimals": null,
          "format": "s",
          "label": "",
@@ -354,7 +351,6 @@
          "show": true
        },
        {
-          "$$hashKey": "object:256",
          "format": "hertz",
          "label": "",
          "logBase": 1,
@@ -429,7 +425,6 @@
      ],
      "thresholds": [
        {
-          "$$hashKey": "object:566",
          "colorMode": "critical",
          "fill": true,
          "line": true,
@@ -457,7 +452,6 @@
      },
      "yaxes": [
        {
-          "$$hashKey": "object:538",
          "decimals": null,
          "format": "percentunit",
          "label": null,
@@ -467,7 +461,6 @@
          "show": true
        },
        {
-          "$$hashKey": "object:539",
          "format": "short",
          "label": null,
          "logBase": 1,
@@ -573,7 +566,6 @@
      },
      "yaxes": [
        {
-          "$$hashKey": "object:1560",
          "format": "bytes",
          "logBase": 1,
          "max": null,
@@ -581,7 +573,6 @@
          "show": true
        },
        {
-          "$$hashKey": "object:1561",
          "format": "short",
          "logBase": 1,
          "max": null,
@@ -641,7 +632,6 @@
      "renderer": "flot",
      "seriesOverrides": [
        {
-          "$$hashKey": "object:639",
          "alias": "/max$/",
          "color": "#890F02",
          "fill": 0,
@@ -693,7 +683,6 @@
      },
      "yaxes": [
        {
-          "$$hashKey": "object:650",
          "decimals": null,
          "format": "none",
          "label": "",
@@ -703,7 +692,6 @@
          "show": true
        },
        {
-          "$$hashKey": "object:651",
          "decimals": null,
          "format": "short",
          "label": null,
@@ -783,11 +771,9 @@
          "renderer": "flot",
          "seriesOverrides": [
            {
-              "$$hashKey": "object:1240",
              "alias": "/user/"
            },
            {
-              "$$hashKey": "object:1241",
              "alias": "/system/"
            }
          ],
@@ -817,7 +803,6 @@
          ],
          "thresholds": [
            {
-              "$$hashKey": "object:1278",
              "colorMode": "custom",
              "fillColor": "rgba(255, 255, 255, 1)",
              "line": true,
@@ -827,7 +812,6 @@
              "yaxis": "left"
            },
            {
-              "$$hashKey": "object:1279",
              "colorMode": "custom",
              "fillColor": "rgba(255, 255, 255, 1)",
              "line": true,
@@ -837,7 +821,6 @@
              "yaxis": "left"
            },
            {
-              "$$hashKey": "object:1498",
              "colorMode": "critical",
              "fill": true,
              "line": true,
@@ -865,7 +848,6 @@
          },
          "yaxes": [
            {
-              "$$hashKey": "object:1250",
              "decimals": null,
              "format": "percentunit",
              "label": "",
@@ -875,7 +857,6 @@
              "show": true
            },
            {
-              "$$hashKey": "object:1251",
              "format": "short",
              "logBase": 1,
              "max": null,
@@ -1427,7 +1408,6 @@
          },
          "yaxes": [
            {
-              "$$hashKey": "object:572",
              "format": "percentunit",
              "label": null,
              "logBase": 1,
@@ -1436,7 +1416,6 @@
              "show": true
            },
            {
-              "$$hashKey": "object:573",
              "format": "short",
              "label": null,
              "logBase": 1,
@@ -1720,7 +1699,6 @@
          },
          "yaxes": [
            {
-              "$$hashKey": "object:102",
              "format": "hertz",
              "logBase": 1,
              "max": null,
@@ -1728,7 +1706,6 @@
              "show": true
            },
            {
-              "$$hashKey": "object:103",
              "format": "short",
              "logBase": 1,
              "max": null,
@@ -3425,7 +3402,7 @@
            "h": 9,
            "w": 12,
            "x": 0,
-            "y": 33
+            "y": 6
          },
          "hiddenSeries": false,
          "id": 79,
@@ -3442,9 +3419,12 @@
          "linewidth": 1,
          "links": [],
          "nullPointMode": "null",
+          "options": {
+            "alertThreshold": true
+          },
          "paceLength": 10,
          "percentage": false,
-          "pluginVersion": "7.1.3",
+          "pluginVersion": "7.3.7",
          "pointradius": 5,
          "points": false,
          "renderer": "flot",
@@ -3526,7 +3506,7 @@
            "h": 9,
            "w": 12,
            "x": 12,
-            "y": 33
+            "y": 6
          },
          "hiddenSeries": false,
          "id": 83,
@@ -3543,9 +3523,12 @@
          "linewidth": 1,
          "links": [],
          "nullPointMode": "null",
+          "options": {
+            "alertThreshold": true
+          },
          "paceLength": 10,
          "percentage": false,
-          "pluginVersion": "7.1.3",
+          "pluginVersion": "7.3.7",
          "pointradius": 5,
          "points": false,
          "renderer": "flot",
@@ -3629,7 +3612,7 @@
            "h": 9,
            "w": 12,
            "x": 0,
-            "y": 42
+            "y": 15
          },
          "hiddenSeries": false,
          "id": 109,
@@ -3646,9 +3629,12 @@
          "linewidth": 1,
          "links": [],
          "nullPointMode": "null",
+          "options": {
+            "alertThreshold": true
+          },
          "paceLength": 10,
          "percentage": false,
-          "pluginVersion": "7.1.3",
+          "pluginVersion": "7.3.7",
          "pointradius": 5,
          "points": false,
          "renderer": "flot",
@@ -3733,7 +3719,7 @@
            "h": 9,
            "w": 12,
            "x": 12,
-            "y": 42
+            "y": 15
          },
          "hiddenSeries": false,
          "id": 111,
@@ -3750,9 +3736,12 @@
          "linewidth": 1,
          "links": [],
          "nullPointMode": "null",
+          "options": {
+            "alertThreshold": true
+          },
          "paceLength": 10,
          "percentage": false,
-          "pluginVersion": "7.1.3",
+          "pluginVersion": "7.3.7",
          "pointradius": 5,
          "points": false,
          "renderer": "flot",
@@ -3831,7 +3820,7 @@
            "h": 8,
            "w": 12,
            "x": 0,
-            "y": 51
+            "y": 24
          },
          "hiddenSeries": false,
          "id": 142,
@@ -3847,8 +3836,11 @@
          "lines": true,
          "linewidth": 1,
          "nullPointMode": "null",
+          "options": {
+            "alertThreshold": true
+          },
          "percentage": false,
-          "pluginVersion": "7.1.3",
+          "pluginVersion": "7.3.7",
          "pointradius": 2,
          "points": false,
          "renderer": "flot",
@@ -3931,7 +3923,7 @@
            "h": 9,
            "w": 12,
            "x": 12,
-            "y": 51
+            "y": 24
          },
          "hiddenSeries": false,
          "id": 140,
@@ -3948,9 +3940,12 @@
          "linewidth": 1,
          "links": [],
          "nullPointMode": "null",
+          "options": {
+            "alertThreshold": true
+          },
          "paceLength": 10,
          "percentage": false,
-          "pluginVersion": "7.1.3",
+          "pluginVersion": "7.3.7",
          "pointradius": 5,
          "points": false,
          "renderer": "flot",
@@ -4079,7 +4074,7 @@
            "h": 9,
            "w": 12,
            "x": 0,
-            "y": 59
+            "y": 32
          },
          "heatmap": {},
          "hideZeroBuckets": false,
@@ -4145,7 +4140,7 @@
            "h": 9,
            "w": 12,
            "x": 12,
-            "y": 60
+            "y": 33
          },
          "hiddenSeries": false,
          "id": 162,
@@ -4163,9 +4158,12 @@
          "linewidth": 0,
          "links": [],
          "nullPointMode": "connected",
+          "options": {
+            "alertThreshold": true
+          },
          "paceLength": 10,
          "percentage": false,
-          "pluginVersion": "7.1.3",
+          "pluginVersion": "7.3.7",
          "pointradius": 5,
          "points": false,
          "renderer": "flot",
@@ -4350,7 +4348,7 @@
            "h": 9,
            "w": 12,
            "x": 0,
-            "y": 68
+            "y": 41
          },
          "heatmap": {},
          "hideZeroBuckets": false,
@@ -4396,6 +4394,311 @@
          "yBucketBound": "auto",
          "yBucketNumber": null,
          "yBucketSize": null
+        },
+        {
+          "aliasColors": {},
+          "bars": false,
+          "dashLength": 10,
+          "dashes": false,
+          "datasource": "$datasource",
+          "editable": true,
+          "error": false,
+          "fieldConfig": {
+            "defaults": {
+              "custom": {},
+              "links": []
+            },
+            "overrides": []
+          },
+          "fill": 1,
+          "fillGradient": 0,
+          "grid": {},
+          "gridPos": {
+            "h": 9,
+            "w": 12,
+            "x": 12,
+            "y": 42
+          },
+          "hiddenSeries": false,
+          "id": 203,
+          "legend": {
+            "avg": false,
+            "current": false,
+            "max": false,
+            "min": false,
+            "show": true,
+            "total": false,
+            "values": false
+          },
+          "lines": true,
+          "linewidth": 2,
+          "links": [],
+          "nullPointMode": "null",
+          "options": {
+            "alertThreshold": true
+          },
+          "paceLength": 10,
+          "percentage": false,
+          "pluginVersion": "7.3.7",
+          "pointradius": 5,
+          "points": false,
+          "renderer": "flot",
+          "seriesOverrides": [],
+          "spaceLength": 10,
+          "stack": false,
+          "steppedLine": false,
+          "targets": [
+            {
+              "expr": "synapse_federation_server_oldest_inbound_pdu_in_staging{job=\"$job\",index=~\"$index\",instance=\"$instance\"}",
+              "format": "time_series",
+              "interval": "",
+              "intervalFactor": 1,
+              "legendFormat": "rss {{index}}",
+              "refId": "A",
+              "step": 4
+            }
+          ],
+          "thresholds": [],
+          "timeFrom": null,
+          "timeRegions": [],
+          "timeShift": null,
+          "title": "Age of oldest event in staging area",
+          "tooltip": {
+            "msResolution": false,
+            "shared": true,
+            "sort": 0,
+            "value_type": "cumulative"
+          },
+          "type": "graph",
+          "xaxis": {
+            "buckets": null,
+            "mode": "time",
+            "name": null,
+            "show": true,
+            "values": []
+          },
+          "yaxes": [
+            {
+              "format": "ms",
+              "label": null,
+              "logBase": 1,
+              "max": null,
+              "min": 0,
+              "show": true
+            },
+            {
+              "format": "short",
+              "label": null,
+              "logBase": 1,
+              "max": null,
+              "min": null,
+              "show": true
+            }
+          ],
+          "yaxis": {
+            "align": false,
+            "alignLevel": null
+          }
+        },
+        {
+          "aliasColors": {},
+          "bars": false,
+          "dashLength": 10,
+          "dashes": false,
+          "datasource": "$datasource",
+          "editable": true,
+          "error": false,
+          "fieldConfig": {
+            "defaults": {
+              "custom": {},
+              "links": []
+            },
+            "overrides": []
+          },
+          "fill": 1,
+          "fillGradient": 0,
+          "grid": {},
+          "gridPos": {
+            "h": 9,
+            "w": 12,
+            "x": 0,
+            "y": 50
+          },
+          "hiddenSeries": false,
+          "id": 202,
+          "legend": {
+            "avg": false,
+            "current": false,
+            "max": false,
+            "min": false,
+            "show": true,
+            "total": false,
+            "values": false
+          },
+          "lines": true,
+          "linewidth": 2,
+          "links": [],
+          "nullPointMode": "null",
+          "options": {
+            "alertThreshold": true
+          },
+          "paceLength": 10,
+          "percentage": false,
+          "pluginVersion": "7.3.7",
+          "pointradius": 5,
+          "points": false,
+          "renderer": "flot",
+          "seriesOverrides": [],
+          "spaceLength": 10,
+          "stack": false,
+          "steppedLine": false,
+          "targets": [
+            {
+              "expr": "synapse_federation_server_number_inbound_pdu_in_staging{job=\"$job\",index=~\"$index\",instance=\"$instance\"}",
+              "format": "time_series",
+              "interval": "",
+              "intervalFactor": 1,
+              "legendFormat": "rss {{index}}",
+              "refId": "A",
+              "step": 4
+            }
+          ],
+          "thresholds": [],
+          "timeFrom": null,
+          "timeRegions": [],
+          "timeShift": null,
+          "title": "Number of events in federation staging area",
+          "tooltip": {
+            "msResolution": false,
+            "shared": true,
+            "sort": 0,
+            "value_type": "cumulative"
+          },
+          "type": "graph",
+          "xaxis": {
+            "buckets": null,
+            "mode": "time",
+            "name": null,
+            "show": true,
+            "values": []
+          },
+          "yaxes": [
+            {
+              "format": "none",
+              "label": null,
+              "logBase": 1,
+              "max": null,
+              "min": 0,
+              "show": true
+            },
+            {
+              "format": "short",
+              "label": null,
+              "logBase": 1,
+              "max": null,
+              "min": null,
+              "show": true
+            }
+          ],
+          "yaxis": {
+            "align": false,
+            "alignLevel": null
+          }
+        },
+        {
+          "aliasColors": {},
+          "bars": false,
+          "dashLength": 10,
+          "dashes": false,
+          "datasource": "${DS_PROMETHEUS}",
+          "fieldConfig": {
+            "defaults": {
+              "custom": {}
+            },
+            "overrides": []
+          },
+          "fill": 1,
+          "fillGradient": 0,
+          "gridPos": {
+            "h": 8,
+            "w": 12,
+            "x": 12,
+            "y": 51
+          },
+          "hiddenSeries": false,
+          "id": 205,
+          "legend": {
+            "avg": false,
+            "current": false,
+            "max": false,
+            "min": false,
+            "show": true,
+            "total": false,
+            "values": false
+          },
+          "lines": true,
+          "linewidth": 1,
+          "nullPointMode": "null",
+          "options": {
+            "alertThreshold": true
+          },
+          "percentage": false,
+          "pluginVersion": "7.3.7",
+          "pointradius": 2,
+          "points": false,
+          "renderer": "flot",
+          "seriesOverrides": [],
+          "spaceLength": 10,
+          "stack": false,
+          "steppedLine": false,
+          "targets": [
+            {
+              "expr": "sum(rate(synapse_federation_soft_failed_events_total{instance=\"$instance\"}[$bucket_size]))",
+              "interval": "",
+              "legendFormat": "soft-failed events",
+              "refId": "A"
+            }
+          ],
+          "thresholds": [],
+          "timeFrom": null,
+          "timeRegions": [],
+          "timeShift": null,
+          "title": "Soft-failed event rate",
+          "tooltip": {
+            "shared": true,
+            "sort": 0,
+            "value_type": "individual"
+          },
+          "type": "graph",
+          "xaxis": {
+            "buckets": null,
+            "mode": "time",
+            "name": null,
+            "show": true,
+            "values": []
+          },
+          "yaxes": [
+            {
+              "format": "hertz",
+              "label": null,
+              "logBase": 1,
+              "max": null,
+              "min": null,
+              "show": true
+            },
+            {
+              "format": "short",
+              "label": null,
+              "logBase": 1,
+              "max": null,
+              "min": null,
+              "show": false
+            }
+          ],
+          "yaxis": {
+            "align": false,
+            "alignLevel": null
+          }
        }
      ],
      "title": "Federation",
@@ -4647,7 +4950,7 @@
            "h": 7,
            "w": 12,
            "x": 0,
-            "y": 8
+            "y": 33
          },
          "hiddenSeries": false,
          "id": 48,
@@ -4749,7 +5052,7 @@
            "h": 7,
            "w": 12,
            "x": 12,
-            "y": 8
+            "y": 33
          },
          "hiddenSeries": false,
          "id": 104,
@@ -4877,7 +5180,7 @@
            "h": 7,
            "w": 12,
            "x": 0,
-            "y": 15
+            "y": 40
          },
          "hiddenSeries": false,
          "id": 10,
@@ -4981,7 +5284,7 @@
            "h": 7,
            "w": 12,
            "x": 12,
-            "y": 15
+            "y": 40
          },
          "hiddenSeries": false,
          "id": 11,
@@ -5086,7 +5389,7 @@
            "h": 7,
            "w": 12,
            "x": 0,
-            "y": 22
+            "y": 47
          },
          "hiddenSeries": false,
          "id": 180,
@@ -5168,6 +5471,126 @@
            "align": false,
            "alignLevel": null
          }
+        },
+        {
+          "aliasColors": {},
+          "bars": false,
+          "dashLength": 10,
+          "dashes": false,
+          "datasource": "$datasource",
+          "fieldConfig": {
+            "defaults": {
+              "custom": {},
+              "links": []
+            },
+            "overrides": []
+          },
+          "fill": 6,
+          "fillGradient": 0,
+          "gridPos": {
+            "h": 9,
+            "w": 12,
+            "x": 12,
+            "y": 47
+          },
+          "hiddenSeries": false,
+          "id": 200,
+          "legend": {
+            "avg": false,
+            "current": false,
+            "max": false,
+            "min": false,
+            "show": true,
+            "total": false,
+            "values": false
+          },
+          "lines": true,
+          "linewidth": 1,
+          "links": [],
+          "nullPointMode": "null",
+          "options": {
+            "alertThreshold": true
+          },
+          "percentage": false,
+          "pluginVersion": "7.3.7",
+          "pointradius": 5,
+          "points": false,
+          "renderer": "flot",
+          "seriesOverrides": [],
+          "spaceLength": 10,
+          "stack": false,
+          "steppedLine": false,
+          "targets": [
+            {
+              "expr": "histogram_quantile(0.99, sum(rate(synapse_storage_schedule_time_bucket{index=~\"$index\",instance=\"$instance\",job=\"$job\"}[$bucket_size])) by (le))",
+              "format": "time_series",
+              "intervalFactor": 1,
+              "legendFormat": "99%",
+              "refId": "D"
+            },
+            {
+              "expr": "histogram_quantile(0.9, sum(rate(synapse_storage_schedule_time_bucket{index=~\"$index\",instance=\"$instance\",job=\"$job\"}[$bucket_size])) by (le))",
+              "format": "time_series",
+              "intervalFactor": 1,
+              "legendFormat": "90%",
+              "refId": "A"
+            },
+            {
+              "expr": "histogram_quantile(0.75, sum(rate(synapse_storage_schedule_time_bucket{index=~\"$index\",instance=\"$instance\",job=\"$job\"}[$bucket_size])) by (le))",
+              "format": "time_series",
+              "intervalFactor": 1,
+              "legendFormat": "75%",
+              "refId": "C"
+            },
+            {
+              "expr": "histogram_quantile(0.5, sum(rate(synapse_storage_schedule_time_bucket{index=~\"$index\",instance=\"$instance\",job=\"$job\"}[$bucket_size])) by (le))",
+              "format": "time_series",
+              "intervalFactor": 1,
+              "legendFormat": "50%",
+              "refId": "B"
+            }
+          ],
+          "thresholds": [],
+          "timeFrom": null,
+          "timeRegions": [],
+          "timeShift": null,
+          "title": "Time waiting for DB connection quantiles",
+          "tooltip": {
+            "shared": true,
+            "sort": 2,
+            "value_type": "individual"
+          },
+          "type": "graph",
+          "xaxis": {
+            "buckets": null,
+            "mode": "time",
+            "name": null,
+            "show": true,
+            "values": []
+          },
+          "yaxes": [
+            {
+              "decimals": null,
+              "format": "s",
+              "label": "",
+              "logBase": 1,
+              "max": null,
+              "min": "0",
+              "show": true
+            },
+            {
+              "format": "short",
+              "label": null,
+              "logBase": 1,
+              "max": null,
+              "min": null,
+              "show": false
+            }
+          ],
+          "yaxis": {
+            "align": false,
+            "alignLevel": null
+          }
        }
      ],
      "repeat": null,
@@ -5916,7 +6339,7 @@
            "h": 10,
            "w": 12,
            "x": 0,
-            "y": 84
+            "y": 35
          },
          "hiddenSeries": false,
          "id": 1,
@@ -6022,7 +6445,7 @@
            "h": 10,
            "w": 12,
            "x": 12,
-            "y": 84
+            "y": 35
          },
          "hiddenSeries": false,
          "id": 8,
@@ -6126,7 +6549,7 @@
            "h": 10,
            "w": 12,
            "x": 0,
-            "y": 94
+            "y": 45
          },
          "hiddenSeries": false,
          "id": 38,
@@ -6226,7 +6649,7 @@
            "h": 10,
            "w": 12,
            "x": 12,
-            "y": 94
+            "y": 45
          },
          "hiddenSeries": false,
          "id": 39,
@@ -6258,8 +6681,9 @@
          "steppedLine": false,
          "targets": [
            {
-              "expr": "topk(10, rate(synapse_util_caches_cache:total{job=\"$job\",index=~\"$index\",instance=\"$instance\"}[$bucket_size]) - rate(synapse_util_caches_cache:hits{job=\"$job\",instance=\"$instance\"}[$bucket_size]))",
+              "expr": "topk(10, rate(synapse_util_caches_cache:total{job=~\"$job\",index=~\"$index\",instance=\"$instance\"}[$bucket_size]) - rate(synapse_util_caches_cache:hits{job=~\"$job\",index=~\"$index\",instance=\"$instance\"}[$bucket_size]))",
              "format": "time_series",
+              "interval": "",
              "intervalFactor": 2,
              "legendFormat": "{{name}} {{job}}-{{index}}",
              "refId": "A",
@@ -6326,7 +6750,7 @@
            "h": 9,
            "w": 12,
            "x": 0,
-            "y": 104
+            "y": 55
          },
          "hiddenSeries": false,
          "id": 65,
@@ -9051,7 +9475,7 @@
            "h": 8,
            "w": 12,
            "x": 0,
-            "y": 119
+            "y": 41
          },
          "hiddenSeries": false,
          "id": 156,
@@ -9089,7 +9513,7 @@
          "steppedLine": false,
          "targets": [
            {
-              "expr": "synapse_admin_mau:current{instance=\"$instance\"}",
+              "expr": "synapse_admin_mau:current{instance=\"$instance\", job=~\"$job\"}",
              "format": "time_series",
              "interval": "",
              "intervalFactor": 1,
@@ -9097,7 +9521,7 @@
              "refId": "A"
            },
            {
-              "expr": "synapse_admin_mau:max{instance=\"$instance\"}",
+              "expr": "synapse_admin_mau:max{instance=\"$instance\", job=~\"$job\"}",
              "format": "time_series",
              "interval": "",
              "intervalFactor": 1,
@@ -9164,7 +9588,7 @@
            "h": 8,
            "w": 12,
            "x": 12,
-            "y": 119
+            "y": 41
          },
          "hiddenSeries": false,
          "id": 160,
@@ -9484,7 +9908,7 @@
            "h": 8,
            "w": 12,
            "x": 0,
-            "y": 73
+            "y": 43
          },
          "hiddenSeries": false,
          "id": 168,
@@ -9516,7 +9940,7 @@
            {
              "expr": "rate(synapse_appservice_api_sent_events{instance=\"$instance\"}[$bucket_size])",
              "interval": "",
-              "legendFormat": "{{exported_service}}",
+              "legendFormat": "{{service}}",
              "refId": "A"
            }
          ],
@@ -9579,7 +10003,7 @@
            "h": 8,
            "w": 12,
            "x": 12,
-            "y": 73
+            "y": 43
          },
          "hiddenSeries": false,
          "id": 171,
@@ -9611,7 +10035,7 @@
            {
              "expr": "rate(synapse_appservice_api_sent_transactions{instance=\"$instance\"}[$bucket_size])",
              "interval": "",
-              "legendFormat": "{{exported_service}}",
+              "legendFormat": "{{service}}",
              "refId": "A"
            }
          ],
@@ -9959,7 +10383,6 @@
          },
          "yaxes": [
            {
-              "$$hashKey": "object:165",
              "format": "hertz",
              "label": null,
              "logBase": 1,
@@ -9968,7 +10391,6 @@
              "show": true
            },
            {
-              "$$hashKey": "object:166",
              "format": "short",
              "label": null,
              "logBase": 1,
@@ -10071,7 +10493,6 @@
          },
          "yaxes": [
            {
-              "$$hashKey": "object:390",
              "format": "hertz",
              "label": null,
              "logBase": 1,
@@ -10080,7 +10501,6 @@
              "show": true
            },
            {
-              "$$hashKey": "object:391",
              "format": "short",
              "label": null,
              "logBase": 1,
@@ -10169,7 +10589,6 @@
          },
          "yaxes": [
            {
-              "$$hashKey": "object:390",
              "format": "hertz",
              "label": null,
              "logBase": 1,
@@ -10178,7 +10597,6 @@
              "show": true
            },
            {
-              "$$hashKey": "object:391",
              "format": "short",
              "label": null,
              "logBase": 1,
@@ -10470,5 +10888,5 @@
  "timezone": "",
  "title": "Synapse",
  "uid": "000000012",
-  "version": 90
+  "version": 99
 }
@@ -100,3 +100,18 @@ esac
 # add a dependency on the right version of python to substvars.
 PYPKG=`basename $SNAKE`
 echo "synapse:pydepends=$PYPKG" >> debian/matrix-synapse-py3.substvars
+
+
+# add a couple of triggers.  This is needed so that dh-virtualenv can rebuild
+# the venv when the system python changes (see
+# https://dh-virtualenv.readthedocs.io/en/latest/tutorial.html#step-2-set-up-packaging-for-your-project)
+#
+# we do it here rather than the more conventional way of just adding it to
+# debian/matrix-synapse-py3.triggers, because we need to add a trigger on the
+# right version of python.
+cat >>"debian/.debhelper/generated/matrix-synapse-py3/triggers" <<EOF
+# triggers for dh-virtualenv
+interest-noawait $SNAKE
+interest dh-virtualenv-interpreter-update
+
+EOF
@@ -20,6 +20,8 @@ matrix-synapse-py3 (1.40.0~rc1) stable; urgency=medium

  [ Richard van der Hoff ]
  * Drop backwards-compatibility code that was required to support Ubuntu Xenial.
+  * Update package triggers so that the virtualenv is correctly rebuilt
+    when the system python is rebuilt, on recent Python versions.

  [ Synapse Packaging team ]
  * New synapse release 1.40.0~rc1.
@@ -1,9 +0,0 @@
-# Register interest in Python interpreter changes and
-# don't make the Python package dependent on the virtualenv package
-# processing (noawait)
-interest-noawait /usr/bin/python3.5
-interest-noawait /usr/bin/python3.6
-interest-noawait /usr/bin/python3.7
-
-# Also provide a symbolic trigger for all dh-virtualenv packages
-interest dh-virtualenv-interpreter-update
@@ -7,6 +7,7 @@
  - [Installation](setup/installation.md)
  - [Using Postgres](postgres.md)
  - [Configuring a Reverse Proxy](reverse_proxy.md)
+  - [Configuring a Forward/Outbound Proxy](setup/forward_proxy.md)
  - [Configuring a Turn Server](turn-howto.md)
  - [Delegation](delegate.md)

@@ -50,12 +51,10 @@
      - [Event Reports](admin_api/event_reports.md)
      - [Media](admin_api/media_admin_api.md)
      - [Purge History](admin_api/purge_history_api.md)
-      - [Purge Rooms](admin_api/purge_room.md)
      - [Register Users](admin_api/register_api.md)
      - [Manipulate Room Membership](admin_api/room_membership.md)
      - [Rooms](admin_api/rooms.md)
      - [Server Notices](admin_api/server_notices.md)
-      - [Shutdown Room](admin_api/shutdown_room.md)
      - [Statistics](admin_api/statistics.md)
      - [Users](admin_api/user_admin_api.md)
      - [Server Version](admin_api/version_api.md)
@@ -12,6 +12,7 @@
 - [Delete local media](#delete-local-media)
  * [Delete a specific local media](#delete-a-specific-local-media)
  * [Delete local media by date or size](#delete-local-media-by-date-or-size)
+  * [Delete media uploaded by a user](#delete-media-uploaded-by-a-user)
 - [Purge Remote Media API](#purge-remote-media-api)

 # Querying media
@@ -47,7 +48,8 @@ The API returns a JSON body like the following:
 ## List all media uploaded by a user

 Listing all media that has been uploaded by a local user can be achieved through
-the use of the [List media of a user](user_admin_api.md#list-media-of-a-user)
+the use of the
+[List media uploaded by a user](user_admin_api.md#list-media-uploaded-by-a-user)
 Admin API.

 # Quarantine media
@@ -281,6 +283,11 @@ The following fields are returned in the JSON response body:
 * `deleted_media`: an array of strings - List of deleted `media_id`
 * `total`: integer - Total number of deleted `media_id`

+## Delete media uploaded by a user
+
+You can find details of how to delete multiple media uploaded by a user in
+[User Admin API](user_admin_api.md#delete-media-uploaded-by-a-user).
+
 # Purge Remote Media API

 The purge remote media API allows server admins to purge old cached remote media.
@@ -1,21 +0,0 @@
-Deprecated: Purge room API
-==========================
-
-**The old Purge room API is deprecated and will be removed in a future release.
-See the new [Delete Room API](rooms.md#delete-room-api) for more details.**
-
-This API will remove all trace of a room from your database.
-
-All local users must have left the room before it can be removed.
-
-The API is:
-
-```
-POST /_synapse/admin/v1/purge_room
-
-{
-    "room_id": "!room:id"
-}
-```
-
-You must authenticate using the access token of an admin user.
@@ -1,102 +0,0 @@
-# Deprecated: Shutdown room API
-
-**The old Shutdown room API is deprecated and will be removed in a future release.
-See the new [Delete Room API](rooms.md#delete-room-api) for more details.**
-
-Shuts down a room, preventing new joins and moves local users and room aliases automatically
-to a new room. The new room will be created with the user specified by the
-`new_room_user_id` parameter as room administrator and will contain a message
-explaining what happened. Users invited to the new room will have power level
-10 by default, and thus be unable to speak. The old room's power levels will be changed to
-disallow any further invites or joins.
-
-The local server will only have the power to move local user and room aliases to
-the new room. Users on other servers will be unaffected.
-
-## API
-
-You will need to authenticate with an access token for an admin user.
-
-### URL
-
-`POST /_synapse/admin/v1/shutdown_room/{room_id}`
-
-### URL Parameters
-
-* `room_id` - The ID of the room (e.g `!someroom:example.com`)
-
-### JSON Body Parameters
-
-* `new_room_user_id` - Required. A string representing the user ID of the user that will admin
-                       the new room that all users in the old room will be moved to.
-* `room_name` - Optional. A string representing the name of the room that new users will be
-                invited to.
-* `message` - Optional. A string containing the first message that will be sent as
-              `new_room_user_id` in the new room. Ideally this will clearly convey why the
-               original room was shut down.
-
-If not specified, the default value of `room_name` is "Content Violation
-Notification". The default value of `message` is "Sharing illegal content on
-othis server is not permitted and rooms in violation will be blocked."
-
-### Response Parameters
-
-* `kicked_users` - An integer number representing the number of users that
-                   were kicked.
-* `failed_to_kick_users` - An integer number representing the number of users
-                           that were not kicked.
-* `local_aliases` - An array of strings representing the local aliases that were migrated from
-                    the old room to the new.
-* `new_room_id` - A string representing the room ID of the new room.
-
-## Example
-
-Request:
-
-```
-POST /_synapse/admin/v1/shutdown_room/!somebadroom%3Aexample.com
-
-{
-    "new_room_user_id": "@someuser:example.com",
-    "room_name": "Content Violation Notification",
-    "message": "Bad Room has been shutdown due to content violations on this server. Please review our Terms of Service."
-}
-```
-
-Response:
-
-```
-{
-    "kicked_users": 5,
-    "failed_to_kick_users": 0,
-    "local_aliases": ["#badroom:example.com", "#evilsaloon:example.com],
-    "new_room_id": "!newroomid:example.com",
-},
-```
-
-## Undoing room shutdowns
-
-*Note*: This guide may be outdated by the time you read it. By nature of room shutdowns being performed at the database level,
-the structure can and does change without notice.
-
-First, it's important to understand that a room shutdown is very destructive. Undoing a shutdown is not as simple as pretending it
-never happened - work has to be done to move forward instead of resetting the past. In fact, in some cases it might not be possible
-to recover at all:
-
-* If the room was invite-only, your users will need to be re-invited.
-* If the room no longer has any members at all, it'll be impossible to rejoin.
-* The first user to rejoin will have to do so via an alias on a different server.
-
-With all that being said, if you still want to try and recover the room:
-
-1. For safety reasons, shut down Synapse.
-2. In the database, run `DELETE FROM blocked_rooms WHERE room_id = '!example:example.org';`
-   * For caution: it's recommended to run this in a transaction: `BEGIN; DELETE ...;`, verify you got 1 result, then `COMMIT;`.
-   * The room ID is the same one supplied to the shutdown room API, not the Content Violation room.
-3. Restart Synapse.
-
-You will have to manually handle, if you so choose, the following:
-
-* Aliases that would have been redirected to the Content Violation room.
-* Users that would have been booted from the room (and will have been force-joined to the Content Violation room).
-* Removal of the Content Violation room if desired.
@@ -81,6 +81,16 @@ with a body of:
            "address": "<user_mail_2>"
        }
    ],
+    "external_ids": [
+        {
+            "auth_provider": "<provider1>",
+            "external_id": "<user_id_provider_1>"
+        },
+        {
+            "auth_provider": "<provider2>",
+            "external_id": "<user_id_provider_2>"
+        }
+    ],
    "avatar_url": "<avatar_url>",
    "admin": false,
    "deactivated": false
@@ -90,26 +100,34 @@ with a body of:
 To use it, you will need to authenticate by providing an `access_token` for a
 server admin: [Admin API](../usage/administration/admin_api)

+Returns HTTP status code:
+- `201` - When a new user object was created.
+- `200` - When a user was modified.
+
 URL parameters:

 - `user_id`: fully-qualified user id: for example, `@user:server.com`.

 Body parameters:

- `password`, optional. If provided, the user's password is updated and all
+- `password` - string, optional. If provided, the user's password is updated and all
  devices are logged out.
-
- `displayname`, optional, defaults to the value of `user_id`.
-
- `threepids`, optional, allows setting the third-party IDs (email, msisdn)
+- `displayname` - string, optional, defaults to the value of `user_id`.
+- `threepids` - array, optional, allows setting the third-party IDs (email, msisdn)
+  - `medium` - string. Kind of third-party ID, either `email` or `msisdn`.
+  - `address` - string. Value of third-party ID.
  belonging to a user.
-
- `avatar_url`, optional, must be a
+- `external_ids` - array, optional. Allow setting the identifier of the external identity
+  provider for SSO (Single sign-on). Details in
+  [Sample Configuration File](../usage/configuration/homeserver_sample_config.html)
+  section `sso` and `oidc_providers`.
+  - `auth_provider` - string. ID of the external identity provider. Value of `idp_id`
+    in homeserver configuration.
+  - `external_id` - string, user ID in the external identity provider.
+- `avatar_url` - string, optional, must be a
  [MXC URI](https://matrix.org/docs/spec/client_server/r0.6.0#matrix-content-mxc-uris).
-
- `admin`, optional, defaults to `false`.
-
- `deactivated`, optional. If unspecified, deactivation state will be left
+- `admin` - bool, optional, defaults to `false`.
+- `deactivated` - bool, optional. If unspecified, deactivation state will be left
  unchanged on existing accounts and set to `false` for new accounts.
  A user cannot be erased by deactivating with this API. For details on
  deactivating users see [Deactivate Account](#deactivate-account).
@@ -443,8 +461,9 @@ The following fields are returned in the JSON response body:
 - `joined_rooms` - An array of `room_id`.
 - `total` - Number of rooms.

+## User media

-## List media of a user
+### List media uploaded by a user
 Gets a list of all local media that a specific `user_id` has created.
 By default, the response is ordered by descending creation date and ascending media ID.
 The newest media is on top. You can change the order with parameters
@@ -543,7 +562,6 @@ The following fields are returned in the JSON response body:

 - `media` - An array of objects, each containing information about a media.
  Media objects contain the following fields:
-
  - `created_ts` - integer - Timestamp when the content was uploaded in ms.
  - `last_access_ts` - integer - Timestamp when the content was last accessed in ms.
  - `media_id` - string - The id used to refer to the media.
@@ -551,13 +569,58 @@ The following fields are returned in the JSON response body:
  - `media_type` - string - The MIME-type of the media.
  - `quarantined_by` - string - The user ID that initiated the quarantine request
    for this media.
-
  - `safe_from_quarantine` - bool - Status if this media is safe from quarantining.
  - `upload_name` - string - The name the media was uploaded with.
-
 - `next_token`: integer - Indication for pagination. See above.
 - `total` - integer - Total number of media.

+### Delete media uploaded by a user
+
+This API deletes the *local* media from the disk of your own server
+that a specific `user_id` has created. This includes any local thumbnails.
+
+This API will not affect media that has been uploaded to external
+media repositories (e.g https://github.com/turt2live/matrix-media-repo/).
+
+By default, the API deletes media ordered by descending creation date and ascending media ID.
+The newest media is deleted first. You can change the order with parameters
+`order_by` and `dir`. If no `limit` is set the API deletes `100` files per request.
+
+The API is:
+
+```
+DELETE /_synapse/admin/v1/users/<user_id>/media
+```
+
+To use it, you will need to authenticate by providing an `access_token` for a
+server admin: [Admin API](../usage/administration/admin_api)
+
+A response body like the following is returned:
+
+```json
+{
+  "deleted_media": [
+    "abcdefghijklmnopqrstuvwx"
+  ],
+  "total": 1
+}
+```
+
+The following fields are returned in the JSON response body:
+
+* `deleted_media`: an array of strings - List of deleted `media_id`
+* `total`: integer - Total number of deleted `media_id`
+
+**Note**: There is no `next_token`. This is not useful for deleting media, because
+after deleting media the remaining media have a new order.
+
+**Parameters**
+
+This API has the same parameters as
+[List media uploaded by a user](#list-media-uploaded-by-a-user).
+With the parameters you can for example limit the number of files to delete at once or
+delete largest/smallest or newest/oldest files first.
+
 ## Login as a user

 Get an access token that can be used to authenticate as that user. Useful for
@@ -1013,3 +1076,22 @@ The following parameters should be set in the URL:
 - `user_id` - The fully qualified MXID: for example, `@user:server.com`. The user must
  be local.

+### Check username availability
+
+Checks to see if a username is available, and valid, for the server. See [the client-server 
+API](https://matrix.org/docs/spec/client_server/r0.6.0#get-matrix-client-r0-register-available)
+for more information.
+
+This endpoint will work even if registration is disabled on the server, unlike 
+`/_matrix/client/r0/register/available`.
+
+The API is:
+
+```
+POST /_synapse/admin/v1/username_availabile?username=$localpart
+```
+
+The request and response format is the same as the [/_matrix/client/r0/register/available](https://matrix.org/docs/spec/client_server/r0.6.0#get-matrix-client-r0-register-available) API.
+
+To use it, you will need to authenticate by providing an `access_token` for a
+server admin: [Admin API](../usage/administration/admin_api)
@@ -67,7 +67,7 @@ This gives a Python REPL in which `hs` gives access to the
 `synapse.server.HomeServer` object - which in turn gives access to many other
 parts of the process.

-Note that any call which returns a coroutine will need to be wrapped in `ensureDeferred`.
+Note that, prior to Synapse 1.41, any call which returns a coroutine will need to be wrapped in `ensureDeferred`.

 As a simple example, retrieving an event from the database:

@@ -282,6 +282,52 @@ the request is a server admin.
 Modules can modify the `request_content` (by e.g. adding events to its `initial_state`),
 or deny the room's creation by raising a `module_api.errors.SynapseError`.

+#### Presence router callbacks
+
+Presence router callbacks allow module developers to specify additional users (local or remote)
+to receive certain presence updates from local users. Presence router callbacks can be 
+registered using the module API's `register_presence_router_callbacks` method.
+
+The available presence router callbacks are:
+
+```python 
+async def get_users_for_states(
+    self,
+    state_updates: Iterable["synapse.api.UserPresenceState"],
+) -> Dict[str, Set["synapse.api.UserPresenceState"]]:
+```
+**Requires** `get_interested_users` to also be registered
+
+Called when processing updates to the presence state of one or more users. This callback can
+be used to instruct the server to forward that presence state to specific users. The module
+must return a dictionary that maps from Matrix user IDs (which can be local or remote) to the
+`UserPresenceState` changes that they should be forwarded.
+
+Synapse will then attempt to send the specified presence updates to each user when possible.
+
+```python
+async def get_interested_users(
+        self, 
+        user_id: str
+) -> Union[Set[str], "synapse.module_api.PRESENCE_ALL_USERS"]
+```
+**Requires** `get_users_for_states` to also be registered
+
+Called when determining which users someone should be able to see the presence state of. This
+callback should return complementary results to `get_users_for_state` or the presence information 
+may not be properly forwarded.
+
+The callback is given the Matrix user ID for a local user that is requesting presence data and
+should return the Matrix user IDs of the users whose presence state they are allowed to
+query. The returned users can be local or remote. 
+
+Alternatively the callback can return `synapse.module_api.PRESENCE_ALL_USERS`
+to indicate that the user should receive updates from all known users.
+
+For example, if the user `@alice:example.org` is passed to this method, and the Set 
+`{"@bob:example.com", "@charlie:somewhere.org"}` is returned, this signifies that Alice 
+should receive presence updates sent by Bob and Charlie, regardless of whether these users 
+share a room.

 ### Porting an existing module that uses the old interface

@@ -1,3 +1,9 @@
+<h2 style="color:red">
+This page of the Synapse documentation is now deprecated. For up to date
+documentation on setting up or writing a presence router module, please see
+<a href="modules.md">this page</a>.
+</h2>
+
 # Presence Router Module

 Synapse supports configuring a module that can specify additional users
@@ -33,6 +33,19 @@ Let's assume that we expect clients to connect to our server at
 `https://example.com:8448`.  The following sections detail the configuration of
 the reverse proxy and the homeserver.

+
+## Homeserver Configuration
+
+The HTTP configuration will need to be updated for Synapse to correctly record 
+client IP addresses and generate redirect URLs while behind a reverse proxy. 
+
+In `homeserver.yaml` set `x_forwarded: true` in the port 8008 section and 
+consider setting `bind_addresses: ['127.0.0.1']` so that the server only
+listens to traffic on localhost. (Do not change `bind_addresses` to `127.0.0.1` 
+when using a containerized Synapse, as that will prevent it from responding
+to proxied traffic.)
+
+
 ## Reverse-proxy configuration examples

 **NOTE**: You only need one of these.
@@ -239,16 +252,6 @@ relay "matrix_federation" {
 }
 ```

-## Homeserver Configuration
-
-You will also want to set `bind_addresses: ['127.0.0.1']` and
-`x_forwarded: true` for port 8008 in `homeserver.yaml` to ensure that
-client IP addresses are recorded correctly.
-
-Having done so, you can then use `https://matrix.example.com` (instead
-of `https://matrix.example.com:8448`) as the "Custom server" when
-connecting to Synapse from a client.
-

 ## Health check endpoint

@@ -108,20 +108,6 @@ presence:
  #
  #enabled: false

-  # Presence routers are third-party modules that can specify additional logic
-  # to where presence updates from users are routed.
-  #
-  presence_router:
-    # The custom module's class. Uncomment to use a custom presence router module.
-    #
-    #module: "my_custom_router.PresenceRouter"
-
-    # Configuration options of the custom module. Refer to your module's
-    # documentation for available options.
-    #
-    #config:
-    #  example_option: 'something'
-
 # Whether to require authentication to retrieve profile data (avatars,
 # display names) of other users through the client API. Defaults to
 # 'false'. Note that profile data is also available via the federation
@@ -210,6 +196,8 @@ presence:
 #
 # This option replaces federation_ip_range_blacklist in Synapse v1.25.0.
 #
+# Note: The value is ignored when an HTTP proxy is in use
+#
 #ip_range_blacklist:
 #  - '127.0.0.0/8'
 #  - '10.0.0.0/8'
@@ -711,6 +699,15 @@ caches:
  #
  #expiry_time: 30m

+  # Controls how long the results of a /sync request are cached for after
+  # a successful response is returned. A higher duration can help clients with
+  # intermittent connections, at the cost of higher memory usage.
+  #
+  # By default, this is zero, which means that sync responses are not cached
+  # at all.
+  #
+  #sync_response_cache_duration: 2m
+

 ## Database ##

@@ -963,6 +960,8 @@ media_store_path: "DATADIR/media_store"
 # This must be specified if url_preview_enabled is set. It is recommended that
 # you uncomment the following list as a starting point.
 #
+# Note: The value is ignored when an HTTP proxy is in use
+#
 #url_preview_ip_range_blacklist:
 #  - '127.0.0.0/8'
 #  - '10.0.0.0/8'
@@ -2229,6 +2228,14 @@ email:
  #
  #require_transport_security: true

+  # Uncomment the following to disable TLS for SMTP.
+  #
+  # By default, if the server supports TLS, it will be used, and the server
+  # must present a certificate that is valid for 'smtp_host'. If this option
+  # is set to false, TLS will not be used.
+  #
+  #enable_tls: false
+
  # notif_from defines the "From" address to use when sending emails.
  # It must be set if email sending is enabled.
  #
@@ -0,0 +1,74 @@
+# Using a forward proxy with Synapse
+
+You can use Synapse with a forward or outbound proxy. An example of when
+this is necessary is in corporate environments behind a DMZ (demilitarized zone).
+Synapse supports routing outbound HTTP(S) requests via a proxy. Only HTTP(S)
+proxy is supported, not SOCKS proxy or anything else.
+
+## Configure
+
+The `http_proxy`, `https_proxy`, `no_proxy` environment variables are used to
+specify proxy settings. The environment variable is not case sensitive.
+- `http_proxy`: Proxy server to use for HTTP requests.
+- `https_proxy`: Proxy server to use for HTTPS requests.
+- `no_proxy`: Comma-separated list of hosts, IP addresses, or IP ranges in CIDR
+  format which should not use the proxy. Synapse will directly connect to these hosts.
+
+The `http_proxy` and `https_proxy` environment variables have the form: `[scheme://][<username>:<password>@]<host>[:<port>]`
+- Supported schemes are `http://` and `https://`. The default scheme is `http://`
+  for compatibility reasons; it is recommended to set a scheme. If scheme is set
+  to `https://` the connection uses TLS between Synapse and the proxy.
+
+  **NOTE**: Synapse validates the certificates. If the certificate is not
+  valid, then the connection is dropped.
+- Default port if not given is `1080`.
+- Username and password are optional and will be used to authenticate against
+  the proxy.
+
+**Examples**
+- HTTP_PROXY=http://USERNAME:PASSWORD@10.0.1.1:8080/
+- HTTPS_PROXY=http://USERNAME:PASSWORD@proxy.example.com:8080/
+- NO_PROXY=master.hostname.example.com,10.1.0.0/16,172.30.0.0/16
+
+**NOTE**:
+Synapse does not apply the IP blacklist to connections through the proxy (since
+the DNS resolution is done by the proxy). It is expected that the proxy or firewall
+will apply blacklisting of IP addresses.
+
+## Connection types
+
+The proxy will be **used** for:
+
+- push
+- url previews
+- phone-home stats
+- recaptcha validation
+- CAS auth validation
+- OpenID Connect
+- Outbound federation
+- Federation (checking public key revocation)
+- Fetching public keys of other servers
+- Downloading remote media
+
+It will **not be used** for:
+
+- Application Services
+- Identity servers
+- In worker configurations
+  - connections between workers
+  - connections from workers to Redis
+
+## Troubleshooting
+
+If a proxy server is used with TLS (HTTPS) and no connections are established,
+it is most likely due to the proxy's certificates. To test this, the validation
+in Synapse can be deactivated.
+
+**NOTE**: This has an impact on security and is for testing purposes only!
+
+To deactivate the certificate validation, the following setting must be made in
+[homserver.yaml](../usage/configuration/homeserver_sample_config.md).
+
+```yaml
+use_insecure_ssl_client_just_for_testing_do_not_use: true
+```
@@ -85,6 +85,54 @@ process, for example:
    dpkg -i matrix-synapse-py3_1.3.0+stretch1_amd64.deb
    ```

+# Upgrading to v1.xx.0
+
+## Removal of old Room Admin API
+
+The following admin APIs were deprecated in [Synapse 1.25](https://github.com/matrix-org/synapse/blob/v1.25.0/CHANGES.md#removal-warning)
+(released on 2021-01-13) and have now been removed:
+
+-   `POST /_synapse/admin/v1/purge_room`
+-   `POST /_synapse/admin/v1/shutdown_room/<room_id>`
+
+Any scripts still using the above APIs should be converted to use the 
+[Delete Room API](https://matrix-org.github.io/synapse/latest/admin_api/rooms.html#delete-room-api).
+
+
+# Upgrading to v1.xx.0
+
+## Add support for routing outbound HTTP requests via a proxy for federation
+
+Since Synapse 1.6.0 (2019-11-26) you can set a proxy for outbound HTTP requests via
+http_proxy/https_proxy environment variables. This proxy was set for:
+- push
+- url previews
+- phone-home stats
+- recaptcha validation
+- CAS auth validation
+- OpenID Connect
+- Federation (checking public key revocation)
+
+In this version we have added support for outbound requests for:
+- Outbound federation
+- Downloading remote media
+- Fetching public keys of other servers
+
+These requests use the same proxy configuration. If you have a proxy configuration we
+recommend to verify the configuration. It may be necessary to adjust the `no_proxy`
+environment variable.
+
+See [using a forward proxy with Synapse documentation](setup/forward_proxy.md) for
+details.
+
+## User-interactive authentication fallback templates can now display errors
+
+This may affect you if you make use of custom HTML templates for the
+[reCAPTCHA](../synapse/res/templates/recaptcha.html) or
+[terms](../synapse/res/templates/terms.html) fallback pages.
+
+The template is now provided an `error` variable if the authentication
+process failed. See the default templates linked above for an example.

 # Upgrading to v1.39.0

@@ -86,6 +86,7 @@ files =
  tests/test_event_auth.py,
  tests/test_utils,
  tests/handlers/test_password_providers.py,
+  tests/handlers/test_room_summary.py,
  tests/rest/client/v1/test_login.py,
  tests/rest/client/v2_alpha/test_auth.py,
  tests/util/test_itertools.py,
@@ -25,7 +25,6 @@ DISTS = (
    "debian:sid",
    "ubuntu:bionic",  # 18.04 LTS (our EOL forced by Py36 on 2021-12-23)
    "ubuntu:focal",  # 20.04 LTS (our EOL forced by Py38 on 2024-10-14)
-    "ubuntu:groovy",  # 20.10 (EOL 2021-07-07)
    "ubuntu:hirsute",  # 21.04 (EOL 2022-01-05)
 )

@@ -11,7 +11,7 @@ set -e
 git remote set-branches --add origin develop
 git fetch -q origin develop

-pr="$BUILDKITE_PULL_REQUEST"
+pr="$PULL_REQUEST_NUMBER"

 # if there are changes in the debian directory, check that the debian changelog
 # has been updated
@@ -65,4 +65,4 @@ if [[ -n "$1" ]]; then
 fi

 # Run the tests!
-go test -v -tags synapse_blacklist,msc2946,msc3083,msc2403 -count=1 $EXTRA_COMPLEMENT_ARGS ./tests/...
+go test -v -tags synapse_blacklist,msc2946,msc3083,msc2403,msc2716 -count=1 $EXTRA_COMPLEMENT_ARGS ./tests/...
@@ -94,7 +94,7 @@ else
          "scripts-dev/build_debian_packages"
          "scripts-dev/sign_json"
          "scripts-dev/update_database"
-          "contrib" "synctl" "setup.py" "synmark" "stubs" ".buildkite"
+          "contrib" "synctl" "setup.py" "synmark" "stubs" ".ci"
      )
  fi
 fi
@@ -76,6 +76,8 @@ class RoomVersion:
    # MSC2716: Adds m.room.power_levels -> content.historical field to control
    # whether "insertion", "chunk", "marker" events can be sent
    msc2716_historical = attr.ib(type=bool)
+    # MSC2716: Adds support for redacting "insertion", "chunk", and "marker" events
+    msc2716_redactions = attr.ib(type=bool)


 class RoomVersions:
@@ -92,6 +94,7 @@ class RoomVersions:
        msc3083_join_rules=False,
        msc2403_knocking=False,
        msc2716_historical=False,
+        msc2716_redactions=False,
    )
    V2 = RoomVersion(
        "2",
@@ -106,6 +109,7 @@ class RoomVersions:
        msc3083_join_rules=False,
        msc2403_knocking=False,
        msc2716_historical=False,
+        msc2716_redactions=False,
    )
    V3 = RoomVersion(
        "3",
@@ -120,6 +124,7 @@ class RoomVersions:
        msc3083_join_rules=False,
        msc2403_knocking=False,
        msc2716_historical=False,
+        msc2716_redactions=False,
    )
    V4 = RoomVersion(
        "4",
@@ -134,6 +139,7 @@ class RoomVersions:
        msc3083_join_rules=False,
        msc2403_knocking=False,
        msc2716_historical=False,
+        msc2716_redactions=False,
    )
    V5 = RoomVersion(
        "5",
@@ -148,6 +154,7 @@ class RoomVersions:
        msc3083_join_rules=False,
        msc2403_knocking=False,
        msc2716_historical=False,
+        msc2716_redactions=False,
    )
    V6 = RoomVersion(
        "6",
@@ -162,6 +169,7 @@ class RoomVersions:
        msc3083_join_rules=False,
        msc2403_knocking=False,
        msc2716_historical=False,
+        msc2716_redactions=False,
    )
    MSC2176 = RoomVersion(
        "org.matrix.msc2176",
@@ -176,6 +184,7 @@ class RoomVersions:
        msc3083_join_rules=False,
        msc2403_knocking=False,
        msc2716_historical=False,
+        msc2716_redactions=False,
    )
    V7 = RoomVersion(
        "7",
@@ -190,20 +199,7 @@ class RoomVersions:
        msc3083_join_rules=False,
        msc2403_knocking=True,
        msc2716_historical=False,
-    )
-    MSC2716 = RoomVersion(
-        "org.matrix.msc2716",
-        RoomDisposition.UNSTABLE,
-        EventFormatVersions.V3,
-        StateResolutionVersions.V2,
-        enforce_key_validity=True,
-        special_case_aliases_auth=False,
-        strict_canonicaljson=True,
-        limit_notifications_power_levels=True,
-        msc2176_redaction_rules=False,
-        msc3083_join_rules=False,
-        msc2403_knocking=True,
-        msc2716_historical=True,
+        msc2716_redactions=False,
    )
    V8 = RoomVersion(
        "8",
@@ -218,6 +214,37 @@ class RoomVersions:
        msc3083_join_rules=True,
        msc2403_knocking=True,
        msc2716_historical=False,
+        msc2716_redactions=False,
+    )
+    MSC2716 = RoomVersion(
+        "org.matrix.msc2716",
+        RoomDisposition.UNSTABLE,
+        EventFormatVersions.V3,
+        StateResolutionVersions.V2,
+        enforce_key_validity=True,
+        special_case_aliases_auth=False,
+        strict_canonicaljson=True,
+        limit_notifications_power_levels=True,
+        msc2176_redaction_rules=False,
+        msc3083_join_rules=False,
+        msc2403_knocking=True,
+        msc2716_historical=True,
+        msc2716_redactions=False,
+    )
+    MSC2716v2 = RoomVersion(
+        "org.matrix.msc2716v2",
+        RoomDisposition.UNSTABLE,
+        EventFormatVersions.V3,
+        StateResolutionVersions.V2,
+        enforce_key_validity=True,
+        special_case_aliases_auth=False,
+        strict_canonicaljson=True,
+        limit_notifications_power_levels=True,
+        msc2176_redaction_rules=False,
+        msc3083_join_rules=False,
+        msc2403_knocking=True,
+        msc2716_historical=True,
+        msc2716_redactions=True,
    )


@@ -37,6 +37,7 @@ from synapse.app import check_bind_error
 from synapse.app.phone_stats_home import start_phone_stats_home
 from synapse.config.homeserver import HomeServerConfig
 from synapse.crypto import context_factory
+from synapse.events.presence_router import load_legacy_presence_router
 from synapse.events.spamcheck import load_legacy_spam_checkers
 from synapse.events.third_party_rules import load_legacy_third_party_event_rules
 from synapse.logging.context import PreserveLoggingContext
@@ -370,6 +371,7 @@ async def start(hs: "HomeServer"):

    load_legacy_spam_checkers(hs)
    load_legacy_third_party_event_rules(hs)
+    load_legacy_presence_router(hs)

    # If we've configured an expiry time for caches, start the background job now.
    setup_expire_lru_cache_entries(hs)
@@ -66,40 +66,40 @@ from synapse.replication.slave.storage.receipts import SlavedReceiptsStore
 from synapse.replication.slave.storage.registration import SlavedRegistrationStore
 from synapse.replication.slave.storage.room import RoomStore
 from synapse.rest.admin import register_servlets_for_media_repo
-from synapse.rest.client.v1 import events, login, presence, room
-from synapse.rest.client.v1.initial_sync import InitialSyncRestServlet
-from synapse.rest.client.v1.profile import (
-    ProfileAvatarURLRestServlet,
-    ProfileDisplaynameRestServlet,
-    ProfileRestServlet,
-)
-from synapse.rest.client.v1.push_rule import PushRuleRestServlet
-from synapse.rest.client.v1.voip import VoipRestServlet
-from synapse.rest.client.v2_alpha import (
+from synapse.rest.client import (
    account_data,
+    events,
    groups,
+    login,
+    presence,
    read_marker,
    receipts,
+    room,
    room_keys,
    sync,
    tags,
    user_directory,
 )
-from synapse.rest.client.v2_alpha._base import client_patterns
-from synapse.rest.client.v2_alpha.account import ThreepidRestServlet
-from synapse.rest.client.v2_alpha.account_data import (
-    AccountDataServlet,
-    RoomAccountDataServlet,
-)
-from synapse.rest.client.v2_alpha.devices import DevicesRestServlet
-from synapse.rest.client.v2_alpha.keys import (
+from synapse.rest.client._base import client_patterns
+from synapse.rest.client.account import ThreepidRestServlet
+from synapse.rest.client.account_data import AccountDataServlet, RoomAccountDataServlet
+from synapse.rest.client.devices import DevicesRestServlet
+from synapse.rest.client.initial_sync import InitialSyncRestServlet
+from synapse.rest.client.keys import (
    KeyChangesServlet,
    KeyQueryServlet,
    OneTimeKeyServlet,
 )
-from synapse.rest.client.v2_alpha.register import RegisterRestServlet
-from synapse.rest.client.v2_alpha.sendtodevice import SendToDeviceRestServlet
+from synapse.rest.client.profile import (
+    ProfileAvatarURLRestServlet,
+    ProfileDisplaynameRestServlet,
+    ProfileRestServlet,
+)
+from synapse.rest.client.push_rule import PushRuleRestServlet
+from synapse.rest.client.register import RegisterRestServlet
+from synapse.rest.client.sendtodevice import SendToDeviceRestServlet
 from synapse.rest.client.versions import VersionsRestServlet
+from synapse.rest.client.voip import VoipRestServlet
 from synapse.rest.health import HealthResource
 from synapse.rest.key.v2 import KeyApiV2Resource
 from synapse.rest.synapse.client import build_synapse_client_resource_tree
@@ -237,13 +237,14 @@ class Config:
    def read_templates(
        self,
        filenames: List[str],
-        custom_template_directory: Optional[str] = None,
+        custom_template_directories: Optional[Iterable[str]] = None,
    ) -> List[jinja2.Template]:
        """Load a list of template files from disk using the given variables.

        This function will attempt to load the given templates from the default Synapse
-        template directory. If `custom_template_directory` is supplied, that directory
-        is tried first.
+        template directory. If `custom_template_directories` is supplied, any directory
+        in this list is tried (in the order they appear in the list) before trying
+        Synapse's default directory.

        Files read are treated as Jinja templates. The templates are not rendered yet
        and have autoescape enabled.
@@ -251,8 +252,8 @@ class Config:
        Args:
            filenames: A list of template filenames to read.

-            custom_template_directory: A directory to try to look for the templates
-                before using the default Synapse template directory instead.
+            custom_template_directories: A list of directory to try to look for the
+                templates before using the default Synapse template directory instead.

        Raises:
            ConfigError: if the file's path is incorrect or otherwise cannot be read.
@@ -260,20 +261,26 @@ class Config:
        Returns:
            A list of jinja2 templates.
        """
-        search_directories = [self.default_template_dir]
+        search_directories = []

-        # The loader will first look in the custom template directory (if specified) for the
-        # given filename. If it doesn't find it, it will use the default template dir instead
-        if custom_template_directory:
-            # Check that the given template directory exists
-            if not self.path_exists(custom_template_directory):
-                raise ConfigError(
-                    "Configured template directory does not exist: %s"
-                    % (custom_template_directory,)
-                )
+        # The loader will first look in the custom template directories (if specified)
+        # for the given filename. If it doesn't find it, it will use the default
+        # template dir instead.
+        if custom_template_directories is not None:
+            for custom_template_directory in custom_template_directories:
+                # Check that the given template directory exists
+                if not self.path_exists(custom_template_directory):
+                    raise ConfigError(
+                        "Configured template directory does not exist: %s"
+                        % (custom_template_directory,)
+                    )

-            # Search the custom template directory as well
-            search_directories.insert(0, custom_template_directory)
+                # Search the custom template directory as well
+                search_directories.append(custom_template_directory)
+
+        # Append the default directory at the end of the list so Jinja can fallback on it
+        # if a template is missing from any custom directory.
+        search_directories.append(self.default_template_dir)

        # TODO: switch to synapse.util.templates.build_jinja_env
        loader = jinja2.FileSystemLoader(search_directories)
@@ -88,5 +88,5 @@ class AccountValidityConfig(Config):
                "account_previously_renewed.html",
                invalid_token_template_filename,
            ],
-            account_validity_template_dir,
+            (td for td in (account_validity_template_dir,) if td),
        )
@@ -151,6 +151,15 @@ class CacheConfig(Config):
          # entries are never evicted based on time.
          #
          #expiry_time: 30m
+
+          # Controls how long the results of a /sync request are cached for after
+          # a successful response is returned. A higher duration can help clients with
+          # intermittent connections, at the cost of higher memory usage.
+          #
+          # By default, this is zero, which means that sync responses are not cached
+          # at all.
+          #
+          #sync_response_cache_duration: 2m
        """

    def read_config(self, config, **kwargs):
@@ -212,6 +221,10 @@ class CacheConfig(Config):
        else:
            self.expiry_time_msec = None

+        self.sync_response_cache_duration = self.parse_duration(
+            cache_config.get("sync_response_cache_duration", 0)
+        )
+
        # Resize all caches (if necessary) with the new factors we've loaded
        self.resize_all_caches()

@@ -80,6 +80,12 @@ class EmailConfig(Config):
        self.require_transport_security = email_config.get(
            "require_transport_security", False
        )
+        self.enable_smtp_tls = email_config.get("enable_tls", True)
+        if self.require_transport_security and not self.enable_smtp_tls:
+            raise ConfigError(
+                "email.require_transport_security requires email.enable_tls to be true"
+            )
+
        if "app_name" in email_config:
            self.email_app_name = email_config["app_name"]
        else:
@@ -251,7 +257,9 @@ class EmailConfig(Config):
                    registration_template_success_html,
                    add_threepid_template_success_html,
                ],
-                template_dir,
+                (
+                    td for td in (template_dir,) if td
+                ),  # Filter out template_dir if not provided
            )

            # Render templates that do not contain any placeholders
@@ -291,7 +299,7 @@ class EmailConfig(Config):
                self.email_notif_template_text,
            ) = self.read_templates(
                [notif_template_html, notif_template_text],
-                template_dir,
+                (td for td in (template_dir,) if td),
            )

            self.email_notif_for_new_users = email_config.get(
@@ -314,7 +322,7 @@ class EmailConfig(Config):
                self.account_validity_template_text,
            ) = self.read_templates(
                [expiry_template_html, expiry_template_text],
-                template_dir,
+                (td for td in (template_dir,) if td),
            )

        subjects_config = email_config.get("subjects", {})
@@ -368,6 +376,14 @@ class EmailConfig(Config):
          #
          #require_transport_security: true

+          # Uncomment the following to disable TLS for SMTP.
+          #
+          # By default, if the server supports TLS, it will be used, and the server
+          # must present a certificate that is valid for 'smtp_host'. If this option
+          # is set to false, TLS will not be used.
+          #
+          #enable_tls: false
+
          # notif_from defines the "From" address to use when sending emails.
          # It must be set if email sending is enabled.
          #
@@ -38,3 +38,6 @@ class ExperimentalConfig(Config):

        # MSC3244 (room version capabilities)
        self.msc3244_enabled: bool = experimental.get("msc3244_enabled", False)
+
+        # MSC3266 (room summary api)
+        self.msc3266_enabled: bool = experimental.get("msc3266_enabled", False)
--- a/Show More
+++ b/Show More
				`@@ -0,0 +1 @@`
				`Improve event caching mechanism to avoid having multiple copies of an event in memory at a time.`
				`@@ -0,0 +1 @@`
				`Add some clarification to the sample config file. Contributed by @Kentokamoto.`
				`@@ -0,0 +1 @@`
				Initial local support for [MSC3266](https://github.com/matrix-org/synapse/pull/10394), Room Summary over the unstable `/rooms/{roomIdOrAlias}/summary` API.
				`@@ -0,0 +1 @@`
				Experimental support for [MSC3288](https://github.com/matrix-org/matrix-doc/pull/3288), sending `room_type` to the identity server for 3pid invites over the `/store-invite` API.
				`@@ -0,0 +1 @@`
				`Add documentation for configuration a forward proxy.`
				`@@ -0,0 +1 @@`
				`Add support for sending federation requests through a proxy. Contributed by @Bubu and @dklimpel.`
				`@@ -0,0 +1 @@`
				`Add support for "marker" events which makes historical events discoverable for servers that already have all of the scrollback history (part of MSC2716).`
				`@@ -0,0 +1 @@`
				`Reduce errors in PostgreSQL logs due to concurrent serialization errors.`
				`@@ -0,0 +1 @@`
				`Include room ID in ignored EDU log messages. Contributed by @ilmari.`
				`@@ -0,0 +1 @@`
				Add a configuration setting for the time a `/sync` response is cached for.