Add further logging for #9533

If in doubt, print it out. Thanks Erik for advising.

.buildkite/.env

@@ -1,13 +0,0 @@
-CI
-BUILDKITE
-BUILDKITE_BUILD_NUMBER
-BUILDKITE_BRANCH
-BUILDKITE_BUILD_NUMBER
-BUILDKITE_JOB_ID
-BUILDKITE_BUILD_URL
-BUILDKITE_PROJECT_SLUG
-BUILDKITE_COMMIT
-BUILDKITE_PULL_REQUEST
-BUILDKITE_TAG
-CODECOV_TOKEN
-TRIAL_FLAGS

.buildkite/merge_base_branch.sh

@@ -1,35 +0,0 @@
-#!/usr/bin/env bash
-
-set -e
-
-if [[ "$BUILDKITE_BRANCH" =~ ^(develop|master|dinsic|shhs|release-.*)$ ]]; then
-    echo "Not merging forward, as this is a release branch"
-    exit 0
-fi
-
-if [[ -z $BUILDKITE_PULL_REQUEST_BASE_BRANCH ]]; then
-    echo "Not a pull request, or hasn't had a PR opened yet..."
-
-    # It probably hasn't had a PR opened yet. Since all PRs land on develop, we
-    # can probably assume it's based on it and will be merged into it.
-    GITBASE="develop"
-else
-    # Get the reference, using the GitHub API
-    GITBASE=$BUILDKITE_PULL_REQUEST_BASE_BRANCH
-fi
-
-echo "--- merge_base_branch $GITBASE"
-
-# Show what we are before
-git --no-pager show -s
-
-# Set up username so it can do a merge
-git config --global user.email bot@matrix.org
-git config --global user.name "A robot"
-
-# Fetch and merge. If it doesn't work, it will raise due to set -e.
-git fetch -u origin $GITBASE
-git merge --no-edit --no-commit origin/$GITBASE
-
-# Show what we are after.
-git --no-pager show -s

.ci/postgres-config.yaml

@@ -3,7 +3,7 @@
 # CI's Docker setup at the point where this file is considered.
 server_name: "localhost:8800"
 
-signing_key_path: ".buildkite/test.signing.key"
+signing_key_path: ".ci/test.signing.key"
 
 report_stats: false
 
@@ -11,7 +11,7 @@ database:
   name: "psycopg2"
   args:
     user: postgres
-    host: postgres
+    host: localhost
     password: postgres
     database: synapse
 

.ci/scripts/postgres_exec.py

@@ -23,7 +23,7 @@ import psycopg2
 # We use "postgres" as a database because it's bound to exist and the "synapse" one
 # doesn't exist yet.
 db_conn = psycopg2.connect(
-    user="postgres", host="postgres", password="postgres", dbname="postgres"
+    user="postgres", host="localhost", password="postgres", dbname="postgres"
 )
 db_conn.autocommit = True
 cur = db_conn.cursor()

.ci/scripts/test_old_deps.sh

@@ -1,6 +1,6 @@
 #!/usr/bin/env bash
 
-# this script is run by buildkite in a plain `bionic` container; it installs the
+# this script is run by GitHub Actions in a plain `bionic` container; it installs the
 # minimal requirements for tox and hands over to the py3-old tox environment.
 
 set -ex

.ci/scripts/test_synapse_port_db.sh

@@ -20,22 +20,22 @@ pip install -e .
 echo "--- Generate the signing key"
 
 # Generate the server's signing key.
-python -m synapse.app.homeserver --generate-keys -c .buildkite/sqlite-config.yaml
+python -m synapse.app.homeserver --generate-keys -c .ci/sqlite-config.yaml
 
 echo "--- Prepare test database"
 
 # Make sure the SQLite3 database is using the latest schema and has no pending background update.
-scripts-dev/update_database --database-config .buildkite/sqlite-config.yaml
+scripts-dev/update_database --database-config .ci/sqlite-config.yaml
 
 # Create the PostgreSQL database.
-./.buildkite/scripts/postgres_exec.py "CREATE DATABASE synapse"
+.ci/scripts/postgres_exec.py "CREATE DATABASE synapse"
 
 echo "+++ Run synapse_port_db against test database"
-coverage run scripts/synapse_port_db --sqlite-database .buildkite/test_db.db --postgres-config .buildkite/postgres-config.yaml
+coverage run scripts/synapse_port_db --sqlite-database .ci/test_db.db --postgres-config .ci/postgres-config.yaml
 
 # We should be able to run twice against the same database.
 echo "+++ Run synapse_port_db a second time"
-coverage run scripts/synapse_port_db --sqlite-database .buildkite/test_db.db --postgres-config .buildkite/postgres-config.yaml
+coverage run scripts/synapse_port_db --sqlite-database .ci/test_db.db --postgres-config .ci/postgres-config.yaml
 
 #####
 
@@ -44,14 +44,14 @@ coverage run scripts/synapse_port_db --sqlite-database .buildkite/test_db.db --p
 echo "--- Prepare empty SQLite database"
 
 # we do this by deleting the sqlite db, and then doing the same again.
-rm .buildkite/test_db.db
+rm .ci/test_db.db
 
-scripts-dev/update_database --database-config .buildkite/sqlite-config.yaml
+scripts-dev/update_database --database-config .ci/sqlite-config.yaml
 
 # re-create the PostgreSQL database.
-./.buildkite/scripts/postgres_exec.py \
+.ci/scripts/postgres_exec.py \
   "DROP DATABASE synapse" \
   "CREATE DATABASE synapse"
 
 echo "+++ Run synapse_port_db against empty database"
-coverage run scripts/synapse_port_db --sqlite-database .buildkite/test_db.db --postgres-config .buildkite/postgres-config.yaml
+coverage run scripts/synapse_port_db --sqlite-database .ci/test_db.db --postgres-config .ci/postgres-config.yaml

.ci/sqlite-config.yaml

@@ -3,14 +3,14 @@
 # schema and run background updates on it.
 server_name: "localhost:8800"
 
-signing_key_path: ".buildkite/test.signing.key"
+signing_key_path: ".ci/test.signing.key"
 
 report_stats: false
 
 database:
   name: "sqlite3"
   args:
-    database: ".buildkite/test_db.db"
+    database: ".ci/test_db.db"
 
 # Suppress the key server warning.
 trusted_key_servers: []

.github/workflows/tests.yml

@@ -8,7 +8,7 @@ on:
 concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}
   cancel-in-progress: true
-  
+
 jobs:
   lint:
     runs-on: ubuntu-latest
@@ -38,20 +38,15 @@ jobs:
     if: ${{ github.base_ref == 'develop'  || contains(github.base_ref, 'release-') }}
     runs-on: ubuntu-latest
     steps:
-      # Note: This and the script can be simplified once we drop Buildkite. See:
-      #   https://github.com/actions/checkout/issues/266#issuecomment-638346893
-      #   https://github.com/actions/checkout/issues/416
       - uses: actions/checkout@v2
         with:
           ref: ${{ github.event.pull_request.head.sha }}
           fetch-depth: 0
       - uses: actions/setup-python@v2
       - run: pip install tox
-      - name: Patch Buildkite-specific test script
-        run: |
-          sed -i -e 's/\$BUILDKITE_PULL_REQUEST/${{ github.event.number }}/' \
-            scripts-dev/check-newsfragment
       - run: scripts-dev/check-newsfragment
+        env:
+          PULL_REQUEST_NUMBER: ${{ github.event.number }}
 
   lint-sdist:
     runs-on: ubuntu-latest
@@ -144,7 +139,7 @@ jobs:
         uses: docker://ubuntu:bionic # For old python and sqlite
         with:
           workdir: /github/workspace
-          entrypoint: .buildkite/scripts/test_old_deps.sh
+          entrypoint: .ci/scripts/test_old_deps.sh
         env:
           TRIAL_FLAGS: "--jobs=2"
       - name: Dump logs
@@ -197,12 +192,12 @@ jobs:
       volumes:
         - ${{ github.workspace }}:/src
       env:
-        BUILDKITE_BRANCH: ${{ github.head_ref }}
         POSTGRES: ${{ matrix.postgres && 1}}
         MULTI_POSTGRES: ${{ (matrix.postgres == 'multi-postgres') && 1}}
         WORKERS: ${{ matrix.workers && 1 }}
         REDIS: ${{ matrix.redis && 1 }}
         BLACKLIST: ${{ matrix.workers && 'synapse-blacklist-with-workers' }}
+        TOP: ${{ github.workspace }}
 
     strategy:
       fail-fast: false
@@ -232,7 +227,7 @@ jobs:
     steps:
       - uses: actions/checkout@v2
       - name: Prepare test blacklist
-        run: cat sytest-blacklist .buildkite/worker-blacklist > synapse-blacklist-with-workers
+        run: cat sytest-blacklist .ci/worker-blacklist > synapse-blacklist-with-workers
       - name: Run SyTest
         run: /bootstrap.sh synapse
         working-directory: /src
@@ -252,6 +247,8 @@ jobs:
     if: ${{ !failure() && !cancelled() }} # Allow previous steps to be skipped, but not fail
     needs: linting-done
     runs-on: ubuntu-latest
+    env:
+      TOP: ${{ github.workspace }}
     strategy:
       matrix:
         include:
@@ -281,13 +278,7 @@ jobs:
       - uses: actions/setup-python@v2
         with:
           python-version: ${{ matrix.python-version }}
-      - name: Patch Buildkite-specific test scripts
-        run: |
-          sed -i -e 's/host="postgres"/host="localhost"/' .buildkite/scripts/postgres_exec.py
-          sed -i -e 's/host: postgres/host: localhost/' .buildkite/postgres-config.yaml
-          sed -i -e 's|/src/||' .buildkite/{sqlite,postgres}-config.yaml
-          sed -i -e 's/\$TOP/\$GITHUB_WORKSPACE/' .coveragerc
-      - run: .buildkite/scripts/test_synapse_port_db.sh
+      - run: .ci/scripts/test_synapse_port_db.sh
 
   complement:
     if: ${{ !failure() && !cancelled() }}
@@ -367,13 +358,21 @@ jobs:
       - name: Set build result
         env:
           NEEDS_CONTEXT: ${{ toJSON(needs) }}
-        # the `jq` incantation dumps out a series of "<job> <result>" lines
+        # the `jq` incantation dumps out a series of "<job> <result>" lines.
+        # we set it to an intermediate variable to avoid a pipe, which makes it
+        # hard to set $rc.
         run: |
-          set -o pipefail
-          jq -r 'to_entries[] | [.key,.value.result] | join(" ")' \
-                          <<< $NEEDS_CONTEXT |
-              while read job result; do
-                  if [ "$result" != "success" ]; then
-                      echo "::set-failed ::Job $job returned $result"
-                  fi
-              done
+          rc=0
+          results=$(jq -r 'to_entries[] | [.key,.value.result] | join(" ")' <<< $NEEDS_CONTEXT)
+          while read job result ; do
+              # The newsfile lint may be skipped on non PR builds
+              if [ $result == "skipped" ] && [ $job == "lint-newsfile" ]; then
+                continue
+              fi
+
+              if [ "$result" != "success" ]; then
+                  echo "::set-failed ::Job $job returned $result"
+                  rc=1
+              fi
+          done <<< $results
+          exit $rc

CHANGES.md

@@ -1,10 +1,132 @@
-Synapse 1.39.0rc2 (2021-07-22)
+Synapse 1.40.0 (2021-08-10)
+===========================
+
+No significant changes.
+
+
+Synapse 1.40.0rc3 (2021-08-09)
+==============================
+
+Features
+--------
+
+- Support [MSC3289: room version 8](https://github.com/matrix-org/matrix-doc/pull/3289). ([\#10449](https://github.com/matrix-org/synapse/issues/10449))
+
+
+Bugfixes
+--------
+
+- Mark the experimental room version from [MSC2716](https://github.com/matrix-org/matrix-doc/pull/2716) as unstable. ([\#10449](https://github.com/matrix-org/synapse/issues/10449))
+
+
+Improved Documentation
+----------------------
+
+- Fix broken links in `upgrade.md`. Contributed by @dklimpel. ([\#10543](https://github.com/matrix-org/synapse/issues/10543))
+
+
+Synapse 1.40.0rc2 (2021-08-04)
 ==============================
 
 Bugfixes
 --------
 
-- Always include `device_one_time_keys_count` key in `/sync` response to work around a bug in Element Android that broke encryption for new devices. ([\#10457](https://github.com/matrix-org/synapse/issues/10457))
+- Fix the `PeriodicallyFlushingMemoryHandler` inhibiting application shutdown because of its background thread. ([\#10517](https://github.com/matrix-org/synapse/issues/10517))
+- Fix a bug introduced in Synapse v1.40.0rc1 that could cause Synapse to respond with an error when clients would update read receipts. ([\#10531](https://github.com/matrix-org/synapse/issues/10531))
+
+
+Internal Changes
+----------------
+
+- Fix release script to open the correct URL for the release. ([\#10516](https://github.com/matrix-org/synapse/issues/10516))
+
+
+Synapse 1.40.0rc1 (2021-08-03)
+==============================
+
+Features
+--------
+
+- Add support for [MSC2033](https://github.com/matrix-org/matrix-doc/pull/2033): `device_id` on `/account/whoami`. ([\#9918](https://github.com/matrix-org/synapse/issues/9918))
+- Update support for [MSC2716 - Incrementally importing history into existing rooms](https://github.com/matrix-org/matrix-doc/pull/2716). ([\#10245](https://github.com/matrix-org/synapse/issues/10245), [\#10432](https://github.com/matrix-org/synapse/issues/10432), [\#10463](https://github.com/matrix-org/synapse/issues/10463))
+- Update support for [MSC3083](https://github.com/matrix-org/matrix-doc/pull/3083) to consider changes in the MSC around which servers can issue join events. ([\#10254](https://github.com/matrix-org/synapse/issues/10254), [\#10447](https://github.com/matrix-org/synapse/issues/10447), [\#10489](https://github.com/matrix-org/synapse/issues/10489))
+- Initial support for [MSC3244](https://github.com/matrix-org/matrix-doc/pull/3244), Room version capabilities over the /capabilities API. ([\#10283](https://github.com/matrix-org/synapse/issues/10283))
+- Add a buffered logging handler which periodically flushes itself. ([\#10407](https://github.com/matrix-org/synapse/issues/10407), [\#10515](https://github.com/matrix-org/synapse/issues/10515))
+- Add support for https connections to a proxy server. Contributed by @Bubu and @dklimpel. ([\#10411](https://github.com/matrix-org/synapse/issues/10411))
+- Support for [MSC2285 (hidden read receipts)](https://github.com/matrix-org/matrix-doc/pull/2285). Contributed by @SimonBrandner. ([\#10413](https://github.com/matrix-org/synapse/issues/10413))
+- Email notifications now state whether an invitation is to a room or a space. ([\#10426](https://github.com/matrix-org/synapse/issues/10426))
+- Allow setting transaction limit for database connections. ([\#10440](https://github.com/matrix-org/synapse/issues/10440), [\#10511](https://github.com/matrix-org/synapse/issues/10511))
+- Add `creation_ts` to "list users" admin API. ([\#10448](https://github.com/matrix-org/synapse/issues/10448))
+
+
+Bugfixes
+--------
+
+- Improve character set detection in URL previews by supporting underscores (in addition to hyphens). Contributed by @srividyut. ([\#10410](https://github.com/matrix-org/synapse/issues/10410))
+- Fix events being incorrectly rejected over federation if they reference auth events that the server needed to fetch. ([\#10439](https://github.com/matrix-org/synapse/issues/10439))
+- Fix `synapse_federation_server_oldest_inbound_pdu_in_staging` Prometheus metric to not report a max age of 51 years when the queue is empty. ([\#10455](https://github.com/matrix-org/synapse/issues/10455))
+- Fix a bug which caused an explicit assignment of power-level 0 to a user to be misinterpreted in rare circumstances. ([\#10499](https://github.com/matrix-org/synapse/issues/10499))
+
+
+Improved Documentation
+----------------------
+
+- Fix hierarchy of providers on the OpenID page. ([\#10445](https://github.com/matrix-org/synapse/issues/10445))
+- Consolidate development documentation to `docs/development/`. ([\#10453](https://github.com/matrix-org/synapse/issues/10453))
+- Add some developer docs to explain room DAG concepts like `outliers`, `state_groups`, `depth`, etc. ([\#10464](https://github.com/matrix-org/synapse/issues/10464))
+- Document how to use Complement while developing a new Synapse feature. ([\#10483](https://github.com/matrix-org/synapse/issues/10483))
+
+
+Internal Changes
+----------------
+
+- Prune inbound federation queues for a room if they get too large. ([\#10390](https://github.com/matrix-org/synapse/issues/10390))
+- Add type hints to `synapse.federation.transport.client` module. ([\#10408](https://github.com/matrix-org/synapse/issues/10408))
+- Remove shebang line from module files. ([\#10415](https://github.com/matrix-org/synapse/issues/10415))
+- Drop backwards-compatibility code that was required to support Ubuntu Xenial. ([\#10429](https://github.com/matrix-org/synapse/issues/10429))
+- Use a docker image cache for the prerequisites for the debian package build. ([\#10431](https://github.com/matrix-org/synapse/issues/10431))
+- Improve servlet type hints. ([\#10437](https://github.com/matrix-org/synapse/issues/10437), [\#10438](https://github.com/matrix-org/synapse/issues/10438))
+- Replace usage of `or_ignore` in `simple_insert` with `simple_upsert` usage, to stop spamming postgres logs with spurious ERROR messages. ([\#10442](https://github.com/matrix-org/synapse/issues/10442))
+- Update the `tests-done` Github Actions status. ([\#10444](https://github.com/matrix-org/synapse/issues/10444), [\#10512](https://github.com/matrix-org/synapse/issues/10512))
+- Update type annotations to work with forthcoming Twisted 21.7.0 release. ([\#10446](https://github.com/matrix-org/synapse/issues/10446), [\#10450](https://github.com/matrix-org/synapse/issues/10450))
+- Cancel redundant GHA workflows when a new commit is pushed. ([\#10451](https://github.com/matrix-org/synapse/issues/10451))
+- Mitigate media repo XSS attacks on IE11 via the non-standard X-Content-Security-Policy header. ([\#10468](https://github.com/matrix-org/synapse/issues/10468))
+- Additional type hints in the state handler. ([\#10482](https://github.com/matrix-org/synapse/issues/10482))
+- Update syntax used to run complement tests. ([\#10488](https://github.com/matrix-org/synapse/issues/10488))
+- Fix up type annotations to work with Twisted 21.7. ([\#10490](https://github.com/matrix-org/synapse/issues/10490))
+- Improve type annotations for `ObservableDeferred`. ([\#10491](https://github.com/matrix-org/synapse/issues/10491))
+- Extend release script to also tag and create GitHub releases. ([\#10496](https://github.com/matrix-org/synapse/issues/10496))
+- Fix a bug which caused production debian packages to be incorrectly marked as 'prerelease'. ([\#10500](https://github.com/matrix-org/synapse/issues/10500))
+
+
+Synapse 1.39.0 (2021-07-29)
+===========================
+
+No significant changes.
+
+
+Synapse 1.39.0rc3 (2021-07-28)
+==============================
+
+Bugfixes
+--------
+
+- Fix a bug introduced in Synapse 1.38 which caused an exception at startup when SAML authentication was enabled. ([\#10477](https://github.com/matrix-org/synapse/issues/10477))
+- Fix a long-standing bug where Synapse would not inform clients that a device had exhausted its one-time-key pool, potentially causing problems decrypting events. ([\#10485](https://github.com/matrix-org/synapse/issues/10485))
+- Fix reporting old R30 stats as R30v2 stats. Introduced in v1.39.0rc1. ([\#10486](https://github.com/matrix-org/synapse/issues/10486))
+
+
+Internal Changes
+----------------
+
+- Fix an error which prevented the Github Actions workflow to build the docker images from running. ([\#10461](https://github.com/matrix-org/synapse/issues/10461))
+- Fix release script to correctly version debian changelog when doing RCs. ([\#10465](https://github.com/matrix-org/synapse/issues/10465))
+
+
+Synapse 1.39.0rc2 (2021-07-22)
+==============================
+
+This release also includes the changes in v1.38.1.
 
 
 Internal Changes

CONTRIBUTING.md

@@ -13,8 +13,9 @@ This document aims to get you started with contributing to this repo!
 - [7. Turn coffee and documentation into code and documentation!](#7-turn-coffee-and-documentation-into-code-and-documentation)
 - [8. Test, test, test!](#8-test-test-test)
   * [Run the linters.](#run-the-linters)
-  * [Run the unit tests.](#run-the-unit-tests)
-  * [Run the integration tests.](#run-the-integration-tests)
+  * [Run the unit tests.](#run-the-unit-tests-twisted-trial)
+  * [Run the integration tests (SyTest).](#run-the-integration-tests-sytest)
+  * [Run the integration tests (Complement).](#run-the-integration-tests-complement)
 - [9. Submit your patch.](#9-submit-your-patch)
   * [Changelog](#changelog)
     + [How do I know what to call the changelog file before I create the PR?](#how-do-i-know-what-to-call-the-changelog-file-before-i-create-the-pr)
@@ -155,7 +156,7 @@ source ./env/bin/activate
 ./scripts-dev/lint.sh path/to/file1.py path/to/file2.py path/to/folder
 ```
 
-## Run the unit tests.
+## Run the unit tests (Twisted trial).
 
 The unit tests run parts of Synapse, including your changes, to see if anything
 was broken. They are slower than the linters but will typically catch more errors.
@@ -186,7 +187,7 @@ SYNAPSE_TEST_LOG_LEVEL=DEBUG trial tests
 ```
 
 
-## Run the integration tests.
+## Run the integration tests ([Sytest](https://github.com/matrix-org/sytest)).
 
 The integration tests are a more comprehensive suite of tests. They
 run a full version of Synapse, including your changes, to check if
@@ -197,12 +198,49 @@ The following command will let you run the integration test with the most common
 configuration:
 
 ```sh
-$ docker run --rm -it -v /path/where/you/have/cloned/the/repository\:/src:ro -v /path/to/where/you/want/logs\:/logs matrixdotorg/sytest-synapse:py37
+$ docker run --rm -it -v /path/where/you/have/cloned/the/repository\:/src:ro -v /path/to/where/you/want/logs\:/logs matrixdotorg/sytest-synapse:buster
 ```
 
 This configuration should generally cover  your needs. For more details about other configurations, see [documentation in the SyTest repo](https://github.com/matrix-org/sytest/blob/develop/docker/README.md).
 
 
+## Run the integration tests ([Complement](https://github.com/matrix-org/complement)).
+
+[Complement](https://github.com/matrix-org/complement) is a suite of black box tests that can be run on any homeserver implementation. It can also be thought of as end-to-end (e2e) tests.
+
+It's often nice to develop on Synapse and write Complement tests at the same time.
+Here is how to run your local Synapse checkout against your local Complement checkout.
+
+(checkout [`complement`](https://github.com/matrix-org/complement) alongside your `synapse` checkout)
+```sh
+COMPLEMENT_DIR=../complement ./scripts-dev/complement.sh
+```
+
+To run a specific test file, you can pass the test name at the end of the command. The name passed comes from the naming structure in your Complement tests. If you're unsure of the name, you can do a full run and copy it from the test output:
+
+```sh
+COMPLEMENT_DIR=../complement ./scripts-dev/complement.sh TestBackfillingHistory
+```
+
+To run a specific test, you can specify the whole name structure:
+
+```sh
+COMPLEMENT_DIR=../complement ./scripts-dev/complement.sh TestBackfillingHistory/parallel/Backfilled_historical_events_resolve_with_proper_state_in_correct_order
+```
+
+
+### Access database for homeserver after Complement test runs.
+
+If you're curious what the database looks like after you run some tests, here are some steps to get you going in Synapse:
+
+ 1. In your Complement test comment out `defer deployment.Destroy(t)` and replace with `defer time.Sleep(2 * time.Hour)` to keep the homeserver running after the tests complete
+ 1. Start the Complement tests
+ 1. Find the name of the container, `docker ps -f name=complement_` (this will filter for just the Compelement related Docker containers)
+ 1. Access the container replacing the name with what you found in the previous step: `docker exec -it complement_1_hs_with_application_service.hs1_2 /bin/bash`
+ 1. Install sqlite (database driver), `apt-get update && apt-get install -y sqlite3`
+ 1. Then run `sqlite3` and open the database `.open /conf/homeserver.db` (this db path comes from the Synapse homeserver.yaml)
+
+
 # 9. Submit your patch.
 
 Once you're happy with your patch, it's time to prepare a Pull Request.
@@ -215,6 +253,7 @@ To prepare a Pull Request, please:
 4. on GitHub, [create the Pull Request](https://docs.github.com/en/github/collaborating-with-issues-and-pull-requests/creating-a-pull-request);
 5. add a [changelog entry](#changelog) and push it to your Pull Request;
 6. for most contributors, that's all - however, if you are a member of the organization `matrix-org`, on GitHub, please request a review from `matrix.org / Synapse Core`.
+7. if you need to update your PR, please avoid rebasing and just add new commits to your branch.
 
 
 ## Changelog

MANIFEST.in

@@ -44,9 +44,9 @@ include book.toml
 include pyproject.toml
 recursive-include changelog.d *
 
-prune .buildkite
 prune .circleci
 prune .github
+prune .ci
 prune contrib
 prune debian
 prune demo/etc

changelog.d/10119.misc

@@ -0,0 +1 @@
+Improve event caching mechanism to avoid having multiple copies of an event in memory at a time.

changelog.d/10129.bugfix

@@ -0,0 +1 @@
+Add some clarification to the sample config file. Contributed by @Kentokamoto.

changelog.d/10254.feature

@@ -1 +0,0 @@
-Update support for [MSC3083](https://github.com/matrix-org/matrix-doc/pull/3083) to consider changes in the MSC around which servers can issue join events.

changelog.d/10283.feature

@@ -1 +0,0 @@
-Initial support for MSC3244, Room version capabilities over the /capabilities API.

changelog.d/10394.feature

@@ -0,0 +1 @@
+Initial local support for [MSC3266](https://github.com/matrix-org/synapse/pull/10394), Room Summary over the unstable `/rooms/{roomIdOrAlias}/summary` API.

changelog.d/10407.feature

@@ -1 +0,0 @@
-Add a buffered logging handler which periodically flushes itself.

changelog.d/10408.misc

@@ -1 +0,0 @@
-Add type hints to `synapse.federation.transport.client` module.

changelog.d/10426.feature

@@ -1 +0,0 @@
-Email notifications now state whether an invitation is to a room or a space.

changelog.d/10429.misc

@@ -1 +0,0 @@
-Drop backwards-compatibility code that was required to support Ubuntu Xenial.

changelog.d/10431.misc

@@ -1 +0,0 @@
-Use a docker image cache for the prerequisites for the debian package build.

changelog.d/10432.misc

@@ -1 +0,0 @@
-Connect historical chunks together with chunk events instead of a content field (MSC2716).

changelog.d/10435.feature

@@ -0,0 +1 @@
+Experimental support for [MSC3288](https://github.com/matrix-org/matrix-doc/pull/3288), sending `room_type` to the identity server for 3pid invites over the `/store-invite` API.

changelog.d/10437.misc

@@ -1 +0,0 @@
-Improve servlet type hints.

changelog.d/10438.misc

@@ -1 +0,0 @@
-Improve servlet type hints.

changelog.d/10442.misc

@@ -1 +0,0 @@
-Replace usage of `or_ignore` in `simple_insert` with `simple_upsert` usage, to stop spamming postgres logs with spurious ERROR messages.

changelog.d/10443.doc

@@ -0,0 +1 @@
+Add documentation for configuration a forward proxy.

changelog.d/10444.misc

@@ -1 +0,0 @@
-Update the `tests-done` Github Actions status.

changelog.d/10445.doc

@@ -1 +0,0 @@
-Fix hierarchy of providers on the OpenID page.

changelog.d/10446.misc

@@ -1 +0,0 @@
-Update type annotations to work with forthcoming Twisted 21.7.0 release.

changelog.d/10448.feature

@@ -1 +0,0 @@
-Add `creation_ts` to list users admin API.

changelog.d/10451.misc

@@ -1 +0,0 @@
-Cancel redundant GHA workflows when a new commit is pushed.

changelog.d/10453.doc

@@ -1 +0,0 @@
-Consolidate development documentation to `docs/development/`.

changelog.d/10461.misc

@@ -1 +0,0 @@
-Fix an error which prevented the Github Actions workflow to build the docker images from running.

changelog.d/10463.misc

@@ -1 +0,0 @@
-Disable `msc2716` Complement tests until Complement updates are merged.

changelog.d/10468.misc

@@ -1 +0,0 @@
-Mitigate media repo XSS attacks on IE11 via the non-standard X-Content-Security-Policy header.

changelog.d/10475.feature

@@ -0,0 +1 @@
+Add support for sending federation requests through a proxy. Contributed by @Bubu and @dklimpel.

changelog.d/10482.misc

@@ -1 +0,0 @@
-Additional type hints in the state handler.

changelog.d/10488.misc

@@ -1 +0,0 @@
-Update syntax used to run complement tests.

changelog.d/10498.feature

@@ -0,0 +1 @@
+Add support for "marker" events which makes historical events discoverable for servers that already have all of the scrollback history (part of MSC2716).

changelog.d/10504.misc

@@ -0,0 +1 @@
+Reduce errors in PostgreSQL logs due to concurrent serialization errors.

changelog.d/10507.misc

@@ -0,0 +1 @@
+Include room ID in ignored EDU log messages. Contributed by @ilmari.

changelog.d/10513.feature

@@ -0,0 +1 @@
+Add a configuration setting for the time a `/sync` response is cached for.

changelog.d/10524.feature

@@ -0,0 +1 @@
+Port the PresenceRouter module interface to the new generic interface.

changelog.d/10527.misc

@@ -0,0 +1 @@
+Add pagination to the spaces summary based on updates to [MSC2946](https://github.com/matrix-org/matrix-doc/pull/2946).

changelog.d/10529.misc

@@ -0,0 +1 @@
+Fix CI to not break when run against branches rather than pull requests.

changelog.d/10530.misc

@@ -0,0 +1 @@
+Add pagination to the spaces summary based on updates to [MSC2946](https://github.com/matrix-org/matrix-doc/pull/2946).

changelog.d/10532.bugfix

@@ -0,0 +1 @@
+Fix a long-standing bug where protocols which are not implemented by any appservices were incorrectly returned via `GET /_matrix/client/r0/thirdparty/protocols`.

changelog.d/10537.misc

@@ -0,0 +1 @@
+Mark all events stemming from the MSC2716 `/batch_send` endpoint as historical.

changelog.d/10538.feature

@@ -0,0 +1 @@
+Add support for new redaction rules for historical events specified in [MSC2716](https://github.com/matrix-org/matrix-doc/pull/2716).

changelog.d/10539.misc

@@ -0,0 +1 @@
+Clean up some of the federation event authentication code for clarity.

changelog.d/10541.bugfix

@@ -0,0 +1 @@
+Fix exceptions in logs when failing to get remote room list.

changelog.d/10542.misc

@@ -0,0 +1 @@
+Convert `Transaction` and `Edu` objects to attrs.

changelog.d/10546.feature

@@ -0,0 +1 @@
+Add a setting to disable TLS when sending email.

changelog.d/10549.feature

@@ -0,0 +1 @@
+Add pagination to the spaces summary based on updates to [MSC2946](https://github.com/matrix-org/matrix-doc/pull/2946).

changelog.d/10550.bugfix

@@ -0,0 +1 @@
+Fix longstanding bug which caused the user "status" to be reset when the user went offline. Contributed by @dklimpel.

changelog.d/10551.doc

@@ -0,0 +1 @@
+Updated the reverse proxy documentation to highlight the homserver configuration that is needed to make Synapse aware that is is intentionally reverse proxied.

changelog.d/10552.misc

@@ -0,0 +1 @@
+Update `/batch_send` endpoint to only return `state_events` created by the `state_events_from_before` passed in.

changelog.d/10558.feature

@@ -0,0 +1 @@
+Admin API to delete several media for a specific user. Contributed by @dklimpel.

changelog.d/10560.feature

@@ -0,0 +1 @@
+Add pagination to the spaces summary based on updates to [MSC2946](https://github.com/matrix-org/matrix-doc/pull/2946).

changelog.d/10561.bugfix

@@ -0,0 +1 @@
+Display an error on User-Interactive Authentication fallback pages when authentication fails. Contributed by Callum Brown.

changelog.d/10563.misc

@@ -0,0 +1 @@
+Update contributing.md to warn against rebasing an open PR.

changelog.d/10569.feature

@@ -0,0 +1 @@
+Add pagination to the spaces summary based on updates to [MSC2946](https://github.com/matrix-org/matrix-doc/pull/2946).

changelog.d/10570.feature

@@ -0,0 +1 @@
+Update the Synapse Grafana dashboard.

changelog.d/10572.misc

@@ -0,0 +1 @@
+Clarify error message when failing to join a restricted room.

changelog.d/10573.misc

@@ -0,0 +1 @@
+Remove references to BuildKite in favour of GitHub Actions.

changelog.d/10574.feature

@@ -0,0 +1 @@
+Add pagination to the spaces summary based on updates to [MSC2946](https://github.com/matrix-org/matrix-doc/pull/2946).

changelog.d/10575.feature

@@ -0,0 +1 @@
+Add pagination to the spaces summary based on updates to [MSC2946](https://github.com/matrix-org/matrix-doc/pull/2946).

changelog.d/10576.misc

@@ -0,0 +1 @@
+Move `/batch_send` endpoint defined by [MSC2716](https://github.com/matrix-org/matrix-doc/pull/2716) to the `/v2_alpha` directory.

changelog.d/10578.feature

@@ -0,0 +1 @@
+Add an admin API (`GET /_synapse/admin/username_available`) to check if a username is available (regardless of registration settings).

changelog.d/10579.feature

@@ -0,0 +1 @@
+Add pagination to the spaces summary based on updates to [MSC2946](https://github.com/matrix-org/matrix-doc/pull/2946).

changelog.d/10580.bugfix

@@ -0,0 +1 @@
+Allow public rooms to be previewed in the spaces summary APIs from [MSC2946](https://github.com/matrix-org/matrix-doc/pull/2946).

changelog.d/10587.misc

@@ -0,0 +1 @@
+Allow multiple custom directories in `read_templates`.

changelog.d/10588.removal

@@ -0,0 +1 @@
+No longer build `.dev` packages for Ubuntu 20.10 LTS Groovy Gorilla, which has now EOLed.

changelog.d/10590.misc

@@ -0,0 +1 @@
+Re-organize the `synapse.federation.transport.server` module to create smaller files.

changelog.d/10591.misc

@@ -0,0 +1 @@
+Clean up some of the federation event authentication code for clarity.

changelog.d/10592.bugfix

@@ -0,0 +1 @@
+Fix a bug introduced in v1.37.1 where an error could occur in the asyncronous processing of PDUs when the queue was empty.

changelog.d/10598.feature

@@ -0,0 +1 @@
+Allow editing a user's `external_ids` via the "Edit User" admin API. Contributed by @dklimpel.

changelog.d/10599.doc

@@ -0,0 +1 @@
+Update CONTRIBUTING.md to fix index links and the instructions for SyTest in docker.

changelog.d/10600.misc

@@ -0,0 +1 @@
+Flatten the `synapse.rest.client` package by moving the contents of `v1` and `v2_alpha` into the parent.

changelog.d/10602.feature

@@ -0,0 +1 @@
+The Synapse manhole no longer needs coroutines to be wrapped in `defer.ensureDeferred`.

changelog.d/10606.bugfix

@@ -0,0 +1 @@
+Fix errors on /sync when read receipt data is a string. Only affects homeservers with the experimental flag for [MSC2285](https://github.com/matrix-org/matrix-doc/pull/2285) enabled. Contributed by @SimonBrandner.

changelog.d/10611.bugfix

@@ -0,0 +1 @@
+Additional validation for the spaces summary API to avoid errors like `ValueError: Stop argument for islice() must be None or an integer`. The missing validation has existed since v1.31.0.

changelog.d/10614.misc

@@ -0,0 +1 @@
+Clean up some of the federation event authentication code for clarity.

changelog.d/10615.misc

@@ -0,0 +1 @@
+Clean up some of the federation event authentication code for clarity.

changelog.d/10620.misc

@@ -0,0 +1 @@
+Fix up a couple of links to the database schema documentation.

changelog.d/10623.bugfix

@@ -0,0 +1 @@
+Revert behaviour introduced in v1.38.0 that strips `org.matrix.msc2732.device_unused_fallback_key_types` from `/sync` when its value is empty. This field should instead always be present according to [MSC2732](https://github.com/matrix-org/matrix-doc/blob/master/proposals/2732-olm-fallback-keys.md).

changelog.d/10629.misc

@@ -0,0 +1 @@
+Convert room member storage tuples to `attrs` classes.

changelog.d/10630.misc

@@ -0,0 +1 @@
+Use auto-attribs for the attrs classes used in sync.

changelog.d/8830.removal

@@ -0,0 +1 @@
+Remove deprecated Shutdown Room and Purge Room Admin API.

changelog.d/9581.feature

@@ -0,0 +1 @@
+Add `get_userinfo_by_id` method to ModuleApi.

changelog.d/9918.feature

@@ -1 +0,0 @@
-Add support for [MSC2033](https://github.com/matrix-org/matrix-doc/pull/2033): `device_id` on `/account/whoami`.

contrib/grafana/synapse.json

@@ -54,7 +54,7 @@
   "gnetId": null,
   "graphTooltip": 0,
   "id": null,
-  "iteration": 1621258266004,
+  "iteration": 1628606819564,
   "links": [
     {
       "asDropdown": false,
@@ -307,7 +307,6 @@
       ],
       "thresholds": [
         {
-          "$$hashKey": "object:283",
           "colorMode": "warning",
           "fill": false,
           "line": true,
@@ -316,7 +315,6 @@
           "yaxis": "left"
         },
         {
-          "$$hashKey": "object:284",
           "colorMode": "critical",
           "fill": false,
           "line": true,
@@ -344,7 +342,6 @@
       },
       "yaxes": [
         {
-          "$$hashKey": "object:255",
           "decimals": null,
           "format": "s",
           "label": "",
@@ -354,7 +351,6 @@
           "show": true
         },
         {
-          "$$hashKey": "object:256",
           "format": "hertz",
           "label": "",
           "logBase": 1,
@@ -429,7 +425,6 @@
       ],
       "thresholds": [
         {
-          "$$hashKey": "object:566",
           "colorMode": "critical",
           "fill": true,
           "line": true,
@@ -457,7 +452,6 @@
       },
       "yaxes": [
         {
-          "$$hashKey": "object:538",
           "decimals": null,
           "format": "percentunit",
           "label": null,
@@ -467,7 +461,6 @@
           "show": true
         },
         {
-          "$$hashKey": "object:539",
           "format": "short",
           "label": null,
           "logBase": 1,
@@ -573,7 +566,6 @@
       },
       "yaxes": [
         {
-          "$$hashKey": "object:1560",
           "format": "bytes",
           "logBase": 1,
           "max": null,
@@ -581,7 +573,6 @@
           "show": true
         },
         {
-          "$$hashKey": "object:1561",
           "format": "short",
           "logBase": 1,
           "max": null,
@@ -641,7 +632,6 @@
       "renderer": "flot",
       "seriesOverrides": [
         {
-          "$$hashKey": "object:639",
           "alias": "/max$/",
           "color": "#890F02",
           "fill": 0,
@@ -693,7 +683,6 @@
       },
       "yaxes": [
         {
-          "$$hashKey": "object:650",
           "decimals": null,
           "format": "none",
           "label": "",
@@ -703,7 +692,6 @@
           "show": true
         },
         {
-          "$$hashKey": "object:651",
           "decimals": null,
           "format": "short",
           "label": null,
@@ -783,11 +771,9 @@
           "renderer": "flot",
           "seriesOverrides": [
             {
-              "$$hashKey": "object:1240",
               "alias": "/user/"
             },
             {
-              "$$hashKey": "object:1241",
               "alias": "/system/"
             }
           ],
@@ -817,7 +803,6 @@
           ],
           "thresholds": [
             {
-              "$$hashKey": "object:1278",
               "colorMode": "custom",
               "fillColor": "rgba(255, 255, 255, 1)",
               "line": true,
@@ -827,7 +812,6 @@
               "yaxis": "left"
             },
             {
-              "$$hashKey": "object:1279",
               "colorMode": "custom",
               "fillColor": "rgba(255, 255, 255, 1)",
               "line": true,
@@ -837,7 +821,6 @@
               "yaxis": "left"
             },
             {
-              "$$hashKey": "object:1498",
               "colorMode": "critical",
               "fill": true,
               "line": true,
@@ -865,7 +848,6 @@
           },
           "yaxes": [
             {
-              "$$hashKey": "object:1250",
               "decimals": null,
               "format": "percentunit",
               "label": "",
@@ -875,7 +857,6 @@
               "show": true
             },
             {
-              "$$hashKey": "object:1251",
               "format": "short",
               "logBase": 1,
               "max": null,
@@ -1427,7 +1408,6 @@
           },
           "yaxes": [
             {
-              "$$hashKey": "object:572",
               "format": "percentunit",
               "label": null,
               "logBase": 1,
@@ -1436,7 +1416,6 @@
               "show": true
             },
             {
-              "$$hashKey": "object:573",
               "format": "short",
               "label": null,
               "logBase": 1,
@@ -1720,7 +1699,6 @@
           },
           "yaxes": [
             {
-              "$$hashKey": "object:102",
               "format": "hertz",
               "logBase": 1,
               "max": null,
@@ -1728,7 +1706,6 @@
               "show": true
             },
             {
-              "$$hashKey": "object:103",
               "format": "short",
               "logBase": 1,
               "max": null,
@@ -3425,7 +3402,7 @@
             "h": 9,
             "w": 12,
             "x": 0,
-            "y": 33
+            "y": 6
           },
           "hiddenSeries": false,
           "id": 79,
@@ -3442,9 +3419,12 @@
           "linewidth": 1,
           "links": [],
           "nullPointMode": "null",
+          "options": {
+            "alertThreshold": true
+          },
           "paceLength": 10,
           "percentage": false,
-          "pluginVersion": "7.1.3",
+          "pluginVersion": "7.3.7",
           "pointradius": 5,
           "points": false,
           "renderer": "flot",
@@ -3526,7 +3506,7 @@
             "h": 9,
             "w": 12,
             "x": 12,
-            "y": 33
+            "y": 6
           },
           "hiddenSeries": false,
           "id": 83,
@@ -3543,9 +3523,12 @@
           "linewidth": 1,
           "links": [],
           "nullPointMode": "null",
+          "options": {
+            "alertThreshold": true
+          },
           "paceLength": 10,
           "percentage": false,
-          "pluginVersion": "7.1.3",
+          "pluginVersion": "7.3.7",
           "pointradius": 5,
           "points": false,
           "renderer": "flot",
@@ -3629,7 +3612,7 @@
             "h": 9,
             "w": 12,
             "x": 0,
-            "y": 42
+            "y": 15
           },
           "hiddenSeries": false,
           "id": 109,
@@ -3646,9 +3629,12 @@
           "linewidth": 1,
           "links": [],
           "nullPointMode": "null",
+          "options": {
+            "alertThreshold": true
+          },
           "paceLength": 10,
           "percentage": false,
-          "pluginVersion": "7.1.3",
+          "pluginVersion": "7.3.7",
           "pointradius": 5,
           "points": false,
           "renderer": "flot",
@@ -3733,7 +3719,7 @@
             "h": 9,
             "w": 12,
             "x": 12,
-            "y": 42
+            "y": 15
           },
           "hiddenSeries": false,
           "id": 111,
@@ -3750,9 +3736,12 @@
           "linewidth": 1,
           "links": [],
           "nullPointMode": "null",
+          "options": {
+            "alertThreshold": true
+          },
           "paceLength": 10,
           "percentage": false,
-          "pluginVersion": "7.1.3",
+          "pluginVersion": "7.3.7",
           "pointradius": 5,
           "points": false,
           "renderer": "flot",
@@ -3831,7 +3820,7 @@
             "h": 8,
             "w": 12,
             "x": 0,
-            "y": 51
+            "y": 24
           },
           "hiddenSeries": false,
           "id": 142,
@@ -3847,8 +3836,11 @@
           "lines": true,
           "linewidth": 1,
           "nullPointMode": "null",
+          "options": {
+            "alertThreshold": true
+          },
           "percentage": false,
-          "pluginVersion": "7.1.3",
+          "pluginVersion": "7.3.7",
           "pointradius": 2,
           "points": false,
           "renderer": "flot",
@@ -3931,7 +3923,7 @@
             "h": 9,
             "w": 12,
             "x": 12,
-            "y": 51
+            "y": 24
           },
           "hiddenSeries": false,
           "id": 140,
@@ -3948,9 +3940,12 @@
           "linewidth": 1,
           "links": [],
           "nullPointMode": "null",
+          "options": {
+            "alertThreshold": true
+          },
           "paceLength": 10,
           "percentage": false,
-          "pluginVersion": "7.1.3",
+          "pluginVersion": "7.3.7",
           "pointradius": 5,
           "points": false,
           "renderer": "flot",
@@ -4079,7 +4074,7 @@
             "h": 9,
             "w": 12,
             "x": 0,
-            "y": 59
+            "y": 32
           },
           "heatmap": {},
           "hideZeroBuckets": false,
@@ -4145,7 +4140,7 @@
             "h": 9,
             "w": 12,
             "x": 12,
-            "y": 60
+            "y": 33
           },
           "hiddenSeries": false,
           "id": 162,
@@ -4163,9 +4158,12 @@
           "linewidth": 0,
           "links": [],
           "nullPointMode": "connected",
+          "options": {
+            "alertThreshold": true
+          },
           "paceLength": 10,
           "percentage": false,
-          "pluginVersion": "7.1.3",
+          "pluginVersion": "7.3.7",
           "pointradius": 5,
           "points": false,
           "renderer": "flot",
@@ -4350,7 +4348,7 @@
             "h": 9,
             "w": 12,
             "x": 0,
-            "y": 68
+            "y": 41
           },
           "heatmap": {},
           "hideZeroBuckets": false,
@@ -4396,6 +4394,311 @@
           "yBucketBound": "auto",
           "yBucketNumber": null,
           "yBucketSize": null
+        },
+        {
+          "aliasColors": {},
+          "bars": false,
+          "dashLength": 10,
+          "dashes": false,
+          "datasource": "$datasource",
+          "editable": true,
+          "error": false,
+          "fieldConfig": {
+            "defaults": {
+              "custom": {},
+              "links": []
+            },
+            "overrides": []
+          },
+          "fill": 1,
+          "fillGradient": 0,
+          "grid": {},
+          "gridPos": {
+            "h": 9,
+            "w": 12,
+            "x": 12,
+            "y": 42
+          },
+          "hiddenSeries": false,
+          "id": 203,
+          "legend": {
+            "avg": false,
+            "current": false,
+            "max": false,
+            "min": false,
+            "show": true,
+            "total": false,
+            "values": false
+          },
+          "lines": true,
+          "linewidth": 2,
+          "links": [],
+          "nullPointMode": "null",
+          "options": {
+            "alertThreshold": true
+          },
+          "paceLength": 10,
+          "percentage": false,
+          "pluginVersion": "7.3.7",
+          "pointradius": 5,
+          "points": false,
+          "renderer": "flot",
+          "seriesOverrides": [],
+          "spaceLength": 10,
+          "stack": false,
+          "steppedLine": false,
+          "targets": [
+            {
+              "expr": "synapse_federation_server_oldest_inbound_pdu_in_staging{job=\"$job\",index=~\"$index\",instance=\"$instance\"}",
+              "format": "time_series",
+              "interval": "",
+              "intervalFactor": 1,
+              "legendFormat": "rss {{index}}",
+              "refId": "A",
+              "step": 4
+            }
+          ],
+          "thresholds": [],
+          "timeFrom": null,
+          "timeRegions": [],
+          "timeShift": null,
+          "title": "Age of oldest event in staging area",
+          "tooltip": {
+            "msResolution": false,
+            "shared": true,
+            "sort": 0,
+            "value_type": "cumulative"
+          },
+          "type": "graph",
+          "xaxis": {
+            "buckets": null,
+            "mode": "time",
+            "name": null,
+            "show": true,
+            "values": []
+          },
+          "yaxes": [
+            {
+              "format": "ms",
+              "label": null,
+              "logBase": 1,
+              "max": null,
+              "min": 0,
+              "show": true
+            },
+            {
+              "format": "short",
+              "label": null,
+              "logBase": 1,
+              "max": null,
+              "min": null,
+              "show": true
+            }
+          ],
+          "yaxis": {
+            "align": false,
+            "alignLevel": null
+          }
+        },
+        {
+          "aliasColors": {},
+          "bars": false,
+          "dashLength": 10,
+          "dashes": false,
+          "datasource": "$datasource",
+          "editable": true,
+          "error": false,
+          "fieldConfig": {
+            "defaults": {
+              "custom": {},
+              "links": []
+            },
+            "overrides": []
+          },
+          "fill": 1,
+          "fillGradient": 0,
+          "grid": {},
+          "gridPos": {
+            "h": 9,
+            "w": 12,
+            "x": 0,
+            "y": 50
+          },
+          "hiddenSeries": false,
+          "id": 202,
+          "legend": {
+            "avg": false,
+            "current": false,
+            "max": false,
+            "min": false,
+            "show": true,
+            "total": false,
+            "values": false
+          },
+          "lines": true,
+          "linewidth": 2,
+          "links": [],
+          "nullPointMode": "null",
+          "options": {
+            "alertThreshold": true
+          },
+          "paceLength": 10,
+          "percentage": false,
+          "pluginVersion": "7.3.7",
+          "pointradius": 5,
+          "points": false,
+          "renderer": "flot",
+          "seriesOverrides": [],
+          "spaceLength": 10,
+          "stack": false,
+          "steppedLine": false,
+          "targets": [
+            {
+              "expr": "synapse_federation_server_number_inbound_pdu_in_staging{job=\"$job\",index=~\"$index\",instance=\"$instance\"}",
+              "format": "time_series",
+              "interval": "",
+              "intervalFactor": 1,
+              "legendFormat": "rss {{index}}",
+              "refId": "A",
+              "step": 4
+            }
+          ],
+          "thresholds": [],
+          "timeFrom": null,
+          "timeRegions": [],
+          "timeShift": null,
+          "title": "Number of events in federation staging area",
+          "tooltip": {
+            "msResolution": false,
+            "shared": true,
+            "sort": 0,
+            "value_type": "cumulative"
+          },
+          "type": "graph",
+          "xaxis": {
+            "buckets": null,
+            "mode": "time",
+            "name": null,
+            "show": true,
+            "values": []
+          },
+          "yaxes": [
+            {
+              "format": "none",
+              "label": null,
+              "logBase": 1,
+              "max": null,
+              "min": 0,
+              "show": true
+            },
+            {
+              "format": "short",
+              "label": null,
+              "logBase": 1,
+              "max": null,
+              "min": null,
+              "show": true
+            }
+          ],
+          "yaxis": {
+            "align": false,
+            "alignLevel": null
+          }
+        },
+        {
+          "aliasColors": {},
+          "bars": false,
+          "dashLength": 10,
+          "dashes": false,
+          "datasource": "${DS_PROMETHEUS}",
+          "fieldConfig": {
+            "defaults": {
+              "custom": {}
+            },
+            "overrides": []
+          },
+          "fill": 1,
+          "fillGradient": 0,
+          "gridPos": {
+            "h": 8,
+            "w": 12,
+            "x": 12,
+            "y": 51
+          },
+          "hiddenSeries": false,
+          "id": 205,
+          "legend": {
+            "avg": false,
+            "current": false,
+            "max": false,
+            "min": false,
+            "show": true,
+            "total": false,
+            "values": false
+          },
+          "lines": true,
+          "linewidth": 1,
+          "nullPointMode": "null",
+          "options": {
+            "alertThreshold": true
+          },
+          "percentage": false,
+          "pluginVersion": "7.3.7",
+          "pointradius": 2,
+          "points": false,
+          "renderer": "flot",
+          "seriesOverrides": [],
+          "spaceLength": 10,
+          "stack": false,
+          "steppedLine": false,
+          "targets": [
+            {
+              "expr": "sum(rate(synapse_federation_soft_failed_events_total{instance=\"$instance\"}[$bucket_size]))",
+              "interval": "",
+              "legendFormat": "soft-failed events",
+              "refId": "A"
+            }
+          ],
+          "thresholds": [],
+          "timeFrom": null,
+          "timeRegions": [],
+          "timeShift": null,
+          "title": "Soft-failed event rate",
+          "tooltip": {
+            "shared": true,
+            "sort": 0,
+            "value_type": "individual"
+          },
+          "type": "graph",
+          "xaxis": {
+            "buckets": null,
+            "mode": "time",
+            "name": null,
+            "show": true,
+            "values": []
+          },
+          "yaxes": [
+            {
+              "format": "hertz",
+              "label": null,
+              "logBase": 1,
+              "max": null,
+              "min": null,
+              "show": true
+            },
+            {
+              "format": "short",
+              "label": null,
+              "logBase": 1,
+              "max": null,
+              "min": null,
+              "show": false
+            }
+          ],
+          "yaxis": {
+            "align": false,
+            "alignLevel": null
+          }
         }
       ],
       "title": "Federation",
@@ -4647,7 +4950,7 @@
             "h": 7,
             "w": 12,
             "x": 0,
-            "y": 8
+            "y": 33
           },
           "hiddenSeries": false,
           "id": 48,
@@ -4749,7 +5052,7 @@
             "h": 7,
             "w": 12,
             "x": 12,
-            "y": 8
+            "y": 33
           },
           "hiddenSeries": false,
           "id": 104,
@@ -4877,7 +5180,7 @@
             "h": 7,
             "w": 12,
             "x": 0,
-            "y": 15
+            "y": 40
           },
           "hiddenSeries": false,
           "id": 10,
@@ -4981,7 +5284,7 @@
             "h": 7,
             "w": 12,
             "x": 12,
-            "y": 15
+            "y": 40
           },
           "hiddenSeries": false,
           "id": 11,
@@ -5086,7 +5389,7 @@
             "h": 7,
             "w": 12,
             "x": 0,
-            "y": 22
+            "y": 47
           },
           "hiddenSeries": false,
           "id": 180,
@@ -5168,6 +5471,126 @@
             "align": false,
             "alignLevel": null
           }
+        },
+        {
+          "aliasColors": {},
+          "bars": false,
+          "dashLength": 10,
+          "dashes": false,
+          "datasource": "$datasource",
+          "fieldConfig": {
+            "defaults": {
+              "custom": {},
+              "links": []
+            },
+            "overrides": []
+          },
+          "fill": 6,
+          "fillGradient": 0,
+          "gridPos": {
+            "h": 9,
+            "w": 12,
+            "x": 12,
+            "y": 47
+          },
+          "hiddenSeries": false,
+          "id": 200,
+          "legend": {
+            "avg": false,
+            "current": false,
+            "max": false,
+            "min": false,
+            "show": true,
+            "total": false,
+            "values": false
+          },
+          "lines": true,
+          "linewidth": 1,
+          "links": [],
+          "nullPointMode": "null",
+          "options": {
+            "alertThreshold": true
+          },
+          "percentage": false,
+          "pluginVersion": "7.3.7",
+          "pointradius": 5,
+          "points": false,
+          "renderer": "flot",
+          "seriesOverrides": [],
+          "spaceLength": 10,
+          "stack": false,
+          "steppedLine": false,
+          "targets": [
+            {
+              "expr": "histogram_quantile(0.99, sum(rate(synapse_storage_schedule_time_bucket{index=~\"$index\",instance=\"$instance\",job=\"$job\"}[$bucket_size])) by (le))",
+              "format": "time_series",
+              "intervalFactor": 1,
+              "legendFormat": "99%",
+              "refId": "D"
+            },
+            {
+              "expr": "histogram_quantile(0.9, sum(rate(synapse_storage_schedule_time_bucket{index=~\"$index\",instance=\"$instance\",job=\"$job\"}[$bucket_size])) by (le))",
+              "format": "time_series",
+              "intervalFactor": 1,
+              "legendFormat": "90%",
+              "refId": "A"
+            },
+            {
+              "expr": "histogram_quantile(0.75, sum(rate(synapse_storage_schedule_time_bucket{index=~\"$index\",instance=\"$instance\",job=\"$job\"}[$bucket_size])) by (le))",
+              "format": "time_series",
+              "intervalFactor": 1,
+              "legendFormat": "75%",
+              "refId": "C"
+            },
+            {
+              "expr": "histogram_quantile(0.5, sum(rate(synapse_storage_schedule_time_bucket{index=~\"$index\",instance=\"$instance\",job=\"$job\"}[$bucket_size])) by (le))",
+              "format": "time_series",
+              "intervalFactor": 1,
+              "legendFormat": "50%",
+              "refId": "B"
+            }
+          ],
+          "thresholds": [],
+          "timeFrom": null,
+          "timeRegions": [],
+          "timeShift": null,
+          "title": "Time waiting for DB connection quantiles",
+          "tooltip": {
+            "shared": true,
+            "sort": 2,
+            "value_type": "individual"
+          },
+          "type": "graph",
+          "xaxis": {
+            "buckets": null,
+            "mode": "time",
+            "name": null,
+            "show": true,
+            "values": []
+          },
+          "yaxes": [
+            {
+              "decimals": null,
+              "format": "s",
+              "label": "",
+              "logBase": 1,
+              "max": null,
+              "min": "0",
+              "show": true
+            },
+            {
+              "format": "short",
+              "label": null,
+              "logBase": 1,
+              "max": null,
+              "min": null,
+              "show": false
+            }
+          ],
+          "yaxis": {
+            "align": false,
+            "alignLevel": null
+          }
         }
       ],
       "repeat": null,
@@ -5916,7 +6339,7 @@
             "h": 10,
             "w": 12,
             "x": 0,
-            "y": 84
+            "y": 35
           },
           "hiddenSeries": false,
           "id": 1,
@@ -6022,7 +6445,7 @@
             "h": 10,
             "w": 12,
             "x": 12,
-            "y": 84
+            "y": 35
           },
           "hiddenSeries": false,
           "id": 8,
@@ -6126,7 +6549,7 @@
             "h": 10,
             "w": 12,
             "x": 0,
-            "y": 94
+            "y": 45
           },
           "hiddenSeries": false,
           "id": 38,
@@ -6226,7 +6649,7 @@
             "h": 10,
             "w": 12,
             "x": 12,
-            "y": 94
+            "y": 45
           },
           "hiddenSeries": false,
           "id": 39,
@@ -6258,8 +6681,9 @@
           "steppedLine": false,
           "targets": [
             {
-              "expr": "topk(10, rate(synapse_util_caches_cache:total{job=\"$job\",index=~\"$index\",instance=\"$instance\"}[$bucket_size]) - rate(synapse_util_caches_cache:hits{job=\"$job\",instance=\"$instance\"}[$bucket_size]))",
+              "expr": "topk(10, rate(synapse_util_caches_cache:total{job=~\"$job\",index=~\"$index\",instance=\"$instance\"}[$bucket_size]) - rate(synapse_util_caches_cache:hits{job=~\"$job\",index=~\"$index\",instance=\"$instance\"}[$bucket_size]))",
               "format": "time_series",
+              "interval": "",
               "intervalFactor": 2,
               "legendFormat": "{{name}} {{job}}-{{index}}",
               "refId": "A",
@@ -6326,7 +6750,7 @@
             "h": 9,
             "w": 12,
             "x": 0,
-            "y": 104
+            "y": 55
           },
           "hiddenSeries": false,
           "id": 65,
@@ -9051,7 +9475,7 @@
             "h": 8,
             "w": 12,
             "x": 0,
-            "y": 119
+            "y": 41
           },
           "hiddenSeries": false,
           "id": 156,
@@ -9089,7 +9513,7 @@
           "steppedLine": false,
           "targets": [
             {
-              "expr": "synapse_admin_mau:current{instance=\"$instance\"}",
+              "expr": "synapse_admin_mau:current{instance=\"$instance\", job=~\"$job\"}",
               "format": "time_series",
               "interval": "",
               "intervalFactor": 1,
@@ -9097,7 +9521,7 @@
               "refId": "A"
             },
             {
-              "expr": "synapse_admin_mau:max{instance=\"$instance\"}",
+              "expr": "synapse_admin_mau:max{instance=\"$instance\", job=~\"$job\"}",
               "format": "time_series",
               "interval": "",
               "intervalFactor": 1,
@@ -9164,7 +9588,7 @@
             "h": 8,
             "w": 12,
             "x": 12,
-            "y": 119
+            "y": 41
           },
           "hiddenSeries": false,
           "id": 160,
@@ -9484,7 +9908,7 @@
             "h": 8,
             "w": 12,
             "x": 0,
-            "y": 73
+            "y": 43
           },
           "hiddenSeries": false,
           "id": 168,
@@ -9516,7 +9940,7 @@
             {
               "expr": "rate(synapse_appservice_api_sent_events{instance=\"$instance\"}[$bucket_size])",
               "interval": "",
-              "legendFormat": "{{exported_service}}",
+              "legendFormat": "{{service}}",
               "refId": "A"
             }
           ],
@@ -9579,7 +10003,7 @@
             "h": 8,
             "w": 12,
             "x": 12,
-            "y": 73
+            "y": 43
           },
           "hiddenSeries": false,
           "id": 171,
@@ -9611,7 +10035,7 @@
             {
               "expr": "rate(synapse_appservice_api_sent_transactions{instance=\"$instance\"}[$bucket_size])",
               "interval": "",
-              "legendFormat": "{{exported_service}}",
+              "legendFormat": "{{service}}",
               "refId": "A"
             }
           ],
@@ -9959,7 +10383,6 @@
           },
           "yaxes": [
             {
-              "$$hashKey": "object:165",
               "format": "hertz",
               "label": null,
               "logBase": 1,
@@ -9968,7 +10391,6 @@
               "show": true
             },
             {
-              "$$hashKey": "object:166",
               "format": "short",
               "label": null,
               "logBase": 1,
@@ -10071,7 +10493,6 @@
           },
           "yaxes": [
             {
-              "$$hashKey": "object:390",
               "format": "hertz",
               "label": null,
               "logBase": 1,
@@ -10080,7 +10501,6 @@
               "show": true
             },
             {
-              "$$hashKey": "object:391",
               "format": "short",
               "label": null,
               "logBase": 1,
@@ -10169,7 +10589,6 @@
           },
           "yaxes": [
             {
-              "$$hashKey": "object:390",
               "format": "hertz",
               "label": null,
               "logBase": 1,
@@ -10178,7 +10597,6 @@
               "show": true
             },
             {
-              "$$hashKey": "object:391",
               "format": "short",
               "label": null,
               "logBase": 1,
@@ -10470,5 +10888,5 @@
   "timezone": "",
   "title": "Synapse",
   "uid": "000000012",
-  "version": 90
+  "version": 99
 }

debian/build_virtualenv

@@ -100,3 +100,18 @@ esac
 # add a dependency on the right version of python to substvars.
 PYPKG=`basename $SNAKE`
 echo "synapse:pydepends=$PYPKG" >> debian/matrix-synapse-py3.substvars
+
+
+# add a couple of triggers.  This is needed so that dh-virtualenv can rebuild
+# the venv when the system python changes (see
+# https://dh-virtualenv.readthedocs.io/en/latest/tutorial.html#step-2-set-up-packaging-for-your-project)
+#
+# we do it here rather than the more conventional way of just adding it to
+# debian/matrix-synapse-py3.triggers, because we need to add a trigger on the
+# right version of python.
+cat >>"debian/.debhelper/generated/matrix-synapse-py3/triggers" <<EOF
+# triggers for dh-virtualenv
+interest-noawait $SNAKE
+interest dh-virtualenv-interpreter-update
+
+EOF

debian/changelog

@@ -1,8 +1,44 @@
-matrix-synapse-py3 (1.39.0ubuntu1) UNRELEASED; urgency=medium
+matrix-synapse-py3 (1.40.0) stable; urgency=medium
 
+  * New synapse release 1.40.0.
+
+ -- Synapse Packaging team <packages@matrix.org>  Tue, 10 Aug 2021 13:50:48 +0100
+
+matrix-synapse-py3 (1.40.0~rc3) stable; urgency=medium
+
+  * New synapse release 1.40.0~rc3.
+
+ -- Synapse Packaging team <packages@matrix.org>  Mon, 09 Aug 2021 13:41:08 +0100
+
+matrix-synapse-py3 (1.40.0~rc2) stable; urgency=medium
+
+  * New synapse release 1.40.0~rc2.
+
+ -- Synapse Packaging team <packages@matrix.org>  Wed, 04 Aug 2021 17:08:55 +0100
+
+matrix-synapse-py3 (1.40.0~rc1) stable; urgency=medium
+
+  [ Richard van der Hoff ]
   * Drop backwards-compatibility code that was required to support Ubuntu Xenial.
+  * Update package triggers so that the virtualenv is correctly rebuilt
+    when the system python is rebuilt, on recent Python versions.
 
- -- Richard van der Hoff <richard@matrix.org>  Tue, 20 Jul 2021 00:10:03 +0100
+  [ Synapse Packaging team ]
+  * New synapse release 1.40.0~rc1.
+
+ -- Synapse Packaging team <packages@matrix.org>  Tue, 03 Aug 2021 11:31:49 +0100
+
+matrix-synapse-py3 (1.39.0) stable; urgency=medium
+
+  * New synapse release 1.39.0.
+
+ -- Synapse Packaging team <packages@matrix.org>  Thu, 29 Jul 2021 09:59:00 +0100
+
+matrix-synapse-py3 (1.39.0~rc3) stable; urgency=medium
+
+  * New synapse release 1.39.0~rc3.
+
+ -- Synapse Packaging team <packages@matrix.org>  Wed, 28 Jul 2021 13:30:58 +0100
 
 matrix-synapse-py3 (1.38.1) stable; urgency=medium
 

debian/matrix-synapse-py3.triggers

@@ -1,9 +0,0 @@
-# Register interest in Python interpreter changes and
-# don't make the Python package dependent on the virtualenv package
-# processing (noawait)
-interest-noawait /usr/bin/python3.5
-interest-noawait /usr/bin/python3.6
-interest-noawait /usr/bin/python3.7
-
-# Also provide a symbolic trigger for all dh-virtualenv packages
-interest dh-virtualenv-interpreter-update

docker/build_debian.sh

@@ -11,10 +11,6 @@ DIST=`cut -d ':' -f2 <<< $distro`
 cp -aT /synapse/source /synapse/build
 cd /synapse/build
 
-# add an entry to the changelog for this distribution
-dch -M -l "+$DIST" "build for $DIST"
-dch -M -r "" --force-distribution --distribution "$DIST"
-
 # if this is a prerelease, set the Section accordingly.
 #
 # When the package is later added to the package repo, reprepro will use the
@@ -23,11 +19,14 @@ dch -M -r "" --force-distribution --distribution "$DIST"
 
 DEB_VERSION=`dpkg-parsechangelog -SVersion`
 case $DEB_VERSION in
-    *rc*|*a*|*b*|*c*)
+    *~rc*|*~a*|*~b*|*~c*)
         sed -ie '/^Section:/c\Section: prerelease' debian/control
         ;;
 esac
 
+# add an entry to the changelog for this distribution
+dch -M -l "+$DIST" "build for $DIST"
+dch -M -r "" --force-distribution --distribution "$DIST"
 
 dpkg-buildpackage -us -uc
 

docs/SUMMARY.md

@@ -7,6 +7,7 @@
   - [Installation](setup/installation.md)
   - [Using Postgres](postgres.md)
   - [Configuring a Reverse Proxy](reverse_proxy.md)
+  - [Configuring a Forward/Outbound Proxy](setup/forward_proxy.md)
   - [Configuring a Turn Server](turn-howto.md)
   - [Delegation](delegate.md)
 
@@ -50,12 +51,10 @@
       - [Event Reports](admin_api/event_reports.md)
       - [Media](admin_api/media_admin_api.md)
       - [Purge History](admin_api/purge_history_api.md)
-      - [Purge Rooms](admin_api/purge_room.md)
       - [Register Users](admin_api/register_api.md)
       - [Manipulate Room Membership](admin_api/room_membership.md)
       - [Rooms](admin_api/rooms.md)
       - [Server Notices](admin_api/server_notices.md)
-      - [Shutdown Room](admin_api/shutdown_room.md)
       - [Statistics](admin_api/statistics.md)
       - [Users](admin_api/user_admin_api.md)
       - [Server Version](admin_api/version_api.md)
@@ -79,6 +78,7 @@
     - [Single Sign-On]()
       - [SAML](development/saml.md)
       - [CAS](development/cas.md)
+    - [Room DAG concepts](development/room-dag-concepts.md)
     - [State Resolution]()
       - [The Auth Chain Difference Algorithm](auth_chain_difference_algorithm.md)
     - [Media Repository](media_repository.md)