Fix changelog number

Co-authored-by: Andrew Morgan <1342360+anoadragon453@users.noreply.github.com>

.buildkite/postgres-config.yaml

@@ -3,7 +3,7 @@
 # CI's Docker setup at the point where this file is considered.
 server_name: "localhost:8800"
 
-signing_key_path: ".buildkite/test.signing.key"
+signing_key_path: "/src/.buildkite/test.signing.key"
 
 report_stats: false
 
@@ -16,4 +16,6 @@ database:
     database: synapse
 
 # Suppress the key server warning.
-trusted_key_servers: []
+trusted_key_servers:
+  - server_name: "matrix.org"
+suppress_key_server_warning: true

.buildkite/scripts/create_postgres_db.py

@@ -0,0 +1,37 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# Copyright 2019 The Matrix.org Foundation C.I.C.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import logging
+
+from synapse.storage.engines import create_engine
+
+logger = logging.getLogger("create_postgres_db")
+
+if __name__ == "__main__":
+    # Create a PostgresEngine.
+    db_engine = create_engine({"name": "psycopg2", "args": {}})
+
+    # Connect to postgres to create the base database.
+    # We use "postgres" as a database because it's bound to exist and the "synapse" one
+    # doesn't exist yet.
+    db_conn = db_engine.module.connect(
+        user="postgres", host="postgres", password="postgres", dbname="postgres"
+    )
+    db_conn.autocommit = True
+    cur = db_conn.cursor()
+    cur.execute("CREATE DATABASE synapse;")
+    cur.close()
+    db_conn.close()

.buildkite/scripts/postgres_exec.py

@@ -1,31 +0,0 @@
-#!/usr/bin/env python
-# Copyright 2019 The Matrix.org Foundation C.I.C.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import sys
-
-import psycopg2
-
-# a very simple replacment for `psql`, to make up for the lack of the postgres client
-# libraries in the synapse docker image.
-
-# We use "postgres" as a database because it's bound to exist and the "synapse" one
-# doesn't exist yet.
-db_conn = psycopg2.connect(
-    user="postgres", host="postgres", password="postgres", dbname="postgres"
-)
-db_conn.autocommit = True
-cur = db_conn.cursor()
-for c in sys.argv[1:]:
-    cur.execute(c)

.buildkite/scripts/test_synapse_port_db.sh

@@ -1,10 +1,10 @@
 #!/usr/bin/env bash
 #
-# Test script for 'synapse_port_db'.
-#   - sets up synapse and deps
-#   - runs the port script on a prepopulated test sqlite db
-#   - also runs it against an new sqlite db
-
+# Test script for 'synapse_port_db', which creates a virtualenv, installs Synapse along
+# with additional dependencies needed for the test (such as coverage or the PostgreSQL
+# driver), update the schema of the test SQLite database and run background updates on it,
+# create an empty test database in PostgreSQL, then run the 'synapse_port_db' script to
+# test porting the SQLite database to the PostgreSQL database (with coverage).
 
 set -xe
 cd `dirname $0`/../..
@@ -22,36 +22,15 @@ echo "--- Generate the signing key"
 # Generate the server's signing key.
 python -m synapse.app.homeserver --generate-keys -c .buildkite/sqlite-config.yaml
 
-echo "--- Prepare test database"
+echo "--- Prepare the databases"
 
 # Make sure the SQLite3 database is using the latest schema and has no pending background update.
 scripts-dev/update_database --database-config .buildkite/sqlite-config.yaml
 
 # Create the PostgreSQL database.
-./.buildkite/scripts/postgres_exec.py "CREATE DATABASE synapse"
+./.buildkite/scripts/create_postgres_db.py
 
-echo "+++ Run synapse_port_db against test database"
-coverage run scripts/synapse_port_db --sqlite-database .buildkite/test_db.db --postgres-config .buildkite/postgres-config.yaml
-
-# We should be able to run twice against the same database.
-echo "+++ Run synapse_port_db a second time"
-coverage run scripts/synapse_port_db --sqlite-database .buildkite/test_db.db --postgres-config .buildkite/postgres-config.yaml
-
-#####
-
-# Now do the same again, on an empty database.
-
-echo "--- Prepare empty SQLite database"
-
-# we do this by deleting the sqlite db, and then doing the same again.
-rm .buildkite/test_db.db
-
-scripts-dev/update_database --database-config .buildkite/sqlite-config.yaml
-
-# re-create the PostgreSQL database.
-./.buildkite/scripts/postgres_exec.py \
-  "DROP DATABASE synapse" \
-  "CREATE DATABASE synapse"
-
-echo "+++ Run synapse_port_db against empty database"
+echo "+++ Run synapse_port_db"
+
+# Run the script
 coverage run scripts/synapse_port_db --sqlite-database .buildkite/test_db.db --postgres-config .buildkite/postgres-config.yaml

.buildkite/sqlite-config.yaml

@@ -3,7 +3,7 @@
 # schema and run background updates on it.
 server_name: "localhost:8800"
 
-signing_key_path: ".buildkite/test.signing.key"
+signing_key_path: "/src/.buildkite/test.signing.key"
 
 report_stats: false
 
@@ -13,4 +13,6 @@ database:
     database: ".buildkite/test_db.db"
 
 # Suppress the key server warning.
-trusted_key_servers: []
+trusted_key_servers:
+  - server_name: "matrix.org"
+suppress_key_server_warning: true

.circleci/config.yml

@@ -41,7 +41,7 @@ workflows:
       - dockerhubuploadlatest:
           filters:
             branches:
-              only: [ master, main ]
+              only: master
 
 commands:
   docker_prepare:

.github/workflows/docs.yaml

@@ -1,31 +0,0 @@
-name: Deploy the documentation
-
-on:
-  push:
-    branches:
-      - develop
-
-  workflow_dispatch:
-
-jobs:
-  pages:
-    name: GitHub Pages
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
-
-      - name: Setup mdbook
-        uses: peaceiris/actions-mdbook@4b5ef36b314c2599664ca107bb8c02412548d79d # v1.1.14
-        with:
-          mdbook-version: '0.4.9'
-
-      - name: Build the documentation
-        run: mdbook build
-
-      - name: Deploy latest documentation
-        uses: peaceiris/actions-gh-pages@068dc23d9710f1ba62e86896f84735d869951305 # v3.8.0
-        with:
-          github_token: ${{ secrets.GITHUB_TOKEN }}
-          keep_files: true
-          publish_dir: ./book
-          destination_dir: ./develop

.github/workflows/tests.yml

@@ -34,13 +34,7 @@ jobs:
     if: ${{ github.base_ref == 'develop'  || contains(github.base_ref, 'release-') }}
     runs-on: ubuntu-latest
     steps:
-      # Note: This and the script can be simplified once we drop Buildkite. See:
-      #   https://github.com/actions/checkout/issues/266#issuecomment-638346893
-      #   https://github.com/actions/checkout/issues/416
       - uses: actions/checkout@v2
-        with:
-          ref: ${{ github.event.pull_request.head.sha }}
-          fetch-depth: 0
       - uses: actions/setup-python@v2
       - run: pip install tox
       - name: Patch Buildkite-specific test script
@@ -232,9 +226,9 @@ jobs:
       - name: Run SyTest
         run: /bootstrap.sh synapse
         working-directory: /src
-      - name: Summarise results.tap
+      - name: Dump results.tap
         if: ${{ always() }}
-        run: /sytest/scripts/tap_to_gha.pl /logs/results.tap
+        run: cat /logs/results.tap
       - name: Upload SyTest logs
         uses: actions/upload-artifact@v2
         if: ${{ always() }}
@@ -279,7 +273,7 @@ jobs:
           python-version: ${{ matrix.python-version }}
       - name: Patch Buildkite-specific test scripts
         run: |
-          sed -i -e 's/host="postgres"/host="localhost"/' .buildkite/scripts/postgres_exec.py
+          sed -i -e 's/host="postgres"/host="localhost"/' .buildkite/scripts/create_postgres_db.py
           sed -i -e 's/host: postgres/host: localhost/' .buildkite/postgres-config.yaml
           sed -i -e 's|/src/||' .buildkite/{sqlite,postgres}-config.yaml
           sed -i -e 's/\$TOP/\$GITHUB_WORKSPACE/' .coveragerc

.gitignore

@@ -46,6 +46,3 @@ __pycache__/
 /docs/build/
 /htmlcov
 /pip-wheel-metadata/
-
-# docs
-book/

CHANGES.md

@@ -1,424 +1,11 @@
-Synapse 1.36.0 (2021-06-15)
-===========================
-
-No significant changes.
-
-
-Synapse 1.36.0rc2 (2021-06-11)
+Synapse 1.32.0rc1 (2021-04-13)
 ==============================
 
-Bugfixes
---------
-
-- Fix a bug which caused  presence updates to stop working some time after a restart, when using a presence writer worker. Broke in v1.33.0. ([\#10149](https://github.com/matrix-org/synapse/issues/10149))
-- Fix a bug when using federation sender worker where it would send out more presence updates than necessary, leading to high resource usage. Broke in v1.33.0. ([\#10163](https://github.com/matrix-org/synapse/issues/10163))
-- Fix a bug where Synapse could send the same presence update to a remote twice. ([\#10165](https://github.com/matrix-org/synapse/issues/10165))
-
-
-Synapse 1.36.0rc1 (2021-06-08)
-==============================
-
-Features
---------
-
-- Add new endpoint `/_matrix/client/r0/rooms/{roomId}/aliases` from Client-Server API r0.6.1 (previously [MSC2432](https://github.com/matrix-org/matrix-doc/pull/2432)). ([\#9224](https://github.com/matrix-org/synapse/issues/9224))
-- Improve performance of incoming federation transactions in large rooms. ([\#9953](https://github.com/matrix-org/synapse/issues/9953), [\#9973](https://github.com/matrix-org/synapse/issues/9973))
-- Rewrite logic around verifying JSON object and fetching server keys to be more performant and use less memory. ([\#10035](https://github.com/matrix-org/synapse/issues/10035))
-- Add new admin APIs for unprotecting local media from quarantine. Contributed by @dklimpel. ([\#10040](https://github.com/matrix-org/synapse/issues/10040))
-- Add new admin APIs to remove media by media ID from quarantine. Contributed by @dklimpel. ([\#10044](https://github.com/matrix-org/synapse/issues/10044))
-- Make reason and score parameters optional for reporting content. Implements [MSC2414](https://github.com/matrix-org/matrix-doc/pull/2414). Contributed by Callum Brown. ([\#10077](https://github.com/matrix-org/synapse/issues/10077))
-- Add support for routing more requests to workers. ([\#10084](https://github.com/matrix-org/synapse/issues/10084))
-- Report OpenTracing spans for database activity. ([\#10113](https://github.com/matrix-org/synapse/issues/10113), [\#10136](https://github.com/matrix-org/synapse/issues/10136), [\#10141](https://github.com/matrix-org/synapse/issues/10141))
-- Significantly reduce memory usage of joining large remote rooms. ([\#10117](https://github.com/matrix-org/synapse/issues/10117))
-
-
-Bugfixes
---------
-
-- Fixed a bug causing replication requests to fail when receiving a lot of events via federation. ([\#10082](https://github.com/matrix-org/synapse/issues/10082))
-- Fix a bug in the `force_tracing_for_users` option introduced in Synapse v1.35 which meant that the OpenTracing spans produced were missing most tags. ([\#10092](https://github.com/matrix-org/synapse/issues/10092))
-- Fixed a bug that could cause Synapse to stop notifying application services. Contributed by Willem Mulder. ([\#10107](https://github.com/matrix-org/synapse/issues/10107))
-- Fix bug where the server would attempt to fetch the same history in the room from a remote server multiple times in parallel. ([\#10116](https://github.com/matrix-org/synapse/issues/10116))
-- Fix a bug introduced in Synapse 1.33.0 which caused replication requests to fail when receiving a lot of very large events via federation. ([\#10118](https://github.com/matrix-org/synapse/issues/10118))
-- Fix bug when using workers where pagination requests failed if a remote server returned zero events from `/backfill`. Introduced in 1.35.0. ([\#10133](https://github.com/matrix-org/synapse/issues/10133))
-
-
-Improved Documentation
-----------------------
-
-- Clarify security note regarding hosting Synapse on the same domain as other web applications. ([\#9221](https://github.com/matrix-org/synapse/issues/9221))
-- Update CAPTCHA documentation to mention turning off the verify origin feature. Contributed by @aaronraimist. ([\#10046](https://github.com/matrix-org/synapse/issues/10046))
-- Tweak wording of database recommendation in `INSTALL.md`. Contributed by @aaronraimist. ([\#10057](https://github.com/matrix-org/synapse/issues/10057))
-- Add initial infrastructure for rendering Synapse documentation with mdbook. ([\#10086](https://github.com/matrix-org/synapse/issues/10086))
-- Convert the remaining Admin API documentation files to markdown. ([\#10089](https://github.com/matrix-org/synapse/issues/10089))
-- Make a link in docs use HTTPS. Contributed by @RhnSharma. ([\#10130](https://github.com/matrix-org/synapse/issues/10130))
-- Fix broken link in Docker docs. ([\#10132](https://github.com/matrix-org/synapse/issues/10132))
-
-
-Deprecations and Removals
--------------------------
-
-- Remove the experimental `spaces_enabled` flag. The spaces features are always available now. ([\#10063](https://github.com/matrix-org/synapse/issues/10063))
-
-
-Internal Changes
-----------------
-
-- Tell CircleCI to build Docker images from `main` branch. ([\#9906](https://github.com/matrix-org/synapse/issues/9906))
-- Simplify naming convention for release branches to only include the major and minor version numbers. ([\#10013](https://github.com/matrix-org/synapse/issues/10013))
-- Add `parse_strings_from_args` for parsing an array from query parameters. ([\#10048](https://github.com/matrix-org/synapse/issues/10048), [\#10137](https://github.com/matrix-org/synapse/issues/10137))
-- Remove some dead code regarding TLS certificate handling. ([\#10054](https://github.com/matrix-org/synapse/issues/10054))
-- Remove redundant, unmaintained `convert_server_keys` script. ([\#10055](https://github.com/matrix-org/synapse/issues/10055))
-- Improve the error message printed by synctl when synapse fails to start. ([\#10059](https://github.com/matrix-org/synapse/issues/10059))
-- Fix GitHub Actions lint for newsfragments. ([\#10069](https://github.com/matrix-org/synapse/issues/10069))
-- Update opentracing to inject the right context into the carrier. ([\#10074](https://github.com/matrix-org/synapse/issues/10074))
-- Fix up `BatchingQueue` implementation. ([\#10078](https://github.com/matrix-org/synapse/issues/10078))
-- Log method and path when dropping request due to size limit. ([\#10091](https://github.com/matrix-org/synapse/issues/10091))
-- In Github Actions workflows, summarize the Sytest results in an easy-to-read format. ([\#10094](https://github.com/matrix-org/synapse/issues/10094))
-- Make `/sync` do fewer state resolutions. ([\#10102](https://github.com/matrix-org/synapse/issues/10102))
-- Add missing type hints to the admin API servlets. ([\#10105](https://github.com/matrix-org/synapse/issues/10105))
-- Improve opentracing annotations for `Notifier`. ([\#10111](https://github.com/matrix-org/synapse/issues/10111))
-- Enable Prometheus metrics for the jaeger client library. ([\#10112](https://github.com/matrix-org/synapse/issues/10112))
-- Work to improve the responsiveness of `/sync` requests. ([\#10124](https://github.com/matrix-org/synapse/issues/10124))
-- OpenTracing: use a consistent name for background processes. ([\#10135](https://github.com/matrix-org/synapse/issues/10135))
-
-
-Synapse 1.35.1 (2021-06-03)
-===========================
-
-Bugfixes
---------
-
-- Fix a bug introduced in v1.35.0 where invite-only rooms would be shown to all users in a space, regardless of if the user had access to it. ([\#10109](https://github.com/matrix-org/synapse/issues/10109))
-
-
-Synapse 1.35.0 (2021-06-01)
-===========================
-
-Note that [the tag](https://github.com/matrix-org/synapse/releases/tag/v1.35.0rc3) and [docker images](https://hub.docker.com/layers/matrixdotorg/synapse/v1.35.0rc3/images/sha256-34ccc87bd99a17e2cbc0902e678b5937d16bdc1991ead097eee6096481ecf2c4?context=explore) for `v1.35.0rc3` were incorrectly built. If you are experiencing issues with either, it is recommended to upgrade to the equivalent tag or docker image for the `v1.35.0` release.
-
-Deprecations and Removals
--------------------------
-
-- The core Synapse development team plan to drop support for the [unstable API of MSC2858](https://github.com/matrix-org/matrix-doc/blob/master/proposals/2858-Multiple-SSO-Identity-Providers.md#unstable-prefix), including the undocumented `experimental.msc2858_enabled` config option, in August 2021. Client authors should ensure that their clients are updated to use the stable API (which has been supported since Synapse 1.30) well before that time, to give their users time to upgrade. ([\#10101](https://github.com/matrix-org/synapse/issues/10101))
-
-Bugfixes
---------
-
-- Fixed a bug causing replication requests to fail when receiving a lot of events via federation. Introduced in v1.33.0. ([\#10082](https://github.com/matrix-org/synapse/issues/10082))
-- Fix HTTP response size limit to allow joining very large rooms over federation. Introduced in v1.33.0. ([\#10093](https://github.com/matrix-org/synapse/issues/10093))
-
-
-Internal Changes
-----------------
-
-- Log method and path when dropping request due to size limit. ([\#10091](https://github.com/matrix-org/synapse/issues/10091))
-
-
-Synapse 1.35.0rc2 (2021-05-27)
-==============================
-
-Bugfixes
---------
-
-- Fix a bug introduced in v1.35.0rc1 when calling the spaces summary API via a GET request. ([\#10079](https://github.com/matrix-org/synapse/issues/10079))
-
-
-Synapse 1.35.0rc1 (2021-05-25)
-==============================
-
-Features
---------
-
-- Add experimental support to allow a user who could join a restricted room to view it in the spaces summary. ([\#9922](https://github.com/matrix-org/synapse/issues/9922), [\#10007](https://github.com/matrix-org/synapse/issues/10007), [\#10038](https://github.com/matrix-org/synapse/issues/10038))
-- Reduce memory usage when joining very large rooms over federation. ([\#9958](https://github.com/matrix-org/synapse/issues/9958))
-- Add a configuration option which allows enabling opentracing by user id. ([\#9978](https://github.com/matrix-org/synapse/issues/9978))
-- Enable experimental support for [MSC2946](https://github.com/matrix-org/matrix-doc/pull/2946) (spaces summary API) and [MSC3083](https://github.com/matrix-org/matrix-doc/pull/3083) (restricted join rules) by default. ([\#10011](https://github.com/matrix-org/synapse/issues/10011))
-
-
-Bugfixes
---------
-
-- Fix a bug introduced in v1.26.0 which meant that `synapse_port_db` would not correctly initialise some postgres sequences, requiring manual updates afterwards. ([\#9991](https://github.com/matrix-org/synapse/issues/9991))
-- Fix `synctl`'s `--no-daemonize` parameter to work correctly with worker processes. ([\#9995](https://github.com/matrix-org/synapse/issues/9995))
-- Fix a validation bug introduced in v1.34.0 in the ordering of spaces in the space summary API. ([\#10002](https://github.com/matrix-org/synapse/issues/10002))
-- Fixed deletion of new presence stream states from database. ([\#10014](https://github.com/matrix-org/synapse/issues/10014), [\#10033](https://github.com/matrix-org/synapse/issues/10033))
-- Fixed a bug with very high resolution image uploads throwing internal server errors. ([\#10029](https://github.com/matrix-org/synapse/issues/10029))
-
-
-Updates to the Docker image
----------------------------
-
-- Fix bug introduced in Synapse 1.33.0 which caused a `Permission denied: '/homeserver.log'` error when starting Synapse with the generated log configuration. Contributed by Sergio Miguéns Iglesias. ([\#10045](https://github.com/matrix-org/synapse/issues/10045))
-
-
-Improved Documentation
-----------------------
-
-- Add hardened systemd files as proposed in [#9760](https://github.com/matrix-org/synapse/issues/9760) and added them to `contrib/`. Change the docs to reflect the presence of these files. ([\#9803](https://github.com/matrix-org/synapse/issues/9803))
-- Clarify documentation around SSO mapping providers generating unique IDs and localparts. ([\#9980](https://github.com/matrix-org/synapse/issues/9980))
-- Updates to the PostgreSQL documentation (`postgres.md`). ([\#9988](https://github.com/matrix-org/synapse/issues/9988), [\#9989](https://github.com/matrix-org/synapse/issues/9989))
-- Fix broken link in user directory documentation. Contributed by @junquera. ([\#10016](https://github.com/matrix-org/synapse/issues/10016))
-- Add missing room state entry to the table of contents of room admin API. ([\#10043](https://github.com/matrix-org/synapse/issues/10043))
-
-
-Deprecations and Removals
--------------------------
-
-- Removed support for the deprecated `tls_fingerprints` configuration setting. Contributed by Jerin J Titus. ([\#9280](https://github.com/matrix-org/synapse/issues/9280))
-
-
-Internal Changes
-----------------
-
-- Allow sending full presence to users via workers other than the one that called `ModuleApi.send_local_online_presence_to`. ([\#9823](https://github.com/matrix-org/synapse/issues/9823))
-- Update comments in the space summary handler. ([\#9974](https://github.com/matrix-org/synapse/issues/9974))
-- Minor enhancements to the `@cachedList` descriptor. ([\#9975](https://github.com/matrix-org/synapse/issues/9975))
-- Split multipart email sending into a dedicated handler. ([\#9977](https://github.com/matrix-org/synapse/issues/9977))
-- Run `black` on files in the `scripts` directory. ([\#9981](https://github.com/matrix-org/synapse/issues/9981))
-- Add missing type hints to `synapse.util` module. ([\#9982](https://github.com/matrix-org/synapse/issues/9982))
-- Simplify a few helper functions. ([\#9984](https://github.com/matrix-org/synapse/issues/9984), [\#9985](https://github.com/matrix-org/synapse/issues/9985), [\#9986](https://github.com/matrix-org/synapse/issues/9986))
-- Remove unnecessary property from SQLBaseStore. ([\#9987](https://github.com/matrix-org/synapse/issues/9987))
-- Remove `keylen` param on `LruCache`. ([\#9993](https://github.com/matrix-org/synapse/issues/9993))
-- Update the Grafana dashboard in `contrib/`. ([\#10001](https://github.com/matrix-org/synapse/issues/10001))
-- Add a batching queue implementation. ([\#10017](https://github.com/matrix-org/synapse/issues/10017))
-- Reduce memory usage when verifying signatures on large numbers of events at once. ([\#10018](https://github.com/matrix-org/synapse/issues/10018))
-- Properly invalidate caches for destination retry timings every (instead of expiring entries every 5 minutes). ([\#10036](https://github.com/matrix-org/synapse/issues/10036))
-- Fix running complement tests with Synapse workers. ([\#10039](https://github.com/matrix-org/synapse/issues/10039))
-- Fix typo in `get_state_ids_for_event` docstring where the return type was incorrect. ([\#10050](https://github.com/matrix-org/synapse/issues/10050))
-
-
-Synapse 1.34.0 (2021-05-17)
-===========================
-
-This release deprecates the `room_invite_state_types` configuration setting. See the [upgrade notes](https://github.com/matrix-org/synapse/blob/release-v1.34.0/UPGRADE.rst#upgrading-to-v1340) for instructions on updating your configuration file to use the new `room_prejoin_state` setting.
-
-This release also deprecates the `POST /_synapse/admin/v1/rooms/<room_id>/delete` admin API route. Server administrators are encouraged to update their scripts to use the new `DELETE /_synapse/admin/v1/rooms/<room_id>` route instead.
-
-
-No significant changes since v1.34.0rc1.
-
-
-Synapse 1.34.0rc1 (2021-05-12)
-==============================
-
-Features
---------
-
-- Add experimental option to track memory usage of the caches. ([\#9881](https://github.com/matrix-org/synapse/issues/9881))
-- Add support for `DELETE /_synapse/admin/v1/rooms/<room_id>`. ([\#9889](https://github.com/matrix-org/synapse/issues/9889))
-- Add limits to how often Synapse will GC, ensuring that large servers do not end up GC thrashing if `gc_thresholds` has not been correctly set. ([\#9902](https://github.com/matrix-org/synapse/issues/9902))
-- Improve performance of sending events for worker-based deployments using Redis. ([\#9905](https://github.com/matrix-org/synapse/issues/9905), [\#9950](https://github.com/matrix-org/synapse/issues/9950), [\#9951](https://github.com/matrix-org/synapse/issues/9951))
-- Improve performance after joining a large room when presence is enabled. ([\#9910](https://github.com/matrix-org/synapse/issues/9910), [\#9916](https://github.com/matrix-org/synapse/issues/9916))
-- Support stable identifiers for [MSC1772](https://github.com/matrix-org/matrix-doc/pull/1772) Spaces. `m.space.child` events will now be taken into account when populating the experimental spaces summary response. Please see [the upgrade notes](https://github.com/matrix-org/synapse/blob/release-v1.34.0/UPGRADE.rst#upgrading-to-v1340) if you have customised `room_invite_state_types` in your configuration. ([\#9915](https://github.com/matrix-org/synapse/issues/9915), [\#9966](https://github.com/matrix-org/synapse/issues/9966))
-- Improve performance of backfilling in large rooms. ([\#9935](https://github.com/matrix-org/synapse/issues/9935))
-- Add a config option to allow you to prevent device display names from being shared over federation. Contributed by @aaronraimist. ([\#9945](https://github.com/matrix-org/synapse/issues/9945))
-- Update support for [MSC2946](https://github.com/matrix-org/matrix-doc/pull/2946): Spaces Summary. ([\#9947](https://github.com/matrix-org/synapse/issues/9947), [\#9954](https://github.com/matrix-org/synapse/issues/9954))
-
-
-Bugfixes
---------
-
-- Fix a bug introduced in v1.32.0 where the associated connection was improperly logged for SQL logging statements. ([\#9895](https://github.com/matrix-org/synapse/issues/9895))
-- Correct the type hint for the `user_may_create_room_alias` method of spam checkers. It is provided a `RoomAlias`, not a `str`. ([\#9896](https://github.com/matrix-org/synapse/issues/9896))
-- Fix bug where user directory could get out of sync if room visibility and membership changed in quick succession. ([\#9910](https://github.com/matrix-org/synapse/issues/9910))
-- Include the `origin_server_ts` property in the experimental [MSC2946](https://github.com/matrix-org/matrix-doc/pull/2946) support to allow clients to properly sort rooms. ([\#9928](https://github.com/matrix-org/synapse/issues/9928))
-- Fix bugs introduced in v1.23.0 which made the PostgreSQL port script fail when run with a newly-created SQLite database. ([\#9930](https://github.com/matrix-org/synapse/issues/9930))
-- Fix a bug introduced in Synapse 1.29.0 which caused `m.room_key_request` to-device messages sent from one user to another to be dropped. ([\#9961](https://github.com/matrix-org/synapse/issues/9961), [\#9965](https://github.com/matrix-org/synapse/issues/9965))
-- Fix a bug introduced in v1.27.0 preventing users and appservices exempt from ratelimiting from creating rooms with many invitees. ([\#9968](https://github.com/matrix-org/synapse/issues/9968))
-
-
-Updates to the Docker image
----------------------------
-
-- Add `startup_delay` to docker healthcheck to reduce waiting time for coming online and update the documentation with extra options. Contributed by @Maquis196. ([\#9913](https://github.com/matrix-org/synapse/issues/9913))
-
-
-Improved Documentation
-----------------------
-
-- Add `port` argument to the Postgres database sample config section. ([\#9911](https://github.com/matrix-org/synapse/issues/9911))
-
-
-Deprecations and Removals
--------------------------
-
-- Mark as deprecated `POST /_synapse/admin/v1/rooms/<room_id>/delete`. ([\#9889](https://github.com/matrix-org/synapse/issues/9889))
-
-
-Internal Changes
-----------------
-
-- Reduce the length of Synapse's access tokens. ([\#5588](https://github.com/matrix-org/synapse/issues/5588))
-- Export jemalloc stats to Prometheus if it is being used. ([\#9882](https://github.com/matrix-org/synapse/issues/9882))
-- Add type hints to presence handler. ([\#9885](https://github.com/matrix-org/synapse/issues/9885))
-- Reduce memory usage of the LRU caches. ([\#9886](https://github.com/matrix-org/synapse/issues/9886))
-- Add type hints to the `synapse.handlers` module. ([\#9896](https://github.com/matrix-org/synapse/issues/9896))
-- Time response time for external cache requests. ([\#9904](https://github.com/matrix-org/synapse/issues/9904))
-- Minor fixes to the `make_full_schema.sh` script. ([\#9931](https://github.com/matrix-org/synapse/issues/9931))
-- Move database schema files into a common directory. ([\#9932](https://github.com/matrix-org/synapse/issues/9932))
-- Add debug logging for lost/delayed to-device messages. ([\#9959](https://github.com/matrix-org/synapse/issues/9959))
-
-
-Synapse 1.33.2 (2021-05-11)
-===========================
-
-Due to the security issue highlighted below, server administrators are encouraged to update Synapse. We are not aware of these vulnerabilities being exploited in the wild.
-
-Security advisory
------------------
-
-This release fixes a denial of service attack ([CVE-2021-29471](https://github.com/matrix-org/synapse/security/advisories/GHSA-x345-32rc-8h85)) against Synapse's push rules implementation. Server admins are encouraged to upgrade.
-
-Internal Changes
-----------------
-
-- Unpin attrs dependency. ([\#9946](https://github.com/matrix-org/synapse/issues/9946))
-
-
-Synapse 1.33.1 (2021-05-06)
-===========================
-
-Bugfixes
---------
-
-- Fix bug where `/sync` would break if using the latest version of `attrs` dependency, by pinning to a previous version. ([\#9937](https://github.com/matrix-org/synapse/issues/9937))
-
-
-Synapse 1.33.0 (2021-05-05)
-===========================
-
-Features
---------
-
-- Build Debian packages for Ubuntu 21.04 (Hirsute Hippo). ([\#9909](https://github.com/matrix-org/synapse/issues/9909))
-
-
-Synapse 1.33.0rc2 (2021-04-29)
-==============================
-
-Bugfixes
---------
-
-- Fix tight loop when handling presence replication when using workers. Introduced in v1.33.0rc1. ([\#9900](https://github.com/matrix-org/synapse/issues/9900))
-
-
-Synapse 1.33.0rc1 (2021-04-28)
-==============================
-
-Features
---------
-
-- Update experimental support for [MSC3083](https://github.com/matrix-org/matrix-doc/pull/3083): restricting room access via group membership. ([\#9800](https://github.com/matrix-org/synapse/issues/9800), [\#9814](https://github.com/matrix-org/synapse/issues/9814))
-- Add experimental support for handling presence on a worker. ([\#9819](https://github.com/matrix-org/synapse/issues/9819), [\#9820](https://github.com/matrix-org/synapse/issues/9820), [\#9828](https://github.com/matrix-org/synapse/issues/9828), [\#9850](https://github.com/matrix-org/synapse/issues/9850))
-- Return a new template when an user attempts to renew their account multiple times with the same token, stating that their account is set to expire. This replaces the invalid token template that would previously be shown in this case. This change concerns the optional account validity feature. ([\#9832](https://github.com/matrix-org/synapse/issues/9832))
-
-
-Bugfixes
---------
-
-- Fixes the OIDC SSO flow when using a `public_baseurl` value including a non-root URL path. ([\#9726](https://github.com/matrix-org/synapse/issues/9726))
-- Fix thumbnail generation for some sites with non-standard content types. Contributed by @rkfg. ([\#9788](https://github.com/matrix-org/synapse/issues/9788))
-- Add some sanity checks to identity server passed to 3PID bind/unbind endpoints. ([\#9802](https://github.com/matrix-org/synapse/issues/9802))
-- Limit the size of HTTP responses read over federation. ([\#9833](https://github.com/matrix-org/synapse/issues/9833))
-- Fix a bug which could cause Synapse to get stuck in a loop of resyncing device lists. ([\#9867](https://github.com/matrix-org/synapse/issues/9867))
-- Fix a long-standing bug where errors from federation did not propagate to the client. ([\#9868](https://github.com/matrix-org/synapse/issues/9868))
-
-
-Improved Documentation
-----------------------
-
-- Add a note to the docker docs mentioning that we mirror upstream's supported Docker platforms. ([\#9801](https://github.com/matrix-org/synapse/issues/9801))
-
-
-Internal Changes
-----------------
-
-- Add a dockerfile for running Synapse in worker-mode under Complement. ([\#9162](https://github.com/matrix-org/synapse/issues/9162))
-- Apply `pyupgrade` across the codebase. ([\#9786](https://github.com/matrix-org/synapse/issues/9786))
-- Move some replication processing out of `generic_worker`. ([\#9796](https://github.com/matrix-org/synapse/issues/9796))
-- Replace `HomeServer.get_config()` with inline references. ([\#9815](https://github.com/matrix-org/synapse/issues/9815))
-- Rename some handlers and config modules to not duplicate the top-level module. ([\#9816](https://github.com/matrix-org/synapse/issues/9816))
-- Fix a long-standing bug which caused `max_upload_size` to not be correctly enforced. ([\#9817](https://github.com/matrix-org/synapse/issues/9817))
-- Reduce CPU usage of the user directory by reusing existing calculated room membership. ([\#9821](https://github.com/matrix-org/synapse/issues/9821))
-- Small speed up for joining large remote rooms. ([\#9825](https://github.com/matrix-org/synapse/issues/9825))
-- Introduce flake8-bugbear to the test suite and fix some of its lint violations. ([\#9838](https://github.com/matrix-org/synapse/issues/9838))
-- Only store the raw data in the in-memory caches, rather than objects that include references to e.g. the data stores. ([\#9845](https://github.com/matrix-org/synapse/issues/9845))
-- Limit length of accepted email addresses. ([\#9855](https://github.com/matrix-org/synapse/issues/9855))
-- Remove redundant `synapse.types.Collection` type definition. ([\#9856](https://github.com/matrix-org/synapse/issues/9856))
-- Handle recently added rate limits correctly when using `--no-rate-limit` with the demo scripts. ([\#9858](https://github.com/matrix-org/synapse/issues/9858))
-- Disable invite rate-limiting by default when running the unit tests. ([\#9871](https://github.com/matrix-org/synapse/issues/9871))
-- Pass a reactor into `SynapseSite` to make testing easier. ([\#9874](https://github.com/matrix-org/synapse/issues/9874))
-- Make `DomainSpecificString` an `attrs` class. ([\#9875](https://github.com/matrix-org/synapse/issues/9875))
-- Add type hints to `synapse.api.auth` and `synapse.api.auth_blocking` modules. ([\#9876](https://github.com/matrix-org/synapse/issues/9876))
-- Remove redundant `_PushHTTPChannel` test class. ([\#9878](https://github.com/matrix-org/synapse/issues/9878))
-- Remove backwards-compatibility code for Python versions < 3.6. ([\#9879](https://github.com/matrix-org/synapse/issues/9879))
-- Small performance improvement around handling new local presence updates. ([\#9887](https://github.com/matrix-org/synapse/issues/9887))
-
-
-Synapse 1.32.2 (2021-04-22)
-===========================
-
-This release includes a fix for a regression introduced in 1.32.0.
-
-Bugfixes
---------
-
-- Fix a regression in Synapse 1.32.0 and 1.32.1 which caused `LoggingContext` errors in plugins. ([\#9857](https://github.com/matrix-org/synapse/issues/9857))
-
-
-Synapse 1.32.1 (2021-04-21)
-===========================
-
-This release fixes [a regression](https://github.com/matrix-org/synapse/issues/9853)
-in Synapse 1.32.0 that caused connected Prometheus instances to become unstable.
-
-However, as this release is still subject to the `LoggingContext` change in 1.32.0,
-it is recommended to remain on or downgrade to 1.31.0.
-
-Bugfixes
---------
-
-- Fix a regression in Synapse 1.32.0 which caused Synapse to report large numbers of Prometheus time series, potentially overwhelming Prometheus instances. ([\#9854](https://github.com/matrix-org/synapse/issues/9854))
-
-
-Synapse 1.32.0 (2021-04-20)
-===========================
-
-**Note:** This release introduces [a regression](https://github.com/matrix-org/synapse/issues/9853)
-that can overwhelm connected Prometheus instances. This issue was not present in
-1.32.0rc1. If affected, it is recommended to downgrade to 1.31.0 in the meantime, and
-follow [these instructions](https://github.com/matrix-org/synapse/pull/9854#issuecomment-823472183)
-to clean up any excess writeahead logs.
-
-**Note:** This release also mistakenly included a change that may affected Synapse
-modules that import `synapse.logging.context.LoggingContext`, such as
-[synapse-s3-storage-provider](https://github.com/matrix-org/synapse-s3-storage-provider).
-This will be fixed in a later Synapse version.
-
 **Note:** This release requires Python 3.6+ and Postgres 9.6+ or SQLite 3.22+.
 
 This release removes the deprecated `GET /_synapse/admin/v1/users/<user_id>` admin API. Please use the [v2 API](https://github.com/matrix-org/synapse/blob/develop/docs/admin_api/user_admin_api.rst#query-user-account) instead, which has improved capabilities.
 
-This release requires Application Services to use type `m.login.application_service` when registering users via the `/_matrix/client/r0/register` endpoint to comply with the spec. Please ensure your Application Services are up to date.
-
-If you are using the `packages.matrix.org` Debian repository for Synapse packages,
-note that we have recently updated the expiry date on the gpg signing key. If you see an
-error similar to `The following signatures were invalid: EXPKEYSIG F473DD4473365DE1`, you
-will need to get a fresh copy of the keys. You can do so with:
-
-```sh
-sudo wget -O /usr/share/keyrings/matrix-org-archive-keyring.gpg https://packages.matrix.org/debian/matrix-org-archive-keyring.gpg
-```
-
-Bugfixes
---------
-
-- Fix the log lines of nested logging contexts. Broke in 1.32.0rc1. ([\#9829](https://github.com/matrix-org/synapse/issues/9829))
-
-
-Synapse 1.32.0rc1 (2021-04-13)
-==============================
+This release requires Application Services to use type `m.login.application_services` when registering users via the `/_matrix/client/r0/register` endpoint to comply with the spec. Please ensure your Application Services are up to date.
 
 Features
 --------

INSTALL.md

@@ -399,9 +399,11 @@ Once you have installed synapse as above, you will need to configure it.
 
 ### Using PostgreSQL
 
-By default Synapse uses an [SQLite](https://sqlite.org/) database and in doing so trades
-performance for convenience. Almost all installations should opt to use [PostgreSQL](https://www.postgresql.org)
-instead. Advantages include:
+By default Synapse uses [SQLite](https://sqlite.org/) and in doing so trades performance for convenience.
+SQLite is only recommended in Synapse for testing purposes or for servers with
+very light workloads.
+
+Almost all installations should opt to use [PostgreSQL](https://www.postgresql.org). Advantages include:
 
 - significant performance improvements due to the superior threading and
   caching model, smarter query optimiser
@@ -410,10 +412,6 @@ instead. Advantages include:
 For information on how to install and use PostgreSQL in Synapse, please see
 [docs/postgres.md](docs/postgres.md)
 
-SQLite is only acceptable for testing purposes. SQLite should not be used in
-a production server. Synapse will perform poorly when using
-SQLite, especially when participating in large rooms.
-
 ### TLS certificates
 
 The default configuration exposes a single HTTP port on the local

MANIFEST.in

@@ -40,7 +40,6 @@ exclude mypy.ini
 exclude sytest-blacklist
 exclude test_postgresql.sh
 
-include book.toml
 include pyproject.toml
 recursive-include changelog.d *
 

README.rst

@@ -149,45 +149,21 @@ For details on having Synapse manage your federation TLS certificates
 automatically, please see `<docs/ACME.md>`_.
 
 
-Security note
+Security Note
 =============
 
-Matrix serves raw, user-supplied data in some APIs -- specifically the `content
-repository endpoints`_.
+Matrix serves raw user generated data in some APIs - specifically the `content
+repository endpoints <https://matrix.org/docs/spec/client_server/latest.html#get-matrix-media-r0-download-servername-mediaid>`_.
 
-.. _content repository endpoints: https://matrix.org/docs/spec/client_server/latest.html#get-matrix-media-r0-download-servername-mediaid
+Whilst we have tried to mitigate against possible XSS attacks (e.g.
+https://github.com/matrix-org/synapse/pull/1021) we recommend running
+matrix homeservers on a dedicated domain name, to limit any malicious user generated
+content served to web browsers a matrix API from being able to attack webapps hosted
+on the same domain.  This is particularly true of sharing a matrix webclient and
+server on the same domain.
 
-Whilst we make a reasonable effort to mitigate against XSS attacks (for
-instance, by using `CSP`_), a Matrix homeserver should not be hosted on a
-domain hosting other web applications. This especially applies to sharing
-the domain with Matrix web clients and other sensitive applications like
-webmail. See
-https://developer.github.com/changes/2014-04-25-user-content-security for more
-information.
-
-.. _CSP: https://github.com/matrix-org/synapse/pull/1021
-
-Ideally, the homeserver should not simply be on a different subdomain, but on
-a completely different `registered domain`_ (also known as top-level site or
-eTLD+1). This is because `some attacks`_ are still possible as long as the two
-applications share the same registered domain.
-
-.. _registered domain: https://tools.ietf.org/html/draft-ietf-httpbis-rfc6265bis-03#section-2.3
-
-.. _some attacks: https://en.wikipedia.org/wiki/Session_fixation#Attacks_using_cross-subdomain_cookie
-
-To illustrate this with an example, if your Element Web or other sensitive web
-application is hosted on ``A.example1.com``, you should ideally host Synapse on
-``example2.com``. Some amount of protection is offered by hosting on
-``B.example1.com`` instead, so this is also acceptable in some scenarios.
-However, you should *not* host your Synapse on ``A.example1.com``.
-
-Note that all of the above refers exclusively to the domain used in Synapse's
-``public_baseurl`` setting. In particular, it has no bearing on the domain
-mentioned in MXIDs hosted on that server.
-
-Following this advice ensures that even if an XSS is found in Synapse, the
-impact to other applications will be minimal.
+See https://github.com/vector-im/riot-web/issues/1977 and
+https://developer.github.com/changes/2014-04-25-user-content-security for more details.
 
 
 Upgrading an existing Synapse

UPGRADE.rst

@@ -85,78 +85,9 @@ for example:
      wget https://packages.matrix.org/debian/pool/main/m/matrix-synapse-py3/matrix-synapse-py3_1.3.0+stretch1_amd64.deb
      dpkg -i matrix-synapse-py3_1.3.0+stretch1_amd64.deb
 
-Upgrading to v1.34.0
-====================
-
-``room_invite_state_types`` configuration setting
------------------------------------------------
-
-The ``room_invite_state_types`` configuration setting has been deprecated and
-replaced with ``room_prejoin_state``. See the `sample configuration file <https://github.com/matrix-org/synapse/blob/v1.34.0/docs/sample_config.yaml#L1515>`_.
-
-If you have set ``room_invite_state_types`` to the default value you should simply
-remove it from your configuration file. The default value used to be:
-
-.. code:: yaml
-
-   room_invite_state_types:
-      - "m.room.join_rules"
-      - "m.room.canonical_alias"
-      - "m.room.avatar"
-      - "m.room.encryption"
-      - "m.room.name"
-
-If you have customised this value, you should remove ``room_invite_state_types`` and
-configure ``room_prejoin_state`` instead.
-
-
-
-Upgrading to v1.33.0
-====================
-
-Account Validity HTML templates can now display a user's expiration date
-------------------------------------------------------------------------
-
-This may affect you if you have enabled the account validity feature, and have made use of a
-custom HTML template specified by the ``account_validity.template_dir`` or ``account_validity.account_renewed_html_path``
-Synapse config options.
-
-The template can now accept an ``expiration_ts`` variable, which represents the unix timestamp in milliseconds for the
-future date of which their account has been renewed until. See the
-`default template <https://github.com/matrix-org/synapse/blob/release-v1.33.0/synapse/res/templates/account_renewed.html>`_
-for an example of usage.
-
-ALso note that a new HTML template, ``account_previously_renewed.html``, has been added. This is is shown to users
-when they attempt to renew their account with a valid renewal token that has already been used before. The default
-template contents can been found
-`here <https://github.com/matrix-org/synapse/blob/release-v1.33.0/synapse/res/templates/account_previously_renewed.html>`_,
-and can also accept an ``expiration_ts`` variable. This template replaces the error message users would previously see
-upon attempting to use a valid renewal token more than once.
-
-
 Upgrading to v1.32.0
 ====================
 
-Regression causing connected Prometheus instances to become overwhelmed
------------------------------------------------------------------------
-
-This release introduces `a regression <https://github.com/matrix-org/synapse/issues/9853>`_
-that can overwhelm connected Prometheus instances. This issue is not present in
-Synapse v1.32.0rc1.
-
-If you have been affected, please downgrade to 1.31.0. You then may need to
-remove excess writeahead logs in order for Prometheus to recover. Instructions
-for doing so are provided
-`here <https://github.com/matrix-org/synapse/pull/9854#issuecomment-823472183>`_.
-
-Dropping support for old Python, Postgres and SQLite versions
--------------------------------------------------------------
-
-In line with our `deprecation policy <https://github.com/matrix-org/synapse/blob/release-v1.32.0/docs/deprecation_policy.md>`_,
-we've dropped support for Python 3.5 and PostgreSQL 9.5, as they are no longer supported upstream.
-
-This release of Synapse requires Python 3.6+ and PostgresSQL 9.6+ or SQLite 3.22+.
-
 Removal of old List Accounts Admin API
 --------------------------------------
 
@@ -167,16 +98,6 @@ has been available since Synapse 1.7.0 (2019-12-13), and is accessible under ``G
 
 The deprecation of the old endpoint was announced with Synapse 1.28.0 (released on 2021-02-25).
 
-Application Services must use type ``m.login.application_service`` when registering users
------------------------------------------------------------------------------------------
-
-In compliance with the
-`Application Service spec <https://matrix.org/docs/spec/application_service/r0.1.2#server-admin-style-permissions>`_,
-Application Services are now required to use the ``m.login.application_service`` type when registering users via the
-``/_matrix/client/r0/register`` endpoint. This behaviour was deprecated in Synapse v1.30.0.
-
-Please ensure your Application Services are up to date.
-
 Upgrading to v1.29.0
 ====================
 

book.toml

@@ -1,39 +0,0 @@
-# Documentation for possible options in this file is at
-# https://rust-lang.github.io/mdBook/format/config.html
-[book]
-title = "Synapse"
-authors = ["The Matrix.org Foundation C.I.C."]
-language = "en"
-multilingual = false
-
-# The directory that documentation files are stored in
-src = "docs"
-
-[build]
-# Prevent markdown pages from being automatically generated when they're
-# linked to in SUMMARY.md
-create-missing = false
-
-[output.html]
-# The URL visitors will be directed to when they try to edit a page
-edit-url-template = "https://github.com/matrix-org/synapse/edit/develop/{path}"
-
-# Remove the numbers that appear before each item in the sidebar, as they can
-# get quite messy as we nest deeper
-no-section-label = true
-
-# The source code URL of the repository
-git-repository-url = "https://github.com/matrix-org/synapse"
-
-# The path that the docs are hosted on
-site-url = "/synapse/"
-
-# Additional HTML, JS, CSS that's injected into each page of the book.
-# More information available in docs/website_files/README.md
-additional-css = [
-    "docs/website_files/table-of-contents.css",
-    "docs/website_files/remove-nav-buttons.css",
-    "docs/website_files/indent-section-headers.css",
-]
-additional-js = ["docs/website_files/table-of-contents.js"]
-theme = "docs/website_files/theme"

changelog.d/9803.doc

@@ -0,0 +1 @@
+Add hardened systemd files as proposed in [#9760](https://github.com/matrix-org/synapse/issues/9760) and added them to `contrib/`. Change the docs to reflect the presence of these files.

contrib/cmdclient/http.py

@@ -1,3 +1,4 @@
+# -*- coding: utf-8 -*-
 # Copyright 2014-2016 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");

contrib/experiments/test_messaging.py

@@ -1,3 +1,4 @@
+# -*- coding: utf-8 -*-
 # Copyright 2014-2016 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");

debian/changelog

@@ -1,66 +1,8 @@
-matrix-synapse-py3 (1.36.0) stable; urgency=medium
+matrix-synapse-py3 (1.31.0+nmu1) UNRELEASED; urgency=medium
 
-  * New synapse release 1.36.0.
-
- -- Synapse Packaging team <packages@matrix.org>  Tue, 15 Jun 2021 15:41:53 +0100
-
-matrix-synapse-py3 (1.35.1) stable; urgency=medium
-
-  * New synapse release 1.35.1.
-
- -- Synapse Packaging team <packages@matrix.org>  Thu, 03 Jun 2021 08:11:29 -0400
-
-matrix-synapse-py3 (1.35.0) stable; urgency=medium
-
-  * New synapse release 1.35.0.
-
- -- Synapse Packaging team <packages@matrix.org>  Tue, 01 Jun 2021 13:23:35 +0100
-
-matrix-synapse-py3 (1.34.0) stable; urgency=medium
-
-  * New synapse release 1.34.0.
-
- -- Synapse Packaging team <packages@matrix.org>  Mon, 17 May 2021 11:34:18 +0100
-
-matrix-synapse-py3 (1.33.2) stable; urgency=medium
-
-  * New synapse release 1.33.2.
-
- -- Synapse Packaging team <packages@matrix.org>  Tue, 11 May 2021 11:17:59 +0100
-
-matrix-synapse-py3 (1.33.1) stable; urgency=medium
-
-  * New synapse release 1.33.1.
-
- -- Synapse Packaging team <packages@matrix.org>  Thu, 06 May 2021 14:06:33 +0100
-
-matrix-synapse-py3 (1.33.0) stable; urgency=medium
-
-  * New synapse release 1.33.0.
-
- -- Synapse Packaging team <packages@matrix.org>  Wed, 05 May 2021 14:15:27 +0100
-
-matrix-synapse-py3 (1.32.2) stable; urgency=medium
-
-  * New synapse release 1.32.2.
-
- -- Synapse Packaging team <packages@matrix.org>  Wed, 22 Apr 2021 12:43:52 +0100
-
-matrix-synapse-py3 (1.32.1) stable; urgency=medium
-
-  * New synapse release 1.32.1.
-
- -- Synapse Packaging team <packages@matrix.org>  Wed, 21 Apr 2021 14:00:55 +0100
-
-matrix-synapse-py3 (1.32.0) stable; urgency=medium
-
-  [ Dan Callahan ]
   * Skip tests when DEB_BUILD_OPTIONS contains "nocheck".
 
-  [ Synapse Packaging team ]
-  * New synapse release 1.32.0.
-
- -- Synapse Packaging team <packages@matrix.org>  Tue, 20 Apr 2021 14:28:39 +0100
+ -- Dan Callahan <danc@element.io>  Mon, 12 Apr 2021 13:07:36 +0000
 
 matrix-synapse-py3 (1.31.0) stable; urgency=medium
 

demo/start.sh

@@ -96,48 +96,18 @@ for port in 8080 8081 8082; do
     # Check script parameters
     if [ $# -eq 1 ]; then
         if [ $1 = "--no-rate-limit" ]; then
+            # messages rate limit
+            echo 'rc_messages_per_second: 1000' >> $DIR/etc/$port.config
+            echo 'rc_message_burst_count: 1000' >> $DIR/etc/$port.config
 
-            # Disable any rate limiting
-            ratelimiting=$(cat <<-RC
-			rc_message:
-			  per_second: 1000
-			  burst_count: 1000
-			rc_registration:
-			  per_second: 1000
-			  burst_count: 1000
-			rc_login:
-			  address:
-			    per_second: 1000
-			    burst_count: 1000
-			  account:
-			    per_second: 1000
-			    burst_count: 1000
-			  failed_attempts:
-			    per_second: 1000
-			    burst_count: 1000
-			rc_admin_redaction:
-			  per_second: 1000
-			  burst_count: 1000
-			rc_joins:
-			  local:
-			    per_second: 1000
-			    burst_count: 1000
-			  remote:
-			    per_second: 1000
-			    burst_count: 1000
-			rc_3pid_validation:
-			  per_second: 1000
-			  burst_count: 1000
-			rc_invites:
-			  per_room:
-			    per_second: 1000
-			    burst_count: 1000
-			  per_user:
-			    per_second: 1000
-			    burst_count: 1000
-			RC
-			)
-            echo "${ratelimiting}" >> $DIR/etc/$port.config
+            # registration rate limit
+            printf 'rc_registration:\n  per_second: 1000\n  burst_count: 1000\n' >> $DIR/etc/$port.config
+
+            # login rate limit
+            echo 'rc_login:' >> $DIR/etc/$port.config
+            printf '  address:\n    per_second: 1000\n    burst_count: 1000\n' >> $DIR/etc/$port.config
+            printf '  account:\n    per_second: 1000\n    burst_count: 1000\n' >> $DIR/etc/$port.config
+            printf '  failed_attempts:\n    per_second: 1000\n    burst_count: 1000\n' >> $DIR/etc/$port.config
         fi
     fi
 

docker/Dockerfile

@@ -88,5 +88,5 @@ EXPOSE 8008/tcp 8009/tcp 8448/tcp
 
 ENTRYPOINT ["/start.py"]
 
-HEALTHCHECK --start-period=5s --interval=15s --timeout=5s \
+HEALTHCHECK --interval=1m --timeout=5s \
     CMD curl -fSs http://localhost:8008/health || exit 1

docker/Dockerfile-workers

@@ -1,23 +0,0 @@
-# Inherit from the official Synapse docker image
-FROM matrixdotorg/synapse
-
-# Install deps
-RUN apt-get update
-RUN apt-get install -y supervisor redis nginx
-
-# Remove the default nginx sites
-RUN rm /etc/nginx/sites-enabled/default
-
-# Copy Synapse worker, nginx and supervisord configuration template files
-COPY ./docker/conf-workers/* /conf/
-
-# Expose nginx listener port
-EXPOSE 8080/tcp
-
-# Volume for user-editable config files, logs etc.
-VOLUME ["/data"]
-
-# A script to read environment variables and create the necessary
-# files to run the desired worker configuration. Will start supervisord.
-COPY ./docker/configure_workers_and_start.py /configure_workers_and_start.py
-ENTRYPOINT ["/configure_workers_and_start.py"]

docker/README-testing.md

@@ -1,140 +0,0 @@
-# Running tests against a dockerised Synapse
-
-It's possible to run integration tests against Synapse
-using [Complement](https://github.com/matrix-org/complement). Complement is a Matrix Spec
-compliance test suite for homeservers, and supports any homeserver docker image configured
-to listen on ports 8008/8448. This document contains instructions for building Synapse
-docker images that can be run inside Complement for testing purposes.
-
-Note that running Synapse's unit tests from within the docker image is not supported.
-
-## Testing with SQLite and single-process Synapse
-
-> Note that `scripts-dev/complement.sh` is a script that will automatically build 
-> and run an SQLite-based, single-process of Synapse against Complement.
-
-The instructions below will set up Complement testing for a single-process, 
-SQLite-based Synapse deployment.
-
-Start by building the base Synapse docker image. If you wish to run tests with the latest
-release of Synapse, instead of your current checkout, you can skip this step. From the
-root of the repository:
-
-```sh
-docker build -t matrixdotorg/synapse -f docker/Dockerfile .
-```
-
-This will build an image with the tag `matrixdotorg/synapse`.
-
-Next, build the Synapse image for Complement. You will need a local checkout 
-of Complement. Change to the root of your Complement checkout and run:
-
-```sh
-docker build -t complement-synapse -f "dockerfiles/Synapse.Dockerfile" dockerfiles
-```
-
-This will build an image with the tag `complement-synapse`, which can be handed to 
-Complement for testing via the `COMPLEMENT_BASE_IMAGE` environment variable. Refer to 
-[Complement's documentation](https://github.com/matrix-org/complement/#running) for 
-how to run the tests, as well as the various available command line flags.
-
-## Testing with PostgreSQL and single or multi-process Synapse
-
-The above docker image only supports running Synapse with SQLite and in a 
-single-process topology. The following instructions are used to build a Synapse image for 
-Complement that supports either single or multi-process topology with a PostgreSQL 
-database backend.
-
-As with the single-process image, build the base Synapse docker image. If you wish to run
-tests with the latest release of Synapse, instead of your current checkout, you can skip
-this step. From the root of the repository:
-
-```sh
-docker build -t matrixdotorg/synapse -f docker/Dockerfile .
-```
-
-This will build an image with the tag `matrixdotorg/synapse`.
-
-Next, we build a new image with worker support based on `matrixdotorg/synapse:latest`. 
-Again, from the root of the repository:
-
-```sh
-docker build -t matrixdotorg/synapse-workers -f docker/Dockerfile-workers .
-```
-
-This will build an image with the tag` matrixdotorg/synapse-workers`.
-
-It's worth noting at this point that this image is fully functional, and 
-can be used for testing against locally. See instructions for using the container 
-under
-[Running the Dockerfile-worker image standalone](#running-the-dockerfile-worker-image-standalone)
-below.
-
-Finally, build the Synapse image for Complement, which is based on
-`matrixdotorg/synapse-workers`. You will need a local checkout of Complement. Change to
-the root of your Complement checkout and run:
-
-```sh
-docker build -t matrixdotorg/complement-synapse-workers -f dockerfiles/SynapseWorkers.Dockerfile dockerfiles
-```
-
-This will build an image with the tag `complement-synapse`, which can be handed to
-Complement for testing via the `COMPLEMENT_BASE_IMAGE` environment variable. Refer to
-[Complement's documentation](https://github.com/matrix-org/complement/#running) for
-how to run the tests, as well as the various available command line flags.
-
-## Running the Dockerfile-worker image standalone
-
-For manual testing of a multi-process Synapse instance in Docker,
-[Dockerfile-workers](Dockerfile-workers) is a Dockerfile that will produce an image
-bundling all necessary components together for a workerised homeserver instance.
-
-This includes any desired Synapse worker processes, a nginx to route traffic accordingly,
-a redis for worker communication and a supervisord instance to start up and monitor all
-processes. You will need to provide your own postgres container to connect to, and TLS 
-is not handled by the container.
-
-Once you've built the image using the above instructions, you can run it. Be sure 
-you've set up a volume according to the [usual Synapse docker instructions](README.md).
-Then run something along the lines of:
-
-```
-docker run -d --name synapse \
-    --mount type=volume,src=synapse-data,dst=/data \
-    -p 8008:8008 \
-    -e SYNAPSE_SERVER_NAME=my.matrix.host \
-    -e SYNAPSE_REPORT_STATS=no \
-    -e POSTGRES_HOST=postgres \
-    -e POSTGRES_USER=postgres \
-    -e POSTGRES_PASSWORD=somesecret \
-    -e SYNAPSE_WORKER_TYPES=synchrotron,media_repository,user_dir \
-    -e SYNAPSE_WORKERS_WRITE_LOGS_TO_DISK=1 \
-    matrixdotorg/synapse-workers
-```
-
-...substituting `POSTGRES*` variables for those that match a postgres host you have 
-available (usually a running postgres docker container).
-
-The `SYNAPSE_WORKER_TYPES` environment variable is a comma-separated list of workers to
-use when running the container. All possible worker names are defined by the keys of the
-`WORKERS_CONFIG` variable in [this script](configure_workers_and_start.py), which the
-Dockerfile makes use of to generate appropriate worker, nginx and supervisord config
-files.
-
-Sharding is supported for a subset of workers, in line with the
-[worker documentation](../docs/workers.md). To run multiple instances of a given worker
-type, simply specify the type multiple times in `SYNAPSE_WORKER_TYPES`
-(e.g `SYNAPSE_WORKER_TYPES=event_creator,event_creator...`).
-
-Otherwise, `SYNAPSE_WORKER_TYPES` can either be left empty or unset to spawn no workers
-(leaving only the main process). The container is configured to use redis-based worker
-mode.
-
-Logs for workers and the main process are logged to stdout and can be viewed with 
-standard `docker logs` tooling. Worker logs contain their worker name 
-after the timestamp.
-
-Setting `SYNAPSE_WORKERS_WRITE_LOGS_TO_DISK=1` will cause worker logs to be written to
-`<data_dir>/logs/<worker_name>.log`. Logs are kept for 1 week and rotate every day at 00:
-00, according to the container's clock. Logging for the main process must still be 
-configured by modifying the homeserver's log config in your Synapse data volume.

docker/README.md

@@ -2,16 +2,13 @@
 
 This Docker image will run Synapse as a single process. By default it uses a
 sqlite database; for production use you should connect it to a separate
-postgres database. The image also does *not* provide a TURN server.
+postgres database.
 
-This image should work on all platforms that are supported by Docker upstream.
-Note that Docker's WS1-backend Linux Containers on Windows
-platform is [experimental](https://github.com/docker/for-win/issues/6470) and
-is not supported by this image.
+The image also does *not* provide a TURN server.
 
 ## Volumes
 
-By default, the image expects a single volume, located at `/data`, that will hold:
+By default, the image expects a single volume, located at ``/data``, that will hold:
 
 * configuration files;
 * uploaded media and thumbnails;
@@ -19,11 +16,11 @@ By default, the image expects a single volume, located at `/data`, that will hol
 * the appservices configuration.
 
 You are free to use separate volumes depending on storage endpoints at your
-disposal. For instance, `/data/media` could be stored on a large but low
+disposal. For instance, ``/data/media`` could be stored on a large but low
 performance hdd storage while other files could be stored on high performance
 endpoints.
 
-In order to setup an application service, simply create an `appservices`
+In order to setup an application service, simply create an ``appservices``
 directory in the data volume and write the application service Yaml
 configuration file there. Multiple application services are supported.
 
@@ -56,8 +53,6 @@ The following environment variables are supported in `generate` mode:
 * `SYNAPSE_SERVER_NAME` (mandatory): the server public hostname.
 * `SYNAPSE_REPORT_STATS` (mandatory, `yes` or `no`): whether to enable
   anonymous statistics reporting.
-* `SYNAPSE_HTTP_PORT`: the port Synapse should listen on for http traffic.
-      Defaults to `8008`.
 * `SYNAPSE_CONFIG_DIR`: where additional config files (such as the log config
   and event signing key) will be stored. Defaults to `/data`.
 * `SYNAPSE_CONFIG_PATH`: path to the file to be generated. Defaults to
@@ -78,8 +73,6 @@ docker run -d --name synapse \
     matrixdotorg/synapse:latest
 ```
 
-(assuming 8008 is the port Synapse is configured to listen on for http traffic.)
-
 You can then check that it has started correctly with:
 
 ```
@@ -191,16 +184,6 @@ whilst running the above `docker run` commands.
 ```
    --no-healthcheck
 ```
-
-## Disabling the healthcheck in docker-compose file
-
-If you wish to disable the healthcheck via docker-compose, append the following to your service configuration.
-
-```
-  healthcheck:
-    disable: true
-```
-
 ## Setting custom healthcheck on docker run
 
 If you wish to point the healthcheck at a different port with docker command, add the following
@@ -212,18 +195,17 @@ If you wish to point the healthcheck at a different port with docker command, ad
 ## Setting the healthcheck in docker-compose file
 
 You can add the following to set a custom healthcheck in a docker compose file.
-You will need docker-compose version >2.1 for this to work. 
+You will need version >2.1 for this to work. 
 
 ```
 healthcheck:
   test: ["CMD", "curl", "-fSs", "http://localhost:8008/health"]
-  interval: 15s
-  timeout: 5s
+  interval: 1m
+  timeout: 10s
   retries: 3
-  start_period: 5s
 ```
 
 ## Using jemalloc
 
 Jemalloc is embedded in the image and will be used instead of the default allocator.
-You can read about jemalloc by reading the Synapse [README](../README.rst).
+You can read about jemalloc by reading the Synapse [README](../README.md)

docker/conf-workers/nginx.conf.j2

@@ -1,27 +0,0 @@
-# This file contains the base config for the reverse proxy, as part of ../Dockerfile-workers.
-# configure_workers_and_start.py uses and amends to this file depending on the workers
-# that have been selected.
-
-{{ upstream_directives }}
-
-server {
-    # Listen on an unoccupied port number
-    listen 8008;
-    listen [::]:8008;
-
-    server_name localhost;
-
-    # Nginx by default only allows file uploads up to 1M in size
-    # Increase client_max_body_size to match max_upload_size defined in homeserver.yaml
-    client_max_body_size 100M;
-
-{{ worker_locations }}
-
-    # Send all other traffic to the main process
-    location ~* ^(\\/_matrix|\\/_synapse) {
-        proxy_pass http://localhost:8080;
-        proxy_set_header X-Forwarded-For $remote_addr;
-        proxy_set_header X-Forwarded-Proto $scheme;
-        proxy_set_header Host $host;
-    }
-}

docker/conf-workers/shared.yaml.j2

@@ -1,9 +0,0 @@
-# This file contains the base for the shared homeserver config file between Synapse workers,
-# as part of ./Dockerfile-workers.
-# configure_workers_and_start.py uses and amends to this file depending on the workers
-# that have been selected.
-
-redis:
-    enabled: true
-
-{{ shared_worker_config }}

docker/conf-workers/supervisord.conf.j2

@@ -1,41 +0,0 @@
-# This file contains the base config for supervisord, as part of ../Dockerfile-workers.
-# configure_workers_and_start.py uses and amends to this file depending on the workers
-# that have been selected.
-[supervisord]
-nodaemon=true
-user=root
-
-[program:nginx]
-command=/usr/sbin/nginx -g "daemon off;"
-priority=500
-stdout_logfile=/dev/stdout
-stdout_logfile_maxbytes=0
-stderr_logfile=/dev/stderr
-stderr_logfile_maxbytes=0
-username=www-data
-autorestart=true
-
-[program:redis]
-command=/usr/bin/redis-server /etc/redis/redis.conf --daemonize no
-priority=1
-stdout_logfile=/dev/stdout
-stdout_logfile_maxbytes=0
-stderr_logfile=/dev/stderr
-stderr_logfile_maxbytes=0
-username=redis
-autorestart=true
-
-[program:synapse_main]
-command=/usr/local/bin/python -m synapse.app.homeserver --config-path="{{ main_config_path }}" --config-path=/conf/workers/shared.yaml
-priority=10
-# Log startup failures to supervisord's stdout/err
-# Regular synapse logs will still go in the configured data directory
-stdout_logfile=/dev/stdout
-stdout_logfile_maxbytes=0
-stderr_logfile=/dev/stderr
-stderr_logfile_maxbytes=0
-autorestart=unexpected
-exitcodes=0
-
-# Additional process blocks
-{{ worker_config }}

docker/conf-workers/worker.yaml.j2

@@ -1,26 +0,0 @@
-# This is a configuration template for a single worker instance, and is
-# used by Dockerfile-workers.
-# Values will be change depending on whichever workers are selected when
-# running that image.
-
-worker_app: "{{ app }}"
-worker_name: "{{ name }}"
-
-# The replication listener on the main synapse process.
-worker_replication_host: 127.0.0.1
-worker_replication_http_port: 9093
-
-worker_listeners:
-  - type: http
-    port: {{ port }}
-{% if listener_resources %}
-    resources:
-      - names:
-{%- for resource in listener_resources %}
-        - {{ resource }}
-{%- endfor %}
-{% endif %}
-
-worker_log_config: {{ worker_log_config_filepath }}
-
-{{ worker_extra_conf }}

docker/conf/homeserver.yaml

@@ -40,9 +40,7 @@ listeners:
         compress: false
   {% endif %}
 
-  # Allow configuring in case we want to reverse proxy 8008
-  # using another process in the same container
-  - port: {{ SYNAPSE_HTTP_PORT or 8008 }}
+  - port: 8008
     tls: false
     bind_addresses: ['::']
     type: http

docker/conf/log.config

@@ -2,36 +2,9 @@ version: 1
 
 formatters:
   precise:
-{% if worker_name %}
-    format: '%(asctime)s - worker:{{ worker_name }} - %(name)s - %(lineno)d - %(levelname)s - %(request)s - %(message)s'
-{% else %}
-    format: '%(asctime)s - %(name)s - %(lineno)d - %(levelname)s - %(request)s - %(message)s'
-{% endif %}
+   format: '%(asctime)s - %(name)s - %(lineno)d - %(levelname)s - %(request)s - %(message)s'
 
 handlers:
-{% if LOG_FILE_PATH %}
-  file:
-    class: logging.handlers.TimedRotatingFileHandler
-    formatter: precise
-    filename: {{ LOG_FILE_PATH }}
-    when: "midnight"
-    backupCount: 6  # Does not include the current log file.
-    encoding: utf8
-
-  # Default to buffering writes to log file for efficiency. This means that
-  # there will be a delay for INFO/DEBUG logs to get written, but WARNING/ERROR
-  # logs will still be flushed immediately.
-  buffer:
-    class: logging.handlers.MemoryHandler
-    target: file
-    # The capacity is the number of log lines that are buffered before
-    # being written to disk. Increasing this will lead to better
-    # performance, at the expensive of it taking longer for log lines to
-    # be written to disk.
-    capacity: 10
-    flushLevel: 30  # Flush for WARNING logs as well
-{% endif %}
-
   console:
     class: logging.StreamHandler
     formatter: precise
@@ -44,11 +17,6 @@ loggers:
 
 root:
     level: {{ SYNAPSE_LOG_LEVEL or "INFO" }}
-
-{% if LOG_FILE_PATH %}
-    handlers: [console, buffer]
-{% else %}
     handlers: [console]
-{% endif %}
 
 disable_existing_loggers: false

docker/configure_workers_and_start.py

@@ -1,558 +0,0 @@
-#!/usr/bin/env python
-# Copyright 2021 The Matrix.org Foundation C.I.C.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# This script reads environment variables and generates a shared Synapse worker,
-# nginx and supervisord configs depending on the workers requested.
-#
-# The environment variables it reads are:
-#   * SYNAPSE_SERVER_NAME: The desired server_name of the homeserver.
-#   * SYNAPSE_REPORT_STATS: Whether to report stats.
-#   * SYNAPSE_WORKER_TYPES: A comma separated list of worker names as specified in WORKER_CONFIG
-#         below. Leave empty for no workers, or set to '*' for all possible workers.
-#
-# NOTE: According to Complement's ENTRYPOINT expectations for a homeserver image (as defined
-# in the project's README), this script may be run multiple times, and functionality should
-# continue to work if so.
-
-import os
-import subprocess
-import sys
-
-import jinja2
-import yaml
-
-MAIN_PROCESS_HTTP_LISTENER_PORT = 8080
-
-
-WORKERS_CONFIG = {
-    "pusher": {
-        "app": "synapse.app.pusher",
-        "listener_resources": [],
-        "endpoint_patterns": [],
-        "shared_extra_conf": {"start_pushers": False},
-        "worker_extra_conf": "",
-    },
-    "user_dir": {
-        "app": "synapse.app.user_dir",
-        "listener_resources": ["client"],
-        "endpoint_patterns": [
-            "^/_matrix/client/(api/v1|r0|unstable)/user_directory/search$"
-        ],
-        "shared_extra_conf": {"update_user_directory": False},
-        "worker_extra_conf": "",
-    },
-    "media_repository": {
-        "app": "synapse.app.media_repository",
-        "listener_resources": ["media"],
-        "endpoint_patterns": [
-            "^/_matrix/media/",
-            "^/_synapse/admin/v1/purge_media_cache$",
-            "^/_synapse/admin/v1/room/.*/media.*$",
-            "^/_synapse/admin/v1/user/.*/media.*$",
-            "^/_synapse/admin/v1/media/.*$",
-            "^/_synapse/admin/v1/quarantine_media/.*$",
-        ],
-        "shared_extra_conf": {"enable_media_repo": False},
-        "worker_extra_conf": "enable_media_repo: true",
-    },
-    "appservice": {
-        "app": "synapse.app.appservice",
-        "listener_resources": [],
-        "endpoint_patterns": [],
-        "shared_extra_conf": {"notify_appservices": False},
-        "worker_extra_conf": "",
-    },
-    "federation_sender": {
-        "app": "synapse.app.federation_sender",
-        "listener_resources": [],
-        "endpoint_patterns": [],
-        "shared_extra_conf": {"send_federation": False},
-        "worker_extra_conf": "",
-    },
-    "synchrotron": {
-        "app": "synapse.app.generic_worker",
-        "listener_resources": ["client"],
-        "endpoint_patterns": [
-            "^/_matrix/client/(v2_alpha|r0)/sync$",
-            "^/_matrix/client/(api/v1|v2_alpha|r0)/events$",
-            "^/_matrix/client/(api/v1|r0)/initialSync$",
-            "^/_matrix/client/(api/v1|r0)/rooms/[^/]+/initialSync$",
-        ],
-        "shared_extra_conf": {},
-        "worker_extra_conf": "",
-    },
-    "federation_reader": {
-        "app": "synapse.app.generic_worker",
-        "listener_resources": ["federation"],
-        "endpoint_patterns": [
-            "^/_matrix/federation/(v1|v2)/event/",
-            "^/_matrix/federation/(v1|v2)/state/",
-            "^/_matrix/federation/(v1|v2)/state_ids/",
-            "^/_matrix/federation/(v1|v2)/backfill/",
-            "^/_matrix/federation/(v1|v2)/get_missing_events/",
-            "^/_matrix/federation/(v1|v2)/publicRooms",
-            "^/_matrix/federation/(v1|v2)/query/",
-            "^/_matrix/federation/(v1|v2)/make_join/",
-            "^/_matrix/federation/(v1|v2)/make_leave/",
-            "^/_matrix/federation/(v1|v2)/send_join/",
-            "^/_matrix/federation/(v1|v2)/send_leave/",
-            "^/_matrix/federation/(v1|v2)/invite/",
-            "^/_matrix/federation/(v1|v2)/query_auth/",
-            "^/_matrix/federation/(v1|v2)/event_auth/",
-            "^/_matrix/federation/(v1|v2)/exchange_third_party_invite/",
-            "^/_matrix/federation/(v1|v2)/user/devices/",
-            "^/_matrix/federation/(v1|v2)/get_groups_publicised$",
-            "^/_matrix/key/v2/query",
-        ],
-        "shared_extra_conf": {},
-        "worker_extra_conf": "",
-    },
-    "federation_inbound": {
-        "app": "synapse.app.generic_worker",
-        "listener_resources": ["federation"],
-        "endpoint_patterns": ["/_matrix/federation/(v1|v2)/send/"],
-        "shared_extra_conf": {},
-        "worker_extra_conf": "",
-    },
-    "event_persister": {
-        "app": "synapse.app.generic_worker",
-        "listener_resources": ["replication"],
-        "endpoint_patterns": [],
-        "shared_extra_conf": {},
-        "worker_extra_conf": "",
-    },
-    "background_worker": {
-        "app": "synapse.app.generic_worker",
-        "listener_resources": [],
-        "endpoint_patterns": [],
-        # This worker cannot be sharded. Therefore there should only ever be one background
-        # worker, and it should be named background_worker1
-        "shared_extra_conf": {"run_background_tasks_on": "background_worker1"},
-        "worker_extra_conf": "",
-    },
-    "event_creator": {
-        "app": "synapse.app.generic_worker",
-        "listener_resources": ["client"],
-        "endpoint_patterns": [
-            "^/_matrix/client/(api/v1|r0|unstable)/rooms/.*/redact",
-            "^/_matrix/client/(api/v1|r0|unstable)/rooms/.*/send",
-            "^/_matrix/client/(api/v1|r0|unstable)/rooms/.*/(join|invite|leave|ban|unban|kick)$",
-            "^/_matrix/client/(api/v1|r0|unstable)/join/",
-            "^/_matrix/client/(api/v1|r0|unstable)/profile/",
-        ],
-        "shared_extra_conf": {},
-        "worker_extra_conf": "",
-    },
-    "frontend_proxy": {
-        "app": "synapse.app.frontend_proxy",
-        "listener_resources": ["client", "replication"],
-        "endpoint_patterns": ["^/_matrix/client/(api/v1|r0|unstable)/keys/upload"],
-        "shared_extra_conf": {},
-        "worker_extra_conf": (
-            "worker_main_http_uri: http://127.0.0.1:%d"
-            % (MAIN_PROCESS_HTTP_LISTENER_PORT,),
-        ),
-    },
-}
-
-# Templates for sections that may be inserted multiple times in config files
-SUPERVISORD_PROCESS_CONFIG_BLOCK = """
-[program:synapse_{name}]
-command=/usr/local/bin/python -m {app} \
-    --config-path="{config_path}" \
-    --config-path=/conf/workers/shared.yaml \
-    --config-path=/conf/workers/{name}.yaml
-autorestart=unexpected
-priority=500
-exitcodes=0
-stdout_logfile=/dev/stdout
-stdout_logfile_maxbytes=0
-stderr_logfile=/dev/stderr
-stderr_logfile_maxbytes=0
-"""
-
-NGINX_LOCATION_CONFIG_BLOCK = """
-    location ~* {endpoint} {{
-        proxy_pass {upstream};
-        proxy_set_header X-Forwarded-For $remote_addr;
-        proxy_set_header X-Forwarded-Proto $scheme;
-        proxy_set_header Host $host;
-    }}
-"""
-
-NGINX_UPSTREAM_CONFIG_BLOCK = """
-upstream {upstream_worker_type} {{
-{body}
-}}
-"""
-
-
-# Utility functions
-def log(txt: str):
-    """Log something to the stdout.
-
-    Args:
-        txt: The text to log.
-    """
-    print(txt)
-
-
-def error(txt: str):
-    """Log something and exit with an error code.
-
-    Args:
-        txt: The text to log in error.
-    """
-    log(txt)
-    sys.exit(2)
-
-
-def convert(src: str, dst: str, **template_vars):
-    """Generate a file from a template
-
-    Args:
-        src: Path to the input file.
-        dst: Path to write to.
-        template_vars: The arguments to replace placeholder variables in the template with.
-    """
-    # Read the template file
-    with open(src) as infile:
-        template = infile.read()
-
-    # Generate a string from the template. We disable autoescape to prevent template
-    # variables from being escaped.
-    rendered = jinja2.Template(template, autoescape=False).render(**template_vars)
-
-    # Write the generated contents to a file
-    #
-    # We use append mode in case the files have already been written to by something else
-    # (for instance, as part of the instructions in a dockerfile).
-    with open(dst, "a") as outfile:
-        # In case the existing file doesn't end with a newline
-        outfile.write("\n")
-
-        outfile.write(rendered)
-
-
-def add_sharding_to_shared_config(
-    shared_config: dict,
-    worker_type: str,
-    worker_name: str,
-    worker_port: int,
-) -> None:
-    """Given a dictionary representing a config file shared across all workers,
-    append sharded worker information to it for the current worker_type instance.
-
-    Args:
-        shared_config: The config dict that all worker instances share (after being converted to YAML)
-        worker_type: The type of worker (one of those defined in WORKERS_CONFIG).
-        worker_name: The name of the worker instance.
-        worker_port: The HTTP replication port that the worker instance is listening on.
-    """
-    # The instance_map config field marks the workers that write to various replication streams
-    instance_map = shared_config.setdefault("instance_map", {})
-
-    # Worker-type specific sharding config
-    if worker_type == "pusher":
-        shared_config.setdefault("pusher_instances", []).append(worker_name)
-
-    elif worker_type == "federation_sender":
-        shared_config.setdefault("federation_sender_instances", []).append(worker_name)
-
-    elif worker_type == "event_persister":
-        # Event persisters write to the events stream, so we need to update
-        # the list of event stream writers
-        shared_config.setdefault("stream_writers", {}).setdefault("events", []).append(
-            worker_name
-        )
-
-        # Map of stream writer instance names to host/ports combos
-        instance_map[worker_name] = {
-            "host": "localhost",
-            "port": worker_port,
-        }
-
-    elif worker_type == "media_repository":
-        # The first configured media worker will run the media background jobs
-        shared_config.setdefault("media_instance_running_background_jobs", worker_name)
-
-
-def generate_base_homeserver_config():
-    """Starts Synapse and generates a basic homeserver config, which will later be
-    modified for worker support.
-
-    Raises: CalledProcessError if calling start.py returned a non-zero exit code.
-    """
-    # start.py already does this for us, so just call that.
-    # note that this script is copied in in the official, monolith dockerfile
-    os.environ["SYNAPSE_HTTP_PORT"] = str(MAIN_PROCESS_HTTP_LISTENER_PORT)
-    subprocess.check_output(["/usr/local/bin/python", "/start.py", "migrate_config"])
-
-
-def generate_worker_files(environ, config_path: str, data_dir: str):
-    """Read the desired list of workers from environment variables and generate
-    shared homeserver, nginx and supervisord configs.
-
-    Args:
-        environ: _Environ[str]
-        config_path: Where to output the generated Synapse main worker config file.
-        data_dir: The location of the synapse data directory. Where log and
-            user-facing config files live.
-    """
-    # Note that yaml cares about indentation, so care should be taken to insert lines
-    # into files at the correct indentation below.
-
-    # shared_config is the contents of a Synapse config file that will be shared amongst
-    # the main Synapse process as well as all workers.
-    # It is intended mainly for disabling functionality when certain workers are spun up,
-    # and adding a replication listener.
-
-    # First read the original config file and extract the listeners block. Then we'll add
-    # another listener for replication. Later we'll write out the result.
-    listeners = [
-        {
-            "port": 9093,
-            "bind_address": "127.0.0.1",
-            "type": "http",
-            "resources": [{"names": ["replication"]}],
-        }
-    ]
-    with open(config_path) as file_stream:
-        original_config = yaml.safe_load(file_stream)
-        original_listeners = original_config.get("listeners")
-        if original_listeners:
-            listeners += original_listeners
-
-    # The shared homeserver config. The contents of which will be inserted into the
-    # base shared worker jinja2 template.
-    #
-    # This config file will be passed to all workers, included Synapse's main process.
-    shared_config = {"listeners": listeners}
-
-    # The supervisord config. The contents of which will be inserted into the
-    # base supervisord jinja2 template.
-    #
-    # Supervisord will be in charge of running everything, from redis to nginx to Synapse
-    # and all of its worker processes. Load the config template, which defines a few
-    # services that are necessary to run.
-    supervisord_config = ""
-
-    # Upstreams for load-balancing purposes. This dict takes the form of a worker type to the
-    # ports of each worker. For example:
-    # {
-    #   worker_type: {1234, 1235, ...}}
-    # }
-    # and will be used to construct 'upstream' nginx directives.
-    nginx_upstreams = {}
-
-    # A map of: {"endpoint": "upstream"}, where "upstream" is a str representing what will be
-    # placed after the proxy_pass directive. The main benefit to representing this data as a
-    # dict over a str is that we can easily deduplicate endpoints across multiple instances
-    # of the same worker.
-    #
-    # An nginx site config that will be amended to depending on the workers that are
-    # spun up. To be placed in /etc/nginx/conf.d.
-    nginx_locations = {}
-
-    # Read the desired worker configuration from the environment
-    worker_types = environ.get("SYNAPSE_WORKER_TYPES")
-    if worker_types is None:
-        # No workers, just the main process
-        worker_types = []
-    else:
-        # Split type names by comma
-        worker_types = worker_types.split(",")
-
-    # Create the worker configuration directory if it doesn't already exist
-    os.makedirs("/conf/workers", exist_ok=True)
-
-    # Start worker ports from this arbitrary port
-    worker_port = 18009
-
-    # A counter of worker_type -> int. Used for determining the name for a given
-    # worker type when generating its config file, as each worker's name is just
-    # worker_type + instance #
-    worker_type_counter = {}
-
-    # For each worker type specified by the user, create config values
-    for worker_type in worker_types:
-        worker_type = worker_type.strip()
-
-        worker_config = WORKERS_CONFIG.get(worker_type)
-        if worker_config:
-            worker_config = worker_config.copy()
-        else:
-            log(worker_type + " is an unknown worker type! It will be ignored")
-            continue
-
-        new_worker_count = worker_type_counter.setdefault(worker_type, 0) + 1
-        worker_type_counter[worker_type] = new_worker_count
-
-        # Name workers by their type concatenated with an incrementing number
-        # e.g. federation_reader1
-        worker_name = worker_type + str(new_worker_count)
-        worker_config.update(
-            {"name": worker_name, "port": worker_port, "config_path": config_path}
-        )
-
-        # Update the shared config with any worker-type specific options
-        shared_config.update(worker_config["shared_extra_conf"])
-
-        # Check if more than one instance of this worker type has been specified
-        worker_type_total_count = worker_types.count(worker_type)
-        if worker_type_total_count > 1:
-            # Update the shared config with sharding-related options if necessary
-            add_sharding_to_shared_config(
-                shared_config, worker_type, worker_name, worker_port
-            )
-
-        # Enable the worker in supervisord
-        supervisord_config += SUPERVISORD_PROCESS_CONFIG_BLOCK.format_map(worker_config)
-
-        # Add nginx location blocks for this worker's endpoints (if any are defined)
-        for pattern in worker_config["endpoint_patterns"]:
-            # Determine whether we need to load-balance this worker
-            if worker_type_total_count > 1:
-                # Create or add to a load-balanced upstream for this worker
-                nginx_upstreams.setdefault(worker_type, set()).add(worker_port)
-
-                # Upstreams are named after the worker_type
-                upstream = "http://" + worker_type
-            else:
-                upstream = "http://localhost:%d" % (worker_port,)
-
-            # Note that this endpoint should proxy to this upstream
-            nginx_locations[pattern] = upstream
-
-        # Write out the worker's logging config file
-
-        # Check whether we should write worker logs to disk, in addition to the console
-        extra_log_template_args = {}
-        if environ.get("SYNAPSE_WORKERS_WRITE_LOGS_TO_DISK"):
-            extra_log_template_args["LOG_FILE_PATH"] = "{dir}/logs/{name}.log".format(
-                dir=data_dir, name=worker_name
-            )
-
-        # Render and write the file
-        log_config_filepath = "/conf/workers/{name}.log.config".format(name=worker_name)
-        convert(
-            "/conf/log.config",
-            log_config_filepath,
-            worker_name=worker_name,
-            **extra_log_template_args,
-        )
-
-        # Then a worker config file
-        convert(
-            "/conf/worker.yaml.j2",
-            "/conf/workers/{name}.yaml".format(name=worker_name),
-            **worker_config,
-            worker_log_config_filepath=log_config_filepath,
-        )
-
-        worker_port += 1
-
-    # Build the nginx location config blocks
-    nginx_location_config = ""
-    for endpoint, upstream in nginx_locations.items():
-        nginx_location_config += NGINX_LOCATION_CONFIG_BLOCK.format(
-            endpoint=endpoint,
-            upstream=upstream,
-        )
-
-    # Determine the load-balancing upstreams to configure
-    nginx_upstream_config = ""
-    for upstream_worker_type, upstream_worker_ports in nginx_upstreams.items():
-        body = ""
-        for port in upstream_worker_ports:
-            body += "    server localhost:%d;\n" % (port,)
-
-        # Add to the list of configured upstreams
-        nginx_upstream_config += NGINX_UPSTREAM_CONFIG_BLOCK.format(
-            upstream_worker_type=upstream_worker_type,
-            body=body,
-        )
-
-    # Finally, we'll write out the config files.
-
-    # Shared homeserver config
-    convert(
-        "/conf/shared.yaml.j2",
-        "/conf/workers/shared.yaml",
-        shared_worker_config=yaml.dump(shared_config),
-    )
-
-    # Nginx config
-    convert(
-        "/conf/nginx.conf.j2",
-        "/etc/nginx/conf.d/matrix-synapse.conf",
-        worker_locations=nginx_location_config,
-        upstream_directives=nginx_upstream_config,
-    )
-
-    # Supervisord config
-    convert(
-        "/conf/supervisord.conf.j2",
-        "/etc/supervisor/conf.d/supervisord.conf",
-        main_config_path=config_path,
-        worker_config=supervisord_config,
-    )
-
-    # Ensure the logging directory exists
-    log_dir = data_dir + "/logs"
-    if not os.path.exists(log_dir):
-        os.mkdir(log_dir)
-
-
-def start_supervisord():
-    """Starts up supervisord which then starts and monitors all other necessary processes
-
-    Raises: CalledProcessError if calling start.py return a non-zero exit code.
-    """
-    subprocess.run(["/usr/bin/supervisord"], stdin=subprocess.PIPE)
-
-
-def main(args, environ):
-    config_dir = environ.get("SYNAPSE_CONFIG_DIR", "/data")
-    config_path = environ.get("SYNAPSE_CONFIG_PATH", config_dir + "/homeserver.yaml")
-    data_dir = environ.get("SYNAPSE_DATA_DIR", "/data")
-
-    # override SYNAPSE_NO_TLS, we don't support TLS in worker mode,
-    # this needs to be handled by a frontend proxy
-    environ["SYNAPSE_NO_TLS"] = "yes"
-
-    # Generate the base homeserver config if one does not yet exist
-    if not os.path.exists(config_path):
-        log("Generating base homeserver config")
-        generate_base_homeserver_config()
-
-    # This script may be run multiple times (mostly by Complement, see note at top of file).
-    # Don't re-configure workers in this instance.
-    mark_filepath = "/conf/workers_have_been_configured"
-    if not os.path.exists(mark_filepath):
-        # Always regenerate all other config files
-        generate_worker_files(environ, config_path, data_dir)
-
-        # Mark workers as being configured
-        with open(mark_filepath, "w") as f:
-            f.write("")
-
-    # Start supervisord, which will start Synapse, all of the configured worker
-    # processes, redis, nginx etc. according to the config we created above.
-    start_supervisord()
-
-
-if __name__ == "__main__":
-    main(sys.argv, os.environ)

docs/CAPTCHA_SETUP.md

@@ -1,37 +1,31 @@
 # Overview
-A captcha can be enabled on your homeserver to help prevent bots from registering
-accounts. Synapse currently uses Google's reCAPTCHA service which requires API keys
-from Google.
+Captcha can be enabled for this home server. This file explains how to do that.
+The captcha mechanism used is Google's ReCaptcha. This requires API keys from Google.
 
-## Getting API keys
+## Getting keys
+
+Requires a site/secret key pair from:
+
+<https://developers.google.com/recaptcha/>
+
+Must be a reCAPTCHA v2 key using the "I'm not a robot" Checkbox option
+
+## Setting ReCaptcha Keys
+
+The keys are a config option on the home server config. If they are not
+visible, you can generate them via `--generate-config`. Set the following value:
 
-1. Create a new site at <https://www.google.com/recaptcha/admin/create>
-1. Set the label to anything you want
-1. Set the type to reCAPTCHA v2 using the "I'm not a robot" Checkbox option.
-This is the only type of captcha that works with Synapse.
-1. Add the public hostname for your server, as set in `public_baseurl`
-in `homeserver.yaml`, to the list of authorized domains. If you have not set
-`public_baseurl`, use `server_name`.
-1. Agree to the terms of service and submit.
-1. Copy your site key and secret key and add them to your `homeserver.yaml`
-configuration file
-    ```
     recaptcha_public_key: YOUR_SITE_KEY
     recaptcha_private_key: YOUR_SECRET_KEY
-    ```
-1. Enable the CAPTCHA for new registrations
-    ```
+
+In addition, you MUST enable captchas via:
+
     enable_registration_captcha: true
-    ```
-1. Go to the settings page for the CAPTCHA you just created
-1. Uncheck the "Verify the origin of reCAPTCHA solutions" checkbox so that the
-captcha can be displayed in any client. If you do not disable this option then you
-must specify the domains of every client that is allowed to display the CAPTCHA.
 
 ## Configuring IP used for auth
 
-The reCAPTCHA API requires that the IP address of the user who solved the
-CAPTCHA is sent. If the client is connecting through a proxy or load balancer,
+The ReCaptcha API requires that the IP address of the user who solved the
+captcha is sent. If the client is connecting through a proxy or load balancer,
 it may be required to use the `X-Forwarded-For` (XFF) header instead of the origin
 IP address. This can be configured using the `x_forwarded` directive in the
-listeners section of the `homeserver.yaml` configuration file.
+listeners section of the homeserver.yaml configuration file.

docs/README.md

@@ -1,72 +1,7 @@
 # Synapse Documentation
 
-**The documentation is currently hosted [here](https://matrix-org.github.io/synapse).**
-Please update any links to point to the new website instead.
+This directory contains documentation specific to the `synapse` homeserver.
 
-## About
+All matrix-generic documentation now lives in its own project, located at [matrix-org/matrix-doc](https://github.com/matrix-org/matrix-doc)
 
-This directory currently holds a series of markdown files documenting how to install, use
-and develop Synapse, the reference Matrix homeserver. The documentation is readable directly
-from this repository, but it is recommended to instead browse through the
-[website](https://matrix-org.github.io/synapse) for easier discoverability.
-
-## Adding to the documentation
-
-Most of the documentation currently exists as top-level files, as when organising them into
-a structured website, these files were kept in place so that existing links would not break.
-The rest of the documentation is stored in folders, such as `setup`, `usage`, and `development`
-etc. **All new documentation files should be placed in structured folders.** For example:
-
-To create a new user-facing documentation page about a new Single Sign-On protocol named
-"MyCoolProtocol", one should create a new file with a relevant name, such as "my_cool_protocol.md".
-This file might fit into the documentation structure at:
-
-- Usage
-  - Configuration
-    - User Authentication
-      - Single Sign-On
-        - **My Cool Protocol**
-
-Given that, one would place the new file under
-`usage/configuration/user_authentication/single_sign_on/my_cool_protocol.md`.
-
-Note that the structure of the documentation (and thus the left sidebar on the website) is determined
-by the list in [SUMMARY.md](SUMMARY.md). The final thing to do when adding a new page is to add a new
-line linking to the new documentation file:
-
-```markdown
-- [My Cool Protocol](usage/configuration/user_authentication/single_sign_on/my_cool_protocol.md)
-```
-
-## Building the documentation
-
-The documentation is built with [mdbook](https://rust-lang.github.io/mdBook/), and the outline of the
-documentation is determined by the structure of [SUMMARY.md](SUMMARY.md).
-
-First, [get mdbook](https://github.com/rust-lang/mdBook#installation). Then, **from the root of the repository**,
-build the documentation with:
-
-```sh
-mdbook build
-```
-
-The rendered contents will be outputted to a new `book/` directory at the root of the repository. You can
-browse the book by opening `book/index.html` in a web browser.
-
-You can also have mdbook host the docs on a local webserver with hot-reload functionality via:
-
-```sh
-mdbook serve
-```
-
-The URL at which the docs can be viewed at will be logged.
-
-## Configuration and theming
-
-The look and behaviour of the website is configured by the [book.toml](../book.toml) file
-at the root of the repository. See
-[mdbook's documentation on configuration](https://rust-lang.github.io/mdBook/format/config.html)
-for available options.
-
-The site can be themed and additionally extended with extra UI and features. See
-[website_files/README.md](website_files/README.md) for details.
+(Note:  some items here may be moved to [matrix-org/matrix-doc](https://github.com/matrix-org/matrix-doc) at some point in the future.)

docs/SUMMARY.md

@@ -1,87 +0,0 @@
-# Summary
-
-# Introduction
-- [Welcome and Overview](welcome_and_overview.md)
-
-# Setup
-  - [Installation](setup/installation.md)
-  - [Using Postgres](postgres.md)
-  - [Configuring a Reverse Proxy](reverse_proxy.md)
-  - [Configuring a Turn Server](turn-howto.md)
-  - [Delegation](delegate.md)
-
-# Upgrading
-  - [Upgrading between Synapse Versions](upgrading/README.md)
-  - [Upgrading from pre-Synapse 1.0](MSC1711_certificates_FAQ.md)
-
-# Usage
-  - [Federation](federate.md)
-  - [Configuration](usage/configuration/README.md)
-    - [Homeserver Sample Config File](usage/configuration/homeserver_sample_config.md)
-    - [Logging Sample Config File](usage/configuration/logging_sample_config.md)
-    - [Structured Logging](structured_logging.md)
-    - [User Authentication](usage/configuration/user_authentication/README.md)
-      - [Single-Sign On]()
-        - [OpenID Connect](openid.md)
-        - [SAML]()
-        - [CAS]()
-        - [SSO Mapping Providers](sso_mapping_providers.md)
-      - [Password Auth Providers](password_auth_providers.md)
-      - [JSON Web Tokens](jwt.md)
-    - [Registration Captcha](CAPTCHA_SETUP.md)
-    - [Application Services](application_services.md)
-    - [Server Notices](server_notices.md)
-    - [Consent Tracking](consent_tracking.md)
-    - [URL Previews](url_previews.md)
-    - [User Directory](user_directory.md)
-    - [Message Retention Policies](message_retention_policies.md)
-    - [Pluggable Modules]()
-      - [Third Party Rules]()
-      - [Spam Checker](spam_checker.md)
-      - [Presence Router](presence_router_module.md)
-      - [Media Storage Providers]()
-    - [Workers](workers.md)
-      - [Using `synctl` with Workers](synctl_workers.md)
-      - [Systemd](systemd-with-workers/README.md)
-  - [Administration](usage/administration/README.md)
-    - [Admin API](usage/administration/admin_api/README.md)
-      - [Account Validity](admin_api/account_validity.md)
-      - [Delete Group](admin_api/delete_group.md)
-      - [Event Reports](admin_api/event_reports.md)
-      - [Media](admin_api/media_admin_api.md)
-      - [Purge History](admin_api/purge_history_api.md)
-      - [Purge Rooms](admin_api/purge_room.md)
-      - [Register Users](admin_api/register_api.md)
-      - [Manipulate Room Membership](admin_api/room_membership.md)
-      - [Rooms](admin_api/rooms.md)
-      - [Server Notices](admin_api/server_notices.md)
-      - [Shutdown Room](admin_api/shutdown_room.md)
-      - [Statistics](admin_api/statistics.md)
-      - [Users](admin_api/user_admin_api.md)
-      - [Server Version](admin_api/version_api.md)
-    - [Manhole](manhole.md)
-    - [Monitoring](metrics-howto.md)
-    - [Scripts]()
-
-# Development
-  - [Contributing Guide](development/contributing_guide.md)
-  - [Code Style](code_style.md)
-  - [Git Usage](dev/git.md)
-  - [Testing]()
-  - [OpenTracing](opentracing.md)
-  - [Synapse Architecture]()
-    - [Log Contexts](log_contexts.md)
-    - [Replication](replication.md)
-    - [TCP Replication](tcp_replication.md)
-  - [Internal Documentation](development/internal_documentation/README.md)
-    - [Single Sign-On]()
-      - [SAML](dev/saml.md)
-      - [CAS](dev/cas.md)
-    - [State Resolution]()
-      - [The Auth Chain Difference Algorithm](auth_chain_difference_algorithm.md)
-    - [Media Repository](media_repository.md)
-    - [Room and User Statistics](room_and_user_statistics.md)
-  - [Scripts]()
-
-# Other
-  - [Dependency Deprecation Policy](deprecation_policy.md)

docs/admin_api/README.rst

@@ -1,14 +1,28 @@
 Admin APIs
 ==========
 
-**Note**: The latest documentation can be viewed `here <https://matrix-org.github.io/synapse>`_.
-See `docs/README.md <../docs/README.md>`_ for more information.
-
-**Please update links to point to the website instead.** Existing files in this directory
-are preserved to maintain historical links, but may be moved in the future.
-
 This directory includes documentation for the various synapse specific admin
-APIs available. Updates to the existing Admin API documentation should still
-be made to these files, but any new documentation files should instead be placed under
-`docs/usage/administration/admin_api <../docs/usage/administration/admin_api>`_.
+APIs available.
 
+Authenticating as a server admin
+--------------------------------
+
+Many of the API calls in the admin api will require an `access_token` for a
+server admin. (Note that a server admin is distinct from a room admin.)
+
+A user can be marked as a server admin by updating the database directly, e.g.:
+
+.. code-block:: sql
+
+    UPDATE users SET admin = 1 WHERE name = '@foo:bar.com';
+
+A new server admin user can also be created using the
+``register_new_matrix_user`` script.
+
+Finding your user's `access_token` is client-dependent, but will usually be shown in the client's settings.
+
+Once you have your `access_token`, to include it in a request, the best option is to add the token to a request header:
+
+``curl --header "Authorization: Bearer <access_token>" <the_rest_of_your_API_request>``
+
+Fore more details, please refer to the complete `matrix spec documentation <https://matrix.org/docs/spec/client_server/r0.5.0#using-access-tokens>`_.

docs/admin_api/account_validity.md

@@ -1,42 +0,0 @@
-# Account validity API
-
-This API allows a server administrator to manage the validity of an account. To
-use it, you must enable the account validity feature (under
-`account_validity`) in Synapse's configuration.
-
-## Renew account
-
-This API extends the validity of an account by as much time as configured in the
-`period` parameter from the `account_validity` configuration.
-
-The API is:
-
-```
-POST /_synapse/admin/v1/account_validity/validity
-```
-
-with the following body:
-
-```json
-{
-    "user_id": "<user ID for the account to renew>",
-    "expiration_ts": 0,
-    "enable_renewal_emails": true
-}
-```
-
-
-`expiration_ts` is an optional parameter and overrides the expiration date,
-which otherwise defaults to now + validity period.
-
-`enable_renewal_emails` is also an optional parameter and enables/disables
-sending renewal emails to the user. Defaults to true.
-
-The API returns with the new expiration date for this account, as a timestamp in
-milliseconds since epoch:
-
-```json
-{
-    "expiration_ts": 0
-}
-```

docs/admin_api/account_validity.rst

@@ -0,0 +1,42 @@
+Account validity API
+====================
+
+This API allows a server administrator to manage the validity of an account. To
+use it, you must enable the account validity feature (under
+``account_validity``) in Synapse's configuration.
+
+Renew account
+-------------
+
+This API extends the validity of an account by as much time as configured in the
+``period`` parameter from the ``account_validity`` configuration.
+
+The API is::
+
+    POST /_synapse/admin/v1/account_validity/validity
+
+with the following body:
+
+.. code:: json
+
+    {
+        "user_id": "<user ID for the account to renew>",
+        "expiration_ts": 0,
+        "enable_renewal_emails": true
+    }
+
+
+``expiration_ts`` is an optional parameter and overrides the expiration date,
+which otherwise defaults to now + validity period.
+
+``enable_renewal_emails`` is also an optional parameter and enables/disables
+sending renewal emails to the user. Defaults to true.
+
+The API returns with the new expiration date for this account, as a timestamp in
+milliseconds since epoch:
+
+.. code:: json
+
+    {
+        "expiration_ts": 0
+    }

docs/admin_api/delete_group.md

@@ -11,4 +11,4 @@ POST /_synapse/admin/v1/delete_group/<group_id>
 ```
 
 To use it, you will need to authenticate by providing an `access_token` for a
-server admin: see [Admin API](../../usage/administration/admin_api).
+server admin: see [README.rst](README.rst).

docs/admin_api/event_reports.md

@@ -7,7 +7,7 @@ The api is:
 GET /_synapse/admin/v1/event_reports?from=0&limit=10
 ```
 To use it, you will need to authenticate by providing an `access_token` for a
-server admin: see [Admin API](../../usage/administration/admin_api).
+server admin: see [README.rst](README.rst).
 
 It returns a JSON body like the following:
 
@@ -75,9 +75,9 @@ The following fields are returned in the JSON response body:
 * `name`: string - The name of the room.
 * `event_id`: string - The ID of the reported event.
 * `user_id`: string - This is the user who reported the event and wrote the reason.
-* `reason`: string - Comment made by the `user_id` in this report. May be blank or `null`.
+* `reason`: string - Comment made by the `user_id` in this report. May be blank.
 * `score`: integer - Content is reported based upon a negative score, where -100 is
-  "most offensive" and 0 is "inoffensive". May be `null`.
+  "most offensive" and 0 is "inoffensive".
 * `sender`: string - This is the ID of the user who sent the original message/event that
   was reported.
 * `canonical_alias`: string - The canonical alias of the room. `null` if the room does not
@@ -95,7 +95,7 @@ The api is:
 GET /_synapse/admin/v1/event_reports/<report_id>
 ```
 To use it, you will need to authenticate by providing an `access_token` for a
-server admin: see [Admin API](../../usage/administration/admin_api).
+server admin: see [README.rst](README.rst).
 
 It returns a JSON body like the following:
 

docs/admin_api/media_admin_api.md

@@ -4,11 +4,9 @@
   * [List all media uploaded by a user](#list-all-media-uploaded-by-a-user)
 - [Quarantine media](#quarantine-media)
   * [Quarantining media by ID](#quarantining-media-by-id)
-  * [Remove media from quarantine by ID](#remove-media-from-quarantine-by-id)
   * [Quarantining media in a room](#quarantining-media-in-a-room)
   * [Quarantining all media of a user](#quarantining-all-media-of-a-user)
   * [Protecting media from being quarantined](#protecting-media-from-being-quarantined)
-  * [Unprotecting media from being quarantined](#unprotecting-media-from-being-quarantined)
 - [Delete local media](#delete-local-media)
   * [Delete a specific local media](#delete-a-specific-local-media)
   * [Delete local media by date or size](#delete-local-media-by-date-or-size)
@@ -28,7 +26,7 @@ The API is:
 GET /_synapse/admin/v1/room/<room_id>/media
 ```
 To use it, you will need to authenticate by providing an `access_token` for a
-server admin: see [Admin API](../../usage/administration/admin_api).
+server admin: see [README.rst](README.rst).
 
 The API returns a JSON body like the following:
 ```json
@@ -78,27 +76,6 @@ Response:
 {}
 ```
 
-## Remove media from quarantine by ID
-
-This API removes a single piece of local or remote media from quarantine.
-
-Request:
-
-```
-POST /_synapse/admin/v1/media/unquarantine/<server_name>/<media_id>
-
-{}
-```
-
-Where `server_name` is in the form of `example.org`, and `media_id` is in the
-form of `abcdefg12345...`.
-
-Response:
-
-```json
-{}
-```
-
 ## Quarantining media in a room
 
 This API quarantines all local and remote media in a room.
@@ -182,26 +159,6 @@ Response:
 {}
 ```
 
-## Unprotecting media from being quarantined
-
-This API reverts the protection of a media.
-
-Request:
-
-```
-POST /_synapse/admin/v1/media/unprotect/<media_id>
-
-{}
-```
-
-Where `media_id` is in the  form of `abcdefg12345...`.
-
-Response:
-
-```json
-{}
-```
-
 # Delete local media
 This API deletes the *local* media from the disk of your own server.
 This includes any local thumbnails and copies of media downloaded from
@@ -311,7 +268,7 @@ The following fields are returned in the JSON response body:
 * `deleted`: integer - The number of media items successfully deleted
 
 To use it, you will need to authenticate by providing an `access_token` for a
-server admin: see [Admin API](../../usage/administration/admin_api).
+server admin: see [README.rst](README.rst).
 
 If the user re-requests purged remote media, synapse will re-request the media
 from the originating server.

docs/admin_api/purge_history_api.rst

@@ -1,4 +1,5 @@
-# Purge History API
+Purge History API
+=================
 
 The purge history API allows server admins to purge historic events from their
 database, reclaiming disk space.
@@ -12,12 +13,10 @@ delete the last message in a room.
 
 The API is:
 
-```
-POST /_synapse/admin/v1/purge_history/<room_id>[/<event_id>]
-```
+``POST /_synapse/admin/v1/purge_history/<room_id>[/<event_id>]``
 
-To use it, you will need to authenticate by providing an `access_token` for a
-server admin: [Admin API](../../usage/administration/admin_api)
+To use it, you will need to authenticate by providing an ``access_token`` for a
+server admin: see `README.rst <README.rst>`_.
 
 By default, events sent by local users are not deleted, as they may represent
 the only copies of this content in existence. (Events sent by remote users are
@@ -25,54 +24,54 @@ deleted.)
 
 Room state data (such as joins, leaves, topic) is always preserved.
 
-To delete local message events as well, set `delete_local_events` in the body:
+To delete local message events as well, set ``delete_local_events`` in the body:
 
-```
-{
-   "delete_local_events": true
-}
-```
+.. code:: json
+
+   {
+       "delete_local_events": true
+   }
 
 The caller must specify the point in the room to purge up to. This can be
 specified by including an event_id in the URI, or by setting a
-`purge_up_to_event_id` or `purge_up_to_ts` in the request body. If an event
+``purge_up_to_event_id`` or ``purge_up_to_ts`` in the request body. If an event
 id is given, that event (and others at the same graph depth) will be retained.
-If `purge_up_to_ts` is given, it should be a timestamp since the unix epoch,
+If ``purge_up_to_ts`` is given, it should be a timestamp since the unix epoch,
 in milliseconds.
 
 The API starts the purge running, and returns immediately with a JSON body with
 a purge id:
 
-```json
-{
-    "purge_id": "<opaque id>"
-}
-```
+.. code:: json
 
-## Purge status query
+    {
+        "purge_id": "<opaque id>"
+    }
+
+Purge status query
+------------------
 
 It is possible to poll for updates on recent purges with a second API;
 
-```
-GET /_synapse/admin/v1/purge_history_status/<purge_id>
-```
+``GET /_synapse/admin/v1/purge_history_status/<purge_id>``
 
-Again, you will need to authenticate by providing an `access_token` for a
+Again, you will need to authenticate by providing an ``access_token`` for a
 server admin.
 
 This API returns a JSON body like the following:
 
-```json
-{
-    "status": "active"
-}
-```
+.. code:: json
 
-The status will be one of `active`, `complete`, or `failed`.
+    {
+        "status": "active"
+    }
 
-## Reclaim disk space (Postgres)
+The status will be one of ``active``, ``complete``, or ``failed``.
+
+Reclaim disk space (Postgres)
+-----------------------------
 
 To reclaim the disk space and return it to the operating system, you need to run
 `VACUUM FULL;` on the database.
 
-<https://www.postgresql.org/docs/current/sql-vacuum.html>
+https://www.postgresql.org/docs/current/sql-vacuum.html

docs/admin_api/register_api.md

@@ -1,73 +0,0 @@
-# Shared-Secret Registration
-
-This API allows for the creation of users in an administrative and
-non-interactive way. This is generally used for bootstrapping a Synapse
-instance with administrator accounts.
-
-To authenticate yourself to the server, you will need both the shared secret
-(`registration_shared_secret` in the homeserver configuration), and a
-one-time nonce. If the registration shared secret is not configured, this API
-is not enabled.
-
-To fetch the nonce, you need to request one from the API:
-
-```
-> GET /_synapse/admin/v1/register
-
-< {"nonce": "thisisanonce"}
-```
-
-Once you have the nonce, you can make a `POST` to the same URL with a JSON
-body containing the nonce, username, password, whether they are an admin
-(optional, False by default), and a HMAC digest of the content. Also you can
-set the displayname (optional, `username` by default).
-
-As an example:
-
-```
-> POST /_synapse/admin/v1/register
-> {
-   "nonce": "thisisanonce",
-   "username": "pepper_roni",
-   "displayname": "Pepper Roni",
-   "password": "pizza",
-   "admin": true,
-   "mac": "mac_digest_here"
-  }
-
-< {
-   "access_token": "token_here",
-   "user_id": "@pepper_roni:localhost",
-   "home_server": "test",
-   "device_id": "device_id_here"
-  }
-```
-
-The MAC is the hex digest output of the HMAC-SHA1 algorithm, with the key being
-the shared secret and the content being the nonce, user, password, either the
-string "admin" or "notadmin", and optionally the user_type
-each separated by NULs. For an example of generation in Python:
-
-```python
-import hmac, hashlib
-
-def generate_mac(nonce, user, password, admin=False, user_type=None):
-
-    mac = hmac.new(
-      key=shared_secret,
-      digestmod=hashlib.sha1,
-    )
-
-    mac.update(nonce.encode('utf8'))
-    mac.update(b"\x00")
-    mac.update(user.encode('utf8'))
-    mac.update(b"\x00")
-    mac.update(password.encode('utf8'))
-    mac.update(b"\x00")
-    mac.update(b"admin" if admin else b"notadmin")
-    if user_type:
-        mac.update(b"\x00")
-        mac.update(user_type.encode('utf8'))
-
-    return mac.hexdigest()
-```

docs/admin_api/register_api.rst

@@ -0,0 +1,68 @@
+Shared-Secret Registration
+==========================
+
+This API allows for the creation of users in an administrative and
+non-interactive way. This is generally used for bootstrapping a Synapse
+instance with administrator accounts.
+
+To authenticate yourself to the server, you will need both the shared secret
+(``registration_shared_secret`` in the homeserver configuration), and a
+one-time nonce. If the registration shared secret is not configured, this API
+is not enabled.
+
+To fetch the nonce, you need to request one from the API::
+
+  > GET /_synapse/admin/v1/register
+
+  < {"nonce": "thisisanonce"}
+
+Once you have the nonce, you can make a ``POST`` to the same URL with a JSON
+body containing the nonce, username, password, whether they are an admin
+(optional, False by default), and a HMAC digest of the content. Also you can
+set the displayname (optional, ``username`` by default).
+
+As an example::
+
+  > POST /_synapse/admin/v1/register
+  > {
+     "nonce": "thisisanonce",
+     "username": "pepper_roni",
+     "displayname": "Pepper Roni",
+     "password": "pizza",
+     "admin": true,
+     "mac": "mac_digest_here"
+    }
+
+  < {
+     "access_token": "token_here",
+     "user_id": "@pepper_roni:localhost",
+     "home_server": "test",
+     "device_id": "device_id_here"
+    }
+
+The MAC is the hex digest output of the HMAC-SHA1 algorithm, with the key being
+the shared secret and the content being the nonce, user, password, either the
+string "admin" or "notadmin", and optionally the user_type
+each separated by NULs. For an example of generation in Python::
+
+  import hmac, hashlib
+
+  def generate_mac(nonce, user, password, admin=False, user_type=None):
+
+      mac = hmac.new(
+        key=shared_secret,
+        digestmod=hashlib.sha1,
+      )
+
+      mac.update(nonce.encode('utf8'))
+      mac.update(b"\x00")
+      mac.update(user.encode('utf8'))
+      mac.update(b"\x00")
+      mac.update(password.encode('utf8'))
+      mac.update(b"\x00")
+      mac.update(b"admin" if admin else b"notadmin")
+      if user_type:
+          mac.update(b"\x00")
+          mac.update(user_type.encode('utf8'))
+
+      return mac.hexdigest()

docs/admin_api/room_membership.md

@@ -24,7 +24,7 @@ POST /_synapse/admin/v1/join/<room_id_or_alias>
 ```
 
 To use it, you will need to authenticate by providing an `access_token` for a
-server admin: see [Admin API](../../usage/administration/admin_api).
+server admin: see [README.rst](README.rst).
 
 Response:
 

docs/admin_api/rooms.md

@@ -4,7 +4,6 @@
   * [Usage](#usage)
 - [Room Details API](#room-details-api)
 - [Room Members API](#room-members-api)
-- [Room State API](#room-state-api)
 - [Delete Room API](#delete-room-api)
   * [Parameters](#parameters-1)
   * [Response](#response)
@@ -428,7 +427,7 @@ the new room. Users on other servers will be unaffected.
 The API is:
 
 ```
-DELETE /_synapse/admin/v1/rooms/<room_id>
+POST /_synapse/admin/v1/rooms/<room_id>/delete
 ```
 
 with a body of:
@@ -443,7 +442,7 @@ with a body of:
 ```
 
 To use it, you will need to authenticate by providing an ``access_token`` for a
-server admin: see [Admin API](../../usage/administration/admin_api).
+server admin: see [README.rst](README.rst).
 
 A response body like the following is returned:
 
@@ -529,15 +528,6 @@ You will have to manually handle, if you so choose, the following:
 * Users that would have been booted from the room (and will have been force-joined to the Content Violation room).
 * Removal of the Content Violation room if desired.
 
-## Deprecated endpoint
-
-The previous deprecated API will be removed in a future release, it was:
-
-```
-POST /_synapse/admin/v1/rooms/<room_id>/delete
-```
-
-It behaves the same way than the current endpoint except the path and the method.
 
 # Make Room Admin API
 

docs/admin_api/statistics.md

@@ -10,7 +10,7 @@ GET /_synapse/admin/v1/statistics/users/media
 ```
 
 To use it, you will need to authenticate by providing an `access_token`
-for a server admin: see [Admin API](../../usage/administration/admin_api).
+for a server admin: see [README.rst](README.rst).
 
 A response body like the following is returned:
 

docs/admin_api/user_admin_api.rst

@@ -0,0 +1,981 @@
+.. contents::
+
+Query User Account
+==================
+
+This API returns information about a specific user account.
+
+The api is::
+
+    GET /_synapse/admin/v2/users/<user_id>
+
+To use it, you will need to authenticate by providing an ``access_token`` for a
+server admin: see `README.rst <README.rst>`_.
+
+It returns a JSON body like the following:
+
+.. code:: json
+
+    {
+        "displayname": "User",
+        "threepids": [
+            {
+                "medium": "email",
+                "address": "<user_mail_1>"
+            },
+            {
+                "medium": "email",
+                "address": "<user_mail_2>"
+            }
+        ],
+        "avatar_url": "<avatar_url>",
+        "admin": 0,
+        "deactivated": 0,
+        "shadow_banned": 0,
+        "password_hash": "$2b$12$p9B4GkqYdRTPGD",
+        "creation_ts": 1560432506,
+        "appservice_id": null,
+        "consent_server_notice_sent": null,
+        "consent_version": null
+    }
+
+URL parameters:
+
+- ``user_id``: fully-qualified user id: for example, ``@user:server.com``.
+
+Create or modify Account
+========================
+
+This API allows an administrator to create or modify a user account with a
+specific ``user_id``.
+
+This api is::
+
+    PUT /_synapse/admin/v2/users/<user_id>
+
+with a body of:
+
+.. code:: json
+
+    {
+        "password": "user_password",
+        "displayname": "User",
+        "threepids": [
+            {
+                "medium": "email",
+                "address": "<user_mail_1>"
+            },
+            {
+                "medium": "email",
+                "address": "<user_mail_2>"
+            }
+        ],
+        "avatar_url": "<avatar_url>",
+        "admin": false,
+        "deactivated": false
+    }
+
+To use it, you will need to authenticate by providing an ``access_token`` for a
+server admin: see `README.rst <README.rst>`_.
+
+URL parameters:
+
+- ``user_id``: fully-qualified user id: for example, ``@user:server.com``.
+
+Body parameters:
+
+- ``password``, optional. If provided, the user's password is updated and all
+  devices are logged out.
+
+- ``displayname``, optional, defaults to the value of ``user_id``.
+
+- ``threepids``, optional, allows setting the third-party IDs (email, msisdn)
+  belonging to a user.
+
+- ``avatar_url``, optional, must be a
+  `MXC URI <https://matrix.org/docs/spec/client_server/r0.6.0#matrix-content-mxc-uris>`_.
+
+- ``admin``, optional, defaults to ``false``.
+
+- ``deactivated``, optional. If unspecified, deactivation state will be left
+  unchanged on existing accounts and set to ``false`` for new accounts.
+  A user cannot be erased by deactivating with this API. For details on deactivating users see
+  `Deactivate Account <#deactivate-account>`_.
+
+If the user already exists then optional parameters default to the current value.
+
+In order to re-activate an account ``deactivated`` must be set to ``false``. If
+users do not login via single-sign-on, a new ``password`` must be provided.
+
+List Accounts
+=============
+
+This API returns all local user accounts.
+By default, the response is ordered by ascending user ID.
+
+The API is::
+
+    GET /_synapse/admin/v2/users?from=0&limit=10&guests=false
+
+To use it, you will need to authenticate by providing an ``access_token`` for a
+server admin: see `README.rst <README.rst>`_.
+
+A response body like the following is returned:
+
+.. code:: json
+
+    {
+        "users": [
+            {
+                "name": "<user_id1>",
+                "is_guest": 0,
+                "admin": 0,
+                "user_type": null,
+                "deactivated": 0,
+                "shadow_banned": 0,
+                "displayname": "<User One>",
+                "avatar_url": null
+            }, {
+                "name": "<user_id2>",
+                "is_guest": 0,
+                "admin": 1,
+                "user_type": null,
+                "deactivated": 0,
+                "shadow_banned": 0,
+                "displayname": "<User Two>",
+                "avatar_url": "<avatar_url>"
+            }
+        ],
+        "next_token": "100",
+        "total": 200
+    }
+
+To paginate, check for ``next_token`` and if present, call the endpoint again
+with ``from`` set to the value of ``next_token``. This will return a new page.
+
+If the endpoint does not return a ``next_token`` then there are no more users
+to paginate through.
+
+**Parameters**
+
+The following parameters should be set in the URL:
+
+- ``user_id`` - Is optional and filters to only return users with user IDs
+  that contain this value. This parameter is ignored when using the ``name`` parameter.
+- ``name`` - Is optional and filters to only return users with user ID localparts
+  **or** displaynames that contain this value.
+- ``guests`` - string representing a bool - Is optional and if ``false`` will **exclude** guest users.
+  Defaults to ``true`` to include guest users.
+- ``deactivated`` - string representing a bool - Is optional and if ``true`` will **include** deactivated users.
+  Defaults to ``false`` to exclude deactivated users.
+- ``limit`` - string representing a positive integer - Is optional but is used for pagination,
+  denoting the maximum number of items to return in this call. Defaults to ``100``.
+- ``from`` - string representing a positive integer - Is optional but used for pagination,
+  denoting the offset in the returned results. This should be treated as an opaque value and
+  not explicitly set to anything other than the return value of ``next_token`` from a previous call.
+  Defaults to ``0``.
+- ``order_by`` - The method by which to sort the returned list of users.
+  If the ordered field has duplicates, the second order is always by ascending ``name``,
+  which guarantees a stable ordering. Valid values are:
+
+  - ``name`` - Users are ordered alphabetically by ``name``. This is the default.
+  - ``is_guest`` - Users are ordered by ``is_guest`` status.
+  - ``admin`` - Users are ordered by ``admin`` status.
+  - ``user_type`` - Users are ordered alphabetically by ``user_type``.
+  - ``deactivated`` - Users are ordered by ``deactivated`` status.
+  - ``shadow_banned`` - Users are ordered by ``shadow_banned`` status.
+  - ``displayname`` - Users are ordered alphabetically by ``displayname``.
+  - ``avatar_url`` - Users are ordered alphabetically by avatar URL.
+
+- ``dir`` - Direction of media order. Either ``f`` for forwards or ``b`` for backwards.
+  Setting this value to ``b`` will reverse the above sort order. Defaults to ``f``.
+
+Caution. The database only has indexes on the columns ``name`` and ``created_ts``.
+This means that if a different sort order is used (``is_guest``, ``admin``,
+``user_type``, ``deactivated``, ``shadow_banned``, ``avatar_url`` or ``displayname``),
+this can cause a large load on the database, especially for large environments.
+
+**Response**
+
+The following fields are returned in the JSON response body:
+
+- ``users`` - An array of objects, each containing information about an user.
+  User objects contain the following fields:
+
+  - ``name`` - string - Fully-qualified user ID (ex. ``@user:server.com``).
+  - ``is_guest`` - bool - Status if that user is a guest account.
+  - ``admin`` - bool - Status if that user is a server administrator.
+  - ``user_type`` - string - Type of the user. Normal users are type ``None``.
+    This allows user type specific behaviour. There are also types ``support`` and ``bot``. 
+  - ``deactivated`` - bool - Status if that user has been marked as deactivated.
+  - ``shadow_banned`` - bool - Status if that user has been marked as shadow banned.
+  - ``displayname`` - string - The user's display name if they have set one.
+  - ``avatar_url`` - string -  The user's avatar URL if they have set one.
+
+- ``next_token``: string representing a positive integer - Indication for pagination. See above.
+- ``total`` - integer - Total number of media.
+
+
+Query current sessions for a user
+=================================
+
+This API returns information about the active sessions for a specific user.
+
+The api is::
+
+    GET /_synapse/admin/v1/whois/<user_id>
+
+and::
+
+    GET /_matrix/client/r0/admin/whois/<userId>
+
+See also: `Client Server API Whois
+<https://matrix.org/docs/spec/client_server/r0.6.1#get-matrix-client-r0-admin-whois-userid>`_
+
+To use it, you will need to authenticate by providing an ``access_token`` for a
+server admin: see `README.rst <README.rst>`_.
+
+It returns a JSON body like the following:
+
+.. code:: json
+
+    {
+        "user_id": "<user_id>",
+        "devices": {
+            "": {
+                "sessions": [
+                    {
+                        "connections": [
+                            {
+                                "ip": "1.2.3.4",
+                                "last_seen": 1417222374433,
+                                "user_agent": "Mozilla/5.0 ..."
+                            },
+                            {
+                                "ip": "1.2.3.10",
+                                "last_seen": 1417222374500,
+                                "user_agent": "Dalvik/2.1.0 ..."
+                            }
+                        ]
+                    }
+                ]
+            }
+        }
+    }
+
+``last_seen`` is measured in milliseconds since the Unix epoch.
+
+Deactivate Account
+==================
+
+This API deactivates an account. It removes active access tokens, resets the
+password, and deletes third-party IDs (to prevent the user requesting a
+password reset).
+
+It can also mark the user as GDPR-erased. This means messages sent by the
+user will still be visible by anyone that was in the room when these messages
+were sent, but hidden from users joining the room afterwards.
+
+The api is::
+
+    POST /_synapse/admin/v1/deactivate/<user_id>
+
+with a body of:
+
+.. code:: json
+
+    {
+        "erase": true
+    }
+
+To use it, you will need to authenticate by providing an ``access_token`` for a
+server admin: see `README.rst <README.rst>`_.
+
+The erase parameter is optional and defaults to ``false``.
+An empty body may be passed for backwards compatibility.
+
+The following actions are performed when deactivating an user:
+
+- Try to unpind 3PIDs from the identity server
+- Remove all 3PIDs from the homeserver
+- Delete all devices and E2EE keys
+- Delete all access tokens
+- Delete the password hash
+- Removal from all rooms the user is a member of
+- Remove the user from the user directory
+- Reject all pending invites
+- Remove all account validity information related to the user
+
+The following additional actions are performed during deactivation if ``erase``
+is set to ``true``:
+
+- Remove the user's display name
+- Remove the user's avatar URL
+- Mark the user as erased
+
+
+Reset password
+==============
+
+Changes the password of another user. This will automatically log the user out of all their devices.
+
+The api is::
+
+    POST /_synapse/admin/v1/reset_password/<user_id>
+
+with a body of:
+
+.. code:: json
+
+   {
+       "new_password": "<secret>",
+       "logout_devices": true
+   }
+
+To use it, you will need to authenticate by providing an ``access_token`` for a
+server admin: see `README.rst <README.rst>`_.
+
+The parameter ``new_password`` is required.
+The parameter ``logout_devices`` is optional and defaults to ``true``.
+
+Get whether a user is a server administrator or not
+===================================================
+
+
+The api is::
+
+    GET /_synapse/admin/v1/users/<user_id>/admin
+
+To use it, you will need to authenticate by providing an ``access_token`` for a
+server admin: see `README.rst <README.rst>`_.
+
+A response body like the following is returned:
+
+.. code:: json
+
+    {
+        "admin": true
+    }
+
+
+Change whether a user is a server administrator or not
+======================================================
+
+Note that you cannot demote yourself.
+
+The api is::
+
+    PUT /_synapse/admin/v1/users/<user_id>/admin
+
+with a body of:
+
+.. code:: json
+
+    {
+        "admin": true
+    }
+
+To use it, you will need to authenticate by providing an ``access_token`` for a
+server admin: see `README.rst <README.rst>`_.
+
+
+List room memberships of an user
+================================
+Gets a list of all ``room_id`` that a specific ``user_id`` is member.
+
+The API is::
+
+  GET /_synapse/admin/v1/users/<user_id>/joined_rooms
+
+To use it, you will need to authenticate by providing an ``access_token`` for a
+server admin: see `README.rst <README.rst>`_.
+
+A response body like the following is returned:
+
+.. code:: json
+
+    {
+        "joined_rooms": [
+            "!DuGcnbhHGaSZQoNQR:matrix.org",
+            "!ZtSaPCawyWtxfWiIy:matrix.org"
+        ],
+        "total": 2
+    }
+
+The server returns the list of rooms of which the user and the server
+are member. If the user is local, all the rooms of which the user is
+member are returned.
+
+**Parameters**
+
+The following parameters should be set in the URL:
+
+- ``user_id`` - fully qualified: for example, ``@user:server.com``.
+
+**Response**
+
+The following fields are returned in the JSON response body:
+
+- ``joined_rooms`` - An array of ``room_id``.
+- ``total`` - Number of rooms.
+
+
+List media of a user
+====================
+Gets a list of all local media that a specific ``user_id`` has created.
+By default, the response is ordered by descending creation date and ascending media ID.
+The newest media is on top. You can change the order with parameters
+``order_by`` and ``dir``.
+
+The API is::
+
+  GET /_synapse/admin/v1/users/<user_id>/media
+
+To use it, you will need to authenticate by providing an ``access_token`` for a
+server admin: see `README.rst <README.rst>`_.
+
+A response body like the following is returned:
+
+.. code:: json
+
+    {
+      "media": [
+        {
+          "created_ts": 100400,
+          "last_access_ts": null,
+          "media_id": "qXhyRzulkwLsNHTbpHreuEgo",
+          "media_length": 67,
+          "media_type": "image/png",
+          "quarantined_by": null,
+          "safe_from_quarantine": false,
+          "upload_name": "test1.png"
+        },
+        {
+          "created_ts": 200400,
+          "last_access_ts": null,
+          "media_id": "FHfiSnzoINDatrXHQIXBtahw",
+          "media_length": 67,
+          "media_type": "image/png",
+          "quarantined_by": null,
+          "safe_from_quarantine": false,
+          "upload_name": "test2.png"
+        }
+      ],
+      "next_token": 3,
+      "total": 2
+    }
+
+To paginate, check for ``next_token`` and if present, call the endpoint again
+with ``from`` set to the value of ``next_token``. This will return a new page.
+
+If the endpoint does not return a ``next_token`` then there are no more
+reports to paginate through.
+
+**Parameters**
+
+The following parameters should be set in the URL:
+
+- ``user_id`` - string - fully qualified: for example, ``@user:server.com``.
+- ``limit``: string representing a positive integer - Is optional but is used for pagination,
+  denoting the maximum number of items to return in this call. Defaults to ``100``.
+- ``from``: string representing a positive integer - Is optional but used for pagination,
+  denoting the offset in the returned results. This should be treated as an opaque value and
+  not explicitly set to anything other than the return value of ``next_token`` from a previous call.
+  Defaults to ``0``.
+- ``order_by`` - The method by which to sort the returned list of media.
+  If the ordered field has duplicates, the second order is always by ascending ``media_id``,
+  which guarantees a stable ordering. Valid values are:
+
+  - ``media_id`` - Media are ordered alphabetically by ``media_id``.
+  - ``upload_name`` - Media are ordered alphabetically by name the media was uploaded with.
+  - ``created_ts`` - Media are ordered by when the content was uploaded in ms.
+    Smallest to largest. This is the default.
+  - ``last_access_ts`` - Media are ordered by when the content was last accessed in ms.
+    Smallest to largest.
+  - ``media_length`` - Media are ordered by length of the media in bytes.
+    Smallest to largest.
+  - ``media_type`` - Media are ordered alphabetically by MIME-type.
+  - ``quarantined_by`` - Media are ordered alphabetically by the user ID that
+    initiated the quarantine request for this media.
+  - ``safe_from_quarantine`` - Media are ordered by the status if this media is safe
+    from quarantining.
+
+- ``dir`` - Direction of media order. Either ``f`` for forwards or ``b`` for backwards.
+  Setting this value to ``b`` will reverse the above sort order. Defaults to ``f``.
+
+If neither ``order_by`` nor ``dir`` is set, the default order is newest media on top
+(corresponds to ``order_by`` = ``created_ts`` and ``dir`` = ``b``).
+
+Caution. The database only has indexes on the columns ``media_id``,
+``user_id`` and ``created_ts``. This means that if a different sort order is used
+(``upload_name``, ``last_access_ts``, ``media_length``, ``media_type``,
+``quarantined_by`` or ``safe_from_quarantine``), this can cause a large load on the
+database, especially for large environments.
+
+**Response**
+
+The following fields are returned in the JSON response body:
+
+- ``media`` - An array of objects, each containing information about a media.
+  Media objects contain the following fields:
+
+  - ``created_ts`` - integer - Timestamp when the content was uploaded in ms.
+  - ``last_access_ts`` - integer - Timestamp when the content was last accessed in ms.
+  - ``media_id`` - string - The id used to refer to the media.
+  - ``media_length`` - integer - Length of the media in bytes.
+  - ``media_type`` - string - The MIME-type of the media.
+  - ``quarantined_by`` - string - The user ID that initiated the quarantine request
+    for this media.
+
+  - ``safe_from_quarantine`` - bool - Status if this media is safe from quarantining.
+  - ``upload_name`` - string - The name the media was uploaded with.
+
+- ``next_token``: integer - Indication for pagination. See above.
+- ``total`` - integer - Total number of media.
+
+Login as a user
+===============
+
+Get an access token that can be used to authenticate as that user. Useful for
+when admins wish to do actions on behalf of a user.
+
+The API is::
+
+    POST /_synapse/admin/v1/users/<user_id>/login
+    {}
+
+An optional ``valid_until_ms`` field can be specified in the request body as an
+integer timestamp that specifies when the token should expire. By default tokens
+do not expire.
+
+A response body like the following is returned:
+
+.. code:: json
+
+    {
+        "access_token": "<opaque_access_token_string>"
+    }
+
+
+This API does *not* generate a new device for the user, and so will not appear
+their ``/devices`` list, and in general the target user should not be able to
+tell they have been logged in as.
+
+To expire the token call the standard ``/logout`` API with the token.
+
+Note: The token will expire if the *admin* user calls ``/logout/all`` from any
+of their devices, but the token will *not* expire if the target user does the
+same.
+
+
+User devices
+============
+
+List all devices
+----------------
+Gets information about all devices for a specific ``user_id``.
+
+The API is::
+
+  GET /_synapse/admin/v2/users/<user_id>/devices
+
+To use it, you will need to authenticate by providing an ``access_token`` for a
+server admin: see `README.rst <README.rst>`_.
+
+A response body like the following is returned:
+
+.. code:: json
+
+    {
+      "devices": [
+        {
+          "device_id": "QBUAZIFURK",
+          "display_name": "android",
+          "last_seen_ip": "1.2.3.4",
+          "last_seen_ts": 1474491775024,
+          "user_id": "<user_id>"
+        },
+        {
+          "device_id": "AUIECTSRND",
+          "display_name": "ios",
+          "last_seen_ip": "1.2.3.5",
+          "last_seen_ts": 1474491775025,
+          "user_id": "<user_id>"
+        }
+      ],
+      "total": 2
+    }
+
+**Parameters**
+
+The following parameters should be set in the URL:
+
+- ``user_id`` - fully qualified: for example, ``@user:server.com``.
+
+**Response**
+
+The following fields are returned in the JSON response body:
+
+- ``devices`` - An array of objects, each containing information about a device.
+  Device objects contain the following fields:
+
+  - ``device_id`` - Identifier of device.
+  - ``display_name`` - Display name set by the user for this device.
+    Absent if no name has been set.
+  - ``last_seen_ip`` - The IP address where this device was last seen.
+    (May be a few minutes out of date, for efficiency reasons).
+  - ``last_seen_ts`` - The timestamp (in milliseconds since the unix epoch) when this
+    devices was last seen. (May be a few minutes out of date, for efficiency reasons).
+  - ``user_id`` - Owner of  device.
+
+- ``total`` - Total number of user's devices.
+
+Delete multiple devices
+------------------
+Deletes the given devices for a specific ``user_id``, and invalidates
+any access token associated with them.
+
+The API is::
+
+    POST /_synapse/admin/v2/users/<user_id>/delete_devices
+
+    {
+      "devices": [
+        "QBUAZIFURK",
+        "AUIECTSRND"
+      ],
+    }
+
+To use it, you will need to authenticate by providing an ``access_token`` for a
+server admin: see `README.rst <README.rst>`_.
+
+An empty JSON dict is returned.
+
+**Parameters**
+
+The following parameters should be set in the URL:
+
+- ``user_id`` - fully qualified: for example, ``@user:server.com``.
+
+The following fields are required in the JSON request body:
+
+- ``devices`` - The list of device IDs to delete.
+
+Show a device
+---------------
+Gets information on a single device, by ``device_id`` for a specific ``user_id``.
+
+The API is::
+
+    GET /_synapse/admin/v2/users/<user_id>/devices/<device_id>
+
+To use it, you will need to authenticate by providing an ``access_token`` for a
+server admin: see `README.rst <README.rst>`_.
+
+A response body like the following is returned:
+
+.. code:: json
+
+    {
+      "device_id": "<device_id>",
+      "display_name": "android",
+      "last_seen_ip": "1.2.3.4",
+      "last_seen_ts": 1474491775024,
+      "user_id": "<user_id>"
+    }
+
+**Parameters**
+
+The following parameters should be set in the URL:
+
+- ``user_id`` - fully qualified: for example, ``@user:server.com``.
+- ``device_id`` - The device to retrieve.
+
+**Response**
+
+The following fields are returned in the JSON response body:
+
+- ``device_id`` - Identifier of device.
+- ``display_name`` - Display name set by the user for this device.
+  Absent if no name has been set.
+- ``last_seen_ip`` - The IP address where this device was last seen.
+  (May be a few minutes out of date, for efficiency reasons).
+- ``last_seen_ts`` - The timestamp (in milliseconds since the unix epoch) when this
+  devices was last seen. (May be a few minutes out of date, for efficiency reasons).
+- ``user_id`` - Owner of  device.
+
+Update a device
+---------------
+Updates the metadata on the given ``device_id`` for a specific ``user_id``.
+
+The API is::
+
+    PUT /_synapse/admin/v2/users/<user_id>/devices/<device_id>
+
+    {
+      "display_name": "My other phone"
+    }
+
+To use it, you will need to authenticate by providing an ``access_token`` for a
+server admin: see `README.rst <README.rst>`_.
+
+An empty JSON dict is returned.
+
+**Parameters**
+
+The following parameters should be set in the URL:
+
+- ``user_id`` - fully qualified: for example, ``@user:server.com``.
+- ``device_id`` - The device to update.
+
+The following fields are required in the JSON request body:
+
+- ``display_name`` - The new display name for this device. If not given,
+  the display name is unchanged.
+
+Delete a device
+---------------
+Deletes the given ``device_id`` for a specific ``user_id``,
+and invalidates any access token associated with it.
+
+The API is::
+
+    DELETE /_synapse/admin/v2/users/<user_id>/devices/<device_id>
+
+    {}
+
+To use it, you will need to authenticate by providing an ``access_token`` for a
+server admin: see `README.rst <README.rst>`_.
+
+An empty JSON dict is returned.
+
+**Parameters**
+
+The following parameters should be set in the URL:
+
+- ``user_id`` - fully qualified: for example, ``@user:server.com``.
+- ``device_id`` - The device to delete.
+
+List all pushers
+================
+Gets information about all pushers for a specific ``user_id``.
+
+The API is::
+
+  GET /_synapse/admin/v1/users/<user_id>/pushers
+
+To use it, you will need to authenticate by providing an ``access_token`` for a
+server admin: see `README.rst <README.rst>`_.
+
+A response body like the following is returned:
+
+.. code:: json
+
+    {
+      "pushers": [
+        {
+          "app_display_name":"HTTP Push Notifications",
+          "app_id":"m.http",
+          "data": {
+            "url":"example.com"
+          },
+          "device_display_name":"pushy push",
+          "kind":"http",
+          "lang":"None",
+          "profile_tag":"",
+          "pushkey":"a@example.com"
+        }
+      ],
+      "total": 1
+    }
+
+**Parameters**
+
+The following parameters should be set in the URL:
+
+- ``user_id`` - fully qualified: for example, ``@user:server.com``.
+
+**Response**
+
+The following fields are returned in the JSON response body:
+
+- ``pushers`` - An array containing the current pushers for the user
+
+  - ``app_display_name`` - string - A string that will allow the user to identify
+    what application owns this pusher.
+
+  - ``app_id`` - string - This is a reverse-DNS style identifier for the application.
+    Max length, 64 chars.
+
+  - ``data`` - A dictionary of information for the pusher implementation itself.
+
+    - ``url`` - string - Required if ``kind`` is ``http``. The URL to use to send
+      notifications to.
+
+    - ``format`` - string - The format to use when sending notifications to the
+      Push Gateway.
+
+  - ``device_display_name`` - string -  A string that will allow the user to identify
+    what device owns this pusher.
+
+  - ``profile_tag`` - string - This string determines which set of device specific rules
+    this pusher executes.
+
+  - ``kind`` - string -  The kind of pusher. "http" is a pusher that sends HTTP pokes.
+  - ``lang`` - string - The preferred language for receiving notifications
+    (e.g. 'en' or 'en-US')
+
+  - ``profile_tag`` - string - This string determines which set of device specific rules
+    this pusher executes.
+
+  - ``pushkey`` - string - This is a unique identifier for this pusher.
+    Max length, 512 bytes.
+
+- ``total`` - integer - Number of pushers.
+
+See also `Client-Server API Spec <https://matrix.org/docs/spec/client_server/latest#get-matrix-client-r0-pushers>`_
+
+Shadow-banning users
+====================
+
+Shadow-banning is a useful tool for moderating malicious or egregiously abusive users.
+A shadow-banned users receives successful responses to their client-server API requests,
+but the events are not propagated into rooms. This can be an effective tool as it
+(hopefully) takes longer for the user to realise they are being moderated before
+pivoting to another account.
+
+Shadow-banning a user should be used as a tool of last resort and may lead to confusing
+or broken behaviour for the client. A shadow-banned user will not receive any
+notification and it is generally more appropriate to ban or kick abusive users.
+A shadow-banned user will be unable to contact anyone on the server.
+
+The API is::
+
+  POST /_synapse/admin/v1/users/<user_id>/shadow_ban
+
+To use it, you will need to authenticate by providing an ``access_token`` for a
+server admin: see `README.rst <README.rst>`_.
+
+An empty JSON dict is returned.
+
+**Parameters**
+
+The following parameters should be set in the URL:
+
+- ``user_id`` - The fully qualified MXID: for example, ``@user:server.com``. The user must
+  be local.
+
+Override ratelimiting for users
+===============================
+
+This API allows to override or disable ratelimiting for a specific user.
+There are specific APIs to set, get and delete a ratelimit.
+
+Get status of ratelimit
+-----------------------
+
+The API is::
+
+  GET /_synapse/admin/v1/users/<user_id>/override_ratelimit
+
+To use it, you will need to authenticate by providing an ``access_token`` for a
+server admin: see `README.rst <README.rst>`_.
+
+A response body like the following is returned:
+
+.. code:: json
+
+    {
+      "messages_per_second": 0,
+      "burst_count": 0
+    }
+
+**Parameters**
+
+The following parameters should be set in the URL:
+
+- ``user_id`` - The fully qualified MXID: for example, ``@user:server.com``. The user must
+  be local.
+
+**Response**
+
+The following fields are returned in the JSON response body:
+
+- ``messages_per_second`` - integer - The number of actions that can
+  be performed in a second. `0` mean that ratelimiting is disabled for this user.
+- ``burst_count`` - integer - How many actions that can be performed before
+  being limited.
+
+If **no** custom ratelimit is set, an empty JSON dict is returned.
+
+.. code:: json
+
+    {}
+
+Set ratelimit
+-------------
+
+The API is::
+
+  POST /_synapse/admin/v1/users/<user_id>/override_ratelimit
+
+To use it, you will need to authenticate by providing an ``access_token`` for a
+server admin: see `README.rst <README.rst>`_.
+
+A response body like the following is returned:
+
+.. code:: json
+
+    {
+      "messages_per_second": 0,
+      "burst_count": 0
+    }
+
+**Parameters**
+
+The following parameters should be set in the URL:
+
+- ``user_id`` - The fully qualified MXID: for example, ``@user:server.com``. The user must
+  be local.
+
+Body parameters:
+
+- ``messages_per_second`` - positive integer, optional. The number of actions that can
+  be performed in a second. Defaults to ``0``.
+- ``burst_count`` - positive integer, optional. How many actions that can be performed
+  before being limited. Defaults to ``0``.
+
+To disable users' ratelimit set both values to ``0``.
+
+**Response**
+
+The following fields are returned in the JSON response body:
+
+- ``messages_per_second`` - integer - The number of actions that can
+  be performed in a second.
+- ``burst_count`` - integer - How many actions that can be performed before
+  being limited.
+
+Delete ratelimit
+----------------
+
+The API is::
+
+  DELETE /_synapse/admin/v1/users/<user_id>/override_ratelimit
+
+To use it, you will need to authenticate by providing an ``access_token`` for a
+server admin: see `README.rst <README.rst>`_.
+
+An empty JSON dict is returned.
+
+.. code:: json
+
+    {}
+
+**Parameters**
+
+The following parameters should be set in the URL:
+
+- ``user_id`` - The fully qualified MXID: for example, ``@user:server.com``. The user must
+  be local.
+

docs/admin_api/version_api.rst

@@ -1,21 +1,20 @@
-# Version API
+Version API
+===========
 
 This API returns the running Synapse version and the Python version
 on which Synapse is being run. This is useful when a Synapse instance
 is behind a proxy that does not forward the 'Server' header (which also
 contains Synapse version information).
 
-The api is:
+The api is::
 
-```
-GET /_synapse/admin/v1/server_version
-```
+    GET /_synapse/admin/v1/server_version
 
 It returns a JSON body like the following:
 
-```json
-{
-    "server_version": "0.99.2rc1 (b=develop, abcdef123)",
-    "python_version": "3.6.8"
-}
-```
+.. code:: json
+
+    {
+        "server_version": "0.99.2rc1 (b=develop, abcdef123)",
+        "python_version": "3.6.8"
+    }

docs/dev/git.md

@@ -122,15 +122,15 @@ So, what counts as a more- or less-stable branch? A little reflection will show
 that our active branches are ordered thus, from more-stable to less-stable:
 
  * `master` (tracks our last release).
- * `release-vX.Y` (the branch where we prepare the next release)<sup
+ * `release-vX.Y.Z` (the branch where we prepare the next release)<sup
    id="a3">[3](#f3)</sup>.
  * PR branches which are targeting the release.
  * `develop` (our "mainline" branch containing our bleeding-edge).
  * regular PR branches.
 
 The corollary is: if you have a bugfix that needs to land in both
-`release-vX.Y` *and* `develop`, then you should base your PR on
-`release-vX.Y`, get it merged there, and then merge from `release-vX.Y` to
+`release-vX.Y.Z` *and* `develop`, then you should base your PR on
+`release-vX.Y.Z`, get it merged there, and then merge from `release-vX.Y.Z` to
 `develop`. (If a fix lands in `develop` and we later need it in a
 release-branch, we can of course cherry-pick it, but landing it in the release
 branch first helps reduce the chance of annoying conflicts.)
@@ -145,4 +145,4 @@ most intuitive name. [^](#a1)
 
 <b id="f3">[3]</b>: Very, very occasionally (I think this has happened once in
 the history of Synapse), we've had two releases in flight at once. Obviously,
-`release-v1.2` is more-stable than `release-v1.3`. [^](#a3)
+`release-v1.2.3` is more-stable than `release-v1.3.0`. [^](#a3)

docs/development/contributing_guide.md

@@ -1,7 +0,0 @@
-<!--
-  Include the contents of CONTRIBUTING.md from the project root (where GitHub likes it 
-  to be)
--->
-# Contributing
-
-{{#include ../../CONTRIBUTING.md}}

docs/development/internal_documentation/README.md

@@ -1,12 +0,0 @@
-# Internal Documentation
-
-This section covers implementation documentation for various parts of Synapse.
-
-If a developer is planning to make a change to a feature of Synapse, it can be useful for
-general documentation of how that feature is implemented to be available. This saves the
-developer time in place of needing to understand how the feature works by reading the
-code.
-
-Documentation that would be more useful for the perspective of a system administrator,
-rather than a developer who's intending to change to code, should instead be placed
-under the Usage section of the documentation.

docs/favicon.svg

@@ -1,58 +0,0 @@
-<?xml version="1.0" encoding="UTF-8" standalone="no"?>
-<svg
-   xmlns:dc="http://purl.org/dc/elements/1.1/"
-   xmlns:cc="http://creativecommons.org/ns#"
-   xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"
-   xmlns:svg="http://www.w3.org/2000/svg"
-   xmlns="http://www.w3.org/2000/svg"
-   xmlns:sodipodi="http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd"
-   xmlns:inkscape="http://www.inkscape.org/namespaces/inkscape"
-   viewBox="0 0 199.7 184.2"
-   version="1.1"
-   id="svg62"
-   sodipodi:docname="mdbook-favicon.svg"
-   inkscape:version="1.0.2 (e86c870879, 2021-01-15, custom)">
-  <metadata
-     id="metadata68">
-    <rdf:RDF>
-      <cc:Work
-         rdf:about="">
-        <dc:format>image/svg+xml</dc:format>
-        <dc:type
-           rdf:resource="http://purl.org/dc/dcmitype/StillImage" />
-      </cc:Work>
-    </rdf:RDF>
-  </metadata>
-  <defs
-     id="defs66" />
-  <sodipodi:namedview
-     pagecolor="#ffffff"
-     bordercolor="#666666"
-     borderopacity="1"
-     objecttolerance="10"
-     gridtolerance="10"
-     guidetolerance="10"
-     inkscape:pageopacity="0"
-     inkscape:pageshadow="2"
-     inkscape:window-width="1920"
-     inkscape:window-height="1026"
-     id="namedview64"
-     showgrid="false"
-     inkscape:zoom="3.2245912"
-     inkscape:cx="84.790185"
-     inkscape:cy="117.96478"
-     inkscape:window-x="0"
-     inkscape:window-y="0"
-     inkscape:window-maximized="1"
-     inkscape:current-layer="svg62" />
-  <style
-     id="style58">
-    @media (prefers-color-scheme: dark) {
-      svg { fill: white; }
-    }
-  </style>
-  <path
-     d="m 189.5,36.8 c 0.2,2.8 0,5.1 -0.6,6.8 L 153,162 c -0.6,2.1 -2,3.7 -4.2,5 -2.2,1.2 -4.4,1.9 -6.7,1.9 H 31.4 c -9.6,0 -15.3,-2.8 -17.3,-8.4 -0.8,-2.2 -0.8,-3.9 0.1,-5.2 0.9,-1.2 2.4,-1.8 4.6,-1.8 H 123 c 7.4,0 12.6,-1.4 15.4,-4.1 2.8,-2.7 5.7,-8.9 8.6,-18.4 L 179.9,22.4 c 1.8,-5.9 1,-11.1 -2.2,-15.6 C 174.5,2.3 169.9,0 164,0 H 72.7 c -1,0 -3.1,0.4 -6.1,1.1 L 66.7,0.7 C 64.5,0.2 62.6,0 61,0.1 c -1.6,0.1 -3,0.5 -4.3,1.4 -1.3,0.9 -2.4,1.8 -3.2,2.8 -0.8,1 -1.5,2.2 -2.3,3.8 -0.8,1.6 -1.4,3 -1.9,4.3 -0.5,1.3 -1.1,2.7 -1.8,4.2 -0.7,1.5 -1.3,2.7 -2,3.7 -0.5,0.6 -1.2,1.5 -2,2.5 -0.8,1 -1.6,2 -2.2,2.8 -0.6,0.8 -0.9,1.5 -1.1,2.2 -0.2,0.7 -0.1,1.8 0.2,3.2 0.3,1.4 0.4,2.4 0.4,3.1 -0.3,3 -1.4,6.9 -3.3,11.6 -1.9,4.7 -3.6,8.1 -5.1,10.1 -0.3,0.4 -1.2,1.3 -2.6,2.7 -1.4,1.4 -2.3,2.6 -2.6,3.7 -0.3,0.4 -0.3,1.5 -0.1,3.4 0.3,1.8 0.4,3.1 0.3,3.8 -0.3,2.7 -1.3,6.3 -3,10.8 -2.406801,6.370944 -3.4,8.2 -5,11 -0.2,0.5 -0.9,1.4 -2,2.8 -1.1,1.4 -1.8,2.5 -2,3.4 -0.2,0.6 -0.1,1.8 0.1,3.4 0.2,1.6 0.2,2.8 -0.1,3.6 -0.6,3 -1.8,6.7 -3.6,11 -1.8,4.3 -3.6,7.9 -5.4,11 -0.5,0.8 -1.1,1.7 -2,2.8 -0.8,1.1 -1.5,2 -2,2.8 -0.5,0.8 -0.8,1.6 -1,2.5 -0.1,0.5 0,1.3 0.4,2.3 0.3,1.1 0.4,1.9 0.4,2.6 -0.1,1.1 -0.2,2.6 -0.5,4.4 -0.2,1.8 -0.4,2.9 -0.4,3.2 -1.8,4.8 -1.7,9.9 0.2,15.2 2.2,6.2 6.2,11.5 11.9,15.8 5.7,4.3 11.7,6.4 17.8,6.4 h 110.7 c 5.2,0 10.1,-1.7 14.7,-5.2 4.6,-3.5 7.7,-7.8 9.2,-12.9 l 33,-108.6 c 1.8,-5.8 1,-10.9 -2.2,-15.5 -1.7,-2.5 -4,-4.2 -7.1,-5.4 z M 38.14858,105.59813 60.882735,41.992545 h 10.8 c 6.340631,0 33.351895,0.778957 70.804135,0.970479 -18.18245,63.254766 0,0 -18.18245,63.254766 -23.00947,-0.10382 -63.362955,-0.6218 -72.55584,-0.51966 -18,0.2 -13.6,-0.1 -13.6,-0.1 z m 80.621,-5.891206 c 15.19043,-50.034423 0,1e-5 15.19043,-50.034423 l -11.90624,-0.13228 2.73304,-9.302941 -44.32863,0.07339 -2.532953,8.036036 -11.321128,-0.18864 -17.955519,51.440073 c 0.02698,0.027 4.954586,0.0514 12.187488,0.0717 l -2.997994,9.804886 c 11.36463,0.0271 1.219679,-0.0736 46.117666,-0.31499 l 2.65246,-9.571696 c 7.08021,0.14819 11.59705,0.13117 12.16138,0.1189 z m -56.149615,-3.855606 13.7,-42.5 h 9.8 l 1.194896,32.99936 23.205109,-32.99936 h 9.9 l -13.6,42.5 h -7.099996 l 12.499996,-35.4 -24.50001,35.4 h -6.799995 l -0.8,-35 -10.8,35 z"
-     id="path60"
-     sodipodi:nodetypes="ccccssccsssccsssccsssssscsssscssscccscscscsccsccccccssssccccccsccsccccccccccccccccccccccccccccc" />
-</svg>

docs/opentracing.md

@@ -42,17 +42,17 @@ To receive OpenTracing spans, start up a Jaeger server. This can be done
 using docker like so:
 
 ```sh
-docker run -d --name jaeger \
+docker run -d --name jaeger
   -p 6831:6831/udp \
   -p 6832:6832/udp \
   -p 5778:5778 \
   -p 16686:16686 \
   -p 14268:14268 \
-  jaegertracing/all-in-one:1
+  jaegertracing/all-in-one:1.13
 ```
 
 Latest documentation is probably at
-https://www.jaegertracing.io/docs/latest/getting-started.
+<https://www.jaegertracing.io/docs/1.13/getting-started/>
 
 ## Enable OpenTracing in Synapse
 
@@ -62,7 +62,7 @@ as shown in the [sample config](./sample_config.yaml). For example:
 
 ```yaml
 opentracing:
-  enabled: true
+  tracer_enabled: true
   homeserver_whitelist:
     - "mytrustedhomeserver.org"
     - "*.myotherhomeservers.com"
@@ -90,4 +90,4 @@ to two problems, namely:
 ## Configuring Jaeger
 
 Sampling strategies can be set as in this document:
-<https://www.jaegertracing.io/docs/latest/sampling/>.
+<https://www.jaegertracing.io/docs/1.13/sampling/>

docs/postgres.md

@@ -1,6 +1,6 @@
 # Using Postgres
 
-Synapse supports PostgreSQL versions 9.6 or later.
+Postgres version 9.5 or later is known to work.
 
 ## Install postgres client libraries
 
@@ -33,15 +33,28 @@ Assuming your PostgreSQL database user is called `postgres`, first authenticate
     # Or, if your system uses sudo to get administrative rights
     sudo -u postgres bash
 
-Then, create a postgres user and a database with:
+Then, create a user ``synapse_user`` with:
 
-    # this will prompt for a password for the new user
     createuser --pwprompt synapse_user
 
-    createdb --encoding=UTF8 --locale=C --template=template0 --owner=synapse_user synapse
+Before you can authenticate with the `synapse_user`, you must create a
+database that it can access. To create a database, first connect to the
+database with your database user:
 
-The above will create a user called `synapse_user`, and a database called
-`synapse`.
+    su - postgres # Or: sudo -u postgres bash
+    psql
+
+and then run:
+
+    CREATE DATABASE synapse
+     ENCODING 'UTF8'
+     LC_COLLATE='C'
+     LC_CTYPE='C'
+     template=template0
+     OWNER synapse_user;
+
+This would create an appropriate database named `synapse` owned by the
+`synapse_user` user (which must already have been created as above).
 
 Note that the PostgreSQL database *must* have the correct encoding set
 (as shown above), otherwise it will not be able to store UTF8 strings.
@@ -50,6 +63,79 @@ You may need to enable password authentication so `synapse_user` can
 connect to the database. See
 <https://www.postgresql.org/docs/current/auth-pg-hba-conf.html>.
 
+If you get an error along the lines of `FATAL:  Ident authentication failed for
+user "synapse_user"`, you may need to use an authentication method other than
+`ident`:
+
+* If the `synapse_user` user has a password, add the password to the `database:`
+  section of `homeserver.yaml`. Then add the following to `pg_hba.conf`:
+
+  ```
+  host    synapse     synapse_user    ::1/128     md5  # or `scram-sha-256` instead of `md5` if you use that
+  ```
+
+* If the `synapse_user` user does not have a password, then a password doesn't
+  have to be added to `homeserver.yaml`. But the following does need to be added
+  to `pg_hba.conf`:
+
+  ```
+  host    synapse     synapse_user    ::1/128     trust
+  ```
+
+Note that line order matters in `pg_hba.conf`, so make sure that if you do add a
+new line, it is inserted before:
+
+```
+host    all         all             ::1/128     ident
+```
+
+### Fixing incorrect `COLLATE` or `CTYPE`
+
+Synapse will refuse to set up a new database if it has the wrong values of
+`COLLATE` and `CTYPE` set, and will log warnings on existing databases. Using
+different locales can cause issues if the locale library is updated from
+underneath the database, or if a different version of the locale is used on any
+replicas.
+
+The safest way to fix the issue is to take a dump and recreate the database with
+the correct `COLLATE` and `CTYPE` parameters (as shown above). It is also possible to change the
+parameters on a live database and run a `REINDEX` on the entire database,
+however extreme care must be taken to avoid database corruption.
+
+Note that the above may fail with an error about duplicate rows if corruption
+has already occurred, and such duplicate rows will need to be manually removed.
+
+
+## Fixing inconsistent sequences error
+
+Synapse uses Postgres sequences to generate IDs for various tables. A sequence
+and associated table can get out of sync if, for example, Synapse has been
+downgraded and then upgraded again.
+
+To fix the issue shut down Synapse (including any and all workers) and run the
+SQL command included in the error message. Once done Synapse should start
+successfully.
+
+
+## Tuning Postgres
+
+The default settings should be fine for most deployments. For larger
+scale deployments tuning some of the settings is recommended, details of
+which can be found at
+<https://wiki.postgresql.org/wiki/Tuning_Your_PostgreSQL_Server>.
+
+In particular, we've found tuning the following values helpful for
+performance:
+
+-   `shared_buffers`
+-   `effective_cache_size`
+-   `work_mem`
+-   `maintenance_work_mem`
+-   `autovacuum_work_mem`
+
+Note that the appropriate values for those fields depend on the amount
+of free memory the database host has available.
+
 ## Synapse config
 
 When you are ready to start using PostgreSQL, edit the `database`
@@ -79,42 +165,18 @@ may block for an extended period while it waits for a response from the
 database server. Example values might be:
 
 ```yaml
-database:
-  args:
-    # ... as above
+# seconds of inactivity after which TCP should send a keepalive message to the server
+keepalives_idle: 10
 
-    # seconds of inactivity after which TCP should send a keepalive message to the server
-    keepalives_idle: 10
+# the number of seconds after which a TCP keepalive message that is not
+# acknowledged by the server should be retransmitted
+keepalives_interval: 10
 
-    # the number of seconds after which a TCP keepalive message that is not
-    # acknowledged by the server should be retransmitted
-    keepalives_interval: 10
-
-    # the number of TCP keepalives that can be lost before the client's connection
-    # to the server is considered dead
-    keepalives_count: 3
+# the number of TCP keepalives that can be lost before the client's connection
+# to the server is considered dead
+keepalives_count: 3
 ```
 
-## Tuning Postgres
-
-The default settings should be fine for most deployments. For larger
-scale deployments tuning some of the settings is recommended, details of
-which can be found at
-<https://wiki.postgresql.org/wiki/Tuning_Your_PostgreSQL_Server>.
-
-In particular, we've found tuning the following values helpful for
-performance:
-
--   `shared_buffers`
--   `effective_cache_size`
--   `work_mem`
--   `maintenance_work_mem`
--   `autovacuum_work_mem`
-
-Note that the appropriate values for those fields depend on the amount
-of free memory the database host has available.
-
-
 ## Porting from SQLite
 
 ### Overview
@@ -123,8 +185,9 @@ The script `synapse_port_db` allows porting an existing synapse server
 backed by SQLite to using PostgreSQL. This is done in as a two phase
 process:
 
-1.  Copy the existing SQLite database to a separate location and run
-    the port script against that offline database.
+1.  Copy the existing SQLite database to a separate location (while the
+    server is down) and running the port script against that offline
+    database.
 2.  Shut down the server. Rerun the port script to port any data that
     has come in since taking the first snapshot. Restart server against
     the PostgreSQL database.
@@ -182,60 +245,3 @@ PostgreSQL database configuration file `homeserver-postgres.yaml`:
     ./synctl start
 
 Synapse should now be running against PostgreSQL.
-
-
-## Troubleshooting
-
-### Alternative auth methods
-
-If you get an error along the lines of `FATAL:  Ident authentication failed for
-user "synapse_user"`, you may need to use an authentication method other than
-`ident`:
-
-* If the `synapse_user` user has a password, add the password to the `database:`
-  section of `homeserver.yaml`. Then add the following to `pg_hba.conf`:
-
-  ```
-  host    synapse     synapse_user    ::1/128     md5  # or `scram-sha-256` instead of `md5` if you use that
-  ```
-
-* If the `synapse_user` user does not have a password, then a password doesn't
-  have to be added to `homeserver.yaml`. But the following does need to be added
-  to `pg_hba.conf`:
-
-  ```
-  host    synapse     synapse_user    ::1/128     trust
-  ```
-
-Note that line order matters in `pg_hba.conf`, so make sure that if you do add a
-new line, it is inserted before:
-
-```
-host    all         all             ::1/128     ident
-```
-
-### Fixing incorrect `COLLATE` or `CTYPE`
-
-Synapse will refuse to set up a new database if it has the wrong values of
-`COLLATE` and `CTYPE` set, and will log warnings on existing databases. Using
-different locales can cause issues if the locale library is updated from
-underneath the database, or if a different version of the locale is used on any
-replicas.
-
-The safest way to fix the issue is to dump the database and recreate it with
-the correct locale parameter (as shown above). It is also possible to change the
-parameters on a live database and run a `REINDEX` on the entire database,
-however extreme care must be taken to avoid database corruption.
-
-Note that the above may fail with an error about duplicate rows if corruption
-has already occurred, and such duplicate rows will need to be manually removed.
-
-### Fixing inconsistent sequences error
-
-Synapse uses Postgres sequences to generate IDs for various tables. A sequence
-and associated table can get out of sync if, for example, Synapse has been
-downgraded and then upgraded again.
-
-To fix the issue shut down Synapse (including any and all workers) and run the
-SQL command included in the error message. Once done Synapse should start
-successfully.

docs/presence_router_module.md

@@ -28,11 +28,7 @@ async def ModuleApi.send_local_online_presence_to(users: Iterable[str]) -> None
 which can be given a list of local or remote MXIDs to broadcast known, online user
 presence to (for those users that the receiving user is considered interested in). 
 It does not include state for users who are currently offline, and it can only be
-called on workers that support sending federation. Additionally, this method must
-only be called from the process that has been configured to write to the
-the [presence stream](https://github.com/matrix-org/synapse/blob/master/docs/workers.md#stream-writers).
-By default, this is the main process, but another worker can be configured to do
-so.
+called on workers that support sending federation.
 
 ### Module structure
 

docs/sample_config.yaml

@@ -152,16 +152,6 @@ presence:
 #
 #gc_thresholds: [700, 10, 10]
 
-# The minimum time in seconds between each GC for a generation, regardless of
-# the GC thresholds. This ensures that we don't do GC too frequently.
-#
-# A value of `[1s, 10s, 30s]` indicates that a second must pass between consecutive
-# generation 0 GCs, etc.
-#
-# Defaults to `[1s, 10s, 30s]`.
-#
-#gc_min_interval: [0.5s, 30s, 1m]
-
 # Set the limit on the returned events in the timeline in the get
 # and sync operations. The default value is 100. -1 means no upper limit.
 #
@@ -683,6 +673,33 @@ acme:
     #
     account_key_file: DATADIR/acme_account.key
 
+# List of allowed TLS fingerprints for this server to publish along
+# with the signing keys for this server. Other matrix servers that
+# make HTTPS requests to this server will check that the TLS
+# certificates returned by this server match one of the fingerprints.
+#
+# Synapse automatically adds the fingerprint of its own certificate
+# to the list. So if federation traffic is handled directly by synapse
+# then no modification to the list is required.
+#
+# If synapse is run behind a load balancer that handles the TLS then it
+# will be necessary to add the fingerprints of the certificates used by
+# the loadbalancers to this list if they are different to the one
+# synapse is using.
+#
+# Homeservers are permitted to cache the list of TLS fingerprints
+# returned in the key responses up to the "valid_until_ts" returned in
+# key. It may be necessary to publish the fingerprints of a new
+# certificate and wait until the "valid_until_ts" of the previous key
+# responses have passed before deploying it.
+#
+# You can calculate a fingerprint from a given TLS listener via:
+# openssl s_client -connect $host:$port < /dev/null 2> /dev/null |
+#   openssl x509 -outform DER | openssl sha256 -binary | base64 | tr -d '='
+# or by checking matrix.org/federationtester/api/report?server_name=$host
+#
+#tls_fingerprints: [{"sha256": "<base64_encoded_sha256_fingerprint>"}]
+
 
 ## Federation ##
 
@@ -714,12 +731,6 @@ acme:
 #
 #allow_profile_lookup_over_federation: false
 
-# Uncomment to disable device display name lookup over federation. By default, the
-# Federation API allows other homeservers to obtain device display names of any user
-# on this homeserver. Defaults to 'true'.
-#
-#allow_device_name_lookup_over_federation: false
-
 
 ## Caching ##
 
@@ -799,7 +810,6 @@ caches:
 #    password: secretpassword
 #    database: synapse
 #    host: localhost
-#    port: 5432
 #    cp_min: 5
 #    cp_max: 10
 #
@@ -1165,6 +1175,69 @@ url_preview_accept_language:
 #
 #enable_registration: false
 
+# Optional account validity configuration. This allows for accounts to be denied
+# any request after a given period.
+#
+# Once this feature is enabled, Synapse will look for registered users without an
+# expiration date at startup and will add one to every account it found using the
+# current settings at that time.
+# This means that, if a validity period is set, and Synapse is restarted (it will
+# then derive an expiration date from the current validity period), and some time
+# after that the validity period changes and Synapse is restarted, the users'
+# expiration dates won't be updated unless their account is manually renewed. This
+# date will be randomly selected within a range [now + period - d ; now + period],
+# where d is equal to 10% of the validity period.
+#
+account_validity:
+  # The account validity feature is disabled by default. Uncomment the
+  # following line to enable it.
+  #
+  #enabled: true
+
+  # The period after which an account is valid after its registration. When
+  # renewing the account, its validity period will be extended by this amount
+  # of time. This parameter is required when using the account validity
+  # feature.
+  #
+  #period: 6w
+
+  # The amount of time before an account's expiry date at which Synapse will
+  # send an email to the account's email address with a renewal link. By
+  # default, no such emails are sent.
+  #
+  # If you enable this setting, you will also need to fill out the 'email' and
+  # 'public_baseurl' configuration sections.
+  #
+  #renew_at: 1w
+
+  # The subject of the email sent out with the renewal link. '%(app)s' can be
+  # used as a placeholder for the 'app_name' parameter from the 'email'
+  # section.
+  #
+  # Note that the placeholder must be written '%(app)s', including the
+  # trailing 's'.
+  #
+  # If this is not set, a default value is used.
+  #
+  #renew_email_subject: "Renew your %(app)s account"
+
+  # Directory in which Synapse will try to find templates for the HTML files to
+  # serve to the user when trying to renew an account. If not set, default
+  # templates from within the Synapse package will be used.
+  #
+  #template_dir: "res/templates"
+
+  # File within 'template_dir' giving the HTML to be displayed to the user after
+  # they successfully renewed their account. If not set, default text is used.
+  #
+  #account_renewed_html_path: "account_renewed.html"
+
+  # File within 'template_dir' giving the HTML to be displayed when the user
+  # tries to renew an account with an invalid renewal token. If not set,
+  # default text is used.
+  #
+  #invalid_token_html_path: "invalid_token.html"
+
 # Time that a user's session remains valid for, after they log in.
 #
 # Note that this is not currently compatible with guest logins.
@@ -1359,91 +1432,6 @@ account_threepid_delegates:
 #auto_join_rooms_for_guests: false
 
 
-## Account Validity ##
-
-# Optional account validity configuration. This allows for accounts to be denied
-# any request after a given period.
-#
-# Once this feature is enabled, Synapse will look for registered users without an
-# expiration date at startup and will add one to every account it found using the
-# current settings at that time.
-# This means that, if a validity period is set, and Synapse is restarted (it will
-# then derive an expiration date from the current validity period), and some time
-# after that the validity period changes and Synapse is restarted, the users'
-# expiration dates won't be updated unless their account is manually renewed. This
-# date will be randomly selected within a range [now + period - d ; now + period],
-# where d is equal to 10% of the validity period.
-#
-account_validity:
-  # The account validity feature is disabled by default. Uncomment the
-  # following line to enable it.
-  #
-  #enabled: true
-
-  # The period after which an account is valid after its registration. When
-  # renewing the account, its validity period will be extended by this amount
-  # of time. This parameter is required when using the account validity
-  # feature.
-  #
-  #period: 6w
-
-  # The amount of time before an account's expiry date at which Synapse will
-  # send an email to the account's email address with a renewal link. By
-  # default, no such emails are sent.
-  #
-  # If you enable this setting, you will also need to fill out the 'email' and
-  # 'public_baseurl' configuration sections.
-  #
-  #renew_at: 1w
-
-  # The subject of the email sent out with the renewal link. '%(app)s' can be
-  # used as a placeholder for the 'app_name' parameter from the 'email'
-  # section.
-  #
-  # Note that the placeholder must be written '%(app)s', including the
-  # trailing 's'.
-  #
-  # If this is not set, a default value is used.
-  #
-  #renew_email_subject: "Renew your %(app)s account"
-
-  # Directory in which Synapse will try to find templates for the HTML files to
-  # serve to the user when trying to renew an account. If not set, default
-  # templates from within the Synapse package will be used.
-  #
-  # The currently available templates are:
-  #
-  # * account_renewed.html: Displayed to the user after they have successfully
-  #       renewed their account.
-  #
-  # * account_previously_renewed.html: Displayed to the user if they attempt to
-  #       renew their account with a token that is valid, but that has already
-  #       been used. In this case the account is not renewed again.
-  #
-  # * invalid_token.html: Displayed to the user when they try to renew an account
-  #       with an unknown or invalid renewal token.
-  #
-  # See https://github.com/matrix-org/synapse/tree/master/synapse/res/templates for
-  # default template contents.
-  #
-  # The file name of some of these templates can be configured below for legacy
-  # reasons.
-  #
-  #template_dir: "res/templates"
-
-  # A custom file name for the 'account_renewed.html' template.
-  #
-  # If not set, the file is assumed to be named "account_renewed.html".
-  #
-  #account_renewed_html_path: "account_renewed.html"
-
-  # A custom file name for the 'invalid_token.html' template.
-  #
-  # If not set, the file is assumed to be named "invalid_token.html".
-  #
-  #invalid_token_html_path: "invalid_token.html"
-
-
 ## Metrics ###
 
 # Enable collection and rendering of performance metrics
@@ -1494,7 +1482,6 @@ room_prejoin_state:
    # - m.room.avatar
    # - m.room.encryption
    # - m.room.name
-   # - m.room.create
    #
    # Uncomment the following to disable these defaults (so that only the event
    # types listed in 'additional_event_types' are shared). Defaults to 'false'.
@@ -1891,7 +1878,7 @@ saml2_config:
 #       sub-properties:
 #
 #       module: The class name of a custom mapping module. Default is
-#           'synapse.handlers.oidc.JinjaOidcMappingProvider'.
+#           'synapse.handlers.oidc_handler.JinjaOidcMappingProvider'.
 #           See https://github.com/matrix-org/synapse/blob/master/docs/sso_mapping_providers.md#openid-mapping-providers
 #           for information on implementing a custom mapping provider.
 #
@@ -2818,8 +2805,7 @@ opentracing:
     #enabled: true
 
     # The list of homeservers we wish to send and receive span contexts and span baggage.
-    # See docs/opentracing.rst.
-    #
+    # See docs/opentracing.rst
     # This is a list of regexes which are matched against the server_name of the
     # homeserver.
     #
@@ -2828,26 +2814,19 @@ opentracing:
     #homeserver_whitelist:
     #  - ".*"
 
-    # A list of the matrix IDs of users whose requests will always be traced,
-    # even if the tracing system would otherwise drop the traces due to
-    # probabilistic sampling.
-    #
-    # By default, the list is empty.
-    #
-    #force_tracing_for_users:
-    #  - "@user1:server_name"
-    #  - "@user2:server_name"
-
     # Jaeger can be configured to sample traces at different rates.
     # All configuration options provided by Jaeger can be set here.
-    # Jaeger's configuration is mostly related to trace sampling which
+    # Jaeger's configuration mostly related to trace sampling which
     # is documented here:
-    # https://www.jaegertracing.io/docs/latest/sampling/.
+    # https://www.jaegertracing.io/docs/1.13/sampling/.
     #
     #jaeger_config:
     #  sampler:
     #    type: const
     #    param: 1
+
+    #  Logging whether spans were started and reported
+    #
     #  logging:
     #    false
 

docs/setup/installation.md

@@ -1,7 +0,0 @@
-<!--
-  Include the contents of INSTALL.md from the project root without moving it, which may
-  break links around the internet. Additionally, note that SUMMARY.md is unable to 
-  directly link to content outside of the docs/ directory. So we use this file as a 
-  redirection.
--->
-{{#include ../../INSTALL.md}}

docs/sso_mapping_providers.md

@@ -67,8 +67,8 @@ A custom mapping provider must specify the following methods:
     - Arguments:
       - `userinfo` - A `authlib.oidc.core.claims.UserInfo` object to extract user
                      information from.
-    - This method must return a string, which is the unique, immutable identifier
-      for the user. Commonly the `sub` claim of the response.
+    - This method must return a string, which is the unique identifier for the
+      user. Commonly the ``sub`` claim of the response.
 * `map_user_attributes(self, userinfo, token, failures)`
     - This method must be async.
     - Arguments:
@@ -87,9 +87,7 @@ A custom mapping provider must specify the following methods:
                      `localpart` value, such as `john.doe1`.
     - Returns a dictionary with two keys:
       - `localpart`: A string, used to generate the Matrix ID. If this is
-        `None`, the user is prompted to pick their own username. This is only used
-        during a user's first login. Once a localpart has been associated with a
-        remote user ID (see `get_remote_user_id`) it cannot be updated.
+        `None`, the user is prompted to pick their own username.
       - `displayname`: An optional string, the display name for the user.
 * `get_extra_attributes(self, userinfo, token)`
     - This method must be async.
@@ -108,7 +106,7 @@ A custom mapping provider must specify the following methods:
 
 Synapse has a built-in OpenID mapping provider if a custom provider isn't
 specified in the config. It is located at
-[`synapse.handlers.oidc.JinjaOidcMappingProvider`](../synapse/handlers/oidc.py).
+[`synapse.handlers.oidc_handler.JinjaOidcMappingProvider`](../synapse/handlers/oidc_handler.py).
 
 ## SAML Mapping Providers
 
@@ -155,8 +153,8 @@ A custom mapping provider must specify the following methods:
                           information from.
       - `client_redirect_url` - A string, the URL that the client will be
                                 redirected to.
-    - This method must return a string, which is the unique, immutable identifier
-      for the user. Commonly the `uid` claim of the response.
+    - This method must return a string, which is the unique identifier for the
+      user. Commonly the ``uid`` claim of the response.
 * `saml_response_to_user_attributes(self, saml_response, failures, client_redirect_url)`
     - Arguments:
       - `saml_response` - A `saml2.response.AuthnResponse` object to extract user
@@ -174,10 +172,8 @@ A custom mapping provider must specify the following methods:
                                 redirected to.
     - This method must return a dictionary, which will then be used by Synapse
       to build a new user. The following keys are allowed:
-       * `mxid_localpart` - A string, the mxid localpart of the new user. If this is
-         `None`, the user is prompted to pick their own username. This is only used
-         during a user's first login. Once a localpart has been associated with a
-         remote user ID (see `get_remote_user_id`) it cannot be updated.
+       * `mxid_localpart` - The mxid localpart of the new user.  If this is
+         `None`, the user is prompted to pick their own username.
        * `displayname` - The displayname of the new user. If not provided, will default to
                          the value of `mxid_localpart`.
        * `emails` - A list of emails for the new user. If not provided, will
@@ -194,4 +190,4 @@ A custom mapping provider must specify the following methods:
 
 Synapse has a built-in SAML mapping provider if a custom provider isn't
 specified in the config. It is located at
-[`synapse.handlers.saml.DefaultSamlMappingProvider`](../synapse/handlers/saml.py).
+[`synapse.handlers.saml_handler.DefaultSamlMappingProvider`](../synapse/handlers/saml_handler.py).

docs/turn-howto.md

@@ -4,7 +4,7 @@ This document explains how to enable VoIP relaying on your Home Server with
 TURN.
 
 The synapse Matrix Home Server supports integration with TURN server via the
-[TURN server REST API](<https://tools.ietf.org/html/draft-uberti-behave-turn-rest-00>). This
+[TURN server REST API](<http://tools.ietf.org/html/draft-uberti-behave-turn-rest-00>). This
 allows the Home Server to generate credentials that are valid for use on the
 TURN server through the use of a secret shared between the Home Server and the
 TURN server.

docs/upgrading/README.md

@@ -1,7 +0,0 @@
-<!--
-  Include the contents of UPGRADE.rst from the project root without moving it, which may
-  break links around the internet. Additionally, note that SUMMARY.md is unable to 
-  directly link to content outside of the docs/ directory. So we use this file as a 
-  redirection.
--->
-{{#include ../../UPGRADE.rst}}

docs/usage/administration/README.md

@@ -1,7 +0,0 @@
-# Administration
-
-This section contains information on managing your Synapse homeserver. This includes:
-
-* Managing users, rooms and media via the Admin API.
-* Setting up metrics and monitoring to give you insight into your homeserver's health.
-* Configuring structured logging.

docs/usage/administration/admin_api/README.md

@@ -1,29 +0,0 @@
-# The Admin API
-
-## Authenticate as a server admin
-
-Many of the API calls in the admin api will require an `access_token` for a
-server admin. (Note that a server admin is distinct from a room admin.)
-
-A user can be marked as a server admin by updating the database directly, e.g.:
-
-```sql
-UPDATE users SET admin = 1 WHERE name = '@foo:bar.com';
-```
-
-A new server admin user can also be created using the `register_new_matrix_user`
-command. This is a script that is located in the `scripts/` directory, or possibly
-already on your `$PATH` depending on how Synapse was installed.
-
-Finding your user's `access_token` is client-dependent, but will usually be shown in the client's settings.
-
-## Making an Admin API request
-Once you have your `access_token`, you will need to authenticate each request to an Admin API endpoint by
-providing the token as either a query parameter or a request header. To add it as a request header in cURL:
-
-```sh
-curl --header "Authorization: Bearer <access_token>" <the_rest_of_your_API_request>
-```
-
-For more details on access tokens in Matrix, please refer to the complete
-[matrix spec documentation](https://matrix.org/docs/spec/client_server/r0.6.1#using-access-tokens).

docs/usage/configuration/README.md

@@ -1,4 +0,0 @@
-# Configuration
-
-This section contains information on tweaking Synapse via the various options in the configuration file. A configuration
-file should have been generated when you [installed Synapse](../../setup/installation.html).

docs/usage/configuration/homeserver_sample_config.md

@@ -1,14 +0,0 @@
-# Homeserver Sample Configuration File
-
-Below is a sample homeserver configuration file. The homeserver configuration file 
-can be tweaked to change the behaviour of your homeserver. A restart of the server is 
-generally required to apply any changes made to this file.
-
-Note that the contents below are *not* intended to be copied and used as the basis for
-a real homeserver.yaml. Instead, if you are starting from scratch, please generate
-a fresh config using Synapse by following the instructions in
-[Installation](../../setup/installation.md).
-
-```yaml
-{{#include ../../sample_config.yaml}}
-```

docs/usage/configuration/logging_sample_config.md

@@ -1,14 +0,0 @@
-# Logging Sample Configuration File
-
-Below is a sample logging configuration file. This file can be tweaked to control how your
-homeserver will output logs. A restart of the server is generally required to apply any
-changes made to this file.
-
-Note that the contents below are *not* intended to be copied and used as the basis for
-a real homeserver.yaml. Instead, if you are starting from scratch, please generate
-a fresh config using Synapse by following the instructions in
-[Installation](../../setup/installation.md).
-
-```yaml
-{{#include ../../sample_log_config.yaml}}
-``__`

docs/usage/configuration/user_authentication/README.md

@@ -1,15 +0,0 @@
-# User Authentication
-
-Synapse supports multiple methods of authenticating users, either out-of-the-box or through custom pluggable
-authentication modules.
-
-Included in Synapse is support for authenticating users via:
-
-* A username and password.
-* An email address and password.
-* Single Sign-On through the SAML, Open ID Connect or CAS protocols.
-* JSON Web Tokens.
-* An administrator's shared secret.
-
-Synapse can additionally be extended to support custom authentication schemes through optional "password auth provider"
-modules.

docs/user_directory.md

@@ -7,6 +7,6 @@ who are present in a publicly viewable room present on the server.
 
 The directory info is stored in various tables, which can (typically after
 DB corruption) get stale or out of sync.  If this happens, for now the
-solution to fix it is to execute the SQL [here](https://github.com/matrix-org/synapse/blob/master/synapse/storage/schema/main/delta/53/user_dir_populate.sql)
+solution to fix it is to execute the SQL [here](../synapse/storage/databases/main/schema/delta/53/user_dir_populate.sql)
 and then restart synapse. This should then start a background task to
 flush the current tables and regenerate the directory.

docs/website_files/README.md

@@ -1,30 +0,0 @@
-# Documentation Website Files and Assets
-
-This directory contains extra files for modifying the look and functionality of 
-[mdbook](https://github.com/rust-lang/mdBook), the documentation software that's
-used to generate Synapse's documentation website.
-
-The configuration options in the `output.html` section of [book.toml](../../book.toml)
-point to additional JS/CSS in this directory that are added on each page load. In
-addition, the `theme` directory contains files that overwrite their counterparts in
-each of the default themes included with mdbook.
-
-Currently we use these files to generate a floating Table of Contents panel. The code for
-which was partially taken from
-[JorelAli/mdBook-pagetoc](https://github.com/JorelAli/mdBook-pagetoc/)
-before being modified such that it scrolls with the content of the page. This is handled
-by the `table-of-contents.js/css` files. The table of contents panel only appears on pages
-that have more than one header, as well as only appearing on desktop-sized monitors.
-
-We remove the navigation arrows which typically appear on the left and right side of the
-screen on desktop as they interfere with the table of contents. This is handled by
-the `remove-nav-buttons.css` file.
-
-Finally, we also stylise the chapter titles in the left sidebar by indenting them
-slightly so that they are more visually distinguishable from the section headers
-(the bold titles). This is done through the `indent-section-headers.css` file.
-
-More information can be found in mdbook's official documentation for
-[injecting page JS/CSS](https://rust-lang.github.io/mdBook/format/config.html)
-and
-[customising the default themes](https://rust-lang.github.io/mdBook/format/theme/index.html).

docs/website_files/indent-section-headers.css

@@ -1,7 +0,0 @@
-/*
- * Indents each chapter title in the left sidebar so that they aren't
- * at the same level as the section headers.
- */
-.chapter-item {
-    margin-left: 1em;
-}

docs/website_files/remove-nav-buttons.css

@@ -1,8 +0,0 @@
-/* Remove the prev, next chapter buttons as they interfere with the
- * table of contents.
- * Note that the table of contents only appears on desktop, thus we
- * only remove the desktop (wide) chapter buttons.
- */
-.nav-wide-wrapper {
-    display: none
-}

docs/website_files/table-of-contents.css

@@ -1,42 +0,0 @@
-@media only screen and (max-width:1439px) {
-    .sidetoc {
-        display: none;
-    }
-}
-
-@media only screen and (min-width:1440px) {
-    main {
-        position: relative;
-        margin-left: 100px !important;
-    }
-    .sidetoc {
-        margin-left: auto;
-        margin-right: auto;
-        left: calc(100% + (var(--content-max-width))/4 - 140px);
-        position: absolute;
-        text-align: right;
-    }
-    .pagetoc {
-        position: fixed;
-        width: 250px;
-        overflow: auto;
-        right: 20px;
-        height: calc(100% - var(--menu-bar-height));
-    }
-    .pagetoc a {
-        color: var(--fg) !important;
-        display: block;
-        padding: 5px 15px 5px 10px;
-        text-align: left;
-        text-decoration: none;
-    }
-    .pagetoc a:hover,
-    .pagetoc a.active {
-        background: var(--sidebar-bg) !important;
-        color: var(--sidebar-fg) !important;
-    }
-    .pagetoc .active {
-        background: var(--sidebar-bg);
-        color: var(--sidebar-fg);
-    }
-}

docs/website_files/table-of-contents.js

@@ -1,134 +0,0 @@
-const getPageToc = () => document.getElementsByClassName('pagetoc')[0];
-
-const pageToc = getPageToc();
-const pageTocChildren = [...pageToc.children];
-const headers = [...document.getElementsByClassName('header')];
-
-
-// Select highlighted item in ToC when clicking an item
-pageTocChildren.forEach(child => {
-    child.addEventHandler('click', () => {
-        pageTocChildren.forEach(child => {
-            child.classList.remove('active');
-        });
-        child.classList.add('active');
-    });
-});
-
-
-/**
- * Test whether a node is in the viewport
- */
-function isInViewport(node) {
-    const rect = node.getBoundingClientRect();
-    return rect.top >= 0 && rect.left >= 0 && rect.bottom <= (window.innerHeight || document.documentElement.clientHeight) && rect.right <= (window.innerWidth || document.documentElement.clientWidth);
-}
-
-
-/**
- * Set a new ToC entry.
- * Clear any previously highlighted ToC items, set the new one,
- * and adjust the ToC scroll position.
- */
-function setTocEntry() {
-    let activeEntry;
-    const pageTocChildren = [...getPageToc().children];
-
-    // Calculate which header is the current one at the top of screen
-    headers.forEach(header => {
-        if (window.pageYOffset >= header.offsetTop) {
-            activeEntry = header;
-        }
-    });
-
-    // Update selected item in ToC when scrolling
-    pageTocChildren.forEach(child => {
-        if (activeEntry.href.localeCompare(child.href) === 0) {
-            child.classList.add('active');
-        } else {
-            child.classList.remove('active');
-        }
-    });
-
-    let tocEntryForLocation = document.querySelector(`nav a[href="${activeEntry.href}"]`);
-    if (tocEntryForLocation) {
-        const headingForLocation = document.querySelector(activeEntry.hash);
-        if (headingForLocation && isInViewport(headingForLocation)) {
-            // Update ToC scroll
-            const nav = getPageToc();
-            const content = document.querySelector('html');
-            if (content.scrollTop !== 0) {
-                nav.scrollTo({
-                    top: tocEntryForLocation.offsetTop - 100,
-                    left: 0,
-                    behavior: 'smooth',
-                });
-            } else {
-                nav.scrollTop = 0;
-            }
-        }
-    }
-}
-
-
-/**
- * Populate sidebar on load
- */
-window.addEventListener('load', () => {
-    // Only create table of contents if there is more than one header on the page
-    if (headers.length <= 1) {
-        return;
-    }
-
-    // Create an entry in the page table of contents for each header in the document
-    headers.forEach((header, index) => {
-        const link = document.createElement('a');
-
-        // Indent shows hierarchy
-        let indent = '0px';
-        switch (header.parentElement.tagName) {
-            case 'H1':
-                indent = '5px';
-                break;
-            case 'H2':
-                indent = '20px';
-                break;
-            case 'H3':
-                indent = '30px';
-                break;
-            case 'H4':
-                indent = '40px';
-                break;
-            case 'H5':
-                indent = '50px';
-                break;
-            case 'H6':
-                indent = '60px';
-                break;
-            default:
-                break;
-        }
-
-        let tocEntry;
-        if (index == 0) {
-            // Create a bolded title for the first element
-            tocEntry = document.createElement("strong");
-            tocEntry.innerHTML = header.text;
-        } else {
-            // All other elements are non-bold
-            tocEntry = document.createTextNode(header.text);
-        }
-        link.appendChild(tocEntry);
-
-        link.style.paddingLeft = indent;
-        link.href = header.href;
-        pageToc.appendChild(link);
-    });
-    setTocEntry.call();
-});
-
-
-// Handle active headers on scroll, if there is more than one header on the page
-if (headers.length > 1) {
-    window.addEventListener('scroll', setTocEntry);
-}

docs/website_files/theme/index.hbs

@@ -1,312 +0,0 @@
-<!DOCTYPE HTML>
-<html lang="{{ language }}" class="sidebar-visible no-js {{ default_theme }}">
-    <head>
-        <!-- Book generated using mdBook -->
-        <meta charset="UTF-8">
-        <title>{{ title }}</title>
-        {{#if is_print }}
-        <meta name="robots" content="noindex" />
-        {{/if}}
-        {{#if base_url}}
-        <base href="{{ base_url }}">
-        {{/if}}
-
-
-        <!-- Custom HTML head -->
-        {{> head}}
-
-        <meta content="text/html; charset=utf-8" http-equiv="Content-Type">
-        <meta name="description" content="{{ description }}">
-        <meta name="viewport" content="width=device-width, initial-scale=1">
-        <meta name="theme-color" content="#ffffff" />
-
-        {{#if favicon_svg}}
-        <link rel="icon" href="{{ path_to_root }}favicon.svg">
-        {{/if}}
-        {{#if favicon_png}}
-        <link rel="shortcut icon" href="{{ path_to_root }}favicon.png">
-        {{/if}}
-        <link rel="stylesheet" href="{{ path_to_root }}css/variables.css">
-        <link rel="stylesheet" href="{{ path_to_root }}css/general.css">
-        <link rel="stylesheet" href="{{ path_to_root }}css/chrome.css">
-        {{#if print_enable}}
-        <link rel="stylesheet" href="{{ path_to_root }}css/print.css" media="print">
-        {{/if}}
-
-        <!-- Fonts -->
-        <link rel="stylesheet" href="{{ path_to_root }}FontAwesome/css/font-awesome.css">
-        {{#if copy_fonts}}
-        <link rel="stylesheet" href="{{ path_to_root }}fonts/fonts.css">
-        {{/if}}
-
-        <!-- Highlight.js Stylesheets -->
-        <link rel="stylesheet" href="{{ path_to_root }}highlight.css">
-        <link rel="stylesheet" href="{{ path_to_root }}tomorrow-night.css">
-        <link rel="stylesheet" href="{{ path_to_root }}ayu-highlight.css">
-
-        <!-- Custom theme stylesheets -->
-        {{#each additional_css}}
-        <link rel="stylesheet" href="{{ ../path_to_root }}{{ this }}">
-        {{/each}}
-
-        {{#if mathjax_support}}
-        <!-- MathJax -->
-        <script async type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.1/MathJax.js?config=TeX-AMS-MML_HTMLorMML"></script>
-        {{/if}}
-    </head>
-    <body>
-        <!-- Provide site root to javascript -->
-        <script type="text/javascript">
-            var path_to_root = "{{ path_to_root }}";
-            var default_theme = window.matchMedia("(prefers-color-scheme: dark)").matches ? "{{ preferred_dark_theme }}" : "{{ default_theme }}";
-        </script>
-
-        <!-- Work around some values being stored in localStorage wrapped in quotes -->
-        <script type="text/javascript">
-            try {
-                var theme = localStorage.getItem('mdbook-theme');
-                var sidebar = localStorage.getItem('mdbook-sidebar');
-                if (theme.startsWith('"') && theme.endsWith('"')) {
-                    localStorage.setItem('mdbook-theme', theme.slice(1, theme.length - 1));
-                }
-                if (sidebar.startsWith('"') && sidebar.endsWith('"')) {
-                    localStorage.setItem('mdbook-sidebar', sidebar.slice(1, sidebar.length - 1));
-                }
-            } catch (e) { }
-        </script>
-
-        <!-- Set the theme before any content is loaded, prevents flash -->
-        <script type="text/javascript">
-            var theme;
-            try { theme = localStorage.getItem('mdbook-theme'); } catch(e) { }
-            if (theme === null || theme === undefined) { theme = default_theme; }
-            var html = document.querySelector('html');
-            html.classList.remove('no-js')
-            html.classList.remove('{{ default_theme }}')
-            html.classList.add(theme);
-            html.classList.add('js');
-        </script>
-
-        <!-- Hide / unhide sidebar before it is displayed -->
-        <script type="text/javascript">
-            var html = document.querySelector('html');
-            var sidebar = 'hidden';
-            if (document.body.clientWidth >= 1080) {
-                try { sidebar = localStorage.getItem('mdbook-sidebar'); } catch(e) { }
-                sidebar = sidebar || 'visible';
-            }
-            html.classList.remove('sidebar-visible');
-            html.classList.add("sidebar-" + sidebar);
-        </script>
-
-        <nav id="sidebar" class="sidebar" aria-label="Table of contents">
-            <div class="sidebar-scrollbox">
-                {{#toc}}{{/toc}}
-            </div>
-            <div id="sidebar-resize-handle" class="sidebar-resize-handle"></div>
-        </nav>
-
-        <div id="page-wrapper" class="page-wrapper">
-
-            <div class="page">
-                {{> header}}
-                <div id="menu-bar-hover-placeholder"></div>
-                <div id="menu-bar" class="menu-bar sticky bordered">
-                    <div class="left-buttons">
-                        <button id="sidebar-toggle" class="icon-button" type="button" title="Toggle Table of Contents" aria-label="Toggle Table of Contents" aria-controls="sidebar">
-                            <i class="fa fa-bars"></i>
-                        </button>
-                        <button id="theme-toggle" class="icon-button" type="button" title="Change theme" aria-label="Change theme" aria-haspopup="true" aria-expanded="false" aria-controls="theme-list">
-                            <i class="fa fa-paint-brush"></i>
-                        </button>
-                        <ul id="theme-list" class="theme-popup" aria-label="Themes" role="menu">
-                            <li role="none"><button role="menuitem" class="theme" id="light">{{ theme_option "Light" }}</button></li>
-                            <li role="none"><button role="menuitem" class="theme" id="rust">{{ theme_option "Rust" }}</button></li>
-                            <li role="none"><button role="menuitem" class="theme" id="coal">{{ theme_option "Coal" }}</button></li>
-                            <li role="none"><button role="menuitem" class="theme" id="navy">{{ theme_option "Navy" }}</button></li>
-                            <li role="none"><button role="menuitem" class="theme" id="ayu">{{ theme_option "Ayu" }}</button></li>
-                        </ul>
-                        {{#if search_enabled}}
-                        <button id="search-toggle" class="icon-button" type="button" title="Search. (Shortkey: s)" aria-label="Toggle Searchbar" aria-expanded="false" aria-keyshortcuts="S" aria-controls="searchbar">
-                            <i class="fa fa-search"></i>
-                        </button>
-                        {{/if}}
-                    </div>
-
-                    <h1 class="menu-title">{{ book_title }}</h1>
-
-                    <div class="right-buttons">
-                        {{#if print_enable}}
-                        <a href="{{ path_to_root }}print.html" title="Print this book" aria-label="Print this book">
-                            <i id="print-button" class="fa fa-print"></i>
-                        </a>
-                        {{/if}}
-                        {{#if git_repository_url}}
-                        <a href="{{git_repository_url}}" title="Git repository" aria-label="Git repository">
-                            <i id="git-repository-button" class="fa {{git_repository_icon}}"></i>
-                        </a>
-                        {{/if}}
-                        {{#if git_repository_edit_url}}
-                        <a href="{{git_repository_edit_url}}" title="Suggest an edit" aria-label="Suggest an edit">
-                            <i id="git-edit-button" class="fa fa-edit"></i>
-                        </a>
-                        {{/if}}
-
-                    </div>
-                </div>
-
-                {{#if search_enabled}}
-                <div id="search-wrapper" class="hidden">
-                    <form id="searchbar-outer" class="searchbar-outer">
-                        <input type="search" id="searchbar" name="searchbar" placeholder="Search this book ..." aria-controls="searchresults-outer" aria-describedby="searchresults-header">
-                    </form>
-                    <div id="searchresults-outer" class="searchresults-outer hidden">
-                        <div id="searchresults-header" class="searchresults-header"></div>
-                        <ul id="searchresults">
-                        </ul>
-                    </div>
-                </div>
-                {{/if}}
-
-                <!-- Apply ARIA attributes after the sidebar and the sidebar toggle button are added to the DOM -->
-                <script type="text/javascript">
-                    document.getElementById('sidebar-toggle').setAttribute('aria-expanded', sidebar === 'visible');
-                    document.getElementById('sidebar').setAttribute('aria-hidden', sidebar !== 'visible');
-                    Array.from(document.querySelectorAll('#sidebar a')).forEach(function(link) {
-                        link.setAttribute('tabIndex', sidebar === 'visible' ? 0 : -1);
-                    });
-                </script>
-
-                <div id="content" class="content">
-                    <main>
-                        <!-- Page table of contents -->
-                        <div class="sidetoc">
-                            <nav class="pagetoc"></nav>
-                        </div>
-
-                        {{{ content }}}
-                    </main>
-
-                    <nav class="nav-wrapper" aria-label="Page navigation">
-                        <!-- Mobile navigation buttons -->
-                        {{#previous}}
-                            <a rel="prev" href="{{ path_to_root }}{{link}}" class="mobile-nav-chapters previous" title="Previous chapter" aria-label="Previous chapter" aria-keyshortcuts="Left">
-                                <i class="fa fa-angle-left"></i>
-                            </a>
-                        {{/previous}}
-
-                        {{#next}}
-                            <a rel="next" href="{{ path_to_root }}{{link}}" class="mobile-nav-chapters next" title="Next chapter" aria-label="Next chapter" aria-keyshortcuts="Right">
-                                <i class="fa fa-angle-right"></i>
-                            </a>
-                        {{/next}}
-
-                        <div style="clear: both"></div>
-                    </nav>
-                </div>
-            </div>
-
-            <nav class="nav-wide-wrapper" aria-label="Page navigation">
-                {{#previous}}
-                    <a rel="prev" href="{{ path_to_root }}{{link}}" class="nav-chapters previous" title="Previous chapter" aria-label="Previous chapter" aria-keyshortcuts="Left">
-                        <i class="fa fa-angle-left"></i>
-                    </a>
-                {{/previous}}
-
-                {{#next}}
-                    <a rel="next" href="{{ path_to_root }}{{link}}" class="nav-chapters next" title="Next chapter" aria-label="Next chapter" aria-keyshortcuts="Right">
-                        <i class="fa fa-angle-right"></i>
-                    </a>
-                {{/next}}
-            </nav>
-
-        </div>
-
-        {{#if livereload}}
-        <!-- Livereload script (if served using the cli tool) -->
-        <script type="text/javascript">
-            var socket = new WebSocket("{{{livereload}}}");
-            socket.onmessage = function (event) {
-                if (event.data === "reload") {
-                    socket.close();
-                    location.reload();
-                }
-            };
-            window.onbeforeunload = function() {
-                socket.close();
-            }
-        </script>
-        {{/if}}
-
-        {{#if google_analytics}}
-        <!-- Google Analytics Tag -->
-        <script type="text/javascript">
-            var localAddrs = ["localhost", "127.0.0.1", ""];
-            // make sure we don't activate google analytics if the developer is
-            // inspecting the book locally...
-            if (localAddrs.indexOf(document.location.hostname) === -1) {
-                (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
-                (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
-                m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
-                })(window,document,'script','https://www.google-analytics.com/analytics.js','ga');
-                ga('create', '{{google_analytics}}', 'auto');
-                ga('send', 'pageview');
-            }
-        </script>
-        {{/if}}
-
-        {{#if playground_line_numbers}}
-        <script type="text/javascript">
-            window.playground_line_numbers = true;
-        </script>
-        {{/if}}
-
-        {{#if playground_copyable}}
-        <script type="text/javascript">
-            window.playground_copyable = true;
-        </script>
-        {{/if}}
-
-        {{#if playground_js}}
-        <script src="{{ path_to_root }}ace.js" type="text/javascript" charset="utf-8"></script>
-        <script src="{{ path_to_root }}editor.js" type="text/javascript" charset="utf-8"></script>
-        <script src="{{ path_to_root }}mode-rust.js" type="text/javascript" charset="utf-8"></script>
-        <script src="{{ path_to_root }}theme-dawn.js" type="text/javascript" charset="utf-8"></script>
-        <script src="{{ path_to_root }}theme-tomorrow_night.js" type="text/javascript" charset="utf-8"></script>
-        {{/if}}
-
-        {{#if search_js}}
-        <script src="{{ path_to_root }}elasticlunr.min.js" type="text/javascript" charset="utf-8"></script>
-        <script src="{{ path_to_root }}mark.min.js" type="text/javascript" charset="utf-8"></script>
-        <script src="{{ path_to_root }}searcher.js" type="text/javascript" charset="utf-8"></script>
-        {{/if}}
-
-        <script src="{{ path_to_root }}clipboard.min.js" type="text/javascript" charset="utf-8"></script>
-        <script src="{{ path_to_root }}highlight.js" type="text/javascript" charset="utf-8"></script>
-        <script src="{{ path_to_root }}book.js" type="text/javascript" charset="utf-8"></script>
-
-        <!-- Custom JS scripts -->
-        {{#each additional_js}}
-        <script type="text/javascript" src="{{ ../path_to_root }}{{this}}"></script>
-        {{/each}}
-
-        {{#if is_print}}
-        {{#if mathjax_support}}
-        <script type="text/javascript">
-        window.addEventListener('load', function() {
-            MathJax.Hub.Register.StartupHook('End', function() {
-                window.setTimeout(window.print, 100);
-            });
-        });
-        </script>
-        {{else}}
-        <script type="text/javascript">
-        window.addEventListener('load', function() {
-            window.setTimeout(window.print, 100);
-        });
-        </script>
-        {{/if}}
-        {{/if}}
-
-    </body>
-</html>

docs/welcome_and_overview.md

@@ -1,4 +0,0 @@
-# Introduction
-
-Welcome to the documentation repository for Synapse, the reference
-[Matrix](https://matrix.org) homeserver implementation.

docs/workers.md

@@ -228,9 +228,6 @@ expressions:
     ^/_matrix/client/(api/v1|r0|unstable)/joined_groups$
     ^/_matrix/client/(api/v1|r0|unstable)/publicised_groups$
     ^/_matrix/client/(api/v1|r0|unstable)/publicised_groups/
-    ^/_matrix/client/(api/v1|r0|unstable)/rooms/.*/event/
-    ^/_matrix/client/(api/v1|r0|unstable)/joined_rooms$
-    ^/_matrix/client/(api/v1|r0|unstable)/search$
 
     # Registration/login requests
     ^/_matrix/client/(api/v1|r0|unstable)/login$

mypy.ini

@@ -32,7 +32,6 @@ files =
   synapse/http/federation/matrix_federation_agent.py,
   synapse/http/federation/well_known_resolver.py,
   synapse/http/matrixfederationclient.py,
-  synapse/http/servlet.py,
   synapse/http/server.py,
   synapse/http/site.py,
   synapse/logging,
@@ -42,6 +41,7 @@ files =
   synapse/push,
   synapse/replication,
   synapse/rest,
+  synapse/secrets.py,
   synapse/server.py,
   synapse/server_notices,
   synapse/spam_checker_api,
@@ -72,13 +72,8 @@ files =
   synapse/types.py,
   synapse/util/async_helpers.py,
   synapse/util/caches,
-  synapse/util/daemonize.py,
-  synapse/util/hash.py,
-  synapse/util/iterutils.py,
   synapse/util/metrics.py,
   synapse/util/macaroons.py,
-  synapse/util/module_loader.py,
-  synapse/util/msisdn.py,
   synapse/util/stringutils.py,
   synapse/visibility.py,
   tests/replication,
@@ -86,7 +81,6 @@ files =
   tests/handlers/test_password_providers.py,
   tests/rest/client/v1/test_login.py,
   tests/rest/client/v2_alpha/test_auth.py,
-  tests/util/test_itertools.py,
   tests/util/test_stream_change_cache.py
 
 [mypy-pymacaroons.*]
@@ -131,7 +125,7 @@ ignore_missing_imports = True
 [mypy-canonicaljson]
 ignore_missing_imports = True
 
-[mypy-jaeger_client.*]
+[mypy-jaeger_client]
 ignore_missing_imports = True
 
 [mypy-jsonschema]
@@ -178,12 +172,3 @@ ignore_missing_imports = True
 
 [mypy-txacme.*]
 ignore_missing_imports = True
-
-[mypy-pympler.*]
-ignore_missing_imports = True
-
-[mypy-phonenumbers.*]
-ignore_missing_imports = True
-
-[mypy-ijson.*]
-ignore_missing_imports = True

scripts-dev/build_debian_packages

@@ -21,18 +21,17 @@ DISTS = (
     "debian:buster",
     "debian:bullseye",
     "debian:sid",
-    "ubuntu:bionic",  # 18.04 LTS (our EOL forced by Py36 on 2021-12-23)
-    "ubuntu:focal",  # 20.04 LTS (our EOL forced by Py38 on 2024-10-14)
-    "ubuntu:groovy",  # 20.10 (EOL 2021-07-07)
-    "ubuntu:hirsute",  # 21.04 (EOL 2022-01-05)
+    "ubuntu:bionic",
+    "ubuntu:focal",
+    "ubuntu:groovy",
 )
 
-DESC = """\
+DESC = '''\
 Builds .debs for synapse, using a Docker image for the build environment.
 
 By default, builds for all known distributions, but a list of distributions
 can be passed on the commandline for debugging.
-"""
+'''
 
 
 class Builder(object):
@@ -46,7 +45,7 @@ class Builder(object):
         """Build deb for a single distribution"""
 
         if self._failed:
-            print("not building %s due to earlier failure" % (dist,))
+            print("not building %s due to earlier failure" % (dist, ))
             raise Exception("failed")
 
         try:
@@ -68,65 +67,48 @@ class Builder(object):
         # we tend to get source packages which are full of debs. (We could hack
         # around that with more magic in the build_debian.sh script, but that
         # doesn't solve the problem for natively-run dpkg-buildpakage).
-        debsdir = os.path.join(projdir, "../debs")
+        debsdir = os.path.join(projdir, '../debs')
         os.makedirs(debsdir, exist_ok=True)
 
         if self.redirect_stdout:
-            logfile = os.path.join(debsdir, "%s.buildlog" % (tag,))
+            logfile = os.path.join(debsdir, "%s.buildlog" % (tag, ))
             print("building %s: directing output to %s" % (dist, logfile))
             stdout = open(logfile, "w")
         else:
             stdout = None
 
         # first build a docker image for the build environment
-        subprocess.check_call(
-            [
-                "docker",
-                "build",
-                "--tag",
-                "dh-venv-builder:" + tag,
-                "--build-arg",
-                "distro=" + dist,
-                "-f",
-                "docker/Dockerfile-dhvirtualenv",
-                "docker",
-            ],
-            stdout=stdout,
-            stderr=subprocess.STDOUT,
-        )
+        subprocess.check_call([
+            "docker", "build",
+            "--tag", "dh-venv-builder:" + tag,
+            "--build-arg", "distro=" + dist,
+            "-f", "docker/Dockerfile-dhvirtualenv",
+            "docker",
+        ], stdout=stdout, stderr=subprocess.STDOUT)
 
         container_name = "synapse_build_" + tag
         with self._lock:
             self.active_containers.add(container_name)
 
         # then run the build itself
-        subprocess.check_call(
-            [
-                "docker",
-                "run",
-                "--rm",
-                "--name",
-                container_name,
-                "--volume=" + projdir + ":/synapse/source:ro",
-                "--volume=" + debsdir + ":/debs",
-                "-e",
-                "TARGET_USERID=%i" % (os.getuid(),),
-                "-e",
-                "TARGET_GROUPID=%i" % (os.getgid(),),
-                "-e",
-                "DEB_BUILD_OPTIONS=%s" % ("nocheck" if skip_tests else ""),
-                "dh-venv-builder:" + tag,
-            ],
-            stdout=stdout,
-            stderr=subprocess.STDOUT,
-        )
+        subprocess.check_call([
+            "docker", "run",
+            "--rm",
+            "--name", container_name,
+            "--volume=" + projdir + ":/synapse/source:ro",
+            "--volume=" + debsdir + ":/debs",
+            "-e", "TARGET_USERID=%i" % (os.getuid(), ),
+            "-e", "TARGET_GROUPID=%i" % (os.getgid(), ),
+            "-e", "DEB_BUILD_OPTIONS=%s" % ("nocheck" if skip_tests else ""),
+            "dh-venv-builder:" + tag,
+        ], stdout=stdout, stderr=subprocess.STDOUT)
 
         with self._lock:
             self.active_containers.remove(container_name)
 
         if stdout is not None:
             stdout.close()
-            print("Completed build of %s" % (dist,))
+            print("Completed build of %s" % (dist, ))
 
     def kill_containers(self):
         with self._lock:
@@ -134,14 +116,9 @@ class Builder(object):
 
         for c in active:
             print("killing container %s" % (c,))
-            subprocess.run(
-                [
-                    "docker",
-                    "kill",
-                    c,
-                ],
-                stdout=subprocess.DEVNULL,
-            )
+            subprocess.run([
+                "docker", "kill", c,
+            ], stdout=subprocess.DEVNULL)
             with self._lock:
                 self.active_containers.remove(c)
 
@@ -152,38 +129,31 @@ def run_builds(dists, jobs=1, skip_tests=False):
     def sig(signum, _frame):
         print("Caught SIGINT")
         builder.kill_containers()
-
     signal.signal(signal.SIGINT, sig)
 
     with ThreadPoolExecutor(max_workers=jobs) as e:
         res = e.map(lambda dist: builder.run_build(dist, skip_tests), dists)
 
     # make sure we consume the iterable so that exceptions are raised.
-    for _ in res:
+    for r in res:
         pass
 
 
-if __name__ == "__main__":
+if __name__ == '__main__':
     parser = argparse.ArgumentParser(
         description=DESC,
     )
     parser.add_argument(
-        "-j",
-        "--jobs",
-        type=int,
-        default=1,
-        help="specify the number of builds to run in parallel",
+        '-j', '--jobs', type=int, default=1,
+        help='specify the number of builds to run in parallel',
     )
     parser.add_argument(
-        "--no-check",
-        action="store_true",
-        help="skip running tests after building",
+        '--no-check', action='store_true',
+        help='skip running tests after building',
     )
     parser.add_argument(
-        "dist",
-        nargs="*",
-        default=DISTS,
-        help="a list of distributions to build for. Default: %(default)s",
+        'dist', nargs='*', default=DISTS,
+        help='a list of distributions to build for. Default: %(default)s',
     )
     args = parser.parse_args()
     run_builds(dists=args.dist, jobs=args.jobs, skip_tests=args.no_check)

scripts-dev/complement.sh

@@ -10,9 +10,6 @@
 # checkout by setting the COMPLEMENT_DIR environment variable to the
 # filepath of a local Complement checkout.
 #
-# By default Synapse is run in monolith mode. This can be overridden by
-# setting the WORKERS environment variable.
-#
 # A regular expression of test method names can be supplied as the first
 # argument to the script. Complement will then only run those tests. If
 # no regex is supplied, all tests are run. For example;
@@ -35,26 +32,10 @@ if [[ -z "$COMPLEMENT_DIR" ]]; then
   echo "Checkout available at 'complement-master'"
 fi
 
-# If we're using workers, modify the docker files slightly.
-if [[ -n "$WORKERS" ]]; then
-  BASE_IMAGE=matrixdotorg/synapse-workers
-  BASE_DOCKERFILE=docker/Dockerfile-workers
-  export COMPLEMENT_BASE_IMAGE=complement-synapse-workers
-  COMPLEMENT_DOCKERFILE=SynapseWorkers.Dockerfile
-  # And provide some more configuration to complement.
-  export COMPLEMENT_CA=true
-  export COMPLEMENT_VERSION_CHECK_ITERATIONS=500
-else
-  BASE_IMAGE=matrixdotorg/synapse
-  BASE_DOCKERFILE=docker/Dockerfile
-  export COMPLEMENT_BASE_IMAGE=complement-synapse
-  COMPLEMENT_DOCKERFILE=Synapse.Dockerfile
-fi
-
 # Build the base Synapse image from the local checkout
-docker build -t $BASE_IMAGE -f "$BASE_DOCKERFILE" .
+docker build -t matrixdotorg/synapse -f docker/Dockerfile .
 # Build the Synapse monolith image from Complement, based on the above image we just built
-docker build -t $COMPLEMENT_BASE_IMAGE -f "$COMPLEMENT_DIR/dockerfiles/$COMPLEMENT_DOCKERFILE" "$COMPLEMENT_DIR/dockerfiles"
+docker build -t complement-synapse -f "$COMPLEMENT_DIR/dockerfiles/Synapse.Dockerfile" "$COMPLEMENT_DIR/dockerfiles"
 
 cd "$COMPLEMENT_DIR"
 
@@ -65,4 +46,4 @@ if [[ -n "$1" ]]; then
 fi
 
 # Run the tests!
-go test -v -tags synapse_blacklist,msc2946,msc3083 -count=1 $EXTRA_COMPLEMENT_ARGS ./tests
+COMPLEMENT_BASE_IMAGE=complement-synapse go test -v -tags synapse_blacklist,msc2946,msc3083 -count=1 $EXTRA_COMPLEMENT_ARGS ./tests

scripts-dev/convert_server_keys.py

@@ -0,0 +1,115 @@
+import hashlib
+import json
+import sys
+import time
+
+import psycopg2
+import yaml
+from canonicaljson import encode_canonical_json
+from signedjson.key import read_signing_keys
+from signedjson.sign import sign_json
+from unpaddedbase64 import encode_base64
+
+db_binary_type = memoryview
+
+
+def select_v1_keys(connection):
+    cursor = connection.cursor()
+    cursor.execute("SELECT server_name, key_id, verify_key FROM server_signature_keys")
+    rows = cursor.fetchall()
+    cursor.close()
+    results = {}
+    for server_name, key_id, verify_key in rows:
+        results.setdefault(server_name, {})[key_id] = encode_base64(verify_key)
+    return results
+
+
+def select_v1_certs(connection):
+    cursor = connection.cursor()
+    cursor.execute("SELECT server_name, tls_certificate FROM server_tls_certificates")
+    rows = cursor.fetchall()
+    cursor.close()
+    results = {}
+    for server_name, tls_certificate in rows:
+        results[server_name] = tls_certificate
+    return results
+
+
+def select_v2_json(connection):
+    cursor = connection.cursor()
+    cursor.execute("SELECT server_name, key_id, key_json FROM server_keys_json")
+    rows = cursor.fetchall()
+    cursor.close()
+    results = {}
+    for server_name, key_id, key_json in rows:
+        results.setdefault(server_name, {})[key_id] = json.loads(
+            str(key_json).decode("utf-8")
+        )
+    return results
+
+
+def convert_v1_to_v2(server_name, valid_until, keys, certificate):
+    return {
+        "old_verify_keys": {},
+        "server_name": server_name,
+        "verify_keys": {key_id: {"key": key} for key_id, key in keys.items()},
+        "valid_until_ts": valid_until,
+        "tls_fingerprints": [fingerprint(certificate)],
+    }
+
+
+def fingerprint(certificate):
+    finger = hashlib.sha256(certificate)
+    return {"sha256": encode_base64(finger.digest())}
+
+
+def rows_v2(server, json):
+    valid_until = json["valid_until_ts"]
+    key_json = encode_canonical_json(json)
+    for key_id in json["verify_keys"]:
+        yield (server, key_id, "-", valid_until, valid_until, db_binary_type(key_json))
+
+
+def main():
+    config = yaml.safe_load(open(sys.argv[1]))
+    valid_until = int(time.time() / (3600 * 24)) * 1000 * 3600 * 24
+
+    server_name = config["server_name"]
+    signing_key = read_signing_keys(open(config["signing_key_path"]))[0]
+
+    database = config["database"]
+    assert database["name"] == "psycopg2", "Can only convert for postgresql"
+    args = database["args"]
+    args.pop("cp_max")
+    args.pop("cp_min")
+    connection = psycopg2.connect(**args)
+    keys = select_v1_keys(connection)
+    certificates = select_v1_certs(connection)
+    json = select_v2_json(connection)
+
+    result = {}
+    for server in keys:
+        if server not in json:
+            v2_json = convert_v1_to_v2(
+                server, valid_until, keys[server], certificates[server]
+            )
+            v2_json = sign_json(v2_json, server_name, signing_key)
+            result[server] = v2_json
+
+    yaml.safe_dump(result, sys.stdout, default_flow_style=False)
+
+    rows = [row for server, json in result.items() for row in rows_v2(server, json)]
+
+    cursor = connection.cursor()
+    cursor.executemany(
+        "INSERT INTO server_keys_json ("
+        " server_name, key_id, from_server,"
+        " ts_added_ms, ts_valid_until_ms, key_json"
+        ") VALUES (%s, %s, %s, %s, %s, %s)",
+        rows,
+    )
+    connection.commit()
+
+
+if __name__ == "__main__":
+    main()

scripts-dev/definitions.py

@@ -140,7 +140,7 @@ if __name__ == "__main__":
 
     definitions = {}
     for directory in args.directories:
-        for root, _, files in os.walk(directory):
+        for root, dirs, files in os.walk(directory):
             for filename in files:
                 if filename.endswith(".py"):
                     filepath = os.path.join(root, filename)

scripts-dev/dump_macaroon.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python2
 
 import sys
 

scripts-dev/lint.sh

@@ -80,22 +80,8 @@ else
   # then lint everything!
   if [[ -z ${files+x} ]]; then
     # Lint all source code files and directories
-    # Note: this list aims to mirror the one in tox.ini
-      files=(
-          "synapse" "docker" "tests"
-          # annoyingly, black doesn't find these so we have to list them
-          "scripts/export_signing_key"
-          "scripts/generate_config"
-          "scripts/generate_log_config"
-          "scripts/hash_password"
-          "scripts/register_new_matrix_user"
-          "scripts/synapse_port_db"
-          "scripts-dev"
-          "scripts-dev/build_debian_packages"
-          "scripts-dev/sign_json"
-          "scripts-dev/update_database"
-          "contrib" "synctl" "setup.py" "synmark" "stubs" ".buildkite"
-      )
+    # Note: this list aims the mirror the one in tox.ini
+    files=("synapse" "docker" "tests" "scripts-dev" "scripts" "contrib" "synctl" "setup.py" "synmark" "stubs" ".buildkite")
   fi
 fi
 

scripts-dev/list_url_patterns.py

@@ -48,7 +48,7 @@ args = parser.parse_args()
 
 
 for directory in args.directories:
-    for root, _, files in os.walk(directory):
+    for root, dirs, files in os.walk(directory):
         for filename in files:
             if filename.endswith(".py"):
                 filepath = os.path.join(root, filename)

scripts-dev/make_full_schema.sh

@@ -6,7 +6,7 @@
 # It does so by having Synapse generate an up-to-date SQLite DB, then running
 # synapse_port_db to convert it to Postgres. It then dumps the contents of both.
 
-export PGHOST="localhost"
+POSTGRES_HOST="localhost"
 POSTGRES_DB_NAME="synapse_full_schema.$$"
 
 SQLITE_FULL_SCHEMA_OUTPUT_FILE="full.sql.sqlite"
@@ -32,7 +32,7 @@ usage() {
 while getopts "p:co:h" opt; do
   case $opt in
     p)
-      export PGUSER=$OPTARG
+      POSTGRES_USERNAME=$OPTARG
       ;;
     c)
       # Print all commands that are being executed
@@ -69,7 +69,7 @@ if [ ${#unsatisfied_requirements} -ne 0 ]; then
   exit 1
 fi
 
-if [ -z "$PGUSER" ]; then
+if [ -z "$POSTGRES_USERNAME" ]; then
   echo "No postgres username supplied"
   usage
   exit 1
@@ -84,9 +84,8 @@ fi
 # Create the output directory if it doesn't exist
 mkdir -p "$OUTPUT_DIR"
 
-read -rsp "Postgres password for '$PGUSER': " PGPASSWORD
+read -rsp "Postgres password for '$POSTGRES_USERNAME': " POSTGRES_PASSWORD
 echo ""
-export PGPASSWORD
 
 # Exit immediately if a command fails
 set -e
@@ -132,9 +131,9 @@ report_stats: false
 database:
   name: "psycopg2"
   args:
-    user: "$PGUSER"
-    host: "$PGHOST"
-    password: "$PGPASSWORD"
+    user: "$POSTGRES_USERNAME"
+    host: "$POSTGRES_HOST"
+    password: "$POSTGRES_PASSWORD"
     database: "$POSTGRES_DB_NAME"
 
 # Suppress the key server warning.
@@ -151,7 +150,7 @@ scripts-dev/update_database --database-config "$SQLITE_CONFIG"
 
 # Create the PostgreSQL database.
 echo "Creating postgres database..."
-createdb --lc-collate=C --lc-ctype=C --template=template0 "$POSTGRES_DB_NAME"
+createdb $POSTGRES_DB_NAME
 
 echo "Copying data from SQLite3 to Postgres with synapse_port_db..."
 if [ -z "$COVERAGE" ]; then
@@ -182,7 +181,7 @@ DROP TABLE user_directory_search_docsize;
 DROP TABLE user_directory_search_stat;
 "
 sqlite3 "$SQLITE_DB" <<< "$SQL"
-psql "$POSTGRES_DB_NAME" -w <<< "$SQL"
+psql $POSTGRES_DB_NAME -U "$POSTGRES_USERNAME" -w <<< "$SQL"
 
 echo "Dumping SQLite3 schema to '$OUTPUT_DIR/$SQLITE_FULL_SCHEMA_OUTPUT_FILE'..."
 sqlite3 "$SQLITE_DB" ".dump" > "$OUTPUT_DIR/$SQLITE_FULL_SCHEMA_OUTPUT_FILE"

scripts-dev/mypy_synapse_plugin.py

@@ -1,3 +1,4 @@
+# -*- coding: utf-8 -*-
 # Copyright 2020 The Matrix.org Foundation C.I.C.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");

scripts-dev/release.py

@@ -139,7 +139,7 @@ def run():
         click.get_current_context().abort()
 
     # Switch to the release branch.
-    release_branch_name = f"release-v{current_version.major}.{current_version.minor}"
+    release_branch_name = f"release-v{base_version}"
     release_branch = find_ref(repo, release_branch_name)
     if release_branch:
         if release_branch.is_remote():

scripts-dev/sign_json

@@ -1,5 +1,6 @@
 #!/usr/bin/env python
 #
+# -*- coding: utf-8 -*-
 # Copyright 2020 The Matrix.org Foundation C.I.C.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");

scripts-dev/update_database

@@ -1,4 +1,5 @@
 #!/usr/bin/env python
+# -*- coding: utf-8 -*-
 # Copyright 2019 The Matrix.org Foundation C.I.C.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");

scripts/export_signing_key

@@ -1,4 +1,5 @@
 #!/usr/bin/env python
+# -*- coding: utf-8 -*-
 # Copyright 2019 The Matrix.org Foundation C.I.C.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -30,11 +31,7 @@ def exit(status: int = 0, message: Optional[str] = None):
 def format_plain(public_key: nacl.signing.VerifyKey):
     print(
         "%s:%s %s"
-        % (
-            public_key.alg,
-            public_key.version,
-            encode_verify_key_base64(public_key),
-        )
+        % (public_key.alg, public_key.version, encode_verify_key_base64(public_key),)
     )
 
 
@@ -54,10 +51,7 @@ if __name__ == "__main__":
     parser = argparse.ArgumentParser()
 
     parser.add_argument(
-        "key_file",
-        nargs="+",
-        type=argparse.FileType("r"),
-        help="The key file to read",
+        "key_file", nargs="+", type=argparse.FileType("r"), help="The key file to read",
     )
 
     parser.add_argument(
@@ -70,7 +64,7 @@ if __name__ == "__main__":
     parser.add_argument(
         "--expiry-ts",
         type=int,
-        default=int(time.time() * 1000) + 6 * 3600000,
+        default=int(time.time() * 1000) + 6*3600000,
         help=(
             "The expiry time to use for -x, in milliseconds since 1970. The default "
             "is (now+6h)."

scripts/generate_config

@@ -11,22 +11,23 @@ if __name__ == "__main__":
     parser.add_argument(
         "--config-dir",
         default="CONFDIR",
+
         help="The path where the config files are kept. Used to create filenames for "
-        "things like the log config and the signing key. Default: %(default)s",
+             "things like the log config and the signing key. Default: %(default)s",
     )
 
     parser.add_argument(
         "--data-dir",
         default="DATADIR",
         help="The path where the data files are kept. Used to create filenames for "
-        "things like the database and media store. Default: %(default)s",
+             "things like the database and media store. Default: %(default)s",
     )
 
     parser.add_argument(
         "--server-name",
         default="SERVERNAME",
         help="The server name. Used to initialise the server_name config param, but also "
-        "used in the names of some of the config files. Default: %(default)s",
+             "used in the names of some of the config files. Default: %(default)s",
     )
 
     parser.add_argument(
@@ -40,22 +41,21 @@ if __name__ == "__main__":
         "--generate-secrets",
         action="store_true",
         help="Enable generation of new secrets for things like the macaroon_secret_key."
-        "By default, these parameters will be left unset.",
+             "By default, these parameters will be left unset."
     )
 
     parser.add_argument(
-        "-o",
-        "--output-file",
-        type=argparse.FileType("w"),
+        "-o", "--output-file",
+        type=argparse.FileType('w'),
         default=sys.stdout,
         help="File to write the configuration to. Default: stdout",
     )
 
     parser.add_argument(
         "--header-file",
-        type=argparse.FileType("r"),
+        type=argparse.FileType('r'),
         help="File from which to read a header, which will be printed before the "
-        "generated config.",
+             "generated config.",
     )
 
     args = parser.parse_args()

scripts/generate_log_config

@@ -1,5 +1,6 @@
 #!/usr/bin/env python3
 
+# -*- coding: utf-8 -*-
 # Copyright 2020 The Matrix.org Foundation C.I.C.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");

scripts/generate_signing_key.py

@@ -1,4 +1,5 @@
 #!/usr/bin/env python
+# -*- coding: utf-8 -*-
 # Copyright 2019 The Matrix.org Foundation C.I.C.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");

scripts/hash_password

@@ -41,7 +41,7 @@ if __name__ == "__main__":
     parser.add_argument(
         "-c",
         "--config",
-        type=argparse.FileType("r"),
+        type=argparse.FileType('r'),
         help=(
             "Path to server config file. "
             "Used to read in bcrypt_rounds and password_pepper."
@@ -72,8 +72,8 @@ if __name__ == "__main__":
     pw = unicodedata.normalize("NFKC", password)
 
     hashed = bcrypt.hashpw(
-        pw.encode("utf8") + password_pepper.encode("utf8"),
+        pw.encode('utf8') + password_pepper.encode("utf8"),
         bcrypt.gensalt(bcrypt_rounds),
-    ).decode("ascii")
+    ).decode('ascii')
 
     print(hashed)

scripts/move_remote_media_to_new_store.py

@@ -1,4 +1,5 @@
 #!/usr/bin/env python
+# -*- coding: utf-8 -*-
 # Copyright 2017 New Vector Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");

scripts/register_new_matrix_user

@@ -1,4 +1,5 @@
 #!/usr/bin/env python
+# -*- coding: utf-8 -*-
 # Copyright 2015, 2016 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");

scripts/synapse_port_db

@@ -1,4 +1,5 @@
 #!/usr/bin/env python
+# -*- coding: utf-8 -*-
 # Copyright 2015, 2016 OpenMarket Ltd
 # Copyright 2018 New Vector Ltd
 # Copyright 2019 The Matrix.org Foundation C.I.C.
@@ -294,7 +295,8 @@ class Porter(object):
         return table, already_ported, total_to_port, forward_chunk, backward_chunk
 
     async def get_table_constraints(self) -> Dict[str, Set[str]]:
-        """Returns a map of tables that have foreign key constraints to tables they depend on."""
+        """Returns a map of tables that have foreign key constraints to tables they depend on.
+        """
 
         def _get_constraints(txn):
             # We can pull the information about foreign key constraints out from
@@ -503,9 +505,7 @@ class Porter(object):
                 return
 
     def build_db_store(
-        self,
-        db_config: DatabaseConnectionConfig,
-        allow_outdated_version: bool = False,
+        self, db_config: DatabaseConnectionConfig, allow_outdated_version: bool = False,
     ):
         """Builds and returns a database store using the provided configuration.
 
@@ -635,11 +635,8 @@ class Porter(object):
                 "device_inbox_sequence", ("device_inbox", "device_federation_outbox")
             )
             await self._setup_sequence(
-                "account_data_sequence",
-                ("room_account_data", "room_tags_revisions", "account_data"),
-            )
-            await self._setup_sequence("receipts_sequence", ("receipts_linearized",))
-            await self._setup_sequence("presence_stream_sequence", ("presence_stream",))
+                "account_data_sequence", ("room_account_data", "room_tags_revisions", "account_data"))
+            await self._setup_sequence("receipts_sequence", ("receipts_linearized", ))
             await self._setup_auth_chain_sequence()
 
             # Step 3. Get tables.
@@ -741,7 +738,7 @@ class Porter(object):
             return col
 
         outrows = []
-        for row in rows:
+        for i, row in enumerate(rows):
             try:
                 outrows.append(
                     tuple(conv(j, col) for j, col in enumerate(row) if j > 0)
@@ -891,7 +888,8 @@ class Porter(object):
         await self.postgres_store.db_pool.runInteraction("setup_user_id_seq", r)
 
     async def _setup_events_stream_seqs(self) -> None:
-        """Set the event stream sequences to the correct values."""
+        """Set the event stream sequences to the correct values.
+        """
 
         # We get called before we've ported the events table, so we need to
         # fetch the current positions from the SQLite store.
@@ -913,21 +911,18 @@ class Porter(object):
                     (curr_forward_id + 1,),
                 )
 
-            if curr_backward_id:
-                txn.execute(
-                    "ALTER SEQUENCE events_backfill_stream_seq RESTART WITH %s",
-                    (curr_backward_id + 1,),
-                )
+            txn.execute(
+                "ALTER SEQUENCE events_backfill_stream_seq RESTART WITH %s",
+                (curr_backward_id + 1,),
+            )
 
         await self.postgres_store.db_pool.runInteraction(
-            "_setup_events_stream_seqs",
-            _setup_events_stream_seqs_set_pos,
+            "_setup_events_stream_seqs", _setup_events_stream_seqs_set_pos,
         )
 
-    async def _setup_sequence(
-        self, sequence_name: str, stream_id_tables: Iterable[str]
-    ) -> None:
-        """Set a sequence to the correct value."""
+    async def _setup_sequence(self, sequence_name: str, stream_id_tables: Iterable[str]) -> None:
+        """Set a sequence to the correct value.
+        """
         current_stream_ids = []
         for stream_id_table in stream_id_tables:
             max_stream_id = await self.sqlite_store.db_pool.simple_select_one_onecol(
@@ -941,32 +936,26 @@ class Porter(object):
         next_id = max(current_stream_ids) + 1
 
         def r(txn):
-            sql = "ALTER SEQUENCE %s RESTART WITH" % (sequence_name,)
-            txn.execute(sql + " %s", (next_id,))
+            sql = "ALTER SEQUENCE %s RESTART WITH" % (sequence_name, )
+            txn.execute(sql + " %s", (next_id, ))
 
-        await self.postgres_store.db_pool.runInteraction(
-            "_setup_%s" % (sequence_name,), r
-        )
+        await self.postgres_store.db_pool.runInteraction("_setup_%s" % (sequence_name,), r)
 
     async def _setup_auth_chain_sequence(self) -> None:
         curr_chain_id = await self.sqlite_store.db_pool.simple_select_one_onecol(
-            table="event_auth_chains",
-            keyvalues={},
-            retcol="MAX(chain_id)",
-            allow_none=True,
+            table="event_auth_chains", keyvalues={}, retcol="MAX(chain_id)", allow_none=True
         )
 
         def r(txn):
             txn.execute(
                 "ALTER SEQUENCE event_auth_chain_id RESTART WITH %s",
-                (curr_chain_id + 1,),
+                (curr_chain_id,),
             )
 
-        if curr_chain_id is not None:
-            await self.postgres_store.db_pool.runInteraction(
-                "_setup_event_auth_chain_id",
-                r,
-            )
+        await self.postgres_store.db_pool.runInteraction(
+            "_setup_event_auth_chain_id", r,
+        )
+
 
 
 ##############################################
@@ -975,7 +964,8 @@ class Porter(object):
 
 
 class Progress(object):
-    """Used to report progress of the port"""
+    """Used to report progress of the port
+    """
 
     def __init__(self):
         self.tables = {}
@@ -1000,7 +990,8 @@ class Progress(object):
 
 
 class CursesProgress(Progress):
-    """Reports progress to a curses window"""
+    """Reports progress to a curses window
+    """
 
     def __init__(self, stdscr):
         self.stdscr = stdscr
@@ -1025,7 +1016,7 @@ class CursesProgress(Progress):
 
         self.total_processed = 0
         self.total_remaining = 0
-        for data in self.tables.values():
+        for table, data in self.tables.items():
             self.total_processed += data["num_done"] - data["start"]
             self.total_remaining += data["total"] - data["num_done"]
 
@@ -1116,7 +1107,8 @@ class CursesProgress(Progress):
 
 
 class TerminalProgress(Progress):
-    """Just prints progress to the terminal"""
+    """Just prints progress to the terminal
+    """
 
     def update(self, table, num_done):
         super(TerminalProgress, self).update(table, num_done)

setup.cfg

@@ -18,7 +18,8 @@ ignore =
 #  E203: whitespace before ':' (which is contrary to pep8?)
 #  E731: do not assign a lambda expression, use a def
 #  E501: Line too long (black enforces this for us)
-ignore=W503,W504,E203,E731,E501
+#  B007: Subsection of the bugbear suite (TODO: add in remaining fixes)
+ignore=W503,W504,E203,E731,E501,B007
 
 [isort]
 line_length = 88