Tidy up hs.get_federation_sender() calls

This a) reduces the API surface and b) means that we calculate where to
send presence on the presence writer, rather than federation senders.

.buildkite/scripts/create_postgres_db.py

@@ -1,5 +1,4 @@
 #!/usr/bin/env python
-# -*- coding: utf-8 -*-
 # Copyright 2019 The Matrix.org Foundation C.I.C.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");

.buildkite/scripts/test_old_deps.sh

@@ -1,16 +1,16 @@
 #!/usr/bin/env bash
 
-# this script is run by buildkite in a plain `xenial` container; it installs the
-# minimal requirements for tox and hands over to the py35-old tox environment.
+# this script is run by buildkite in a plain `bionic` container; it installs the
+# minimal requirements for tox and hands over to the py3-old tox environment.
 
 set -ex
 
 apt-get update
-apt-get install -y python3.5 python3.5-dev python3-pip libxml2-dev libxslt-dev xmlsec1 zlib1g-dev tox
+apt-get install -y python3 python3-dev python3-pip libxml2-dev libxslt-dev xmlsec1 zlib1g-dev tox
 
 export LANG="C.UTF-8"
 
 # Prevent virtualenv from auto-updating pip to an incompatible version
 export VIRTUALENV_NO_DOWNLOAD=1
 
-exec tox -e py35-old,combine
+exec tox -e py3-old,combine

.github/workflows/tests.yml

@@ -0,0 +1,322 @@
+name: Tests
+
+on:
+  push:
+    branches: ["develop", "release-*"]
+  pull_request:
+
+jobs:
+  lint:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        toxenv:
+          - "check-sampleconfig"
+          - "check_codestyle"
+          - "check_isort"
+          - "mypy"
+          - "packaging"
+
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/setup-python@v2
+      - run: pip install tox
+      - run: tox -e ${{ matrix.toxenv }}
+
+  lint-crlf:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - name: Check line endings
+        run: scripts-dev/check_line_terminators.sh
+
+  lint-newsfile:
+    if: ${{ github.base_ref == 'develop'  || contains(github.base_ref, 'release-') }}
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/setup-python@v2
+      - run: pip install tox
+      - name: Patch Buildkite-specific test script
+        run: |
+          sed -i -e 's/\$BUILDKITE_PULL_REQUEST/${{ github.event.number }}/' \
+            scripts-dev/check-newsfragment
+      - run: scripts-dev/check-newsfragment
+
+  lint-sdist:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/setup-python@v2
+        with:
+          python-version: "3.x"
+      - run: pip install wheel
+      - run: python setup.py sdist bdist_wheel
+      - uses: actions/upload-artifact@v2
+        with:
+          name: Python Distributions
+          path: dist/*
+
+  # Dummy step to gate other tests on without repeating the whole list
+  linting-done:
+    if: ${{ always() }} # Run this even if prior jobs were skipped
+    needs: [lint, lint-crlf, lint-newsfile, lint-sdist]
+    runs-on: ubuntu-latest
+    steps:
+      - run: "true"
+
+  trial:
+    if: ${{ !failure() }} # Allow previous steps to be skipped, but not fail
+    needs: linting-done
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: ["3.6", "3.7", "3.8", "3.9"]
+        database: ["sqlite"]
+        include:
+          # Newest Python without optional deps
+          - python-version: "3.9"
+            toxenv: "py-noextras,combine"
+
+          # Oldest Python with PostgreSQL
+          - python-version: "3.6"
+            database: "postgres"
+            postgres-version: "9.6"
+
+          # Newest Python with PostgreSQL
+          - python-version: "3.9"
+            database: "postgres"
+            postgres-version: "13"
+
+    steps:
+      - uses: actions/checkout@v2
+      - run: sudo apt-get -qq install xmlsec1
+      - name: Set up PostgreSQL ${{ matrix.postgres-version }}
+        if: ${{ matrix.postgres-version }}
+        run: |
+          docker run -d -p 5432:5432 \
+            -e POSTGRES_PASSWORD=postgres \
+            -e POSTGRES_INITDB_ARGS="--lc-collate C --lc-ctype C --encoding UTF8" \
+            postgres:${{ matrix.postgres-version }}
+      - uses: actions/setup-python@v2
+        with:
+          python-version: ${{ matrix.python-version }}
+      - run: pip install tox
+      - name: Await PostgreSQL
+        if: ${{ matrix.postgres-version }}
+        timeout-minutes: 2
+        run: until pg_isready -h localhost; do sleep 1; done
+      - run: tox -e py,combine
+        env:
+          TRIAL_FLAGS: "--jobs=2"
+          SYNAPSE_POSTGRES: ${{ matrix.database == 'postgres' || '' }}
+          SYNAPSE_POSTGRES_HOST: localhost
+          SYNAPSE_POSTGRES_USER: postgres
+          SYNAPSE_POSTGRES_PASSWORD: postgres
+      - name: Dump logs
+        # Note: Dumps to workflow logs instead of using actions/upload-artifact
+        #       This keeps logs colocated with failing jobs
+        #       It also ignores find's exit code; this is a best effort affair
+        run: >-
+          find _trial_temp -name '*.log'
+          -exec echo "::group::{}" \;
+          -exec cat {} \;
+          -exec echo "::endgroup::" \;
+          || true
+
+  trial-olddeps:
+    if: ${{ !failure() }} # Allow previous steps to be skipped, but not fail
+    needs: linting-done
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - name: Test with old deps
+        uses: docker://ubuntu:bionic # For old python and sqlite
+        with:
+          workdir: /github/workspace
+          entrypoint: .buildkite/scripts/test_old_deps.sh
+        env:
+          TRIAL_FLAGS: "--jobs=2"
+      - name: Dump logs
+        # Note: Dumps to workflow logs instead of using actions/upload-artifact
+        #       This keeps logs colocated with failing jobs
+        #       It also ignores find's exit code; this is a best effort affair
+        run: >-
+          find _trial_temp -name '*.log'
+          -exec echo "::group::{}" \;
+          -exec cat {} \;
+          -exec echo "::endgroup::" \;
+          || true
+
+  trial-pypy:
+    # Very slow; only run if the branch name includes 'pypy'
+    if: ${{ contains(github.ref, 'pypy') && !failure() }}
+    needs: linting-done
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: ["pypy-3.6"]
+
+    steps:
+      - uses: actions/checkout@v2
+      - run: sudo apt-get -qq install xmlsec1 libxml2-dev libxslt-dev
+      - uses: actions/setup-python@v2
+        with:
+          python-version: ${{ matrix.python-version }}
+      - run: pip install tox
+      - run: tox -e py,combine
+        env:
+          TRIAL_FLAGS: "--jobs=2"
+      - name: Dump logs
+        # Note: Dumps to workflow logs instead of using actions/upload-artifact
+        #       This keeps logs colocated with failing jobs
+        #       It also ignores find's exit code; this is a best effort affair
+        run: >-
+          find _trial_temp -name '*.log'
+          -exec echo "::group::{}" \;
+          -exec cat {} \;
+          -exec echo "::endgroup::" \;
+          || true
+
+  sytest:
+    if: ${{ !failure() }}
+    needs: linting-done
+    runs-on: ubuntu-latest
+    container:
+      image: matrixdotorg/sytest-synapse:${{ matrix.sytest-tag }}
+      volumes:
+        - ${{ github.workspace }}:/src
+      env:
+        BUILDKITE_BRANCH: ${{ github.head_ref }}
+        POSTGRES: ${{ matrix.postgres && 1}}
+        MULTI_POSTGRES: ${{ (matrix.postgres == 'multi-postgres') && 1}}
+        WORKERS: ${{ matrix.workers && 1 }}
+        REDIS: ${{ matrix.redis && 1 }}
+        BLACKLIST: ${{ matrix.workers && 'synapse-blacklist-with-workers' }}
+
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - sytest-tag: bionic
+
+          - sytest-tag: bionic
+            postgres: postgres
+
+          - sytest-tag: testing
+            postgres: postgres
+
+          - sytest-tag: bionic
+            postgres: multi-postgres
+            workers: workers
+
+          - sytest-tag: buster
+            postgres: multi-postgres
+            workers: workers
+
+          - sytest-tag: buster
+            postgres: postgres
+            workers: workers
+            redis: redis
+
+    steps:
+      - uses: actions/checkout@v2
+      - name: Prepare test blacklist
+        run: cat sytest-blacklist .buildkite/worker-blacklist > synapse-blacklist-with-workers
+      - name: Run SyTest
+        run: /bootstrap.sh synapse
+        working-directory: /src
+      - name: Dump results.tap
+        if: ${{ always() }}
+        run: cat /logs/results.tap
+      - name: Upload SyTest logs
+        uses: actions/upload-artifact@v2
+        if: ${{ always() }}
+        with:
+          name: Sytest Logs - ${{ job.status }} - (${{ join(matrix.*, ', ') }})
+          path: |
+            /logs/results.tap
+            /logs/**/*.log*
+
+  portdb:
+    if: ${{ !failure() }} # Allow previous steps to be skipped, but not fail
+    needs: linting-done
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        include:
+          - python-version: "3.6"
+            postgres-version: "9.6"
+
+          - python-version: "3.9"
+            postgres-version: "13"
+
+    services:
+      postgres:
+        image: postgres:${{ matrix.postgres-version }}
+        ports:
+          - 5432:5432
+        env:
+          POSTGRES_PASSWORD: "postgres"
+          POSTGRES_INITDB_ARGS: "--lc-collate C --lc-ctype C --encoding UTF8"
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+
+    steps:
+      - uses: actions/checkout@v2
+      - run: sudo apt-get -qq install xmlsec1
+      - uses: actions/setup-python@v2
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Patch Buildkite-specific test scripts
+        run: |
+          sed -i -e 's/host="postgres"/host="localhost"/' .buildkite/scripts/create_postgres_db.py
+          sed -i -e 's/host: postgres/host: localhost/' .buildkite/postgres-config.yaml
+          sed -i -e 's|/src/||' .buildkite/{sqlite,postgres}-config.yaml
+          sed -i -e 's/\$TOP/\$GITHUB_WORKSPACE/' .coveragerc
+      - run: .buildkite/scripts/test_synapse_port_db.sh
+
+  complement:
+    if: ${{ !failure() }}
+    needs: linting-done
+    runs-on: ubuntu-latest
+    container:
+      # https://github.com/matrix-org/complement/blob/master/dockerfiles/ComplementCIBuildkite.Dockerfile
+      image: matrixdotorg/complement:latest
+      env:
+        CI: true
+      ports:
+        - 8448:8448
+      volumes:
+        - /var/run/docker.sock:/var/run/docker.sock
+
+    steps:
+      - name: Run actions/checkout@v2 for synapse
+        uses: actions/checkout@v2
+        with:
+          path: synapse
+
+      - name: Run actions/checkout@v2 for complement
+        uses: actions/checkout@v2
+        with:
+          repository: "matrix-org/complement"
+          path: complement
+
+      # Build initial Synapse image
+      - run: docker build -t matrixdotorg/synapse:latest -f docker/Dockerfile .
+        working-directory: synapse
+
+      # Build a ready-to-run Synapse image based on the initial image above.
+      # This new image includes a config file, keys for signing and TLS, and
+      # other settings to make it suitable for testing under Complement.
+      - run: docker build -t complement-synapse -f Synapse.Dockerfile .
+        working-directory: complement/dockerfiles
+
+      # Run Complement
+      - run: go test -v -tags synapse_blacklist ./tests
+        env:
+          COMPLEMENT_BASE_IMAGE: complement-synapse:latest
+        working-directory: complement

CHANGES.md

@@ -1,10 +1,99 @@
-Synapse 1.31.0rc1 (2021-03-30)
+Synapse 1.32.0rc1 (2021-04-13)
 ==============================
 
-**Note:** As announced in v1.25.0, and in line with the deprecation policy for platform dependencies, this is the last release to support Python 3.5 and PostgreSQL 9.5. Future versions of Synapse will require Python 3.6+ and PostgreSQL 9.6+.
+**Note:** This release requires Python 3.6+ and Postgres 9.6+ or SQLite 3.22+.
+
+This release removes the deprecated `GET /_synapse/admin/v1/users/<user_id>` admin API. Please use the [v2 API](https://github.com/matrix-org/synapse/blob/develop/docs/admin_api/user_admin_api.rst#query-user-account) instead, which has improved capabilities.
+
+This release requires Application Services to use type `m.login.application_services` when registering users via the `/_matrix/client/r0/register` endpoint to comply with the spec. Please ensure your Application Services are up to date.
+
+Features
+--------
+
+- Add a Synapse module for routing presence updates between users. ([\#9491](https://github.com/matrix-org/synapse/issues/9491))
+- Add an admin API to manage ratelimit for a specific user. ([\#9648](https://github.com/matrix-org/synapse/issues/9648))
+- Include request information in structured logging output. ([\#9654](https://github.com/matrix-org/synapse/issues/9654))
+- Add `order_by` to the admin API `GET /_synapse/admin/v2/users`. Contributed by @dklimpel. ([\#9691](https://github.com/matrix-org/synapse/issues/9691))
+- Replace the `room_invite_state_types` configuration setting with `room_prejoin_state`. ([\#9700](https://github.com/matrix-org/synapse/issues/9700))
+- Add experimental support for [MSC3083](https://github.com/matrix-org/matrix-doc/pull/3083): restricting room access via group membership. ([\#9717](https://github.com/matrix-org/synapse/issues/9717), [\#9735](https://github.com/matrix-org/synapse/issues/9735))
+- Update experimental support for Spaces: include `m.room.create` in the room state sent with room-invites. ([\#9710](https://github.com/matrix-org/synapse/issues/9710))
+- Synapse now requires Python 3.6 or later. It also requires Postgres 9.6 or later or SQLite 3.22 or later. ([\#9766](https://github.com/matrix-org/synapse/issues/9766))
+
+
+Bugfixes
+--------
+
+- Prevent `synapse_forward_extremities` and `synapse_excess_extremity_events` Prometheus metrics from initially reporting zero-values after startup. ([\#8926](https://github.com/matrix-org/synapse/issues/8926))
+- Fix recently added ratelimits to correctly honour the application service `rate_limited` flag. ([\#9711](https://github.com/matrix-org/synapse/issues/9711))
+- Fix longstanding bug which caused `duplicate key value violates unique constraint "remote_media_cache_thumbnails_media_origin_media_id_thumbna_key"` errors. ([\#9725](https://github.com/matrix-org/synapse/issues/9725))
+- Fix bug where sharded federation senders could get stuck repeatedly querying the DB in a loop, using lots of CPU. ([\#9770](https://github.com/matrix-org/synapse/issues/9770))
+- Fix duplicate logging of exceptions thrown during federation transaction processing. ([\#9780](https://github.com/matrix-org/synapse/issues/9780))
+
+
+Updates to the Docker image
+---------------------------
+
+- Move opencontainers labels to the final Docker image such that users can inspect them. ([\#9765](https://github.com/matrix-org/synapse/issues/9765))
+
+
+Improved Documentation
+----------------------
+
+- Make the `allowed_local_3pids` regex example in the sample config stricter. ([\#9719](https://github.com/matrix-org/synapse/issues/9719))
+
+
+Deprecations and Removals
+-------------------------
+
+- Remove old admin API `GET /_synapse/admin/v1/users/<user_id>`. ([\#9401](https://github.com/matrix-org/synapse/issues/9401))
+- Make `/_matrix/client/r0/register` expect a type of `m.login.application_service` when an Application Service registers a user, to align with [the relevant spec](https://spec.matrix.org/unstable/application-service-api/#server-admin-style-permissions). ([\#9548](https://github.com/matrix-org/synapse/issues/9548))
+
+
+Internal Changes
+----------------
+
+- Replace deprecated `imp` module with successor `importlib`. Contributed by Cristina Muñoz. ([\#9718](https://github.com/matrix-org/synapse/issues/9718))
+- Experiment with GitHub Actions for CI. ([\#9661](https://github.com/matrix-org/synapse/issues/9661))
+- Introduce flake8-bugbear to the test suite and fix some of its lint violations. ([\#9682](https://github.com/matrix-org/synapse/issues/9682))
+- Update `scripts-dev/complement.sh` to use a local checkout of Complement, allow running a subset of tests and have it use Synapse's Complement test blacklist. ([\#9685](https://github.com/matrix-org/synapse/issues/9685))
+- Improve Jaeger tracing for `to_device` messages. ([\#9686](https://github.com/matrix-org/synapse/issues/9686))
+- Add release helper script for automating part of the Synapse release process. ([\#9713](https://github.com/matrix-org/synapse/issues/9713))
+- Add type hints to expiring cache. ([\#9730](https://github.com/matrix-org/synapse/issues/9730))
+- Convert various testcases to `HomeserverTestCase`. ([\#9736](https://github.com/matrix-org/synapse/issues/9736))
+- Start linting mypy with `no_implicit_optional`. ([\#9742](https://github.com/matrix-org/synapse/issues/9742))
+- Add missing type hints to federation handler and server. ([\#9743](https://github.com/matrix-org/synapse/issues/9743))
+- Check that a `ConfigError` is raised, rather than simply `Exception`, when appropriate in homeserver config file generation tests. ([\#9753](https://github.com/matrix-org/synapse/issues/9753))
+- Fix incompatibility with `tox` 2.5. ([\#9769](https://github.com/matrix-org/synapse/issues/9769))
+- Enable Complement tests for [MSC2946](https://github.com/matrix-org/matrix-doc/pull/2946): Spaces Summary API. ([\#9771](https://github.com/matrix-org/synapse/issues/9771))
+- Use mock from the standard library instead of a separate package. ([\#9772](https://github.com/matrix-org/synapse/issues/9772))
+- Update Black configuration to target Python 3.6. ([\#9781](https://github.com/matrix-org/synapse/issues/9781))
+- Add option to skip unit tests when building Debian packages. ([\#9793](https://github.com/matrix-org/synapse/issues/9793))
+
+
+Synapse 1.31.0 (2021-04-06)
+===========================
+
+**Note:** As announced in v1.25.0, and in line with the deprecation policy for platform dependencies, this is the last release to support Python 3.5 and PostgreSQL 9.5. Future versions of Synapse will require Python 3.6+ and PostgreSQL 9.6+, as per our [deprecation policy](docs/deprecation_policy.md).
 
 This is also the last release that the Synapse team will be publishing packages for Debian Stretch and Ubuntu Xenial.
 
+
+Improved Documentation
+----------------------
+
+- Add a document describing the deprecation policy for platform dependencies. ([\#9723](https://github.com/matrix-org/synapse/issues/9723))
+
+
+Internal Changes
+----------------
+
+- Revert using `dmypy run` in lint script. ([\#9720](https://github.com/matrix-org/synapse/issues/9720))
+- Pin flake8-bugbear's version. ([\#9734](https://github.com/matrix-org/synapse/issues/9734))
+
+
+Synapse 1.31.0rc1 (2021-03-30)
+==============================
+
 Features
 --------
 

INSTALL.md

@@ -38,6 +38,7 @@ There are 3 steps to follow under **Installation Instructions**.
     - [URL previews](#url-previews)
     - [Troubleshooting Installation](#troubleshooting-installation)
 
+
 ## Choosing your server name
 
 It is important to choose the name for your server before you install Synapse,
@@ -542,7 +543,7 @@ Alternatively, you can do so from the command line. This can be done as follows:
     register_new_matrix_user -c homeserver.yaml http://localhost:8008
     ```
 
-This will prompt you to add details for the new user, and will then connect to 
+This will prompt you to add details for the new user, and will then connect to
 the running Synapse to create the new user. For example:
 ```
 New user localpart: erikj

README.rst

@@ -314,6 +314,15 @@ Testing with SyTest is recommended for verifying that changes related to the
 Client-Server API are functioning correctly. See the `installation instructions
 <https://github.com/matrix-org/sytest#installing>`_ for details.
 
+
+Platform dependencies
+=====================
+
+Synapse uses a number of platform dependencies such as Python and PostgreSQL,
+and aims to follow supported upstream versions. See the
+`<docs/deprecation_policy.md>`_ document for more details.
+
+
 Troubleshooting
 ===============
 
@@ -384,12 +393,17 @@ massive excess of outgoing federation requests (see `discussion
 indicate that your server is also issuing far more outgoing federation
 requests than can be accounted for by your users' activity, this is a
 likely cause. The misbehavior can be worked around by setting
-``use_presence: false`` in the Synapse config file.
+the following in the Synapse config file:
+
+.. code-block:: yaml
+
+   presence:
+       enabled: false
 
 People can't accept room invitations from me
 --------------------------------------------
 
-The typical failure mode here is that you send an invitation to someone 
+The typical failure mode here is that you send an invitation to someone
 to join a room or direct chat, but when they go to accept it, they get an
 error (typically along the lines of "Invalid signature"). They might see
 something like the following in their logs::

UPGRADE.rst

@@ -85,6 +85,19 @@ for example:
      wget https://packages.matrix.org/debian/pool/main/m/matrix-synapse-py3/matrix-synapse-py3_1.3.0+stretch1_amd64.deb
      dpkg -i matrix-synapse-py3_1.3.0+stretch1_amd64.deb
 
+Upgrading to v1.32.0
+====================
+
+Removal of old List Accounts Admin API
+--------------------------------------
+
+The deprecated v1 "list accounts" admin API (``GET /_synapse/admin/v1/users/<user_id>``) has been removed in this version.
+
+The `v2 list accounts API <https://github.com/matrix-org/synapse/blob/master/docs/admin_api/user_admin_api.rst#list-accounts>`_
+has been available since Synapse 1.7.0 (2019-12-13), and is accessible under ``GET /_synapse/admin/v2/users``.
+
+The deprecation of the old endpoint was announced with Synapse 1.28.0 (released on 2021-02-25).
+
 Upgrading to v1.29.0
 ====================
 

changelog.d/9162.misc

@@ -0,0 +1 @@
+Add a dockerfile for running Synapse in worker-mode under Complement.

changelog.d/9685.misc

@@ -1 +0,0 @@
-Update `scripts-dev/complement.sh` to use a local checkout of Complement, allow running a subset of tests and have it use Synapse's Complement test blacklist.

changelog.d/9691.feature

@@ -1 +0,0 @@
-Add `order_by` to the admin API `GET /_synapse/admin/v2/users`. Contributed by @dklimpel. 

changelog.d/9700.feature

@@ -1 +0,0 @@
-Replace the `room_invite_state_types` configuration setting with `room_prejoin_state`.

changelog.d/9702.misc

@@ -0,0 +1 @@
+Speed up federation transmission by using fewer database calls. Contributed by @ShadowJonathan.

changelog.d/9710.feature

@@ -1 +0,0 @@
-Experimental Spaces support: include `m.room.create` in the room state sent with room-invites.

changelog.d/9711.bugfix

@@ -1 +0,0 @@
-Fix recently added ratelimits to correctly honour the application service `rate_limited` flag.

changelog.d/9717.feature

@@ -1 +0,0 @@
-Add experimental support for [MSC3083](https://github.com/matrix-org/matrix-doc/pull/3083): restricting room access via group membership.

changelog.d/9718.removal

@@ -1 +0,0 @@
-Replace deprecated `imp` module with successor `importlib`. Contributed by Cristina Muñoz.

changelog.d/9719.doc

@@ -1 +0,0 @@
-Make the allowed_local_3pids regex example in the sample config stricter.

changelog.d/9720.misc

@@ -1 +0,0 @@
-Revert using `dmypy run` in lint script.

changelog.d/9786.misc

@@ -0,0 +1 @@
+Apply `pyupgrade` across the codebase.

changelog.d/9788.bugfix

@@ -0,0 +1 @@
+Fix thumbnail generation for some sites with non-standard content types. Contributed by @rkfg.

changelog.d/9796.misc

@@ -0,0 +1 @@
+Move some replication processing out of `generic_worker`.

changelog.d/9800.feature

@@ -0,0 +1 @@
+Update experimental support for [MSC3083](https://github.com/matrix-org/matrix-doc/pull/3083): restricting room access via group membership.

changelog.d/9801.doc

@@ -0,0 +1 @@
+Add a note to the docker docs mentioning that we mirror upstream's supported Docker platforms.

changelog.d/9815.misc

@@ -0,0 +1 @@
+Replace `HomeServer.get_config()` with inline references.

changelog.d/9819.feature

@@ -0,0 +1 @@
+Add experimental support for handling presence on a worker.

contrib/cmdclient/console.py

@@ -24,6 +24,7 @@ import sys
 import time
 import urllib
 from http import TwistedHttpClient
+from typing import Optional
 
 import nacl.encoding
 import nacl.signing
@@ -718,7 +719,7 @@ class SynapseCmd(cmd.Cmd):
         method,
         path,
         data=None,
-        query_params={"access_token": None},
+        query_params: Optional[dict] = None,
         alt_text=None,
     ):
         """Runs an HTTP request and pretty prints the output.
@@ -729,6 +730,8 @@ class SynapseCmd(cmd.Cmd):
             data: Raw JSON data if any
             query_params: dict of query parameters to add to the url
         """
+        query_params = query_params or {"access_token": None}
+
         url = self._url() + path
         if "access_token" in query_params:
             query_params["access_token"] = self._tok()

contrib/cmdclient/http.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2014-2016 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -16,6 +15,7 @@
 import json
 import urllib
 from pprint import pformat
+from typing import Optional
 
 from twisted.internet import defer, reactor
 from twisted.web.client import Agent, readBody
@@ -85,8 +85,9 @@ class TwistedHttpClient(HttpClient):
         body = yield readBody(response)
         defer.returnValue(json.loads(body))
 
-    def _create_put_request(self, url, json_data, headers_dict={}):
+    def _create_put_request(self, url, json_data, headers_dict: Optional[dict] = None):
         """Wrapper of _create_request to issue a PUT request"""
+        headers_dict = headers_dict or {}
 
         if "Content-Type" not in headers_dict:
             raise defer.error(RuntimeError("Must include Content-Type header for PUTs"))
@@ -95,14 +96,22 @@ class TwistedHttpClient(HttpClient):
             "PUT", url, producer=_JsonProducer(json_data), headers_dict=headers_dict
         )
 
-    def _create_get_request(self, url, headers_dict={}):
+    def _create_get_request(self, url, headers_dict: Optional[dict] = None):
         """Wrapper of _create_request to issue a GET request"""
-        return self._create_request("GET", url, headers_dict=headers_dict)
+        return self._create_request("GET", url, headers_dict=headers_dict or {})
 
     @defer.inlineCallbacks
     def do_request(
-        self, method, url, data=None, qparams=None, jsonreq=True, headers={}
+        self,
+        method,
+        url,
+        data=None,
+        qparams=None,
+        jsonreq=True,
+        headers: Optional[dict] = None,
     ):
+        headers = headers or {}
+
         if qparams:
             url = "%s?%s" % (url, urllib.urlencode(qparams, True))
 
@@ -123,8 +132,12 @@ class TwistedHttpClient(HttpClient):
         defer.returnValue(json.loads(body))
 
     @defer.inlineCallbacks
-    def _create_request(self, method, url, producer=None, headers_dict={}):
+    def _create_request(
+        self, method, url, producer=None, headers_dict: Optional[dict] = None
+    ):
         """Creates and sends a request to the given url"""
+        headers_dict = headers_dict or {}
+
         headers_dict["User-Agent"] = ["Synapse Cmd Client"]
 
         retries_left = 5

contrib/experiments/test_messaging.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2014-2016 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -225,14 +224,16 @@ class HomeServer(ReplicationHandler):
         destinations = yield self.get_servers_for_context(room_name)
 
         try:
-            yield self.replication_layer.send_pdu(
-                Pdu.create_new(
-                    context=room_name,
-                    pdu_type="sy.room.message",
-                    content={"sender": sender, "body": body},
-                    origin=self.server_name,
-                    destinations=destinations,
-                )
+            yield self.replication_layer.send_pdus(
+                [
+                    Pdu.create_new(
+                        context=room_name,
+                        pdu_type="sy.room.message",
+                        content={"sender": sender, "body": body},
+                        origin=self.server_name,
+                        destinations=destinations,
+                    )
+                ]
             )
         except Exception as e:
             logger.exception(e)
@@ -254,7 +255,7 @@ class HomeServer(ReplicationHandler):
                 origin=self.server_name,
                 destinations=destinations,
             )
-            yield self.replication_layer.send_pdu(pdu)
+            yield self.replication_layer.send_pdus([pdu])
         except Exception as e:
             logger.exception(e)
 
@@ -266,16 +267,18 @@ class HomeServer(ReplicationHandler):
         destinations = yield self.get_servers_for_context(room_name)
 
         try:
-            yield self.replication_layer.send_pdu(
-                Pdu.create_new(
-                    context=room_name,
-                    is_state=True,
-                    pdu_type="sy.room.member",
-                    state_key=invitee,
-                    content={"membership": "invite"},
-                    origin=self.server_name,
-                    destinations=destinations,
-                )
+            yield self.replication_layer.send_pdus(
+                [
+                    Pdu.create_new(
+                        context=room_name,
+                        is_state=True,
+                        pdu_type="sy.room.member",
+                        state_key=invitee,
+                        content={"membership": "invite"},
+                        origin=self.server_name,
+                        destinations=destinations,
+                    )
+                ]
             )
         except Exception as e:
             logger.exception(e)

debian/build_virtualenv

@@ -50,15 +50,24 @@ PACKAGE_BUILD_DIR="debian/matrix-synapse-py3"
 VIRTUALENV_DIR="${PACKAGE_BUILD_DIR}${DH_VIRTUALENV_INSTALL_ROOT}/matrix-synapse"
 TARGET_PYTHON="${VIRTUALENV_DIR}/bin/python"
 
-# we copy the tests to a temporary directory so that we can put them on the
-# PYTHONPATH without putting the uninstalled synapse on the pythonpath.
-tmpdir=`mktemp -d`
-trap "rm -r $tmpdir" EXIT
+case "$DEB_BUILD_OPTIONS" in
+    *nocheck*)
+        # Skip running tests if "nocheck" present in $DEB_BUILD_OPTIONS
+        ;;
 
-cp -r tests "$tmpdir"
+    *)
+        # Copy tests to a temporary directory so that we can put them on the
+        # PYTHONPATH without putting the uninstalled synapse on the pythonpath.
+        tmpdir=`mktemp -d`
+        trap "rm -r $tmpdir" EXIT
 
-PYTHONPATH="$tmpdir" \
-    "${TARGET_PYTHON}" -m twisted.trial --reporter=text -j2 tests
+        cp -r tests "$tmpdir"
+
+        PYTHONPATH="$tmpdir" \
+            "${TARGET_PYTHON}" -m twisted.trial --reporter=text -j2 tests
+
+        ;;
+esac
 
 # build the config file
 "${TARGET_PYTHON}" "${VIRTUALENV_DIR}/bin/generate_config" \

debian/changelog

@@ -1,3 +1,15 @@
+matrix-synapse-py3 (1.31.0+nmu1) UNRELEASED; urgency=medium
+
+  * Skip tests when DEB_BUILD_OPTIONS contains "nocheck".
+
+ -- Dan Callahan <danc@element.io>  Mon, 12 Apr 2021 13:07:36 +0000
+
+matrix-synapse-py3 (1.31.0) stable; urgency=medium
+
+  * New synapse release 1.31.0.
+
+ -- Synapse Packaging team <packages@matrix.org>  Tue, 06 Apr 2021 13:08:29 +0100
+
 matrix-synapse-py3 (1.30.1) stable; urgency=medium
 
   * New synapse release 1.30.1.

docker/Dockerfile

@@ -18,11 +18,6 @@ ARG PYTHON_VERSION=3.8
 ###
 FROM docker.io/python:${PYTHON_VERSION}-slim as builder
 
-LABEL org.opencontainers.image.url='https://matrix.org/docs/projects/server/synapse'
-LABEL org.opencontainers.image.documentation='https://github.com/matrix-org/synapse/blob/master/docker/README.md'
-LABEL org.opencontainers.image.source='https://github.com/matrix-org/synapse.git'
-LABEL org.opencontainers.image.licenses='Apache-2.0'
-
 # install the OS build deps
 RUN apt-get update && apt-get install -y \
     build-essential \
@@ -66,6 +61,11 @@ RUN pip install --prefix="/install" --no-deps --no-warn-script-location /synapse
 
 FROM docker.io/python:${PYTHON_VERSION}-slim
 
+LABEL org.opencontainers.image.url='https://matrix.org/docs/projects/server/synapse'
+LABEL org.opencontainers.image.documentation='https://github.com/matrix-org/synapse/blob/master/docker/README.md'
+LABEL org.opencontainers.image.source='https://github.com/matrix-org/synapse.git'
+LABEL org.opencontainers.image.licenses='Apache-2.0'
+
 RUN apt-get update && apt-get install -y \
     curl \
     gosu \

docker/Dockerfile-workers

@@ -0,0 +1,23 @@
+# Inherit from the official Synapse docker image
+FROM matrixdotorg/synapse
+
+# Install deps
+RUN apt-get update
+RUN apt-get install -y supervisor redis nginx
+
+# Remove the default nginx sites
+RUN rm /etc/nginx/sites-enabled/default
+
+# Copy Synapse worker, nginx and supervisord configuration template files
+COPY ./docker/conf-workers/* /conf/
+
+# Expose nginx listener port
+EXPOSE 8080/tcp
+
+# Volume for user-editable config files, logs etc.
+VOLUME ["/data"]
+
+# A script to read environment variables and create the necessary
+# files to run the desired worker configuration. Will start supervisord.
+COPY ./docker/configure_workers_and_start.py /configure_workers_and_start.py
+ENTRYPOINT ["/configure_workers_and_start.py"]

docker/README-testing.md

@@ -0,0 +1,140 @@
+# Running tests against a dockerised Synapse
+
+It's possible to run integration tests against Synapse
+using [Complement](https://github.com/matrix-org/complement). Complement is a Matrix Spec
+compliance test suite for homeservers, and supports any homeserver docker image configured
+to listen on ports 8008/8448. This document contains instructions for building Synapse
+docker images that can be run inside Complement for testing purposes.
+
+Note that running Synapse's unit tests from within the docker image is not supported.
+
+## Testing with SQLite and single-process Synapse
+
+> Note that `scripts-dev/complement.sh` is a script that will automatically build 
+> and run an SQLite-based, single-process of Synapse against Complement.
+
+The instructions below will set up Complement testing for a single-process, 
+SQLite-based Synapse deployment.
+
+Start by building the base Synapse docker image. If you wish to run tests with the latest
+release of Synapse, instead of your current checkout, you can skip this step. From the
+root of the repository:
+
+```sh
+docker build -t matrixdotorg/synapse -f docker/Dockerfile .
+```
+
+This will build an image with the tag `matrixdotorg/synapse`.
+
+Next, build the Synapse image for Complement. You will need a local checkout 
+of Complement. Change to the root of your Complement checkout and run:
+
+```sh
+docker build -t complement-synapse -f "dockerfiles/Synapse.Dockerfile" dockerfiles
+```
+
+This will build an image with the tag `complement-synapse`, which can be handed to 
+Complement for testing via the `COMPLEMENT_BASE_IMAGE` environment variable. Refer to 
+[Complement's documentation](https://github.com/matrix-org/complement/#running) for 
+how to run the tests, as well as the various available command line flags.
+
+## Testing with PostgreSQL and single or multi-process Synapse
+
+The above docker image only supports running Synapse with SQLite and in a 
+single-process topology. The following instructions are used to build a Synapse image for 
+Complement that supports either single or multi-process topology with a PostgreSQL 
+database backend.
+
+As with the single-process image, build the base Synapse docker image. If you wish to run
+tests with the latest release of Synapse, instead of your current checkout, you can skip
+this step. From the root of the repository:
+
+```sh
+docker build -t matrixdotorg/synapse -f docker/Dockerfile .
+```
+
+This will build an image with the tag `matrixdotorg/synapse`.
+
+Next, we build a new image with worker support based on `matrixdotorg/synapse:latest`. 
+Again, from the root of the repository:
+
+```sh
+docker build -t matrixdotorg/synapse-workers -f docker/Dockerfile-workers .
+```
+
+This will build an image with the tag` matrixdotorg/synapse-workers`.
+
+It's worth noting at this point that this image is fully functional, and 
+can be used for testing against locally. See instructions for using the container 
+under
+[Running the Dockerfile-worker image standalone](#running-the-dockerfile-worker-image-standalone)
+below.
+
+Finally, build the Synapse image for Complement, which is based on
+`matrixdotorg/synapse-workers`. You will need a local checkout of Complement. Change to
+the root of your Complement checkout and run:
+
+```sh
+docker build -t matrixdotorg/complement-synapse-workers -f dockerfiles/SynapseWorkers.Dockerfile dockerfiles
+```
+
+This will build an image with the tag `complement-synapse`, which can be handed to
+Complement for testing via the `COMPLEMENT_BASE_IMAGE` environment variable. Refer to
+[Complement's documentation](https://github.com/matrix-org/complement/#running) for
+how to run the tests, as well as the various available command line flags.
+
+## Running the Dockerfile-worker image standalone
+
+For manual testing of a multi-process Synapse instance in Docker,
+[Dockerfile-workers](Dockerfile-workers) is a Dockerfile that will produce an image
+bundling all necessary components together for a workerised homeserver instance.
+
+This includes any desired Synapse worker processes, a nginx to route traffic accordingly,
+a redis for worker communication and a supervisord instance to start up and monitor all
+processes. You will need to provide your own postgres container to connect to, and TLS 
+is not handled by the container.
+
+Once you've built the image using the above instructions, you can run it. Be sure 
+you've set up a volume according to the [usual Synapse docker instructions](README.md).
+Then run something along the lines of:
+
+```
+docker run -d --name synapse \
+    --mount type=volume,src=synapse-data,dst=/data \
+    -p 8008:8008 \
+    -e SYNAPSE_SERVER_NAME=my.matrix.host \
+    -e SYNAPSE_REPORT_STATS=no \
+    -e POSTGRES_HOST=postgres \
+    -e POSTGRES_USER=postgres \
+    -e POSTGRES_PASSWORD=somesecret \
+    -e SYNAPSE_WORKER_TYPES=synchrotron,media_repository,user_dir \
+    -e SYNAPSE_WORKERS_WRITE_LOGS_TO_DISK=1 \
+    matrixdotorg/synapse-workers
+```
+
+...substituting `POSTGRES*` variables for those that match a postgres host you have 
+available (usually a running postgres docker container).
+
+The `SYNAPSE_WORKER_TYPES` environment variable is a comma-separated list of workers to
+use when running the container. All possible worker names are defined by the keys of the
+`WORKERS_CONFIG` variable in [this script](configure_workers_and_start.py), which the
+Dockerfile makes use of to generate appropriate worker, nginx and supervisord config
+files.
+
+Sharding is supported for a subset of workers, in line with the
+[worker documentation](../docs/workers.md). To run multiple instances of a given worker
+type, simply specify the type multiple times in `SYNAPSE_WORKER_TYPES`
+(e.g `SYNAPSE_WORKER_TYPES=event_creator,event_creator...`).
+
+Otherwise, `SYNAPSE_WORKER_TYPES` can either be left empty or unset to spawn no workers
+(leaving only the main process). The container is configured to use redis-based worker
+mode.
+
+Logs for workers and the main process are logged to stdout and can be viewed with 
+standard `docker logs` tooling. Worker logs contain their worker name 
+after the timestamp.
+
+Setting `SYNAPSE_WORKERS_WRITE_LOGS_TO_DISK=1` will cause worker logs to be written to
+`<data_dir>/logs/<worker_name>.log`. Logs are kept for 1 week and rotate every day at 00:
+00, according to the container's clock. Logging for the main process must still be 
+configured by modifying the homeserver's log config in your Synapse data volume.

docker/README.md

@@ -2,13 +2,16 @@
 
 This Docker image will run Synapse as a single process. By default it uses a
 sqlite database; for production use you should connect it to a separate
-postgres database.
+postgres database. The image also does *not* provide a TURN server.
 
-The image also does *not* provide a TURN server.
+This image should work on all platforms that are supported by Docker upstream.
+Note that Docker's WS1-backend Linux Containers on Windows
+platform is [experimental](https://github.com/docker/for-win/issues/6470) and
+is not supported by this image.
 
 ## Volumes
 
-By default, the image expects a single volume, located at ``/data``, that will hold:
+By default, the image expects a single volume, located at `/data`, that will hold:
 
 * configuration files;
 * uploaded media and thumbnails;
@@ -16,11 +19,11 @@ By default, the image expects a single volume, located at ``/data``, that will h
 * the appservices configuration.
 
 You are free to use separate volumes depending on storage endpoints at your
-disposal. For instance, ``/data/media`` could be stored on a large but low
+disposal. For instance, `/data/media` could be stored on a large but low
 performance hdd storage while other files could be stored on high performance
 endpoints.
 
-In order to setup an application service, simply create an ``appservices``
+In order to setup an application service, simply create an `appservices`
 directory in the data volume and write the application service Yaml
 configuration file there. Multiple application services are supported.
 
@@ -53,6 +56,8 @@ The following environment variables are supported in `generate` mode:
 * `SYNAPSE_SERVER_NAME` (mandatory): the server public hostname.
 * `SYNAPSE_REPORT_STATS` (mandatory, `yes` or `no`): whether to enable
   anonymous statistics reporting.
+* `SYNAPSE_HTTP_PORT`: the port Synapse should listen on for http traffic.
+      Defaults to `8008`.
 * `SYNAPSE_CONFIG_DIR`: where additional config files (such as the log config
   and event signing key) will be stored. Defaults to `/data`.
 * `SYNAPSE_CONFIG_PATH`: path to the file to be generated. Defaults to
@@ -73,6 +78,8 @@ docker run -d --name synapse \
     matrixdotorg/synapse:latest
 ```
 
+(assuming 8008 is the port Synapse is configured to listen on for http traffic.)
+
 You can then check that it has started correctly with:
 
 ```
@@ -208,4 +215,4 @@ healthcheck:
 ## Using jemalloc
 
 Jemalloc is embedded in the image and will be used instead of the default allocator.
-You can read about jemalloc by reading the Synapse [README](../README.md)
+You can read about jemalloc by reading the Synapse [README](../README.md).

docker/conf-workers/nginx.conf.j2

@@ -0,0 +1,27 @@
+# This file contains the base config for the reverse proxy, as part of ../Dockerfile-workers.
+# configure_workers_and_start.py uses and amends to this file depending on the workers
+# that have been selected.
+
+{{ upstream_directives }}
+
+server {
+    # Listen on an unoccupied port number
+    listen 8008;
+    listen [::]:8008;
+
+    server_name localhost;
+
+    # Nginx by default only allows file uploads up to 1M in size
+    # Increase client_max_body_size to match max_upload_size defined in homeserver.yaml
+    client_max_body_size 100M;
+
+{{ worker_locations }}
+
+    # Send all other traffic to the main process
+    location ~* ^(\\/_matrix|\\/_synapse) {
+        proxy_pass http://localhost:8080;
+        proxy_set_header X-Forwarded-For $remote_addr;
+        proxy_set_header X-Forwarded-Proto $scheme;
+        proxy_set_header Host $host;
+    }
+}

docker/conf-workers/shared.yaml.j2

@@ -0,0 +1,9 @@
+# This file contains the base for the shared homeserver config file between Synapse workers,
+# as part of ./Dockerfile-workers.
+# configure_workers_and_start.py uses and amends to this file depending on the workers
+# that have been selected.
+
+redis:
+    enabled: true
+
+{{ shared_worker_config }}

docker/conf-workers/supervisord.conf.j2

@@ -0,0 +1,41 @@
+# This file contains the base config for supervisord, as part of ../Dockerfile-workers.
+# configure_workers_and_start.py uses and amends to this file depending on the workers
+# that have been selected.
+[supervisord]
+nodaemon=true
+user=root
+
+[program:nginx]
+command=/usr/sbin/nginx -g "daemon off;"
+priority=500
+stdout_logfile=/dev/stdout
+stdout_logfile_maxbytes=0
+stderr_logfile=/dev/stderr
+stderr_logfile_maxbytes=0
+username=www-data
+autorestart=true
+
+[program:redis]
+command=/usr/bin/redis-server /etc/redis/redis.conf --daemonize no
+priority=1
+stdout_logfile=/dev/stdout
+stdout_logfile_maxbytes=0
+stderr_logfile=/dev/stderr
+stderr_logfile_maxbytes=0
+username=redis
+autorestart=true
+
+[program:synapse_main]
+command=/usr/local/bin/python -m synapse.app.homeserver --config-path="{{ main_config_path }}" --config-path=/conf/workers/shared.yaml
+priority=10
+# Log startup failures to supervisord's stdout/err
+# Regular synapse logs will still go in the configured data directory
+stdout_logfile=/dev/stdout
+stdout_logfile_maxbytes=0
+stderr_logfile=/dev/stderr
+stderr_logfile_maxbytes=0
+autorestart=unexpected
+exitcodes=0
+
+# Additional process blocks
+{{ worker_config }}

docker/conf-workers/worker.yaml.j2

@@ -0,0 +1,26 @@
+# This is a configuration template for a single worker instance, and is
+# used by Dockerfile-workers.
+# Values will be change depending on whichever workers are selected when
+# running that image.
+
+worker_app: "{{ app }}"
+worker_name: "{{ name }}"
+
+# The replication listener on the main synapse process.
+worker_replication_host: 127.0.0.1
+worker_replication_http_port: 9093
+
+worker_listeners:
+  - type: http
+    port: {{ port }}
+{% if listener_resources %}
+    resources:
+      - names:
+{%- for resource in listener_resources %}
+        - {{ resource }}
+{%- endfor %}
+{% endif %}
+
+worker_log_config: {{ worker_log_config_filepath }}
+
+{{ worker_extra_conf }}

docker/conf/homeserver.yaml

@@ -40,7 +40,9 @@ listeners:
         compress: false
   {% endif %}
 
-  - port: 8008
+  # Allow configuring in case we want to reverse proxy 8008
+  # using another process in the same container
+  - port: {{ SYNAPSE_HTTP_PORT or 8008 }}
     tls: false
     bind_addresses: ['::']
     type: http

docker/conf/log.config

@@ -2,9 +2,34 @@ version: 1
 
 formatters:
   precise:
-   format: '%(asctime)s - %(name)s - %(lineno)d - %(levelname)s - %(request)s - %(message)s'
+{% if worker_name %}
+    format: '%(asctime)s - worker:{{ worker_name }} - %(name)s - %(lineno)d - %(levelname)s - %(request)s - %(message)s'
+{% else %}
+    format: '%(asctime)s - %(name)s - %(lineno)d - %(levelname)s - %(request)s - %(message)s'
+{% endif %}
 
 handlers:
+  file:
+    class: logging.handlers.TimedRotatingFileHandler
+    formatter: precise
+    filename: {{ LOG_FILE_PATH or "homeserver.log" }}
+    when: "midnight"
+    backupCount: 6  # Does not include the current log file.
+    encoding: utf8
+
+  # Default to buffering writes to log file for efficiency. This means that
+  # there will be a delay for INFO/DEBUG logs to get written, but WARNING/ERROR
+  # logs will still be flushed immediately.
+  buffer:
+    class: logging.handlers.MemoryHandler
+    target: file
+    # The capacity is the number of log lines that are buffered before
+    # being written to disk. Increasing this will lead to better
+    # performance, at the expensive of it taking longer for log lines to
+    # be written to disk.
+    capacity: 10
+    flushLevel: 30  # Flush for WARNING logs as well
+
   console:
     class: logging.StreamHandler
     formatter: precise
@@ -17,6 +42,11 @@ loggers:
 
 root:
     level: {{ SYNAPSE_LOG_LEVEL or "INFO" }}
+
+{% if LOG_FILE_PATH %}
+    handlers: [console, buffer]
+{% else %}
     handlers: [console]
+{% endif %}
 
 disable_existing_loggers: false

docker/configure_workers_and_start.py

@@ -0,0 +1,558 @@
+#!/usr/bin/env python
+# Copyright 2021 The Matrix.org Foundation C.I.C.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This script reads environment variables and generates a shared Synapse worker,
+# nginx and supervisord configs depending on the workers requested.
+#
+# The environment variables it reads are:
+#   * SYNAPSE_SERVER_NAME: The desired server_name of the homeserver.
+#   * SYNAPSE_REPORT_STATS: Whether to report stats.
+#   * SYNAPSE_WORKER_TYPES: A comma separated list of worker names as specified in WORKER_CONFIG
+#         below. Leave empty for no workers, or set to '*' for all possible workers.
+#
+# NOTE: According to Complement's ENTRYPOINT expectations for a homeserver image (as defined
+# in the project's README), this script may be run multiple times, and functionality should
+# continue to work if so.
+
+import os
+import subprocess
+import sys
+
+import jinja2
+import yaml
+
+MAIN_PROCESS_HTTP_LISTENER_PORT = 8080
+
+
+WORKERS_CONFIG = {
+    "pusher": {
+        "app": "synapse.app.pusher",
+        "listener_resources": [],
+        "endpoint_patterns": [],
+        "shared_extra_conf": {"start_pushers": False},
+        "worker_extra_conf": "",
+    },
+    "user_dir": {
+        "app": "synapse.app.user_dir",
+        "listener_resources": ["client"],
+        "endpoint_patterns": [
+            "^/_matrix/client/(api/v1|r0|unstable)/user_directory/search$"
+        ],
+        "shared_extra_conf": {"update_user_directory": False},
+        "worker_extra_conf": "",
+    },
+    "media_repository": {
+        "app": "synapse.app.media_repository",
+        "listener_resources": ["media"],
+        "endpoint_patterns": [
+            "^/_matrix/media/",
+            "^/_synapse/admin/v1/purge_media_cache$",
+            "^/_synapse/admin/v1/room/.*/media.*$",
+            "^/_synapse/admin/v1/user/.*/media.*$",
+            "^/_synapse/admin/v1/media/.*$",
+            "^/_synapse/admin/v1/quarantine_media/.*$",
+        ],
+        "shared_extra_conf": {"enable_media_repo": False},
+        "worker_extra_conf": "enable_media_repo: true",
+    },
+    "appservice": {
+        "app": "synapse.app.appservice",
+        "listener_resources": [],
+        "endpoint_patterns": [],
+        "shared_extra_conf": {"notify_appservices": False},
+        "worker_extra_conf": "",
+    },
+    "federation_sender": {
+        "app": "synapse.app.federation_sender",
+        "listener_resources": [],
+        "endpoint_patterns": [],
+        "shared_extra_conf": {"send_federation": False},
+        "worker_extra_conf": "",
+    },
+    "synchrotron": {
+        "app": "synapse.app.generic_worker",
+        "listener_resources": ["client"],
+        "endpoint_patterns": [
+            "^/_matrix/client/(v2_alpha|r0)/sync$",
+            "^/_matrix/client/(api/v1|v2_alpha|r0)/events$",
+            "^/_matrix/client/(api/v1|r0)/initialSync$",
+            "^/_matrix/client/(api/v1|r0)/rooms/[^/]+/initialSync$",
+        ],
+        "shared_extra_conf": {},
+        "worker_extra_conf": "",
+    },
+    "federation_reader": {
+        "app": "synapse.app.generic_worker",
+        "listener_resources": ["federation"],
+        "endpoint_patterns": [
+            "^/_matrix/federation/(v1|v2)/event/",
+            "^/_matrix/federation/(v1|v2)/state/",
+            "^/_matrix/federation/(v1|v2)/state_ids/",
+            "^/_matrix/federation/(v1|v2)/backfill/",
+            "^/_matrix/federation/(v1|v2)/get_missing_events/",
+            "^/_matrix/federation/(v1|v2)/publicRooms",
+            "^/_matrix/federation/(v1|v2)/query/",
+            "^/_matrix/federation/(v1|v2)/make_join/",
+            "^/_matrix/federation/(v1|v2)/make_leave/",
+            "^/_matrix/federation/(v1|v2)/send_join/",
+            "^/_matrix/federation/(v1|v2)/send_leave/",
+            "^/_matrix/federation/(v1|v2)/invite/",
+            "^/_matrix/federation/(v1|v2)/query_auth/",
+            "^/_matrix/federation/(v1|v2)/event_auth/",
+            "^/_matrix/federation/(v1|v2)/exchange_third_party_invite/",
+            "^/_matrix/federation/(v1|v2)/user/devices/",
+            "^/_matrix/federation/(v1|v2)/get_groups_publicised$",
+            "^/_matrix/key/v2/query",
+        ],
+        "shared_extra_conf": {},
+        "worker_extra_conf": "",
+    },
+    "federation_inbound": {
+        "app": "synapse.app.generic_worker",
+        "listener_resources": ["federation"],
+        "endpoint_patterns": ["/_matrix/federation/(v1|v2)/send/"],
+        "shared_extra_conf": {},
+        "worker_extra_conf": "",
+    },
+    "event_persister": {
+        "app": "synapse.app.generic_worker",
+        "listener_resources": ["replication"],
+        "endpoint_patterns": [],
+        "shared_extra_conf": {},
+        "worker_extra_conf": "",
+    },
+    "background_worker": {
+        "app": "synapse.app.generic_worker",
+        "listener_resources": [],
+        "endpoint_patterns": [],
+        # This worker cannot be sharded. Therefore there should only ever be one background
+        # worker, and it should be named background_worker1
+        "shared_extra_conf": {"run_background_tasks_on": "background_worker1"},
+        "worker_extra_conf": "",
+    },
+    "event_creator": {
+        "app": "synapse.app.generic_worker",
+        "listener_resources": ["client"],
+        "endpoint_patterns": [
+            "^/_matrix/client/(api/v1|r0|unstable)/rooms/.*/redact",
+            "^/_matrix/client/(api/v1|r0|unstable)/rooms/.*/send",
+            "^/_matrix/client/(api/v1|r0|unstable)/rooms/.*/(join|invite|leave|ban|unban|kick)$",
+            "^/_matrix/client/(api/v1|r0|unstable)/join/",
+            "^/_matrix/client/(api/v1|r0|unstable)/profile/",
+        ],
+        "shared_extra_conf": {},
+        "worker_extra_conf": "",
+    },
+    "frontend_proxy": {
+        "app": "synapse.app.frontend_proxy",
+        "listener_resources": ["client", "replication"],
+        "endpoint_patterns": ["^/_matrix/client/(api/v1|r0|unstable)/keys/upload"],
+        "shared_extra_conf": {},
+        "worker_extra_conf": (
+            "worker_main_http_uri: http://127.0.0.1:%d"
+            % (MAIN_PROCESS_HTTP_LISTENER_PORT,),
+        ),
+    },
+}
+
+# Templates for sections that may be inserted multiple times in config files
+SUPERVISORD_PROCESS_CONFIG_BLOCK = """
+[program:synapse_{name}]
+command=/usr/local/bin/python -m {app} \
+    --config-path="{config_path}" \
+    --config-path=/conf/workers/shared.yaml \
+    --config-path=/conf/workers/{name}.yaml
+autorestart=unexpected
+priority=500
+exitcodes=0
+stdout_logfile=/dev/stdout
+stdout_logfile_maxbytes=0
+stderr_logfile=/dev/stderr
+stderr_logfile_maxbytes=0
+"""
+
+NGINX_LOCATION_CONFIG_BLOCK = """
+    location ~* {endpoint} {
+        proxy_pass {upstream};
+        proxy_set_header X-Forwarded-For $remote_addr;
+        proxy_set_header X-Forwarded-Proto $scheme;
+        proxy_set_header Host $host;
+    }
+"""
+
+NGINX_UPSTREAM_CONFIG_BLOCK = """
+upstream {upstream_worker_type} {
+{body}
+}
+"""
+
+
+# Utility functions
+def log(txt: str):
+    """Log something to the stdout.
+
+    Args:
+        txt: The text to log.
+    """
+    print(txt)
+
+
+def error(txt: str):
+    """Log something and exit with an error code.
+
+    Args:
+        txt: The text to log in error.
+    """
+    log(txt)
+    sys.exit(2)
+
+
+def convert(src: str, dst: str, **template_vars):
+    """Generate a file from a template
+
+    Args:
+        src: Path to the input file.
+        dst: Path to write to.
+        template_vars: The arguments to replace placeholder variables in the template with.
+    """
+    # Read the template file
+    with open(src) as infile:
+        template = infile.read()
+
+    # Generate a string from the template. We disable autoescape to prevent template
+    # variables from being escaped.
+    rendered = jinja2.Template(template, autoescape=False).render(**template_vars)
+
+    # Write the generated contents to a file
+    #
+    # We use append mode in case the files have already been written to by something else
+    # (for instance, as part of the instructions in a dockerfile).
+    with open(dst, "a") as outfile:
+        # In case the existing file doesn't end with a newline
+        outfile.write("\n")
+
+        outfile.write(rendered)
+
+
+def add_sharding_to_shared_config(
+    shared_config: dict,
+    worker_type: str,
+    worker_name: str,
+    worker_port: int,
+) -> None:
+    """Given a dictionary representing a config file shared across all workers,
+    append sharded worker information to it for the current worker_type instance.
+
+    Args:
+        shared_config: The config dict that all worker instances share (after being converted to YAML)
+        worker_type: The type of worker (one of those defined in WORKERS_CONFIG).
+        worker_name: The name of the worker instance.
+        worker_port: The HTTP replication port that the worker instance is listening on.
+    """
+    # The instance_map config field marks the workers that write to various replication streams
+    instance_map = shared_config.setdefault("instance_map", {})
+
+    # Worker-type specific sharding config
+    if worker_type == "pusher":
+        shared_config.setdefault("pusher_instances", []).append(worker_name)
+
+    elif worker_type == "federation_sender":
+        shared_config.setdefault("federation_sender_instances", []).append(worker_name)
+
+    elif worker_type == "event_persister":
+        # Event persisters write to the events stream, so we need to update
+        # the list of event stream writers
+        shared_config.setdefault("stream_writers", {}).setdefault("events", []).append(
+            worker_name
+        )
+
+        # Map of stream writer instance names to host/ports combos
+        instance_map[worker_name] = {
+            "host": "localhost",
+            "port": worker_port,
+        }
+
+    elif worker_type == "media_repository":
+        # The first configured media worker will run the media background jobs
+        shared_config.setdefault("media_instance_running_background_jobs", worker_name)
+
+
+def generate_base_homeserver_config():
+    """Starts Synapse and generates a basic homeserver config, which will later be
+    modified for worker support.
+
+    Raises: CalledProcessError if calling start.py returned a non-zero exit code.
+    """
+    # start.py already does this for us, so just call that.
+    # note that this script is copied in in the official, monolith dockerfile
+    os.environ["SYNAPSE_HTTP_PORT"] = str(MAIN_PROCESS_HTTP_LISTENER_PORT)
+    subprocess.check_output(["/usr/local/bin/python", "/start.py", "migrate_config"])
+
+
+def generate_worker_files(environ, config_path: str, data_dir: str):
+    """Read the desired list of workers from environment variables and generate
+    shared homeserver, nginx and supervisord configs.
+
+    Args:
+        environ: _Environ[str]
+        config_path: Where to output the generated Synapse main worker config file.
+        data_dir: The location of the synapse data directory. Where log and
+            user-facing config files live.
+    """
+    # Note that yaml cares about indentation, so care should be taken to insert lines
+    # into files at the correct indentation below.
+
+    # shared_config is the contents of a Synapse config file that will be shared amongst
+    # the main Synapse process as well as all workers.
+    # It is intended mainly for disabling functionality when certain workers are spun up,
+    # and adding a replication listener.
+
+    # First read the original config file and extract the listeners block. Then we'll add
+    # another listener for replication. Later we'll write out the result.
+    listeners = [
+        {
+            "port": 9093,
+            "bind_address": "127.0.0.1",
+            "type": "http",
+            "resources": [{"names": ["replication"]}],
+        }
+    ]
+    with open(config_path) as file_stream:
+        original_config = yaml.safe_load(file_stream)
+        original_listeners = original_config.get("listeners")
+        if original_listeners:
+            listeners += original_listeners
+
+    # The shared homeserver config. The contents of which will be inserted into the
+    # base shared worker jinja2 template.
+    #
+    # This config file will be passed to all workers, included Synapse's main process.
+    shared_config = {"listeners": listeners}
+
+    # The supervisord config. The contents of which will be inserted into the
+    # base supervisord jinja2 template.
+    #
+    # Supervisord will be in charge of running everything, from redis to nginx to Synapse
+    # and all of its worker processes. Load the config template, which defines a few
+    # services that are necessary to run.
+    supervisord_config = ""
+
+    # Upstreams for load-balancing purposes. This dict takes the form of a worker type to the
+    # ports of each worker. For example:
+    # {
+    #   worker_type: {1234, 1235, ...}}
+    # }
+    # and will be used to construct 'upstream' nginx directives.
+    nginx_upstreams = {}
+
+    # A map of: {"endpoint": "upstream"}, where "upstream" is a str representing what will be
+    # placed after the proxy_pass directive. The main benefit to representing this data as a
+    # dict over a str is that we can easily deduplicate endpoints across multiple instances
+    # of the same worker.
+    #
+    # An nginx site config that will be amended to depending on the workers that are
+    # spun up. To be placed in /etc/nginx/conf.d.
+    nginx_locations = {}
+
+    # Read the desired worker configuration from the environment
+    worker_types = environ.get("SYNAPSE_WORKER_TYPES")
+    if worker_types is None:
+        # No workers, just the main process
+        worker_types = []
+    else:
+        # Split type names by comma
+        worker_types = worker_types.split(",")
+
+    # Create the worker configuration directory if it doesn't already exist
+    os.makedirs("/conf/workers", exist_ok=True)
+
+    # Start worker ports from this arbitrary port
+    worker_port = 18009
+
+    # A counter of worker_type -> int. Used for determining the name for a given
+    # worker type when generating its config file, as each worker's name is just
+    # worker_type + instance #
+    worker_type_counter = {}
+
+    # For each worker type specified by the user, create config values
+    for worker_type in worker_types:
+        worker_type = worker_type.strip()
+
+        worker_config = WORKERS_CONFIG.get(worker_type)
+        if worker_config:
+            worker_config = worker_config.copy()
+        else:
+            log(worker_type + " is an unknown worker type! It will be ignored")
+            continue
+
+        new_worker_count = worker_type_counter.setdefault(worker_type, 0) + 1
+        worker_type_counter[worker_type] = new_worker_count
+
+        # Name workers by their type concatenated with an incrementing number
+        # e.g. federation_reader1
+        worker_name = worker_type + str(new_worker_count)
+        worker_config.update(
+            {"name": worker_name, "port": worker_port, "config_path": config_path}
+        )
+
+        # Update the shared config with any worker-type specific options
+        shared_config.update(worker_config["shared_extra_conf"])
+
+        # Check if more than one instance of this worker type has been specified
+        worker_type_total_count = worker_types.count(worker_type)
+        if worker_type_total_count > 1:
+            # Update the shared config with sharding-related options if necessary
+            add_sharding_to_shared_config(
+                shared_config, worker_type, worker_name, worker_port
+            )
+
+        # Enable the worker in supervisord
+        supervisord_config += SUPERVISORD_PROCESS_CONFIG_BLOCK.format_map(worker_config)
+
+        # Add nginx location blocks for this worker's endpoints (if any are defined)
+        for pattern in worker_config["endpoint_patterns"]:
+            # Determine whether we need to load-balance this worker
+            if worker_type_total_count > 1:
+                # Create or add to a load-balanced upstream for this worker
+                nginx_upstreams.setdefault(worker_type, set()).add(worker_port)
+
+                # Upstreams are named after the worker_type
+                upstream = "http://" + worker_type
+            else:
+                upstream = "http://localhost:%d" % (worker_port,)
+
+            # Note that this endpoint should proxy to this upstream
+            nginx_locations[pattern] = upstream
+
+        # Write out the worker's logging config file
+
+        # Check whether we should write worker logs to disk, in addition to the console
+        extra_log_template_args = {}
+        if environ.get("SYNAPSE_WORKERS_WRITE_LOGS_TO_DISK"):
+            extra_log_template_args["LOG_FILE_PATH"] = "{dir}/logs/{name}.log".format(
+                dir=data_dir, name=worker_name
+            )
+
+        # Render and write the file
+        log_config_filepath = "/conf/workers/{name}.log.config".format(name=worker_name)
+        convert(
+            "/conf/log.config",
+            log_config_filepath,
+            worker_name=worker_name,
+            **extra_log_template_args,
+        )
+
+        # Then a worker config file
+        convert(
+            "/conf/worker.yaml.j2",
+            "/conf/workers/{name}.yaml".format(name=worker_name),
+            **worker_config,
+            worker_log_config_filepath=log_config_filepath,
+        )
+
+        worker_port += 1
+
+    # Build the nginx location config blocks
+    nginx_location_config = ""
+    for endpoint, upstream in nginx_locations.items():
+        nginx_location_config += NGINX_LOCATION_CONFIG_BLOCK.format(
+            endpoint=endpoint,
+            upstream=upstream,
+        )
+
+    # Determine the load-balancing upstreams to configure
+    nginx_upstream_config = ""
+    for upstream_worker_type, upstream_worker_ports in nginx_upstreams.items():
+        body = ""
+        for port in upstream_worker_ports:
+            body += "    server localhost:%d;\n" % (port,)
+
+        # Add to the list of configured upstreams
+        nginx_upstream_config += NGINX_UPSTREAM_CONFIG_BLOCK.format(
+            upstream_worker_type=upstream_worker_type,
+            body=body,
+        )
+
+    # Finally, we'll write out the config files.
+
+    # Shared homeserver config
+    convert(
+        "/conf/shared.yaml.j2",
+        "/conf/workers/shared.yaml",
+        shared_worker_config=yaml.dump(shared_config),
+    )
+
+    # Nginx config
+    convert(
+        "/conf/nginx.conf.j2",
+        "/etc/nginx/conf.d/matrix-synapse.conf",
+        worker_locations=nginx_location_config,
+        upstream_directives=nginx_upstream_config,
+    )
+
+    # Supervisord config
+    convert(
+        "/conf/supervisord.conf.j2",
+        "/etc/supervisor/conf.d/supervisord.conf",
+        main_config_path=config_path,
+        worker_config=supervisord_config,
+    )
+
+    # Ensure the logging directory exists
+    log_dir = data_dir + "/logs"
+    if not os.path.exists(log_dir):
+        os.mkdir(log_dir)
+
+
+def start_supervisord():
+    """Starts up supervisord which then starts and monitors all other necessary processes
+
+    Raises: CalledProcessError if calling start.py return a non-zero exit code.
+    """
+    subprocess.run(["/usr/bin/supervisord"], stdin=subprocess.PIPE)
+
+
+def main(args, environ):
+    config_dir = environ.get("SYNAPSE_CONFIG_DIR", "/data")
+    config_path = environ.get("SYNAPSE_CONFIG_PATH", config_dir + "/homeserver.yaml")
+    data_dir = environ.get("SYNAPSE_DATA_DIR", "/data")
+
+    # override SYNAPSE_NO_TLS, we don't support TLS in worker mode,
+    # this needs to be handled by a frontend proxy
+    environ["SYNAPSE_NO_TLS"] = "yes"
+
+    # Generate the base homeserver config if one does not yet exist
+    if not os.path.exists(config_path):
+        log("Generating base homeserver config")
+        generate_base_homeserver_config()
+
+    # This script may be run multiple times (mostly by Complement, see note at top of file).
+    # Don't re-configure workers in this instance.
+    mark_filepath = "/conf/workers_have_been_configured"
+    if not os.path.exists(mark_filepath):
+        # Always regenerate all other config files
+        generate_worker_files(environ, config_path, data_dir)
+
+        # Mark workers as being configured
+        with open(mark_filepath, "w") as f:
+            f.write("")
+
+    # Start supervisord, which will start Synapse, all of the configured worker
+    # processes, redis, nginx etc. according to the config we created above.
+    start_supervisord()
+
+
+if __name__ == "__main__":
+    main(sys.argv, os.environ)

docs/admin_api/user_admin_api.rst

@@ -202,7 +202,7 @@ The following fields are returned in the JSON response body:
 - ``users`` - An array of objects, each containing information about an user.
   User objects contain the following fields:
 
-  - ``name`` - string - Fully-qualified user ID (ex. `@user:server.com`).
+  - ``name`` - string - Fully-qualified user ID (ex. ``@user:server.com``).
   - ``is_guest`` - bool - Status if that user is a guest account.
   - ``admin`` - bool - Status if that user is a server administrator.
   - ``user_type`` - string - Type of the user. Normal users are type ``None``.
@@ -864,3 +864,118 @@ The following parameters should be set in the URL:
 
 - ``user_id`` - The fully qualified MXID: for example, ``@user:server.com``. The user must
   be local.
+
+Override ratelimiting for users
+===============================
+
+This API allows to override or disable ratelimiting for a specific user.
+There are specific APIs to set, get and delete a ratelimit.
+
+Get status of ratelimit
+-----------------------
+
+The API is::
+
+  GET /_synapse/admin/v1/users/<user_id>/override_ratelimit
+
+To use it, you will need to authenticate by providing an ``access_token`` for a
+server admin: see `README.rst <README.rst>`_.
+
+A response body like the following is returned:
+
+.. code:: json
+
+    {
+      "messages_per_second": 0,
+      "burst_count": 0
+    }
+
+**Parameters**
+
+The following parameters should be set in the URL:
+
+- ``user_id`` - The fully qualified MXID: for example, ``@user:server.com``. The user must
+  be local.
+
+**Response**
+
+The following fields are returned in the JSON response body:
+
+- ``messages_per_second`` - integer - The number of actions that can
+  be performed in a second. `0` mean that ratelimiting is disabled for this user.
+- ``burst_count`` - integer - How many actions that can be performed before
+  being limited.
+
+If **no** custom ratelimit is set, an empty JSON dict is returned.
+
+.. code:: json
+
+    {}
+
+Set ratelimit
+-------------
+
+The API is::
+
+  POST /_synapse/admin/v1/users/<user_id>/override_ratelimit
+
+To use it, you will need to authenticate by providing an ``access_token`` for a
+server admin: see `README.rst <README.rst>`_.
+
+A response body like the following is returned:
+
+.. code:: json
+
+    {
+      "messages_per_second": 0,
+      "burst_count": 0
+    }
+
+**Parameters**
+
+The following parameters should be set in the URL:
+
+- ``user_id`` - The fully qualified MXID: for example, ``@user:server.com``. The user must
+  be local.
+
+Body parameters:
+
+- ``messages_per_second`` - positive integer, optional. The number of actions that can
+  be performed in a second. Defaults to ``0``.
+- ``burst_count`` - positive integer, optional. How many actions that can be performed
+  before being limited. Defaults to ``0``.
+
+To disable users' ratelimit set both values to ``0``.
+
+**Response**
+
+The following fields are returned in the JSON response body:
+
+- ``messages_per_second`` - integer - The number of actions that can
+  be performed in a second.
+- ``burst_count`` - integer - How many actions that can be performed before
+  being limited.
+
+Delete ratelimit
+----------------
+
+The API is::
+
+  DELETE /_synapse/admin/v1/users/<user_id>/override_ratelimit
+
+To use it, you will need to authenticate by providing an ``access_token`` for a
+server admin: see `README.rst <README.rst>`_.
+
+An empty JSON dict is returned.
+
+.. code:: json
+
+    {}
+
+**Parameters**
+
+The following parameters should be set in the URL:
+
+- ``user_id`` - The fully qualified MXID: for example, ``@user:server.com``. The user must
+  be local.
+

docs/deprecation_policy.md

@@ -0,0 +1,33 @@
+Deprecation Policy for Platform Dependencies
+============================================
+
+Synapse has a number of platform dependencies, including Python and PostgreSQL.
+This document outlines the policy towards which versions we support, and when we
+drop support for versions in the future.
+
+
+Policy
+------
+
+Synapse follows the upstream support life cycles for Python and PostgreSQL,
+i.e. when a version reaches End of Life Synapse will withdraw support for that
+version in future releases.
+
+Details on the upstream support life cycles for Python and PostgreSQL are
+documented at https://endoflife.date/python and
+https://endoflife.date/postgresql.
+
+
+Context
+-------
+
+It is important for system admins to have a clear understanding of the platform
+requirements of Synapse and its deprecation policies so that they can
+effectively plan upgrading their infrastructure ahead of time. This is
+especially important in contexts where upgrading the infrastructure requires
+auditing and approval from a security team, or where otherwise upgrading is a
+long process.
+
+By following the upstream support life cycles Synapse can ensure that its
+dependencies continue to get security patches, while not requiring system admins
+to constantly update their platform dependencies to the latest versions.

docs/presence_router_module.md

@@ -0,0 +1,235 @@
+# Presence Router Module
+
+Synapse supports configuring a module that can specify additional users
+(local or remote) to should receive certain presence updates from local
+users.
+
+Note that routing presence via Application Service transactions is not
+currently supported.
+
+The presence routing module is implemented as a Python class, which will
+be imported by the running Synapse.
+
+## Python Presence Router Class
+
+The Python class is instantiated with two objects:
+
+* A configuration object of some type (see below).
+* An instance of `synapse.module_api.ModuleApi`.
+
+It then implements methods related to presence routing.
+
+Note that one method of `ModuleApi` that may be useful is:
+
+```python
+async def ModuleApi.send_local_online_presence_to(users: Iterable[str]) -> None
+```
+
+which can be given a list of local or remote MXIDs to broadcast known, online user
+presence to (for those users that the receiving user is considered interested in). 
+It does not include state for users who are currently offline, and it can only be
+called on workers that support sending federation.
+
+### Module structure
+
+Below is a list of possible methods that can be implemented, and whether they are
+required.
+
+#### `parse_config`
+
+```python
+def parse_config(config_dict: dict) -> Any
+```
+
+**Required.** A static method that is passed a dictionary of config options, and
+  should return a validated config object. This method is described further in
+  [Configuration](#configuration).
+
+#### `get_users_for_states`
+
+```python
+async def get_users_for_states(
+    self,
+    state_updates: Iterable[UserPresenceState],
+) -> Dict[str, Set[UserPresenceState]]:
+```
+
+**Required.** An asynchronous method that is passed an iterable of user presence
+state. This method can determine whether a given presence update should be sent to certain
+users. It does this by returning a dictionary with keys representing local or remote
+Matrix User IDs, and values being a python set
+of `synapse.handlers.presence.UserPresenceState` instances.
+
+Synapse will then attempt to send the specified presence updates to each user when
+possible.
+
+#### `get_interested_users`
+
+```python
+async def get_interested_users(self, user_id: str) -> Union[Set[str], str]
+```
+
+**Required.** An asynchronous method that is passed a single Matrix User ID. This
+method is expected to return the users that the passed in user may be interested in the
+presence of. Returned users may be local or remote. The presence routed as a result of
+what this method returns is sent in addition to the updates already sent between users
+that share a room together. Presence updates are deduplicated.
+
+This method should return a python set of Matrix User IDs, or the object
+`synapse.events.presence_router.PresenceRouter.ALL_USERS` to indicate that the passed
+user should receive presence information for *all* known users.
+
+For clarity, if the user `@alice:example.org` is passed to this method, and the Set
+`{"@bob:example.com", "@charlie:somewhere.org"}` is returned, this signifies that Alice
+should receive presence updates sent by Bob and Charlie, regardless of whether these
+users share a room.
+
+### Example
+
+Below is an example implementation of a presence router class.
+
+```python
+from typing import Dict, Iterable, Set, Union
+from synapse.events.presence_router import PresenceRouter
+from synapse.handlers.presence import UserPresenceState
+from synapse.module_api import ModuleApi
+
+class PresenceRouterConfig:
+    def __init__(self):
+        # Config options with their defaults
+        # A list of users to always send all user presence updates to
+        self.always_send_to_users = []  # type: List[str]
+        
+        # A list of users to ignore presence updates for. Does not affect
+        # shared-room presence relationships
+        self.blacklisted_users = []  # type: List[str]
+
+class ExamplePresenceRouter:
+    """An example implementation of synapse.presence_router.PresenceRouter.
+    Supports routing all presence to a configured set of users, or a subset
+    of presence from certain users to members of certain rooms.
+
+    Args:
+        config: A configuration object.
+        module_api: An instance of Synapse's ModuleApi.
+    """
+    def __init__(self, config: PresenceRouterConfig, module_api: ModuleApi):
+        self._config = config
+        self._module_api = module_api
+
+    @staticmethod
+    def parse_config(config_dict: dict) -> PresenceRouterConfig:
+        """Parse a configuration dictionary from the homeserver config, do
+        some validation and return a typed PresenceRouterConfig.
+
+        Args:
+            config_dict: The configuration dictionary.
+
+        Returns:
+            A validated config object.
+        """
+        # Initialise a typed config object
+        config = PresenceRouterConfig()
+        always_send_to_users = config_dict.get("always_send_to_users")
+        blacklisted_users = config_dict.get("blacklisted_users")
+
+        # Do some validation of config options... otherwise raise a
+        # synapse.config.ConfigError.
+        config.always_send_to_users = always_send_to_users
+        config.blacklisted_users = blacklisted_users
+
+        return config
+
+    async def get_users_for_states(
+        self,
+        state_updates: Iterable[UserPresenceState],
+    ) -> Dict[str, Set[UserPresenceState]]:
+        """Given an iterable of user presence updates, determine where each one
+        needs to go. Returned results will not affect presence updates that are
+        sent between users who share a room.
+
+        Args:
+            state_updates: An iterable of user presence state updates.
+
+        Returns:
+          A dictionary of user_id -> set of UserPresenceState that the user should 
+          receive.
+        """
+        destination_users = {}  # type: Dict[str, Set[UserPresenceState]
+
+        # Ignore any updates for blacklisted users
+        desired_updates = set()
+        for update in state_updates:
+            if update.state_key not in self._config.blacklisted_users:
+                desired_updates.add(update)
+
+        # Send all presence updates to specific users
+        for user_id in self._config.always_send_to_users:
+            destination_users[user_id] = desired_updates
+
+        return destination_users
+
+    async def get_interested_users(
+        self,
+        user_id: str,
+    ) -> Union[Set[str], PresenceRouter.ALL_USERS]:
+        """
+        Retrieve a list of users that `user_id` is interested in receiving the
+        presence of. This will be in addition to those they share a room with.
+        Optionally, the object PresenceRouter.ALL_USERS can be returned to indicate
+        that this user should receive all incoming local and remote presence updates.
+
+        Note that this method will only be called for local users.
+
+        Args:
+          user_id: A user requesting presence updates.
+
+        Returns:
+          A set of user IDs to return additional presence updates for, or
+          PresenceRouter.ALL_USERS to return presence updates for all other users.
+        """
+        if user_id in self._config.always_send_to_users:
+            return PresenceRouter.ALL_USERS
+
+        return set()
+```
+
+#### A note on `get_users_for_states` and `get_interested_users`
+
+Both of these methods are effectively two different sides of the same coin. The logic
+regarding which users should receive updates for other users should be the same 
+between them.
+
+`get_users_for_states` is called when presence updates come in from either federation 
+or local users, and is used to either direct local presence to remote users, or to
+wake up the sync streams of local users to collect remote presence.
+
+In contrast, `get_interested_users` is used to determine the users that presence should
+be fetched for when a local user is syncing. This presence is then retrieved, before
+being fed through `get_users_for_states` once again, with only the syncing user's
+routing information pulled from the resulting dictionary.
+
+Their routing logic should thus line up, else you may run into unintended behaviour.
+
+## Configuration
+
+Once you've crafted your module and installed it into the same Python environment as
+Synapse, amend your homeserver config file with the following.
+
+```yaml
+presence:
+  routing_module:
+    module: my_module.ExamplePresenceRouter
+    config:
+      # Any configuration options for your module. The below is an example.
+      # of setting options for ExamplePresenceRouter.
+      always_send_to_users: ["@presence_gobbler:example.org"]
+      blacklisted_users:
+        - "@alice:example.com"
+        - "@bob:example.com"
+      ...
+```
+
+The contents of `config` will be passed as a Python dictionary to the static
+`parse_config` method of your class. The object returned by this method will
+then be passed to the `__init__` method of your module as `config`.

docs/sample_config.yaml

@@ -82,9 +82,28 @@ pid_file: DATADIR/homeserver.pid
 #
 #soft_file_limit: 0
 
-# Set to false to disable presence tracking on this homeserver.
+# Presence tracking allows users to see the state (e.g online/offline)
+# of other local and remote users.
 #
-#use_presence: false
+presence:
+  # Uncomment to disable presence tracking on this homeserver. This option
+  # replaces the previous top-level 'use_presence' option.
+  #
+  #enabled: false
+
+  # Presence routers are third-party modules that can specify additional logic
+  # to where presence updates from users are routed.
+  #
+  presence_router:
+    # The custom module's class. Uncomment to use a custom presence router module.
+    #
+    #module: "my_custom_router.PresenceRouter"
+
+    # Configuration options of the custom module. Refer to your module's
+    # documentation for available options.
+    #
+    #config:
+    #  example_option: 'something'
 
 # Whether to require authentication to retrieve profile data (avatars,
 # display names) of other users through the client API. Defaults to

docs/tcp_replication.md

@@ -216,10 +216,6 @@ Asks the server for the current position of all streams.
 
    This is used when a worker is shutting down.
 
-#### FEDERATION_ACK (C)
-
-   Acknowledge receipt of some federation data
-
 ### REMOTE_SERVER_UP (S, C)
 
    Inform other processes that a remote server may have come back online.

mypy.ini

@@ -8,6 +8,7 @@ show_traceback = True
 mypy_path = stubs
 warn_unreachable = True
 local_partial_types = True
+no_implicit_optional = True
 
 # To find all folders that pass mypy you run:
 #

pyproject.toml

@@ -35,7 +35,7 @@
         showcontent = true
 
 [tool.black]
-target-version = ['py35']
+target-version = ['py36']
 exclude = '''
 
 (

scripts-dev/build_debian_packages

@@ -18,11 +18,9 @@ import threading
 from concurrent.futures import ThreadPoolExecutor
 
 DISTS = (
-    "debian:stretch",
     "debian:buster",
     "debian:bullseye",
     "debian:sid",
-    "ubuntu:xenial",
     "ubuntu:bionic",
     "ubuntu:focal",
     "ubuntu:groovy",
@@ -43,7 +41,7 @@ class Builder(object):
         self._lock = threading.Lock()
         self._failed = False
 
-    def run_build(self, dist):
+    def run_build(self, dist, skip_tests=False):
         """Build deb for a single distribution"""
 
         if self._failed:
@@ -51,13 +49,13 @@ class Builder(object):
             raise Exception("failed")
 
         try:
-            self._inner_build(dist)
+            self._inner_build(dist, skip_tests)
         except Exception as e:
             print("build of %s failed: %s" % (dist, e), file=sys.stderr)
             self._failed = True
             raise
 
-    def _inner_build(self, dist):
+    def _inner_build(self, dist, skip_tests=False):
         projdir = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
         os.chdir(projdir)
 
@@ -101,6 +99,7 @@ class Builder(object):
             "--volume=" + debsdir + ":/debs",
             "-e", "TARGET_USERID=%i" % (os.getuid(), ),
             "-e", "TARGET_GROUPID=%i" % (os.getgid(), ),
+            "-e", "DEB_BUILD_OPTIONS=%s" % ("nocheck" if skip_tests else ""),
             "dh-venv-builder:" + tag,
         ], stdout=stdout, stderr=subprocess.STDOUT)
 
@@ -124,7 +123,7 @@ class Builder(object):
                 self.active_containers.remove(c)
 
 
-def run_builds(dists, jobs=1):
+def run_builds(dists, jobs=1, skip_tests=False):
     builder = Builder(redirect_stdout=(jobs > 1))
 
     def sig(signum, _frame):
@@ -133,7 +132,7 @@ def run_builds(dists, jobs=1):
     signal.signal(signal.SIGINT, sig)
 
     with ThreadPoolExecutor(max_workers=jobs) as e:
-        res = e.map(builder.run_build, dists)
+        res = e.map(lambda dist: builder.run_build(dist, skip_tests), dists)
 
     # make sure we consume the iterable so that exceptions are raised.
     for r in res:
@@ -148,9 +147,13 @@ if __name__ == '__main__':
         '-j', '--jobs', type=int, default=1,
         help='specify the number of builds to run in parallel',
     )
+    parser.add_argument(
+        '--no-check', action='store_true',
+        help='skip running tests after building',
+    )
     parser.add_argument(
         'dist', nargs='*', default=DISTS,
         help='a list of distributions to build for. Default: %(default)s',
     )
     args = parser.parse_args()
-    run_builds(dists=args.dist, jobs=args.jobs)
+    run_builds(dists=args.dist, jobs=args.jobs, skip_tests=args.no_check)

scripts-dev/complement.sh

@@ -46,4 +46,4 @@ if [[ -n "$1" ]]; then
 fi
 
 # Run the tests!
-COMPLEMENT_BASE_IMAGE=complement-synapse go test -v -tags synapse_blacklist -count=1 $EXTRA_COMPLEMENT_ARGS ./tests
+COMPLEMENT_BASE_IMAGE=complement-synapse go test -v -tags synapse_blacklist,msc2946,msc3083 -count=1 $EXTRA_COMPLEMENT_ARGS ./tests

scripts-dev/mypy_synapse_plugin.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2020 The Matrix.org Foundation C.I.C.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");

scripts-dev/release.py

@@ -0,0 +1,244 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# Copyright 2020 The Matrix.org Foundation C.I.C.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""An interactive script for doing a release. See `run()` below.
+"""
+
+import subprocess
+import sys
+from typing import Optional
+
+import click
+import git
+from packaging import version
+from redbaron import RedBaron
+
+
+@click.command()
+def run():
+    """An interactive script to walk through the initial stages of creating a
+    release, including creating release branch, updating changelog and pushing to
+    GitHub.
+
+    Requires the dev dependencies be installed, which can be done via:
+
+        pip install -e .[dev]
+
+    """
+
+    # Make sure we're in a git repo.
+    try:
+        repo = git.Repo()
+    except git.InvalidGitRepositoryError:
+        raise click.ClickException("Not in Synapse repo.")
+
+    if repo.is_dirty():
+        raise click.ClickException("Uncommitted changes exist.")
+
+    click.secho("Updating git repo...")
+    repo.remote().fetch()
+
+    # Parse the AST and load the `__version__` node so that we can edit it
+    # later.
+    with open("synapse/__init__.py") as f:
+        red = RedBaron(f.read())
+
+    version_node = None
+    for node in red:
+        if node.type != "assignment":
+            continue
+
+        if node.target.type != "name":
+            continue
+
+        if node.target.value != "__version__":
+            continue
+
+        version_node = node
+        break
+
+    if not version_node:
+        print("Failed to find '__version__' definition in synapse/__init__.py")
+        sys.exit(1)
+
+    # Parse the current version.
+    current_version = version.parse(version_node.value.value.strip('"'))
+    assert isinstance(current_version, version.Version)
+
+    # Figure out what sort of release we're doing and calcuate the new version.
+    rc = click.confirm("RC", default=True)
+    if current_version.pre:
+        # If the current version is an RC we don't need to bump any of the
+        # version numbers (other than the RC number).
+        base_version = "{}.{}.{}".format(
+            current_version.major,
+            current_version.minor,
+            current_version.micro,
+        )
+
+        if rc:
+            new_version = "{}.{}.{}rc{}".format(
+                current_version.major,
+                current_version.minor,
+                current_version.micro,
+                current_version.pre[1] + 1,
+            )
+        else:
+            new_version = base_version
+    else:
+        # If this is a new release cycle then we need to know if its a major
+        # version bump or a hotfix.
+        release_type = click.prompt(
+            "Release type",
+            type=click.Choice(("major", "hotfix")),
+            show_choices=True,
+            default="major",
+        )
+
+        if release_type == "major":
+            base_version = new_version = "{}.{}.{}".format(
+                current_version.major,
+                current_version.minor + 1,
+                0,
+            )
+            if rc:
+                new_version = "{}.{}.{}rc1".format(
+                    current_version.major,
+                    current_version.minor + 1,
+                    0,
+                )
+
+        else:
+            base_version = new_version = "{}.{}.{}".format(
+                current_version.major,
+                current_version.minor,
+                current_version.micro + 1,
+            )
+            if rc:
+                new_version = "{}.{}.{}rc1".format(
+                    current_version.major,
+                    current_version.minor,
+                    current_version.micro + 1,
+                )
+
+    # Confirm the calculated version is OK.
+    if not click.confirm(f"Create new version: {new_version}?", default=True):
+        click.get_current_context().abort()
+
+    # Switch to the release branch.
+    release_branch_name = f"release-v{base_version}"
+    release_branch = find_ref(repo, release_branch_name)
+    if release_branch:
+        if release_branch.is_remote():
+            # If the release branch only exists on the remote we check it out
+            # locally.
+            repo.git.checkout(release_branch_name)
+            release_branch = repo.active_branch
+    else:
+        # If a branch doesn't exist we create one. We ask which one branch it
+        # should be based off, defaulting to sensible values depending on the
+        # release type.
+        if current_version.is_prerelease:
+            default = release_branch_name
+        elif release_type == "major":
+            default = "develop"
+        else:
+            default = "master"
+
+        branch_name = click.prompt(
+            "Which branch should the release be based on?", default=default
+        )
+
+        base_branch = find_ref(repo, branch_name)
+        if not base_branch:
+            print(f"Could not find base branch {branch_name}!")
+            click.get_current_context().abort()
+
+        # Check out the base branch and ensure it's up to date
+        repo.head.reference = base_branch
+        repo.head.reset(index=True, working_tree=True)
+        if not base_branch.is_remote():
+            update_branch(repo)
+
+        # Create the new release branch
+        release_branch = repo.create_head(release_branch_name, commit=base_branch)
+
+    # Switch to the release branch and ensure its up to date.
+    repo.git.checkout(release_branch_name)
+    update_branch(repo)
+
+    # Update the `__version__` variable and write it back to the file.
+    version_node.value = '"' + new_version + '"'
+    with open("synapse/__init__.py", "w") as f:
+        f.write(red.dumps())
+
+    # Generate changelogs
+    subprocess.run("python3 -m towncrier", shell=True)
+
+    # Generate debian changelogs if its not an RC.
+    if not rc:
+        subprocess.run(
+            f'dch -M -v {new_version} "New synapse release {new_version}."', shell=True
+        )
+        subprocess.run('dch -M -r -D stable ""', shell=True)
+
+    # Show the user the changes and ask if they want to edit the change log.
+    repo.git.add("-u")
+    subprocess.run("git diff --cached", shell=True)
+
+    if click.confirm("Edit changelog?", default=False):
+        click.edit(filename="CHANGES.md")
+
+    # Commit the changes.
+    repo.git.add("-u")
+    repo.git.commit(f"-m {new_version}")
+
+    # We give the option to bail here in case the user wants to make sure things
+    # are OK before pushing.
+    if not click.confirm("Push branch to github?", default=True):
+        print("")
+        print("Run when ready to push:")
+        print("")
+        print(f"\tgit push -u {repo.remote().name} {repo.active_branch.name}")
+        print("")
+        sys.exit(0)
+
+    # Otherwise, push and open the changelog in the browser.
+    repo.git.push("-u", repo.remote().name, repo.active_branch.name)
+
+    click.launch(
+        f"https://github.com/matrix-org/synapse/blob/{repo.active_branch.name}/CHANGES.md"
+    )
+
+
+def find_ref(repo: git.Repo, ref_name: str) -> Optional[git.HEAD]:
+    """Find the branch/ref, looking first locally then in the remote."""
+    if ref_name in repo.refs:
+        return repo.refs[ref_name]
+    elif ref_name in repo.remote().refs:
+        return repo.remote().refs[ref_name]
+    else:
+        return None
+
+
+def update_branch(repo: git.Repo):
+    """Ensure branch is up to date if it has a remote"""
+    if repo.active_branch.tracking_branch():
+        repo.git.merge(repo.active_branch.tracking_branch().name)
+
+
+if __name__ == "__main__":
+    run()

scripts-dev/sign_json

@@ -1,6 +1,5 @@
 #!/usr/bin/env python
 #
-# -*- coding: utf-8 -*-
 # Copyright 2020 The Matrix.org Foundation C.I.C.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");

scripts-dev/update_database

@@ -1,5 +1,4 @@
 #!/usr/bin/env python
-# -*- coding: utf-8 -*-
 # Copyright 2019 The Matrix.org Foundation C.I.C.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");

scripts/export_signing_key

@@ -1,5 +1,4 @@
 #!/usr/bin/env python
-# -*- coding: utf-8 -*-
 # Copyright 2019 The Matrix.org Foundation C.I.C.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");

scripts/generate_log_config

@@ -1,6 +1,5 @@
 #!/usr/bin/env python3
 
-# -*- coding: utf-8 -*-
 # Copyright 2020 The Matrix.org Foundation C.I.C.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");

scripts/generate_signing_key.py

@@ -1,5 +1,4 @@
 #!/usr/bin/env python
-# -*- coding: utf-8 -*-
 # Copyright 2019 The Matrix.org Foundation C.I.C.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");

scripts/move_remote_media_to_new_store.py

@@ -1,5 +1,4 @@
 #!/usr/bin/env python
-# -*- coding: utf-8 -*-
 # Copyright 2017 New Vector Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");

scripts/register_new_matrix_user

@@ -1,5 +1,4 @@
 #!/usr/bin/env python
-# -*- coding: utf-8 -*-
 # Copyright 2015, 2016 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");

scripts/synapse_port_db

@@ -1,5 +1,4 @@
 #!/usr/bin/env python
-# -*- coding: utf-8 -*-
 # Copyright 2015, 2016 OpenMarket Ltd
 # Copyright 2018 New Vector Ltd
 # Copyright 2019 The Matrix.org Foundation C.I.C.

setup.cfg

@@ -18,16 +18,15 @@ ignore =
 #  E203: whitespace before ':' (which is contrary to pep8?)
 #  E731: do not assign a lambda expression, use a def
 #  E501: Line too long (black enforces this for us)
-#  B00*: Subsection of the bugbear suite (TODO: add in remaining fixes)
-ignore=W503,W504,E203,E731,E501,B006,B007,B008
+#  B007: Subsection of the bugbear suite (TODO: add in remaining fixes)
+ignore=W503,W504,E203,E731,E501,B007
 
 [isort]
 line_length = 88
-sections=FUTURE,STDLIB,COMPAT,THIRDPARTY,TWISTED,FIRSTPARTY,TESTS,LOCALFOLDER
+sections=FUTURE,STDLIB,THIRDPARTY,TWISTED,FIRSTPARTY,TESTS,LOCALFOLDER
 default_section=THIRDPARTY
 known_first_party = synapse
 known_tests=tests
-known_compat = mock
 known_twisted=twisted,OpenSSL
 multi_line_output=3
 include_trailing_comma=true

setup.py

@@ -99,10 +99,17 @@ CONDITIONAL_REQUIREMENTS["lint"] = [
     "isort==5.7.0",
     "black==20.8b1",
     "flake8-comprehensions",
-    "flake8-bugbear",
+    "flake8-bugbear==21.3.2",
     "flake8",
 ]
 
+CONDITIONAL_REQUIREMENTS["dev"] = CONDITIONAL_REQUIREMENTS["lint"] + [
+    # The following are used by the release script
+    "click==7.1.2",
+    "redbaron==0.9.2",
+    "GitPython==3.1.14",
+]
+
 CONDITIONAL_REQUIREMENTS["mypy"] = ["mypy==0.812", "mypy-zope==0.2.13"]
 
 # Dependencies which are exclusively required by unit test code. This is
@@ -110,7 +117,7 @@ CONDITIONAL_REQUIREMENTS["mypy"] = ["mypy==0.812", "mypy-zope==0.2.13"]
 # Tests assume that all optional dependencies are installed.
 #
 # parameterized_class decorator was introduced in parameterized 0.7.0
-CONDITIONAL_REQUIREMENTS["test"] = ["mock>=2.0", "parameterized>=0.7.0"]
+CONDITIONAL_REQUIREMENTS["test"] = ["parameterized>=0.7.0"]
 
 setup(
     name="matrix-synapse",
@@ -123,13 +130,12 @@ setup(
     zip_safe=False,
     long_description=long_description,
     long_description_content_type="text/x-rst",
-    python_requires="~=3.5",
+    python_requires="~=3.6",
     classifiers=[
         "Development Status :: 5 - Production/Stable",
         "Topic :: Communications :: Chat",
         "License :: OSI Approved :: Apache Software License",
         "Programming Language :: Python :: 3 :: Only",
-        "Programming Language :: Python :: 3.5",
         "Programming Language :: Python :: 3.6",
         "Programming Language :: Python :: 3.7",
         "Programming Language :: Python :: 3.8",

stubs/frozendict.pyi

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2020 The Matrix.org Foundation C.I.C.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");

stubs/txredisapi.pyi

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2020 The Matrix.org Foundation C.I.C.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");

synapse/__init__.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2014-2016 OpenMarket Ltd
 # Copyright 2018-9 New Vector Ltd
 #
@@ -48,7 +47,7 @@ try:
 except ImportError:
     pass
 
-__version__ = "1.31.0rc1"
+__version__ = "1.32.0rc1"
 
 if bool(os.environ.get("SYNAPSE_TEST_PATCH_LOG_CONTEXTS", False)):
     # We import here so that we don't have to install a bunch of deps when

synapse/_scripts/register_new_matrix_user.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2015, 2016 OpenMarket Ltd
 # Copyright 2018 New Vector
 #

synapse/api/__init__.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2014-2016 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");

synapse/api/auth.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2014 - 2016 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");

synapse/api/auth_blocking.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2020 The Matrix.org Foundation C.I.C.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");

synapse/api/constants.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2014-2016 OpenMarket Ltd
 # Copyright 2017 Vector Creations Ltd
 # Copyright 2018-2019 New Vector Ltd
@@ -73,6 +72,11 @@ class LoginType:
     DUMMY = "m.login.dummy"
 
 
+# This is used in the `type` parameter for /register when called by
+# an appservice to register a new user.
+APP_SERVICE_REGISTRATION_TYPE = "m.login.application_service"
+
+
 class EventTypes:
     Member = "m.room.member"
     Create = "m.room.create"

synapse/api/errors.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2014-2016 OpenMarket Ltd
 # Copyright 2018 New Vector Ltd
 #

synapse/api/filtering.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2015, 2016 OpenMarket Ltd
 # Copyright 2017 Vector Creations Ltd
 # Copyright 2018-2019 New Vector Ltd

synapse/api/presence.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2014-2016 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");

synapse/api/room_versions.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2019 New Vector Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");

synapse/api/urls.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2014-2016 OpenMarket Ltd
 # Copyright 2018 New Vector Ltd
 #

synapse/app/__init__.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2014-2016 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");

synapse/app/_base.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2017 New Vector Ltd
 # Copyright 2019-2021 The Matrix.org Foundation C.I.C
 #

synapse/app/admin_cmd.py

@@ -1,5 +1,4 @@
 #!/usr/bin/env python
-# -*- coding: utf-8 -*-
 # Copyright 2019 Matrix.org Foundation C.I.C.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");

synapse/app/appservice.py

@@ -1,5 +1,4 @@
 #!/usr/bin/env python
-# -*- coding: utf-8 -*-
 # Copyright 2016 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");

synapse/app/client_reader.py

@@ -1,5 +1,4 @@
 #!/usr/bin/env python
-# -*- coding: utf-8 -*-
 # Copyright 2016 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");

synapse/app/event_creator.py

@@ -1,5 +1,4 @@
 #!/usr/bin/env python
-# -*- coding: utf-8 -*-
 # Copyright 2018 New Vector Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");

synapse/app/federation_reader.py

@@ -1,5 +1,4 @@
 #!/usr/bin/env python
-# -*- coding: utf-8 -*-
 # Copyright 2016 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");

synapse/app/federation_sender.py

@@ -1,5 +1,4 @@
 #!/usr/bin/env python
-# -*- coding: utf-8 -*-
 # Copyright 2016 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");

synapse/app/frontend_proxy.py

@@ -1,5 +1,4 @@
 #!/usr/bin/env python
-# -*- coding: utf-8 -*-
 # Copyright 2016 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");

synapse/app/generic_worker.py

@@ -1,5 +1,4 @@
 #!/usr/bin/env python
-# -*- coding: utf-8 -*-
 # Copyright 2016 OpenMarket Ltd
 # Copyright 2020 The Matrix.org Foundation C.I.C.
 #
@@ -14,12 +13,9 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-import contextlib
 import logging
 import sys
-from typing import Dict, Iterable, Optional, Set
-
-from typing_extensions import ContextManager
+from typing import Dict, Iterable, Optional
 
 from twisted.internet import address
 from twisted.web.resource import IResource
@@ -41,24 +37,13 @@ from synapse.config._base import ConfigError
 from synapse.config.homeserver import HomeServerConfig
 from synapse.config.logger import setup_logging
 from synapse.config.server import ListenerConfig
-from synapse.federation import send_queue
 from synapse.federation.transport.server import TransportLayerServer
-from synapse.handlers.presence import (
-    BasePresenceHandler,
-    PresenceState,
-    get_interested_parties,
-)
 from synapse.http.server import JsonResource, OptionsResource
 from synapse.http.servlet import RestServlet, parse_json_object_from_request
 from synapse.http.site import SynapseSite
 from synapse.logging.context import LoggingContext
 from synapse.metrics import METRICS_PREFIX, MetricsResource, RegistryProxy
-from synapse.metrics.background_process_metrics import run_as_background_process
 from synapse.replication.http import REPLICATION_PREFIX, ReplicationRestResource
-from synapse.replication.http.presence import (
-    ReplicationBumpPresenceActiveTime,
-    ReplicationPresenceSetState,
-)
 from synapse.replication.slave.storage._base import BaseSlavedStore
 from synapse.replication.slave.storage.account_data import SlavedAccountDataStore
 from synapse.replication.slave.storage.appservice import SlavedApplicationServiceStore
@@ -78,19 +63,6 @@ from synapse.replication.slave.storage.receipts import SlavedReceiptsStore
 from synapse.replication.slave.storage.registration import SlavedRegistrationStore
 from synapse.replication.slave.storage.room import RoomStore
 from synapse.replication.slave.storage.transactions import SlavedTransactionStore
-from synapse.replication.tcp.client import ReplicationDataHandler
-from synapse.replication.tcp.commands import ClearUserSyncsCommand
-from synapse.replication.tcp.streams import (
-    AccountDataStream,
-    DeviceListsStream,
-    GroupServerStream,
-    PresenceStream,
-    PushersStream,
-    PushRulesStream,
-    ReceiptsStream,
-    TagAccountDataStream,
-    ToDeviceStream,
-)
 from synapse.rest.admin import register_servlets_for_media_repo
 from synapse.rest.client.v1 import events, login, room
 from synapse.rest.client.v1.initial_sync import InitialSyncRestServlet
@@ -129,7 +101,7 @@ from synapse.rest.client.versions import VersionsRestServlet
 from synapse.rest.health import HealthResource
 from synapse.rest.key.v2 import KeyApiV2Resource
 from synapse.rest.synapse.client import build_synapse_client_resource_tree
-from synapse.server import HomeServer, cache_in_self
+from synapse.server import HomeServer
 from synapse.storage.databases.main.censor_events import CensorEventsStore
 from synapse.storage.databases.main.client_ips import ClientIpWorkerStore
 from synapse.storage.databases.main.e2e_room_keys import EndToEndRoomKeyStore
@@ -138,14 +110,11 @@ from synapse.storage.databases.main.metrics import ServerMetricsStore
 from synapse.storage.databases.main.monthly_active_users import (
     MonthlyActiveUsersWorkerStore,
 )
-from synapse.storage.databases.main.presence import UserPresenceState
 from synapse.storage.databases.main.search import SearchWorkerStore
 from synapse.storage.databases.main.stats import StatsStore
 from synapse.storage.databases.main.transactions import TransactionWorkerStore
 from synapse.storage.databases.main.ui_auth import UIAuthWorkerStore
 from synapse.storage.databases.main.user_directory import UserDirectoryStore
-from synapse.types import ReadReceipt
-from synapse.util.async_helpers import Linearizer
 from synapse.util.httpresourcetree import create_resource_tree
 from synapse.util.versionstring import get_version_string
 
@@ -265,213 +234,6 @@ class KeyUploadServlet(RestServlet):
             return 200, {"one_time_key_counts": result}
 
 
-class _NullContextManager(ContextManager[None]):
-    """A context manager which does nothing."""
-
-    def __exit__(self, exc_type, exc_val, exc_tb):
-        pass
-
-
-UPDATE_SYNCING_USERS_MS = 10 * 1000
-
-
-class GenericWorkerPresence(BasePresenceHandler):
-    def __init__(self, hs):
-        super().__init__(hs)
-        self.hs = hs
-        self.is_mine_id = hs.is_mine_id
-
-        self._presence_enabled = hs.config.use_presence
-
-        # The number of ongoing syncs on this process, by user id.
-        # Empty if _presence_enabled is false.
-        self._user_to_num_current_syncs = {}  # type: Dict[str, int]
-
-        self.notifier = hs.get_notifier()
-        self.instance_id = hs.get_instance_id()
-
-        # user_id -> last_sync_ms. Lists the users that have stopped syncing
-        # but we haven't notified the master of that yet
-        self.users_going_offline = {}
-
-        self._bump_active_client = ReplicationBumpPresenceActiveTime.make_client(hs)
-        self._set_state_client = ReplicationPresenceSetState.make_client(hs)
-
-        self._send_stop_syncing_loop = self.clock.looping_call(
-            self.send_stop_syncing, UPDATE_SYNCING_USERS_MS
-        )
-
-        self._busy_presence_enabled = hs.config.experimental.msc3026_enabled
-
-        hs.get_reactor().addSystemEventTrigger(
-            "before",
-            "shutdown",
-            run_as_background_process,
-            "generic_presence.on_shutdown",
-            self._on_shutdown,
-        )
-
-    def _on_shutdown(self):
-        if self._presence_enabled:
-            self.hs.get_tcp_replication().send_command(
-                ClearUserSyncsCommand(self.instance_id)
-            )
-
-    def send_user_sync(self, user_id, is_syncing, last_sync_ms):
-        if self._presence_enabled:
-            self.hs.get_tcp_replication().send_user_sync(
-                self.instance_id, user_id, is_syncing, last_sync_ms
-            )
-
-    def mark_as_coming_online(self, user_id):
-        """A user has started syncing. Send a UserSync to the master, unless they
-        had recently stopped syncing.
-
-        Args:
-            user_id (str)
-        """
-        going_offline = self.users_going_offline.pop(user_id, None)
-        if not going_offline:
-            # Safe to skip because we haven't yet told the master they were offline
-            self.send_user_sync(user_id, True, self.clock.time_msec())
-
-    def mark_as_going_offline(self, user_id):
-        """A user has stopped syncing. We wait before notifying the master as
-        its likely they'll come back soon. This allows us to avoid sending
-        a stopped syncing immediately followed by a started syncing notification
-        to the master
-
-        Args:
-            user_id (str)
-        """
-        self.users_going_offline[user_id] = self.clock.time_msec()
-
-    def send_stop_syncing(self):
-        """Check if there are any users who have stopped syncing a while ago
-        and haven't come back yet. If there are poke the master about them.
-        """
-        now = self.clock.time_msec()
-        for user_id, last_sync_ms in list(self.users_going_offline.items()):
-            if now - last_sync_ms > UPDATE_SYNCING_USERS_MS:
-                self.users_going_offline.pop(user_id, None)
-                self.send_user_sync(user_id, False, last_sync_ms)
-
-    async def user_syncing(
-        self, user_id: str, affect_presence: bool
-    ) -> ContextManager[None]:
-        """Record that a user is syncing.
-
-        Called by the sync and events servlets to record that a user has connected to
-        this worker and is waiting for some events.
-        """
-        if not affect_presence or not self._presence_enabled:
-            return _NullContextManager()
-
-        curr_sync = self._user_to_num_current_syncs.get(user_id, 0)
-        self._user_to_num_current_syncs[user_id] = curr_sync + 1
-
-        # If we went from no in flight sync to some, notify replication
-        if self._user_to_num_current_syncs[user_id] == 1:
-            self.mark_as_coming_online(user_id)
-
-        def _end():
-            # We check that the user_id is in user_to_num_current_syncs because
-            # user_to_num_current_syncs may have been cleared if we are
-            # shutting down.
-            if user_id in self._user_to_num_current_syncs:
-                self._user_to_num_current_syncs[user_id] -= 1
-
-                # If we went from one in flight sync to non, notify replication
-                if self._user_to_num_current_syncs[user_id] == 0:
-                    self.mark_as_going_offline(user_id)
-
-        @contextlib.contextmanager
-        def _user_syncing():
-            try:
-                yield
-            finally:
-                _end()
-
-        return _user_syncing()
-
-    async def notify_from_replication(self, states, stream_id):
-        parties = await get_interested_parties(self.store, states)
-        room_ids_to_states, users_to_states = parties
-
-        self.notifier.on_new_event(
-            "presence_key",
-            stream_id,
-            rooms=room_ids_to_states.keys(),
-            users=users_to_states.keys(),
-        )
-
-    async def process_replication_rows(self, token, rows):
-        states = [
-            UserPresenceState(
-                row.user_id,
-                row.state,
-                row.last_active_ts,
-                row.last_federation_update_ts,
-                row.last_user_sync_ts,
-                row.status_msg,
-                row.currently_active,
-            )
-            for row in rows
-        ]
-
-        for state in states:
-            self.user_to_current_state[state.user_id] = state
-
-        stream_id = token
-        await self.notify_from_replication(states, stream_id)
-
-    def get_currently_syncing_users_for_replication(self) -> Iterable[str]:
-        return [
-            user_id
-            for user_id, count in self._user_to_num_current_syncs.items()
-            if count > 0
-        ]
-
-    async def set_state(self, target_user, state, ignore_status_msg=False):
-        """Set the presence state of the user."""
-        presence = state["presence"]
-
-        valid_presence = (
-            PresenceState.ONLINE,
-            PresenceState.UNAVAILABLE,
-            PresenceState.OFFLINE,
-            PresenceState.BUSY,
-        )
-
-        if presence not in valid_presence or (
-            presence == PresenceState.BUSY and not self._busy_presence_enabled
-        ):
-            raise SynapseError(400, "Invalid presence state")
-
-        user_id = target_user.to_string()
-
-        # If presence is disabled, no-op
-        if not self.hs.config.use_presence:
-            return
-
-        # Proxy request to master
-        await self._set_state_client(
-            user_id=user_id, state=state, ignore_status_msg=ignore_status_msg
-        )
-
-    async def bump_presence_active_time(self, user):
-        """We've seen the user do something that indicates they're interacting
-        with the app.
-        """
-        # If presence is disabled, no-op
-        if not self.hs.config.use_presence:
-            return
-
-        # Proxy request to master
-        user_id = user.to_string()
-        await self._bump_active_client(user_id=user_id)
-
-
 class GenericWorkerSlavedStore(
     # FIXME(#3714): We need to add UserDirectoryStore as we write directly
     # rather than going via the correct worker.
@@ -643,7 +405,7 @@ class GenericWorkerServer(HomeServer):
                     listener.bind_addresses, listener.port, manhole_globals={"hs": self}
                 )
             elif listener.type == "metrics":
-                if not self.get_config().enable_metrics:
+                if not self.config.enable_metrics:
                     logger.warning(
                         (
                             "Metrics listener configured, but "
@@ -657,234 +419,6 @@ class GenericWorkerServer(HomeServer):
 
         self.get_tcp_replication().start_replication(self)
 
-    @cache_in_self
-    def get_replication_data_handler(self):
-        return GenericWorkerReplicationHandler(self)
-
-    @cache_in_self
-    def get_presence_handler(self):
-        return GenericWorkerPresence(self)
-
-
-class GenericWorkerReplicationHandler(ReplicationDataHandler):
-    def __init__(self, hs):
-        super().__init__(hs)
-
-        self.store = hs.get_datastore()
-        self.presence_handler = hs.get_presence_handler()  # type: GenericWorkerPresence
-        self.notifier = hs.get_notifier()
-
-        self.notify_pushers = hs.config.start_pushers
-        self.pusher_pool = hs.get_pusherpool()
-
-        self.send_handler = None  # type: Optional[FederationSenderHandler]
-        if hs.config.send_federation:
-            self.send_handler = FederationSenderHandler(hs)
-
-    async def on_rdata(self, stream_name, instance_name, token, rows):
-        await super().on_rdata(stream_name, instance_name, token, rows)
-        await self._process_and_notify(stream_name, instance_name, token, rows)
-
-    async def _process_and_notify(self, stream_name, instance_name, token, rows):
-        try:
-            if self.send_handler:
-                await self.send_handler.process_replication_rows(
-                    stream_name, token, rows
-                )
-
-            if stream_name == PushRulesStream.NAME:
-                self.notifier.on_new_event(
-                    "push_rules_key", token, users=[row.user_id for row in rows]
-                )
-            elif stream_name in (AccountDataStream.NAME, TagAccountDataStream.NAME):
-                self.notifier.on_new_event(
-                    "account_data_key", token, users=[row.user_id for row in rows]
-                )
-            elif stream_name == ReceiptsStream.NAME:
-                self.notifier.on_new_event(
-                    "receipt_key", token, rooms=[row.room_id for row in rows]
-                )
-                await self.pusher_pool.on_new_receipts(
-                    token, token, {row.room_id for row in rows}
-                )
-            elif stream_name == ToDeviceStream.NAME:
-                entities = [row.entity for row in rows if row.entity.startswith("@")]
-                if entities:
-                    self.notifier.on_new_event("to_device_key", token, users=entities)
-            elif stream_name == DeviceListsStream.NAME:
-                all_room_ids = set()  # type: Set[str]
-                for row in rows:
-                    if row.entity.startswith("@"):
-                        room_ids = await self.store.get_rooms_for_user(row.entity)
-                        all_room_ids.update(room_ids)
-                self.notifier.on_new_event("device_list_key", token, rooms=all_room_ids)
-            elif stream_name == PresenceStream.NAME:
-                await self.presence_handler.process_replication_rows(token, rows)
-            elif stream_name == GroupServerStream.NAME:
-                self.notifier.on_new_event(
-                    "groups_key", token, users=[row.user_id for row in rows]
-                )
-            elif stream_name == PushersStream.NAME:
-                for row in rows:
-                    if row.deleted:
-                        self.stop_pusher(row.user_id, row.app_id, row.pushkey)
-                    else:
-                        await self.start_pusher(row.user_id, row.app_id, row.pushkey)
-        except Exception:
-            logger.exception("Error processing replication")
-
-    async def on_position(self, stream_name: str, instance_name: str, token: int):
-        await super().on_position(stream_name, instance_name, token)
-        # Also call on_rdata to ensure that stream positions are properly reset.
-        await self.on_rdata(stream_name, instance_name, token, [])
-
-    def stop_pusher(self, user_id, app_id, pushkey):
-        if not self.notify_pushers:
-            return
-
-        key = "%s:%s" % (app_id, pushkey)
-        pushers_for_user = self.pusher_pool.pushers.get(user_id, {})
-        pusher = pushers_for_user.pop(key, None)
-        if pusher is None:
-            return
-        logger.info("Stopping pusher %r / %r", user_id, key)
-        pusher.on_stop()
-
-    async def start_pusher(self, user_id, app_id, pushkey):
-        if not self.notify_pushers:
-            return
-
-        key = "%s:%s" % (app_id, pushkey)
-        logger.info("Starting pusher %r / %r", user_id, key)
-        return await self.pusher_pool.start_pusher_by_id(app_id, pushkey, user_id)
-
-    def on_remote_server_up(self, server: str):
-        """Called when get a new REMOTE_SERVER_UP command."""
-
-        # Let's wake up the transaction queue for the server in case we have
-        # pending stuff to send to it.
-        if self.send_handler:
-            self.send_handler.wake_destination(server)
-
-
-class FederationSenderHandler:
-    """Processes the fedration replication stream
-
-    This class is only instantiate on the worker responsible for sending outbound
-    federation transactions. It receives rows from the replication stream and forwards
-    the appropriate entries to the FederationSender class.
-    """
-
-    def __init__(self, hs: GenericWorkerServer):
-        self.store = hs.get_datastore()
-        self._is_mine_id = hs.is_mine_id
-        self.federation_sender = hs.get_federation_sender()
-        self._hs = hs
-
-        # Stores the latest position in the federation stream we've gotten up
-        # to. This is always set before we use it.
-        self.federation_position = None
-
-        self._fed_position_linearizer = Linearizer(name="_fed_position_linearizer")
-
-    def wake_destination(self, server: str):
-        self.federation_sender.wake_destination(server)
-
-    async def process_replication_rows(self, stream_name, token, rows):
-        # The federation stream contains things that we want to send out, e.g.
-        # presence, typing, etc.
-        if stream_name == "federation":
-            send_queue.process_rows_for_federation(self.federation_sender, rows)
-            await self.update_token(token)
-
-        # ... and when new receipts happen
-        elif stream_name == ReceiptsStream.NAME:
-            await self._on_new_receipts(rows)
-
-        # ... as well as device updates and messages
-        elif stream_name == DeviceListsStream.NAME:
-            # The entities are either user IDs (starting with '@') whose devices
-            # have changed, or remote servers that we need to tell about
-            # changes.
-            hosts = {row.entity for row in rows if not row.entity.startswith("@")}
-            for host in hosts:
-                self.federation_sender.send_device_messages(host)
-
-        elif stream_name == ToDeviceStream.NAME:
-            # The to_device stream includes stuff to be pushed to both local
-            # clients and remote servers, so we ignore entities that start with
-            # '@' (since they'll be local users rather than destinations).
-            hosts = {row.entity for row in rows if not row.entity.startswith("@")}
-            for host in hosts:
-                self.federation_sender.send_device_messages(host)
-
-    async def _on_new_receipts(self, rows):
-        """
-        Args:
-            rows (Iterable[synapse.replication.tcp.streams.ReceiptsStream.ReceiptsStreamRow]):
-                new receipts to be processed
-        """
-        for receipt in rows:
-            # we only want to send on receipts for our own users
-            if not self._is_mine_id(receipt.user_id):
-                continue
-            receipt_info = ReadReceipt(
-                receipt.room_id,
-                receipt.receipt_type,
-                receipt.user_id,
-                [receipt.event_id],
-                receipt.data,
-            )
-            await self.federation_sender.send_read_receipt(receipt_info)
-
-    async def update_token(self, token):
-        """Update the record of where we have processed to in the federation stream.
-
-        Called after we have processed a an update received over replication. Sends
-        a FEDERATION_ACK back to the master, and stores the token that we have processed
-         in `federation_stream_position` so that we can restart where we left off.
-        """
-        self.federation_position = token
-
-        # We save and send the ACK to master asynchronously, so we don't block
-        # processing on persistence. We don't need to do this operation for
-        # every single RDATA we receive, we just need to do it periodically.
-
-        if self._fed_position_linearizer.is_queued(None):
-            # There is already a task queued up to save and send the token, so
-            # no need to queue up another task.
-            return
-
-        run_as_background_process("_save_and_send_ack", self._save_and_send_ack)
-
-    async def _save_and_send_ack(self):
-        """Save the current federation position in the database and send an ACK
-        to master with where we're up to.
-        """
-        try:
-            # We linearize here to ensure we don't have races updating the token
-            #
-            # XXX this appears to be redundant, since the ReplicationCommandHandler
-            # has a linearizer which ensures that we only process one line of
-            # replication data at a time. Should we remove it, or is it doing useful
-            # service for robustness? Or could we replace it with an assertion that
-            # we're not being re-entered?
-
-            with (await self._fed_position_linearizer.queue(None)):
-                # We persist and ack the same position, so we take a copy of it
-                # here as otherwise it can get modified from underneath us.
-                current_position = self.federation_position
-
-                await self.store.update_federation_out_pos(
-                    "federation", current_position
-                )
-
-                # We ACK this token over replication so that the master can drop
-                # its in memory queues
-                self._hs.get_tcp_replication().send_federation_ack(current_position)
-        except Exception:
-            logger.exception("Error updating federation stream position")
-
 
 def start(config_options):
     try:

synapse/app/homeserver.py

@@ -1,5 +1,4 @@
 #!/usr/bin/env python
-# -*- coding: utf-8 -*-
 # Copyright 2014-2016 OpenMarket Ltd
 # Copyright 2019 New Vector Ltd
 #
@@ -192,7 +191,7 @@ class SynapseHomeServer(HomeServer):
                 }
             )
 
-            if self.get_config().threepid_behaviour_email == ThreepidBehaviour.LOCAL:
+            if self.config.threepid_behaviour_email == ThreepidBehaviour.LOCAL:
                 from synapse.rest.synapse.client.password_reset import (
                     PasswordResetSubmitTokenResource,
                 )
@@ -231,7 +230,7 @@ class SynapseHomeServer(HomeServer):
             )
 
         if name in ["media", "federation", "client"]:
-            if self.get_config().enable_media_repo:
+            if self.config.enable_media_repo:
                 media_repo = self.get_media_repository_resource()
                 resources.update(
                     {MEDIA_PREFIX: media_repo, LEGACY_MEDIA_PREFIX: media_repo}
@@ -245,7 +244,7 @@ class SynapseHomeServer(HomeServer):
             resources[SERVER_KEY_V2_PREFIX] = KeyApiV2Resource(self)
 
         if name == "webclient":
-            webclient_loc = self.get_config().web_client_location
+            webclient_loc = self.config.web_client_location
 
             if webclient_loc is None:
                 logger.warning(
@@ -266,7 +265,7 @@ class SynapseHomeServer(HomeServer):
                 # https://twistedmatrix.com/trac/ticket/7678
                 resources[WEB_CLIENT_PREFIX] = File(webclient_loc)
 
-        if name == "metrics" and self.get_config().enable_metrics:
+        if name == "metrics" and self.config.enable_metrics:
             resources[METRICS_PREFIX] = MetricsResource(RegistryProxy)
 
         if name == "replication":
@@ -275,9 +274,7 @@ class SynapseHomeServer(HomeServer):
         return resources
 
     def start_listening(self, listeners: Iterable[ListenerConfig]):
-        config = self.get_config()
-
-        if config.redis_enabled:
+        if self.config.redis_enabled:
             # If redis is enabled we connect via the replication command handler
             # in the same way as the workers (since we're effectively a client
             # rather than a server).
@@ -285,7 +282,9 @@ class SynapseHomeServer(HomeServer):
 
         for listener in listeners:
             if listener.type == "http":
-                self._listening_services.extend(self._listener_http(config, listener))
+                self._listening_services.extend(
+                    self._listener_http(self.config, listener)
+                )
             elif listener.type == "manhole":
                 _base.listen_manhole(
                     listener.bind_addresses, listener.port, manhole_globals={"hs": self}
@@ -299,7 +298,7 @@ class SynapseHomeServer(HomeServer):
                 for s in services:
                     reactor.addSystemEventTrigger("before", "shutdown", s.stopListening)
             elif listener.type == "metrics":
-                if not self.get_config().enable_metrics:
+                if not self.config.enable_metrics:
                     logger.warning(
                         (
                             "Metrics listener configured, but "

synapse/app/media_repository.py

@@ -1,5 +1,4 @@
 #!/usr/bin/env python
-# -*- coding: utf-8 -*-
 # Copyright 2016 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");

synapse/app/pusher.py

@@ -1,5 +1,4 @@
 #!/usr/bin/env python
-# -*- coding: utf-8 -*-
 # Copyright 2016 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");

synapse/app/synchrotron.py

@@ -1,5 +1,4 @@
 #!/usr/bin/env python
-# -*- coding: utf-8 -*-
 # Copyright 2016 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");

synapse/app/user_dir.py

@@ -1,5 +1,4 @@
 #!/usr/bin/env python
-# -*- coding: utf-8 -*-
 # Copyright 2017 Vector Creations Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");

synapse/appservice/__init__.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2015, 2016 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");

synapse/appservice/api.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2015, 2016 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");

synapse/appservice/scheduler.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2015, 2016 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -49,7 +48,7 @@ This is all tied together by the AppServiceScheduler which DIs the required
 components.
 """
 import logging
-from typing import List
+from typing import List, Optional
 
 from synapse.appservice import ApplicationService, ApplicationServiceState
 from synapse.events import EventBase
@@ -191,11 +190,11 @@ class _TransactionController:
         self,
         service: ApplicationService,
         events: List[EventBase],
-        ephemeral: List[JsonDict] = [],
+        ephemeral: Optional[List[JsonDict]] = None,
     ):
         try:
             txn = await self.store.create_appservice_txn(
-                service=service, events=events, ephemeral=ephemeral
+                service=service, events=events, ephemeral=ephemeral or []
             )
             service_is_up = await self._is_service_up(service)
             if service_is_up:

synapse/config/__init__.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2014-2016 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");

synapse/config/__main__.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2015, 2016 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");

synapse/config/_base.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2014-2016 OpenMarket Ltd
 # Copyright 2017-2018 New Vector Ltd
 # Copyright 2019 The Matrix.org Foundation C.I.C.

synapse/config/_util.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2020 The Matrix.org Foundation C.I.C.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");

synapse/config/auth.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2015, 2016 OpenMarket Ltd
 # Copyright 2020 The Matrix.org Foundation C.I.C.
 #

synapse/config/cache.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2019 Matrix.org Foundation C.I.C.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");

synapse/config/cas.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # Copyright 2015, 2016 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");