update comment

When configuring the return values of mocks, prefer awaitables from
`make_awaitable` over `defer.succeed`. `Deferred`s are only awaitable
once, so it is inappropriate for a mock to return the same `Deferred`
multiple times.

Also update `run_in_background` to support functions that return
arbitrary awaitables.

Signed-off-by: Sean Quah <seanq@element.io>

.ci/latest_deps_build_failed_issue_template.md

@@ -0,0 +1,4 @@
+---
+title: CI run against latest deps is failing
+---
+See https://github.com/{{env.GITHUB_REPOSITORY}}/actions/runs/{{env.GITHUB_RUN_ID}}

.ci/patch_for_twisted_trunk.sh

@@ -1,8 +0,0 @@
-#!/bin/sh
-
-# replaces the dependency on Twisted in `python_dependencies` with trunk.
-
-set -e
-cd "$(dirname "$0")"/..
-
-sed -i -e 's#"Twisted.*"#"Twisted @ git+https://github.com/twisted/twisted"#' synapse/python_dependencies.py

.ci/scripts/test_export_data_command.sh

@@ -2,29 +2,24 @@
 
 # Test for the export-data admin command against sqlite and postgres
 
+# Expects Synapse to have been already installed with `poetry install --extras postgres`.
+# Expects `poetry` to be available on the `PATH`.
+
 set -xe
 cd "$(dirname "$0")/../.."
 
-echo "--- Install dependencies"
-
-# Install dependencies for this test.
-pip install psycopg2
-
-# Install Synapse itself. This won't update any libraries.
-pip install -e .
-
 echo "--- Generate the signing key"
 
 # Generate the server's signing key.
-python -m synapse.app.homeserver --generate-keys -c .ci/sqlite-config.yaml
+poetry run synapse_homeserver --generate-keys -c .ci/sqlite-config.yaml
 
 echo "--- Prepare test database"
 
 # Make sure the SQLite3 database is using the latest schema and has no pending background update.
-update_synapse_database --database-config .ci/sqlite-config.yaml --run-background-updates
+poetry run update_synapse_database --database-config .ci/sqlite-config.yaml --run-background-updates
 
 # Run the export-data command on the sqlite test database
-python -m synapse.app.admin_cmd -c .ci/sqlite-config.yaml  export-data @anon-20191002_181700-832:localhost:8800 \
+poetry run python -m synapse.app.admin_cmd -c .ci/sqlite-config.yaml  export-data @anon-20191002_181700-832:localhost:8800 \
 --output-directory /tmp/export_data
 
 # Test that the output directory exists and contains the rooms directory
@@ -37,14 +32,14 @@ else
 fi
 
 # Create the PostgreSQL database.
-.ci/scripts/postgres_exec.py "CREATE DATABASE synapse"
+poetry run .ci/scripts/postgres_exec.py "CREATE DATABASE synapse"
 
 # Port the SQLite databse to postgres so we can check command works against postgres
 echo "+++ Port SQLite3 databse to postgres"
-synapse_port_db --sqlite-database .ci/test_db.db --postgres-config .ci/postgres-config.yaml
+poetry run synapse_port_db --sqlite-database .ci/test_db.db --postgres-config .ci/postgres-config.yaml
 
 # Run the export-data command on postgres database
-python -m synapse.app.admin_cmd -c .ci/postgres-config.yaml  export-data @anon-20191002_181700-832:localhost:8800 \
+poetry run python -m synapse.app.admin_cmd -c .ci/postgres-config.yaml  export-data @anon-20191002_181700-832:localhost:8800 \
 --output-directory /tmp/export_data2
 
 # Test that the output directory exists and contains the rooms directory

.ci/scripts/test_old_deps.sh

@@ -1,6 +1,9 @@
 #!/usr/bin/env bash
-# this script is run by GitHub Actions in a plain `focal` container; it installs the
-# minimal requirements for tox and hands over to the py3-old tox environment.
+# this script is run by GitHub Actions in a plain `focal` container; it
+# - installs the minimal system requirements, and poetry;
+# - patches the project definition file to refer to old versions only;
+# - creates a venv with these old versions using poetry; and finally
+# - invokes `trial` to run the tests with old deps.
 
 # Prevent tzdata from asking for user input
 export DEBIAN_FRONTEND=noninteractive
@@ -9,12 +12,70 @@ set -ex
 
 apt-get update
 apt-get install -y \
-        python3 python3-dev python3-pip python3-venv \
-        libxml2-dev libxslt-dev xmlsec1 zlib1g-dev tox libjpeg-dev libwebp-dev
+        python3 python3-dev python3-pip python3-venv pipx \
+        libxml2-dev libxslt-dev xmlsec1 zlib1g-dev libjpeg-dev libwebp-dev
 
 export LANG="C.UTF-8"
 
 # Prevent virtualenv from auto-updating pip to an incompatible version
 export VIRTUALENV_NO_DOWNLOAD=1
 
-exec tox -e py3-old
+# TODO: in the future, we could use an implementation of
+#   https://github.com/python-poetry/poetry/issues/3527
+#   https://github.com/pypa/pip/issues/8085
+# to select the lowest possible versions, rather than resorting to this sed script.
+
+# Patch the project definitions in-place:
+# - Replace all lower and tilde bounds with exact bounds
+# - Make the pyopenssl 17.0, which is the oldest version that works with
+#   a `cryptography` compiled against OpenSSL 1.1.
+# - Delete all lines referring to psycopg2 --- so no testing of postgres support.
+# - Omit systemd: we're not logging to journal here.
+
+# TODO: also replace caret bounds, see https://python-poetry.org/docs/dependency-specification/#version-constraints
+# We don't use these yet, but IIRC they are the default bound used when you `poetry add`.
+# The sed expression 's/\^/==/g' ought to do the trick. But it would also change
+# `python = "^3.7"` to `python = "==3.7", which would mean we fail because olddeps
+# runs on 3.8 (#12343).
+
+sed -i \
+   -e "s/[~>]=/==/g" \
+   -e "/psycopg2/d" \
+   -e 's/pyOpenSSL = "==16.0.0"/pyOpenSSL = "==17.0.0"/' \
+   -e '/systemd/d' \
+   pyproject.toml
+
+# Use poetry to do the installation. This ensures that the versions are all mutually
+# compatible (as far the package metadata declares, anyway); pip's package resolver
+# is more lax.
+#
+# Rather than `poetry install --no-dev`, we drop all dev dependencies from the
+# toml file. This means we don't have to ensure compatibility between old deps and
+# dev tools.
+
+pip install --user toml
+
+REMOVE_DEV_DEPENDENCIES="
+import toml
+with open('pyproject.toml', 'r') as f:
+    data = toml.loads(f.read())
+
+del data['tool']['poetry']['dev-dependencies']
+
+with open('pyproject.toml', 'w') as f:
+    toml.dump(data, f)
+"
+python3 -c "$REMOVE_DEV_DEPENDENCIES"
+
+pipx install poetry==1.1.12
+~/.local/bin/poetry lock
+
+echo "::group::Patched pyproject.toml"
+cat pyproject.toml
+echo "::endgroup::"
+echo "::group::Lockfile after patch"
+cat poetry.lock
+echo "::endgroup::"
+
+~/.local/bin/poetry install -E "all test"
+~/.local/bin/poetry run trial --jobs=2 tests

.ci/scripts/test_synapse_port_db.sh

@@ -1,43 +1,37 @@
 #!/usr/bin/env bash
 #
 # Test script for 'synapse_port_db'.
-#   - sets up synapse and deps
+#   - configures synapse and a postgres server.
 #   - runs the port script on a prepopulated test sqlite db
 #   - also runs it against an new sqlite db
-
+#
+# Expects Synapse to have been already installed with `poetry install --extras postgres`.
+# Expects `poetry` to be available on the `PATH`.
 
 set -xe
 cd "$(dirname "$0")/../.."
 
-echo "--- Install dependencies"
-
-# Install dependencies for this test.
-pip install psycopg2 coverage coverage-enable-subprocess
-
-# Install Synapse itself. This won't update any libraries.
-pip install -e .
-
 echo "--- Generate the signing key"
 
 # Generate the server's signing key.
-python -m synapse.app.homeserver --generate-keys -c .ci/sqlite-config.yaml
+poetry run synapse_homeserver --generate-keys -c .ci/sqlite-config.yaml
 
 echo "--- Prepare test database"
 
 # Make sure the SQLite3 database is using the latest schema and has no pending background update.
-update_synapse_database --database-config .ci/sqlite-config.yaml --run-background-updates
+poetry run update_synapse_database --database-config .ci/sqlite-config.yaml --run-background-updates
 
 # Create the PostgreSQL database.
-.ci/scripts/postgres_exec.py "CREATE DATABASE synapse"
+poetry run .ci/scripts/postgres_exec.py "CREATE DATABASE synapse"
 
 echo "+++ Run synapse_port_db against test database"
 # TODO: this invocation of synapse_port_db (and others below) used to be prepended with `coverage run`,
 # but coverage seems unable to find the entrypoints installed by `pip install -e .`.
-synapse_port_db --sqlite-database .ci/test_db.db --postgres-config .ci/postgres-config.yaml
+poetry run synapse_port_db --sqlite-database .ci/test_db.db --postgres-config .ci/postgres-config.yaml
 
 # We should be able to run twice against the same database.
 echo "+++ Run synapse_port_db a second time"
-synapse_port_db --sqlite-database .ci/test_db.db --postgres-config .ci/postgres-config.yaml
+poetry run synapse_port_db --sqlite-database .ci/test_db.db --postgres-config .ci/postgres-config.yaml
 
 #####
 
@@ -48,12 +42,12 @@ echo "--- Prepare empty SQLite database"
 # we do this by deleting the sqlite db, and then doing the same again.
 rm .ci/test_db.db
 
-update_synapse_database --database-config .ci/sqlite-config.yaml --run-background-updates
+poetry run update_synapse_database --database-config .ci/sqlite-config.yaml --run-background-updates
 
 # re-create the PostgreSQL database.
-.ci/scripts/postgres_exec.py \
+poetry run .ci/scripts/postgres_exec.py \
   "DROP DATABASE synapse" \
   "CREATE DATABASE synapse"
 
 echo "+++ Run synapse_port_db against empty database"
-synapse_port_db --sqlite-database .ci/test_db.db --postgres-config .ci/postgres-config.yaml
+poetry run synapse_port_db --sqlite-database .ci/test_db.db --postgres-config .ci/postgres-config.yaml

.dockerignore

@@ -4,8 +4,8 @@
 # things to include
 !docker
 !synapse
-!MANIFEST.in
 !README.rst
-!setup.py
+!pyproject.toml
+!poetry.lock
 
 **/__pycache__

.github/workflows/docs.yaml

@@ -22,7 +22,7 @@ jobs:
       - name: Setup mdbook
         uses: peaceiris/actions-mdbook@4b5ef36b314c2599664ca107bb8c02412548d79d # v1.1.14
         with:
-          mdbook-version: '0.4.9'
+          mdbook-version: '0.4.17'
 
       - name: Build the documentation
         # mdbook will only create an index.html if we're including docs/README.md in SUMMARY.md.

.github/workflows/latest_deps.yml

@@ -0,0 +1,156 @@
+# People who are freshly `pip install`ing from PyPI will pull in the latest versions of
+# dependencies which match the broad requirements. Since most CI runs are against
+# the locked poetry environment, run specifically against the latest dependencies to
+# know if there's an upcoming breaking change.
+#
+# As an overview this workflow:
+# - checks out develop,
+# - installs from source, pulling in the dependencies like a fresh `pip install` would, and 
+# - runs mypy and test suites in that checkout.
+#
+# Based on the twisted trunk CI job.
+
+name: Latest dependencies
+
+on:
+  schedule:
+    - cron: 0 7 * * *
+  workflow_dispatch:
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  mypy:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      # The dev dependencies aren't exposed in the wheel metadata (at least with current
+      # poetry-core versions), so we install with poetry.
+      - uses: matrix-org/setup-python-poetry@v1
+        with:
+          python-version: "3.x"
+          poetry-version: "1.2.0b1"
+      # Dump installed versions for debugging.
+      - run: poetry run pip list > before.txt
+      # Upgrade all runtime dependencies only. This is intended to mimic a fresh
+      # `pip install matrix-synapse[all]` as closely as possible.
+      - run: poetry update --no-dev
+      - run: poetry run pip list > after.txt && (diff -u before.txt after.txt || true)
+      - run: poetry run mypy
+  trial:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        include:
+          - database: "sqlite"
+          - database: "postgres"
+            postgres-version: "14"
+
+    steps:
+      - uses: actions/checkout@v2
+      - run: sudo apt-get -qq install xmlsec1
+      - name: Set up PostgreSQL ${{ matrix.postgres-version }}
+        if: ${{ matrix.postgres-version }}
+        run: |
+          docker run -d -p 5432:5432 \
+            -e POSTGRES_PASSWORD=postgres \
+            -e POSTGRES_INITDB_ARGS="--lc-collate C --lc-ctype C --encoding UTF8" \
+            postgres:${{ matrix.postgres-version }}
+      - uses: actions/setup-python@v2
+        with:
+          python-version: "3.x"
+      - run: pip install .[all,test]
+      - name: Await PostgreSQL
+        if: ${{ matrix.postgres-version }}
+        timeout-minutes: 2
+        run: until pg_isready -h localhost; do sleep 1; done
+      - run: python -m twisted.trial --jobs=2 tests
+        env:
+          SYNAPSE_POSTGRES: ${{ matrix.database == 'postgres' || '' }}
+          SYNAPSE_POSTGRES_HOST: localhost
+          SYNAPSE_POSTGRES_USER: postgres
+          SYNAPSE_POSTGRES_PASSWORD: postgres
+      - name: Dump logs
+        # Logs are most useful when the command fails, always include them.
+        if: ${{ always() }}
+        # Note: Dumps to workflow logs instead of using actions/upload-artifact
+        #       This keeps logs colocated with failing jobs
+        #       It also ignores find's exit code; this is a best effort affair
+        run: >-
+          find _trial_temp -name '*.log'
+          -exec echo "::group::{}" \;
+          -exec cat {} \;
+          -exec echo "::endgroup::" \;
+          || true
+
+
+  sytest:
+    runs-on: ubuntu-latest
+    container:
+      image: matrixdotorg/sytest-synapse:testing
+      volumes:
+        - ${{ github.workspace }}:/src
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - sytest-tag: focal
+
+          - sytest-tag: focal
+            postgres: postgres
+            workers: workers
+            redis: redis
+    env:
+      POSTGRES: ${{ matrix.postgres && 1}}
+      WORKERS: ${{ matrix.workers && 1 }}
+      REDIS: ${{ matrix.redis && 1 }}
+      BLACKLIST: ${{ matrix.workers && 'synapse-blacklist-with-workers' }}
+
+    steps:
+      - uses: actions/checkout@v2
+      - name: Ensure sytest runs `pip install`
+        # Delete the lockfile so sytest will `pip install` rather than `poetry install`
+        run: rm /src/poetry.lock
+        working-directory: /src
+      - name: Prepare test blacklist
+        run: cat sytest-blacklist .ci/worker-blacklist > synapse-blacklist-with-workers
+      - name: Run SyTest
+        run: /bootstrap.sh synapse
+        working-directory: /src
+      - name: Summarise results.tap
+        if: ${{ always() }}
+        run: /sytest/scripts/tap_to_gha.pl /logs/results.tap
+      - name: Upload SyTest logs
+        uses: actions/upload-artifact@v2
+        if: ${{ always() }}
+        with:
+          name: Sytest Logs - ${{ job.status }} - (${{ join(matrix.*, ', ') }})
+          path: |
+            /logs/results.tap
+            /logs/**/*.log*
+
+
+  # TODO: run complement (as with twisted trunk, see #12473).
+
+  # open an issue if the build fails, so we know about it.
+  open-issue:
+    if: failure()
+    needs:
+      # TODO: should mypy be included here? It feels more brittle than the other two.
+      - mypy
+      - trial
+      - sytest
+
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v2
+      - uses: JasonEtco/create-an-issue@5d9504915f79f9cc6d791934b8ef34f2353dd74d # v2.5.0, 2020-12-06
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        with:
+          update_existing: true
+          filename: .ci/latest_deps_build_failed_issue_template.md
+

.github/workflows/tests.yml

@@ -15,25 +15,14 @@ jobs:
     steps:
       - uses: actions/checkout@v2
       - uses: actions/setup-python@v2
-      - run: pip install -e .
+      - run: pip install .
       - run: scripts-dev/generate_sample_config.sh --check
       - run: scripts-dev/config-lint.sh
 
   lint:
-    runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        toxenv:
-          - "check_codestyle"
-          - "check_isort"
-          - "mypy"
-          - "packaging"
-
-    steps:
-      - uses: actions/checkout@v2
-      - uses: actions/setup-python@v2
-      - run: pip install tox
-      - run: tox -e ${{ matrix.toxenv }}
+    uses: "matrix-org/backend-meta/.github/workflows/python-poetry-ci.yml@v1"
+    with:
+      typechecking-extras: "all"
 
   lint-crlf:
     runs-on: ubuntu-latest
@@ -72,23 +61,23 @@ jobs:
       matrix:
         python-version: ["3.7", "3.8", "3.9", "3.10"]
         database: ["sqlite"]
-        toxenv: ["py"]
+        extras: ["all"]
         include:
           # Newest Python without optional deps
           - python-version: "3.10"
-            toxenv: "py-noextras"
+            extras: ""
 
           # Oldest Python with PostgreSQL
           - python-version: "3.7"
             database: "postgres"
             postgres-version: "10"
-            toxenv: "py"
+            extras: "all"
 
           # Newest Python with newest PostgreSQL
           - python-version: "3.10"
             database: "postgres"
             postgres-version: "14"
-            toxenv: "py"
+            extras: "all"
 
     steps:
       - uses: actions/checkout@v2
@@ -100,17 +89,16 @@ jobs:
             -e POSTGRES_PASSWORD=postgres \
             -e POSTGRES_INITDB_ARGS="--lc-collate C --lc-ctype C --encoding UTF8" \
             postgres:${{ matrix.postgres-version }}
-      - uses: actions/setup-python@v2
+      - uses: matrix-org/setup-python-poetry@v1
         with:
           python-version: ${{ matrix.python-version }}
-      - run: pip install tox
+          extras: ${{ matrix.extras }}
       - name: Await PostgreSQL
         if: ${{ matrix.postgres-version }}
         timeout-minutes: 2
         run: until pg_isready -h localhost; do sleep 1; done
-      - run: tox -e ${{ matrix.toxenv }}
+      - run: poetry run trial --jobs=2 tests
         env:
-          TRIAL_FLAGS: "--jobs=2"
           SYNAPSE_POSTGRES: ${{ matrix.database == 'postgres' || '' }}
           SYNAPSE_POSTGRES_HOST: localhost
           SYNAPSE_POSTGRES_USER: postgres
@@ -129,6 +117,7 @@ jobs:
           || true
 
   trial-olddeps:
+    # Note: sqlite only; no postgres
     if: ${{ !cancelled() && !failure() }} # Allow previous steps to be skipped, but not fail
     needs: linting-done
     runs-on: ubuntu-latest
@@ -136,11 +125,11 @@ jobs:
       - uses: actions/checkout@v2
       - name: Test with old deps
         uses: docker://ubuntu:focal # For old python and sqlite
+        # Note: focal seems to be using 3.8, but the oldest is 3.7?
+        # See https://github.com/matrix-org/synapse/issues/12343
         with:
           workdir: /github/workspace
           entrypoint: .ci/scripts/test_old_deps.sh
-        env:
-          TRIAL_FLAGS: "--jobs=2"
       - name: Dump logs
         # Logs are most useful when the command fails, always include them.
         if: ${{ always() }}
@@ -156,23 +145,24 @@ jobs:
 
   trial-pypy:
     # Very slow; only run if the branch name includes 'pypy'
+    # Note: sqlite only; no postgres. Completely untested since poetry move.
     if: ${{ contains(github.ref, 'pypy') && !failure() && !cancelled() }}
     needs: linting-done
     runs-on: ubuntu-latest
     strategy:
       matrix:
         python-version: ["pypy-3.7"]
+        extras: ["all"]
 
     steps:
       - uses: actions/checkout@v2
+      # Install libs necessary for PyPy to build binary wheels for dependencies
       - run: sudo apt-get -qq install xmlsec1 libxml2-dev libxslt-dev
-      - uses: actions/setup-python@v2
+      - uses: matrix-org/setup-python-poetry@v1
         with:
           python-version: ${{ matrix.python-version }}
-      - run: pip install tox
-      - run: tox -e py
-        env:
-          TRIAL_FLAGS: "--jobs=2"
+          extras: ${{ matrix.extras }}
+      - run: poetry run trial --jobs=2 tests
       - name: Dump logs
         # Logs are most useful when the command fails, always include them.
         if: ${{ always() }}
@@ -271,9 +261,10 @@ jobs:
     steps:
       - uses: actions/checkout@v2
       - run: sudo apt-get -qq install xmlsec1
-      - uses: actions/setup-python@v2
+      - uses: matrix-org/setup-python-poetry@v1
         with:
-          python-version: "3.9"
+          python-version: ${{ matrix.python-version }}
+          extras: "postgres"
       - run: .ci/scripts/test_export_data_command.sh
 
   portdb:
@@ -308,9 +299,10 @@ jobs:
     steps:
       - uses: actions/checkout@v2
       - run: sudo apt-get -qq install xmlsec1
-      - uses: actions/setup-python@v2
+      - uses: matrix-org/setup-python-poetry@v1
         with:
           python-version: ${{ matrix.python-version }}
+          extras: "postgres"
       - run: .ci/scripts/test_synapse_port_db.sh
 
   complement:
@@ -362,27 +354,11 @@ jobs:
             (wget -O - "https://github.com/matrix-org/complement/archive/$BRANCH_NAME.tar.gz" | tar -xz --strip-components=1 -C complement) && break
           done
 
-      # Build initial Synapse image
-      - run: docker build -t matrixdotorg/synapse:latest -f docker/Dockerfile .
-        working-directory: synapse
-        env:
-          DOCKER_BUILDKIT: 1
-
-      # Build a ready-to-run Synapse image based on the initial image above.
-      # This new image includes a config file, keys for signing and TLS, and
-      # other settings to make it suitable for testing under Complement.
-      - run: docker build -t complement-synapse -f Synapse.Dockerfile .
-        working-directory: complement/dockerfiles
-
-      # Run Complement
       - run: |
           set -o pipefail
-          go test -v -json -p 1 -tags synapse_blacklist,msc2403,msc2716,msc3030 ./tests/... 2>&1 | gotestfmt
+          COMPLEMENT_DIR=`pwd`/complement synapse/scripts-dev/complement.sh -json 2>&1 | gotestfmt
         shell: bash
         name: Run Complement Tests
-        env:
-          COMPLEMENT_BASE_IMAGE: complement-synapse:latest
-        working-directory: complement
 
   # a job which marks all the other jobs as complete, thus allowing PRs to be merged.
   tests-done:

.github/workflows/twisted_trunk.yml

@@ -6,16 +6,25 @@ on:
 
   workflow_dispatch:
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
 jobs:
   mypy:
     runs-on: ubuntu-latest
 
     steps:
       - uses: actions/checkout@v2
-      - uses: actions/setup-python@v2
-      - run: .ci/patch_for_twisted_trunk.sh
-      - run: pip install tox
-      - run: tox -e mypy
+      - uses: matrix-org/setup-python-poetry@v1
+        with:
+          python-version: "3.x"
+          extras: "all"
+      - run: |
+          poetry remove twisted
+          poetry add --extras tls git+https://github.com/twisted/twisted.git#trunk
+          poetry install --no-interaction --extras "all test"
+      - run: poetry run mypy
 
   trial:
     runs-on: ubuntu-latest
@@ -23,14 +32,15 @@ jobs:
     steps:
       - uses: actions/checkout@v2
       - run: sudo apt-get -qq install xmlsec1
-      - uses: actions/setup-python@v2
+      - uses: matrix-org/setup-python-poetry@v1
         with:
-          python-version: 3.7
-      - run: .ci/patch_for_twisted_trunk.sh
-      - run: pip install tox
-      - run: tox -e py
-        env:
-          TRIAL_FLAGS: "--jobs=2"
+          python-version: "3.x"
+          extras: "all test"
+      - run: |
+          poetry remove twisted
+          poetry add --extras tls git+https://github.com/twisted/twisted.git#trunk
+          poetry install --no-interaction --extras "all test"
+      - run: poetry run trial --jobs 2 tests
 
       - name: Dump logs
         # Logs are most useful when the command fails, always include them.
@@ -55,11 +65,23 @@ jobs:
     steps:
       - uses: actions/checkout@v2
       - name: Patch dependencies
-        run: .ci/patch_for_twisted_trunk.sh
+        # Note: The poetry commands want to create a virtualenv in /src/.venv/,
+        #       but the sytest-synapse container expects it to be in /venv/.
+        #       We symlink it before running poetry so that poetry actually
+        #       ends up installing to `/venv`.
+        run: |
+          ln -s -T /venv /src/.venv
+          poetry remove twisted
+          poetry add --extras tls git+https://github.com/twisted/twisted.git#trunk
+          poetry install --no-interaction --extras "all test"
         working-directory: /src
       - name: Run SyTest
         run: /bootstrap.sh synapse
         working-directory: /src
+        env:
+          # Use offline mode to avoid reinstalling the pinned version of
+          # twisted.
+          OFFLINE: 1
       - name: Summarise results.tap
         if: ${{ always() }}
         run: /sytest/scripts/tap_to_gha.pl /logs/results.tap

.gitignore

@@ -15,6 +15,9 @@ _trial_temp*/
 .DS_Store
 __pycache__/
 
+# We do want the poetry lockfile.
+!poetry.lock
+
 # stuff that is likely to exist when you run a server locally
 /*.db
 /*.log
@@ -30,6 +33,9 @@ __pycache__/
 /media_store/
 /uploads
 
+# For direnv users
+/.envrc
+
 # IDEs
 /.idea/
 /.ropeproject/

MANIFEST.in

@@ -1,54 +0,0 @@
-include LICENSE
-include VERSION
-include *.rst
-include *.md
-include demo/README
-include demo/demo.tls.dh
-include demo/*.py
-include demo/*.sh
-
-include synapse/py.typed
-recursive-include synapse/storage *.sql
-recursive-include synapse/storage *.sql.postgres
-recursive-include synapse/storage *.sql.sqlite
-recursive-include synapse/storage *.py
-recursive-include synapse/storage *.txt
-recursive-include synapse/storage *.md
-
-recursive-include docs *
-recursive-include scripts-dev *
-recursive-include synapse *.pyi
-recursive-include tests *.py
-recursive-include tests *.pem
-recursive-include tests *.p8
-recursive-include tests *.crt
-recursive-include tests *.key
-
-recursive-include synapse/res *
-recursive-include synapse/static *.css
-recursive-include synapse/static *.gif
-recursive-include synapse/static *.html
-recursive-include synapse/static *.js
-
-exclude .codecov.yml
-exclude .coveragerc
-exclude .dockerignore
-exclude .editorconfig
-exclude Dockerfile
-exclude mypy.ini
-exclude sytest-blacklist
-exclude test_postgresql.sh
-
-include book.toml
-include pyproject.toml
-recursive-include changelog.d *
-
-include .flake8
-prune .circleci
-prune .github
-prune .ci
-prune contrib
-prune debian
-prune demo/etc
-prune docker
-prune stubs

README.rst

@@ -293,39 +293,42 @@ directory of your choice::
     git clone https://github.com/matrix-org/synapse.git
     cd synapse
 
-Synapse has a number of external dependencies, that are easiest
-to install using pip and a virtualenv::
+Synapse has a number of external dependencies. We maintain a fixed development
+environment using [poetry](https://python-poetry.org/). First, install poetry. We recommend
 
-    python3 -m venv ./env
-    source ./env/bin/activate
-    pip install -e ".[all,dev]"
+    pip install --user pipx
+    pipx install poetry
+
+as described `here <https://python-poetry.org/docs/#installing-with-pipx>`_.
+(See `poetry's installation docs <https://python-poetry.org/docs/#installation>`
+for other installation methods.) Then ask poetry to create a virtual environment
+from the project and install Synapse's dependencies::
+
+    poetry install --extras "all test"
 
 This will run a process of downloading and installing all the needed
-dependencies into a virtual env. If any dependencies fail to install,
-try installing the failing modules individually::
-
-    pip install -e "module-name"
+dependencies into a virtual env.
 
 We recommend using the demo which starts 3 federated instances running on ports `8080` - `8082`
 
-    ./demo/start.sh
+    poetry run ./demo/start.sh
 
-(to stop, you can use `./demo/stop.sh`)
+(to stop, you can use `poetry run ./demo/stop.sh`)
 
-See the [demo documentation](https://matrix-org.github.io/synapse/develop/development/demo.html)
+See the `demo documentation <https://matrix-org.github.io/synapse/develop/development/demo.html>`_
 for more information.
 
 If you just want to start a single instance of the app and run it directly::
 
     # Create the homeserver.yaml config once
-    python -m synapse.app.homeserver \
+    poetry run synapse_homeserver \
       --server-name my.domain.name \
       --config-path homeserver.yaml \
       --generate-config \
       --report-stats=[yes|no]
 
     # Start the app
-    python -m synapse.app.homeserver --config-path homeserver.yaml
+    poetry run synapse_homeserver --config-path homeserver.yaml
 
 
 Running the unit tests
@@ -334,7 +337,7 @@ Running the unit tests
 After getting up and running, you may wish to run Synapse's unit tests to
 check that everything is installed correctly::
 
-    trial tests
+    poetry run trial tests
 
 This should end with a 'PASSED' result (note that exact numbers will
 differ)::

book.toml

@@ -33,7 +33,7 @@ site-url = "/synapse/"
 additional-css = [
     "docs/website_files/table-of-contents.css",
     "docs/website_files/remove-nav-buttons.css",
-    "docs/website_files/section-headers.css",
+    "docs/website_files/indent-section-headers.css",
 ]
 additional-js = ["docs/website_files/table-of-contents.js"]
 theme = "docs/website_files/theme"

changelog.d/11700.removal

@@ -1 +0,0 @@
-Remove workaround introduced in Synapse 1.50.0 for Mjolnir compatibility. Breaks compatibility with Mjolnir 1.3.1 and earlier.

changelog.d/11915.misc

@@ -1 +0,0 @@
-Simplify the `ApplicationService` class' set of public methods related to interest checking.

changelog.d/11998.doc

@@ -1 +0,0 @@
-Fix complexity checking config example in [Resource Constrained Devices](https://matrix-org.github.io/synapse/v1.54/other/running_synapse_on_single_board_computers.html) docs page.

changelog.d/12028.feature

@@ -1 +0,0 @@
-Add third-party rules rules callbacks `check_can_shutdown_room` and `check_can_deactivate_user`.

changelog.d/12042.misc

@@ -1 +0,0 @@
-Correct type hints for txredis.

changelog.d/12090.bugfix

@@ -1 +0,0 @@
-Use the proper serialization format for bundled thread aggregations. The bug has existed since Synapse v1.48.0.

changelog.d/12101.misc

@@ -1 +0,0 @@
-Limit the size of `aggregation_key` on annotations.

changelog.d/12108.misc

@@ -1 +0,0 @@
-Add type hints to `tests/rest/client`.

changelog.d/12113.bugfix

@@ -1 +0,0 @@
-Fix a long-standing bug when redacting events with relations.

changelog.d/12118.misc

@@ -1 +0,0 @@
-Move scripts to Synapse package and expose as setuptools entry points.

changelog.d/12121.bugfix

@@ -1 +0,0 @@
-Fix a long-standing bug when redacting events with relations.

changelog.d/12128.misc

@@ -1 +0,0 @@
-Fix data validation to compare to lists, not sequences.

changelog.d/12130.bugfix

@@ -1 +0,0 @@
-Fix a long-standing bug when redacting events with relations.

changelog.d/12131.misc

@@ -1 +0,0 @@
-Fix CI not attaching source distributions and wheels to the GitHub releases.

changelog.d/12132.feature

@@ -1 +0,0 @@
-Improve performance of logging in for large accounts.

changelog.d/12135.feature

@@ -1 +0,0 @@
-Add experimental env var `SYNAPSE_ASYNC_IO_REACTOR` that causes Synapse to use the asyncio reactor for Twisted.

changelog.d/12136.misc

@@ -1 +0,0 @@
-Remove unused mocks from `test_typing`.

changelog.d/12137.misc

@@ -1 +0,0 @@
-Give `scripts-dev` scripts suffixes for neater CI config.

changelog.d/12138.removal

@@ -1 +0,0 @@
-Remove backwards compatibilty with pagination tokens from the `/relations` and `/aggregations` endpoints generated from Synapse < v1.52.0.

changelog.d/12140.misc

@@ -1 +0,0 @@
-Move `synctl` into `synapse._scripts` and expose as an entry point.

changelog.d/12142.misc

@@ -1 +0,0 @@
-Move the snapcraft configuration file to `contrib`.

changelog.d/12143.doc

@@ -1 +0,0 @@
-Improve documentation for demo scripts.

changelog.d/12144.misc

@@ -1 +0,0 @@
-Enable [MSC3030](https://github.com/matrix-org/matrix-doc/pull/3030) Complement tests in CI.

changelog.d/12145.misc

@@ -1 +0,0 @@
-Enable [MSC2716](https://github.com/matrix-org/matrix-doc/pull/2716) Complement tests in CI.

changelog.d/12146.misc

@@ -1 +0,0 @@
-Add type hints to `tests/rest`.

changelog.d/12149.misc

@@ -1 +0,0 @@
-Add test for `ObservableDeferred`'s cancellation behaviour.

changelog.d/12150.misc

@@ -1 +0,0 @@
-Use `ParamSpec` in type hints for `synapse.logging.context`.

changelog.d/12151.feature

@@ -1 +0,0 @@
-Support the stable identifiers from [MSC3440](https://github.com/matrix-org/matrix-doc/pull/3440): threads.

changelog.d/12152.misc

@@ -1 +0,0 @@
-Prune unused jobs from `tox` config.

changelog.d/12153.misc

@@ -1 +0,0 @@
-Move CI checks out of tox, to facilitate a move to using poetry.

changelog.d/12154.misc

@@ -1 +0,0 @@
-Avoid generating state groups for local out-of-band leaves.

changelog.d/12155.misc

@@ -1 +0,0 @@
-Avoid trying to calculate the state at outlier events.

changelog.d/12156.misc

@@ -1 +0,0 @@
-Fix some type annotations.

changelog.d/12157.bugfix

@@ -1 +0,0 @@
-Fix a bug introduced in #4864 whereby background updates are never run with the default background batch size.

changelog.d/12159.misc

@@ -1 +0,0 @@
-Add type hints for `ObservableDeferred` attributes.

changelog.d/12161.misc

@@ -1 +0,0 @@
-Use a prebuilt Action for the `tests-done` CI job.

changelog.d/12163.misc

@@ -1 +0,0 @@
-Reduce number of DB queries made during processing of `/sync`.

changelog.d/12173.misc

@@ -1 +0,0 @@
-Avoid trying to calculate the state at outlier events.

changelog.d/12175.bugfix

@@ -1 +0,0 @@
-Fix a bug where non-standard information was returned from the `/hierarchy` API. Introduced in Synapse v1.41.0.

changelog.d/12179.doc

@@ -1 +0,0 @@
-Updates to the Room DAG concepts development document.

changelog.d/12182.misc

@@ -1 +0,0 @@
-Retry HTTP replication failures, this should prevent 502's when restarting stateful workers (main, event persisters, stream writers). Contributed by Nick @ Beeper.

changelog.d/12187.misc

@@ -1 +0,0 @@
-Remove unused variables.

changelog.d/12189.bugfix

@@ -1 +0,0 @@
-Fix a long-standing bug when redacting events with relations.

changelog.d/12192.misc

@@ -1 +0,0 @@
-Rename `HomeServer.get_tcp_replication` to `get_replication_command_handler`.

changelog.d/12197.misc

@@ -1 +0,0 @@
-Remove some dead code.

changelog.d/12356.misc

@@ -0,0 +1 @@
+Fix scripts-dev to pass typechecking.

changelog.d/12406.feature

@@ -0,0 +1 @@
+Add a module API to allow modules to change actions for existing push rules of local users.

changelog.d/12480.misc

@@ -0,0 +1 @@
+Use supervisord to supervise Postgres and Caddy in the Complement image to reduce restart time.

changelog.d/12485.misc

@@ -0,0 +1 @@
+Add some type hints to datastore.

changelog.d/12505.misc

@@ -0,0 +1 @@
+Use `make_awaitable` instead of `defer.succeed` for return values of mocks in tests.

changelog.d/12526.feature

@@ -0,0 +1 @@
+Add new `enable_registration_token_3pid_bypass` configuration option to allow registrations via token as an alternative to verifying a 3pid.

changelog.d/12531.misc

@@ -0,0 +1 @@
+Remove unused `# type: ignore`s.

changelog.d/12541.docker

@@ -0,0 +1 @@
+Explicitly opt-in to using [BuildKit-specific features](https://github.com/moby/buildkit/blob/master/frontend/dockerfile/docs/syntax.md) in the Dockerfile. This fixes issues with building images in some GitLab CI environments.

changelog.d/12544.bugfix

@@ -0,0 +1 @@
+Fix a bug where attempting to send a large amount of read receipts to an application service all at once would result in duplicate content and abnormally high memory usage. Contributed by Brad & Nick @ Beeper.

changelog.d/12564.misc

@@ -0,0 +1 @@
+Consistently check if an object is a `frozendict`.

changelog.d/12572.feature

@@ -0,0 +1 @@
+Add the Synapse function `types.map_username_to_mxid_localpart` to the Module API.

contrib/jitsimeetbridge/jitsimeetbridge.py

@@ -193,12 +193,15 @@ class TrivialXmppClient:
         time.sleep(7)
         print("SSRC spammer started")
         while self.running:
-            ssrcMsg = "<presence to='%(tojid)s' xmlns='jabber:client'><x xmlns='http://jabber.org/protocol/muc'/><c xmlns='http://jabber.org/protocol/caps' hash='sha-1' node='http://jitsi.org/jitsimeet' ver='0WkSdhFnAUxrz4ImQQLdB80GFlE='/><nick xmlns='http://jabber.org/protocol/nick'>%(nick)s</nick><stats xmlns='http://jitsi.org/jitmeet/stats'><stat name='bitrate_download' value='175'/><stat name='bitrate_upload' value='176'/><stat name='packetLoss_total' value='0'/><stat name='packetLoss_download' value='0'/><stat name='packetLoss_upload' value='0'/></stats><media xmlns='http://estos.de/ns/mjs'><source type='audio' ssrc='%(assrc)s' direction='sendre'/><source type='video' ssrc='%(vssrc)s' direction='sendre'/></media></presence>" % {
-                "tojid": "%s@%s/%s" % (ROOMNAME, ROOMDOMAIN, self.shortJid),
-                "nick": self.userId,
-                "assrc": self.ssrcs["audio"],
-                "vssrc": self.ssrcs["video"],
-            }
+            ssrcMsg = (
+                "<presence to='%(tojid)s' xmlns='jabber:client'><x xmlns='http://jabber.org/protocol/muc'/><c xmlns='http://jabber.org/protocol/caps' hash='sha-1' node='http://jitsi.org/jitsimeet' ver='0WkSdhFnAUxrz4ImQQLdB80GFlE='/><nick xmlns='http://jabber.org/protocol/nick'>%(nick)s</nick><stats xmlns='http://jitsi.org/jitmeet/stats'><stat name='bitrate_download' value='175'/><stat name='bitrate_upload' value='176'/><stat name='packetLoss_total' value='0'/><stat name='packetLoss_download' value='0'/><stat name='packetLoss_upload' value='0'/></stats><media xmlns='http://estos.de/ns/mjs'><source type='audio' ssrc='%(assrc)s' direction='sendre'/><source type='video' ssrc='%(vssrc)s' direction='sendre'/></media></presence>"
+                % {
+                    "tojid": "%s@%s/%s" % (ROOMNAME, ROOMDOMAIN, self.shortJid),
+                    "nick": self.userId,
+                    "assrc": self.ssrcs["audio"],
+                    "vssrc": self.ssrcs["video"],
+                }
+            )
             res = self.sendIq(ssrcMsg)
             print("reply from ssrc announce: ", res)
             time.sleep(10)

debian/build_virtualenv

@@ -30,9 +30,19 @@ case $(dpkg-architecture -q DEB_HOST_ARCH) in
         ;;
 esac
 
-# Use --builtin-venv to use the better `venv` module from CPython 3.4+ rather
-# than the 2/3 compatible `virtualenv`.
+# Manually install Poetry and export a pip-compatible `requirements.txt`
+# We need a Poetry pre-release as the export command is buggy in < 1.2
+TEMP_VENV="$(mktemp -d)"
+python3 -m venv "$TEMP_VENV"
+source "$TEMP_VENV/bin/activate"
+pip install -U pip
+pip install poetry==1.2.0b1
+poetry export --extras all --extras test -o exported_requirements.txt
+deactivate
+rm -rf "$TEMP_VENV"
 
+# Use --no-deps to only install pinned versions in exported_requirements.txt,
+# and to avoid https://github.com/pypa/pip/issues/9644
 dh_virtualenv \
     --install-suffix "matrix-synapse" \
     --builtin-venv \
@@ -41,9 +51,11 @@ dh_virtualenv \
     --preinstall="lxml" \
     --preinstall="mock" \
     --preinstall="wheel" \
+    --extra-pip-arg="--no-deps" \
     --extra-pip-arg="--no-cache-dir" \
     --extra-pip-arg="--compile" \
-    --extras="all,systemd,test"
+    --extras="all,systemd,test" \
+    --requirements="exported_requirements.txt"
 
 PACKAGE_BUILD_DIR="debian/matrix-synapse-py3"
 VIRTUALENV_DIR="${PACKAGE_BUILD_DIR}${DH_VIRTUALENV_INSTALL_ROOT}/matrix-synapse"

debian/changelog

@@ -1,3 +1,70 @@
+matrix-synapse-py3 (1.58.0~rc2) stable; urgency=medium
+
+  * New Synapse release 1.58.0rc2.
+
+ -- Synapse Packaging team <packages@matrix.org>  Tue, 26 Apr 2022 17:14:56 +0100
+
+matrix-synapse-py3 (1.58.0~rc1) stable; urgency=medium
+
+  * Use poetry to manage the bundled virtualenv included with this package.
+  * New Synapse release 1.58.0rc1.
+
+ -- Synapse Packaging team <packages@matrix.org>  Tue, 26 Apr 2022 11:15:20 +0100
+
+matrix-synapse-py3 (1.57.1) stable; urgency=medium
+
+  * New synapse release 1.57.1.
+
+ -- Synapse Packaging team <packages@matrix.org>  Wed, 20 Apr 2022 15:27:21 +0100
+
+matrix-synapse-py3 (1.57.0) stable; urgency=medium
+
+  * New synapse release 1.57.0.
+
+ -- Synapse Packaging team <packages@matrix.org>  Tue, 19 Apr 2022 10:58:42 +0100
+
+matrix-synapse-py3 (1.57.0~rc1) stable; urgency=medium
+
+  * New synapse release 1.57.0~rc1.
+
+ -- Synapse Packaging team <packages@matrix.org>  Tue, 12 Apr 2022 13:36:25 +0100
+
+matrix-synapse-py3 (1.56.0) stable; urgency=medium
+
+  * New synapse release 1.56.0.
+
+ -- Synapse Packaging team <packages@matrix.org>  Tue, 05 Apr 2022 12:38:39 +0100
+
+matrix-synapse-py3 (1.56.0~rc1) stable; urgency=medium
+
+  * New synapse release 1.56.0~rc1.
+
+ -- Synapse Packaging team <packages@matrix.org>  Tue, 29 Mar 2022 10:40:50 +0100
+
+matrix-synapse-py3 (1.55.2) stable; urgency=medium
+
+  * New synapse release 1.55.2.
+
+ -- Synapse Packaging team <packages@matrix.org>  Thu, 24 Mar 2022 19:07:11 +0000
+
+matrix-synapse-py3 (1.55.1) stable; urgency=medium
+
+  * New synapse release 1.55.1.
+
+ -- Synapse Packaging team <packages@matrix.org>  Thu, 24 Mar 2022 17:44:23 +0000
+
+matrix-synapse-py3 (1.55.0) stable; urgency=medium
+
+  * New synapse release 1.55.0.
+
+ -- Synapse Packaging team <packages@matrix.org>  Tue, 22 Mar 2022 13:59:26 +0000
+
+matrix-synapse-py3 (1.55.0~rc1) stable; urgency=medium
+
+  * New synapse release 1.55.0~rc1.
+
+ -- Synapse Packaging team <packages@matrix.org>  Tue, 15 Mar 2022 10:59:31 +0000
+
 matrix-synapse-py3 (1.54.0) stable; urgency=medium
 
   * New synapse release 1.54.0.

debian/clean

@@ -0,0 +1 @@
+exported_requirements.txt

demo/start.sh

@@ -38,6 +38,7 @@ for port in 8080 8081 8082; do
             printf '\n\n# Customisation made by demo/start.sh\n\n'
             echo "public_baseurl: http://localhost:$port/"
             echo 'enable_registration: true'
+            echo 'enable_registration_without_verification: true'
             echo ''
 
 			# Warning, this heredoc depends on the interaction of tabs and spaces.

docker/Dockerfile

@@ -1,3 +1,4 @@
+# syntax=docker/dockerfile:1
 # Dockerfile to build the matrixdotorg/synapse docker images.
 #
 # Note that it uses features which are only available in BuildKit - see
@@ -14,20 +15,61 @@
 #    DOCKER_BUILDKIT=1 docker build -f docker/Dockerfile --build-arg PYTHON_VERSION=3.10 .
 #
 
+# Irritatingly, there is no blessed guide on how to distribute an application with its
+# poetry-managed environment in a docker image. We have opted for
+# `poetry export | pip install -r /dev/stdin`, but there are known bugs in
+# in `poetry export` whose fixes (scheduled for poetry 1.2) have yet to be released.
+# In case we get bitten by those bugs in the future, the recommendations here might
+# be useful:
+#     https://github.com/python-poetry/poetry/discussions/1879#discussioncomment-216865
+#     https://stackoverflow.com/questions/53835198/integrating-python-poetry-with-docker?answertab=scoredesc
+
+
+
 ARG PYTHON_VERSION=3.9
 
 ###
-### Stage 0: builder
+### Stage 0: generate requirements.txt
+###
+FROM docker.io/python:${PYTHON_VERSION}-slim as requirements
+
+# RUN --mount is specific to buildkit and is documented at
+# https://github.com/moby/buildkit/blob/master/frontend/dockerfile/docs/syntax.md#build-mounts-run---mount.
+# Here we use it to set up a cache for apt (and below for pip), to improve
+# rebuild speeds on slow connections.
+RUN \
+   --mount=type=cache,target=/var/cache/apt,sharing=locked \
+   --mount=type=cache,target=/var/lib/apt,sharing=locked \
+ apt-get update && apt-get install -y git \
+    && rm -rf /var/lib/apt/lists/*
+
+# We install poetry in its own build stage to avoid its dependencies conflicting with
+# synapse's dependencies.
+# We use a specific commit from poetry's master branch instead of our usual 1.1.12,
+# to incorporate fixes to some bugs in `poetry export`. This commit corresponds to
+#    https://github.com/python-poetry/poetry/pull/5156 and
+#    https://github.com/python-poetry/poetry/issues/5141 ;
+# without it, we generate a requirements.txt with incorrect environment markers,
+# which causes necessary packages to be omitted when we `pip install`.
+#
+# NB: In poetry 1.2 `poetry export` will be moved into a plugin; we'll need to also
+# pip install poetry-plugin-export (https://github.com/python-poetry/poetry-plugin-export).
+RUN --mount=type=cache,target=/root/.cache/pip \
+  pip install --user git+https://github.com/python-poetry/poetry.git@fb13b3a676f476177f7937ffa480ee5cff9a90a5
+
+WORKDIR /synapse
+
+# Copy just what we need to run `poetry export`...
+COPY pyproject.toml poetry.lock /synapse/
+
+RUN /root/.local/bin/poetry export --extras all -o /synapse/requirements.txt
+
+###
+### Stage 1: builder
 ###
 FROM docker.io/python:${PYTHON_VERSION}-slim as builder
 
 # install the OS build deps
-#
-# RUN --mount is specific to buildkit and is documented at
-# https://github.com/moby/buildkit/blob/master/frontend/dockerfile/docs/syntax.md#build-mounts-run---mount.
-# Here we use it to set up a cache for apt, to improve rebuild speeds on
-# slow connections.
-#
 RUN \
    --mount=type=cache,target=/var/cache/apt,sharing=locked \
    --mount=type=cache,target=/var/lib/apt,sharing=locked \
@@ -45,30 +87,25 @@ RUN \
     zlib1g-dev \
     && rm -rf /var/lib/apt/lists/*
 
-# Copy just what we need to pip install
-COPY MANIFEST.in README.rst setup.py /synapse/
-COPY synapse/__init__.py /synapse/synapse/__init__.py
-COPY synapse/python_dependencies.py /synapse/synapse/python_dependencies.py
-
 # To speed up rebuilds, install all of the dependencies before we copy over
-# the whole synapse project so that we this layer in the Docker cache can be
+# the whole synapse project, so that this layer in the Docker cache can be
 # used while you develop on the source
 #
-# This is aiming at installing the `install_requires` and `extras_require` from `setup.py`
+# This is aiming at installing the `[tool.poetry.depdendencies]` from pyproject.toml.
+COPY --from=requirements /synapse/requirements.txt /synapse/
 RUN --mount=type=cache,target=/root/.cache/pip \
-  pip install --prefix="/install" --no-warn-script-location \
-    /synapse[all]
+  pip install --prefix="/install" --no-deps --no-warn-script-location -r /synapse/requirements.txt
 
-# Copy over the rest of the project
+# Copy over the rest of the synapse source code.
 COPY synapse /synapse/synapse/
+# ... and what we need to `pip install`.
+COPY pyproject.toml README.rst /synapse/
 
-# Install the synapse package itself and all of its children packages.
-#
-# This is aiming at installing only the `packages=find_packages(...)` from `setup.py
+# Install the synapse package itself.
 RUN pip install --prefix="/install" --no-deps --no-warn-script-location /synapse
 
 ###
-### Stage 1: runtime
+### Stage 2: runtime
 ###
 
 FROM docker.io/python:${PYTHON_VERSION}-slim

docker/Dockerfile-pgtests

@@ -1,30 +0,0 @@
-# Use the Sytest image that comes with a lot of the build dependencies
-# pre-installed
-FROM matrixdotorg/sytest:focal
-
-# The Sytest image doesn't come with python, so install that
-RUN apt-get update && apt-get -qq install -y python3 python3-dev python3-pip
-
-# We need tox to run the tests in run_pg_tests.sh
-RUN python3 -m pip install tox
-
-# Initialise the db
-RUN su -c '/usr/lib/postgresql/10/bin/initdb -D /var/lib/postgresql/data -E "UTF-8" --lc-collate="C.UTF-8" --lc-ctype="C.UTF-8" --username=postgres' postgres
-
-# Add a user with our UID and GID so that files get created on the host owned
-# by us, not root.
-ARG UID
-ARG GID
-RUN groupadd --gid $GID user
-RUN useradd --uid $UID --gid $GID --groups sudo --no-create-home user
-
-# Ensure we can start postgres by sudo-ing as the postgres user.
-RUN apt-get update && apt-get -qq install -y sudo
-RUN echo "user ALL=(ALL) NOPASSWD: ALL" >> /etc/sudoers
-
-ADD run_pg_tests.sh /run_pg_tests.sh
-# Use the "exec form" of ENTRYPOINT (https://docs.docker.com/engine/reference/builder/#entrypoint)
-# so that we can `docker run` this container and pass arguments to pg_tests.sh
-ENTRYPOINT ["/run_pg_tests.sh"]
-
-USER user

docker/Dockerfile-workers

@@ -2,25 +2,36 @@
 FROM matrixdotorg/synapse
 
 # Install deps
-RUN apt-get update
-RUN apt-get install -y supervisor redis nginx
+RUN \
+   --mount=type=cache,target=/var/cache/apt,sharing=locked \
+   --mount=type=cache,target=/var/lib/apt,sharing=locked \
+  apt-get update && \
+  DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
+     redis-server nginx-light
 
-# Remove the default nginx sites
+# Install supervisord with pip instead of apt, to avoid installing a second
+# copy of python.
+RUN --mount=type=cache,target=/root/.cache/pip \
+    pip install supervisor~=4.2
+
+# Disable the default nginx sites
 RUN rm /etc/nginx/sites-enabled/default
 
 # Copy Synapse worker, nginx and supervisord configuration template files
 COPY ./docker/conf-workers/* /conf/
 
+# Copy a script to prefix log lines with the supervisor program name
+COPY ./docker/prefix-log /usr/local/bin/
+
 # Expose nginx listener port
 EXPOSE 8080/tcp
 
-# Volume for user-editable config files, logs etc.
-VOLUME ["/data"]
-
 # A script to read environment variables and create the necessary
 # files to run the desired worker configuration. Will start supervisord.
 COPY ./docker/configure_workers_and_start.py /configure_workers_and_start.py
 ENTRYPOINT ["/configure_workers_and_start.py"]
 
+# Replace the healthcheck with one which checks *all* the workers. The script
+# is generated by configure_workers_and_start.py.
 HEALTHCHECK --start-period=5s --interval=15s --timeout=5s \
     CMD /bin/sh /healthcheck.sh

docker/README-testing.md

@@ -10,10 +10,10 @@ Note that running Synapse's unit tests from within the docker image is not suppo
 
 ## Testing with SQLite and single-process Synapse
 
-> Note that `scripts-dev/complement.sh` is a script that will automatically build 
+> Note that `scripts-dev/complement.sh` is a script that will automatically build
 > and run an SQLite-based, single-process of Synapse against Complement.
 
-The instructions below will set up Complement testing for a single-process, 
+The instructions below will set up Complement testing for a single-process,
 SQLite-based Synapse deployment.
 
 Start by building the base Synapse docker image. If you wish to run tests with the latest
@@ -26,23 +26,22 @@ docker build -t matrixdotorg/synapse -f docker/Dockerfile .
 
 This will build an image with the tag `matrixdotorg/synapse`.
 
-Next, build the Synapse image for Complement. You will need a local checkout 
-of Complement. Change to the root of your Complement checkout and run:
+Next, build the Synapse image for Complement.
 
 ```sh
-docker build -t complement-synapse -f "dockerfiles/Synapse.Dockerfile" dockerfiles
+docker build -t complement-synapse -f "docker/complement/Dockerfile" docker/complement
 ```
 
-This will build an image with the tag `complement-synapse`, which can be handed to 
-Complement for testing via the `COMPLEMENT_BASE_IMAGE` environment variable. Refer to 
-[Complement's documentation](https://github.com/matrix-org/complement/#running) for 
+This will build an image with the tag `complement-synapse`, which can be handed to
+Complement for testing via the `COMPLEMENT_BASE_IMAGE` environment variable. Refer to
+[Complement's documentation](https://github.com/matrix-org/complement/#running) for
 how to run the tests, as well as the various available command line flags.
 
 ## Testing with PostgreSQL and single or multi-process Synapse
 
-The above docker image only supports running Synapse with SQLite and in a 
-single-process topology. The following instructions are used to build a Synapse image for 
-Complement that supports either single or multi-process topology with a PostgreSQL 
+The above docker image only supports running Synapse with SQLite and in a
+single-process topology. The following instructions are used to build a Synapse image for
+Complement that supports either single or multi-process topology with a PostgreSQL
 database backend.
 
 As with the single-process image, build the base Synapse docker image. If you wish to run
@@ -55,7 +54,7 @@ docker build -t matrixdotorg/synapse -f docker/Dockerfile .
 
 This will build an image with the tag `matrixdotorg/synapse`.
 
-Next, we build a new image with worker support based on `matrixdotorg/synapse:latest`. 
+Next, we build a new image with worker support based on `matrixdotorg/synapse:latest`.
 Again, from the root of the repository:
 
 ```sh
@@ -64,21 +63,20 @@ docker build -t matrixdotorg/synapse-workers -f docker/Dockerfile-workers .
 
 This will build an image with the tag` matrixdotorg/synapse-workers`.
 
-It's worth noting at this point that this image is fully functional, and 
-can be used for testing against locally. See instructions for using the container 
+It's worth noting at this point that this image is fully functional, and
+can be used for testing against locally. See instructions for using the container
 under
 [Running the Dockerfile-worker image standalone](#running-the-dockerfile-worker-image-standalone)
 below.
 
 Finally, build the Synapse image for Complement, which is based on
-`matrixdotorg/synapse-workers`. You will need a local checkout of Complement. Change to
-the root of your Complement checkout and run:
+`matrixdotorg/synapse-workers`.
 
 ```sh
-docker build -t matrixdotorg/complement-synapse-workers -f dockerfiles/SynapseWorkers.Dockerfile dockerfiles
+docker build -t matrixdotorg/complement-synapse-workers -f docker/complement/SynapseWorkers.Dockerfile docker/complement
 ```
 
-This will build an image with the tag `complement-synapse`, which can be handed to
+This will build an image with the tag `complement-synapse-workers`, which can be handed to
 Complement for testing via the `COMPLEMENT_BASE_IMAGE` environment variable. Refer to
 [Complement's documentation](https://github.com/matrix-org/complement/#running) for
 how to run the tests, as well as the various available command line flags.
@@ -91,10 +89,10 @@ bundling all necessary components together for a workerised homeserver instance.
 
 This includes any desired Synapse worker processes, a nginx to route traffic accordingly,
 a redis for worker communication and a supervisord instance to start up and monitor all
-processes. You will need to provide your own postgres container to connect to, and TLS 
+processes. You will need to provide your own postgres container to connect to, and TLS
 is not handled by the container.
 
-Once you've built the image using the above instructions, you can run it. Be sure 
+Once you've built the image using the above instructions, you can run it. Be sure
 you've set up a volume according to the [usual Synapse docker instructions](README.md).
 Then run something along the lines of:
 
@@ -112,7 +110,7 @@ docker run -d --name synapse \
     matrixdotorg/synapse-workers
 ```
 
-...substituting `POSTGRES*` variables for those that match a postgres host you have 
+...substituting `POSTGRES*` variables for those that match a postgres host you have
 available (usually a running postgres docker container).
 
 The `SYNAPSE_WORKER_TYPES` environment variable is a comma-separated list of workers to
@@ -130,11 +128,11 @@ Otherwise, `SYNAPSE_WORKER_TYPES` can either be left empty or unset to spawn no
 (leaving only the main process). The container is configured to use redis-based worker
 mode.
 
-Logs for workers and the main process are logged to stdout and can be viewed with 
-standard `docker logs` tooling. Worker logs contain their worker name 
+Logs for workers and the main process are logged to stdout and can be viewed with
+standard `docker logs` tooling. Worker logs contain their worker name
 after the timestamp.
 
 Setting `SYNAPSE_WORKERS_WRITE_LOGS_TO_DISK=1` will cause worker logs to be written to
 `<data_dir>/logs/<worker_name>.log`. Logs are kept for 1 week and rotate every day at 00:
-00, according to the container's clock. Logging for the main process must still be 
+00, according to the container's clock. Logging for the main process must still be
 configured by modifying the homeserver's log config in your Synapse data volume.

docker/complement/Dockerfile

@@ -0,0 +1,22 @@
+# A dockerfile which builds an image suitable for testing Synapse under
+# complement.
+
+ARG SYNAPSE_VERSION=latest
+
+FROM matrixdotorg/synapse:${SYNAPSE_VERSION}
+
+ENV SERVER_NAME=localhost
+
+COPY conf/* /conf/
+
+# generate a signing key
+RUN generate_signing_key -o /conf/server.signing.key
+
+WORKDIR /data
+
+EXPOSE 8008 8448
+
+ENTRYPOINT ["/conf/start.sh"]
+
+HEALTHCHECK --start-period=5s --interval=1s --timeout=1s \
+    CMD curl -fSs http://localhost:8008/health || exit 1

docker/complement/README.md

@@ -0,0 +1 @@
+Stuff for building the docker image used for testing under complement.

docker/complement/SynapseWorkers.Dockerfile

@@ -0,0 +1,50 @@
+# This dockerfile builds on top of 'docker/Dockerfile-worker' in matrix-org/synapse
+# by including a built-in postgres instance, as well as setting up the homeserver so
+# that it is ready for testing via Complement.
+#
+# Instructions for building this image from those it depends on is detailed in this guide:
+# https://github.com/matrix-org/synapse/blob/develop/docker/README-testing.md#testing-with-postgresql-and-single-or-multi-process-synapse
+FROM matrixdotorg/synapse-workers
+
+# Download a caddy server to stand in front of nginx and terminate TLS using Complement's
+# custom CA.
+# We include this near the top of the file in order to cache the result.
+RUN curl -OL "https://github.com/caddyserver/caddy/releases/download/v2.3.0/caddy_2.3.0_linux_amd64.tar.gz" && \
+  tar xzf caddy_2.3.0_linux_amd64.tar.gz && rm caddy_2.3.0_linux_amd64.tar.gz && mv caddy /root
+
+# Install postgresql
+RUN apt-get update && \
+  DEBIAN_FRONTEND=noninteractive apt-get install --no-install-recommends -y postgresql-13
+
+# Configure a user and create a database for Synapse
+RUN pg_ctlcluster 13 main start &&  su postgres -c "echo \
+ \"ALTER USER postgres PASSWORD 'somesecret'; \
+ CREATE DATABASE synapse \
+  ENCODING 'UTF8' \
+  LC_COLLATE='C' \
+  LC_CTYPE='C' \
+  template=template0;\" | psql" && pg_ctlcluster 13 main stop
+
+# Modify the shared homeserver config with postgres support, certificate setup
+# and the disabling of rate-limiting
+COPY conf-workers/workers-shared.yaml /conf/workers/shared.yaml
+
+WORKDIR /data
+
+# Copy the caddy config
+COPY conf-workers/caddy.complement.json /root/caddy.json
+
+COPY conf-workers/postgres.supervisord.conf /etc/supervisor/conf.d/postgres.conf
+COPY conf-workers/caddy.supervisord.conf /etc/supervisor/conf.d/caddy.conf
+
+# Copy the entrypoint
+COPY conf-workers/start-complement-synapse-workers.sh /
+
+# Expose caddy's listener ports
+EXPOSE 8008 8448
+
+ENTRYPOINT ["/start-complement-synapse-workers.sh"]
+
+# Update the healthcheck to have a shorter check interval
+HEALTHCHECK --start-period=5s --interval=1s --timeout=1s \
+    CMD /bin/sh /healthcheck.sh

docker/complement/conf-workers/caddy.complement.json

@@ -0,0 +1,72 @@
+{
+    "apps": {
+      "http": {
+        "servers": {
+          "srv0": {
+            "listen": [
+              ":8448"
+            ],
+            "routes": [
+              {
+                "match": [
+                  {
+                    "host": [
+                      "{{ server_name }}"
+                    ]
+                  }
+                ],
+                "handle": [
+                  {
+                    "handler": "subroute",
+                    "routes": [
+                      {
+                        "handle": [
+                          {
+                            "handler": "reverse_proxy",
+                            "upstreams": [
+                              {
+                                "dial": "localhost:8008"
+                              }
+                            ]
+                          }
+                        ]
+                      }
+                    ]
+                  }
+                ],
+                "terminal": true
+              }
+            ]
+          }
+        }
+      },
+      "tls": {
+        "automation": {
+          "policies": [
+            {
+              "subjects": [
+                "{{ server_name }}"
+              ],
+              "issuers": [
+                {
+                  "module": "internal"
+                }
+              ],
+              "on_demand": true
+            }
+          ]
+        }
+      },
+      "pki": {
+        "certificate_authorities": {
+          "local": {
+            "name": "Complement CA",
+            "root": {
+              "certificate": "/complement/ca/ca.crt",
+              "private_key": "/complement/ca/ca.key"
+            }
+          }
+        }
+      }
+    }
+  }

docker/complement/conf-workers/caddy.supervisord.conf

@@ -0,0 +1,7 @@
+[program:caddy]
+command=/usr/local/bin/prefix-log /root/caddy run --config /root/caddy.json
+autorestart=unexpected
+stdout_logfile=/dev/stdout
+stdout_logfile_maxbytes=0
+stderr_logfile=/dev/stderr
+stderr_logfile_maxbytes=0

docker/complement/conf-workers/postgres.supervisord.conf

@@ -0,0 +1,16 @@
+[program:postgres]
+command=/usr/local/bin/prefix-log /usr/bin/pg_ctlcluster 13 main start --foreground
+
+# Lower priority number = starts first
+priority=1
+
+autorestart=unexpected
+stdout_logfile=/dev/stdout
+stdout_logfile_maxbytes=0
+stderr_logfile=/dev/stderr
+stderr_logfile_maxbytes=0
+
+# Use 'Fast Shutdown' mode which aborts current transactions and closes connections quickly.
+# (Default (TERM) is 'Smart Shutdown' which stops accepting new connections but
+#  lets existing connections close gracefully.)
+stopsignal=INT

docker/complement/conf-workers/start-complement-synapse-workers.sh

@@ -0,0 +1,44 @@
+#!/bin/bash
+#
+# Default ENTRYPOINT for the docker image used for testing synapse with workers under complement
+
+set -e
+
+function log {
+    d=$(date +"%Y-%m-%d %H:%M:%S,%3N")
+    echo "$d $@"
+}
+
+# Replace the server name in the caddy config
+sed -i "s/{{ server_name }}/${SERVER_NAME}/g" /root/caddy.json
+
+# Set the server name of the homeserver
+export SYNAPSE_SERVER_NAME=${SERVER_NAME}
+
+# No need to report stats here
+export SYNAPSE_REPORT_STATS=no
+
+# Set postgres authentication details which will be placed in the homeserver config file
+export POSTGRES_PASSWORD=somesecret
+export POSTGRES_USER=postgres
+export POSTGRES_HOST=localhost
+
+# Specify the workers to test with
+export SYNAPSE_WORKER_TYPES="\
+    event_persister, \
+    event_persister, \
+    background_worker, \
+    frontend_proxy, \
+    event_creator, \
+    user_dir, \
+    media_repository, \
+    federation_inbound, \
+    federation_reader, \
+    federation_sender, \
+    synchrotron, \
+    appservice, \
+    pusher"
+
+# Run the script that writes the necessary config files and starts supervisord, which in turn
+# starts everything else
+exec /configure_workers_and_start.py

docker/complement/conf-workers/workers-shared.yaml

@@ -0,0 +1,72 @@
+## Server ##
+report_stats: False
+trusted_key_servers: []
+enable_registration: true
+enable_registration_without_verification: true
+bcrypt_rounds: 4
+
+## Federation ##
+
+# trust certs signed by Complement's CA
+federation_custom_ca_list:
+- /complement/ca/ca.crt
+
+# unblacklist RFC1918 addresses
+federation_ip_range_blacklist: []
+
+# Disable server rate-limiting
+rc_federation:
+  window_size: 1000
+  sleep_limit: 10
+  sleep_delay: 500
+  reject_limit: 99999
+  concurrent: 3
+
+rc_message:
+  per_second: 9999
+  burst_count: 9999
+
+rc_registration:
+  per_second: 9999
+  burst_count: 9999
+
+rc_login:
+  address:
+    per_second: 9999
+    burst_count: 9999
+  account:
+    per_second: 9999
+    burst_count: 9999
+  failed_attempts:
+    per_second: 9999
+    burst_count: 9999
+
+rc_admin_redaction:
+  per_second: 9999
+  burst_count: 9999
+
+rc_joins:
+  local:
+    per_second: 9999
+    burst_count: 9999
+  remote:
+    per_second: 9999
+    burst_count: 9999
+
+federation_rr_transactions_per_room_per_second: 9999
+
+## Experimental Features ##
+
+experimental_features:
+  # Enable history backfilling support
+  msc2716_enabled: true
+  # Enable spaces support
+  spaces_enabled: true
+  # Enable jump to date endpoint
+  msc3030_enabled: true
+
+server_notices:
+  system_mxid_localpart: _server
+  system_mxid_display_name: "Server Alert"
+  system_mxid_avatar_url: ""
+  room_name: "Server Alert"

docker/complement/conf/homeserver.yaml

@@ -0,0 +1,117 @@
+## Server ##
+
+server_name: SERVER_NAME
+log_config: /conf/log_config.yaml
+report_stats: False
+signing_key_path: /conf/server.signing.key
+trusted_key_servers: []
+enable_registration: true
+enable_registration_without_verification: true
+
+## Listeners ##
+
+tls_certificate_path: /conf/server.tls.crt
+tls_private_key_path: /conf/server.tls.key
+bcrypt_rounds: 4
+registration_shared_secret: complement
+
+listeners:
+  - port: 8448
+    bind_addresses: ['::']
+    type: http
+    tls: true
+    resources:
+      - names: [federation]
+
+  - port: 8008
+    bind_addresses: ['::']
+    type: http
+
+    resources:
+      - names: [client]
+
+## Database ##
+
+database:
+  name: "sqlite3"
+  args:
+    # We avoid /data, as it is a volume and is not transferred when the container is committed,
+    # which is a fundamental necessity in complement.
+    database: "/conf/homeserver.db"
+
+## Federation ##
+
+# trust certs signed by the complement CA
+federation_custom_ca_list:
+- /complement/ca/ca.crt
+
+# unblacklist RFC1918 addresses
+ip_range_blacklist: []
+
+# Disable server rate-limiting
+rc_federation:
+  window_size: 1000
+  sleep_limit: 10
+  sleep_delay: 500
+  reject_limit: 99999
+  concurrent: 3
+
+rc_message:
+  per_second: 9999
+  burst_count: 9999
+
+rc_registration:
+  per_second: 9999
+  burst_count: 9999
+
+rc_login:
+  address:
+    per_second: 9999
+    burst_count: 9999
+  account:
+    per_second: 9999
+    burst_count: 9999
+  failed_attempts:
+    per_second: 9999
+    burst_count: 9999
+
+rc_admin_redaction:
+  per_second: 9999
+  burst_count: 9999
+
+rc_joins:
+  local:
+    per_second: 9999
+    burst_count: 9999
+  remote:
+    per_second: 9999
+    burst_count: 9999
+
+federation_rr_transactions_per_room_per_second: 9999
+
+## API Configuration ##
+
+# A list of application service config files to use
+#
+app_service_config_files:
+AS_REGISTRATION_FILES  
+
+## Experimental Features ##
+
+experimental_features:
+  # Enable spaces support
+  spaces_enabled: true
+  # Enable history backfilling support
+  msc2716_enabled: true
+  # server-side support for partial state in /send_join responses
+  msc3706_enabled: true
+  # client-side support for partial state in /send_join responses
+  faster_joins: true
+  # Enable jump to date endpoint
+  msc3030_enabled: true
+
+server_notices:
+  system_mxid_localpart: _server
+  system_mxid_display_name: "Server Alert"
+  system_mxid_avatar_url: ""
+  room_name: "Server Alert"

docker/complement/conf/log_config.yaml

@@ -0,0 +1,24 @@
+version: 1
+
+formatters:
+  precise:
+   format: '%(asctime)s - %(name)s - %(lineno)d - %(levelname)s - %(request)s - %(message)s'
+
+filters:
+  context:
+    (): synapse.logging.context.LoggingContextFilter
+    request: ""
+
+handlers:
+  console:
+    class: logging.StreamHandler
+    formatter: precise
+    filters: [context]
+    # log to stdout, for easier use with 'docker logs'
+    stream: 'ext://sys.stdout'
+
+root:
+    level: INFO
+    handlers: [console]
+
+disable_existing_loggers: false

docker/complement/conf/start.sh

@@ -0,0 +1,30 @@
+#!/bin/sh
+
+set -e
+
+sed -i "s/SERVER_NAME/${SERVER_NAME}/g" /conf/homeserver.yaml
+
+# Add the application service registration files to the homeserver.yaml config
+for filename in /complement/appservice/*.yaml; do
+  [ -f "$filename" ] || break
+
+  as_id=$(basename "$filename" .yaml)
+
+  # Insert the path to the registration file and the AS_REGISTRATION_FILES marker after 
+  # so we can add the next application service in the next iteration of this for loop
+  sed -i "s/AS_REGISTRATION_FILES/  - \/complement\/appservice\/${as_id}.yaml\nAS_REGISTRATION_FILES/g" /conf/homeserver.yaml
+done
+# Remove the AS_REGISTRATION_FILES entry
+sed -i "s/AS_REGISTRATION_FILES//g" /conf/homeserver.yaml
+
+# generate an ssl key and cert for the server, signed by the complement CA
+openssl genrsa -out /conf/server.tls.key 2048
+
+openssl req -new -key /conf/server.tls.key -out /conf/server.tls.csr \
+  -subj "/CN=${SERVER_NAME}"
+openssl x509 -req -in /conf/server.tls.csr \
+  -CA /complement/ca/ca.crt -CAkey /complement/ca/ca.key -set_serial 1 \
+  -out /conf/server.tls.crt
+
+exec python -m synapse.app.homeserver -c /conf/homeserver.yaml "$@"
+

docker/conf-workers/supervisord.conf.j2

@@ -5,6 +5,9 @@
 nodaemon=true
 user=root
 
+[include]
+files = /etc/supervisor/conf.d/*.conf
+
 [program:nginx]
 command=/usr/sbin/nginx -g "daemon off;"
 priority=500

docker/conf/log.config

@@ -2,11 +2,7 @@ version: 1
 
 formatters:
   precise:
-{% if worker_name %}
-    format: '%(asctime)s - worker:{{ worker_name }} - %(name)s - %(lineno)d - %(levelname)s - %(request)s - %(message)s'
-{% else %}
     format: '%(asctime)s - %(name)s - %(lineno)d - %(levelname)s - %(request)s - %(message)s'
-{% endif %}
 
 handlers:
 {% if LOG_FILE_PATH %}

docker/configure_workers_and_start.py

@@ -29,6 +29,7 @@
 import os
 import subprocess
 import sys
+from typing import Any, Dict, List, Mapping, MutableMapping, NoReturn, Set
 
 import jinja2
 import yaml
@@ -36,7 +37,7 @@ import yaml
 MAIN_PROCESS_HTTP_LISTENER_PORT = 8080
 
 
-WORKERS_CONFIG = {
+WORKERS_CONFIG: Dict[str, Dict[str, Any]] = {
     "pusher": {
         "app": "synapse.app.pusher",
         "listener_resources": [],
@@ -170,7 +171,7 @@ WORKERS_CONFIG = {
 # Templates for sections that may be inserted multiple times in config files
 SUPERVISORD_PROCESS_CONFIG_BLOCK = """
 [program:synapse_{name}]
-command=/usr/local/bin/python -m {app} \
+command=/usr/local/bin/prefix-log /usr/local/bin/python -m {app} \
     --config-path="{config_path}" \
     --config-path=/conf/workers/shared.yaml \
     --config-path=/conf/workers/{name}.yaml
@@ -200,7 +201,7 @@ upstream {upstream_worker_type} {{
 
 
 # Utility functions
-def log(txt: str):
+def log(txt: str) -> None:
     """Log something to the stdout.
 
     Args:
@@ -209,7 +210,7 @@ def log(txt: str):
     print(txt)
 
 
-def error(txt: str):
+def error(txt: str) -> NoReturn:
     """Log something and exit with an error code.
 
     Args:
@@ -219,7 +220,7 @@ def error(txt: str):
     sys.exit(2)
 
 
-def convert(src: str, dst: str, **template_vars):
+def convert(src: str, dst: str, **template_vars: object) -> None:
     """Generate a file from a template
 
     Args:
@@ -289,7 +290,7 @@ def add_sharding_to_shared_config(
         shared_config.setdefault("media_instance_running_background_jobs", worker_name)
 
 
-def generate_base_homeserver_config():
+def generate_base_homeserver_config() -> None:
     """Starts Synapse and generates a basic homeserver config, which will later be
     modified for worker support.
 
@@ -301,13 +302,15 @@ def generate_base_homeserver_config():
     subprocess.check_output(["/usr/local/bin/python", "/start.py", "migrate_config"])
 
 
-def generate_worker_files(environ, config_path: str, data_dir: str):
+def generate_worker_files(
+    environ: Mapping[str, str], config_path: str, data_dir: str
+) -> None:
     """Read the desired list of workers from environment variables and generate
     shared homeserver, nginx and supervisord configs.
 
     Args:
-        environ: _Environ[str]
-        config_path: Where to output the generated Synapse main worker config file.
+        environ: os.environ instance.
+        config_path: The location of the generated Synapse main worker config file.
         data_dir: The location of the synapse data directory. Where log and
             user-facing config files live.
     """
@@ -320,7 +323,8 @@ def generate_worker_files(environ, config_path: str, data_dir: str):
     # and adding a replication listener.
 
     # First read the original config file and extract the listeners block. Then we'll add
-    # another listener for replication. Later we'll write out the result.
+    # another listener for replication. Later we'll write out the result to the shared
+    # config file.
     listeners = [
         {
             "port": 9093,
@@ -339,7 +343,7 @@ def generate_worker_files(environ, config_path: str, data_dir: str):
     # base shared worker jinja2 template.
     #
     # This config file will be passed to all workers, included Synapse's main process.
-    shared_config = {"listeners": listeners}
+    shared_config: Dict[str, Any] = {"listeners": listeners}
 
     # The supervisord config. The contents of which will be inserted into the
     # base supervisord jinja2 template.
@@ -355,7 +359,7 @@ def generate_worker_files(environ, config_path: str, data_dir: str):
     #   worker_type: {1234, 1235, ...}}
     # }
     # and will be used to construct 'upstream' nginx directives.
-    nginx_upstreams = {}
+    nginx_upstreams: Dict[str, Set[int]] = {}
 
     # A map of: {"endpoint": "upstream"}, where "upstream" is a str representing what will be
     # placed after the proxy_pass directive. The main benefit to representing this data as a
@@ -367,13 +371,13 @@ def generate_worker_files(environ, config_path: str, data_dir: str):
     nginx_locations = {}
 
     # Read the desired worker configuration from the environment
-    worker_types = environ.get("SYNAPSE_WORKER_TYPES")
-    if worker_types is None:
+    worker_types_env = environ.get("SYNAPSE_WORKER_TYPES")
+    if worker_types_env is None:
         # No workers, just the main process
         worker_types = []
     else:
         # Split type names by comma
-        worker_types = worker_types.split(",")
+        worker_types = worker_types_env.split(",")
 
     # Create the worker configuration directory if it doesn't already exist
     os.makedirs("/conf/workers", exist_ok=True)
@@ -384,7 +388,11 @@ def generate_worker_files(environ, config_path: str, data_dir: str):
     # A counter of worker_type -> int. Used for determining the name for a given
     # worker type when generating its config file, as each worker's name is just
     # worker_type + instance #
-    worker_type_counter = {}
+    worker_type_counter: Dict[str, int] = {}
+
+    # A list of internal endpoints to healthcheck, starting with the main process
+    # which exists even if no workers do.
+    healthcheck_urls = ["http://localhost:8080/health"]
 
     # For each worker type specified by the user, create config values
     for worker_type in worker_types:
@@ -404,12 +412,14 @@ def generate_worker_files(environ, config_path: str, data_dir: str):
         # e.g. federation_reader1
         worker_name = worker_type + str(new_worker_count)
         worker_config.update(
-            {"name": worker_name, "port": worker_port, "config_path": config_path}
+            {"name": worker_name, "port": str(worker_port), "config_path": config_path}
         )
 
         # Update the shared config with any worker-type specific options
         shared_config.update(worker_config["shared_extra_conf"])
 
+        healthcheck_urls.append("http://localhost:%d/health" % (worker_port,))
+
         # Check if more than one instance of this worker type has been specified
         worker_type_total_count = worker_types.count(worker_type)
         if worker_type_total_count > 1:
@@ -438,21 +448,7 @@ def generate_worker_files(environ, config_path: str, data_dir: str):
 
         # Write out the worker's logging config file
 
-        # Check whether we should write worker logs to disk, in addition to the console
-        extra_log_template_args = {}
-        if environ.get("SYNAPSE_WORKERS_WRITE_LOGS_TO_DISK"):
-            extra_log_template_args["LOG_FILE_PATH"] = "{dir}/logs/{name}.log".format(
-                dir=data_dir, name=worker_name
-            )
-
-        # Render and write the file
-        log_config_filepath = "/conf/workers/{name}.log.config".format(name=worker_name)
-        convert(
-            "/conf/log.config",
-            log_config_filepath,
-            worker_name=worker_name,
-            **extra_log_template_args,
-        )
+        log_config_filepath = generate_worker_log_config(environ, worker_name, data_dir)
 
         # Then a worker config file
         convert(
@@ -475,15 +471,10 @@ def generate_worker_files(environ, config_path: str, data_dir: str):
     # Determine the load-balancing upstreams to configure
     nginx_upstream_config = ""
 
-    # At the same time, prepare a list of internal endpoints to healthcheck
-    # starting with the main process which exists even if no workers do.
-    healthcheck_urls = ["http://localhost:8080/health"]
-
     for upstream_worker_type, upstream_worker_ports in nginx_upstreams.items():
         body = ""
         for port in upstream_worker_ports:
             body += "    server localhost:%d;\n" % (port,)
-            healthcheck_urls.append("http://localhost:%d/health" % (port,))
 
         # Add to the list of configured upstreams
         nginx_upstream_config += NGINX_UPSTREAM_CONFIG_BLOCK.format(
@@ -493,6 +484,10 @@ def generate_worker_files(environ, config_path: str, data_dir: str):
 
     # Finally, we'll write out the config files.
 
+    # log config for the master process
+    master_log_config = generate_worker_log_config(environ, "master", data_dir)
+    shared_config["log_config"] = master_log_config
+
     # Shared homeserver config
     convert(
         "/conf/shared.yaml.j2",
@@ -509,9 +504,10 @@ def generate_worker_files(environ, config_path: str, data_dir: str):
     )
 
     # Supervisord config
+    os.makedirs("/etc/supervisor", exist_ok=True)
     convert(
         "/conf/supervisord.conf.j2",
-        "/etc/supervisor/conf.d/supervisord.conf",
+        "/etc/supervisor/supervisord.conf",
         main_config_path=config_path,
         worker_config=supervisord_config,
     )
@@ -529,15 +525,31 @@ def generate_worker_files(environ, config_path: str, data_dir: str):
         os.mkdir(log_dir)
 
 
-def start_supervisord():
-    """Starts up supervisord which then starts and monitors all other necessary processes
+def generate_worker_log_config(
+    environ: Mapping[str, str], worker_name: str, data_dir: str
+) -> str:
+    """Generate a log.config file for the given worker.
 
-    Raises: CalledProcessError if calling start.py return a non-zero exit code.
+    Returns: the path to the generated file
     """
-    subprocess.run(["/usr/bin/supervisord"], stdin=subprocess.PIPE)
+    # Check whether we should write worker logs to disk, in addition to the console
+    extra_log_template_args = {}
+    if environ.get("SYNAPSE_WORKERS_WRITE_LOGS_TO_DISK"):
+        extra_log_template_args["LOG_FILE_PATH"] = "{dir}/logs/{name}.log".format(
+            dir=data_dir, name=worker_name
+        )
+    # Render and write the file
+    log_config_filepath = "/conf/workers/{name}.log.config".format(name=worker_name)
+    convert(
+        "/conf/log.config",
+        log_config_filepath,
+        worker_name=worker_name,
+        **extra_log_template_args,
+    )
+    return log_config_filepath
 
 
-def main(args, environ):
+def main(args: List[str], environ: MutableMapping[str, str]) -> None:
     config_dir = environ.get("SYNAPSE_CONFIG_DIR", "/data")
     config_path = environ.get("SYNAPSE_CONFIG_PATH", config_dir + "/homeserver.yaml")
     data_dir = environ.get("SYNAPSE_DATA_DIR", "/data")
@@ -564,7 +576,13 @@ def main(args, environ):
 
     # Start supervisord, which will start Synapse, all of the configured worker
     # processes, redis, nginx etc. according to the config we created above.
-    start_supervisord()
+    log("Starting supervisord")
+    os.execl(
+        "/usr/local/bin/supervisord",
+        "supervisord",
+        "-c",
+        "/etc/supervisor/supervisord.conf",
+    )
 
 
 if __name__ == "__main__":

docker/prefix-log

@@ -0,0 +1,12 @@
+#!/bin/bash
+#
+# Prefixes all lines on stdout and stderr with the process name (as determined by
+# the SUPERVISOR_PROCESS_NAME env var, which is automatically set by Supervisor).
+#
+# Usage:
+#   prefix-log command [args...]
+#
+
+exec 1> >(awk '{print "'"${SUPERVISOR_PROCESS_NAME}"' | "$0}' >&1)
+exec 2> >(awk '{print "'"${SUPERVISOR_PROCESS_NAME}"' | "$0}' >&2)
+exec "$@"

docker/run_pg_tests.sh

@@ -1,19 +0,0 @@
-#!/usr/bin/env bash
-
-# This script runs the PostgreSQL tests inside a Docker container. It expects
-# the relevant source files to be mounted into /src (done automatically by the
-# caller script). It will set up the database, run it, and then use the tox
-# configuration to run the tests.
-
-set -e
-
-# Set PGUSER so Synapse's tests know what user to connect to the database with
-export PGUSER=postgres
-
-# Start the database
-sudo -u postgres /usr/lib/postgresql/10/bin/pg_ctl -w -D /var/lib/postgresql/data start
-
-# Run the tests
-cd /src
-export TRIAL_FLAGS="-j 4"
-tox --workdir=./.tox-pg-container -e py37-postgres "$@"

docker/start.py

@@ -6,27 +6,28 @@ import os
 import platform
 import subprocess
 import sys
+from typing import Any, Dict, List, Mapping, MutableMapping, NoReturn, Optional
 
 import jinja2
 
 
 # Utility functions
-def log(txt):
+def log(txt: str) -> None:
     print(txt, file=sys.stderr)
 
 
-def error(txt):
+def error(txt: str) -> NoReturn:
     log(txt)
     sys.exit(2)
 
 
-def convert(src, dst, environ):
+def convert(src: str, dst: str, environ: Mapping[str, object]) -> None:
     """Generate a file from a template
 
     Args:
-        src (str): path to input file
-        dst (str): path to file to write
-        environ (dict): environment dictionary, for replacement mappings.
+        src: path to input file
+        dst: path to file to write
+        environ: environment dictionary, for replacement mappings.
     """
     with open(src) as infile:
         template = infile.read()
@@ -35,25 +36,30 @@ def convert(src, dst, environ):
         outfile.write(rendered)
 
 
-def generate_config_from_template(config_dir, config_path, environ, ownership):
+def generate_config_from_template(
+    config_dir: str,
+    config_path: str,
+    os_environ: Mapping[str, str],
+    ownership: Optional[str],
+) -> None:
     """Generate a homeserver.yaml from environment variables
 
     Args:
-        config_dir (str): where to put generated config files
-        config_path (str): where to put the main config file
-        environ (dict): environment dictionary
-        ownership (str|None): "<user>:<group>" string which will be used to set
+        config_dir: where to put generated config files
+        config_path: where to put the main config file
+        os_environ: environment mapping
+        ownership: "<user>:<group>" string which will be used to set
             ownership of the generated configs. If None, ownership will not change.
     """
     for v in ("SYNAPSE_SERVER_NAME", "SYNAPSE_REPORT_STATS"):
-        if v not in environ:
+        if v not in os_environ:
             error(
                 "Environment variable '%s' is mandatory when generating a config file."
                 % (v,)
             )
 
     # populate some params from data files (if they exist, else create new ones)
-    environ = environ.copy()
+    environ: Dict[str, Any] = dict(os_environ)
     secrets = {
         "registration": "SYNAPSE_REGISTRATION_SHARED_SECRET",
         "macaroon": "SYNAPSE_MACAROON_SECRET_KEY",
@@ -108,7 +114,7 @@ def generate_config_from_template(config_dir, config_path, environ, ownership):
 
     # Hopefully we already have a signing key, but generate one if not.
     args = [
-        "python",
+        sys.executable,
         "-m",
         "synapse.app.homeserver",
         "--config-path",
@@ -127,12 +133,12 @@ def generate_config_from_template(config_dir, config_path, environ, ownership):
     subprocess.check_output(args)
 
 
-def run_generate_config(environ, ownership):
+def run_generate_config(environ: Mapping[str, str], ownership: Optional[str]) -> None:
     """Run synapse with a --generate-config param to generate a template config file
 
     Args:
-        environ (dict): env var dict
-        ownership (str|None): "userid:groupid" arg for chmod. If None, ownership will not change.
+        environ: env vars from `os.enrivon`.
+        ownership: "userid:groupid" arg for chmod. If None, ownership will not change.
 
     Never returns.
     """
@@ -158,7 +164,7 @@ def run_generate_config(environ, ownership):
 
     # generate the main config file, and a signing key.
     args = [
-        "python",
+        sys.executable,
         "-m",
         "synapse.app.homeserver",
         "--server-name",
@@ -175,10 +181,10 @@ def run_generate_config(environ, ownership):
         "--open-private-ports",
     ]
     # log("running %s" % (args, ))
-    os.execv("/usr/local/bin/python", args)
+    os.execv(sys.executable, args)
 
 
-def main(args, environ):
+def main(args: List[str], environ: MutableMapping[str, str]) -> None:
     mode = args[1] if len(args) > 1 else "run"
 
     # if we were given an explicit user to switch to, do so
@@ -254,12 +260,12 @@ running with 'migrate_config'. See the README for more details.
 
     log("Starting synapse with args " + " ".join(args))
 
-    args = ["python"] + args
+    args = [sys.executable] + args
     if ownership is not None:
         args = ["gosu", ownership] + args
         os.execve("/usr/sbin/gosu", args, environ)
     else:
-        os.execve("/usr/local/bin/python", args, environ)
+        os.execve(sys.executable, args, environ)
 
 
 if __name__ == "__main__":

docs/SUMMARY.md

@@ -17,6 +17,7 @@
 # Usage
   - [Federation](federate.md)
   - [Configuration](usage/configuration/README.md)
+    - [Configuration Manual](usage/configuration/config_documentation.md) 
     - [Homeserver Sample Config File](usage/configuration/homeserver_sample_config.md)
     - [Logging Sample Config File](usage/configuration/logging_sample_config.md)
     - [Structured Logging](structured_logging.md)
@@ -45,6 +46,7 @@
         - [Account validity callbacks](modules/account_validity_callbacks.md)
         - [Password auth provider callbacks](modules/password_auth_provider_callbacks.md)
         - [Background update controller callbacks](modules/background_update_controller_callbacks.md)
+        - [Account data callbacks](modules/account_data_callbacks.md)
         - [Porting a legacy module to the new interface](modules/porting_legacy_module.md)
     - [Workers](workers.md)
       - [Using `synctl` with Workers](synctl_workers.md)

docs/admin_api/user_admin_api.md

@@ -804,7 +804,7 @@ POST /_synapse/admin/v2/users/<user_id>/delete_devices
   "devices": [
     "QBUAZIFURK",
     "AUIECTSRND"
-  ],
+  ]
 }
 ```