update comment

update newsfragment number
lints
2022-04-28 11:55:00 -07:00 · 2022-04-27 15:38:02 -07:00 · 2022-04-27 15:34:11 -07:00 · 2022-04-27 15:17:45 -07:00 · 2022-04-27 15:17:33 -07:00 · 2022-04-27 14:58:26 +01:00
397 changed files with 25144 additions and 14202 deletions
@@ -0,0 +1,4 @@
+---
+title: CI run against latest deps is failing
+---
+See https://github.com/{{env.GITHUB_REPOSITORY}}/actions/runs/{{env.GITHUB_RUN_ID}}
@@ -1,8 +0,0 @@
-#!/bin/sh
-
-# replaces the dependency on Twisted in `python_dependencies` with trunk.
-
-set -e
-cd "$(dirname "$0")"/..
-
-sed -i -e 's#"Twisted.*"#"Twisted @ git+https://github.com/twisted/twisted"#' synapse/python_dependencies.py
@@ -2,29 +2,24 @@

 # Test for the export-data admin command against sqlite and postgres

+# Expects Synapse to have been already installed with `poetry install --extras postgres`.
+# Expects `poetry` to be available on the `PATH`.
+
 set -xe
 cd "$(dirname "$0")/../.."

-echo "--- Install dependencies"
-
-# Install dependencies for this test.
-pip install psycopg2
-
-# Install Synapse itself. This won't update any libraries.
-pip install -e .
-
 echo "--- Generate the signing key"

 # Generate the server's signing key.
-python -m synapse.app.homeserver --generate-keys -c .ci/sqlite-config.yaml
+poetry run synapse_homeserver --generate-keys -c .ci/sqlite-config.yaml

 echo "--- Prepare test database"

 # Make sure the SQLite3 database is using the latest schema and has no pending background update.
-update_synapse_database --database-config .ci/sqlite-config.yaml --run-background-updates
+poetry run update_synapse_database --database-config .ci/sqlite-config.yaml --run-background-updates

 # Run the export-data command on the sqlite test database
-python -m synapse.app.admin_cmd -c .ci/sqlite-config.yaml  export-data @anon-20191002_181700-832:localhost:8800 \
+poetry run python -m synapse.app.admin_cmd -c .ci/sqlite-config.yaml  export-data @anon-20191002_181700-832:localhost:8800 \
 --output-directory /tmp/export_data

 # Test that the output directory exists and contains the rooms directory
@@ -37,14 +32,14 @@ else
 fi

 # Create the PostgreSQL database.
-.ci/scripts/postgres_exec.py "CREATE DATABASE synapse"
+poetry run .ci/scripts/postgres_exec.py "CREATE DATABASE synapse"

 # Port the SQLite databse to postgres so we can check command works against postgres
 echo "+++ Port SQLite3 databse to postgres"
-synapse_port_db --sqlite-database .ci/test_db.db --postgres-config .ci/postgres-config.yaml
+poetry run synapse_port_db --sqlite-database .ci/test_db.db --postgres-config .ci/postgres-config.yaml

 # Run the export-data command on postgres database
-python -m synapse.app.admin_cmd -c .ci/postgres-config.yaml  export-data @anon-20191002_181700-832:localhost:8800 \
+poetry run python -m synapse.app.admin_cmd -c .ci/postgres-config.yaml  export-data @anon-20191002_181700-832:localhost:8800 \
 --output-directory /tmp/export_data2

 # Test that the output directory exists and contains the rooms directory
@@ -1,6 +1,9 @@
 #!/usr/bin/env bash
-# this script is run by GitHub Actions in a plain `focal` container; it installs the
-# minimal requirements for tox and hands over to the py3-old tox environment.
+# this script is run by GitHub Actions in a plain `focal` container; it
+# - installs the minimal system requirements, and poetry;
+# - patches the project definition file to refer to old versions only;
+# - creates a venv with these old versions using poetry; and finally
+# - invokes `trial` to run the tests with old deps.

 # Prevent tzdata from asking for user input
 export DEBIAN_FRONTEND=noninteractive
@@ -9,12 +12,70 @@ set -ex

 apt-get update
 apt-get install -y \
-        python3 python3-dev python3-pip python3-venv \
-        libxml2-dev libxslt-dev xmlsec1 zlib1g-dev tox libjpeg-dev libwebp-dev
+        python3 python3-dev python3-pip python3-venv pipx \
+        libxml2-dev libxslt-dev xmlsec1 zlib1g-dev libjpeg-dev libwebp-dev

 export LANG="C.UTF-8"

 # Prevent virtualenv from auto-updating pip to an incompatible version
 export VIRTUALENV_NO_DOWNLOAD=1

-exec tox -e py3-old
+# TODO: in the future, we could use an implementation of
+#   https://github.com/python-poetry/poetry/issues/3527
+#   https://github.com/pypa/pip/issues/8085
+# to select the lowest possible versions, rather than resorting to this sed script.
+
+# Patch the project definitions in-place:
+# - Replace all lower and tilde bounds with exact bounds
+# - Make the pyopenssl 17.0, which is the oldest version that works with
+#   a `cryptography` compiled against OpenSSL 1.1.
+# - Delete all lines referring to psycopg2 --- so no testing of postgres support.
+# - Omit systemd: we're not logging to journal here.
+
+# TODO: also replace caret bounds, see https://python-poetry.org/docs/dependency-specification/#version-constraints
+# We don't use these yet, but IIRC they are the default bound used when you `poetry add`.
+# The sed expression 's/\^/==/g' ought to do the trick. But it would also change
+# `python = "^3.7"` to `python = "==3.7", which would mean we fail because olddeps
+# runs on 3.8 (#12343).
+
+sed -i \
+   -e "s/[~>]=/==/g" \
+   -e "/psycopg2/d" \
+   -e 's/pyOpenSSL = "==16.0.0"/pyOpenSSL = "==17.0.0"/' \
+   -e '/systemd/d' \
+   pyproject.toml
+
+# Use poetry to do the installation. This ensures that the versions are all mutually
+# compatible (as far the package metadata declares, anyway); pip's package resolver
+# is more lax.
+#
+# Rather than `poetry install --no-dev`, we drop all dev dependencies from the
+# toml file. This means we don't have to ensure compatibility between old deps and
+# dev tools.
+
+pip install --user toml
+
+REMOVE_DEV_DEPENDENCIES="
+import toml
+with open('pyproject.toml', 'r') as f:
+    data = toml.loads(f.read())
+
+del data['tool']['poetry']['dev-dependencies']
+
+with open('pyproject.toml', 'w') as f:
+    toml.dump(data, f)
+"
+python3 -c "$REMOVE_DEV_DEPENDENCIES"
+
+pipx install poetry==1.1.12
+~/.local/bin/poetry lock
+
+echo "::group::Patched pyproject.toml"
+cat pyproject.toml
+echo "::endgroup::"
+echo "::group::Lockfile after patch"
+cat poetry.lock
+echo "::endgroup::"
+
+~/.local/bin/poetry install -E "all test"
+~/.local/bin/poetry run trial --jobs=2 tests
@@ -1,43 +1,37 @@
 #!/usr/bin/env bash
 #
 # Test script for 'synapse_port_db'.
-#   - sets up synapse and deps
+#   - configures synapse and a postgres server.
 #   - runs the port script on a prepopulated test sqlite db
 #   - also runs it against an new sqlite db
-
+#
+# Expects Synapse to have been already installed with `poetry install --extras postgres`.
+# Expects `poetry` to be available on the `PATH`.

 set -xe
 cd "$(dirname "$0")/../.."

-echo "--- Install dependencies"
-
-# Install dependencies for this test.
-pip install psycopg2 coverage coverage-enable-subprocess
-
-# Install Synapse itself. This won't update any libraries.
-pip install -e .
-
 echo "--- Generate the signing key"

 # Generate the server's signing key.
-python -m synapse.app.homeserver --generate-keys -c .ci/sqlite-config.yaml
+poetry run synapse_homeserver --generate-keys -c .ci/sqlite-config.yaml

 echo "--- Prepare test database"

 # Make sure the SQLite3 database is using the latest schema and has no pending background update.
-update_synapse_database --database-config .ci/sqlite-config.yaml --run-background-updates
+poetry run update_synapse_database --database-config .ci/sqlite-config.yaml --run-background-updates

 # Create the PostgreSQL database.
-.ci/scripts/postgres_exec.py "CREATE DATABASE synapse"
+poetry run .ci/scripts/postgres_exec.py "CREATE DATABASE synapse"

 echo "+++ Run synapse_port_db against test database"
 # TODO: this invocation of synapse_port_db (and others below) used to be prepended with `coverage run`,
 # but coverage seems unable to find the entrypoints installed by `pip install -e .`.
-synapse_port_db --sqlite-database .ci/test_db.db --postgres-config .ci/postgres-config.yaml
+poetry run synapse_port_db --sqlite-database .ci/test_db.db --postgres-config .ci/postgres-config.yaml

 # We should be able to run twice against the same database.
 echo "+++ Run synapse_port_db a second time"
-synapse_port_db --sqlite-database .ci/test_db.db --postgres-config .ci/postgres-config.yaml
+poetry run synapse_port_db --sqlite-database .ci/test_db.db --postgres-config .ci/postgres-config.yaml

 #####

@@ -48,12 +42,12 @@ echo "--- Prepare empty SQLite database"
 # we do this by deleting the sqlite db, and then doing the same again.
 rm .ci/test_db.db

-update_synapse_database --database-config .ci/sqlite-config.yaml --run-background-updates
+poetry run update_synapse_database --database-config .ci/sqlite-config.yaml --run-background-updates

 # re-create the PostgreSQL database.
-.ci/scripts/postgres_exec.py \
+poetry run .ci/scripts/postgres_exec.py \
  "DROP DATABASE synapse" \
  "CREATE DATABASE synapse"

 echo "+++ Run synapse_port_db against empty database"
-synapse_port_db --sqlite-database .ci/test_db.db --postgres-config .ci/postgres-config.yaml
+poetry run synapse_port_db --sqlite-database .ci/test_db.db --postgres-config .ci/postgres-config.yaml
@@ -4,8 +4,8 @@
 # things to include
 !docker
 !synapse
-!MANIFEST.in
 !README.rst
-!setup.py
+!pyproject.toml
+!poetry.lock

 **/__pycache__
@@ -22,7 +22,7 @@ jobs:
      - name: Setup mdbook
        uses: peaceiris/actions-mdbook@4b5ef36b314c2599664ca107bb8c02412548d79d # v1.1.14
        with:
-          mdbook-version: '0.4.9'
+          mdbook-version: '0.4.17'

      - name: Build the documentation
        # mdbook will only create an index.html if we're including docs/README.md in SUMMARY.md.
@@ -0,0 +1,156 @@
+# People who are freshly `pip install`ing from PyPI will pull in the latest versions of
+# dependencies which match the broad requirements. Since most CI runs are against
+# the locked poetry environment, run specifically against the latest dependencies to
+# know if there's an upcoming breaking change.
+#
+# As an overview this workflow:
+# - checks out develop,
+# - installs from source, pulling in the dependencies like a fresh `pip install` would, and 
+# - runs mypy and test suites in that checkout.
+#
+# Based on the twisted trunk CI job.
+
+name: Latest dependencies
+
+on:
+  schedule:
+    - cron: 0 7 * * *
+  workflow_dispatch:
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  mypy:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      # The dev dependencies aren't exposed in the wheel metadata (at least with current
+      # poetry-core versions), so we install with poetry.
+      - uses: matrix-org/setup-python-poetry@v1
+        with:
+          python-version: "3.x"
+          poetry-version: "1.2.0b1"
+      # Dump installed versions for debugging.
+      - run: poetry run pip list > before.txt
+      # Upgrade all runtime dependencies only. This is intended to mimic a fresh
+      # `pip install matrix-synapse[all]` as closely as possible.
+      - run: poetry update --no-dev
+      - run: poetry run pip list > after.txt && (diff -u before.txt after.txt || true)
+      - run: poetry run mypy
+  trial:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        include:
+          - database: "sqlite"
+          - database: "postgres"
+            postgres-version: "14"
+
+    steps:
+      - uses: actions/checkout@v2
+      - run: sudo apt-get -qq install xmlsec1
+      - name: Set up PostgreSQL ${{ matrix.postgres-version }}
+        if: ${{ matrix.postgres-version }}
+        run: |
+          docker run -d -p 5432:5432 \
+            -e POSTGRES_PASSWORD=postgres \
+            -e POSTGRES_INITDB_ARGS="--lc-collate C --lc-ctype C --encoding UTF8" \
+            postgres:${{ matrix.postgres-version }}
+      - uses: actions/setup-python@v2
+        with:
+          python-version: "3.x"
+      - run: pip install .[all,test]
+      - name: Await PostgreSQL
+        if: ${{ matrix.postgres-version }}
+        timeout-minutes: 2
+        run: until pg_isready -h localhost; do sleep 1; done
+      - run: python -m twisted.trial --jobs=2 tests
+        env:
+          SYNAPSE_POSTGRES: ${{ matrix.database == 'postgres' || '' }}
+          SYNAPSE_POSTGRES_HOST: localhost
+          SYNAPSE_POSTGRES_USER: postgres
+          SYNAPSE_POSTGRES_PASSWORD: postgres
+      - name: Dump logs
+        # Logs are most useful when the command fails, always include them.
+        if: ${{ always() }}
+        # Note: Dumps to workflow logs instead of using actions/upload-artifact
+        #       This keeps logs colocated with failing jobs
+        #       It also ignores find's exit code; this is a best effort affair
+        run: >-
+          find _trial_temp -name '*.log'
+          -exec echo "::group::{}" \;
+          -exec cat {} \;
+          -exec echo "::endgroup::" \;
+          || true
+
+
+  sytest:
+    runs-on: ubuntu-latest
+    container:
+      image: matrixdotorg/sytest-synapse:testing
+      volumes:
+        - ${{ github.workspace }}:/src
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - sytest-tag: focal
+
+          - sytest-tag: focal
+            postgres: postgres
+            workers: workers
+            redis: redis
+    env:
+      POSTGRES: ${{ matrix.postgres && 1}}
+      WORKERS: ${{ matrix.workers && 1 }}
+      REDIS: ${{ matrix.redis && 1 }}
+      BLACKLIST: ${{ matrix.workers && 'synapse-blacklist-with-workers' }}
+
+    steps:
+      - uses: actions/checkout@v2
+      - name: Ensure sytest runs `pip install`
+        # Delete the lockfile so sytest will `pip install` rather than `poetry install`
+        run: rm /src/poetry.lock
+        working-directory: /src
+      - name: Prepare test blacklist
+        run: cat sytest-blacklist .ci/worker-blacklist > synapse-blacklist-with-workers
+      - name: Run SyTest
+        run: /bootstrap.sh synapse
+        working-directory: /src
+      - name: Summarise results.tap
+        if: ${{ always() }}
+        run: /sytest/scripts/tap_to_gha.pl /logs/results.tap
+      - name: Upload SyTest logs
+        uses: actions/upload-artifact@v2
+        if: ${{ always() }}
+        with:
+          name: Sytest Logs - ${{ job.status }} - (${{ join(matrix.*, ', ') }})
+          path: |
+            /logs/results.tap
+            /logs/**/*.log*
+
+
+  # TODO: run complement (as with twisted trunk, see #12473).
+
+  # open an issue if the build fails, so we know about it.
+  open-issue:
+    if: failure()
+    needs:
+      # TODO: should mypy be included here? It feels more brittle than the other two.
+      - mypy
+      - trial
+      - sytest
+
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v2
+      - uses: JasonEtco/create-an-issue@5d9504915f79f9cc6d791934b8ef34f2353dd74d # v2.5.0, 2020-12-06
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        with:
+          update_existing: true
+          filename: .ci/latest_deps_build_failed_issue_template.md
+
@@ -15,25 +15,14 @@ jobs:
    steps:
      - uses: actions/checkout@v2
      - uses: actions/setup-python@v2
-      - run: pip install -e .
+      - run: pip install .
      - run: scripts-dev/generate_sample_config.sh --check
      - run: scripts-dev/config-lint.sh

  lint:
-    runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        toxenv:
-          - "check_codestyle"
-          - "check_isort"
-          - "mypy"
-          - "packaging"
-
-    steps:
-      - uses: actions/checkout@v2
-      - uses: actions/setup-python@v2
-      - run: pip install tox
-      - run: tox -e ${{ matrix.toxenv }}
+    uses: "matrix-org/backend-meta/.github/workflows/python-poetry-ci.yml@v1"
+    with:
+      typechecking-extras: "all"

  lint-crlf:
    runs-on: ubuntu-latest
@@ -72,23 +61,23 @@ jobs:
      matrix:
        python-version: ["3.7", "3.8", "3.9", "3.10"]
        database: ["sqlite"]
-        toxenv: ["py"]
+        extras: ["all"]
        include:
          # Newest Python without optional deps
          - python-version: "3.10"
-            toxenv: "py-noextras"
+            extras: ""

          # Oldest Python with PostgreSQL
          - python-version: "3.7"
            database: "postgres"
            postgres-version: "10"
-            toxenv: "py"
+            extras: "all"

          # Newest Python with newest PostgreSQL
          - python-version: "3.10"
            database: "postgres"
            postgres-version: "14"
-            toxenv: "py"
+            extras: "all"

    steps:
      - uses: actions/checkout@v2
@@ -100,17 +89,16 @@ jobs:
            -e POSTGRES_PASSWORD=postgres \
            -e POSTGRES_INITDB_ARGS="--lc-collate C --lc-ctype C --encoding UTF8" \
            postgres:${{ matrix.postgres-version }}
-      - uses: actions/setup-python@v2
+      - uses: matrix-org/setup-python-poetry@v1
        with:
          python-version: ${{ matrix.python-version }}
-      - run: pip install tox
+          extras: ${{ matrix.extras }}
      - name: Await PostgreSQL
        if: ${{ matrix.postgres-version }}
        timeout-minutes: 2
        run: until pg_isready -h localhost; do sleep 1; done
-      - run: tox -e ${{ matrix.toxenv }}
+      - run: poetry run trial --jobs=2 tests
        env:
-          TRIAL_FLAGS: "--jobs=2"
          SYNAPSE_POSTGRES: ${{ matrix.database == 'postgres' || '' }}
          SYNAPSE_POSTGRES_HOST: localhost
          SYNAPSE_POSTGRES_USER: postgres
@@ -129,6 +117,7 @@ jobs:
          || true

  trial-olddeps:
+    # Note: sqlite only; no postgres
    if: ${{ !cancelled() && !failure() }} # Allow previous steps to be skipped, but not fail
    needs: linting-done
    runs-on: ubuntu-latest
@@ -136,11 +125,11 @@ jobs:
      - uses: actions/checkout@v2
      - name: Test with old deps
        uses: docker://ubuntu:focal # For old python and sqlite
+        # Note: focal seems to be using 3.8, but the oldest is 3.7?
+        # See https://github.com/matrix-org/synapse/issues/12343
        with:
          workdir: /github/workspace
          entrypoint: .ci/scripts/test_old_deps.sh
-        env:
-          TRIAL_FLAGS: "--jobs=2"
      - name: Dump logs
        # Logs are most useful when the command fails, always include them.
        if: ${{ always() }}
@@ -156,23 +145,24 @@ jobs:

  trial-pypy:
    # Very slow; only run if the branch name includes 'pypy'
+    # Note: sqlite only; no postgres. Completely untested since poetry move.
    if: ${{ contains(github.ref, 'pypy') && !failure() && !cancelled() }}
    needs: linting-done
    runs-on: ubuntu-latest
    strategy:
      matrix:
        python-version: ["pypy-3.7"]
+        extras: ["all"]

    steps:
      - uses: actions/checkout@v2
+      # Install libs necessary for PyPy to build binary wheels for dependencies
      - run: sudo apt-get -qq install xmlsec1 libxml2-dev libxslt-dev
-      - uses: actions/setup-python@v2
+      - uses: matrix-org/setup-python-poetry@v1
        with:
          python-version: ${{ matrix.python-version }}
-      - run: pip install tox
-      - run: tox -e py
-        env:
-          TRIAL_FLAGS: "--jobs=2"
+          extras: ${{ matrix.extras }}
+      - run: poetry run trial --jobs=2 tests
      - name: Dump logs
        # Logs are most useful when the command fails, always include them.
        if: ${{ always() }}
@@ -271,9 +261,10 @@ jobs:
    steps:
      - uses: actions/checkout@v2
      - run: sudo apt-get -qq install xmlsec1
-      - uses: actions/setup-python@v2
+      - uses: matrix-org/setup-python-poetry@v1
        with:
-          python-version: "3.9"
+          python-version: ${{ matrix.python-version }}
+          extras: "postgres"
      - run: .ci/scripts/test_export_data_command.sh

  portdb:
@@ -308,9 +299,10 @@ jobs:
    steps:
      - uses: actions/checkout@v2
      - run: sudo apt-get -qq install xmlsec1
-      - uses: actions/setup-python@v2
+      - uses: matrix-org/setup-python-poetry@v1
        with:
          python-version: ${{ matrix.python-version }}
+          extras: "postgres"
      - run: .ci/scripts/test_synapse_port_db.sh

  complement:
@@ -362,27 +354,11 @@ jobs:
            (wget -O - "https://github.com/matrix-org/complement/archive/$BRANCH_NAME.tar.gz" | tar -xz --strip-components=1 -C complement) && break
          done

-      # Build initial Synapse image
-      - run: docker build -t matrixdotorg/synapse:latest -f docker/Dockerfile .
-        working-directory: synapse
-        env:
-          DOCKER_BUILDKIT: 1
-
-      # Build a ready-to-run Synapse image based on the initial image above.
-      # This new image includes a config file, keys for signing and TLS, and
-      # other settings to make it suitable for testing under Complement.
-      - run: docker build -t complement-synapse -f Synapse.Dockerfile .
-        working-directory: complement/dockerfiles
-
-      # Run Complement
      - run: |
          set -o pipefail
-          go test -v -json -tags synapse_blacklist,msc2403,msc2716,msc3030 ./tests/... 2>&1 | gotestfmt
+          COMPLEMENT_DIR=`pwd`/complement synapse/scripts-dev/complement.sh -json 2>&1 | gotestfmt
        shell: bash
        name: Run Complement Tests
-        env:
-          COMPLEMENT_BASE_IMAGE: complement-synapse:latest
-        working-directory: complement

  # a job which marks all the other jobs as complete, thus allowing PRs to be merged.
  tests-done:
@@ -6,16 +6,25 @@ on:

  workflow_dispatch:

+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
 jobs:
  mypy:
    runs-on: ubuntu-latest

    steps:
      - uses: actions/checkout@v2
-      - uses: actions/setup-python@v2
-      - run: .ci/patch_for_twisted_trunk.sh
-      - run: pip install tox
-      - run: tox -e mypy
+      - uses: matrix-org/setup-python-poetry@v1
+        with:
+          python-version: "3.x"
+          extras: "all"
+      - run: |
+          poetry remove twisted
+          poetry add --extras tls git+https://github.com/twisted/twisted.git#trunk
+          poetry install --no-interaction --extras "all test"
+      - run: poetry run mypy

  trial:
    runs-on: ubuntu-latest
@@ -23,14 +32,15 @@ jobs:
    steps:
      - uses: actions/checkout@v2
      - run: sudo apt-get -qq install xmlsec1
-      - uses: actions/setup-python@v2
+      - uses: matrix-org/setup-python-poetry@v1
        with:
-          python-version: 3.7
-      - run: .ci/patch_for_twisted_trunk.sh
-      - run: pip install tox
-      - run: tox -e py
-        env:
-          TRIAL_FLAGS: "--jobs=2"
+          python-version: "3.x"
+          extras: "all test"
+      - run: |
+          poetry remove twisted
+          poetry add --extras tls git+https://github.com/twisted/twisted.git#trunk
+          poetry install --no-interaction --extras "all test"
+      - run: poetry run trial --jobs 2 tests

      - name: Dump logs
        # Logs are most useful when the command fails, always include them.
@@ -55,11 +65,23 @@ jobs:
    steps:
      - uses: actions/checkout@v2
      - name: Patch dependencies
-        run: .ci/patch_for_twisted_trunk.sh
+        # Note: The poetry commands want to create a virtualenv in /src/.venv/,
+        #       but the sytest-synapse container expects it to be in /venv/.
+        #       We symlink it before running poetry so that poetry actually
+        #       ends up installing to `/venv`.
+        run: |
+          ln -s -T /venv /src/.venv
+          poetry remove twisted
+          poetry add --extras tls git+https://github.com/twisted/twisted.git#trunk
+          poetry install --no-interaction --extras "all test"
        working-directory: /src
      - name: Run SyTest
        run: /bootstrap.sh synapse
        working-directory: /src
+        env:
+          # Use offline mode to avoid reinstalling the pinned version of
+          # twisted.
+          OFFLINE: 1
      - name: Summarise results.tap
        if: ${{ always() }}
        run: /sytest/scripts/tap_to_gha.pl /logs/results.tap
@@ -15,6 +15,9 @@ _trial_temp*/
 .DS_Store
 __pycache__/

+# We do want the poetry lockfile.
+!poetry.lock
+
 # stuff that is likely to exist when you run a server locally
 /*.db
 /*.log
@@ -30,6 +33,9 @@ __pycache__/
 /media_store/
 /uploads

+# For direnv users
+/.envrc
+
 # IDEs
 /.idea/
 /.ropeproject/
@@ -1,54 +0,0 @@
-include LICENSE
-include VERSION
-include *.rst
-include *.md
-include demo/README
-include demo/demo.tls.dh
-include demo/*.py
-include demo/*.sh
-
-include synapse/py.typed
-recursive-include synapse/storage *.sql
-recursive-include synapse/storage *.sql.postgres
-recursive-include synapse/storage *.sql.sqlite
-recursive-include synapse/storage *.py
-recursive-include synapse/storage *.txt
-recursive-include synapse/storage *.md
-
-recursive-include docs *
-recursive-include scripts-dev *
-recursive-include synapse *.pyi
-recursive-include tests *.py
-recursive-include tests *.pem
-recursive-include tests *.p8
-recursive-include tests *.crt
-recursive-include tests *.key
-
-recursive-include synapse/res *
-recursive-include synapse/static *.css
-recursive-include synapse/static *.gif
-recursive-include synapse/static *.html
-recursive-include synapse/static *.js
-
-exclude .codecov.yml
-exclude .coveragerc
-exclude .dockerignore
-exclude .editorconfig
-exclude Dockerfile
-exclude mypy.ini
-exclude sytest-blacklist
-exclude test_postgresql.sh
-
-include book.toml
-include pyproject.toml
-recursive-include changelog.d *
-
-include .flake8
-prune .circleci
-prune .github
-prune .ci
-prune contrib
-prune debian
-prune demo/etc
-prune docker
-prune stubs
@@ -293,39 +293,42 @@ directory of your choice::
    git clone https://github.com/matrix-org/synapse.git
    cd synapse

-Synapse has a number of external dependencies, that are easiest
-to install using pip and a virtualenv::
+Synapse has a number of external dependencies. We maintain a fixed development
+environment using [poetry](https://python-poetry.org/). First, install poetry. We recommend

-    python3 -m venv ./env
-    source ./env/bin/activate
-    pip install -e ".[all,dev]"
+    pip install --user pipx
+    pipx install poetry
+
+as described `here <https://python-poetry.org/docs/#installing-with-pipx>`_.
+(See `poetry's installation docs <https://python-poetry.org/docs/#installation>`
+for other installation methods.) Then ask poetry to create a virtual environment
+from the project and install Synapse's dependencies::
+
+    poetry install --extras "all test"

 This will run a process of downloading and installing all the needed
-dependencies into a virtual env. If any dependencies fail to install,
-try installing the failing modules individually::
-
-    pip install -e "module-name"
+dependencies into a virtual env.

 We recommend using the demo which starts 3 federated instances running on ports `8080` - `8082`

-    ./demo/start.sh
+    poetry run ./demo/start.sh

-(to stop, you can use `./demo/stop.sh`)
+(to stop, you can use `poetry run ./demo/stop.sh`)

-See the [demo documentation](https://matrix-org.github.io/synapse/develop/development/demo.html)
+See the `demo documentation <https://matrix-org.github.io/synapse/develop/development/demo.html>`_
 for more information.

 If you just want to start a single instance of the app and run it directly::

    # Create the homeserver.yaml config once
-    python -m synapse.app.homeserver \
+    poetry run synapse_homeserver \
      --server-name my.domain.name \
      --config-path homeserver.yaml \
      --generate-config \
      --report-stats=[yes|no]

    # Start the app
-    python -m synapse.app.homeserver --config-path homeserver.yaml
+    poetry run synapse_homeserver --config-path homeserver.yaml


 Running the unit tests
@@ -334,7 +337,7 @@ Running the unit tests
 After getting up and running, you may wish to run Synapse's unit tests to
 check that everything is installed correctly::

-    trial tests
+    poetry run trial tests

 This should end with a 'PASSED' result (note that exact numbers will
 differ)::
@@ -1 +0,0 @@
-Rename `shared_rooms` to `mutual_rooms` (MSC2666), as per proposal changes.
@@ -1 +0,0 @@
-Remove check on `update_user_directory` for shared rooms handler (MSC2666), and update/expand documentation.
@@ -1 +0,0 @@
-Refactor `create_new_client_event` to use a new parameter, `state_event_ids`, which accurately describes the usage with [MSC2716](https://github.com/matrix-org/matrix-doc/pull/2716) instead of abusing `auth_event_ids`.
@@ -1 +0,0 @@
-Fix a long-standing bug which caused the `/_matrix/federation/v1/state` and `.../state_ids` endpoints to return incorrect or invalid data when called for an event which we have stored as an "outlier".
@@ -1 +0,0 @@
-Refuse to start if registration is enabled without email, captcha, or token-based verification unless new config flag `enable_registration_without_verification` is set.
@@ -1 +0,0 @@
-Allow modules to store already existing 3PID associations.
@@ -1 +0,0 @@
-Add tests for database transaction callbacks.
@@ -1 +0,0 @@
-Handle cancellation in `DatabasePool.runInteraction()`.
@@ -1 +0,0 @@
-Add missing type hints for cache storage.
@@ -1 +0,0 @@
-Clean-up logic around rebasing URLs for URL image previews.
@@ -1 +0,0 @@
-Add type hints to tests files.
@@ -1 +0,0 @@
-Use the `ignored_users` table in additional places instead of re-parsing the account data.
@@ -1 +0,0 @@
-Fix a long-standing bug where events from ignored users were still considered for relations.
@@ -1 +0,0 @@
-Fix a bug introduced in v1.53.0 where an unnecessary query could be performed when fetching bundled aggregations for threads.
@@ -1 +0,0 @@
-Fix the link to the module documentation in the legacy spam checker warning message.
@@ -1 +0,0 @@
-Fix a long-standing bug where events from ignored users were still considered for relations.
@@ -1 +0,0 @@
-Refactor the relations endpoints to add a `RelationsHandler`.
@@ -1 +0,0 @@
-Add type hints to tests files.
@@ -1 +0,0 @@
-Generate announcement links in the release script.
@@ -1 +0,0 @@
-Remove incorrect prefixes in the worker documentation for some endpoints.
@@ -1 +0,0 @@
-Improve error message when dependencies check finds a broken installation.
@@ -1 +0,0 @@
-Correct `check_username_for_spam` annotations and docs.
@@ -1 +0,0 @@
-Add missing type hints for storage.
@@ -1 +0,0 @@
-Allow registering admin users using the module API. Contributed by Famedly.
@@ -1 +0,0 @@
-Add missing type hints for storage.
@@ -1 +0,0 @@
-Add type hints to tests files.
@@ -1 +0,0 @@
-Compress metrics HTTP resource when enabled. Contributed by Nick @ Beeper.
@@ -1 +0,0 @@
-Fix a bug introduced in Synapse 1.52 where admins could not deactivate and GDPR-erase a user if Synapse was configured with limits on avatars.
@@ -1 +0,0 @@
-Refuse to start if DB has non-`C` locale, unless config flag `allow_unsafe_db_locale` is set to true.
@@ -1 +0,0 @@
-Optionally include account validity expiration information to experimental [MSC3720](https://github.com/matrix-org/matrix-doc/pull/3720) account status responses.
@@ -1 +0,0 @@
-Use type stubs for `psycopg2`.
@@ -1 +0,0 @@
-Add a new cache `_get_membership_from_event_id` to speed up push rule calculations in large rooms.
@@ -1 +0,0 @@
-Corrected Authentik OpenID typo, added helpful note for troubleshooting. Contributed by @IronTooch.
@@ -1 +0,0 @@
-HAProxy reverse proxy guide update to stop sending IPv4-mapped address to homeserver. Contributed by @villepeh.
@@ -1 +0,0 @@
-Re-enable Complement concurrency in CI.
@@ -1 +0,0 @@
-Fix a long-standing bug where events from ignored users were still considered for relations.
@@ -1 +0,0 @@
-Refuse to start if DB has non-`C` locale, unless config flag `allow_unsafe_db_locale` is set to true.
@@ -1 +0,0 @@
-Remove unused test utilities.
@@ -1 +0,0 @@
-Enhance logging for inbound federation events.
@@ -1 +0,0 @@
-Refactor `create_new_client_event` to use a new parameter, `state_event_ids`, which accurately describes the usage with [MSC2716](https://github.com/matrix-org/matrix-doc/pull/2716) instead of abusing `auth_event_ids`.
@@ -1 +0,0 @@
-Improve type annotations for `execute_values`.
@@ -1 +0,0 @@
-Fix compatibility with the recently-released Jinja 3.1.
@@ -1 +0,0 @@
-Avoid trying to calculate the state at outlier events.
@@ -1 +0,0 @@
-Document the behaviour of `LoggingTransaction.call_after` and `LoggingTransaction.call_on_exception` methods when transactions are retried.
@@ -1 +0,0 @@
-Bump the version of `black` for compatibility with the latest `click` release.
@@ -0,0 +1 @@
+Fix scripts-dev to pass typechecking.
@@ -0,0 +1 @@
+Add a module API to allow modules to change actions for existing push rules of local users.
@@ -0,0 +1 @@
+Use supervisord to supervise Postgres and Caddy in the Complement image to reduce restart time.
@@ -0,0 +1 @@
+Add some type hints to datastore.
@@ -0,0 +1 @@
+Use `make_awaitable` instead of `defer.succeed` for return values of mocks in tests.
@@ -0,0 +1 @@
+Add new `enable_registration_token_3pid_bypass` configuration option to allow registrations via token as an alternative to verifying a 3pid.
@@ -0,0 +1 @@
+Remove unused `# type: ignore`s.
@@ -0,0 +1 @@
+Explicitly opt-in to using [BuildKit-specific features](https://github.com/moby/buildkit/blob/master/frontend/dockerfile/docs/syntax.md) in the Dockerfile. This fixes issues with building images in some GitLab CI environments.
@@ -0,0 +1 @@
+Fix a bug where attempting to send a large amount of read receipts to an application service all at once would result in duplicate content and abnormally high memory usage. Contributed by Brad & Nick @ Beeper.
@@ -0,0 +1 @@
+Consistently check if an object is a `frozendict`.
@@ -0,0 +1 @@
+Add the Synapse function `types.map_username_to_mxid_localpart` to the Module API.
@@ -30,9 +30,19 @@ case $(dpkg-architecture -q DEB_HOST_ARCH) in
        ;;
 esac

-# Use --builtin-venv to use the better `venv` module from CPython 3.4+ rather
-# than the 2/3 compatible `virtualenv`.
+# Manually install Poetry and export a pip-compatible `requirements.txt`
+# We need a Poetry pre-release as the export command is buggy in < 1.2
+TEMP_VENV="$(mktemp -d)"
+python3 -m venv "$TEMP_VENV"
+source "$TEMP_VENV/bin/activate"
+pip install -U pip
+pip install poetry==1.2.0b1
+poetry export --extras all --extras test -o exported_requirements.txt
+deactivate
+rm -rf "$TEMP_VENV"

+# Use --no-deps to only install pinned versions in exported_requirements.txt,
+# and to avoid https://github.com/pypa/pip/issues/9644
 dh_virtualenv \
    --install-suffix "matrix-synapse" \
    --builtin-venv \
@@ -41,9 +51,11 @@ dh_virtualenv \
    --preinstall="lxml" \
    --preinstall="mock" \
    --preinstall="wheel" \
+    --extra-pip-arg="--no-deps" \
    --extra-pip-arg="--no-cache-dir" \
    --extra-pip-arg="--compile" \
-    --extras="all,systemd,test"
+    --extras="all,systemd,test" \
+    --requirements="exported_requirements.txt"

 PACKAGE_BUILD_DIR="debian/matrix-synapse-py3"
 VIRTUALENV_DIR="${PACKAGE_BUILD_DIR}${DH_VIRTUALENV_INSTALL_ROOT}/matrix-synapse"
@@ -1,3 +1,46 @@
+matrix-synapse-py3 (1.58.0~rc2) stable; urgency=medium
+
+  * New Synapse release 1.58.0rc2.
+
+ -- Synapse Packaging team <packages@matrix.org>  Tue, 26 Apr 2022 17:14:56 +0100
+
+matrix-synapse-py3 (1.58.0~rc1) stable; urgency=medium
+
+  * Use poetry to manage the bundled virtualenv included with this package.
+  * New Synapse release 1.58.0rc1.
+
+ -- Synapse Packaging team <packages@matrix.org>  Tue, 26 Apr 2022 11:15:20 +0100
+
+matrix-synapse-py3 (1.57.1) stable; urgency=medium
+
+  * New synapse release 1.57.1.
+
+ -- Synapse Packaging team <packages@matrix.org>  Wed, 20 Apr 2022 15:27:21 +0100
+
+matrix-synapse-py3 (1.57.0) stable; urgency=medium
+
+  * New synapse release 1.57.0.
+
+ -- Synapse Packaging team <packages@matrix.org>  Tue, 19 Apr 2022 10:58:42 +0100
+
+matrix-synapse-py3 (1.57.0~rc1) stable; urgency=medium
+
+  * New synapse release 1.57.0~rc1.
+
+ -- Synapse Packaging team <packages@matrix.org>  Tue, 12 Apr 2022 13:36:25 +0100
+
+matrix-synapse-py3 (1.56.0) stable; urgency=medium
+
+  * New synapse release 1.56.0.
+
+ -- Synapse Packaging team <packages@matrix.org>  Tue, 05 Apr 2022 12:38:39 +0100
+
+matrix-synapse-py3 (1.56.0~rc1) stable; urgency=medium
+
+  * New synapse release 1.56.0~rc1.
+
+ -- Synapse Packaging team <packages@matrix.org>  Tue, 29 Mar 2022 10:40:50 +0100
+
 matrix-synapse-py3 (1.55.2) stable; urgency=medium

  * New synapse release 1.55.2.
@@ -0,0 +1 @@
+exported_requirements.txt
@@ -1,3 +1,4 @@
+# syntax=docker/dockerfile:1
 # Dockerfile to build the matrixdotorg/synapse docker images.
 #
 # Note that it uses features which are only available in BuildKit - see
@@ -14,20 +15,61 @@
 #    DOCKER_BUILDKIT=1 docker build -f docker/Dockerfile --build-arg PYTHON_VERSION=3.10 .
 #

+# Irritatingly, there is no blessed guide on how to distribute an application with its
+# poetry-managed environment in a docker image. We have opted for
+# `poetry export | pip install -r /dev/stdin`, but there are known bugs in
+# in `poetry export` whose fixes (scheduled for poetry 1.2) have yet to be released.
+# In case we get bitten by those bugs in the future, the recommendations here might
+# be useful:
+#     https://github.com/python-poetry/poetry/discussions/1879#discussioncomment-216865
+#     https://stackoverflow.com/questions/53835198/integrating-python-poetry-with-docker?answertab=scoredesc
+
+
+
 ARG PYTHON_VERSION=3.9

 ###
-### Stage 0: builder
+### Stage 0: generate requirements.txt
+###
+FROM docker.io/python:${PYTHON_VERSION}-slim as requirements
+
+# RUN --mount is specific to buildkit and is documented at
+# https://github.com/moby/buildkit/blob/master/frontend/dockerfile/docs/syntax.md#build-mounts-run---mount.
+# Here we use it to set up a cache for apt (and below for pip), to improve
+# rebuild speeds on slow connections.
+RUN \
+   --mount=type=cache,target=/var/cache/apt,sharing=locked \
+   --mount=type=cache,target=/var/lib/apt,sharing=locked \
+ apt-get update && apt-get install -y git \
+    && rm -rf /var/lib/apt/lists/*
+
+# We install poetry in its own build stage to avoid its dependencies conflicting with
+# synapse's dependencies.
+# We use a specific commit from poetry's master branch instead of our usual 1.1.12,
+# to incorporate fixes to some bugs in `poetry export`. This commit corresponds to
+#    https://github.com/python-poetry/poetry/pull/5156 and
+#    https://github.com/python-poetry/poetry/issues/5141 ;
+# without it, we generate a requirements.txt with incorrect environment markers,
+# which causes necessary packages to be omitted when we `pip install`.
+#
+# NB: In poetry 1.2 `poetry export` will be moved into a plugin; we'll need to also
+# pip install poetry-plugin-export (https://github.com/python-poetry/poetry-plugin-export).
+RUN --mount=type=cache,target=/root/.cache/pip \
+  pip install --user git+https://github.com/python-poetry/poetry.git@fb13b3a676f476177f7937ffa480ee5cff9a90a5
+
+WORKDIR /synapse
+
+# Copy just what we need to run `poetry export`...
+COPY pyproject.toml poetry.lock /synapse/
+
+RUN /root/.local/bin/poetry export --extras all -o /synapse/requirements.txt
+
+###
+### Stage 1: builder
 ###
 FROM docker.io/python:${PYTHON_VERSION}-slim as builder

 # install the OS build deps
-#
-# RUN --mount is specific to buildkit and is documented at
-# https://github.com/moby/buildkit/blob/master/frontend/dockerfile/docs/syntax.md#build-mounts-run---mount.
-# Here we use it to set up a cache for apt, to improve rebuild speeds on
-# slow connections.
-#
 RUN \
   --mount=type=cache,target=/var/cache/apt,sharing=locked \
   --mount=type=cache,target=/var/lib/apt,sharing=locked \
@@ -45,30 +87,25 @@ RUN \
    zlib1g-dev \
    && rm -rf /var/lib/apt/lists/*

-# Copy just what we need to pip install
-COPY MANIFEST.in README.rst setup.py /synapse/
-COPY synapse/__init__.py /synapse/synapse/__init__.py
-COPY synapse/python_dependencies.py /synapse/synapse/python_dependencies.py
-
 # To speed up rebuilds, install all of the dependencies before we copy over
-# the whole synapse project so that we this layer in the Docker cache can be
+# the whole synapse project, so that this layer in the Docker cache can be
 # used while you develop on the source
 #
-# This is aiming at installing the `install_requires` and `extras_require` from `setup.py`
+# This is aiming at installing the `[tool.poetry.depdendencies]` from pyproject.toml.
+COPY --from=requirements /synapse/requirements.txt /synapse/
 RUN --mount=type=cache,target=/root/.cache/pip \
-  pip install --prefix="/install" --no-warn-script-location \
-    /synapse[all]
+  pip install --prefix="/install" --no-deps --no-warn-script-location -r /synapse/requirements.txt

-# Copy over the rest of the project
+# Copy over the rest of the synapse source code.
 COPY synapse /synapse/synapse/
+# ... and what we need to `pip install`.
+COPY pyproject.toml README.rst /synapse/

-# Install the synapse package itself and all of its children packages.
-#
-# This is aiming at installing only the `packages=find_packages(...)` from `setup.py
+# Install the synapse package itself.
 RUN pip install --prefix="/install" --no-deps --no-warn-script-location /synapse

 ###
-### Stage 1: runtime
+### Stage 2: runtime
 ###

 FROM docker.io/python:${PYTHON_VERSION}-slim
@@ -1,30 +0,0 @@
-# Use the Sytest image that comes with a lot of the build dependencies
-# pre-installed
-FROM matrixdotorg/sytest:focal
-
-# The Sytest image doesn't come with python, so install that
-RUN apt-get update && apt-get -qq install -y python3 python3-dev python3-pip
-
-# We need tox to run the tests in run_pg_tests.sh
-RUN python3 -m pip install tox
-
-# Initialise the db
-RUN su -c '/usr/lib/postgresql/10/bin/initdb -D /var/lib/postgresql/data -E "UTF-8" --lc-collate="C.UTF-8" --lc-ctype="C.UTF-8" --username=postgres' postgres
-
-# Add a user with our UID and GID so that files get created on the host owned
-# by us, not root.
-ARG UID
-ARG GID
-RUN groupadd --gid $GID user
-RUN useradd --uid $UID --gid $GID --groups sudo --no-create-home user
-
-# Ensure we can start postgres by sudo-ing as the postgres user.
-RUN apt-get update && apt-get -qq install -y sudo
-RUN echo "user ALL=(ALL) NOPASSWD: ALL" >> /etc/sudoers
-
-ADD run_pg_tests.sh /run_pg_tests.sh
-# Use the "exec form" of ENTRYPOINT (https://docs.docker.com/engine/reference/builder/#entrypoint)
-# so that we can `docker run` this container and pass arguments to pg_tests.sh
-ENTRYPOINT ["/run_pg_tests.sh"]
-
-USER user
@@ -2,25 +2,36 @@
 FROM matrixdotorg/synapse

 # Install deps
-RUN apt-get update
-RUN apt-get install -y supervisor redis nginx
+RUN \
+   --mount=type=cache,target=/var/cache/apt,sharing=locked \
+   --mount=type=cache,target=/var/lib/apt,sharing=locked \
+  apt-get update && \
+  DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
+     redis-server nginx-light

-# Remove the default nginx sites
+# Install supervisord with pip instead of apt, to avoid installing a second
+# copy of python.
+RUN --mount=type=cache,target=/root/.cache/pip \
+    pip install supervisor~=4.2
+
+# Disable the default nginx sites
 RUN rm /etc/nginx/sites-enabled/default

 # Copy Synapse worker, nginx and supervisord configuration template files
 COPY ./docker/conf-workers/* /conf/

+# Copy a script to prefix log lines with the supervisor program name
+COPY ./docker/prefix-log /usr/local/bin/
+
 # Expose nginx listener port
 EXPOSE 8080/tcp

-# Volume for user-editable config files, logs etc.
-VOLUME ["/data"]
-
 # A script to read environment variables and create the necessary
 # files to run the desired worker configuration. Will start supervisord.
 COPY ./docker/configure_workers_and_start.py /configure_workers_and_start.py
 ENTRYPOINT ["/configure_workers_and_start.py"]

+# Replace the healthcheck with one which checks *all* the workers. The script
+# is generated by configure_workers_and_start.py.
 HEALTHCHECK --start-period=5s --interval=15s --timeout=5s \
    CMD /bin/sh /healthcheck.sh
@@ -10,10 +10,10 @@ Note that running Synapse's unit tests from within the docker image is not suppo

 ## Testing with SQLite and single-process Synapse

-> Note that `scripts-dev/complement.sh` is a script that will automatically build 
+> Note that `scripts-dev/complement.sh` is a script that will automatically build
 > and run an SQLite-based, single-process of Synapse against Complement.

-The instructions below will set up Complement testing for a single-process, 
+The instructions below will set up Complement testing for a single-process,
 SQLite-based Synapse deployment.

 Start by building the base Synapse docker image. If you wish to run tests with the latest
@@ -26,23 +26,22 @@ docker build -t matrixdotorg/synapse -f docker/Dockerfile .

 This will build an image with the tag `matrixdotorg/synapse`.

-Next, build the Synapse image for Complement. You will need a local checkout 
-of Complement. Change to the root of your Complement checkout and run:
+Next, build the Synapse image for Complement.

 ```sh
-docker build -t complement-synapse -f "dockerfiles/Synapse.Dockerfile" dockerfiles
+docker build -t complement-synapse -f "docker/complement/Dockerfile" docker/complement
 ```

-This will build an image with the tag `complement-synapse`, which can be handed to 
-Complement for testing via the `COMPLEMENT_BASE_IMAGE` environment variable. Refer to 
-[Complement's documentation](https://github.com/matrix-org/complement/#running) for 
+This will build an image with the tag `complement-synapse`, which can be handed to
+Complement for testing via the `COMPLEMENT_BASE_IMAGE` environment variable. Refer to
+[Complement's documentation](https://github.com/matrix-org/complement/#running) for
 how to run the tests, as well as the various available command line flags.

 ## Testing with PostgreSQL and single or multi-process Synapse

-The above docker image only supports running Synapse with SQLite and in a 
-single-process topology. The following instructions are used to build a Synapse image for 
-Complement that supports either single or multi-process topology with a PostgreSQL 
+The above docker image only supports running Synapse with SQLite and in a
+single-process topology. The following instructions are used to build a Synapse image for
+Complement that supports either single or multi-process topology with a PostgreSQL
 database backend.

 As with the single-process image, build the base Synapse docker image. If you wish to run
@@ -55,7 +54,7 @@ docker build -t matrixdotorg/synapse -f docker/Dockerfile .

 This will build an image with the tag `matrixdotorg/synapse`.

-Next, we build a new image with worker support based on `matrixdotorg/synapse:latest`. 
+Next, we build a new image with worker support based on `matrixdotorg/synapse:latest`.
 Again, from the root of the repository:

 ```sh
@@ -64,21 +63,20 @@ docker build -t matrixdotorg/synapse-workers -f docker/Dockerfile-workers .

 This will build an image with the tag` matrixdotorg/synapse-workers`.

-It's worth noting at this point that this image is fully functional, and 
-can be used for testing against locally. See instructions for using the container 
+It's worth noting at this point that this image is fully functional, and
+can be used for testing against locally. See instructions for using the container
 under
 [Running the Dockerfile-worker image standalone](#running-the-dockerfile-worker-image-standalone)
 below.

 Finally, build the Synapse image for Complement, which is based on
-`matrixdotorg/synapse-workers`. You will need a local checkout of Complement. Change to
-the root of your Complement checkout and run:
+`matrixdotorg/synapse-workers`.

 ```sh
-docker build -t matrixdotorg/complement-synapse-workers -f dockerfiles/SynapseWorkers.Dockerfile dockerfiles
+docker build -t matrixdotorg/complement-synapse-workers -f docker/complement/SynapseWorkers.Dockerfile docker/complement
 ```

-This will build an image with the tag `complement-synapse`, which can be handed to
+This will build an image with the tag `complement-synapse-workers`, which can be handed to
 Complement for testing via the `COMPLEMENT_BASE_IMAGE` environment variable. Refer to
 [Complement's documentation](https://github.com/matrix-org/complement/#running) for
 how to run the tests, as well as the various available command line flags.
@@ -91,10 +89,10 @@ bundling all necessary components together for a workerised homeserver instance.

 This includes any desired Synapse worker processes, a nginx to route traffic accordingly,
 a redis for worker communication and a supervisord instance to start up and monitor all
-processes. You will need to provide your own postgres container to connect to, and TLS 
+processes. You will need to provide your own postgres container to connect to, and TLS
 is not handled by the container.

-Once you've built the image using the above instructions, you can run it. Be sure 
+Once you've built the image using the above instructions, you can run it. Be sure
 you've set up a volume according to the [usual Synapse docker instructions](README.md).
 Then run something along the lines of:

@@ -112,7 +110,7 @@ docker run -d --name synapse \
    matrixdotorg/synapse-workers
 ```

-...substituting `POSTGRES*` variables for those that match a postgres host you have 
+...substituting `POSTGRES*` variables for those that match a postgres host you have
 available (usually a running postgres docker container).

 The `SYNAPSE_WORKER_TYPES` environment variable is a comma-separated list of workers to
@@ -130,11 +128,11 @@ Otherwise, `SYNAPSE_WORKER_TYPES` can either be left empty or unset to spawn no
 (leaving only the main process). The container is configured to use redis-based worker
 mode.

-Logs for workers and the main process are logged to stdout and can be viewed with 
-standard `docker logs` tooling. Worker logs contain their worker name 
+Logs for workers and the main process are logged to stdout and can be viewed with
+standard `docker logs` tooling. Worker logs contain their worker name
 after the timestamp.

 Setting `SYNAPSE_WORKERS_WRITE_LOGS_TO_DISK=1` will cause worker logs to be written to
 `<data_dir>/logs/<worker_name>.log`. Logs are kept for 1 week and rotate every day at 00:
-00, according to the container's clock. Logging for the main process must still be 
+00, according to the container's clock. Logging for the main process must still be
 configured by modifying the homeserver's log config in your Synapse data volume.
@@ -0,0 +1,22 @@
+# A dockerfile which builds an image suitable for testing Synapse under
+# complement.
+
+ARG SYNAPSE_VERSION=latest
+
+FROM matrixdotorg/synapse:${SYNAPSE_VERSION}
+
+ENV SERVER_NAME=localhost
+
+COPY conf/* /conf/
+
+# generate a signing key
+RUN generate_signing_key -o /conf/server.signing.key
+
+WORKDIR /data
+
+EXPOSE 8008 8448
+
+ENTRYPOINT ["/conf/start.sh"]
+
+HEALTHCHECK --start-period=5s --interval=1s --timeout=1s \
+    CMD curl -fSs http://localhost:8008/health || exit 1
@@ -0,0 +1 @@
+Stuff for building the docker image used for testing under complement.
@@ -0,0 +1,50 @@
+# This dockerfile builds on top of 'docker/Dockerfile-worker' in matrix-org/synapse
+# by including a built-in postgres instance, as well as setting up the homeserver so
+# that it is ready for testing via Complement.
+#
+# Instructions for building this image from those it depends on is detailed in this guide:
+# https://github.com/matrix-org/synapse/blob/develop/docker/README-testing.md#testing-with-postgresql-and-single-or-multi-process-synapse
+FROM matrixdotorg/synapse-workers
+
+# Download a caddy server to stand in front of nginx and terminate TLS using Complement's
+# custom CA.
+# We include this near the top of the file in order to cache the result.
+RUN curl -OL "https://github.com/caddyserver/caddy/releases/download/v2.3.0/caddy_2.3.0_linux_amd64.tar.gz" && \
+  tar xzf caddy_2.3.0_linux_amd64.tar.gz && rm caddy_2.3.0_linux_amd64.tar.gz && mv caddy /root
+
+# Install postgresql
+RUN apt-get update && \
+  DEBIAN_FRONTEND=noninteractive apt-get install --no-install-recommends -y postgresql-13
+
+# Configure a user and create a database for Synapse
+RUN pg_ctlcluster 13 main start &&  su postgres -c "echo \
+ \"ALTER USER postgres PASSWORD 'somesecret'; \
+ CREATE DATABASE synapse \
+  ENCODING 'UTF8' \
+  LC_COLLATE='C' \
+  LC_CTYPE='C' \
+  template=template0;\" | psql" && pg_ctlcluster 13 main stop
+
+# Modify the shared homeserver config with postgres support, certificate setup
+# and the disabling of rate-limiting
+COPY conf-workers/workers-shared.yaml /conf/workers/shared.yaml
+
+WORKDIR /data
+
+# Copy the caddy config
+COPY conf-workers/caddy.complement.json /root/caddy.json
+
+COPY conf-workers/postgres.supervisord.conf /etc/supervisor/conf.d/postgres.conf
+COPY conf-workers/caddy.supervisord.conf /etc/supervisor/conf.d/caddy.conf
+
+# Copy the entrypoint
+COPY conf-workers/start-complement-synapse-workers.sh /
+
+# Expose caddy's listener ports
+EXPOSE 8008 8448
+
+ENTRYPOINT ["/start-complement-synapse-workers.sh"]
+
+# Update the healthcheck to have a shorter check interval
+HEALTHCHECK --start-period=5s --interval=1s --timeout=1s \
+    CMD /bin/sh /healthcheck.sh
@@ -0,0 +1,72 @@
+{
+    "apps": {
+      "http": {
+        "servers": {
+          "srv0": {
+            "listen": [
+              ":8448"
+            ],
+            "routes": [
+              {
+                "match": [
+                  {
+                    "host": [
+                      "{{ server_name }}"
+                    ]
+                  }
+                ],
+                "handle": [
+                  {
+                    "handler": "subroute",
+                    "routes": [
+                      {
+                        "handle": [
+                          {
+                            "handler": "reverse_proxy",
+                            "upstreams": [
+                              {
+                                "dial": "localhost:8008"
+                              }
+                            ]
+                          }
+                        ]
+                      }
+                    ]
+                  }
+                ],
+                "terminal": true
+              }
+            ]
+          }
+        }
+      },
+      "tls": {
+        "automation": {
+          "policies": [
+            {
+              "subjects": [
+                "{{ server_name }}"
+              ],
+              "issuers": [
+                {
+                  "module": "internal"
+                }
+              ],
+              "on_demand": true
+            }
+          ]
+        }
+      },
+      "pki": {
+        "certificate_authorities": {
+          "local": {
+            "name": "Complement CA",
+            "root": {
+              "certificate": "/complement/ca/ca.crt",
+              "private_key": "/complement/ca/ca.key"
+            }
+          }
+        }
+      }
+    }
+  }
@@ -0,0 +1,7 @@
+[program:caddy]
+command=/usr/local/bin/prefix-log /root/caddy run --config /root/caddy.json
+autorestart=unexpected
+stdout_logfile=/dev/stdout
+stdout_logfile_maxbytes=0
+stderr_logfile=/dev/stderr
+stderr_logfile_maxbytes=0
@@ -0,0 +1,16 @@
+[program:postgres]
+command=/usr/local/bin/prefix-log /usr/bin/pg_ctlcluster 13 main start --foreground
+
+# Lower priority number = starts first
+priority=1
+
+autorestart=unexpected
+stdout_logfile=/dev/stdout
+stdout_logfile_maxbytes=0
+stderr_logfile=/dev/stderr
+stderr_logfile_maxbytes=0
+
+# Use 'Fast Shutdown' mode which aborts current transactions and closes connections quickly.
+# (Default (TERM) is 'Smart Shutdown' which stops accepting new connections but
+#  lets existing connections close gracefully.)
+stopsignal=INT
@@ -0,0 +1,44 @@
+#!/bin/bash
+#
+# Default ENTRYPOINT for the docker image used for testing synapse with workers under complement
+
+set -e
+
+function log {
+    d=$(date +"%Y-%m-%d %H:%M:%S,%3N")
+    echo "$d $@"
+}
+
+# Replace the server name in the caddy config
+sed -i "s/{{ server_name }}/${SERVER_NAME}/g" /root/caddy.json
+
+# Set the server name of the homeserver
+export SYNAPSE_SERVER_NAME=${SERVER_NAME}
+
+# No need to report stats here
+export SYNAPSE_REPORT_STATS=no
+
+# Set postgres authentication details which will be placed in the homeserver config file
+export POSTGRES_PASSWORD=somesecret
+export POSTGRES_USER=postgres
+export POSTGRES_HOST=localhost
+
+# Specify the workers to test with
+export SYNAPSE_WORKER_TYPES="\
+    event_persister, \
+    event_persister, \
+    background_worker, \
+    frontend_proxy, \
+    event_creator, \
+    user_dir, \
+    media_repository, \
+    federation_inbound, \
+    federation_reader, \
+    federation_sender, \
+    synchrotron, \
+    appservice, \
+    pusher"
+
+# Run the script that writes the necessary config files and starts supervisord, which in turn
+# starts everything else
+exec /configure_workers_and_start.py
@@ -0,0 +1,72 @@
+## Server ##
+report_stats: False
+trusted_key_servers: []
+enable_registration: true
+enable_registration_without_verification: true
+bcrypt_rounds: 4
+
+## Federation ##
+
+# trust certs signed by Complement's CA
+federation_custom_ca_list:
+- /complement/ca/ca.crt
+
+# unblacklist RFC1918 addresses
+federation_ip_range_blacklist: []
+
+# Disable server rate-limiting
+rc_federation:
+  window_size: 1000
+  sleep_limit: 10
+  sleep_delay: 500
+  reject_limit: 99999
+  concurrent: 3
+
+rc_message:
+  per_second: 9999
+  burst_count: 9999
+
+rc_registration:
+  per_second: 9999
+  burst_count: 9999
+
+rc_login:
+  address:
+    per_second: 9999
+    burst_count: 9999
+  account:
+    per_second: 9999
+    burst_count: 9999
+  failed_attempts:
+    per_second: 9999
+    burst_count: 9999
+
+rc_admin_redaction:
+  per_second: 9999
+  burst_count: 9999
+
+rc_joins:
+  local:
+    per_second: 9999
+    burst_count: 9999
+  remote:
+    per_second: 9999
+    burst_count: 9999
+
+federation_rr_transactions_per_room_per_second: 9999
+
+## Experimental Features ##
+
+experimental_features:
+  # Enable history backfilling support
+  msc2716_enabled: true
+  # Enable spaces support
+  spaces_enabled: true
+  # Enable jump to date endpoint
+  msc3030_enabled: true
+
+server_notices:
+  system_mxid_localpart: _server
+  system_mxid_display_name: "Server Alert"
+  system_mxid_avatar_url: ""
+  room_name: "Server Alert"
@@ -0,0 +1,117 @@
+## Server ##
+
+server_name: SERVER_NAME
+log_config: /conf/log_config.yaml
+report_stats: False
+signing_key_path: /conf/server.signing.key
+trusted_key_servers: []
+enable_registration: true
+enable_registration_without_verification: true
+
+## Listeners ##
+
+tls_certificate_path: /conf/server.tls.crt
+tls_private_key_path: /conf/server.tls.key
+bcrypt_rounds: 4
+registration_shared_secret: complement
+
+listeners:
+  - port: 8448
+    bind_addresses: ['::']
+    type: http
+    tls: true
+    resources:
+      - names: [federation]
+
+  - port: 8008
+    bind_addresses: ['::']
+    type: http
+
+    resources:
+      - names: [client]
+
+## Database ##
+
+database:
+  name: "sqlite3"
+  args:
+    # We avoid /data, as it is a volume and is not transferred when the container is committed,
+    # which is a fundamental necessity in complement.
+    database: "/conf/homeserver.db"
+
+## Federation ##
+
+# trust certs signed by the complement CA
+federation_custom_ca_list:
+- /complement/ca/ca.crt
+
+# unblacklist RFC1918 addresses
+ip_range_blacklist: []
+
+# Disable server rate-limiting
+rc_federation:
+  window_size: 1000
+  sleep_limit: 10
+  sleep_delay: 500
+  reject_limit: 99999
+  concurrent: 3
+
+rc_message:
+  per_second: 9999
+  burst_count: 9999
+
+rc_registration:
+  per_second: 9999
+  burst_count: 9999
+
+rc_login:
+  address:
+    per_second: 9999
+    burst_count: 9999
+  account:
+    per_second: 9999
+    burst_count: 9999
+  failed_attempts:
+    per_second: 9999
+    burst_count: 9999
+
+rc_admin_redaction:
+  per_second: 9999
+  burst_count: 9999
+
+rc_joins:
+  local:
+    per_second: 9999
+    burst_count: 9999
+  remote:
+    per_second: 9999
+    burst_count: 9999
+
+federation_rr_transactions_per_room_per_second: 9999
+
+## API Configuration ##
+
+# A list of application service config files to use
+#
+app_service_config_files:
+AS_REGISTRATION_FILES  
+
+## Experimental Features ##
+
+experimental_features:
+  # Enable spaces support
+  spaces_enabled: true
+  # Enable history backfilling support
+  msc2716_enabled: true
+  # server-side support for partial state in /send_join responses
+  msc3706_enabled: true
+  # client-side support for partial state in /send_join responses
+  faster_joins: true
+  # Enable jump to date endpoint
+  msc3030_enabled: true
+
+server_notices:
+  system_mxid_localpart: _server
+  system_mxid_display_name: "Server Alert"
+  system_mxid_avatar_url: ""
+  room_name: "Server Alert"
@@ -0,0 +1,24 @@
+version: 1
+
+formatters:
+  precise:
+   format: '%(asctime)s - %(name)s - %(lineno)d - %(levelname)s - %(request)s - %(message)s'
+
+filters:
+  context:
+    (): synapse.logging.context.LoggingContextFilter
+    request: ""
+
+handlers:
+  console:
+    class: logging.StreamHandler
+    formatter: precise
+    filters: [context]
+    # log to stdout, for easier use with 'docker logs'
+    stream: 'ext://sys.stdout'
+
+root:
+    level: INFO
+    handlers: [console]
+
+disable_existing_loggers: false
@@ -0,0 +1,30 @@
+#!/bin/sh
+
+set -e
+
+sed -i "s/SERVER_NAME/${SERVER_NAME}/g" /conf/homeserver.yaml
+
+# Add the application service registration files to the homeserver.yaml config
+for filename in /complement/appservice/*.yaml; do
+  [ -f "$filename" ] || break
+
+  as_id=$(basename "$filename" .yaml)
+
+  # Insert the path to the registration file and the AS_REGISTRATION_FILES marker after 
+  # so we can add the next application service in the next iteration of this for loop
+  sed -i "s/AS_REGISTRATION_FILES/  - \/complement\/appservice\/${as_id}.yaml\nAS_REGISTRATION_FILES/g" /conf/homeserver.yaml
+done
+# Remove the AS_REGISTRATION_FILES entry
+sed -i "s/AS_REGISTRATION_FILES//g" /conf/homeserver.yaml
+
+# generate an ssl key and cert for the server, signed by the complement CA
+openssl genrsa -out /conf/server.tls.key 2048
+
+openssl req -new -key /conf/server.tls.key -out /conf/server.tls.csr \
+  -subj "/CN=${SERVER_NAME}"
+openssl x509 -req -in /conf/server.tls.csr \
+  -CA /complement/ca/ca.crt -CAkey /complement/ca/ca.key -set_serial 1 \
+  -out /conf/server.tls.crt
+
+exec python -m synapse.app.homeserver -c /conf/homeserver.yaml "$@"
+
@@ -5,6 +5,9 @@
 nodaemon=true
 user=root

+[include]
+files = /etc/supervisor/conf.d/*.conf
+
 [program:nginx]
 command=/usr/sbin/nginx -g "daemon off;"
 priority=500
@@ -2,11 +2,7 @@ version: 1

 formatters:
  precise:
-{% if worker_name %}
-    format: '%(asctime)s - worker:{{ worker_name }} - %(name)s - %(lineno)d - %(levelname)s - %(request)s - %(message)s'
-{% else %}
    format: '%(asctime)s - %(name)s - %(lineno)d - %(levelname)s - %(request)s - %(message)s'
-{% endif %}

 handlers:
 {% if LOG_FILE_PATH %}
@@ -29,6 +29,7 @@
 import os
 import subprocess
 import sys
+from typing import Any, Dict, List, Mapping, MutableMapping, NoReturn, Set

 import jinja2
 import yaml
@@ -36,7 +37,7 @@ import yaml
 MAIN_PROCESS_HTTP_LISTENER_PORT = 8080


-WORKERS_CONFIG = {
+WORKERS_CONFIG: Dict[str, Dict[str, Any]] = {
    "pusher": {
        "app": "synapse.app.pusher",
        "listener_resources": [],
@@ -170,7 +171,7 @@ WORKERS_CONFIG = {
 # Templates for sections that may be inserted multiple times in config files
 SUPERVISORD_PROCESS_CONFIG_BLOCK = """
 [program:synapse_{name}]
-command=/usr/local/bin/python -m {app} \
+command=/usr/local/bin/prefix-log /usr/local/bin/python -m {app} \
    --config-path="{config_path}" \
    --config-path=/conf/workers/shared.yaml \
    --config-path=/conf/workers/{name}.yaml
@@ -200,7 +201,7 @@ upstream {upstream_worker_type} {{


 # Utility functions
-def log(txt: str):
+def log(txt: str) -> None:
    """Log something to the stdout.

    Args:
@@ -209,7 +210,7 @@ def log(txt: str):
    print(txt)


-def error(txt: str):
+def error(txt: str) -> NoReturn:
    """Log something and exit with an error code.

    Args:
@@ -219,7 +220,7 @@ def error(txt: str):
    sys.exit(2)


-def convert(src: str, dst: str, **template_vars):
+def convert(src: str, dst: str, **template_vars: object) -> None:
    """Generate a file from a template

    Args:
@@ -289,7 +290,7 @@ def add_sharding_to_shared_config(
        shared_config.setdefault("media_instance_running_background_jobs", worker_name)


-def generate_base_homeserver_config():
+def generate_base_homeserver_config() -> None:
    """Starts Synapse and generates a basic homeserver config, which will later be
    modified for worker support.

@@ -301,13 +302,15 @@ def generate_base_homeserver_config():
    subprocess.check_output(["/usr/local/bin/python", "/start.py", "migrate_config"])


-def generate_worker_files(environ, config_path: str, data_dir: str):
+def generate_worker_files(
+    environ: Mapping[str, str], config_path: str, data_dir: str
+) -> None:
    """Read the desired list of workers from environment variables and generate
    shared homeserver, nginx and supervisord configs.

    Args:
-        environ: _Environ[str]
-        config_path: Where to output the generated Synapse main worker config file.
+        environ: os.environ instance.
+        config_path: The location of the generated Synapse main worker config file.
        data_dir: The location of the synapse data directory. Where log and
            user-facing config files live.
    """
@@ -320,7 +323,8 @@ def generate_worker_files(environ, config_path: str, data_dir: str):
    # and adding a replication listener.

    # First read the original config file and extract the listeners block. Then we'll add
-    # another listener for replication. Later we'll write out the result.
+    # another listener for replication. Later we'll write out the result to the shared
+    # config file.
    listeners = [
        {
            "port": 9093,
@@ -339,7 +343,7 @@ def generate_worker_files(environ, config_path: str, data_dir: str):
    # base shared worker jinja2 template.
    #
    # This config file will be passed to all workers, included Synapse's main process.
-    shared_config = {"listeners": listeners}
+    shared_config: Dict[str, Any] = {"listeners": listeners}

    # The supervisord config. The contents of which will be inserted into the
    # base supervisord jinja2 template.
@@ -355,7 +359,7 @@ def generate_worker_files(environ, config_path: str, data_dir: str):
    #   worker_type: {1234, 1235, ...}}
    # }
    # and will be used to construct 'upstream' nginx directives.
-    nginx_upstreams = {}
+    nginx_upstreams: Dict[str, Set[int]] = {}

    # A map of: {"endpoint": "upstream"}, where "upstream" is a str representing what will be
    # placed after the proxy_pass directive. The main benefit to representing this data as a
@@ -367,13 +371,13 @@ def generate_worker_files(environ, config_path: str, data_dir: str):
    nginx_locations = {}

    # Read the desired worker configuration from the environment
-    worker_types = environ.get("SYNAPSE_WORKER_TYPES")
-    if worker_types is None:
+    worker_types_env = environ.get("SYNAPSE_WORKER_TYPES")
+    if worker_types_env is None:
        # No workers, just the main process
        worker_types = []
    else:
        # Split type names by comma
-        worker_types = worker_types.split(",")
+        worker_types = worker_types_env.split(",")

    # Create the worker configuration directory if it doesn't already exist
    os.makedirs("/conf/workers", exist_ok=True)
@@ -384,7 +388,11 @@ def generate_worker_files(environ, config_path: str, data_dir: str):
    # A counter of worker_type -> int. Used for determining the name for a given
    # worker type when generating its config file, as each worker's name is just
    # worker_type + instance #
-    worker_type_counter = {}
+    worker_type_counter: Dict[str, int] = {}
+
+    # A list of internal endpoints to healthcheck, starting with the main process
+    # which exists even if no workers do.
+    healthcheck_urls = ["http://localhost:8080/health"]

    # For each worker type specified by the user, create config values
    for worker_type in worker_types:
@@ -404,12 +412,14 @@ def generate_worker_files(environ, config_path: str, data_dir: str):
        # e.g. federation_reader1
        worker_name = worker_type + str(new_worker_count)
        worker_config.update(
-            {"name": worker_name, "port": worker_port, "config_path": config_path}
+            {"name": worker_name, "port": str(worker_port), "config_path": config_path}
        )

        # Update the shared config with any worker-type specific options
        shared_config.update(worker_config["shared_extra_conf"])

+        healthcheck_urls.append("http://localhost:%d/health" % (worker_port,))
+
        # Check if more than one instance of this worker type has been specified
        worker_type_total_count = worker_types.count(worker_type)
        if worker_type_total_count > 1:
@@ -438,21 +448,7 @@ def generate_worker_files(environ, config_path: str, data_dir: str):

        # Write out the worker's logging config file

-        # Check whether we should write worker logs to disk, in addition to the console
-        extra_log_template_args = {}
-        if environ.get("SYNAPSE_WORKERS_WRITE_LOGS_TO_DISK"):
-            extra_log_template_args["LOG_FILE_PATH"] = "{dir}/logs/{name}.log".format(
-                dir=data_dir, name=worker_name
-            )
-
-        # Render and write the file
-        log_config_filepath = "/conf/workers/{name}.log.config".format(name=worker_name)
-        convert(
-            "/conf/log.config",
-            log_config_filepath,
-            worker_name=worker_name,
-            **extra_log_template_args,
-        )
+        log_config_filepath = generate_worker_log_config(environ, worker_name, data_dir)

        # Then a worker config file
        convert(
@@ -475,15 +471,10 @@ def generate_worker_files(environ, config_path: str, data_dir: str):
    # Determine the load-balancing upstreams to configure
    nginx_upstream_config = ""

-    # At the same time, prepare a list of internal endpoints to healthcheck
-    # starting with the main process which exists even if no workers do.
-    healthcheck_urls = ["http://localhost:8080/health"]
-
    for upstream_worker_type, upstream_worker_ports in nginx_upstreams.items():
        body = ""
        for port in upstream_worker_ports:
            body += "    server localhost:%d;\n" % (port,)
-            healthcheck_urls.append("http://localhost:%d/health" % (port,))

        # Add to the list of configured upstreams
        nginx_upstream_config += NGINX_UPSTREAM_CONFIG_BLOCK.format(
@@ -493,6 +484,10 @@ def generate_worker_files(environ, config_path: str, data_dir: str):

    # Finally, we'll write out the config files.

+    # log config for the master process
+    master_log_config = generate_worker_log_config(environ, "master", data_dir)
+    shared_config["log_config"] = master_log_config
+
    # Shared homeserver config
    convert(
        "/conf/shared.yaml.j2",
@@ -509,9 +504,10 @@ def generate_worker_files(environ, config_path: str, data_dir: str):
    )

    # Supervisord config
+    os.makedirs("/etc/supervisor", exist_ok=True)
    convert(
        "/conf/supervisord.conf.j2",
-        "/etc/supervisor/conf.d/supervisord.conf",
+        "/etc/supervisor/supervisord.conf",
        main_config_path=config_path,
        worker_config=supervisord_config,
    )
@@ -529,15 +525,31 @@ def generate_worker_files(environ, config_path: str, data_dir: str):
        os.mkdir(log_dir)


-def start_supervisord():
-    """Starts up supervisord which then starts and monitors all other necessary processes
+def generate_worker_log_config(
+    environ: Mapping[str, str], worker_name: str, data_dir: str
+) -> str:
+    """Generate a log.config file for the given worker.

-    Raises: CalledProcessError if calling start.py return a non-zero exit code.
+    Returns: the path to the generated file
    """
-    subprocess.run(["/usr/bin/supervisord"], stdin=subprocess.PIPE)
+    # Check whether we should write worker logs to disk, in addition to the console
+    extra_log_template_args = {}
+    if environ.get("SYNAPSE_WORKERS_WRITE_LOGS_TO_DISK"):
+        extra_log_template_args["LOG_FILE_PATH"] = "{dir}/logs/{name}.log".format(
+            dir=data_dir, name=worker_name
+        )
+    # Render and write the file
+    log_config_filepath = "/conf/workers/{name}.log.config".format(name=worker_name)
+    convert(
+        "/conf/log.config",
+        log_config_filepath,
+        worker_name=worker_name,
+        **extra_log_template_args,
+    )
+    return log_config_filepath


-def main(args, environ):
+def main(args: List[str], environ: MutableMapping[str, str]) -> None:
    config_dir = environ.get("SYNAPSE_CONFIG_DIR", "/data")
    config_path = environ.get("SYNAPSE_CONFIG_PATH", config_dir + "/homeserver.yaml")
    data_dir = environ.get("SYNAPSE_DATA_DIR", "/data")
@@ -564,7 +576,13 @@ def main(args, environ):

    # Start supervisord, which will start Synapse, all of the configured worker
    # processes, redis, nginx etc. according to the config we created above.
-    start_supervisord()
+    log("Starting supervisord")
+    os.execl(
+        "/usr/local/bin/supervisord",
+        "supervisord",
+        "-c",
+        "/etc/supervisor/supervisord.conf",
+    )


 if __name__ == "__main__":
@@ -0,0 +1,12 @@
+#!/bin/bash
+#
+# Prefixes all lines on stdout and stderr with the process name (as determined by
+# the SUPERVISOR_PROCESS_NAME env var, which is automatically set by Supervisor).
+#
+# Usage:
+#   prefix-log command [args...]
+#
+
+exec 1> >(awk '{print "'"${SUPERVISOR_PROCESS_NAME}"' | "$0}' >&1)
+exec 2> >(awk '{print "'"${SUPERVISOR_PROCESS_NAME}"' | "$0}' >&2)
+exec "$@"
@@ -1,19 +0,0 @@
-#!/usr/bin/env bash
-
-# This script runs the PostgreSQL tests inside a Docker container. It expects
-# the relevant source files to be mounted into /src (done automatically by the
-# caller script). It will set up the database, run it, and then use the tox
-# configuration to run the tests.
-
-set -e
-
-# Set PGUSER so Synapse's tests know what user to connect to the database with
-export PGUSER=postgres
-
-# Start the database
-sudo -u postgres /usr/lib/postgresql/10/bin/pg_ctl -w -D /var/lib/postgresql/data start
-
-# Run the tests
-cd /src
-export TRIAL_FLAGS="-j 4"
-tox --workdir=./.tox-pg-container -e py37-postgres "$@"
@@ -6,27 +6,28 @@ import os
 import platform
 import subprocess
 import sys
+from typing import Any, Dict, List, Mapping, MutableMapping, NoReturn, Optional

 import jinja2


 # Utility functions
-def log(txt):
+def log(txt: str) -> None:
    print(txt, file=sys.stderr)


-def error(txt):
+def error(txt: str) -> NoReturn:
    log(txt)
    sys.exit(2)


-def convert(src, dst, environ):
+def convert(src: str, dst: str, environ: Mapping[str, object]) -> None:
    """Generate a file from a template

    Args:
-        src (str): path to input file
-        dst (str): path to file to write
-        environ (dict): environment dictionary, for replacement mappings.
+        src: path to input file
+        dst: path to file to write
+        environ: environment dictionary, for replacement mappings.
    """
    with open(src) as infile:
        template = infile.read()
@@ -35,25 +36,30 @@ def convert(src, dst, environ):
        outfile.write(rendered)


-def generate_config_from_template(config_dir, config_path, environ, ownership):
+def generate_config_from_template(
+    config_dir: str,
+    config_path: str,
+    os_environ: Mapping[str, str],
+    ownership: Optional[str],
+) -> None:
    """Generate a homeserver.yaml from environment variables

    Args:
-        config_dir (str): where to put generated config files
-        config_path (str): where to put the main config file
-        environ (dict): environment dictionary
-        ownership (str|None): "<user>:<group>" string which will be used to set
+        config_dir: where to put generated config files
+        config_path: where to put the main config file
+        os_environ: environment mapping
+        ownership: "<user>:<group>" string which will be used to set
            ownership of the generated configs. If None, ownership will not change.
    """
    for v in ("SYNAPSE_SERVER_NAME", "SYNAPSE_REPORT_STATS"):
-        if v not in environ:
+        if v not in os_environ:
            error(
                "Environment variable '%s' is mandatory when generating a config file."
                % (v,)
            )

    # populate some params from data files (if they exist, else create new ones)
-    environ = environ.copy()
+    environ: Dict[str, Any] = dict(os_environ)
    secrets = {
        "registration": "SYNAPSE_REGISTRATION_SHARED_SECRET",
        "macaroon": "SYNAPSE_MACAROON_SECRET_KEY",
@@ -108,7 +114,7 @@ def generate_config_from_template(config_dir, config_path, environ, ownership):

    # Hopefully we already have a signing key, but generate one if not.
    args = [
-        "python",
+        sys.executable,
        "-m",
        "synapse.app.homeserver",
        "--config-path",
@@ -127,12 +133,12 @@ def generate_config_from_template(config_dir, config_path, environ, ownership):
    subprocess.check_output(args)


-def run_generate_config(environ, ownership):
+def run_generate_config(environ: Mapping[str, str], ownership: Optional[str]) -> None:
    """Run synapse with a --generate-config param to generate a template config file

    Args:
-        environ (dict): env var dict
-        ownership (str|None): "userid:groupid" arg for chmod. If None, ownership will not change.
+        environ: env vars from `os.enrivon`.
+        ownership: "userid:groupid" arg for chmod. If None, ownership will not change.

    Never returns.
    """
@@ -158,7 +164,7 @@ def run_generate_config(environ, ownership):

    # generate the main config file, and a signing key.
    args = [
-        "python",
+        sys.executable,
        "-m",
        "synapse.app.homeserver",
        "--server-name",
@@ -175,10 +181,10 @@ def run_generate_config(environ, ownership):
        "--open-private-ports",
    ]
    # log("running %s" % (args, ))
-    os.execv("/usr/local/bin/python", args)
+    os.execv(sys.executable, args)


-def main(args, environ):
+def main(args: List[str], environ: MutableMapping[str, str]) -> None:
    mode = args[1] if len(args) > 1 else "run"

    # if we were given an explicit user to switch to, do so
@@ -254,12 +260,12 @@ running with 'migrate_config'. See the README for more details.

    log("Starting synapse with args " + " ".join(args))

-    args = ["python"] + args
+    args = [sys.executable] + args
    if ownership is not None:
        args = ["gosu", ownership] + args
        os.execve("/usr/sbin/gosu", args, environ)
    else:
-        os.execve("/usr/local/bin/python", args, environ)
+        os.execve(sys.executable, args, environ)


 if __name__ == "__main__":
@@ -17,6 +17,7 @@
 # Usage
  - [Federation](federate.md)
  - [Configuration](usage/configuration/README.md)
+    - [Configuration Manual](usage/configuration/config_documentation.md) 
    - [Homeserver Sample Config File](usage/configuration/homeserver_sample_config.md)
    - [Logging Sample Config File](usage/configuration/logging_sample_config.md)
    - [Structured Logging](structured_logging.md)
@@ -45,6 +46,7 @@
        - [Account validity callbacks](modules/account_validity_callbacks.md)
        - [Password auth provider callbacks](modules/password_auth_provider_callbacks.md)
        - [Background update controller callbacks](modules/background_update_controller_callbacks.md)
+        - [Account data callbacks](modules/account_data_callbacks.md)
        - [Porting a legacy module to the new interface](modules/porting_legacy_module.md)
    - [Workers](workers.md)
      - [Using `synctl` with Workers](synctl_workers.md)
@@ -804,7 +804,7 @@ POST /_synapse/admin/v2/users/<user_id>/delete_devices
  "devices": [
    "QBUAZIFURK",
    "AUIECTSRND"
-  ],
+  ]
 }
 ```

--- a/Show More
+++ b/Show More
				`@@ -1 +0,0 @@`
				Rename `shared_rooms` to `mutual_rooms` (MSC2666), as per proposal changes.
				`@@ -1 +0,0 @@`
				Remove check on `update_user_directory` for shared rooms handler (MSC2666), and update/expand documentation.
				`@@ -1 +0,0 @@`
				Refactor `create_new_client_event` to use a new parameter, `state_event_ids`, which accurately describes the usage with [MSC2716](https://github.com/matrix-org/matrix-doc/pull/2716) instead of abusing `auth_event_ids`.
				`@@ -1 +0,0 @@`
				Fix a long-standing bug which caused the `/_matrix/federation/v1/state` and `.../state_ids` endpoints to return incorrect or invalid data when called for an event which we have stored as an "outlier".
				`@@ -1 +0,0 @@`
				Refuse to start if registration is enabled without email, captcha, or token-based verification unless new config flag `enable_registration_without_verification` is set.
				`@@ -1 +0,0 @@`
				`Allow modules to store already existing 3PID associations.`
				`@@ -1 +0,0 @@`
				`Add tests for database transaction callbacks.`
				`@@ -1 +0,0 @@`
				Handle cancellation in `DatabasePool.runInteraction()`.
				`@@ -1 +0,0 @@`
				`Clean-up logic around rebasing URLs for URL image previews.`
				`@@ -1 +0,0 @@`
				Use the `ignored_users` table in additional places instead of re-parsing the account data.