Update v1.58.1 changelog entry with more familiar language

Co-authored-by: Richard van der Hoff <1389908+richvdh@users.noreply.github.com>

.ci/latest_deps_build_failed_issue_template.md

@@ -0,0 +1,4 @@
+---
+title: CI run against latest deps is failing
+---
+See https://github.com/{{env.GITHUB_REPOSITORY}}/actions/runs/{{env.GITHUB_RUN_ID}}

.ci/patch_for_twisted_trunk.sh

@@ -1,8 +0,0 @@
-#!/bin/sh
-
-# replaces the dependency on Twisted in `python_dependencies` with trunk.
-
-set -e
-cd "$(dirname "$0")"/..
-
-sed -i -e 's#"Twisted.*"#"Twisted @ git+https://github.com/twisted/twisted"#' synapse/python_dependencies.py

.ci/scripts/test_export_data_command.sh

@@ -2,29 +2,24 @@
 
 # Test for the export-data admin command against sqlite and postgres
 
+# Expects Synapse to have been already installed with `poetry install --extras postgres`.
+# Expects `poetry` to be available on the `PATH`.
+
 set -xe
 cd "$(dirname "$0")/../.."
 
-echo "--- Install dependencies"
-
-# Install dependencies for this test.
-pip install psycopg2
-
-# Install Synapse itself. This won't update any libraries.
-pip install -e .
-
 echo "--- Generate the signing key"
 
 # Generate the server's signing key.
-python -m synapse.app.homeserver --generate-keys -c .ci/sqlite-config.yaml
+poetry run synapse_homeserver --generate-keys -c .ci/sqlite-config.yaml
 
 echo "--- Prepare test database"
 
 # Make sure the SQLite3 database is using the latest schema and has no pending background update.
-scripts/update_synapse_database --database-config .ci/sqlite-config.yaml --run-background-updates
+poetry run update_synapse_database --database-config .ci/sqlite-config.yaml --run-background-updates
 
 # Run the export-data command on the sqlite test database
-python -m synapse.app.admin_cmd -c .ci/sqlite-config.yaml  export-data @anon-20191002_181700-832:localhost:8800 \
+poetry run python -m synapse.app.admin_cmd -c .ci/sqlite-config.yaml  export-data @anon-20191002_181700-832:localhost:8800 \
 --output-directory /tmp/export_data
 
 # Test that the output directory exists and contains the rooms directory
@@ -37,14 +32,14 @@ else
 fi
 
 # Create the PostgreSQL database.
-.ci/scripts/postgres_exec.py "CREATE DATABASE synapse"
+poetry run .ci/scripts/postgres_exec.py "CREATE DATABASE synapse"
 
 # Port the SQLite databse to postgres so we can check command works against postgres
 echo "+++ Port SQLite3 databse to postgres"
-scripts/synapse_port_db --sqlite-database .ci/test_db.db --postgres-config .ci/postgres-config.yaml
+poetry run synapse_port_db --sqlite-database .ci/test_db.db --postgres-config .ci/postgres-config.yaml
 
 # Run the export-data command on postgres database
-python -m synapse.app.admin_cmd -c .ci/postgres-config.yaml  export-data @anon-20191002_181700-832:localhost:8800 \
+poetry run python -m synapse.app.admin_cmd -c .ci/postgres-config.yaml  export-data @anon-20191002_181700-832:localhost:8800 \
 --output-directory /tmp/export_data2
 
 # Test that the output directory exists and contains the rooms directory

.ci/scripts/test_old_deps.sh

@@ -1,6 +1,9 @@
 #!/usr/bin/env bash
-# this script is run by GitHub Actions in a plain `focal` container; it installs the
-# minimal requirements for tox and hands over to the py3-old tox environment.
+# this script is run by GitHub Actions in a plain `focal` container; it
+# - installs the minimal system requirements, and poetry;
+# - patches the project definition file to refer to old versions only;
+# - creates a venv with these old versions using poetry; and finally
+# - invokes `trial` to run the tests with old deps.
 
 # Prevent tzdata from asking for user input
 export DEBIAN_FRONTEND=noninteractive
@@ -9,12 +12,70 @@ set -ex
 
 apt-get update
 apt-get install -y \
-        python3 python3-dev python3-pip python3-venv \
-        libxml2-dev libxslt-dev xmlsec1 zlib1g-dev tox libjpeg-dev libwebp-dev
+        python3 python3-dev python3-pip python3-venv pipx \
+        libxml2-dev libxslt-dev xmlsec1 zlib1g-dev libjpeg-dev libwebp-dev
 
 export LANG="C.UTF-8"
 
 # Prevent virtualenv from auto-updating pip to an incompatible version
 export VIRTUALENV_NO_DOWNLOAD=1
 
-exec tox -e py3-old
+# TODO: in the future, we could use an implementation of
+#   https://github.com/python-poetry/poetry/issues/3527
+#   https://github.com/pypa/pip/issues/8085
+# to select the lowest possible versions, rather than resorting to this sed script.
+
+# Patch the project definitions in-place:
+# - Replace all lower and tilde bounds with exact bounds
+# - Make the pyopenssl 17.0, which is the oldest version that works with
+#   a `cryptography` compiled against OpenSSL 1.1.
+# - Delete all lines referring to psycopg2 --- so no testing of postgres support.
+# - Omit systemd: we're not logging to journal here.
+
+# TODO: also replace caret bounds, see https://python-poetry.org/docs/dependency-specification/#version-constraints
+# We don't use these yet, but IIRC they are the default bound used when you `poetry add`.
+# The sed expression 's/\^/==/g' ought to do the trick. But it would also change
+# `python = "^3.7"` to `python = "==3.7", which would mean we fail because olddeps
+# runs on 3.8 (#12343).
+
+sed -i \
+   -e "s/[~>]=/==/g" \
+   -e "/psycopg2/d" \
+   -e 's/pyOpenSSL = "==16.0.0"/pyOpenSSL = "==17.0.0"/' \
+   -e '/systemd/d' \
+   pyproject.toml
+
+# Use poetry to do the installation. This ensures that the versions are all mutually
+# compatible (as far the package metadata declares, anyway); pip's package resolver
+# is more lax.
+#
+# Rather than `poetry install --no-dev`, we drop all dev dependencies from the
+# toml file. This means we don't have to ensure compatibility between old deps and
+# dev tools.
+
+pip install --user toml
+
+REMOVE_DEV_DEPENDENCIES="
+import toml
+with open('pyproject.toml', 'r') as f:
+    data = toml.loads(f.read())
+
+del data['tool']['poetry']['dev-dependencies']
+
+with open('pyproject.toml', 'w') as f:
+    toml.dump(data, f)
+"
+python3 -c "$REMOVE_DEV_DEPENDENCIES"
+
+pipx install poetry==1.1.12
+~/.local/bin/poetry lock
+
+echo "::group::Patched pyproject.toml"
+cat pyproject.toml
+echo "::endgroup::"
+echo "::group::Lockfile after patch"
+cat poetry.lock
+echo "::endgroup::"
+
+~/.local/bin/poetry install -E "all test"
+~/.local/bin/poetry run trial --jobs=2 tests

.ci/scripts/test_synapse_port_db.sh

@@ -1,41 +1,37 @@
 #!/usr/bin/env bash
 #
 # Test script for 'synapse_port_db'.
-#   - sets up synapse and deps
+#   - configures synapse and a postgres server.
 #   - runs the port script on a prepopulated test sqlite db
 #   - also runs it against an new sqlite db
-
+#
+# Expects Synapse to have been already installed with `poetry install --extras postgres`.
+# Expects `poetry` to be available on the `PATH`.
 
 set -xe
 cd "$(dirname "$0")/../.."
 
-echo "--- Install dependencies"
-
-# Install dependencies for this test.
-pip install psycopg2 coverage coverage-enable-subprocess
-
-# Install Synapse itself. This won't update any libraries.
-pip install -e .
-
 echo "--- Generate the signing key"
 
 # Generate the server's signing key.
-python -m synapse.app.homeserver --generate-keys -c .ci/sqlite-config.yaml
+poetry run synapse_homeserver --generate-keys -c .ci/sqlite-config.yaml
 
 echo "--- Prepare test database"
 
 # Make sure the SQLite3 database is using the latest schema and has no pending background update.
-scripts/update_synapse_database --database-config .ci/sqlite-config.yaml --run-background-updates
+poetry run update_synapse_database --database-config .ci/sqlite-config.yaml --run-background-updates
 
 # Create the PostgreSQL database.
-.ci/scripts/postgres_exec.py "CREATE DATABASE synapse"
+poetry run .ci/scripts/postgres_exec.py "CREATE DATABASE synapse"
 
 echo "+++ Run synapse_port_db against test database"
-coverage run scripts/synapse_port_db --sqlite-database .ci/test_db.db --postgres-config .ci/postgres-config.yaml
+# TODO: this invocation of synapse_port_db (and others below) used to be prepended with `coverage run`,
+# but coverage seems unable to find the entrypoints installed by `pip install -e .`.
+poetry run synapse_port_db --sqlite-database .ci/test_db.db --postgres-config .ci/postgres-config.yaml
 
 # We should be able to run twice against the same database.
 echo "+++ Run synapse_port_db a second time"
-coverage run scripts/synapse_port_db --sqlite-database .ci/test_db.db --postgres-config .ci/postgres-config.yaml
+poetry run synapse_port_db --sqlite-database .ci/test_db.db --postgres-config .ci/postgres-config.yaml
 
 #####
 
@@ -46,12 +42,12 @@ echo "--- Prepare empty SQLite database"
 # we do this by deleting the sqlite db, and then doing the same again.
 rm .ci/test_db.db
 
-scripts/update_synapse_database --database-config .ci/sqlite-config.yaml --run-background-updates
+poetry run update_synapse_database --database-config .ci/sqlite-config.yaml --run-background-updates
 
 # re-create the PostgreSQL database.
-.ci/scripts/postgres_exec.py \
+poetry run .ci/scripts/postgres_exec.py \
   "DROP DATABASE synapse" \
   "CREATE DATABASE synapse"
 
 echo "+++ Run synapse_port_db against empty database"
-coverage run scripts/synapse_port_db --sqlite-database .ci/test_db.db --postgres-config .ci/postgres-config.yaml
+poetry run synapse_port_db --sqlite-database .ci/test_db.db --postgres-config .ci/postgres-config.yaml

.dockerignore

@@ -3,11 +3,9 @@
 
 # things to include
 !docker
-!scripts
 !synapse
-!MANIFEST.in
 !README.rst
-!setup.py
-!synctl
+!pyproject.toml
+!poetry.lock
 
 **/__pycache__

.github/workflows/docs.yaml

@@ -22,7 +22,7 @@ jobs:
       - name: Setup mdbook
         uses: peaceiris/actions-mdbook@4b5ef36b314c2599664ca107bb8c02412548d79d # v1.1.14
         with:
-          mdbook-version: '0.4.9'
+          mdbook-version: '0.4.17'
 
       - name: Build the documentation
         # mdbook will only create an index.html if we're including docs/README.md in SUMMARY.md.

.github/workflows/latest_deps.yml

@@ -0,0 +1,156 @@
+# People who are freshly `pip install`ing from PyPI will pull in the latest versions of
+# dependencies which match the broad requirements. Since most CI runs are against
+# the locked poetry environment, run specifically against the latest dependencies to
+# know if there's an upcoming breaking change.
+#
+# As an overview this workflow:
+# - checks out develop,
+# - installs from source, pulling in the dependencies like a fresh `pip install` would, and 
+# - runs mypy and test suites in that checkout.
+#
+# Based on the twisted trunk CI job.
+
+name: Latest dependencies
+
+on:
+  schedule:
+    - cron: 0 7 * * *
+  workflow_dispatch:
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  mypy:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      # The dev dependencies aren't exposed in the wheel metadata (at least with current
+      # poetry-core versions), so we install with poetry.
+      - uses: matrix-org/setup-python-poetry@v1
+        with:
+          python-version: "3.x"
+          poetry-version: "1.2.0b1"
+      # Dump installed versions for debugging.
+      - run: poetry run pip list > before.txt
+      # Upgrade all runtime dependencies only. This is intended to mimic a fresh
+      # `pip install matrix-synapse[all]` as closely as possible.
+      - run: poetry update --no-dev
+      - run: poetry run pip list > after.txt && (diff -u before.txt after.txt || true)
+      - run: poetry run mypy
+  trial:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        include:
+          - database: "sqlite"
+          - database: "postgres"
+            postgres-version: "14"
+
+    steps:
+      - uses: actions/checkout@v2
+      - run: sudo apt-get -qq install xmlsec1
+      - name: Set up PostgreSQL ${{ matrix.postgres-version }}
+        if: ${{ matrix.postgres-version }}
+        run: |
+          docker run -d -p 5432:5432 \
+            -e POSTGRES_PASSWORD=postgres \
+            -e POSTGRES_INITDB_ARGS="--lc-collate C --lc-ctype C --encoding UTF8" \
+            postgres:${{ matrix.postgres-version }}
+      - uses: actions/setup-python@v2
+        with:
+          python-version: "3.x"
+      - run: pip install .[all,test]
+      - name: Await PostgreSQL
+        if: ${{ matrix.postgres-version }}
+        timeout-minutes: 2
+        run: until pg_isready -h localhost; do sleep 1; done
+      - run: python -m twisted.trial --jobs=2 tests
+        env:
+          SYNAPSE_POSTGRES: ${{ matrix.database == 'postgres' || '' }}
+          SYNAPSE_POSTGRES_HOST: localhost
+          SYNAPSE_POSTGRES_USER: postgres
+          SYNAPSE_POSTGRES_PASSWORD: postgres
+      - name: Dump logs
+        # Logs are most useful when the command fails, always include them.
+        if: ${{ always() }}
+        # Note: Dumps to workflow logs instead of using actions/upload-artifact
+        #       This keeps logs colocated with failing jobs
+        #       It also ignores find's exit code; this is a best effort affair
+        run: >-
+          find _trial_temp -name '*.log'
+          -exec echo "::group::{}" \;
+          -exec cat {} \;
+          -exec echo "::endgroup::" \;
+          || true
+
+
+  sytest:
+    runs-on: ubuntu-latest
+    container:
+      image: matrixdotorg/sytest-synapse:testing
+      volumes:
+        - ${{ github.workspace }}:/src
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - sytest-tag: focal
+
+          - sytest-tag: focal
+            postgres: postgres
+            workers: workers
+            redis: redis
+    env:
+      POSTGRES: ${{ matrix.postgres && 1}}
+      WORKERS: ${{ matrix.workers && 1 }}
+      REDIS: ${{ matrix.redis && 1 }}
+      BLACKLIST: ${{ matrix.workers && 'synapse-blacklist-with-workers' }}
+
+    steps:
+      - uses: actions/checkout@v2
+      - name: Ensure sytest runs `pip install`
+        # Delete the lockfile so sytest will `pip install` rather than `poetry install`
+        run: rm /src/poetry.lock
+        working-directory: /src
+      - name: Prepare test blacklist
+        run: cat sytest-blacklist .ci/worker-blacklist > synapse-blacklist-with-workers
+      - name: Run SyTest
+        run: /bootstrap.sh synapse
+        working-directory: /src
+      - name: Summarise results.tap
+        if: ${{ always() }}
+        run: /sytest/scripts/tap_to_gha.pl /logs/results.tap
+      - name: Upload SyTest logs
+        uses: actions/upload-artifact@v2
+        if: ${{ always() }}
+        with:
+          name: Sytest Logs - ${{ job.status }} - (${{ join(matrix.*, ', ') }})
+          path: |
+            /logs/results.tap
+            /logs/**/*.log*
+
+
+  # TODO: run complement (as with twisted trunk, see #12473).
+
+  # open an issue if the build fails, so we know about it.
+  open-issue:
+    if: failure()
+    needs:
+      # TODO: should mypy be included here? It feels more brittle than the other two.
+      - mypy
+      - trial
+      - sytest
+
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v2
+      - uses: JasonEtco/create-an-issue@5d9504915f79f9cc6d791934b8ef34f2353dd74d # v2.5.0, 2020-12-06
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        with:
+          update_existing: true
+          filename: .ci/latest_deps_build_failed_issue_template.md
+

.github/workflows/release-artifacts.yml

@@ -31,7 +31,7 @@ jobs:
           # if we're running from a tag, get the full list of distros; otherwise just use debian:sid
           dists='["debian:sid"]'
           if [[ $GITHUB_REF == refs/tags/* ]]; then
-              dists=$(scripts-dev/build_debian_packages --show-dists-json)
+              dists=$(scripts-dev/build_debian_packages.py --show-dists-json)
           fi
           echo "::set-output name=distros::$dists"
     # map the step outputs to job outputs
@@ -74,7 +74,7 @@ jobs:
         # see https://github.com/docker/build-push-action/issues/252
         # for the cache magic here
         run: |
-          ./src/scripts-dev/build_debian_packages \
+          ./src/scripts-dev/build_debian_packages.py \
             --docker-build-arg=--cache-from=type=local,src=/tmp/.buildx-cache \
             --docker-build-arg=--cache-to=type=local,mode=max,dest=/tmp/.buildx-cache-new \
             --docker-build-arg=--progress=plain \
@@ -112,7 +112,8 @@ jobs:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         with:
           files: |
-            python-dist/*
+            Sdist/*
+            Wheel/*
             debs.tar.xz
           # if it's not already published, keep the release as a draft.
           draft: true

.github/workflows/tests.yml

@@ -15,24 +15,18 @@ jobs:
     steps:
       - uses: actions/checkout@v2
       - uses: actions/setup-python@v2
-      - run: pip install -e .
-      - run: scripts-dev/generate_sample_config --check
+      - run: pip install .
+      - run: scripts-dev/generate_sample_config.sh --check
+      - run: scripts-dev/config-lint.sh
 
   lint:
-    runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        toxenv:
-          - "check_codestyle"
-          - "check_isort"
-          - "mypy"
-          - "packaging"
-
-    steps:
-      - uses: actions/checkout@v2
-      - uses: actions/setup-python@v2
-      - run: pip install tox
-      - run: tox -e ${{ matrix.toxenv }}
+    # This does a vanilla `poetry install` - no extras. I'm slightly anxious
+    # that we might skip some typechecks on code that uses extras. However,
+    # I think the right way to fix this is to mark any extras needed for
+    # typechecking as development dependencies. To detect this, we ought to
+    # turn up mypy's strictness: disallow unknown imports and be accept fewer
+    # uses of `Any`.
+    uses: "matrix-org/backend-meta/.github/workflows/python-poetry-ci.yml@v1"
 
   lint-crlf:
     runs-on: ubuntu-latest
@@ -51,7 +45,7 @@ jobs:
           fetch-depth: 0
       - uses: actions/setup-python@v2
       - run: "pip install 'towncrier>=18.6.0rc1'"
-      - run: scripts-dev/check-newsfragment
+      - run: scripts-dev/check-newsfragment.sh
         env:
           PULL_REQUEST_NUMBER: ${{ github.event.number }}
 
@@ -71,23 +65,23 @@ jobs:
       matrix:
         python-version: ["3.7", "3.8", "3.9", "3.10"]
         database: ["sqlite"]
-        toxenv: ["py"]
+        extras: ["all"]
         include:
           # Newest Python without optional deps
           - python-version: "3.10"
-            toxenv: "py-noextras"
+            extras: ""
 
           # Oldest Python with PostgreSQL
           - python-version: "3.7"
             database: "postgres"
             postgres-version: "10"
-            toxenv: "py"
+            extras: "all"
 
           # Newest Python with newest PostgreSQL
           - python-version: "3.10"
             database: "postgres"
             postgres-version: "14"
-            toxenv: "py"
+            extras: "all"
 
     steps:
       - uses: actions/checkout@v2
@@ -99,17 +93,16 @@ jobs:
             -e POSTGRES_PASSWORD=postgres \
             -e POSTGRES_INITDB_ARGS="--lc-collate C --lc-ctype C --encoding UTF8" \
             postgres:${{ matrix.postgres-version }}
-      - uses: actions/setup-python@v2
+      - uses: matrix-org/setup-python-poetry@v1
         with:
           python-version: ${{ matrix.python-version }}
-      - run: pip install tox
+          extras: ${{ matrix.extras }}
       - name: Await PostgreSQL
         if: ${{ matrix.postgres-version }}
         timeout-minutes: 2
         run: until pg_isready -h localhost; do sleep 1; done
-      - run: tox -e ${{ matrix.toxenv }}
+      - run: poetry run trial --jobs=2 tests
         env:
-          TRIAL_FLAGS: "--jobs=2"
           SYNAPSE_POSTGRES: ${{ matrix.database == 'postgres' || '' }}
           SYNAPSE_POSTGRES_HOST: localhost
           SYNAPSE_POSTGRES_USER: postgres
@@ -128,6 +121,7 @@ jobs:
           || true
 
   trial-olddeps:
+    # Note: sqlite only; no postgres
     if: ${{ !cancelled() && !failure() }} # Allow previous steps to be skipped, but not fail
     needs: linting-done
     runs-on: ubuntu-latest
@@ -135,11 +129,11 @@ jobs:
       - uses: actions/checkout@v2
       - name: Test with old deps
         uses: docker://ubuntu:focal # For old python and sqlite
+        # Note: focal seems to be using 3.8, but the oldest is 3.7?
+        # See https://github.com/matrix-org/synapse/issues/12343
         with:
           workdir: /github/workspace
           entrypoint: .ci/scripts/test_old_deps.sh
-        env:
-          TRIAL_FLAGS: "--jobs=2"
       - name: Dump logs
         # Logs are most useful when the command fails, always include them.
         if: ${{ always() }}
@@ -155,23 +149,24 @@ jobs:
 
   trial-pypy:
     # Very slow; only run if the branch name includes 'pypy'
+    # Note: sqlite only; no postgres. Completely untested since poetry move.
     if: ${{ contains(github.ref, 'pypy') && !failure() && !cancelled() }}
     needs: linting-done
     runs-on: ubuntu-latest
     strategy:
       matrix:
         python-version: ["pypy-3.7"]
+        extras: ["all"]
 
     steps:
       - uses: actions/checkout@v2
+      # Install libs necessary for PyPy to build binary wheels for dependencies
       - run: sudo apt-get -qq install xmlsec1 libxml2-dev libxslt-dev
-      - uses: actions/setup-python@v2
+      - uses: matrix-org/setup-python-poetry@v1
         with:
           python-version: ${{ matrix.python-version }}
-      - run: pip install tox
-      - run: tox -e py
-        env:
-          TRIAL_FLAGS: "--jobs=2"
+          extras: ${{ matrix.extras }}
+      - run: poetry run trial --jobs=2 tests
       - name: Dump logs
         # Logs are most useful when the command fails, always include them.
         if: ${{ always() }}
@@ -270,9 +265,10 @@ jobs:
     steps:
       - uses: actions/checkout@v2
       - run: sudo apt-get -qq install xmlsec1
-      - uses: actions/setup-python@v2
+      - uses: matrix-org/setup-python-poetry@v1
         with:
-          python-version: "3.9"
+          python-version: ${{ matrix.python-version }}
+          extras: "postgres"
       - run: .ci/scripts/test_export_data_command.sh
 
   portdb:
@@ -307,9 +303,10 @@ jobs:
     steps:
       - uses: actions/checkout@v2
       - run: sudo apt-get -qq install xmlsec1
-      - uses: actions/setup-python@v2
+      - uses: matrix-org/setup-python-poetry@v1
         with:
           python-version: ${{ matrix.python-version }}
+          extras: "postgres"
       - run: .ci/scripts/test_synapse_port_db.sh
 
   complement:
@@ -361,60 +358,32 @@ jobs:
             (wget -O - "https://github.com/matrix-org/complement/archive/$BRANCH_NAME.tar.gz" | tar -xz --strip-components=1 -C complement) && break
           done
 
-      # Build initial Synapse image
-      - run: docker build -t matrixdotorg/synapse:latest -f docker/Dockerfile .
-        working-directory: synapse
-        env:
-          DOCKER_BUILDKIT: 1
-
-      # Build a ready-to-run Synapse image based on the initial image above.
-      # This new image includes a config file, keys for signing and TLS, and
-      # other settings to make it suitable for testing under Complement.
-      - run: docker build -t complement-synapse -f Synapse.Dockerfile .
-        working-directory: complement/dockerfiles
-
-      # Run Complement
       - run: |
           set -o pipefail
-          go test -v -json -p 1 -tags synapse_blacklist,msc2403 ./tests/... 2>&1 | gotestfmt
+          COMPLEMENT_DIR=`pwd`/complement synapse/scripts-dev/complement.sh -json 2>&1 | gotestfmt
         shell: bash
         name: Run Complement Tests
-        env:
-          COMPLEMENT_BASE_IMAGE: complement-synapse:latest
-        working-directory: complement
 
   # a job which marks all the other jobs as complete, thus allowing PRs to be merged.
   tests-done:
     if: ${{ always() }}
     needs:
+      - check-sampleconfig
       - lint
       - lint-crlf
       - lint-newsfile
       - trial
       - trial-olddeps
       - sytest
+      - export-data
       - portdb
       - complement
     runs-on: ubuntu-latest
     steps:
-      - name: Set build result
-        env:
-          NEEDS_CONTEXT: ${{ toJSON(needs) }}
-        # the `jq` incantation dumps out a series of "<job> <result>" lines.
-        # we set it to an intermediate variable to avoid a pipe, which makes it
-        # hard to set $rc.
-        run: |
-          rc=0
-          results=$(jq -r 'to_entries[] | [.key,.value.result] | join(" ")' <<< $NEEDS_CONTEXT)
-          while read job result ; do
-              # The newsfile lint may be skipped on non PR builds
-              if [ $result == "skipped" ] && [ $job == "lint-newsfile" ]; then
-                continue
-              fi
+      - uses: matrix-org/done-action@v2
+        with:
+          needs: ${{ toJSON(needs) }}
 
-              if [ "$result" != "success" ]; then
-                  echo "::set-failed ::Job $job returned $result"
-                  rc=1
-              fi
-          done <<< $results
-          exit $rc
+          # The newsfile lint may be skipped on non PR builds
+          skippable:
+            lint-newsfile

.github/workflows/twisted_trunk.yml

@@ -6,16 +6,25 @@ on:
 
   workflow_dispatch:
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
 jobs:
   mypy:
     runs-on: ubuntu-latest
 
     steps:
       - uses: actions/checkout@v2
-      - uses: actions/setup-python@v2
-      - run: .ci/patch_for_twisted_trunk.sh
-      - run: pip install tox
-      - run: tox -e mypy
+      - uses: matrix-org/setup-python-poetry@v1
+        with:
+          python-version: "3.x"
+          extras: "all"
+      - run: |
+          poetry remove twisted
+          poetry add --extras tls git+https://github.com/twisted/twisted.git#trunk
+          poetry install --no-interaction --extras "all test"
+      - run: poetry run mypy
 
   trial:
     runs-on: ubuntu-latest
@@ -23,14 +32,15 @@ jobs:
     steps:
       - uses: actions/checkout@v2
       - run: sudo apt-get -qq install xmlsec1
-      - uses: actions/setup-python@v2
+      - uses: matrix-org/setup-python-poetry@v1
         with:
-          python-version: 3.7
-      - run: .ci/patch_for_twisted_trunk.sh
-      - run: pip install tox
-      - run: tox -e py
-        env:
-          TRIAL_FLAGS: "--jobs=2"
+          python-version: "3.x"
+          extras: "all test"
+      - run: |
+          poetry remove twisted
+          poetry add --extras tls git+https://github.com/twisted/twisted.git#trunk
+          poetry install --no-interaction --extras "all test"
+      - run: poetry run trial --jobs 2 tests
 
       - name: Dump logs
         # Logs are most useful when the command fails, always include them.
@@ -55,11 +65,23 @@ jobs:
     steps:
       - uses: actions/checkout@v2
       - name: Patch dependencies
-        run: .ci/patch_for_twisted_trunk.sh
+        # Note: The poetry commands want to create a virtualenv in /src/.venv/,
+        #       but the sytest-synapse container expects it to be in /venv/.
+        #       We symlink it before running poetry so that poetry actually
+        #       ends up installing to `/venv`.
+        run: |
+          ln -s -T /venv /src/.venv
+          poetry remove twisted
+          poetry add --extras tls git+https://github.com/twisted/twisted.git#trunk
+          poetry install --no-interaction --extras "all test"
         working-directory: /src
       - name: Run SyTest
         run: /bootstrap.sh synapse
         working-directory: /src
+        env:
+          # Use offline mode to avoid reinstalling the pinned version of
+          # twisted.
+          OFFLINE: 1
       - name: Summarise results.tap
         if: ${{ always() }}
         run: /sytest/scripts/tap_to_gha.pl /logs/results.tap

.gitignore

@@ -15,6 +15,9 @@ _trial_temp*/
 .DS_Store
 __pycache__/
 
+# We do want the poetry lockfile.
+!poetry.lock
+
 # stuff that is likely to exist when you run a server locally
 /*.db
 /*.log
@@ -30,6 +33,9 @@ __pycache__/
 /media_store/
 /uploads
 
+# For direnv users
+/.envrc
+
 # IDEs
 /.idea/
 /.ropeproject/

MANIFEST.in

@@ -1,57 +0,0 @@
-include synctl
-include LICENSE
-include VERSION
-include *.rst
-include *.md
-include demo/README
-include demo/demo.tls.dh
-include demo/*.py
-include demo/*.sh
-
-include synapse/py.typed
-recursive-include synapse/storage *.sql
-recursive-include synapse/storage *.sql.postgres
-recursive-include synapse/storage *.sql.sqlite
-recursive-include synapse/storage *.py
-recursive-include synapse/storage *.txt
-recursive-include synapse/storage *.md
-
-recursive-include docs *
-recursive-include scripts *
-recursive-include scripts-dev *
-recursive-include synapse *.pyi
-recursive-include tests *.py
-recursive-include tests *.pem
-recursive-include tests *.p8
-recursive-include tests *.crt
-recursive-include tests *.key
-
-recursive-include synapse/res *
-recursive-include synapse/static *.css
-recursive-include synapse/static *.gif
-recursive-include synapse/static *.html
-recursive-include synapse/static *.js
-
-exclude .codecov.yml
-exclude .coveragerc
-exclude .dockerignore
-exclude .editorconfig
-exclude Dockerfile
-exclude mypy.ini
-exclude sytest-blacklist
-exclude test_postgresql.sh
-
-include book.toml
-include pyproject.toml
-recursive-include changelog.d *
-
-include .flake8
-prune .circleci
-prune .github
-prune .ci
-prune contrib
-prune debian
-prune demo/etc
-prune docker
-prune snap
-prune stubs

README.rst

@@ -293,36 +293,42 @@ directory of your choice::
     git clone https://github.com/matrix-org/synapse.git
     cd synapse
 
-Synapse has a number of external dependencies, that are easiest
-to install using pip and a virtualenv::
+Synapse has a number of external dependencies. We maintain a fixed development
+environment using [poetry](https://python-poetry.org/). First, install poetry. We recommend
 
-    python3 -m venv ./env
-    source ./env/bin/activate
-    pip install -e ".[all,dev]"
+    pip install --user pipx
+    pipx install poetry
+
+as described `here <https://python-poetry.org/docs/#installing-with-pipx>`_.
+(See `poetry's installation docs <https://python-poetry.org/docs/#installation>`
+for other installation methods.) Then ask poetry to create a virtual environment
+from the project and install Synapse's dependencies::
+
+    poetry install --extras "all test"
 
 This will run a process of downloading and installing all the needed
-dependencies into a virtual env. If any dependencies fail to install,
-try installing the failing modules individually::
-
-    pip install -e "module-name"
+dependencies into a virtual env.
 
 We recommend using the demo which starts 3 federated instances running on ports `8080` - `8082`
 
-    ./demo/start.sh
+    poetry run ./demo/start.sh
 
-(to stop, you can use `./demo/stop.sh`)
+(to stop, you can use `poetry run ./demo/stop.sh`)
+
+See the `demo documentation <https://matrix-org.github.io/synapse/develop/development/demo.html>`_
+for more information.
 
 If you just want to start a single instance of the app and run it directly::
 
     # Create the homeserver.yaml config once
-    python -m synapse.app.homeserver \
+    poetry run synapse_homeserver \
       --server-name my.domain.name \
       --config-path homeserver.yaml \
       --generate-config \
       --report-stats=[yes|no]
 
     # Start the app
-    python -m synapse.app.homeserver --config-path homeserver.yaml
+    poetry run synapse_homeserver --config-path homeserver.yaml
 
 
 Running the unit tests
@@ -331,7 +337,7 @@ Running the unit tests
 After getting up and running, you may wish to run Synapse's unit tests to
 check that everything is installed correctly::
 
-    trial tests
+    poetry run trial tests
 
 This should end with a 'PASSED' result (note that exact numbers will
 differ)::

book.toml

@@ -34,14 +34,6 @@ additional-css = [
     "docs/website_files/table-of-contents.css",
     "docs/website_files/remove-nav-buttons.css",
     "docs/website_files/indent-section-headers.css",
-    "docs/website_files/version-picker.css",
 ]
-additional-js = [
-    "docs/website_files/table-of-contents.js",
-    "docs/website_files/version-picker.js",
-    "docs/website_files/version.js",
-]
-theme = "docs/website_files/theme"
-
-[preprocessor.schema_versions]
-command = "./scripts-dev/schema_versions.py"
+additional-js = ["docs/website_files/table-of-contents.js"]
+theme = "docs/website_files/theme"

contrib/jitsimeetbridge/jitsimeetbridge.py

@@ -193,12 +193,15 @@ class TrivialXmppClient:
         time.sleep(7)
         print("SSRC spammer started")
         while self.running:
-            ssrcMsg = "<presence to='%(tojid)s' xmlns='jabber:client'><x xmlns='http://jabber.org/protocol/muc'/><c xmlns='http://jabber.org/protocol/caps' hash='sha-1' node='http://jitsi.org/jitsimeet' ver='0WkSdhFnAUxrz4ImQQLdB80GFlE='/><nick xmlns='http://jabber.org/protocol/nick'>%(nick)s</nick><stats xmlns='http://jitsi.org/jitmeet/stats'><stat name='bitrate_download' value='175'/><stat name='bitrate_upload' value='176'/><stat name='packetLoss_total' value='0'/><stat name='packetLoss_download' value='0'/><stat name='packetLoss_upload' value='0'/></stats><media xmlns='http://estos.de/ns/mjs'><source type='audio' ssrc='%(assrc)s' direction='sendre'/><source type='video' ssrc='%(vssrc)s' direction='sendre'/></media></presence>" % {
-                "tojid": "%s@%s/%s" % (ROOMNAME, ROOMDOMAIN, self.shortJid),
-                "nick": self.userId,
-                "assrc": self.ssrcs["audio"],
-                "vssrc": self.ssrcs["video"],
-            }
+            ssrcMsg = (
+                "<presence to='%(tojid)s' xmlns='jabber:client'><x xmlns='http://jabber.org/protocol/muc'/><c xmlns='http://jabber.org/protocol/caps' hash='sha-1' node='http://jitsi.org/jitsimeet' ver='0WkSdhFnAUxrz4ImQQLdB80GFlE='/><nick xmlns='http://jabber.org/protocol/nick'>%(nick)s</nick><stats xmlns='http://jitsi.org/jitmeet/stats'><stat name='bitrate_download' value='175'/><stat name='bitrate_upload' value='176'/><stat name='packetLoss_total' value='0'/><stat name='packetLoss_download' value='0'/><stat name='packetLoss_upload' value='0'/></stats><media xmlns='http://estos.de/ns/mjs'><source type='audio' ssrc='%(assrc)s' direction='sendre'/><source type='video' ssrc='%(vssrc)s' direction='sendre'/></media></presence>"
+                % {
+                    "tojid": "%s@%s/%s" % (ROOMNAME, ROOMDOMAIN, self.shortJid),
+                    "nick": self.userId,
+                    "assrc": self.ssrcs["audio"],
+                    "vssrc": self.ssrcs["video"],
+                }
+            )
             res = self.sendIq(ssrcMsg)
             print("reply from ssrc announce: ", res)
             time.sleep(10)

contrib/snap/snapcraft.yaml

@@ -20,7 +20,7 @@ apps:
   generate-config:
     command: generate_config
   generate-signing-key:
-    command: generate_signing_key.py
+    command: generate_signing_key
   register-new-matrix-user:
     command: register_new_matrix_user
     plugs: [network]

debian/build_virtualenv

@@ -30,9 +30,24 @@ case $(dpkg-architecture -q DEB_HOST_ARCH) in
         ;;
 esac
 
-# Use --builtin-venv to use the better `venv` module from CPython 3.4+ rather
-# than the 2/3 compatible `virtualenv`.
+# Manually install Poetry and export a pip-compatible `requirements.txt`
+# We need a Poetry pre-release as the export command is buggy in < 1.2
+TEMP_VENV="$(mktemp -d)"
+python3 -m venv "$TEMP_VENV"
+source "$TEMP_VENV/bin/activate"
+pip install -U pip
+pip install poetry==1.2.0b1
+poetry export \
+    --extras all \
+    --extras test \
+    --extras systemd \
+    --extras cache_memory \
+    -o exported_requirements.txt
+deactivate
+rm -rf "$TEMP_VENV"
 
+# Use --no-deps to only install pinned versions in exported_requirements.txt,
+# and to avoid https://github.com/pypa/pip/issues/9644
 dh_virtualenv \
     --install-suffix "matrix-synapse" \
     --builtin-venv \
@@ -41,9 +56,11 @@ dh_virtualenv \
     --preinstall="lxml" \
     --preinstall="mock" \
     --preinstall="wheel" \
+    --extra-pip-arg="--no-deps" \
     --extra-pip-arg="--no-cache-dir" \
     --extra-pip-arg="--compile" \
-    --extras="all,systemd,test"
+    --extras="all,systemd,test" \
+    --requirements="exported_requirements.txt"
 
 PACKAGE_BUILD_DIR="debian/matrix-synapse-py3"
 VIRTUALENV_DIR="${PACKAGE_BUILD_DIR}${DH_VIRTUALENV_INSTALL_ROOT}/matrix-synapse"

debian/changelog

@@ -1,3 +1,84 @@
+matrix-synapse-py3 (1.58.1) stable; urgency=medium
+
+  * Include python dependencies from the `systemd` and `cache_memory` extras package groups, which
+    were incorrectly omitted from the 1.58.0 package.
+  * New Synapse release 1.58.1.
+
+ -- Synapse Packaging team <packages@matrix.org>  Thu, 05 May 2022 14:58:23 +0100
+
+matrix-synapse-py3 (1.58.0) stable; urgency=medium
+
+  * New Synapse release 1.58.0.
+
+ -- Synapse Packaging team <packages@matrix.org>  Tue, 03 May 2022 10:52:58 +0100
+
+matrix-synapse-py3 (1.58.0~rc2) stable; urgency=medium
+
+  * New Synapse release 1.58.0rc2.
+
+ -- Synapse Packaging team <packages@matrix.org>  Tue, 26 Apr 2022 17:14:56 +0100
+
+matrix-synapse-py3 (1.58.0~rc1) stable; urgency=medium
+
+  * Use poetry to manage the bundled virtualenv included with this package.
+  * New Synapse release 1.58.0rc1.
+
+ -- Synapse Packaging team <packages@matrix.org>  Tue, 26 Apr 2022 11:15:20 +0100
+
+matrix-synapse-py3 (1.57.1) stable; urgency=medium
+
+  * New synapse release 1.57.1.
+
+ -- Synapse Packaging team <packages@matrix.org>  Wed, 20 Apr 2022 15:27:21 +0100
+
+matrix-synapse-py3 (1.57.0) stable; urgency=medium
+
+  * New synapse release 1.57.0.
+
+ -- Synapse Packaging team <packages@matrix.org>  Tue, 19 Apr 2022 10:58:42 +0100
+
+matrix-synapse-py3 (1.57.0~rc1) stable; urgency=medium
+
+  * New synapse release 1.57.0~rc1.
+
+ -- Synapse Packaging team <packages@matrix.org>  Tue, 12 Apr 2022 13:36:25 +0100
+
+matrix-synapse-py3 (1.56.0) stable; urgency=medium
+
+  * New synapse release 1.56.0.
+
+ -- Synapse Packaging team <packages@matrix.org>  Tue, 05 Apr 2022 12:38:39 +0100
+
+matrix-synapse-py3 (1.56.0~rc1) stable; urgency=medium
+
+  * New synapse release 1.56.0~rc1.
+
+ -- Synapse Packaging team <packages@matrix.org>  Tue, 29 Mar 2022 10:40:50 +0100
+
+matrix-synapse-py3 (1.55.2) stable; urgency=medium
+
+  * New synapse release 1.55.2.
+
+ -- Synapse Packaging team <packages@matrix.org>  Thu, 24 Mar 2022 19:07:11 +0000
+
+matrix-synapse-py3 (1.55.1) stable; urgency=medium
+
+  * New synapse release 1.55.1.
+
+ -- Synapse Packaging team <packages@matrix.org>  Thu, 24 Mar 2022 17:44:23 +0000
+
+matrix-synapse-py3 (1.55.0) stable; urgency=medium
+
+  * New synapse release 1.55.0.
+
+ -- Synapse Packaging team <packages@matrix.org>  Tue, 22 Mar 2022 13:59:26 +0000
+
+matrix-synapse-py3 (1.55.0~rc1) stable; urgency=medium
+
+  * New synapse release 1.55.0~rc1.
+
+ -- Synapse Packaging team <packages@matrix.org>  Tue, 15 Mar 2022 10:59:31 +0000
+
 matrix-synapse-py3 (1.54.0) stable; urgency=medium
 
   * New synapse release 1.54.0.

debian/clean

@@ -0,0 +1 @@
+exported_requirements.txt

demo/.gitignore

@@ -1,7 +1,4 @@
-*.db
-*.log
-*.log.*
-*.pid
-
-/media_store.*
-/etc
+# Ignore all the temporary files from the demo servers.
+8080/
+8081/
+8082/

demo/README

@@ -1,26 +0,0 @@
-DO NOT USE THESE DEMO SERVERS IN PRODUCTION
-
-Requires you to have done:
-    python setup.py develop
-
-
-The demo start.sh will start three synapse servers on ports 8080, 8081 and 8082, with host names localhost:$port. This can be easily changed to `hostname`:$port in start.sh if required.
-
-To enable the servers to communicate untrusted ssl certs are used. In order to do this the servers do not check the certs
-and are configured in a highly insecure way. Do not use these configuration files in production.
-
-stop.sh will stop the synapse servers and the webclient.
-
-clean.sh will delete the databases and log files.
-
-To start a completely new set of servers, run:
-
-    ./demo/stop.sh; ./demo/clean.sh && ./demo/start.sh
-
-
-Logs and sqlitedb will be stored in demo/808{0,1,2}.{log,db}
-
-
-
-Also note that when joining a public room on a different HS via "#foo:bar.net", then you are (in the current impl) joining a room with room_id "foo". This means that it won't work if your HS already has a room with that name.
-

demo/clean.sh

@@ -4,6 +4,9 @@ set -e
 
 DIR="$( cd "$( dirname "$0" )" && pwd )"
 
+# Ensure that the servers are stopped.
+$DIR/stop.sh
+
 PID_FILE="$DIR/servers.pid"
 
 if [ -f "$PID_FILE" ]; then

demo/start.sh

@@ -6,8 +6,6 @@ CWD=$(pwd)
 
 cd "$DIR/.." || exit
 
-mkdir -p demo/etc
-
 PYTHONPATH=$(readlink -f "$(pwd)")
 export PYTHONPATH
 
@@ -21,22 +19,27 @@ for port in 8080 8081 8082; do
     mkdir -p demo/$port
     pushd demo/$port || exit
 
-    #rm $DIR/etc/$port.config
+    # Generate the configuration for the homeserver at localhost:848x.
     python3 -m synapse.app.homeserver \
         --generate-config \
-        -H "localhost:$https_port" \
-        --config-path "$DIR/etc/$port.config" \
+        --server-name "localhost:$port" \
+        --config-path "$port.config" \
         --report-stats no
 
-    if ! grep -F "Customisation made by demo/start.sh" -q "$DIR/etc/$port.config"; then
-        # Generate tls keys
-        openssl req -x509 -newkey rsa:4096 -keyout "$DIR/etc/localhost:$https_port.tls.key" -out "$DIR/etc/localhost:$https_port.tls.crt" -days 365 -nodes -subj "/O=matrix"
+    if ! grep -F "Customisation made by demo/start.sh" -q "$port.config"; then
+        # Generate TLS keys.
+        openssl req -x509 -newkey rsa:4096 \
+          -keyout "localhost:$port.tls.key" \
+          -out "localhost:$port.tls.crt" \
+          -days 365 -nodes -subj "/O=matrix"
 
-        # Regenerate configuration
+        # Add customisations to the configuration.
         {
-            printf '\n\n# Customisation made by demo/start.sh\n'
+            printf '\n\n# Customisation made by demo/start.sh\n\n'
             echo "public_baseurl: http://localhost:$port/"
             echo 'enable_registration: true'
+            echo 'enable_registration_without_verification: true'
+            echo ''
 
 			# Warning, this heredoc depends on the interaction of tabs and spaces.
 			# Please don't accidentaly bork me with your fancy settings.
@@ -63,38 +66,34 @@ for port in 8080 8081 8082; do
 
             echo "${listeners}"
 
-            # Disable tls for the servers
-            printf '\n\n# Disable tls on the servers.'
+            # Disable TLS for the servers
+            printf '\n\n# Disable TLS for the servers.'
             echo '# DO NOT USE IN PRODUCTION'
             echo 'use_insecure_ssl_client_just_for_testing_do_not_use: true'
             echo 'federation_verify_certificates: false'
 
-            # Set tls paths
-            echo "tls_certificate_path: \"$DIR/etc/localhost:$https_port.tls.crt\""
-            echo "tls_private_key_path: \"$DIR/etc/localhost:$https_port.tls.key\""
+            # Set paths for the TLS certificates.
+            echo "tls_certificate_path: \"$DIR/$port/localhost:$port.tls.crt\""
+            echo "tls_private_key_path: \"$DIR/$port/localhost:$port.tls.key\""
 
             # Ignore keys from the trusted keys server
             echo '# Ignore keys from the trusted keys server'
             echo 'trusted_key_servers:'
             echo '  - server_name: "matrix.org"'
             echo '    accept_keys_insecurely: true'
+            echo ''
 
-			# Reduce the blacklist
-			blacklist=$(cat <<-BLACK
-			# Set the blacklist so that it doesn't include 127.0.0.1, ::1
-			federation_ip_range_blacklist:
-			  - '10.0.0.0/8'
-			  - '172.16.0.0/12'
-			  - '192.168.0.0/16'
-			  - '100.64.0.0/10'
-			  - '169.254.0.0/16'
-			  - 'fe80::/64'
-			  - 'fc00::/7'
-			BLACK
+			# Allow the servers to communicate over localhost.
+			allow_list=$(cat <<-ALLOW_LIST
+			# Allow the servers to communicate over localhost.
+			ip_range_whitelist:
+			  - '127.0.0.1/8'
+			  - '::1/128'
+			ALLOW_LIST
 			)
 
-            echo "${blacklist}"
-        } >> "$DIR/etc/$port.config"
+            echo "${allow_list}"
+        } >> "$port.config"
     fi
 
     # Check script parameters
@@ -141,19 +140,18 @@ for port in 8080 8081 8082; do
 			    burst_count: 1000
 			RC
 			)
-            echo "${ratelimiting}" >> "$DIR/etc/$port.config"
+            echo "${ratelimiting}" >> "$port.config"
         fi
     fi
 
-    if ! grep -F "full_twisted_stacktraces" -q  "$DIR/etc/$port.config"; then
-        echo "full_twisted_stacktraces: true" >> "$DIR/etc/$port.config"
-    fi
-    if ! grep -F "report_stats" -q  "$DIR/etc/$port.config" ; then
-        echo "report_stats: false" >> "$DIR/etc/$port.config"
+    # Always disable reporting of stats if the option is not there.
+    if ! grep -F "report_stats" -q  "$port.config" ; then
+        echo "report_stats: false" >> "$port.config"
     fi
 
+    # Run the homeserver in the background.
     python3 -m synapse.app.homeserver \
-        --config-path "$DIR/etc/$port.config" \
+        --config-path "$port.config" \
         -D \
 
     popd || exit

docker/Dockerfile

@@ -14,20 +14,61 @@
 #    DOCKER_BUILDKIT=1 docker build -f docker/Dockerfile --build-arg PYTHON_VERSION=3.10 .
 #
 
+# Irritatingly, there is no blessed guide on how to distribute an application with its
+# poetry-managed environment in a docker image. We have opted for
+# `poetry export | pip install -r /dev/stdin`, but there are known bugs in
+# in `poetry export` whose fixes (scheduled for poetry 1.2) have yet to be released.
+# In case we get bitten by those bugs in the future, the recommendations here might
+# be useful:
+#     https://github.com/python-poetry/poetry/discussions/1879#discussioncomment-216865
+#     https://stackoverflow.com/questions/53835198/integrating-python-poetry-with-docker?answertab=scoredesc
+
+
+
 ARG PYTHON_VERSION=3.9
 
 ###
-### Stage 0: builder
+### Stage 0: generate requirements.txt
+###
+FROM docker.io/python:${PYTHON_VERSION}-slim as requirements
+
+# RUN --mount is specific to buildkit and is documented at
+# https://github.com/moby/buildkit/blob/master/frontend/dockerfile/docs/syntax.md#build-mounts-run---mount.
+# Here we use it to set up a cache for apt (and below for pip), to improve
+# rebuild speeds on slow connections.
+RUN \
+   --mount=type=cache,target=/var/cache/apt,sharing=locked \
+   --mount=type=cache,target=/var/lib/apt,sharing=locked \
+ apt-get update && apt-get install -y git \
+    && rm -rf /var/lib/apt/lists/*
+
+# We install poetry in its own build stage to avoid its dependencies conflicting with
+# synapse's dependencies.
+# We use a specific commit from poetry's master branch instead of our usual 1.1.12,
+# to incorporate fixes to some bugs in `poetry export`. This commit corresponds to
+#    https://github.com/python-poetry/poetry/pull/5156 and
+#    https://github.com/python-poetry/poetry/issues/5141 ;
+# without it, we generate a requirements.txt with incorrect environment markers,
+# which causes necessary packages to be omitted when we `pip install`.
+#
+# NB: In poetry 1.2 `poetry export` will be moved into a plugin; we'll need to also
+# pip install poetry-plugin-export (https://github.com/python-poetry/poetry-plugin-export).
+RUN --mount=type=cache,target=/root/.cache/pip \
+  pip install --user git+https://github.com/python-poetry/poetry.git@fb13b3a676f476177f7937ffa480ee5cff9a90a5
+
+WORKDIR /synapse
+
+# Copy just what we need to run `poetry export`...
+COPY pyproject.toml poetry.lock /synapse/
+
+RUN /root/.local/bin/poetry export --extras all -o /synapse/requirements.txt
+
+###
+### Stage 1: builder
 ###
 FROM docker.io/python:${PYTHON_VERSION}-slim as builder
 
 # install the OS build deps
-#
-# RUN --mount is specific to buildkit and is documented at
-# https://github.com/moby/buildkit/blob/master/frontend/dockerfile/docs/syntax.md#build-mounts-run---mount.
-# Here we use it to set up a cache for apt, to improve rebuild speeds on
-# slow connections.
-#
 RUN \
    --mount=type=cache,target=/var/cache/apt,sharing=locked \
    --mount=type=cache,target=/var/lib/apt,sharing=locked \
@@ -45,31 +86,25 @@ RUN \
     zlib1g-dev \
     && rm -rf /var/lib/apt/lists/*
 
-# Copy just what we need to pip install
-COPY scripts /synapse/scripts/
-COPY MANIFEST.in README.rst setup.py synctl /synapse/
-COPY synapse/__init__.py /synapse/synapse/__init__.py
-COPY synapse/python_dependencies.py /synapse/synapse/python_dependencies.py
-
 # To speed up rebuilds, install all of the dependencies before we copy over
-# the whole synapse project so that we this layer in the Docker cache can be
+# the whole synapse project, so that this layer in the Docker cache can be
 # used while you develop on the source
 #
-# This is aiming at installing the `install_requires` and `extras_require` from `setup.py`
+# This is aiming at installing the `[tool.poetry.depdendencies]` from pyproject.toml.
+COPY --from=requirements /synapse/requirements.txt /synapse/
 RUN --mount=type=cache,target=/root/.cache/pip \
-  pip install --prefix="/install" --no-warn-script-location \
-    /synapse[all]
+  pip install --prefix="/install" --no-deps --no-warn-script-location -r /synapse/requirements.txt
 
-# Copy over the rest of the project
+# Copy over the rest of the synapse source code.
 COPY synapse /synapse/synapse/
+# ... and what we need to `pip install`.
+COPY pyproject.toml README.rst /synapse/
 
-# Install the synapse package itself and all of its children packages.
-#
-# This is aiming at installing only the `packages=find_packages(...)` from `setup.py
+# Install the synapse package itself.
 RUN pip install --prefix="/install" --no-deps --no-warn-script-location /synapse
 
 ###
-### Stage 1: runtime
+### Stage 2: runtime
 ###
 
 FROM docker.io/python:${PYTHON_VERSION}-slim

docker/Dockerfile-pgtests

@@ -1,30 +0,0 @@
-# Use the Sytest image that comes with a lot of the build dependencies
-# pre-installed
-FROM matrixdotorg/sytest:focal
-
-# The Sytest image doesn't come with python, so install that
-RUN apt-get update && apt-get -qq install -y python3 python3-dev python3-pip
-
-# We need tox to run the tests in run_pg_tests.sh
-RUN python3 -m pip install tox
-
-# Initialise the db
-RUN su -c '/usr/lib/postgresql/10/bin/initdb -D /var/lib/postgresql/data -E "UTF-8" --lc-collate="C.UTF-8" --lc-ctype="C.UTF-8" --username=postgres' postgres
-
-# Add a user with our UID and GID so that files get created on the host owned
-# by us, not root.
-ARG UID
-ARG GID
-RUN groupadd --gid $GID user
-RUN useradd --uid $UID --gid $GID --groups sudo --no-create-home user
-
-# Ensure we can start postgres by sudo-ing as the postgres user.
-RUN apt-get update && apt-get -qq install -y sudo
-RUN echo "user ALL=(ALL) NOPASSWD: ALL" >> /etc/sudoers
-
-ADD run_pg_tests.sh /run_pg_tests.sh
-# Use the "exec form" of ENTRYPOINT (https://docs.docker.com/engine/reference/builder/#entrypoint)
-# so that we can `docker run` this container and pass arguments to pg_tests.sh
-ENTRYPOINT ["/run_pg_tests.sh"]
-
-USER user

docker/Dockerfile-workers

@@ -2,10 +2,19 @@
 FROM matrixdotorg/synapse
 
 # Install deps
-RUN apt-get update
-RUN apt-get install -y supervisor redis nginx
+RUN \
+   --mount=type=cache,target=/var/cache/apt,sharing=locked \
+   --mount=type=cache,target=/var/lib/apt,sharing=locked \
+  apt-get update && \
+  DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
+     redis-server nginx-light
 
-# Remove the default nginx sites
+# Install supervisord with pip instead of apt, to avoid installing a second
+# copy of python.
+RUN --mount=type=cache,target=/root/.cache/pip \
+    pip install supervisor~=4.2
+
+# Disable the default nginx sites
 RUN rm /etc/nginx/sites-enabled/default
 
 # Copy Synapse worker, nginx and supervisord configuration template files
@@ -14,13 +23,12 @@ COPY ./docker/conf-workers/* /conf/
 # Expose nginx listener port
 EXPOSE 8080/tcp
 
-# Volume for user-editable config files, logs etc.
-VOLUME ["/data"]
-
 # A script to read environment variables and create the necessary
 # files to run the desired worker configuration. Will start supervisord.
 COPY ./docker/configure_workers_and_start.py /configure_workers_and_start.py
 ENTRYPOINT ["/configure_workers_and_start.py"]
 
+# Replace the healthcheck with one which checks *all* the workers. The script
+# is generated by configure_workers_and_start.py.
 HEALTHCHECK --start-period=5s --interval=15s --timeout=5s \
     CMD /bin/sh /healthcheck.sh

docker/README-testing.md

@@ -10,10 +10,10 @@ Note that running Synapse's unit tests from within the docker image is not suppo
 
 ## Testing with SQLite and single-process Synapse
 
-> Note that `scripts-dev/complement.sh` is a script that will automatically build 
+> Note that `scripts-dev/complement.sh` is a script that will automatically build
 > and run an SQLite-based, single-process of Synapse against Complement.
 
-The instructions below will set up Complement testing for a single-process, 
+The instructions below will set up Complement testing for a single-process,
 SQLite-based Synapse deployment.
 
 Start by building the base Synapse docker image. If you wish to run tests with the latest
@@ -26,23 +26,22 @@ docker build -t matrixdotorg/synapse -f docker/Dockerfile .
 
 This will build an image with the tag `matrixdotorg/synapse`.
 
-Next, build the Synapse image for Complement. You will need a local checkout 
-of Complement. Change to the root of your Complement checkout and run:
+Next, build the Synapse image for Complement.
 
 ```sh
-docker build -t complement-synapse -f "dockerfiles/Synapse.Dockerfile" dockerfiles
+docker build -t complement-synapse -f "docker/complement/Dockerfile" docker/complement
 ```
 
-This will build an image with the tag `complement-synapse`, which can be handed to 
-Complement for testing via the `COMPLEMENT_BASE_IMAGE` environment variable. Refer to 
-[Complement's documentation](https://github.com/matrix-org/complement/#running) for 
+This will build an image with the tag `complement-synapse`, which can be handed to
+Complement for testing via the `COMPLEMENT_BASE_IMAGE` environment variable. Refer to
+[Complement's documentation](https://github.com/matrix-org/complement/#running) for
 how to run the tests, as well as the various available command line flags.
 
 ## Testing with PostgreSQL and single or multi-process Synapse
 
-The above docker image only supports running Synapse with SQLite and in a 
-single-process topology. The following instructions are used to build a Synapse image for 
-Complement that supports either single or multi-process topology with a PostgreSQL 
+The above docker image only supports running Synapse with SQLite and in a
+single-process topology. The following instructions are used to build a Synapse image for
+Complement that supports either single or multi-process topology with a PostgreSQL
 database backend.
 
 As with the single-process image, build the base Synapse docker image. If you wish to run
@@ -55,7 +54,7 @@ docker build -t matrixdotorg/synapse -f docker/Dockerfile .
 
 This will build an image with the tag `matrixdotorg/synapse`.
 
-Next, we build a new image with worker support based on `matrixdotorg/synapse:latest`. 
+Next, we build a new image with worker support based on `matrixdotorg/synapse:latest`.
 Again, from the root of the repository:
 
 ```sh
@@ -64,21 +63,20 @@ docker build -t matrixdotorg/synapse-workers -f docker/Dockerfile-workers .
 
 This will build an image with the tag` matrixdotorg/synapse-workers`.
 
-It's worth noting at this point that this image is fully functional, and 
-can be used for testing against locally. See instructions for using the container 
+It's worth noting at this point that this image is fully functional, and
+can be used for testing against locally. See instructions for using the container
 under
 [Running the Dockerfile-worker image standalone](#running-the-dockerfile-worker-image-standalone)
 below.
 
 Finally, build the Synapse image for Complement, which is based on
-`matrixdotorg/synapse-workers`. You will need a local checkout of Complement. Change to
-the root of your Complement checkout and run:
+`matrixdotorg/synapse-workers`.
 
 ```sh
-docker build -t matrixdotorg/complement-synapse-workers -f dockerfiles/SynapseWorkers.Dockerfile dockerfiles
+docker build -t matrixdotorg/complement-synapse-workers -f docker/complement/SynapseWorkers.Dockerfile docker/complement
 ```
 
-This will build an image with the tag `complement-synapse`, which can be handed to
+This will build an image with the tag `complement-synapse-workers`, which can be handed to
 Complement for testing via the `COMPLEMENT_BASE_IMAGE` environment variable. Refer to
 [Complement's documentation](https://github.com/matrix-org/complement/#running) for
 how to run the tests, as well as the various available command line flags.
@@ -91,10 +89,10 @@ bundling all necessary components together for a workerised homeserver instance.
 
 This includes any desired Synapse worker processes, a nginx to route traffic accordingly,
 a redis for worker communication and a supervisord instance to start up and monitor all
-processes. You will need to provide your own postgres container to connect to, and TLS 
+processes. You will need to provide your own postgres container to connect to, and TLS
 is not handled by the container.
 
-Once you've built the image using the above instructions, you can run it. Be sure 
+Once you've built the image using the above instructions, you can run it. Be sure
 you've set up a volume according to the [usual Synapse docker instructions](README.md).
 Then run something along the lines of:
 
@@ -112,7 +110,7 @@ docker run -d --name synapse \
     matrixdotorg/synapse-workers
 ```
 
-...substituting `POSTGRES*` variables for those that match a postgres host you have 
+...substituting `POSTGRES*` variables for those that match a postgres host you have
 available (usually a running postgres docker container).
 
 The `SYNAPSE_WORKER_TYPES` environment variable is a comma-separated list of workers to
@@ -130,11 +128,11 @@ Otherwise, `SYNAPSE_WORKER_TYPES` can either be left empty or unset to spawn no
 (leaving only the main process). The container is configured to use redis-based worker
 mode.
 
-Logs for workers and the main process are logged to stdout and can be viewed with 
-standard `docker logs` tooling. Worker logs contain their worker name 
+Logs for workers and the main process are logged to stdout and can be viewed with
+standard `docker logs` tooling. Worker logs contain their worker name
 after the timestamp.
 
 Setting `SYNAPSE_WORKERS_WRITE_LOGS_TO_DISK=1` will cause worker logs to be written to
 `<data_dir>/logs/<worker_name>.log`. Logs are kept for 1 week and rotate every day at 00:
-00, according to the container's clock. Logging for the main process must still be 
+00, according to the container's clock. Logging for the main process must still be
 configured by modifying the homeserver's log config in your Synapse data volume.

docker/complement/Dockerfile

@@ -0,0 +1,22 @@
+# A dockerfile which builds an image suitable for testing Synapse under
+# complement.
+
+ARG SYNAPSE_VERSION=latest
+
+FROM matrixdotorg/synapse:${SYNAPSE_VERSION}
+
+ENV SERVER_NAME=localhost
+
+COPY conf/* /conf/
+
+# generate a signing key
+RUN generate_signing_key -o /conf/server.signing.key
+
+WORKDIR /data
+
+EXPOSE 8008 8448
+
+ENTRYPOINT ["/conf/start.sh"]
+
+HEALTHCHECK --start-period=5s --interval=1s --timeout=1s \
+    CMD curl -fSs http://localhost:8008/health || exit 1

docker/complement/README.md

@@ -0,0 +1 @@
+Stuff for building the docker image used for testing under complement.

docker/complement/SynapseWorkers.Dockerfile

@@ -0,0 +1,47 @@
+# This dockerfile builds on top of 'docker/Dockerfile-worker' in matrix-org/synapse
+# by including a built-in postgres instance, as well as setting up the homeserver so
+# that it is ready for testing via Complement.
+#
+# Instructions for building this image from those it depends on is detailed in this guide:
+# https://github.com/matrix-org/synapse/blob/develop/docker/README-testing.md#testing-with-postgresql-and-single-or-multi-process-synapse
+FROM matrixdotorg/synapse-workers
+
+# Download a caddy server to stand in front of nginx and terminate TLS using Complement's
+# custom CA.
+# We include this near the top of the file in order to cache the result.
+RUN curl -OL "https://github.com/caddyserver/caddy/releases/download/v2.3.0/caddy_2.3.0_linux_amd64.tar.gz" && \
+  tar xzf caddy_2.3.0_linux_amd64.tar.gz && rm caddy_2.3.0_linux_amd64.tar.gz && mv caddy /root
+
+# Install postgresql
+RUN apt-get update && \
+  DEBIAN_FRONTEND=noninteractive apt-get install --no-install-recommends -y postgresql-13
+
+# Configure a user and create a database for Synapse
+RUN pg_ctlcluster 13 main start &&  su postgres -c "echo \
+ \"ALTER USER postgres PASSWORD 'somesecret'; \
+ CREATE DATABASE synapse \
+  ENCODING 'UTF8' \
+  LC_COLLATE='C' \
+  LC_CTYPE='C' \
+  template=template0;\" | psql" && pg_ctlcluster 13 main stop
+
+# Modify the shared homeserver config with postgres support, certificate setup
+# and the disabling of rate-limiting
+COPY conf-workers/workers-shared.yaml /conf/workers/shared.yaml
+
+WORKDIR /data
+
+# Copy the caddy config
+COPY conf-workers/caddy.complement.json /root/caddy.json
+
+# Copy the entrypoint
+COPY conf-workers/start-complement-synapse-workers.sh /
+
+# Expose caddy's listener ports
+EXPOSE 8008 8448
+
+ENTRYPOINT /start-complement-synapse-workers.sh
+
+# Update the healthcheck to have a shorter check interval
+HEALTHCHECK --start-period=5s --interval=1s --timeout=1s \
+    CMD /bin/sh /healthcheck.sh

docker/complement/conf-workers/caddy.complement.json

@@ -0,0 +1,72 @@
+{
+    "apps": {
+      "http": {
+        "servers": {
+          "srv0": {
+            "listen": [
+              ":8448"
+            ],
+            "routes": [
+              {
+                "match": [
+                  {
+                    "host": [
+                      "{{ server_name }}"
+                    ]
+                  }
+                ],
+                "handle": [
+                  {
+                    "handler": "subroute",
+                    "routes": [
+                      {
+                        "handle": [
+                          {
+                            "handler": "reverse_proxy",
+                            "upstreams": [
+                              {
+                                "dial": "localhost:8008"
+                              }
+                            ]
+                          }
+                        ]
+                      }
+                    ]
+                  }
+                ],
+                "terminal": true
+              }
+            ]
+          }
+        }
+      },
+      "tls": {
+        "automation": {
+          "policies": [
+            {
+              "subjects": [
+                "{{ server_name }}"
+              ],
+              "issuers": [
+                {
+                  "module": "internal"
+                }
+              ],
+              "on_demand": true
+            }
+          ]
+        }
+      },
+      "pki": {
+        "certificate_authorities": {
+          "local": {
+            "name": "Complement CA",
+            "root": {
+              "certificate": "/complement/ca/ca.crt",
+              "private_key": "/complement/ca/ca.key"
+            }
+          }
+        }
+      }
+    }
+  }

docker/complement/conf-workers/start-complement-synapse-workers.sh

@@ -0,0 +1,50 @@
+#!/bin/bash
+#
+# Default ENTRYPOINT for the docker image used for testing synapse with workers under complement
+
+set -e
+
+function log {
+    d=$(date +"%Y-%m-%d %H:%M:%S,%3N")
+    echo "$d $@"
+}
+
+# Replace the server name in the caddy config
+sed -i "s/{{ server_name }}/${SERVER_NAME}/g" /root/caddy.json
+
+log "starting postgres"
+pg_ctlcluster 13 main start
+
+log "starting caddy"
+/root/caddy start --config /root/caddy.json
+
+# Set the server name of the homeserver
+export SYNAPSE_SERVER_NAME=${SERVER_NAME}
+
+# No need to report stats here
+export SYNAPSE_REPORT_STATS=no
+
+# Set postgres authentication details which will be placed in the homeserver config file
+export POSTGRES_PASSWORD=somesecret
+export POSTGRES_USER=postgres
+export POSTGRES_HOST=localhost
+
+# Specify the workers to test with
+export SYNAPSE_WORKER_TYPES="\
+    event_persister, \
+    event_persister, \
+    background_worker, \
+    frontend_proxy, \
+    event_creator, \
+    user_dir, \
+    media_repository, \
+    federation_inbound, \
+    federation_reader, \
+    federation_sender, \
+    synchrotron, \
+    appservice, \
+    pusher"
+
+# Run the script that writes the necessary config files and starts supervisord, which in turn
+# starts everything else
+exec /configure_workers_and_start.py

docker/complement/conf-workers/workers-shared.yaml

@@ -0,0 +1,72 @@
+## Server ##
+report_stats: False
+trusted_key_servers: []
+enable_registration: true
+enable_registration_without_verification: true
+bcrypt_rounds: 4
+
+## Federation ##
+
+# trust certs signed by Complement's CA
+federation_custom_ca_list:
+- /complement/ca/ca.crt
+
+# unblacklist RFC1918 addresses
+federation_ip_range_blacklist: []
+
+# Disable server rate-limiting
+rc_federation:
+  window_size: 1000
+  sleep_limit: 10
+  sleep_delay: 500
+  reject_limit: 99999
+  concurrent: 3
+
+rc_message:
+  per_second: 9999
+  burst_count: 9999
+
+rc_registration:
+  per_second: 9999
+  burst_count: 9999
+
+rc_login:
+  address:
+    per_second: 9999
+    burst_count: 9999
+  account:
+    per_second: 9999
+    burst_count: 9999
+  failed_attempts:
+    per_second: 9999
+    burst_count: 9999
+
+rc_admin_redaction:
+  per_second: 9999
+  burst_count: 9999
+
+rc_joins:
+  local:
+    per_second: 9999
+    burst_count: 9999
+  remote:
+    per_second: 9999
+    burst_count: 9999
+
+federation_rr_transactions_per_room_per_second: 9999
+
+## Experimental Features ##
+
+experimental_features:
+  # Enable history backfilling support
+  msc2716_enabled: true
+  # Enable spaces support
+  spaces_enabled: true
+  # Enable jump to date endpoint
+  msc3030_enabled: true
+
+server_notices:
+  system_mxid_localpart: _server
+  system_mxid_display_name: "Server Alert"
+  system_mxid_avatar_url: ""
+  room_name: "Server Alert"

docker/complement/conf/homeserver.yaml

@@ -0,0 +1,117 @@
+## Server ##
+
+server_name: SERVER_NAME
+log_config: /conf/log_config.yaml
+report_stats: False
+signing_key_path: /conf/server.signing.key
+trusted_key_servers: []
+enable_registration: true
+enable_registration_without_verification: true
+
+## Listeners ##
+
+tls_certificate_path: /conf/server.tls.crt
+tls_private_key_path: /conf/server.tls.key
+bcrypt_rounds: 4
+registration_shared_secret: complement
+
+listeners:
+  - port: 8448
+    bind_addresses: ['::']
+    type: http
+    tls: true
+    resources:
+      - names: [federation]
+
+  - port: 8008
+    bind_addresses: ['::']
+    type: http
+
+    resources:
+      - names: [client]
+
+## Database ##
+
+database:
+  name: "sqlite3"
+  args:
+    # We avoid /data, as it is a volume and is not transferred when the container is committed,
+    # which is a fundamental necessity in complement.
+    database: "/conf/homeserver.db"
+
+## Federation ##
+
+# trust certs signed by the complement CA
+federation_custom_ca_list:
+- /complement/ca/ca.crt
+
+# unblacklist RFC1918 addresses
+ip_range_blacklist: []
+
+# Disable server rate-limiting
+rc_federation:
+  window_size: 1000
+  sleep_limit: 10
+  sleep_delay: 500
+  reject_limit: 99999
+  concurrent: 3
+
+rc_message:
+  per_second: 9999
+  burst_count: 9999
+
+rc_registration:
+  per_second: 9999
+  burst_count: 9999
+
+rc_login:
+  address:
+    per_second: 9999
+    burst_count: 9999
+  account:
+    per_second: 9999
+    burst_count: 9999
+  failed_attempts:
+    per_second: 9999
+    burst_count: 9999
+
+rc_admin_redaction:
+  per_second: 9999
+  burst_count: 9999
+
+rc_joins:
+  local:
+    per_second: 9999
+    burst_count: 9999
+  remote:
+    per_second: 9999
+    burst_count: 9999
+
+federation_rr_transactions_per_room_per_second: 9999
+
+## API Configuration ##
+
+# A list of application service config files to use
+#
+app_service_config_files:
+AS_REGISTRATION_FILES  
+
+## Experimental Features ##
+
+experimental_features:
+  # Enable spaces support
+  spaces_enabled: true
+  # Enable history backfilling support
+  msc2716_enabled: true
+  # server-side support for partial state in /send_join responses
+  msc3706_enabled: true
+  # client-side support for partial state in /send_join responses
+  faster_joins: true
+  # Enable jump to date endpoint
+  msc3030_enabled: true
+
+server_notices:
+  system_mxid_localpart: _server
+  system_mxid_display_name: "Server Alert"
+  system_mxid_avatar_url: ""
+  room_name: "Server Alert"

docker/complement/conf/log_config.yaml

@@ -0,0 +1,24 @@
+version: 1
+
+formatters:
+  precise:
+   format: '%(asctime)s - %(name)s - %(lineno)d - %(levelname)s - %(request)s - %(message)s'
+
+filters:
+  context:
+    (): synapse.logging.context.LoggingContextFilter
+    request: ""
+
+handlers:
+  console:
+    class: logging.StreamHandler
+    formatter: precise
+    filters: [context]
+    # log to stdout, for easier use with 'docker logs'
+    stream: 'ext://sys.stdout'
+
+root:
+    level: INFO
+    handlers: [console]
+
+disable_existing_loggers: false

docker/complement/conf/start.sh

@@ -0,0 +1,30 @@
+#!/bin/sh
+
+set -e
+
+sed -i "s/SERVER_NAME/${SERVER_NAME}/g" /conf/homeserver.yaml
+
+# Add the application service registration files to the homeserver.yaml config
+for filename in /complement/appservice/*.yaml; do
+  [ -f "$filename" ] || break
+
+  as_id=$(basename "$filename" .yaml)
+
+  # Insert the path to the registration file and the AS_REGISTRATION_FILES marker after 
+  # so we can add the next application service in the next iteration of this for loop
+  sed -i "s/AS_REGISTRATION_FILES/  - \/complement\/appservice\/${as_id}.yaml\nAS_REGISTRATION_FILES/g" /conf/homeserver.yaml
+done
+# Remove the AS_REGISTRATION_FILES entry
+sed -i "s/AS_REGISTRATION_FILES//g" /conf/homeserver.yaml
+
+# generate an ssl key and cert for the server, signed by the complement CA
+openssl genrsa -out /conf/server.tls.key 2048
+
+openssl req -new -key /conf/server.tls.key -out /conf/server.tls.csr \
+  -subj "/CN=${SERVER_NAME}"
+openssl x509 -req -in /conf/server.tls.csr \
+  -CA /complement/ca/ca.crt -CAkey /complement/ca/ca.key -set_serial 1 \
+  -out /conf/server.tls.crt
+
+exec python -m synapse.app.homeserver -c /conf/homeserver.yaml "$@"
+

docker/conf-workers/supervisord.conf.j2

@@ -5,6 +5,9 @@
 nodaemon=true
 user=root
 
+[include]
+files = /etc/supervisor/conf.d/*.conf
+
 [program:nginx]
 command=/usr/sbin/nginx -g "daemon off;"
 priority=500

docker/configure_workers_and_start.py

@@ -29,6 +29,7 @@
 import os
 import subprocess
 import sys
+from typing import Any, Dict, List, Mapping, MutableMapping, NoReturn, Set
 
 import jinja2
 import yaml
@@ -36,7 +37,7 @@ import yaml
 MAIN_PROCESS_HTTP_LISTENER_PORT = 8080
 
 
-WORKERS_CONFIG = {
+WORKERS_CONFIG: Dict[str, Dict[str, Any]] = {
     "pusher": {
         "app": "synapse.app.pusher",
         "listener_resources": [],
@@ -200,7 +201,7 @@ upstream {upstream_worker_type} {{
 
 
 # Utility functions
-def log(txt: str):
+def log(txt: str) -> None:
     """Log something to the stdout.
 
     Args:
@@ -209,7 +210,7 @@ def log(txt: str):
     print(txt)
 
 
-def error(txt: str):
+def error(txt: str) -> NoReturn:
     """Log something and exit with an error code.
 
     Args:
@@ -219,7 +220,7 @@ def error(txt: str):
     sys.exit(2)
 
 
-def convert(src: str, dst: str, **template_vars):
+def convert(src: str, dst: str, **template_vars: object) -> None:
     """Generate a file from a template
 
     Args:
@@ -289,7 +290,7 @@ def add_sharding_to_shared_config(
         shared_config.setdefault("media_instance_running_background_jobs", worker_name)
 
 
-def generate_base_homeserver_config():
+def generate_base_homeserver_config() -> None:
     """Starts Synapse and generates a basic homeserver config, which will later be
     modified for worker support.
 
@@ -301,13 +302,15 @@ def generate_base_homeserver_config():
     subprocess.check_output(["/usr/local/bin/python", "/start.py", "migrate_config"])
 
 
-def generate_worker_files(environ, config_path: str, data_dir: str):
+def generate_worker_files(
+    environ: Mapping[str, str], config_path: str, data_dir: str
+) -> None:
     """Read the desired list of workers from environment variables and generate
     shared homeserver, nginx and supervisord configs.
 
     Args:
-        environ: _Environ[str]
-        config_path: Where to output the generated Synapse main worker config file.
+        environ: os.environ instance.
+        config_path: The location of the generated Synapse main worker config file.
         data_dir: The location of the synapse data directory. Where log and
             user-facing config files live.
     """
@@ -320,7 +323,8 @@ def generate_worker_files(environ, config_path: str, data_dir: str):
     # and adding a replication listener.
 
     # First read the original config file and extract the listeners block. Then we'll add
-    # another listener for replication. Later we'll write out the result.
+    # another listener for replication. Later we'll write out the result to the shared
+    # config file.
     listeners = [
         {
             "port": 9093,
@@ -339,7 +343,7 @@ def generate_worker_files(environ, config_path: str, data_dir: str):
     # base shared worker jinja2 template.
     #
     # This config file will be passed to all workers, included Synapse's main process.
-    shared_config = {"listeners": listeners}
+    shared_config: Dict[str, Any] = {"listeners": listeners}
 
     # The supervisord config. The contents of which will be inserted into the
     # base supervisord jinja2 template.
@@ -355,7 +359,7 @@ def generate_worker_files(environ, config_path: str, data_dir: str):
     #   worker_type: {1234, 1235, ...}}
     # }
     # and will be used to construct 'upstream' nginx directives.
-    nginx_upstreams = {}
+    nginx_upstreams: Dict[str, Set[int]] = {}
 
     # A map of: {"endpoint": "upstream"}, where "upstream" is a str representing what will be
     # placed after the proxy_pass directive. The main benefit to representing this data as a
@@ -367,13 +371,13 @@ def generate_worker_files(environ, config_path: str, data_dir: str):
     nginx_locations = {}
 
     # Read the desired worker configuration from the environment
-    worker_types = environ.get("SYNAPSE_WORKER_TYPES")
-    if worker_types is None:
+    worker_types_env = environ.get("SYNAPSE_WORKER_TYPES")
+    if worker_types_env is None:
         # No workers, just the main process
         worker_types = []
     else:
         # Split type names by comma
-        worker_types = worker_types.split(",")
+        worker_types = worker_types_env.split(",")
 
     # Create the worker configuration directory if it doesn't already exist
     os.makedirs("/conf/workers", exist_ok=True)
@@ -384,7 +388,11 @@ def generate_worker_files(environ, config_path: str, data_dir: str):
     # A counter of worker_type -> int. Used for determining the name for a given
     # worker type when generating its config file, as each worker's name is just
     # worker_type + instance #
-    worker_type_counter = {}
+    worker_type_counter: Dict[str, int] = {}
+
+    # A list of internal endpoints to healthcheck, starting with the main process
+    # which exists even if no workers do.
+    healthcheck_urls = ["http://localhost:8080/health"]
 
     # For each worker type specified by the user, create config values
     for worker_type in worker_types:
@@ -404,12 +412,14 @@ def generate_worker_files(environ, config_path: str, data_dir: str):
         # e.g. federation_reader1
         worker_name = worker_type + str(new_worker_count)
         worker_config.update(
-            {"name": worker_name, "port": worker_port, "config_path": config_path}
+            {"name": worker_name, "port": str(worker_port), "config_path": config_path}
         )
 
         # Update the shared config with any worker-type specific options
         shared_config.update(worker_config["shared_extra_conf"])
 
+        healthcheck_urls.append("http://localhost:%d/health" % (worker_port,))
+
         # Check if more than one instance of this worker type has been specified
         worker_type_total_count = worker_types.count(worker_type)
         if worker_type_total_count > 1:
@@ -438,21 +448,7 @@ def generate_worker_files(environ, config_path: str, data_dir: str):
 
         # Write out the worker's logging config file
 
-        # Check whether we should write worker logs to disk, in addition to the console
-        extra_log_template_args = {}
-        if environ.get("SYNAPSE_WORKERS_WRITE_LOGS_TO_DISK"):
-            extra_log_template_args["LOG_FILE_PATH"] = "{dir}/logs/{name}.log".format(
-                dir=data_dir, name=worker_name
-            )
-
-        # Render and write the file
-        log_config_filepath = "/conf/workers/{name}.log.config".format(name=worker_name)
-        convert(
-            "/conf/log.config",
-            log_config_filepath,
-            worker_name=worker_name,
-            **extra_log_template_args,
-        )
+        log_config_filepath = generate_worker_log_config(environ, worker_name, data_dir)
 
         # Then a worker config file
         convert(
@@ -475,15 +471,10 @@ def generate_worker_files(environ, config_path: str, data_dir: str):
     # Determine the load-balancing upstreams to configure
     nginx_upstream_config = ""
 
-    # At the same time, prepare a list of internal endpoints to healthcheck
-    # starting with the main process which exists even if no workers do.
-    healthcheck_urls = ["http://localhost:8080/health"]
-
     for upstream_worker_type, upstream_worker_ports in nginx_upstreams.items():
         body = ""
         for port in upstream_worker_ports:
             body += "    server localhost:%d;\n" % (port,)
-            healthcheck_urls.append("http://localhost:%d/health" % (port,))
 
         # Add to the list of configured upstreams
         nginx_upstream_config += NGINX_UPSTREAM_CONFIG_BLOCK.format(
@@ -493,6 +484,10 @@ def generate_worker_files(environ, config_path: str, data_dir: str):
 
     # Finally, we'll write out the config files.
 
+    # log config for the master process
+    master_log_config = generate_worker_log_config(environ, "master", data_dir)
+    shared_config["log_config"] = master_log_config
+
     # Shared homeserver config
     convert(
         "/conf/shared.yaml.j2",
@@ -509,9 +504,10 @@ def generate_worker_files(environ, config_path: str, data_dir: str):
     )
 
     # Supervisord config
+    os.makedirs("/etc/supervisor", exist_ok=True)
     convert(
         "/conf/supervisord.conf.j2",
-        "/etc/supervisor/conf.d/supervisord.conf",
+        "/etc/supervisor/supervisord.conf",
         main_config_path=config_path,
         worker_config=supervisord_config,
     )
@@ -529,15 +525,31 @@ def generate_worker_files(environ, config_path: str, data_dir: str):
         os.mkdir(log_dir)
 
 
-def start_supervisord():
-    """Starts up supervisord which then starts and monitors all other necessary processes
+def generate_worker_log_config(
+    environ: Mapping[str, str], worker_name: str, data_dir: str
+) -> str:
+    """Generate a log.config file for the given worker.
 
-    Raises: CalledProcessError if calling start.py return a non-zero exit code.
+    Returns: the path to the generated file
     """
-    subprocess.run(["/usr/bin/supervisord"], stdin=subprocess.PIPE)
+    # Check whether we should write worker logs to disk, in addition to the console
+    extra_log_template_args = {}
+    if environ.get("SYNAPSE_WORKERS_WRITE_LOGS_TO_DISK"):
+        extra_log_template_args["LOG_FILE_PATH"] = "{dir}/logs/{name}.log".format(
+            dir=data_dir, name=worker_name
+        )
+    # Render and write the file
+    log_config_filepath = "/conf/workers/{name}.log.config".format(name=worker_name)
+    convert(
+        "/conf/log.config",
+        log_config_filepath,
+        worker_name=worker_name,
+        **extra_log_template_args,
+    )
+    return log_config_filepath
 
 
-def main(args, environ):
+def main(args: List[str], environ: MutableMapping[str, str]) -> None:
     config_dir = environ.get("SYNAPSE_CONFIG_DIR", "/data")
     config_path = environ.get("SYNAPSE_CONFIG_PATH", config_dir + "/homeserver.yaml")
     data_dir = environ.get("SYNAPSE_DATA_DIR", "/data")
@@ -564,7 +576,13 @@ def main(args, environ):
 
     # Start supervisord, which will start Synapse, all of the configured worker
     # processes, redis, nginx etc. according to the config we created above.
-    start_supervisord()
+    log("Starting supervisord")
+    os.execl(
+        "/usr/local/bin/supervisord",
+        "supervisord",
+        "-c",
+        "/etc/supervisor/supervisord.conf",
+    )
 
 
 if __name__ == "__main__":

docker/run_pg_tests.sh

@@ -1,19 +0,0 @@
-#!/usr/bin/env bash
-
-# This script runs the PostgreSQL tests inside a Docker container. It expects
-# the relevant source files to be mounted into /src (done automatically by the
-# caller script). It will set up the database, run it, and then use the tox
-# configuration to run the tests.
-
-set -e
-
-# Set PGUSER so Synapse's tests know what user to connect to the database with
-export PGUSER=postgres
-
-# Start the database
-sudo -u postgres /usr/lib/postgresql/10/bin/pg_ctl -w -D /var/lib/postgresql/data start
-
-# Run the tests
-cd /src
-export TRIAL_FLAGS="-j 4"
-tox --workdir=./.tox-pg-container -e py37-postgres "$@"

docker/start.py

@@ -6,27 +6,28 @@ import os
 import platform
 import subprocess
 import sys
+from typing import Any, Dict, List, Mapping, MutableMapping, NoReturn, Optional
 
 import jinja2
 
 
 # Utility functions
-def log(txt):
+def log(txt: str) -> None:
     print(txt, file=sys.stderr)
 
 
-def error(txt):
+def error(txt: str) -> NoReturn:
     log(txt)
     sys.exit(2)
 
 
-def convert(src, dst, environ):
+def convert(src: str, dst: str, environ: Mapping[str, object]) -> None:
     """Generate a file from a template
 
     Args:
-        src (str): path to input file
-        dst (str): path to file to write
-        environ (dict): environment dictionary, for replacement mappings.
+        src: path to input file
+        dst: path to file to write
+        environ: environment dictionary, for replacement mappings.
     """
     with open(src) as infile:
         template = infile.read()
@@ -35,25 +36,30 @@ def convert(src, dst, environ):
         outfile.write(rendered)
 
 
-def generate_config_from_template(config_dir, config_path, environ, ownership):
+def generate_config_from_template(
+    config_dir: str,
+    config_path: str,
+    os_environ: Mapping[str, str],
+    ownership: Optional[str],
+) -> None:
     """Generate a homeserver.yaml from environment variables
 
     Args:
-        config_dir (str): where to put generated config files
-        config_path (str): where to put the main config file
-        environ (dict): environment dictionary
-        ownership (str|None): "<user>:<group>" string which will be used to set
+        config_dir: where to put generated config files
+        config_path: where to put the main config file
+        os_environ: environment mapping
+        ownership: "<user>:<group>" string which will be used to set
             ownership of the generated configs. If None, ownership will not change.
     """
     for v in ("SYNAPSE_SERVER_NAME", "SYNAPSE_REPORT_STATS"):
-        if v not in environ:
+        if v not in os_environ:
             error(
                 "Environment variable '%s' is mandatory when generating a config file."
                 % (v,)
             )
 
     # populate some params from data files (if they exist, else create new ones)
-    environ = environ.copy()
+    environ: Dict[str, Any] = dict(os_environ)
     secrets = {
         "registration": "SYNAPSE_REGISTRATION_SHARED_SECRET",
         "macaroon": "SYNAPSE_MACAROON_SECRET_KEY",
@@ -108,7 +114,7 @@ def generate_config_from_template(config_dir, config_path, environ, ownership):
 
     # Hopefully we already have a signing key, but generate one if not.
     args = [
-        "python",
+        sys.executable,
         "-m",
         "synapse.app.homeserver",
         "--config-path",
@@ -127,12 +133,12 @@ def generate_config_from_template(config_dir, config_path, environ, ownership):
     subprocess.check_output(args)
 
 
-def run_generate_config(environ, ownership):
+def run_generate_config(environ: Mapping[str, str], ownership: Optional[str]) -> None:
     """Run synapse with a --generate-config param to generate a template config file
 
     Args:
-        environ (dict): env var dict
-        ownership (str|None): "userid:groupid" arg for chmod. If None, ownership will not change.
+        environ: env vars from `os.enrivon`.
+        ownership: "userid:groupid" arg for chmod. If None, ownership will not change.
 
     Never returns.
     """
@@ -158,7 +164,7 @@ def run_generate_config(environ, ownership):
 
     # generate the main config file, and a signing key.
     args = [
-        "python",
+        sys.executable,
         "-m",
         "synapse.app.homeserver",
         "--server-name",
@@ -175,10 +181,10 @@ def run_generate_config(environ, ownership):
         "--open-private-ports",
     ]
     # log("running %s" % (args, ))
-    os.execv("/usr/local/bin/python", args)
+    os.execv(sys.executable, args)
 
 
-def main(args, environ):
+def main(args: List[str], environ: MutableMapping[str, str]) -> None:
     mode = args[1] if len(args) > 1 else "run"
 
     # if we were given an explicit user to switch to, do so
@@ -254,12 +260,12 @@ running with 'migrate_config'. See the README for more details.
 
     log("Starting synapse with args " + " ".join(args))
 
-    args = ["python"] + args
+    args = [sys.executable] + args
     if ownership is not None:
         args = ["gosu", ownership] + args
         os.execve("/usr/sbin/gosu", args, environ)
     else:
-        os.execve("/usr/local/bin/python", args, environ)
+        os.execve(sys.executable, args, environ)
 
 
 if __name__ == "__main__":

docs/SUMMARY.md

@@ -17,6 +17,7 @@
 # Usage
   - [Federation](federate.md)
   - [Configuration](usage/configuration/README.md)
+    - [Configuration Manual](usage/configuration/config_documentation.md) 
     - [Homeserver Sample Config File](usage/configuration/homeserver_sample_config.md)
     - [Logging Sample Config File](usage/configuration/logging_sample_config.md)
     - [Structured Logging](structured_logging.md)
@@ -45,6 +46,7 @@
         - [Account validity callbacks](modules/account_validity_callbacks.md)
         - [Password auth provider callbacks](modules/password_auth_provider_callbacks.md)
         - [Background update controller callbacks](modules/background_update_controller_callbacks.md)
+        - [Account data callbacks](modules/account_data_callbacks.md)
         - [Porting a legacy module to the new interface](modules/porting_legacy_module.md)
     - [Workers](workers.md)
       - [Using `synctl` with Workers](synctl_workers.md)
@@ -82,6 +84,7 @@
   - [Release Cycle](development/releases.md)
   - [Git Usage](development/git.md)
   - [Testing]()
+    - [Demo scripts](development/demo.md)
   - [OpenTracing](opentracing.md)
   - [Database Schemas](development/database_schema.md)
   - [Experimental features](development/experimental_features.md)

docs/admin_api/user_admin_api.md

@@ -804,7 +804,7 @@ POST /_synapse/admin/v2/users/<user_id>/delete_devices
   "devices": [
     "QBUAZIFURK",
     "AUIECTSRND"
-  ],
+  ]
 }
 ```
 

docs/changelogs/README.md

@@ -0,0 +1 @@
+This directory contains changelogs for previous years.

docs/code_style.md

@@ -6,62 +6,36 @@ The Synapse codebase uses a number of code formatting tools in order to
 quickly and automatically check for formatting (and sometimes logical)
 errors in code.
 
-The necessary tools are detailed below.
+The necessary tools are:
 
-First install them with:
+- [black](https://black.readthedocs.io/en/stable/), a source code formatter;
+- [isort](https://pycqa.github.io/isort/), which organises each file's imports;
+- [flake8](https://flake8.pycqa.org/en/latest/), which can spot common errors; and
+- [mypy](https://mypy.readthedocs.io/en/stable/), a type checker.
+
+Install them with:
 
 ```sh
 pip install -e ".[lint,mypy]"
 ```
 
--   **black**
+The easiest way to run the lints is to invoke the linter script as follows.
 
-    The Synapse codebase uses [black](https://pypi.org/project/black/)
-    as an opinionated code formatter, ensuring all comitted code is
-    properly formatted.
-
-    Have `black` auto-format your code (it shouldn't change any
-    functionality) with:
-
-    ```sh
-    black . --exclude="\.tox|build|env"
-    ```
-
--   **flake8**
-
-    `flake8` is a code checking tool. We require code to pass `flake8`
-    before being merged into the codebase.
-
-    Check all application and test code with:
-
-    ```sh
-    flake8 synapse tests
-    ```
-
--   **isort**
-
-    `isort` ensures imports are nicely formatted, and can suggest and
-    auto-fix issues such as double-importing.
-
-    Auto-fix imports with:
-
-    ```sh
-    isort -rc synapse tests
-    ```
-
-    `-rc` means to recursively search the given directories.
+```sh
+scripts-dev/lint.sh
+```
 
 It's worth noting that modern IDEs and text editors can run these tools
 automatically on save. It may be worth looking into whether this
 functionality is supported in your editor for a more convenient
-development workflow. It is not, however, recommended to run `flake8` on
-save as it takes a while and is very resource intensive.
+development workflow. It is not, however, recommended to run `flake8` or `mypy`
+on save as they take a while and can be very resource intensive.
 
 ## General rules
 
 -   **Naming**:
-    -   Use camel case for class and type names
-    -   Use underscores for functions and variables.
+    -   Use `CamelCase` for class and type names
+    -   Use underscores for `function_names` and `variable_names`.
 -   **Docstrings**: should follow the [google code
     style](https://google.github.io/styleguide/pyguide.html#38-comments-and-docstrings).
     See the
@@ -172,6 +146,6 @@ frobber:
 ```
 
 Note that the sample configuration is generated from the synapse code
-and is maintained by a script, `scripts-dev/generate_sample_config`.
+and is maintained by a script, `scripts-dev/generate_sample_config.sh`.
 Making sure that the output from this script matches the desired format
 is left as an exercise for the reader!

docs/development/contributing_guide.md

@@ -48,19 +48,28 @@ can find many good git tutorials on the web.
 
 # 4. Install the dependencies
 
-Once you have installed Python 3 and added the source, please open a terminal and
-setup a *virtualenv*, as follows:
+Synapse uses the [poetry](https://python-poetry.org/) project to manage its dependencies
+and development environment. Once you have installed Python 3 and added the
+source, you should install `poetry`.
+Of their installation methods, we recommend
+[installing `poetry` using `pipx`](https://python-poetry.org/docs/#installing-with-pipx),
+
+```shell
+pip install --user pipx
+pipx install poetry
+```
+
+but see poetry's [installation instructions](https://python-poetry.org/docs/#installation)
+for other installation methods.
+
+Next, open a terminal and install dependencies as follows:
 
 ```sh
 cd path/where/you/have/cloned/the/repository
-python3 -m venv ./env
-source ./env/bin/activate
-pip install wheel
-pip install -e ".[all,dev]"
-pip install tox
+poetry install --extras all
 ```
 
-This will install the developer dependencies for the project.
+This will install the runtime and developer dependencies for the project.
 
 
 # 5. Get in touch.
@@ -117,11 +126,10 @@ The linters look at your code and do two things:
 - ensure that your code follows the coding style adopted by the project;
 - catch a number of errors in your code.
 
-The linter takes no time at all to run as soon as you've [downloaded the dependencies into your python virtual environment](#4-install-the-dependencies).
+The linter takes no time at all to run as soon as you've [downloaded the dependencies](#4-install-the-dependencies).
 
 ```sh
-source ./env/bin/activate
-./scripts-dev/lint.sh
+poetry run ./scripts-dev/lint.sh
 ```
 
 Note that this script *will modify your files* to fix styling errors.
@@ -131,15 +139,13 @@ If you wish to restrict the linters to only the files changed since the last com
 (much faster!), you can instead run:
 
 ```sh
-source ./env/bin/activate
-./scripts-dev/lint.sh -d
+poetry run ./scripts-dev/lint.sh -d
 ```
 
 Or if you know exactly which files you wish to lint, you can instead run:
 
 ```sh
-source ./env/bin/activate
-./scripts-dev/lint.sh path/to/file1.py path/to/file2.py path/to/folder
+poetry run ./scripts-dev/lint.sh path/to/file1.py path/to/file2.py path/to/folder
 ```
 
 ## Run the unit tests (Twisted trial).
@@ -148,16 +154,14 @@ The unit tests run parts of Synapse, including your changes, to see if anything
 was broken. They are slower than the linters but will typically catch more errors.
 
 ```sh
-source ./env/bin/activate
-trial tests
+poetry run trial tests
 ```
 
 If you wish to only run *some* unit tests, you may specify
 another module instead of `tests` - or a test class or a method:
 
 ```sh
-source ./env/bin/activate
-trial tests.rest.admin.test_room tests.handlers.test_admin.ExfiltrateData.test_invite
+poetry run trial tests.rest.admin.test_room tests.handlers.test_admin.ExfiltrateData.test_invite
 ```
 
 If your tests fail, you may wish to look at the logs (the default log level is `ERROR`):
@@ -169,7 +173,7 @@ less _trial_temp/test.log
 To increase the log level for the tests, set `SYNAPSE_TEST_LOG_LEVEL`:
 
 ```sh
-SYNAPSE_TEST_LOG_LEVEL=DEBUG trial tests
+SYNAPSE_TEST_LOG_LEVEL=DEBUG poetry run trial tests
 ```
 
 By default, tests will use an in-memory SQLite database for test data. For additional
@@ -180,7 +184,7 @@ database state to be stored in a file named `test.db` under the trial process'
 working directory. Typically, this ends up being `_trial_temp/test.db`. For example:
 
 ```sh
-SYNAPSE_TEST_PERSIST_SQLITE_DB=1 trial tests
+SYNAPSE_TEST_PERSIST_SQLITE_DB=1 poetry run trial tests
 ```
 
 The database file can then be inspected with:
@@ -206,9 +210,10 @@ To do so, [configure Postgres](../postgres.md) and run `trial` with the
 following environment variables matching your configuration:
 
 - `SYNAPSE_POSTGRES` to anything nonempty
-- `SYNAPSE_POSTGRES_HOST`
-- `SYNAPSE_POSTGRES_USER`
-- `SYNAPSE_POSTGRES_PASSWORD`
+- `SYNAPSE_POSTGRES_HOST` (optional if it's the default: UNIX socket)
+- `SYNAPSE_POSTGRES_PORT` (optional if it's the default: 5432)
+- `SYNAPSE_POSTGRES_USER` (optional if using a UNIX socket)
+- `SYNAPSE_POSTGRES_PASSWORD` (optional if using a UNIX socket)
 
 For example:
 
@@ -220,26 +225,12 @@ export SYNAPSE_POSTGRES_PASSWORD=mydevenvpassword
 trial
 ```
 
-#### Prebuilt container
+You don't need to specify the host, user, port or password if your Postgres
+server is set to authenticate you over the UNIX socket (i.e. if the `psql` command
+works without further arguments).
 
-Since configuring PostgreSQL can be fiddly, we can make use of a pre-made
-Docker container to set up PostgreSQL and run our tests for us. To do so, run
+Your Postgres account needs to be able to create databases.
 
-```shell
-scripts-dev/test_postgresql.sh
-```
-
-Any extra arguments to the script will be passed to `tox` and then to `trial`,
-so we can run a specific test in this container with e.g.
-
-```shell
-scripts-dev/test_postgresql.sh tests.replication.test_sharded_event_persister.EventPersisterShardTestCase
-```
-
-The container creates a folder in your Synapse checkout called
-`.tox-pg-container` and uses this as a tox environment. The output of any
-`trial` runs goes into `_trial_temp` in your synapse source directory — the same
-as running `trial` directly on your host machine.
 
 ## Run the integration tests ([Sytest](https://github.com/matrix-org/sytest)).
 
@@ -254,8 +245,14 @@ configuration:
 ```sh
 $ docker run --rm -it -v /path/where/you/have/cloned/the/repository\:/src:ro -v /path/to/where/you/want/logs\:/logs matrixdotorg/sytest-synapse:buster
 ```
+(Note that the paths must be full paths! You could also write `$(realpath relative/path)` if needed.)
 
-This configuration should generally cover  your needs. For more details about other configurations, see [documentation in the SyTest repo](https://github.com/matrix-org/sytest/blob/develop/docker/README.md).
+This configuration should generally cover your needs.
+
+- To run with Postgres, supply the `-e POSTGRES=1 -e MULTI_POSTGRES=1` environment flags.
+- To run with Synapse in worker mode, supply the `-e WORKERS=1 -e REDIS=1` environment flags (in addition to the Postgres flags).
+
+For more details about other configurations, see the [Docker-specific documentation in the SyTest repo](https://github.com/matrix-org/sytest/blob/develop/docker/README.md).
 
 
 ## Run the integration tests ([Complement](https://github.com/matrix-org/complement)).
@@ -458,6 +455,17 @@ Git allows you to add this signoff automatically when using the `-s`
 flag to `git commit`, which uses the name and email set in your
 `user.name` and `user.email` git configs.
 
+### Private Sign off
+
+If you would like to provide your legal name privately to the Matrix.org
+Foundation (instead of in a public commit or comment), you can do so
+by emailing your legal name and a link to the pull request to
+[dco@matrix.org](mailto:dco@matrix.org?subject=Private%20sign%20off).
+It helps to include "sign off" or similar in the subject line. You will then
+be instructed further.
+
+Once private sign off is complete, doing so for future contributions will not
+be required.
 
 # 10. Turn feedback into better code.
 

docs/development/database_schema.md

@@ -158,9 +158,9 @@ same as integers.
 There are three separate aspects to this:
 
  * Any new boolean column must be added to the `BOOLEAN_COLUMNS` list in
-   `scripts/synapse_port_db`. This tells the port script to cast the integer
-   value from SQLite to a boolean before writing the value to the postgres
-   database.
+   `synapse/_scripts/synapse_port_db.py`. This tells the port script to cast
+   the integer value from SQLite to a boolean before writing the value to the
+   postgres database.
 
  * Before SQLite 3.23, `TRUE` and `FALSE` were not recognised as constants by
    SQLite, and the `IS [NOT] TRUE`/`IS [NOT] FALSE` operators were not

docs/development/demo.md

@@ -0,0 +1,41 @@
+# Synapse demo setup
+
+**DO NOT USE THESE DEMO SERVERS IN PRODUCTION**
+
+Requires you to have a [Synapse development environment setup](https://matrix-org.github.io/synapse/develop/development/contributing_guide.html#4-install-the-dependencies).
+
+The demo setup allows running three federation Synapse servers, with server
+names `localhost:8080`, `localhost:8081`, and `localhost:8082`.
+
+You can access them via any Matrix client over HTTP at `localhost:8080`,
+`localhost:8081`, and `localhost:8082` or over HTTPS at `localhost:8480`,
+`localhost:8481`, and `localhost:8482`.
+
+To enable the servers to communicate, self-signed SSL certificates are generated
+and the servers are configured in a highly insecure way, including:
+
+* Not checking certificates over federation.
+* Not verifying keys.
+
+The servers are configured to store their data under `demo/8080`, `demo/8081`, and
+`demo/8082`. This includes configuration, logs, SQLite databases, and media.
+
+Note that when joining a public room on a different HS via "#foo:bar.net", then
+you are (in the current impl) joining a room with room_id "foo". This means that
+it won't work if your HS already has a room with that name.
+
+## Using the demo scripts
+
+There's three main scripts with straightforward purposes:
+
+* `start.sh` will start the Synapse servers, generating any missing configuration.
+  * This accepts a single parameter `--no-rate-limit` to "disable" rate limits
+    (they actually still exist, but are very high).
+* `stop.sh` will stop the Synapse servers.
+* `clean.sh` will delete the configuration, databases, log files, etc.
+
+To start a completely new set of servers, run:
+
+```sh
+./demo/stop.sh; ./demo/clean.sh && ./demo/start.sh
+```

docs/development/dependencies.md

@@ -0,0 +1,239 @@
+# Managing dependencies with Poetry
+
+This is a quick cheat sheet for developers on how to use [`poetry`](https://python-poetry.org/).
+
+# Background
+
+Synapse uses a variety of third-party Python packages to function as a homeserver.
+Some of these are direct dependencies, listed in `pyproject.toml` under the
+`[tool.poetry.dependencies]` section. The rest are transitive dependencies (the
+things that our direct dependencies themselves depend on, and so on recursively.)
+
+We maintain a locked list of all our dependencies (transitive included) so that
+we can track exactly which version of each dependency appears in a given release.
+See [here](https://github.com/matrix-org/synapse/issues/11537#issue-1074469665)
+for discussion of why we wanted this for Synapse. We chose to use
+[`poetry`](https://python-poetry.org/) to manage this locked list; see
+[this comment](https://github.com/matrix-org/synapse/issues/11537#issuecomment-1015975819)
+for the reasoning.
+
+The locked dependencies get included in our "self-contained" releases: namely,
+our docker images and our debian packages. We also use the locked dependencies
+in development and our continuous integration.
+
+Separately, our "broad" dependencies—the version ranges specified in
+`pyproject.toml`—are included as metadata in our "sdists" and "wheels" [uploaded
+to PyPI](https://pypi.org/project/matrix-synapse). Installing from PyPI or from
+the Synapse source tree directly will _not_ use the locked dependencies; instead,
+they'll pull in the latest version of each package available at install time.
+
+## Example dependency
+
+An example may help. We have a broad dependency on
+[`phonenumbers`](https://pypi.org/project/phonenumbers/), as declared in
+this snippet from pyproject.toml [as of Synapse 1.57](
+https://github.com/matrix-org/synapse/blob/release-v1.57/pyproject.toml#L133
+):
+
+```toml
+[tool.poetry.dependencies]
+# ...
+phonenumbers = ">=8.2.0"
+```
+
+In our lockfile this is
+[pinned]( https://github.com/matrix-org/synapse/blob/dfc7646504cef3e4ff396c36089e1c6f1b1634de/poetry.lock#L679-L685)
+to version 8.12.44, even though
+[newer versions are available](https://pypi.org/project/phonenumbers/#history).
+
+```toml
+[[package]]
+name = "phonenumbers"
+version = "8.12.44"
+description = "Python version of Google's common library for parsing, formatting, storing and validating international phone numbers."
+category = "main"
+optional = false
+python-versions = "*"
+```
+
+The lockfile also includes a
+[cryptographic checksum](https://github.com/matrix-org/synapse/blob/release-v1.57/poetry.lock#L2178-L2181)
+of the sdists and wheels provided for this version of `phonenumbers`.
+
+```toml
+[metadata.files]
+# ...
+phonenumbers = [
+    {file = "phonenumbers-8.12.44-py2.py3-none-any.whl", hash = "sha256:cc1299cf37b309ecab6214297663ab86cb3d64ae37fd5b88e904fe7983a874a6"},
+    {file = "phonenumbers-8.12.44.tar.gz", hash = "sha256:26cfd0257d1704fe2f88caff2caabb70d16a877b1e65b6aae51f9fbbe10aa8ce"},
+]
+```
+
+We can see this pinned version inside the docker image for that release:
+
+```
+$ docker pull matrixdotorg/synapse:v1.57.0
+...
+$ docker run --entrypoint pip matrixdotorg/synapse:v1.57.0 show phonenumbers
+Name: phonenumbers
+Version: 8.12.44
+Summary: Python version of Google's common library for parsing, formatting, storing and validating international phone numbers.
+Home-page: https://github.com/daviddrysdale/python-phonenumbers
+Author: David Drysdale
+Author-email: dmd@lurklurk.org
+License: Apache License 2.0
+Location: /usr/local/lib/python3.9/site-packages
+Requires:
+Required-by: matrix-synapse
+```
+
+Whereas the wheel metadata just contains the broad dependencies:
+
+```
+$ cd /tmp
+$ wget https://files.pythonhosted.org/packages/ca/5e/d722d572cc5b3092402b783d6b7185901b444427633bd8a6b00ea0dd41b7/matrix_synapse-1.57.0rc1-py3-none-any.whl
+...
+$ unzip -c matrix_synapse-1.57.0rc1-py3-none-any.whl matrix_synapse-1.57.0rc1.dist-info/METADATA | grep phonenumbers
+Requires-Dist: phonenumbers (>=8.2.0)
+```
+
+# Tooling recommendation: direnv
+
+[`direnv`](https://direnv.net/) is a tool for activating environments in your
+shell inside a given directory. Its support for poetry is unofficial (a
+community wiki recipe only), but works solidly in our experience. We thoroughly
+recommend it for daily use. To use it:
+
+1. [Install `direnv`](https://direnv.net/docs/installation.html) - it's likely
+   packaged for your system already.
+2. Teach direnv about poetry. The [shell config here](https://github.com/direnv/direnv/wiki/Python#poetry)
+   needs to be added to `~/.config/direnv/direnvrc` (or more generally `$XDG_CONFIG_HOME/direnv/direnvrc`).
+3. Mark the synapse checkout as a poetry project: `echo layout poetry > .envrc`.
+4. Convince yourself that you trust this `.envrc` configuration and project.
+   Then formally confirm this to `direnv` by running `direnv allow`.
+
+Then whenever you navigate to the synapse checkout, you should be able to run
+e.g. `mypy` instead of `poetry run mypy`; `python` instead of
+`poetry run python`; and your shell commands will automatically run in the
+context of poetry's venv, without having to run `poetry shell` beforehand.
+
+
+# How do I...
+
+## ...reset my venv to the locked environment?
+
+```shell
+poetry install --extras all --remove-untracked
+```
+
+## ...run a command in the `poetry` virtualenv?
+
+Use `poetry run cmd args` when you need the python virtualenv context.
+To avoid typing `poetry run` all the time, you can run  `poetry shell`
+to start a new shell in the poetry virtualenv context. Within `poetry shell`,
+`python`, `pip`, `mypy`, `trial`, etc. are all run inside the project virtualenv
+and isolated from the rest o the system.
+
+Roughly speaking, the translation from a traditional virtualenv is:
+- `env/bin/activate` -> `poetry shell`, and
+- `deactivate` -> close the terminal (Ctrl-D, `exit`, etc.)
+
+See also the direnv recommendation above, which makes `poetry run` and
+`poetry shell` unnecessary.
+
+
+## ...inspect the `poetry` virtualenv?
+
+Some suggestions:
+
+```shell
+# Current env only
+poetry env info
+# All envs: this allows you to have e.g. a poetry managed venv for Python 3.7,
+# and another for Python 3.10.
+poetry env list --full-path
+poetry run pip list
+```
+
+Note that `poetry show` describes the abstract *lock file* rather than your
+on-disk environment. With that said, `poetry show --tree` can sometimes be
+useful.
+
+
+## ...add a new dependency?
+
+Either:
+- manually update `pyproject.toml`; then `poetry lock --no-update`; or else
+- `poetry add packagename`. See `poetry add --help`; note the `--dev`,
+  `--extras` and `--optional` flags in particular.
+  - **NB**: this specifies the new package with a version given by a "caret bound". This won't get forced to its lowest version in the old deps CI job: see [this TODO](https://github.com/matrix-org/synapse/blob/4e1374373857f2f7a911a31c50476342d9070681/.ci/scripts/test_old_deps.sh#L35-L39).
+
+Include the updated `pyproject.toml` and `poetry.lock` files in your commit.
+
+## ...remove a dependency?
+
+This is not done often and is untested, but
+
+```shell
+poetry remove packagename
+```
+
+ought to do the trick. Alternatively, manually update `pyproject.toml` and
+`poetry lock --no-update`. Include the updated `pyproject.toml` and poetry.lock`
+files in your commit.
+
+## ...update the version range for an existing dependency?
+
+Best done by manually editing `pyproject.toml`, then `poetry lock --no-update`.
+Include the updated `pyproject.toml` and `poetry.lock` in your commit.
+
+## ...update a dependency in the locked environment?
+
+Use
+
+```shell
+poetry update packagename
+```
+
+to use the latest version of `packagename` in the locked environment, without
+affecting the broad dependencies listed in the wheel.
+
+There doesn't seem to be a way to do this whilst locking a _specific_ version of
+`packagename`. We can workaround this (crudely) as follows:
+
+```shell
+poetry add packagename==1.2.3
+# This should update pyproject.lock.
+
+# Now undo the changes to pyproject.toml. For example
+# git restore pyproject.toml
+
+# Get poetry to recompute the content-hash of pyproject.toml without changing
+# the locked package versions.
+poetry lock --no-update
+```
+
+Either way, include the updated `poetry.lock` file in your commit.
+
+## ...export a `requirements.txt` file?
+
+```shell
+poetry export --extras all
+```
+
+Be wary of bugs in `poetry export` and `pip install -r requirements.txt`.
+
+Note: `poetry export` will be made a plugin in Poetry 1.2. Additional config may
+be required.
+
+## ...build a test wheel?
+
+I usually use
+
+```shell
+poetry run pip install build && poetry run python -m build
+```
+
+because [`build`](https://github.com/pypa/build) is a standardish tool which
+doesn't require poetry. (It's what we use in CI too). However, you could try
+`poetry build` too.

docs/development/room-dag-concepts.md

@@ -30,13 +30,58 @@ rather than skipping any that arrived late; whereas if you're looking at a
 historical section of timeline (i.e. `/messages`), you want to see the best
 representation of the state of the room as others were seeing it at the time.
 
+## Outliers
+
+We mark an event as an `outlier` when we haven't figured out the state for the
+room at that point in the DAG yet. They are "floating" events that we haven't
+yet correlated to the DAG.
+
+Outliers typically arise when we fetch the auth chain or state for a given
+event. When that happens, we just grab the events in the state/auth chain,
+without calculating the state at those events, or backfilling their
+`prev_events`. Since we don't have the state at any events fetched in that
+way, we mark them as outliers.
+
+So, typically, we won't have the `prev_events` of an `outlier` in the database,
+(though it's entirely possible that we *might* have them for some other
+reason). Other things that make outliers different from regular events:
+
+ * We don't have state for them, so there should be no entry in
+   `event_to_state_groups` for an outlier. (In practice this isn't always
+   the case, though I'm not sure why: see https://github.com/matrix-org/synapse/issues/12201).
+
+ * We don't record entries for them in the `event_edges`,
+   `event_forward_extremeties` or `event_backward_extremities` tables.
+
+Since outliers are not tied into the DAG, they do not normally form part of the
+timeline sent down to clients via `/sync` or `/messages`; however there is an
+exception:
+
+### Out-of-band membership events
+
+A special case of outlier events are some membership events for federated rooms
+that we aren't full members of. For example:
+
+ * invites received over federation, before we join the room
+ * *rejections* for said invites
+ * knock events for rooms that we would like to join but have not yet joined.
+
+In all the above cases, we don't have the state for the room, which is why they
+are treated as outliers. They are a bit special though, in that they are
+proactively sent to clients via `/sync`.
 
 ## Forward extremity
 
-Most-recent-in-time events in the DAG which are not referenced by any other events' `prev_events` yet.
+Most-recent-in-time events in the DAG which are not referenced by any other
+events' `prev_events` yet. (In this definition, outliers, rejected events, and
+soft-failed events don't count.)
 
-The forward extremities of a room are used as the `prev_events` when the next event is sent.
+The forward extremities of a room (or at least, a subset of them, if there are
+more than ten) are used as the `prev_events` when the next event is sent.
 
+The "current state" of a room (ie: the state which would be used if we
+generated a new event) is, therefore, the resolution of the room states
+at each of the forward extremities.
 
 ## Backward extremity
 
@@ -44,23 +89,14 @@ The current marker of where we have backfilled up to and will generally be the
 `prev_events` of the oldest-in-time events we have in the DAG. This gives a starting point when
 backfilling history.
 
-When we persist a non-outlier event, we clear it as a backward extremity and set
-all of its `prev_events` as the new backward extremities if they aren't already
-persisted in the `events` table.
-
-
-## Outliers
-
-We mark an event as an `outlier` when we haven't figured out the state for the
-room at that point in the DAG yet.
-
-We won't *necessarily* have the `prev_events` of an `outlier` in the database,
-but it's entirely possible that we *might*.
-
-For example, when we fetch the event auth chain or state for a given event, we
-mark all of those claimed auth events as outliers because we haven't done the
-state calculation ourself.
+Note that, unlike forward extremities, we typically don't have any backward
+extremity events themselves in the database - or, if we do, they will be "outliers" (see
+above). Either way, we don't expect to have the room state at a backward extremity.
 
+When we persist a non-outlier event, if it was previously a backward extremity,
+we clear it as a backward extremity and set all of its `prev_events` as the new
+backward extremities if they aren't already persisted as non-outliers. This
+therefore keeps the backward extremities up-to-date.
 
 ## State groups
 

docs/federate.md

@@ -63,4 +63,5 @@ release of Synapse.
 
 If you want to get up and running quickly with a trio of homeservers in a
 private federation, there is a script in the `demo` directory. This is mainly
-useful just for development purposes. See [demo/README](https://github.com/matrix-org/synapse/tree/develop/demo/).
+useful just for development purposes. See
+[demo scripts](https://matrix-org.github.io/synapse/develop/development/demo.html).

docs/modules/account_data_callbacks.md

@@ -0,0 +1,106 @@
+# Account data callbacks
+
+Account data callbacks allow module developers to react to changes of the account data
+of local users. Account data callbacks can be registered using the module API's
+`register_account_data_callbacks` method.
+
+## Callbacks
+
+The available account data callbacks are:
+
+### `on_account_data_updated`
+
+_First introduced in Synapse v1.57.0_
+
+```python
+async def on_account_data_updated(
+    user_id: str,
+    room_id: Optional[str],
+    account_data_type: str,
+    content: "synapse.module_api.JsonDict",
+) -> None:
+```
+
+Called after user's account data has been updated. The module is given the
+Matrix ID of the user whose account data is changing, the room ID the data is associated
+with, the type associated with the change, as well as the new content. If the account
+data is not associated with a specific room, then the room ID is `None`.
+
+This callback is triggered when new account data is added or when the data associated with
+a given type (and optionally room) changes. This includes deletion, since in Matrix,
+deleting account data consists of replacing the data associated with a given type
+(and optionally room) with an empty dictionary (`{}`).
+
+Note that this doesn't trigger when changing the tags associated with a room, as these are
+processed separately by Synapse.
+
+If multiple modules implement this callback, Synapse runs them all in order.
+
+## Example
+
+The example below is a module that implements the `on_account_data_updated` callback, and
+sends an event to an audit room when a user changes their account data.
+
+```python
+import json
+import attr
+from typing import Any, Dict, Optional
+
+from synapse.module_api import JsonDict, ModuleApi
+from synapse.module_api.errors import ConfigError
+
+
+@attr.s(auto_attribs=True)
+class CustomAccountDataConfig:
+    audit_room: str
+    sender: str
+
+
+class CustomAccountDataModule:
+    def __init__(self, config: CustomAccountDataConfig, api: ModuleApi):
+        self.api = api
+        self.config = config
+
+        self.api.register_account_data_callbacks(
+            on_account_data_updated=self.log_new_account_data,
+        )
+
+    @staticmethod
+    def parse_config(config: Dict[str, Any]) -> CustomAccountDataConfig:
+        def check_in_config(param: str):
+            if param not in config:
+                raise ConfigError(f"'{param}' is required")
+
+        check_in_config("audit_room")
+        check_in_config("sender")
+
+        return CustomAccountDataConfig(
+            audit_room=config["audit_room"],
+            sender=config["sender"],
+        )
+
+    async def log_new_account_data(
+        self,
+        user_id: str,
+        room_id: Optional[str],
+        account_data_type: str,
+        content: JsonDict,
+    ) -> None:
+        content_raw = json.dumps(content)
+        msg_content = f"{user_id} has changed their account data for type {account_data_type} to: {content_raw}"
+
+        if room_id is not None:
+            msg_content += f" (in room {room_id})"
+
+        await self.api.create_and_send_event_into_room(
+            {
+                "room_id": self.config.audit_room,
+                "sender": self.config.sender,
+                "type": "m.room.message",
+                "content": {
+                    "msgtype": "m.text",
+                    "body": msg_content
+                }
+            }
+        )
+```

docs/modules/spam_checker_callbacks.md

@@ -172,7 +172,7 @@ any of the subsequent implementations of this callback.
 _First introduced in Synapse v1.37.0_
 
 ```python
-async def check_username_for_spam(user_profile: Dict[str, str]) -> bool
+async def check_username_for_spam(user_profile: synapse.module_api.UserProfile) -> bool
 ```
 
 Called when computing search results in the user directory. The module must return a
@@ -182,9 +182,11 @@ search results; otherwise return `False`.
 
 The profile is represented as a dictionary with the following keys:
 
-* `user_id`: The Matrix ID for this user.
-* `display_name`: The user's display name.
-* `avatar_url`: The `mxc://` URL to the user's avatar.
+* `user_id: str`. The Matrix ID for this user.
+* `display_name: Optional[str]`. The user's display name, or `None` if this user
+  has not set a display name.
+* `avatar_url: Optional[str]`. The `mxc://` URL to the user's avatar, or `None`
+  if this user has not set an avatar.
 
 The module is given a copy of the original dictionary, so modifying it from within the
 module cannot modify a user's profile when included in user directory search results.

docs/modules/third_party_rules_callbacks.md

@@ -148,6 +148,49 @@ deny an incoming event, see [`check_event_for_spam`](spam_checker_callbacks.md#c
 
 If multiple modules implement this callback, Synapse runs them all in order.
 
+### `check_can_shutdown_room`
+
+_First introduced in Synapse v1.55.0_
+
+```python
+async def check_can_shutdown_room(
+    user_id: str, room_id: str,
+) -> bool:
+```
+
+Called when an admin user requests the shutdown of a room. The module must return a
+boolean indicating whether the shutdown can go through. If the callback returns `False`,
+the shutdown will not proceed and the caller will see a `M_FORBIDDEN` error.
+
+If multiple modules implement this callback, they will be considered in order. If a
+callback returns `True`, Synapse falls through to the next one. The value of the first
+callback that does not return `True` will be used. If this happens, Synapse will not call
+any of the subsequent implementations of this callback.
+
+### `check_can_deactivate_user`
+
+_First introduced in Synapse v1.55.0_
+
+```python
+async def check_can_deactivate_user(
+    user_id: str, by_admin: bool,
+) -> bool:
+```
+
+Called when the deactivation of a user is requested. User deactivation can be
+performed by an admin or the user themselves, so developers are encouraged to check the
+requester when implementing this callback. The module must return a
+boolean indicating whether the deactivation can go through. If the callback returns `False`,
+the deactivation will not proceed and the caller will see a `M_FORBIDDEN` error.
+
+The module is passed two parameters, `user_id` which is the ID of the user being deactivated, and `by_admin` which is `True` if the request is made by a serve admin, and `False` otherwise.
+
+If multiple modules implement this callback, they will be considered in order. If a
+callback returns `True`, Synapse falls through to the next one. The value of the first
+callback that does not return `True` will be used. If this happens, Synapse will not call
+any of the subsequent implementations of this callback.
+
+
 ### `on_profile_update`
 
 _First introduced in Synapse v1.54.0_
@@ -204,6 +247,24 @@ admin API.
 
 If multiple modules implement this callback, Synapse runs them all in order.
 
+### `on_threepid_bind`
+
+_First introduced in Synapse v1.56.0_
+
+```python
+async def on_threepid_bind(user_id: str, medium: str, address: str) -> None:
+```
+
+Called after creating an association between a local user and a third-party identifier
+(email address, phone number). The module is given the Matrix ID of the user the
+association is for, as well as the medium (`email` or `msisdn`) and address of the
+third-party identifier.
+
+Note that this callback is _not_ called after a successful association on an _identity
+server_.
+
+If multiple modules implement this callback, Synapse runs them all in order.
+
 ## Example
 
 The example below is a module that implements the third-party rules callback

docs/modules/writing_a_module.md

@@ -33,7 +33,7 @@ A module can implement the following static method:
 
 ```python
 @staticmethod
-def parse_config(config: dict) -> dict
+def parse_config(config: dict) -> Any
 ```
 
 This method is given a dictionary resulting from parsing the YAML configuration for the

docs/openid.md

@@ -225,6 +225,8 @@ oidc_providers:
 3. Create an application for synapse in Authentik and link it to the provider.
 4. Note the slug of your application, Client ID and Client Secret.
 
+Note: RSA keys must be used for signing for Authentik, ECC keys do not work.
+
 Synapse config:
 ```yaml
 oidc_providers:
@@ -240,7 +242,7 @@ oidc_providers:
       - "email"
     user_mapping_provider:
       config:
-        localpart_template: "{{ user.preferred_username }}}"
+        localpart_template: "{{ user.preferred_username }}"
         display_name_template: "{{ user.preferred_username|capitalize }}" # TO BE FILLED: If your users have names in Authentik and you want those in Synapse, this should be replaced with user.name|capitalize.
 ```
 

docs/other/running_synapse_on_single_board_computers.md

@@ -31,28 +31,29 @@ Anything that requires modifying the device list [#7721](https://github.com/matr
 Put the below in a new file at /etc/matrix-synapse/conf.d/sbc.yaml to override the defaults in homeserver.yaml.
 
 ```
-# Set to false to disable presence tracking on this homeserver.
+# Disable presence tracking, which is currently fairly resource intensive
+# More info: https://github.com/matrix-org/synapse/issues/9478
 use_presence: false
 
-# When this is enabled, the room "complexity" will be checked before a user
-# joins a new remote room. If it is above the complexity limit, the server will
-# disallow joining, or will instantly leave.
+# Set a small complexity limit, preventing users from joining large rooms
+# which may be resource-intensive to remain a part of.
+#
+# Note that this will not prevent users from joining smaller rooms that
+# eventually become complex.
 limit_remote_rooms:
-  # Uncomment to enable room complexity checking.
-  #enabled: true
+  enabled: true
   complexity: 3.0
 
 # Database configuration
 database:
+  # Use postgres for the best performance
   name: psycopg2
   args:
     user: matrix-synapse
-    # Generate a long, secure one with a password manager
+    # Generate a long, secure password using a password manager
     password: hunter2
     database: matrix-synapse
     host: localhost
-    cp_min: 5
-    cp_max: 10
 ```
 
 Currently the complexity is measured by [current_state_events / 500](https://github.com/matrix-org/synapse/blob/v1.20.1/synapse/storage/databases/main/events_worker.py#L986). You can find join times and your most complex rooms like this:

docs/postgres.md

@@ -153,9 +153,9 @@ database file (typically `homeserver.db`) to another location. Once the
 copy is complete, restart synapse. For instance:
 
 ```sh
-./synctl stop
+synctl stop
 cp homeserver.db homeserver.db.snapshot
-./synctl start
+synctl start
 ```
 
 Copy the old config file into a new config file:
@@ -192,10 +192,10 @@ Once that has completed, change the synapse config to point at the
 PostgreSQL database configuration file `homeserver-postgres.yaml`:
 
 ```sh
-./synctl stop
+synctl stop
 mv homeserver.yaml homeserver-old-sqlite.yaml
 mv homeserver-postgres.yaml homeserver.yaml
-./synctl start
+synctl start
 ```
 
 Synapse should now be running against PostgreSQL.
@@ -234,12 +234,13 @@ host    all         all             ::1/128     ident
 ### Fixing incorrect `COLLATE` or `CTYPE`
 
 Synapse will refuse to set up a new database if it has the wrong values of
-`COLLATE` and `CTYPE` set, and will log warnings on existing databases. Using
-different locales can cause issues if the locale library is updated from
+`COLLATE` and `CTYPE` set. Synapse will also refuse to start an existing database with incorrect values
+of `COLLATE` and `CTYPE` unless the config flag `allow_unsafe_locale`, found in the 
+`database` section of the config, is set to true. Using different locales can cause issues if the locale library is updated from
 underneath the database, or if a different version of the locale is used on any
 replicas.
 
-The safest way to fix the issue is to dump the database and recreate it with
+If you have a databse with an unsafe locale, the safest way to fix the issue is to dump the database and recreate it with
 the correct locale parameter (as shown above). It is also possible to change the
 parameters on a live database and run a `REINDEX` on the entire database,
 however extreme care must be taken to avoid database corruption.

docs/reverse_proxy.md

@@ -182,7 +182,7 @@ matrix.example.com {
 
 ```
 frontend https
-  bind :::443 v4v6 ssl crt /etc/ssl/haproxy/ strict-sni alpn h2,http/1.1
+  bind *:443,[::]:443 ssl crt /etc/ssl/haproxy/ strict-sni alpn h2,http/1.1
   http-request set-header X-Forwarded-Proto https if { ssl_fc }
   http-request set-header X-Forwarded-Proto http if !{ ssl_fc }
   http-request set-header X-Forwarded-For %[src]
@@ -195,7 +195,7 @@ frontend https
   use_backend matrix if matrix-host matrix-path
 
 frontend matrix-federation
-  bind :::8448 v4v6 ssl crt /etc/ssl/haproxy/synapse.pem alpn h2,http/1.1
+  bind *:8448,[::]:8448 ssl crt /etc/ssl/haproxy/synapse.pem alpn h2,http/1.1
   http-request set-header X-Forwarded-Proto https if { ssl_fc }
   http-request set-header X-Forwarded-Proto http if !{ ssl_fc }
   http-request set-header X-Forwarded-For %[src]
@@ -206,6 +206,28 @@ backend matrix
   server matrix 127.0.0.1:8008
 ```
 
+
+[Delegation](delegate.md) example:
+```
+frontend https
+  acl matrix-well-known-client-path path /.well-known/matrix/client
+  acl matrix-well-known-server-path path /.well-known/matrix/server
+  use_backend matrix-well-known-client if matrix-well-known-client-path
+  use_backend matrix-well-known-server if matrix-well-known-server-path
+ 
+backend matrix-well-known-client
+  http-after-response set-header Access-Control-Allow-Origin "*"
+  http-after-response set-header Access-Control-Allow-Methods "GET, POST, PUT, DELETE, OPTIONS"
+  http-after-response set-header Access-Control-Allow-Headers "Origin, X-Requested-With, Content-Type, Accept, Authorization"
+  http-request return status 200 content-type application/json string '{"m.homeserver":{"base_url":"https://matrix.example.com"},"m.identity_server":{"base_url":"https://identity.example.com"}}'
+
+backend matrix-well-known-server
+  http-after-response set-header Access-Control-Allow-Origin "*"
+  http-after-response set-header Access-Control-Allow-Methods "GET, POST, PUT, DELETE, OPTIONS"
+  http-after-response set-header Access-Control-Allow-Headers "Origin, X-Requested-With, Content-Type, Accept, Authorization"
+  http-request return status 200 content-type application/json string '{"m.server":"matrix.example.com:443"}'
+```
+
 ### Relayd
 
 ```

docs/sample_config.yaml

@@ -539,6 +539,15 @@ templates:
   #
   #custom_template_directory: /path/to/custom/templates/
 
+# List of rooms to exclude from sync responses. This is useful for server
+# administrators wishing to group users into a room without these users being able
+# to see it from their client.
+#
+# By default, no room is excluded.
+#
+#exclude_rooms_from_sync:
+#    - !foo:example.com
+
 
 # Message retention policy at the server level.
 #
@@ -783,6 +792,12 @@ caches:
 # 'txn_limit' gives the maximum number of transactions to run per connection
 # before reconnecting. Defaults to 0, which means no limit.
 #
+# 'allow_unsafe_locale' is an option specific to Postgres. Under the default behavior, Synapse will refuse to
+# start if the postgres db is set to a non-C locale. You can override this behavior (which is *not* recommended)
+# by setting 'allow_unsafe_locale' to true. Note that doing so may corrupt your database. You can find more information
+# here: https://matrix-org.github.io/synapse/latest/postgres.html#fixing-incorrect-collate-or-ctype and here:
+# https://wiki.postgresql.org/wiki/Locale_data_changes
+#
 # 'args' gives options which are passed through to the database engine,
 # except for options starting 'cp_', which are used to configure the Twisted
 # connection pool. For a reference to valid arguments, see:
@@ -1212,10 +1227,18 @@ oembed:
 # Registration can be rate-limited using the parameters in the "Ratelimiting"
 # section of this file.
 
-# Enable registration for new users.
+# Enable registration for new users. Defaults to 'false'. It is highly recommended that if you enable registration,
+# you use either captcha, email, or token-based verification to verify that new users are not bots. In order to enable registration
+# without any verification, you must also set `enable_registration_without_verification`, found below.
 #
 #enable_registration: false
 
+# Enable registration without email or captcha verification. Note: this option is *not* recommended,
+# as registration without verification is a known vector for spam and abuse. Defaults to false. Has no effect
+# unless `enable_registration` is also enabled.
+#
+#enable_registration_without_verification: true
+
 # Time that a user's session remains valid for, after they log in.
 #
 # Note that this is not currently compatible with guest logins.
@@ -1947,8 +1970,14 @@ saml2_config:
 #
 #             localpart_template: Jinja2 template for the localpart of the MXID.
 #                 If this is not set, the user will be prompted to choose their
-#                 own username (see 'sso_auth_account_details.html' in the 'sso'
-#                 section of this file).
+#                 own username (see the documentation for the
+#                 'sso_auth_account_details.html' template). This template can
+#                 use the 'localpart_from_email' filter.
+#
+#             confirm_localpart: Whether to prompt the user to validate (or
+#                 change) the generated localpart (see the documentation for the
+#                 'sso_auth_account_details.html' template), instead of
+#                 registering the account right away.
 #
 #             display_name_template: Jinja2 template for the display name to set
 #                 on first login. If unset, no displayname will be set.
@@ -2729,3 +2758,35 @@ redis:
   # Optional password if configured on the Redis instance
   #
   #password: <secret_password>
+
+
+## Background Updates ##
+
+# Background updates are database updates that are run in the background in batches.
+# The duration, minimum batch size, default batch size, whether to sleep between batches and if so, how long to
+# sleep can all be configured. This is helpful to speed up or slow down the updates.
+#
+background_updates:
+    # How long in milliseconds to run a batch of background updates for. Defaults to 100. Uncomment and set
+    # a time to change the default.
+    #
+    #background_update_duration_ms: 500
+
+    # Whether to sleep between updates. Defaults to True. Uncomment to change the default.
+    #
+    #sleep_enabled: false
+
+    # If sleeping between updates, how long in milliseconds to sleep for. Defaults to 1000. Uncomment
+    # and set a duration to change the default.
+    #
+    #sleep_duration_ms: 300
+
+    # Minimum size a batch of background updates can be. Must be greater than 0. Defaults to 1. Uncomment and
+    # set a size to change the default.
+    #
+    #min_batch_size: 10
+
+    # The batch size to use for the first iteration of a new background update. The default is 100.
+    # Uncomment and set a size to change the default.
+    #
+    #default_batch_size: 50

docs/structured_logging.md

@@ -78,82 +78,3 @@ loggers:
 The above logging config will set Synapse as 'INFO' logging level by default,
 with the SQL layer at 'WARNING', and will log JSON formatted messages to a
 remote endpoint at 10.1.2.3:9999.
-
-## Upgrading from legacy structured logging configuration
-
-Versions of Synapse prior to v1.54.0 automatically converted the legacy
-structured logging configuration, which was deprecated in v1.23.0, to the standard
-library logging configuration.
-
-The following reference can be used to update your configuration. Based on the
-drain `type`, we can pick a new handler:
-
-1. For a type of `console`, `console_json`, or `console_json_terse`: a handler
-   with a class of `logging.StreamHandler` and a `stream` of `ext://sys.stdout`
-   or `ext://sys.stderr` should be used.
-2. For a type of `file` or `file_json`: a handler of `logging.FileHandler` with
-   a location of the file path should be used.
-3. For a type of `network_json_terse`: a handler of `synapse.logging.RemoteHandler`
-   with the host and port should be used.
-
-Then based on the drain `type` we can pick a new formatter:
-
-1. For a type of `console` or `file` no formatter is necessary.
-2. For a type of `console_json` or `file_json`: a formatter of
-   `synapse.logging.JsonFormatter` should be used.
-3. For a type of `console_json_terse` or `network_json_terse`: a formatter of
-   `synapse.logging.TerseJsonFormatter` should be used.
-
-For each new handler and formatter they should be added to the logging configuration
-and then assigned to either a logger or the root logger.
-
-An example legacy configuration:
-
-```yaml
-structured: true
-
-loggers:
-    synapse:
-        level: INFO
-    synapse.storage.SQL:
-        level: WARNING
-
-drains:
-    console:
-        type: console
-        location: stdout
-    file:
-        type: file_json
-        location: homeserver.log
-```
-
-Would be converted into a new configuration:
-
-```yaml
-version: 1
-
-formatters:
-    json:
-        class: synapse.logging.JsonFormatter
-
-handlers:
-    console:
-        class: logging.StreamHandler
-        stream: ext://sys.stdout
-    file:
-        class: logging.FileHandler
-        formatter: json
-        filename: homeserver.log
-
-loggers:
-    synapse:
-        level: INFO
-        handlers: [console, file]
-    synapse.storage.SQL:
-        level: WARNING
-```
-
-The new logging configuration is a bit more verbose, but significantly more
-flexible. It allows for configuration that were not previously possible, such as
-sending plain logs over the network, or using different handlers for different
-modules.

docs/systemd-with-workers/README.md

@@ -10,15 +10,15 @@ See the folder [system](https://github.com/matrix-org/synapse/tree/develop/docs/
 for the systemd unit files.
 
 The folder [workers](https://github.com/matrix-org/synapse/tree/develop/docs/systemd-with-workers/workers/)
-contains an example configuration for the `federation_reader` worker.
+contains an example configuration for the `generic_worker` worker.
 
 ## Synapse configuration files
 
 See [the worker documentation](../workers.md) for information on how to set up the
 configuration files and reverse-proxy correctly.
-Below is a sample `federation_reader` worker configuration file.
+Below is a sample `generic_worker` worker configuration file.
 ```yaml
-{{#include workers/federation_reader.yaml}}
+{{#include workers/generic_worker.yaml}}
 ```
 
 Systemd manages daemonization itself, so ensure that none of the configuration
@@ -61,9 +61,9 @@ systemctl stop matrix-synapse.target
 # Restart the master alone
 systemctl start matrix-synapse.service
 
-# Restart a specific worker (eg. federation_reader); the master is
+# Restart a specific worker (eg. generic_worker); the master is
 # unaffected by this.
-systemctl restart matrix-synapse-worker@federation_reader.service
+systemctl restart matrix-synapse-worker@generic_worker.service
 
 # Add a new worker (assuming all configs are set up already)
 systemctl enable matrix-synapse-worker@federation_writer.service

docs/systemd-with-workers/workers/background_worker.yaml

@@ -0,0 +1,8 @@
+worker_app: synapse.app.generic_worker
+worker_name: background_worker
+
+# The replication listener on the main synapse process.
+worker_replication_host: 127.0.0.1
+worker_replication_http_port: 9093
+
+worker_log_config: /etc/matrix-synapse/background-worker-log.yaml

docs/systemd-with-workers/workers/event_persister.yaml

@@ -0,0 +1,23 @@
+worker_app: synapse.app.generic_worker
+worker_name: event_persister1 
+
+# The replication listener on the main synapse process.
+worker_replication_host: 127.0.0.1
+worker_replication_http_port: 9093
+
+worker_listeners:
+  - type: http
+    port: 8034
+    resources:
+      - names: [replication]
+
+  # Enable listener if this stream writer handles endpoints for the `typing` or
+  # `to_device` streams. Uses a different port to the `replication` listener to
+  # avoid exposing the `replication` listener publicly.
+  #
+  #- type: http
+  #  port: 8035
+  #  resources:
+  #    - names: [client]
+
+worker_log_config: /etc/matrix-synapse/event-persister-log.yaml

docs/systemd-with-workers/workers/federation_reader.yaml

@@ -1,13 +0,0 @@
-worker_app: synapse.app.federation_reader
-worker_name: federation_reader1
-
-worker_replication_host: 127.0.0.1
-worker_replication_http_port: 9093
-
-worker_listeners:
-    - type: http
-      port: 8011
-      resources:
-          - names: [federation]
-
-worker_log_config: /etc/matrix-synapse/federation-reader-log.yaml

docs/systemd-with-workers/workers/generic_worker.yaml

@@ -0,0 +1,14 @@
+worker_app: synapse.app.generic_worker
+worker_name: generic_worker1
+
+# The replication listener on the main synapse process.
+worker_replication_host: 127.0.0.1
+worker_replication_http_port: 9093
+
+worker_listeners:
+  - type: http
+    port: 8083
+    resources:
+      - names: [client, federation]
+
+worker_log_config: /etc/matrix-synapse/generic-worker-log.yaml

docs/templates.md

@@ -36,6 +36,13 @@ Turns a `mxc://` URL for media content into an HTTP(S) one using the homeserver'
 
 Example: `message.sender_avatar_url|mxc_to_http(32,32)`
 
+```python
+localpart_from_email(address: str) -> str
+```
+
+Returns the local part of an email address (e.g. `alice` in `alice@example.com`).
+
+Example: `user.email_address|localpart_from_email`
 
 ## Email templates
 
@@ -176,8 +183,11 @@ Below are the templates Synapse will look for when generating pages related to S
              for the brand of the IdP
     * `user_attributes`: an object containing details about the user that
       we received from the IdP. May have the following attributes:
-        * display_name: the user's display_name
-        * emails: a list of email addresses
+        * `display_name`: the user's display name
+        * `emails`: a list of email addresses
+        * `localpart`: the local part of the Matrix user ID to register,
+          if `localpart_template` is set in the mapping provider configuration (empty
+          string if not)
   The template should render a form which submits the following fields:
     * `username`: the localpart of the user's chosen user id
 * `sso_new_user_consent.html`: HTML page allowing the user to consent to the

docs/turn-howto.md

@@ -238,8 +238,9 @@ After updating the homeserver configuration, you must restart synapse:
 
   * If you use synctl:
     ```sh
-    cd /where/you/run/synapse
-    ./synctl restart
+    # Depending on how Synapse is installed, synctl may already be on
+    # your PATH. If not, you may need to activate a virtual environment.
+    synctl restart
     ```
   * If you use systemd:
     ```sh
@@ -301,14 +302,14 @@ Here are a few things to try:
 
    (Understanding the output is beyond the scope of this document!)
 
- * You can test your Matrix homeserver TURN setup with https://test.voip.librepush.net/.
+ * You can test your Matrix homeserver TURN setup with <https://test.voip.librepush.net/>.
    Note that this test is not fully reliable yet, so don't be discouraged if
    the test fails.
    [Here](https://github.com/matrix-org/voip-tester) is the github repo of the
    source of the tester, where you can file bug reports.
 
  * There is a WebRTC test tool at
-   https://webrtc.github.io/samples/src/content/peerconnection/trickle-ice/. To
+   <https://webrtc.github.io/samples/src/content/peerconnection/trickle-ice/>. To
    use it, you will need a username/password for your TURN server. You can
    either:
 

docs/upgrade.md

@@ -19,35 +19,39 @@ this document.
     packages](setup/installation.md#prebuilt-packages), you will need to follow the
     normal process for upgrading those packages.
 
+-   If Synapse was installed using pip then upgrade to the latest
+    version by running:
+
+    ```bash
+    pip install --upgrade matrix-synapse
+    ```
+
 -   If Synapse was installed from source, then:
 
-    1.  Activate the virtualenv before upgrading. For example, if
-        Synapse is installed in a virtualenv in `~/synapse/env` then
+    1.  Obtain the latest version of the source code. Git users can run
+        `git pull` to do this.
+
+    2.  If you're running Synapse in a virtualenv, make sure to activate it before
+        upgrading. For example, if Synapse is installed in a virtualenv in `~/synapse/env` then
         run:
 
         ```bash
         source ~/synapse/env/bin/activate
-        ```
-
-    2.  If Synapse was installed using pip then upgrade to the latest
-        version by running:
-
-        ```bash
-        pip install --upgrade matrix-synapse
-        ```
-
-        If Synapse was installed using git then upgrade to the latest
-        version by running:
-
-        ```bash
-        git pull
         pip install --upgrade .
         ```
+        Include any relevant extras between square brackets, e.g. `pip install --upgrade ".[postgres,oidc]"`.
 
-    3.  Restart Synapse:
+    3.  If you're using `poetry` to manage a Synapse installation, run:
+        ```bash
+        poetry install
+        ```
+        Include any relevant extras with `--extras`, e.g. `poetry install --extras postgres --extras oidc`.
+        It's probably easiest to run `poetry install --extras all`.
+
+    4.  Restart Synapse:
 
         ```bash
-        ./synctl restart
+        synctl restart
         ```
 
 To check whether your update was successful, you can check the running
@@ -85,6 +89,120 @@ process, for example:
     dpkg -i matrix-synapse-py3_1.3.0+stretch1_amd64.deb
     ```
 
+# Upgrading to v1.58.0
+
+## Groups/communities feature has been disabled by default
+
+The non-standard groups/communities feature in Synapse has been disabled by default
+and will be removed in Synapse v1.61.0.
+
+# Upgrading to v1.57.0
+
+## Changes to database schema for application services
+
+Synapse v1.57.0 includes a [change](https://github.com/matrix-org/synapse/pull/12209) to the
+way transaction IDs are managed for application services. If your deployment uses a dedicated
+worker for application service traffic, **it must be stopped** when the database is upgraded
+(which normally happens when the main process is upgraded), to ensure the change is made safely
+without any risk of reusing transaction IDs.
+
+Deployments which do not use separate worker processes can be upgraded as normal. Similarly,
+deployments where no application services are in use can be upgraded as normal.
+
+<details>
+<summary><b>Recovering from an incorrect upgrade</b></summary>
+
+If the database schema is upgraded *without* stopping the worker responsible
+for AS traffic, then the following error may be given when attempting to start
+a Synapse worker or master process:
+
+```
+**********************************************************************************
+ Error during initialisation:
+
+ Postgres sequence 'application_services_txn_id_seq' is inconsistent with associated
+ table 'application_services_txns'. This can happen if Synapse has been downgraded and
+ then upgraded again, or due to a bad migration.
+
+ To fix this error, shut down Synapse (including any and all workers)
+ and run the following SQL:
+
+     SELECT setval('application_services_txn_id_seq', (
+         SELECT GREATEST(MAX(txn_id), 0) FROM application_services_txns
+     ));
+
+ See docs/postgres.md for more information.
+
+ There may be more information in the logs.
+**********************************************************************************
+```
+
+This error may also be seen if Synapse is *downgraded* to an earlier version,
+and then upgraded again to v1.57.0 or later.
+
+In either case:
+
+ 1. Ensure that the worker responsible for AS traffic is stopped.
+ 2. Run the SQL command given in the error message via `psql`.
+
+Synapse should then start correctly.
+</details>
+
+# Upgrading to v1.56.0
+
+## Open registration without verification is now disabled by default
+
+Synapse will refuse to start if registration is enabled without email, captcha, or token-based verification unless the new config
+flag `enable_registration_without_verification` is set to "true".
+
+## Groups/communities feature has been deprecated
+
+The non-standard groups/communities feature in Synapse has been deprecated and will
+be disabled by default in Synapse v1.58.0.
+
+You can test disabling it by adding the following to your homeserver configuration:
+
+```yaml
+experimental_features:
+  groups_enabled: false
+```
+
+## Change in behaviour for PostgreSQL databases with unsafe locale
+
+Synapse now refuses to start when using PostgreSQL with non-`C` values for `COLLATE` and
+`CTYPE` unless the config flag `allow_unsafe_locale`, found in the database section of
+the configuration file, is set to `true`. See the [PostgreSQL documentation](https://matrix-org.github.io/synapse/latest/postgres.html#fixing-incorrect-collate-or-ctype)
+for more information and instructions on how to fix a database with incorrect values.
+
+# Upgrading to v1.55.0
+
+## `synctl` script has been moved
+
+The `synctl` script
+[has been made](https://github.com/matrix-org/synapse/pull/12140) an
+[entry point](https://packaging.python.org/en/latest/specifications/entry-points/)
+and no longer exists at the root of Synapse's source tree. If you wish to use
+`synctl` to manage your homeserver, you should invoke `synctl` directly, e.g.
+`synctl start` instead of `./synctl start` or `/path/to/synctl start`.
+
+You will need to ensure `synctl` is on your `PATH`.
+  - This is automatically the case when using
+    [Debian packages](https://packages.matrix.org/debian/) or
+    [docker images](https://hub.docker.com/r/matrixdotorg/synapse)
+    provided by Matrix.org.
+  - When installing from a wheel, sdist, or PyPI, a `synctl` executable is added
+    to your Python installation's `bin`. This should be on your `PATH`
+    automatically, though you might need to activate a virtual environment
+    depending on how you installed Synapse.
+
+
+## Compatibility dropped for Mjolnir 1.3.1 and earlier
+
+Synapse v1.55.0 drops support for Mjolnir 1.3.1 and earlier.
+If you use the Mjolnir module to moderate your homeserver,
+please upgrade Mjolnir to version 1.3.2 or later before upgrading Synapse.
+
+
 # Upgrading to v1.54.0
 
 ## Legacy structured logging configuration removal
@@ -92,7 +210,7 @@ process, for example:
 This release removes support for the `structured: true` logging configuration
 which was deprecated in Synapse v1.23.0. If your logging configuration contains
 `structured: true` then it should be modified based on the
-[structured logging documentation](structured_logging.md).
+[structured logging documentation](https://matrix-org.github.io/synapse/v1.56/structured_logging.html#upgrading-from-legacy-structured-logging-configuration).
 
 # Upgrading to v1.53.0
 
@@ -108,8 +226,8 @@ the `/_matrix/client/` path.
 
 ## Stablisation of MSC3231
 
-The unstable validity-check endpoint for the 
-[Registration Tokens](https://spec.matrix.org/v1.2/client-server-api/#get_matrixclientv1registermloginregistration_tokenvalidity) 
+The unstable validity-check endpoint for the
+[Registration Tokens](https://spec.matrix.org/v1.2/client-server-api/#get_matrixclientv1registermloginregistration_tokenvalidity)
 feature has been stabilised and moved from:
 
 `/_matrix/client/unstable/org.matrix.msc3231/register/org.matrix.msc3231.login.registration_token/validity`
@@ -123,9 +241,9 @@ Please update any relevant reverse proxy or firewall configurations appropriatel
 ## Time-based cache expiry is now enabled by default
 
 Formerly, entries in the cache were not evicted regardless of whether they were accessed after storing.
-This behavior has now changed. By default entries in the cache are now evicted after 30m of not being accessed. 
-To change the default behavior, go to the `caches` section of the config and change the `expire_caches` and 
-`cache_entry_ttl` flags as necessary. Please note that these flags replace the `expiry_time` flag in the config.  
+This behavior has now changed. By default entries in the cache are now evicted after 30m of not being accessed.
+To change the default behavior, go to the `caches` section of the config and change the `expire_caches` and
+`cache_entry_ttl` flags as necessary. Please note that these flags replace the `expiry_time` flag in the config.
 The `expiry_time` flag will still continue to work, but it has been deprecated and will be removed in the future.
 
 ## Deprecation of `capability` `org.matrix.msc3283.*`
@@ -709,7 +827,7 @@ lock down external access to the Admin API endpoints.
 This release deprecates use of the `structured: true` logging
 configuration for structured logging. If your logging configuration
 contains `structured: true` then it should be modified based on the
-[structured logging documentation](structured_logging.md).
+[structured logging documentation](https://matrix-org.github.io/synapse/v1.56/structured_logging.html#upgrading-from-legacy-structured-logging-configuration).
 
 The `structured` and `drains` logging options are now deprecated and
 should be replaced by standard logging configuration of `handlers` and

docs/usage/administration/admin_api/README.md

@@ -12,7 +12,7 @@ UPDATE users SET admin = 1 WHERE name = '@foo:bar.com';
 ```
 
 A new server admin user can also be created using the `register_new_matrix_user`
-command. This is a script that is located in the `scripts/` directory, or possibly
+command. This is a script that is distributed as part of synapse. It is possibly
 already on your `$PATH` depending on how Synapse was installed.
 
 Finding your user's `access_token` is client-dependent, but will usually be shown in the client's settings.

docs/website_files/README.md

@@ -24,11 +24,6 @@ Finally, we also stylise the chapter titles in the left sidebar by indenting the
 slightly so that they are more visually distinguishable from the section headers
 (the bold titles). This is done through the `indent-section-headers.css` file.
 
-In addition to these modifications, we have added a version picker to the documentation.
-Users can switch between documentations for different versions of Synapse.
-This functionality was implemented through the `version-picker.js` and
-`version-picker.css` files.
-
 More information can be found in mdbook's official documentation for
 [injecting page JS/CSS](https://rust-lang.github.io/mdBook/format/config.html)
 and

docs/website_files/table-of-contents.js

@@ -75,6 +75,20 @@ function setTocEntry() {
  * Populate sidebar on load
  */
 window.addEventListener('load', () => {
+    // Prevent rendering the table of contents of the "print book" page, as it
+    // will end up being rendered into the output (in a broken-looking way)
+
+    // Get the name of the current page (i.e. 'print.html')
+    const pageNameExtension = window.location.pathname.split('/').pop();
+
+    // Split off the extension (as '.../print' is also a valid page name), which
+    // should result in 'print'
+    const pageName = pageNameExtension.split('.')[0];
+    if (pageName === "print") {
+        // Don't render the table of contents on this page
+        return;
+    }
+
     // Only create table of contents if there is more than one header on the page
     if (headers.length <= 1) {
         return;

docs/website_files/theme/index.hbs

@@ -131,18 +131,6 @@
                             <i class="fa fa-search"></i>
                         </button>
                         {{/if}}
-                        <div class="version-picker">
-                            <div class="dropdown">
-                                <div class="select">
-                                    <span></span>
-                                    <i class="fa fa-chevron-down"></i>
-                                </div>
-                                <input type="hidden" name="version">
-                                <ul class="dropdown-menu">
-                                    <!-- Versions will be added dynamically in version-picker.js -->
-                                </ul>
-                            </div>
-                        </div>      
                     </div>
 
                     <h1 class="menu-title">{{ book_title }}</h1>
@@ -321,4 +309,4 @@
         {{/if}}
 
     </body>
-</html>
+</html>

docs/website_files/version-picker.css

@@ -1,78 +0,0 @@
-.version-picker {
-    display: flex;
-    align-items: center;
-}
-
-.version-picker .dropdown {
-    width: 130px;
-    max-height: 29px;
-    margin-left: 10px;
-    display: inline-block;
-    border-radius: 4px;
-    border: 1px solid var(--theme-popup-border);
-    position: relative;
-    font-size: 13px;
-    color: var(--fg);
-    height: 100%;
-    text-align: left;
-}
-.version-picker .dropdown .select {
-    cursor: pointer;
-    display: block;
-    padding: 5px 2px 5px 15px;
-}
-.version-picker .dropdown .select > i {
-    font-size: 10px;
-    color: var(--fg);
-    cursor: pointer;
-    float: right;
-    line-height: 20px !important;
-}
-.version-picker .dropdown:hover {
-    border: 1px solid var(--theme-popup-border);
-}
-.version-picker .dropdown:active {
-    background-color: var(--theme-popup-bg);
-}
-.version-picker .dropdown.active:hover,
-.version-picker .dropdown.active {
-    border: 1px solid var(--theme-popup-border);
-    border-radius: 2px 2px 0 0;
-    background-color: var(--theme-popup-bg);
-}
-.version-picker .dropdown.active .select > i {
-    transform: rotate(-180deg);
-}
-.version-picker .dropdown .dropdown-menu {
-    position: absolute;
-    background-color: var(--theme-popup-bg);
-    width: 100%;
-    left: -1px;
-    right: 1px;
-    margin-top: 1px;
-    border: 1px solid var(--theme-popup-border);
-    border-radius: 0 0 4px 4px;
-    overflow: hidden;
-    display: none;
-    max-height: 300px;
-    overflow-y: auto;
-    z-index: 9;
-}
-.version-picker .dropdown .dropdown-menu li {
-    font-size: 12px;
-    padding: 6px 20px;
-    cursor: pointer;
-} 
-.version-picker .dropdown .dropdown-menu {
-    padding: 0;
-    list-style: none;
-}
-.version-picker .dropdown .dropdown-menu li:hover {
-    background-color: var(--theme-hover);
-}
-.version-picker .dropdown .dropdown-menu li.active::before {
-    display: inline-block;
-    content: "✓";
-    margin-inline-start: -14px;
-    width: 14px;
-}

docs/website_files/version-picker.js

@@ -1,127 +0,0 @@
-
-const dropdown = document.querySelector('.version-picker .dropdown');
-const dropdownMenu = dropdown.querySelector('.dropdown-menu');
-
-fetchVersions(dropdown, dropdownMenu).then(() => {
-    initializeVersionDropdown(dropdown, dropdownMenu);
-});
-
-/**
- * Initialize the dropdown functionality for version selection.
- * 
- * @param {Element} dropdown - The dropdown element.
- * @param {Element} dropdownMenu - The dropdown menu element.
- */
-function initializeVersionDropdown(dropdown, dropdownMenu) {
-    // Toggle the dropdown menu on click
-    dropdown.addEventListener('click', function () {
-        this.setAttribute('tabindex', 1);
-        this.classList.toggle('active');
-        dropdownMenu.style.display = (dropdownMenu.style.display === 'block') ? 'none' : 'block';
-    });
-  
-    // Remove the 'active' class and hide the dropdown menu on focusout
-    dropdown.addEventListener('focusout', function () {
-        this.classList.remove('active');
-        dropdownMenu.style.display = 'none';
-    });
-  
-    // Handle item selection within the dropdown menu
-    const dropdownMenuItems = dropdownMenu.querySelectorAll('li');    
-    dropdownMenuItems.forEach(function (item) {
-        item.addEventListener('click', function () {
-            dropdownMenuItems.forEach(function (item) {
-                item.classList.remove('active');
-            });
-            this.classList.add('active');
-            dropdown.querySelector('span').textContent = this.textContent;
-            dropdown.querySelector('input').value = this.getAttribute('id');
-
-            window.location.href = changeVersion(window.location.href, this.textContent);
-        });
-    });
-};
-
-/**
- * This function fetches the available versions from a GitHub repository
- * and inserts them into the version picker.
- * 
- * @param {Element} dropdown - The dropdown element.
- * @param {Element} dropdownMenu - The dropdown menu element.
- * @returns {Promise<Array<string>>} A promise that resolves with an array of available versions.
- */
-function fetchVersions(dropdown, dropdownMenu) {
-    return new Promise((resolve, reject) => {
-        window.addEventListener("load", () => {
-
-            fetch("https://api.github.com/repos/matrix-org/synapse/git/trees/gh-pages", {
-                cache: "force-cache",
-            }).then(res => 
-                res.json()
-            ).then(resObject => {
-                const excluded = ['dev-docs', 'v1.91.0', 'v1.80.0', 'v1.69.0'];
-                const tree = resObject.tree.filter(item => item.type === "tree" && !excluded.includes(item.path));
-                const versions = tree.map(item => item.path).sort(sortVersions);
-
-                // Create a list of <li> items for versions
-                versions.forEach((version) => {
-                    const li = document.createElement("li");
-                    li.textContent = version;
-                    li.id = version;
-    
-                    if (window.SYNAPSE_VERSION === version) {
-                        li.classList.add('active');
-                        dropdown.querySelector('span').textContent = version;
-                        dropdown.querySelector('input').value = version;
-                    }
-    
-                    dropdownMenu.appendChild(li);
-                });
-
-                resolve(versions);
-
-            }).catch(ex => {
-                console.error("Failed to fetch version data", ex);
-                reject(ex);
-            })
-        });
-    });
-}
-
-/**
- * Custom sorting function to sort an array of version strings.
- *
- * @param {string} a - The first version string to compare.
- * @param {string} b - The second version string to compare.
- * @returns {number} - A negative number if a should come before b, a positive number if b should come before a, or 0 if they are equal.
- */
-function sortVersions(a, b) {
-    // Put 'develop' and 'latest' at the top
-    if (a === 'develop' || a === 'latest') return -1;
-    if (b === 'develop' || b === 'latest') return 1;
-
-    const versionA = (a.match(/v\d+(\.\d+)+/) || [])[0];
-    const versionB = (b.match(/v\d+(\.\d+)+/) || [])[0];
-
-    return versionB.localeCompare(versionA);
-}
-
-/**
- * Change the version in a URL path.
- *
- * @param {string} url - The original URL to be modified.
- * @param {string} newVersion - The new version to replace the existing version in the URL.
- * @returns {string} The updated URL with the new version.
- */
-function changeVersion(url, newVersion) {
-    const parsedURL = new URL(url);
-    const pathSegments = parsedURL.pathname.split('/');
-  
-    // Modify the version
-    pathSegments[2] = newVersion;
-
-    // Reconstruct the URL
-    parsedURL.pathname = pathSegments.join('/');
-  
-    return parsedURL.href;
-}

docs/website_files/version.js

@@ -1 +0,0 @@
-window.SYNAPSE_VERSION = 'v1.54';

docs/workers.md

@@ -27,7 +27,7 @@ feeds streams of newly written data between processes so they can be kept in
 sync with the database state.
 
 When configured to do so, Synapse uses a
-[Redis pub/sub channel](https://redis.io/topics/pubsub) to send the replication
+[Redis pub/sub channel](https://redis.io/docs/manual/pubsub/) to send the replication
 stream between all configured Synapse processes. Additionally, processes may
 make HTTP requests to each other, primarily for operations which need to wait
 for a reply ─ such as sending an event.
@@ -138,22 +138,7 @@ as the `listeners` option in the shared config.
 For example:
 
 ```yaml
-worker_app: synapse.app.generic_worker
-worker_name: worker1
-
-# The replication listener on the main synapse process.
-worker_replication_host: 127.0.0.1
-worker_replication_http_port: 9093
-
-worker_listeners:
- - type: http
-   port: 8083
-   resources:
-     - names:
-       - client
-       - federation
-
-worker_log_config: /home/matrix/synapse/config/worker1_log_config.yaml
+{{#include systemd-with-workers/workers/generic_worker.yaml}}
 ```
 
 ...is a full configuration for a generic worker instance, which will expose a
@@ -185,8 +170,8 @@ worker: refer to the [stream writers](#stream-writers) section below for further
 information.
 
     # Sync requests
-    ^/_matrix/client/(v2_alpha|r0|v3)/sync$
-    ^/_matrix/client/(api/v1|v2_alpha|r0|v3)/events$
+    ^/_matrix/client/(r0|v3)/sync$
+    ^/_matrix/client/(api/v1|r0|v3)/events$
     ^/_matrix/client/(api/v1|r0|v3)/initialSync$
     ^/_matrix/client/(api/v1|r0|v3)/rooms/[^/]+/initialSync$
 
@@ -200,13 +185,9 @@ information.
     ^/_matrix/federation/v1/query/
     ^/_matrix/federation/v1/make_join/
     ^/_matrix/federation/v1/make_leave/
-    ^/_matrix/federation/v1/send_join/
-    ^/_matrix/federation/v2/send_join/
-    ^/_matrix/federation/v1/send_leave/
-    ^/_matrix/federation/v2/send_leave/
-    ^/_matrix/federation/v1/invite/
-    ^/_matrix/federation/v2/invite/
-    ^/_matrix/federation/v1/query_auth/
+    ^/_matrix/federation/(v1|v2)/send_join/
+    ^/_matrix/federation/(v1|v2)/send_leave/
+    ^/_matrix/federation/(v1|v2)/invite/
     ^/_matrix/federation/v1/event_auth/
     ^/_matrix/federation/v1/exchange_third_party_invite/
     ^/_matrix/federation/v1/user/devices/
@@ -274,6 +255,8 @@ information.
 Additionally, the following REST endpoints can be handled for GET requests:
 
     ^/_matrix/federation/v1/groups/
+    ^/_matrix/client/(api/v1|r0|v3|unstable)/pushrules/
+    ^/_matrix/client/(r0|v3|unstable)/groups/
 
 Pagination requests can also be handled, but all requests for a given
 room must be routed to the same instance. Additionally, care must be taken to
@@ -345,14 +328,17 @@ effects of bursts of events from that bridge on events sent by normal users.
 
 #### Stream writers
 
-Additionally, there is *experimental* support for moving writing of specific
-streams (such as events) off of the main process to a particular worker. (This
-is only supported with Redis-based replication.)
+Additionally, the writing of specific streams (such as events) can be moved off
+of the main process to a particular worker.
+(This is only supported with Redis-based replication.)
 
 To enable this, the worker must have a HTTP replication listener configured,
 have a `worker_name` and be listed in the `instance_map` config. The same worker
-can handle multiple streams. For example, to move event persistence off to a
-dedicated worker, the shared configuration would include:
+can handle multiple streams, but unless otherwise documented, each stream can only
+have a single writer.
+
+For example, to move event persistence off to a dedicated worker, the shared
+configuration would include:
 
 ```yaml
 instance_map:
@@ -364,14 +350,20 @@ stream_writers:
     events: event_persister1
 ```
 
+An example for a stream writer instance:
+
+```yaml
+{{#include systemd-with-workers/workers/event_persister.yaml}}
+```
+
 Some of the streams have associated endpoints which, for maximum efficiency, should
 be routed to the workers handling that stream. See below for the currently supported
 streams and the endpoints associated with them:
 
 ##### The `events` stream
 
-The `events` stream also experimentally supports having multiple writers, where
-work is sharded between them by room ID. Note that you *must* restart all worker
+The `events` stream experimentally supports having multiple writers, where work
+is sharded between them by room ID. Note that you *must* restart all worker
 instances when adding or removing event persisters. An example `stream_writers`
 configuration with multiple writers:
 
@@ -384,44 +376,44 @@ stream_writers:
 
 ##### The `typing` stream
 
-The following endpoints should be routed directly to the workers configured as
-stream writers for the `typing` stream:
+The following endpoints should be routed directly to the worker configured as
+the stream writer for the `typing` stream:
 
     ^/_matrix/client/(api/v1|r0|v3|unstable)/rooms/.*/typing
 
 ##### The `to_device` stream
 
-The following endpoints should be routed directly to the workers configured as
-stream writers for the `to_device` stream:
+The following endpoints should be routed directly to the worker configured as
+the stream writer for the `to_device` stream:
 
-    ^/_matrix/client/(api/v1|r0|v3|unstable)/sendToDevice/
+    ^/_matrix/client/(r0|v3|unstable)/sendToDevice/
 
 ##### The `account_data` stream
 
-The following endpoints should be routed directly to the workers configured as
-stream writers for the `account_data` stream:
+The following endpoints should be routed directly to the worker configured as
+the stream writer for the `account_data` stream:
 
-    ^/_matrix/client/(api/v1|r0|v3|unstable)/.*/tags
-    ^/_matrix/client/(api/v1|r0|v3|unstable)/.*/account_data
+    ^/_matrix/client/(r0|v3|unstable)/.*/tags
+    ^/_matrix/client/(r0|v3|unstable)/.*/account_data
 
 ##### The `receipts` stream
 
-The following endpoints should be routed directly to the workers configured as
-stream writers for the `receipts` stream:
+The following endpoints should be routed directly to the worker configured as
+the stream writer for the `receipts` stream:
 
-    ^/_matrix/client/(api/v1|r0|v3|unstable)/rooms/.*/receipt
-    ^/_matrix/client/(api/v1|r0|v3|unstable)/rooms/.*/read_markers
+    ^/_matrix/client/(r0|v3|unstable)/rooms/.*/receipt
+    ^/_matrix/client/(r0|v3|unstable)/rooms/.*/read_markers
 
 ##### The `presence` stream
 
-The following endpoints should be routed directly to the workers configured as
-stream writers for the `presence` stream:
+The following endpoints should be routed directly to the worker configured as
+the stream writer for the `presence` stream:
 
     ^/_matrix/client/(api/v1|r0|v3|unstable)/presence/
 
 #### Background tasks
 
-There is also *experimental* support for moving background tasks to a separate
+There is also support for moving background tasks to a separate
 worker. Background tasks are run periodically or started via replication. Exactly
 which tasks are configured to run depends on your Synapse configuration (e.g. if
 stats is enabled).
@@ -437,6 +429,12 @@ run_background_tasks_on: background_worker
 You might also wish to investigate the `update_user_directory` and
 `media_instance_running_background_jobs` settings.
 
+An example for a dedicated background worker instance:
+
+```yaml
+{{#include systemd-with-workers/workers/background_worker.yaml}}
+```
+
 ### `synapse.app.pusher`
 
 Handles sending push notifications to sygnal and email. Doesn't handle any
@@ -525,19 +523,28 @@ Note that if a reverse proxy is used , then `/_matrix/media/` must be routed for
 Handles searches in the user directory. It can handle REST endpoints matching
 the following regular expressions:
 
-    ^/_matrix/client/(api/v1|r0|v3|unstable)/user_directory/search$
+    ^/_matrix/client/(r0|v3|unstable)/user_directory/search$
 
-When using this worker you must also set `update_user_directory: False` in the
+When using this worker you must also set `update_user_directory: false` in the
 shared configuration file to stop the main synapse running background
 jobs related to updating the user directory.
 
+Above endpoint is not *required* to be routed to this worker. By default,
+`update_user_directory` is set to `true`, which means the main process
+will handle updates. All workers configured with `client` can handle the above
+endpoint as long as either this worker or the main process are configured to
+handle it, and are online.
+
+If `update_user_directory` is set to `false`, and this worker is not running,
+the above endpoint may give outdated results.
+
 ### `synapse.app.frontend_proxy`
 
 Proxies some frequently-requested client endpoints to add caching and remove
 load from the main synapse. It can handle REST endpoints matching the following
 regular expressions:
 
-    ^/_matrix/client/(api/v1|r0|v3|unstable)/keys/upload
+    ^/_matrix/client/(r0|v3|unstable)/keys/upload
 
 If `use_presence` is False in the homeserver config, it can also handle REST
 endpoints matching the following regular expressions:
@@ -607,14 +614,14 @@ The following shows an example setup using Redis and a reverse proxy:
 |   Main       |  |   Generic    |  |   Generic    |  |  Event       |
 |   Process    |  |   Worker 1   |  |   Worker 2   |  |  Persister   |
 +--------------+  +--------------+  +--------------+  +--------------+
-      ^    ^          |   ^   |         |   ^   |          ^    ^
-      |    |          |   |   |         |   |   |          |    |
-      |    |          |   |   |  HTTP   |   |   |          |    |
-      |    +----------+<--|---|---------+   |   |          |    |
-      |                   |   +-------------|-->+----------+    |
-      |                   |                 |                   |
-      |                   |                 |                   |
-      v                   v                 v                   v
-====================================================================
+      ^    ^          |   ^   |         |   ^   |         |   ^   ^
+      |    |          |   |   |         |   |   |         |   |   |
+      |    |          |   |   |  HTTP   |   |   |         |   |   |
+      |    +----------+<--|---|---------+<--|---|---------+   |   |
+      |                   |   +-------------|-->+-------------+   |
+      |                   |                 |                     |
+      |                   |                 |                     |
+      v                   v                 v                     v
+======================================================================
                                                          Redis pub/sub channel
 ```

mypy.ini

@@ -11,8 +11,8 @@ local_partial_types = True
 no_implicit_optional = True
 
 files =
-  scripts-dev/sign_json,
-  setup.py,
+  docker/,
+  scripts-dev/,
   synapse/,
   tests/
 
@@ -23,19 +23,16 @@ files =
 # https://docs.python.org/3/library/re.html#re.X
 exclude = (?x)
   ^(
+   |scripts-dev/build_debian_packages.py
+   |scripts-dev/federation_client.py
+   |scripts-dev/release.py
+
    |synapse/storage/databases/__init__.py
-   |synapse/storage/databases/main/__init__.py
    |synapse/storage/databases/main/cache.py
    |synapse/storage/databases/main/devices.py
    |synapse/storage/databases/main/event_federation.py
-   |synapse/storage/databases/main/group_server.py
-   |synapse/storage/databases/main/metrics.py
-   |synapse/storage/databases/main/monthly_active_users.py
    |synapse/storage/databases/main/push_rule.py
-   |synapse/storage/databases/main/receipts.py
    |synapse/storage/databases/main/roommember.py
-   |synapse/storage/databases/main/search.py
-   |synapse/storage/databases/main/state.py
    |synapse/storage/schema/
 
    |tests/api/test_auth.py
@@ -52,14 +49,6 @@ exclude = (?x)
    |tests/federation/test_federation_server.py
    |tests/federation/transport/test_knocking.py
    |tests/federation/transport/test_server.py
-   |tests/handlers/test_cas.py
-   |tests/handlers/test_directory.py
-   |tests/handlers/test_e2e_keys.py
-   |tests/handlers/test_federation.py
-   |tests/handlers/test_oidc.py
-   |tests/handlers/test_presence.py
-   |tests/handlers/test_profile.py
-   |tests/handlers/test_saml.py
    |tests/handlers/test_typing.py
    |tests/http/federation/test_matrix_federation_agent.py
    |tests/http/federation/test_srv_resolver.py
@@ -71,37 +60,20 @@ exclude = (?x)
    |tests/logging/test_terse_json.py
    |tests/module_api/test_api.py
    |tests/push/test_email.py
-   |tests/push/test_http.py
    |tests/push/test_presentable_names.py
    |tests/push/test_push_rule_evaluator.py
-   |tests/rest/client/test_account.py
-   |tests/rest/client/test_filter.py
-   |tests/rest/client/test_report_event.py
-   |tests/rest/client/test_rooms.py
-   |tests/rest/client/test_third_party_rules.py
    |tests/rest/client/test_transactions.py
-   |tests/rest/client/test_typing.py
-   |tests/rest/key/v2/test_remote_key_resource.py
-   |tests/rest/media/v1/test_base.py
    |tests/rest/media/v1/test_media_storage.py
-   |tests/rest/media/v1/test_url_preview.py
    |tests/scripts/test_new_matrix_user.py
    |tests/server.py
    |tests/server_notices/test_resource_limits_server_notices.py
    |tests/state/test_v2.py
-   |tests/storage/test_background_update.py
    |tests/storage/test_base.py
-   |tests/storage/test_client_ips.py
-   |tests/storage/test_database.py
-   |tests/storage/test_event_federation.py
-   |tests/storage/test_id_generators.py
    |tests/storage/test_roommember.py
    |tests/test_metrics.py
-   |tests/test_phone_home.py
    |tests/test_server.py
    |tests/test_state.py
    |tests/test_terms_auth.py
-   |tests/unittest.py
    |tests/util/caches/test_cached_call.py
    |tests/util/caches/test_deferred_cache.py
    |tests/util/caches/test_descriptors.py
@@ -120,6 +92,9 @@ exclude = (?x)
    |tests/utils.py
    )$
 
+[mypy-synapse._scripts.*]
+disallow_untyped_defs = True
+
 [mypy-synapse.api.*]
 disallow_untyped_defs = True
 
@@ -129,7 +104,7 @@ disallow_untyped_defs = True
 [mypy-synapse.appservice.*]
 disallow_untyped_defs = True
 
-[mypy-synapse.config._base]
+[mypy-synapse.config.*]
 disallow_untyped_defs = True
 
 [mypy-synapse.crypto.*]
@@ -246,10 +221,7 @@ disallow_untyped_defs = True
 [mypy-tests.storage.test_user_directory]
 disallow_untyped_defs = True
 
-[mypy-tests.rest.admin.*]
-disallow_untyped_defs = True
-
-[mypy-tests.rest.client.*]
+[mypy-tests.rest.*]
 disallow_untyped_defs = True
 
 [mypy-tests.federation.transport.test_client]
@@ -261,8 +233,8 @@ disallow_untyped_defs = True
 ;; The `typeshed` project maintains stubs here:
 ;;     https://github.com/python/typeshed/tree/master/stubs
 ;; and for each package `foo` there's a corresponding `types-foo` package on PyPI,
-;; which we can pull in as a dev dependency by adding to `setup.py`'s
-;; `CONDITIONAL_REQUIREMENTS["mypy"]` list.
+;; which we can pull in as a dev dependency by adding to `pyproject.toml`'s
+;; `[tool.poetry.dev-dependencies]` list.
 
 [mypy-authlib.*]
 ignore_missing_imports = True
@@ -291,6 +263,9 @@ ignore_missing_imports = True
 [mypy-ijson.*]
 ignore_missing_imports = True
 
+[mypy-importlib_metadata.*]
+ignore_missing_imports = True
+
 [mypy-jaeger_client.*]
 ignore_missing_imports = True
 
@@ -350,3 +325,6 @@ ignore_missing_imports = True
 
 [mypy-zope]
 ignore_missing_imports = True
+
+[mypy-incremental.*]
+ignore_missing_imports = True

pyproject.toml

@@ -36,24 +36,9 @@
 
 [tool.black]
 target-version = ['py37', 'py38', 'py39', 'py310']
-exclude = '''
-
-(
-  /(
-      \.eggs         # exclude a few common directories in the
-    | \.git          # root of the project
-    | \.tox
-    | \.venv
-    | \.env
-    | env
-    | _build
-    | _trial_temp.*
-    | build
-    | dist
-    | debian
-  )/
-)
-'''
+# black ignores everything in .gitignore by default, see
+# https://black.readthedocs.io/en/stable/usage_and_configuration/file_collection_and_discovery.html#gitignore
+# Use `extend-exclude` if you want to exclude something in addition to this.
 
 [tool.isort]
 line_length = 88
@@ -65,4 +50,234 @@ known_twisted = ["twisted", "OpenSSL"]
 multi_line_output = 3
 include_trailing_comma = true
 combine_as_imports = true
+skip_gitignore = true
 
+[tool.poetry]
+name = "matrix-synapse"
+version = "1.58.1"
+description = "Homeserver for the Matrix decentralised comms protocol"
+authors = ["Matrix.org Team and Contributors <packages@matrix.org>"]
+license = "Apache-2.0"
+readme = "README.rst"
+repository = "https://github.com/matrix-org/synapse"
+packages = [
+    { include = "synapse" },
+]
+classifiers = [
+    "Development Status :: 5 - Production/Stable",
+    "Topic :: Communications :: Chat",
+]
+include = [
+    { path = "AUTHORS.rst", format = "sdist" },
+    { path = "book.toml", format = "sdist" },
+    { path = "changelog.d", format = "sdist" },
+    { path = "CHANGES.md", format = "sdist" },
+    { path = "CONTRIBUTING.md", format = "sdist" },
+    { path = "demo", format = "sdist" },
+    { path = "docs", format = "sdist" },
+    { path = "INSTALL.md", format = "sdist" },
+    { path = "mypy.ini", format = "sdist" },
+    { path = "scripts-dev", format = "sdist" },
+    { path = "synmark", format="sdist" },
+    { path = "sytest-blacklist", format = "sdist" },
+    { path = "tests", format = "sdist" },
+    { path = "UPGRADE.rst", format = "sdist" },
+]
+
+[tool.poetry.scripts]
+synapse_homeserver = "synapse.app.homeserver:main"
+synapse_worker = "synapse.app.generic_worker:main"
+synctl = "synapse._scripts.synctl:main"
+
+export_signing_key = "synapse._scripts.export_signing_key:main"
+generate_config = "synapse._scripts.generate_config:main"
+generate_log_config = "synapse._scripts.generate_log_config:main"
+generate_signing_key = "synapse._scripts.generate_signing_key:main"
+hash_password = "synapse._scripts.hash_password:main"
+register_new_matrix_user = "synapse._scripts.register_new_matrix_user:main"
+synapse_port_db = "synapse._scripts.synapse_port_db:main"
+synapse_review_recent_signups = "synapse._scripts.review_recent_signups:main"
+update_synapse_database = "synapse._scripts.update_synapse_database:main"
+
+[tool.poetry.dependencies]
+python = "^3.7"
+
+# Mandatory Dependencies
+# ----------------------
+# we use the TYPE_CHECKER.redefine method added in jsonschema 3.0.0
+jsonschema = ">=3.0.0"
+# frozendict 2.1.2 is broken on Debian 10: https://github.com/Marco-Sulla/python-frozendict/issues/41
+frozendict = ">=1,!=2.1.2"
+# We require 2.1.0 or higher for type hints. Previous guard was >= 1.1.0
+unpaddedbase64 = ">=2.1.0"
+canonicaljson = ">=1.4.0"
+# we use the type definitions added in signedjson 1.1.
+signedjson = ">=1.1.0"
+PyNaCl = ">=1.2.1"
+# validating SSL certs for IP addresses requires service_identity 18.1.
+service-identity = ">=18.1.0"
+# Twisted 18.9 introduces some logger improvements that the structured
+# logger utilises
+Twisted = {extras = ["tls"], version = ">=18.9.0"}
+treq = ">=15.1"
+# Twisted has required pyopenssl 16.0 since about Twisted 16.6.
+pyOpenSSL = ">=16.0.0"
+PyYAML = ">=3.11"
+pyasn1 = ">=0.1.9"
+pyasn1-modules = ">=0.0.7"
+bcrypt = ">=3.1.0"
+Pillow = ">=5.4.0"
+sortedcontainers = ">=1.4.4"
+pymacaroons = ">=0.13.0"
+msgpack = ">=0.5.2"
+phonenumbers = ">=8.2.0"
+# we use GaugeHistogramMetric, which was added in prom-client 0.4.0.
+prometheus-client = ">=0.4.0"
+# we use `order`, which arrived in attrs 19.2.0.
+# Note: 21.1.0 broke `/sync`, see #9936
+attrs = ">=19.2.0,!=21.1.0"
+netaddr = ">=0.7.18"
+# Jinja 2.x is incompatible with MarkupSafe>=2.1. To ensure that admins do not
+# end up with a broken installation, with recent MarkupSafe but old Jinja, we
+# add a lower bound to the Jinja2 dependency.
+Jinja2 = ">=3.0"
+bleach = ">=1.4.3"
+# We use `ParamSpec`, which was added in `typing-extensions` 3.10.0.0.
+typing-extensions = ">=3.10.0"
+# We enforce that we have a `cryptography` version that bundles an `openssl`
+# with the latest security patches.
+cryptography = ">=3.4.7"
+# ijson 3.1.4 fixes a bug with "." in property names
+ijson = ">=3.1.4"
+matrix-common = "~=1.1.0"
+# We need packaging.requirements.Requirement, added in 16.1.
+packaging = ">=16.1"
+# At the time of writing, we only use functions from the version `importlib.metadata`
+# which shipped in Python 3.8. This corresponds to version 1.4 of the backport.
+importlib_metadata = { version = ">=1.4", python = "<3.8" }
+
+
+# Optional Dependencies
+# ---------------------
+matrix-synapse-ldap3 = { version = ">=0.1", optional = true }
+psycopg2 = { version = ">=2.8", markers = "platform_python_implementation != 'PyPy'", optional = true }
+psycopg2cffi = { version = ">=2.8", markers = "platform_python_implementation == 'PyPy'", optional = true }
+psycopg2cffi-compat = { version = "==1.1", markers = "platform_python_implementation == 'PyPy'", optional = true }
+pysaml2 = { version = ">=4.5.0", optional = true }
+authlib = { version = ">=0.14.0", optional = true }
+# systemd-python is necessary for logging to the systemd journal via
+# `systemd.journal.JournalHandler`, as is documented in
+# `contrib/systemd/log_config.yaml`.
+# Note: systemd-python 231 appears to have been yanked from pypi
+systemd-python = { version = ">=231", optional = true }
+lxml = { version = ">=4.2.0", optional = true }
+sentry-sdk = { version = ">=0.7.2", optional = true }
+opentracing = { version = ">=2.2.0", optional = true }
+jaeger-client = { version = ">=4.0.0", optional = true }
+pyjwt = { version = ">=1.6.4", optional = true }
+txredisapi = { version = ">=1.4.7", optional = true }
+hiredis = { version = "*", optional = true }
+Pympler = { version = "*", optional = true }
+parameterized = { version = ">=0.7.4", optional = true }
+idna = { version = ">=2.5", optional = true }
+
+[tool.poetry.extras]
+# NB: Packages that should be part of `pip install matrix-synapse[all]` need to be specified
+# twice: once here, and once in the `all` extra.
+matrix-synapse-ldap3 = ["matrix-synapse-ldap3"]
+postgres = ["psycopg2", "psycopg2cffi", "psycopg2cffi-compat"]
+saml2 = ["pysaml2"]
+oidc = ["authlib"]
+# systemd-python is necessary for logging to the systemd journal via
+# `systemd.journal.JournalHandler`, as is documented in
+# `contrib/systemd/log_config.yaml`.
+systemd = ["systemd-python"]
+url_preview = ["lxml"]
+sentry = ["sentry-sdk"]
+opentracing = ["jaeger-client", "opentracing"]
+jwt = ["pyjwt"]
+# hiredis is not a *strict* dependency, but it makes things much faster.
+# (if it is not installed, we fall back to slow code.)
+redis = ["txredisapi", "hiredis"]
+# Required to use experimental `caches.track_memory_usage` config option.
+cache_memory = ["pympler"]
+test = ["parameterized", "idna"]
+
+# The duplication here is awful. I hate hate hate hate hate it. However, for now I want
+# to ensure you can still `pip install matrix-synapse[all]` like today. Two motivations:
+# 1) for new installations, I want instructions in existing documentation and tutorials
+#    out there to still work.
+# 2) I don't want to hard-code a list of extras into CI if I can help it. The ideal
+#    solution here would be something like https://github.com/python-poetry/poetry/issues/3413
+# Poetry 1.2's dependency groups might make this easier. But I'm not trying that out
+# until there's a stable release of 1.2.
+#
+# NB: the strings in this list must be *package* names, not extra names.
+# Some of our extra names _are_ package names, which can lead to great confusion.
+all = [
+    # matrix-synapse-ldap3
+    "matrix-synapse-ldap3",
+    # postgres
+    "psycopg2", "psycopg2cffi", "psycopg2cffi-compat",
+    # saml2
+    "pysaml2",
+    # oidc
+    "authlib",
+    # url_preview
+    "lxml",
+    # sentry
+    "sentry-sdk",
+    # opentracing
+    "jaeger-client", "opentracing",
+    # jwt
+    "pyjwt",
+    #redis
+    "txredisapi", "hiredis"
+    # omitted:
+    #   - cache_memory: this is an experimental option
+    #   - test: it's useful to have this separate from dev deps in the olddeps job
+    #   - systemd: this is a system-based requirement
+]
+
+[tool.poetry.dev-dependencies]
+## We pin black so that our tests don't start failing on new releases.
+isort = "==5.7.0"
+black = "==22.3.0"
+flake8-comprehensions = "*"
+flake8-bugbear = "==21.3.2"
+flake8 = "*"
+
+# Typechecking
+mypy = "==0.931"
+mypy-zope = "==0.3.5"
+types-bleach = ">=4.1.0"
+types-jsonschema = ">=3.2.0"
+types-opentracing = ">=2.4.2"
+types-Pillow = ">=8.3.4"
+types-psycopg2 = ">=2.9.9"
+types-pyOpenSSL = ">=20.0.7"
+types-PyYAML = ">=5.4.10"
+types-requests = ">=2.26.0"
+types-setuptools = ">=57.4.0"
+
+# Dependencies which are exclusively required by unit test code. This is
+# NOT a list of all modules that are necessary to run the unit tests.
+# Tests assume that all optional dependencies are installed.
+# parameterized<0.7.4 can create classes with names that would normally be invalid
+# identifiers. trial really does not like this when running with multiple workers.
+parameterized = ">=0.7.4"
+idna = ">=2.5"
+
+# The following are used by the release script
+click = "==8.1.0"
+GitPython = "==3.1.14"
+commonmark = "==0.9.1"
+pygithub = "==1.55"
+# The following are executed as commands by the release script.
+twine = "*"
+# Towncrier min version comes from #3425. Rationale unclear.
+towncrier = ">=18.6.0rc1"
+
+[build-system]
+requires = ["poetry-core>=1.0.0"]
+build-backend = "poetry.core.masonry.api"

scripts-dev/build_debian_packages.py

@@ -26,6 +26,7 @@ DISTS = (
     "debian:sid",
     "ubuntu:focal",  # 20.04 LTS (our EOL forced by Py38 on 2024-10-14)
     "ubuntu:impish",  # 21.10  (EOL 2022-07)
+    "ubuntu:jammy",  # 22.04 LTS (EOL 2027-04)
 )
 
 DESC = """\

scripts-dev/check-newsfragment.sh

@@ -19,7 +19,7 @@ if ! git diff --quiet FETCH_HEAD... -- debian; then
     if git diff --quiet FETCH_HEAD... -- debian/changelog; then
         echo "Updates to debian directory, but no update to the changelog." >&2
         echo "!! Please see the contributing guide for help writing your changelog entry:" >&2
-        echo "https://github.com/matrix-org/synapse/blob/develop/CONTRIBUTING.md#debian-changelog" >&2
+        echo "https://matrix-org.github.io/synapse/latest/development/contributing_guide.html#debian-changelog" >&2
         exit 1
     fi
 fi
@@ -32,7 +32,7 @@ fi
 
 # Print a link to the contributing guide if the user makes a mistake
 CONTRIBUTING_GUIDE_TEXT="!! Please see the contributing guide for help writing your changelog entry:
-https://github.com/matrix-org/synapse/blob/develop/CONTRIBUTING.md#changelog"
+https://matrix-org.github.io/synapse/latest/development/contributing_guide.html#changelog"
 
 # If check-newsfragment returns a non-zero exit code, print the contributing guide and exit
 python -m towncrier.check --compare-with=origin/develop || (echo -e "$CONTRIBUTING_GUIDE_TEXT" >&2 && exit 1)

scripts-dev/check_signature.py

@@ -1,72 +0,0 @@
-import argparse
-import json
-import logging
-import sys
-
-import dns.resolver
-import urllib2
-from signedjson.key import decode_verify_key_bytes, write_signing_keys
-from signedjson.sign import verify_signed_json
-from unpaddedbase64 import decode_base64
-
-
-def get_targets(server_name):
-    if ":" in server_name:
-        target, port = server_name.split(":")
-        yield (target, int(port))
-        return
-    try:
-        answers = dns.resolver.query("_matrix._tcp." + server_name, "SRV")
-        for srv in answers:
-            yield (srv.target, srv.port)
-    except dns.resolver.NXDOMAIN:
-        yield (server_name, 8448)
-
-
-def get_server_keys(server_name, target, port):
-    url = "https://%s:%i/_matrix/key/v1" % (target, port)
-    keys = json.load(urllib2.urlopen(url))
-    verify_keys = {}
-    for key_id, key_base64 in keys["verify_keys"].items():
-        verify_key = decode_verify_key_bytes(key_id, decode_base64(key_base64))
-        verify_signed_json(keys, server_name, verify_key)
-        verify_keys[key_id] = verify_key
-    return verify_keys
-
-
-def main():
-
-    parser = argparse.ArgumentParser()
-    parser.add_argument("signature_name")
-    parser.add_argument(
-        "input_json", nargs="?", type=argparse.FileType("r"), default=sys.stdin
-    )
-
-    args = parser.parse_args()
-    logging.basicConfig()
-
-    server_name = args.signature_name
-    keys = {}
-    for target, port in get_targets(server_name):
-        try:
-            keys = get_server_keys(server_name, target, port)
-            print("Using keys from https://%s:%s/_matrix/key/v1" % (target, port))
-            write_signing_keys(sys.stdout, keys.values())
-            break
-        except Exception:
-            logging.exception("Error talking to %s:%s", target, port)
-
-    json_to_check = json.load(args.input_json)
-    print("Checking JSON:")
-    for key_id in json_to_check["signatures"][args.signature_name]:
-        try:
-            key = keys[key_id]
-            verify_signed_json(json_to_check, args.signature_name, key)
-            print("PASS %s" % (key_id,))
-        except Exception:
-            logging.exception("Check for key %s failed" % (key_id,))
-            print("FAIL %s" % (key_id,))
-
-
-if __name__ == "__main__":
-    main()

scripts-dev/complement.sh

@@ -50,25 +50,18 @@ if [[ -n "$WORKERS" ]]; then
 
   export COMPLEMENT_BASE_IMAGE=complement-synapse-workers
   COMPLEMENT_DOCKERFILE=SynapseWorkers.Dockerfile
+
   # And provide some more configuration to complement.
-  export COMPLEMENT_CA=true
-  export COMPLEMENT_SPAWN_HS_TIMEOUT_SECS=25
+  export COMPLEMENT_SPAWN_HS_TIMEOUT_SECS=60
 else
   export COMPLEMENT_BASE_IMAGE=complement-synapse
-  COMPLEMENT_DOCKERFILE=Synapse.Dockerfile
+  COMPLEMENT_DOCKERFILE=Dockerfile
 fi
 
 # Build the Complement image from the Synapse image we just built.
-docker build -t $COMPLEMENT_BASE_IMAGE -f "$COMPLEMENT_DIR/dockerfiles/$COMPLEMENT_DOCKERFILE" "$COMPLEMENT_DIR/dockerfiles"
-
-cd "$COMPLEMENT_DIR"
-
-EXTRA_COMPLEMENT_ARGS=""
-if [[ -n "$1" ]]; then
-  # A test name regex has been set, supply it to Complement
-  EXTRA_COMPLEMENT_ARGS+="-run $1 "
-fi
+docker build -t $COMPLEMENT_BASE_IMAGE -f "docker/complement/$COMPLEMENT_DOCKERFILE" "docker/complement"
 
 # Run the tests!
 echo "Images built; running complement"
-go test -v -tags synapse_blacklist,msc2403 -count=1 $EXTRA_COMPLEMENT_ARGS ./tests/...
+cd "$COMPLEMENT_DIR"
+go test -v -tags synapse_blacklist,msc2716,msc3030,faster_joins -count=1 "$@" ./tests/...

scripts-dev/definitions.py

@@ -1,208 +0,0 @@
-#! /usr/bin/python
-
-import argparse
-import ast
-import os
-import re
-import sys
-
-import yaml
-
-
-class DefinitionVisitor(ast.NodeVisitor):
-    def __init__(self):
-        super().__init__()
-        self.functions = {}
-        self.classes = {}
-        self.names = {}
-        self.attrs = set()
-        self.definitions = {
-            "def": self.functions,
-            "class": self.classes,
-            "names": self.names,
-            "attrs": self.attrs,
-        }
-
-    def visit_Name(self, node):
-        self.names.setdefault(type(node.ctx).__name__, set()).add(node.id)
-
-    def visit_Attribute(self, node):
-        self.attrs.add(node.attr)
-        for child in ast.iter_child_nodes(node):
-            self.visit(child)
-
-    def visit_ClassDef(self, node):
-        visitor = DefinitionVisitor()
-        self.classes[node.name] = visitor.definitions
-        for child in ast.iter_child_nodes(node):
-            visitor.visit(child)
-
-    def visit_FunctionDef(self, node):
-        visitor = DefinitionVisitor()
-        self.functions[node.name] = visitor.definitions
-        for child in ast.iter_child_nodes(node):
-            visitor.visit(child)
-
-
-def non_empty(defs):
-    functions = {name: non_empty(f) for name, f in defs["def"].items()}
-    classes = {name: non_empty(f) for name, f in defs["class"].items()}
-    result = {}
-    if functions:
-        result["def"] = functions
-    if classes:
-        result["class"] = classes
-    names = defs["names"]
-    uses = []
-    for name in names.get("Load", ()):
-        if name not in names.get("Param", ()) and name not in names.get("Store", ()):
-            uses.append(name)
-    uses.extend(defs["attrs"])
-    if uses:
-        result["uses"] = uses
-    result["names"] = names
-    result["attrs"] = defs["attrs"]
-    return result
-
-
-def definitions_in_code(input_code):
-    input_ast = ast.parse(input_code)
-    visitor = DefinitionVisitor()
-    visitor.visit(input_ast)
-    definitions = non_empty(visitor.definitions)
-    return definitions
-
-
-def definitions_in_file(filepath):
-    with open(filepath) as f:
-        return definitions_in_code(f.read())
-
-
-def defined_names(prefix, defs, names):
-    for name, funcs in defs.get("def", {}).items():
-        names.setdefault(name, {"defined": []})["defined"].append(prefix + name)
-        defined_names(prefix + name + ".", funcs, names)
-
-    for name, funcs in defs.get("class", {}).items():
-        names.setdefault(name, {"defined": []})["defined"].append(prefix + name)
-        defined_names(prefix + name + ".", funcs, names)
-
-
-def used_names(prefix, item, defs, names):
-    for name, funcs in defs.get("def", {}).items():
-        used_names(prefix + name + ".", name, funcs, names)
-
-    for name, funcs in defs.get("class", {}).items():
-        used_names(prefix + name + ".", name, funcs, names)
-
-    path = prefix.rstrip(".")
-    for used in defs.get("uses", ()):
-        if used in names:
-            if item:
-                names[item].setdefault("uses", []).append(used)
-            names[used].setdefault("used", {}).setdefault(item, []).append(path)
-
-
-if __name__ == "__main__":
-
-    parser = argparse.ArgumentParser(description="Find definitions.")
-    parser.add_argument(
-        "--unused", action="store_true", help="Only list unused definitions"
-    )
-    parser.add_argument(
-        "--ignore", action="append", metavar="REGEXP", help="Ignore a pattern"
-    )
-    parser.add_argument(
-        "--pattern", action="append", metavar="REGEXP", help="Search for a pattern"
-    )
-    parser.add_argument(
-        "directories",
-        nargs="+",
-        metavar="DIR",
-        help="Directories to search for definitions",
-    )
-    parser.add_argument(
-        "--referrers",
-        default=0,
-        type=int,
-        help="Include referrers up to the given depth",
-    )
-    parser.add_argument(
-        "--referred",
-        default=0,
-        type=int,
-        help="Include referred down to the given depth",
-    )
-    parser.add_argument(
-        "--format", default="yaml", help="Output format, one of 'yaml' or 'dot'"
-    )
-    args = parser.parse_args()
-
-    definitions = {}
-    for directory in args.directories:
-        for root, _, files in os.walk(directory):
-            for filename in files:
-                if filename.endswith(".py"):
-                    filepath = os.path.join(root, filename)
-                    definitions[filepath] = definitions_in_file(filepath)
-
-    names = {}
-    for filepath, defs in definitions.items():
-        defined_names(filepath + ":", defs, names)
-
-    for filepath, defs in definitions.items():
-        used_names(filepath + ":", None, defs, names)
-
-    patterns = [re.compile(pattern) for pattern in args.pattern or ()]
-    ignore = [re.compile(pattern) for pattern in args.ignore or ()]
-
-    result = {}
-    for name, definition in names.items():
-        if patterns and not any(pattern.match(name) for pattern in patterns):
-            continue
-        if ignore and any(pattern.match(name) for pattern in ignore):
-            continue
-        if args.unused and definition.get("used"):
-            continue
-        result[name] = definition
-
-    referrer_depth = args.referrers
-    referrers = set()
-    while referrer_depth:
-        referrer_depth -= 1
-        for entry in result.values():
-            for used_by in entry.get("used", ()):
-                referrers.add(used_by)
-        for name, definition in names.items():
-            if name not in referrers:
-                continue
-            if ignore and any(pattern.match(name) for pattern in ignore):
-                continue
-            result[name] = definition
-
-    referred_depth = args.referred
-    referred = set()
-    while referred_depth:
-        referred_depth -= 1
-        for entry in result.values():
-            for uses in entry.get("uses", ()):
-                referred.add(uses)
-        for name, definition in names.items():
-            if name not in referred:
-                continue
-            if ignore and any(pattern.match(name) for pattern in ignore):
-                continue
-            result[name] = definition
-
-    if args.format == "yaml":
-        yaml.dump(result, sys.stdout, default_flow_style=False)
-    elif args.format == "dot":
-        print("digraph {")
-        for name, entry in result.items():
-            print(name)
-            for used_by in entry.get("used", ()):
-                if used_by in result:
-                    print(used_by, "->", name)
-        print("}")
-    else:
-        raise ValueError("Unknown format %r" % (args.format))

scripts-dev/federation_client.py

@@ -124,7 +124,12 @@ def request(
     authorization_headers = []
 
     for key, sig in signed_json["signatures"][origin_name].items():
-        header = 'X-Matrix origin=%s,key="%s",sig="%s"' % (origin_name, key, sig)
+        header = 'X-Matrix origin=%s,key="%s",sig="%s",destination="%s"' % (
+            origin_name,
+            key,
+            sig,
+            destination,
+        )
         authorization_headers.append(header.encode("ascii"))
         print("Authorization: %s" % header, file=sys.stderr)
 

scripts-dev/generate_sample_config

@@ -1,28 +0,0 @@
-#!/usr/bin/env bash
-#
-# Update/check the docs/sample_config.yaml
-
-set -e
-
-cd "$(dirname "$0")/.."
-
-SAMPLE_CONFIG="docs/sample_config.yaml"
-SAMPLE_LOG_CONFIG="docs/sample_log_config.yaml"
-
-check() {
-    diff -u "$SAMPLE_LOG_CONFIG" <(./scripts/generate_log_config) >/dev/null || return 1
-}
-
-if [ "$1" == "--check" ]; then
-    diff -u "$SAMPLE_CONFIG" <(./scripts/generate_config --header-file docs/.sample_config_header.yaml) >/dev/null || {
-        echo -e "\e[1m\e[31m$SAMPLE_CONFIG is not up-to-date. Regenerate it with \`scripts-dev/generate_sample_config\`.\e[0m" >&2
-        exit 1
-    }
-    diff -u "$SAMPLE_LOG_CONFIG" <(./scripts/generate_log_config) >/dev/null || {
-        echo -e "\e[1m\e[31m$SAMPLE_LOG_CONFIG is not up-to-date. Regenerate it with \`scripts-dev/generate_sample_config\`.\e[0m" >&2
-        exit 1
-    }
-else
-    ./scripts/generate_config --header-file docs/.sample_config_header.yaml -o "$SAMPLE_CONFIG"
-    ./scripts/generate_log_config -o "$SAMPLE_LOG_CONFIG"
-fi

scripts-dev/generate_sample_config.sh

@@ -0,0 +1,28 @@
+#!/usr/bin/env bash
+#
+# Update/check the docs/sample_config.yaml
+
+set -e
+
+cd "$(dirname "$0")/.."
+
+SAMPLE_CONFIG="docs/sample_config.yaml"
+SAMPLE_LOG_CONFIG="docs/sample_log_config.yaml"
+
+check() {
+    diff -u "$SAMPLE_LOG_CONFIG" <(synapse/_scripts/generate_log_config.py) >/dev/null || return 1
+}
+
+if [ "$1" == "--check" ]; then
+    diff -u "$SAMPLE_CONFIG" <(synapse/_scripts/generate_config.py --header-file docs/.sample_config_header.yaml) >/dev/null || {
+        echo -e "\e[1m\e[31m$SAMPLE_CONFIG is not up-to-date. Regenerate it with \`scripts-dev/generate_sample_config.sh\`.\e[0m" >&2
+        exit 1
+    }
+    diff -u "$SAMPLE_LOG_CONFIG" <(synapse/_scripts/generate_log_config.py) >/dev/null || {
+        echo -e "\e[1m\e[31m$SAMPLE_LOG_CONFIG is not up-to-date. Regenerate it with \`scripts-dev/generate_sample_config.sh\`.\e[0m" >&2
+        exit 1
+    }
+else
+    synapse/_scripts/generate_config.py --header-file docs/.sample_config_header.yaml -o "$SAMPLE_CONFIG"
+    synapse/_scripts/generate_log_config.py -o "$SAMPLE_LOG_CONFIG"
+fi

scripts-dev/hash_history.py

@@ -1,81 +0,0 @@
-import sqlite3
-import sys
-
-from unpaddedbase64 import decode_base64, encode_base64
-
-from synapse.crypto.event_signing import (
-    add_event_pdu_content_hash,
-    compute_pdu_event_reference_hash,
-)
-from synapse.federation.units import Pdu
-from synapse.storage._base import SQLBaseStore
-from synapse.storage.pdu import PduStore
-from synapse.storage.signatures import SignatureStore
-
-
-class Store:
-    _get_pdu_tuples = PduStore.__dict__["_get_pdu_tuples"]
-    _get_pdu_content_hashes_txn = SignatureStore.__dict__["_get_pdu_content_hashes_txn"]
-    _get_prev_pdu_hashes_txn = SignatureStore.__dict__["_get_prev_pdu_hashes_txn"]
-    _get_pdu_origin_signatures_txn = SignatureStore.__dict__[
-        "_get_pdu_origin_signatures_txn"
-    ]
-    _store_pdu_content_hash_txn = SignatureStore.__dict__["_store_pdu_content_hash_txn"]
-    _store_pdu_reference_hash_txn = SignatureStore.__dict__[
-        "_store_pdu_reference_hash_txn"
-    ]
-    _store_prev_pdu_hash_txn = SignatureStore.__dict__["_store_prev_pdu_hash_txn"]
-    simple_insert_txn = SQLBaseStore.__dict__["simple_insert_txn"]
-
-
-store = Store()
-
-
-def select_pdus(cursor):
-    cursor.execute("SELECT pdu_id, origin FROM pdus ORDER BY depth ASC")
-
-    ids = cursor.fetchall()
-
-    pdu_tuples = store._get_pdu_tuples(cursor, ids)
-
-    pdus = [Pdu.from_pdu_tuple(p) for p in pdu_tuples]
-
-    reference_hashes = {}
-
-    for pdu in pdus:
-        try:
-            if pdu.prev_pdus:
-                print("PROCESS", pdu.pdu_id, pdu.origin, pdu.prev_pdus)
-                for pdu_id, origin, hashes in pdu.prev_pdus:
-                    ref_alg, ref_hsh = reference_hashes[(pdu_id, origin)]
-                    hashes[ref_alg] = encode_base64(ref_hsh)
-                    store._store_prev_pdu_hash_txn(
-                        cursor, pdu.pdu_id, pdu.origin, pdu_id, origin, ref_alg, ref_hsh
-                    )
-                print("SUCCESS", pdu.pdu_id, pdu.origin, pdu.prev_pdus)
-            pdu = add_event_pdu_content_hash(pdu)
-            ref_alg, ref_hsh = compute_pdu_event_reference_hash(pdu)
-            reference_hashes[(pdu.pdu_id, pdu.origin)] = (ref_alg, ref_hsh)
-            store._store_pdu_reference_hash_txn(
-                cursor, pdu.pdu_id, pdu.origin, ref_alg, ref_hsh
-            )
-
-            for alg, hsh_base64 in pdu.hashes.items():
-                print(alg, hsh_base64)
-                store._store_pdu_content_hash_txn(
-                    cursor, pdu.pdu_id, pdu.origin, alg, decode_base64(hsh_base64)
-                )
-
-        except Exception:
-            print("FAILED_", pdu.pdu_id, pdu.origin, pdu.prev_pdus)
-
-
-def main():
-    conn = sqlite3.connect(sys.argv[1])
-    cursor = conn.cursor()
-    select_pdus(cursor)
-    conn.commit()
-
-
-if __name__ == "__main__":
-    main()

scripts-dev/lint.sh

@@ -79,22 +79,19 @@ else
   # If we were not asked to lint changed files, and no paths were found as a result,
   # then lint everything!
   if [[ -z ${files+x} ]]; then
-    # Lint all source code files and directories
-    # Note: this list aims to mirror the one in tox.ini
+      # CI runs each linter on the entire checkout, e.g. `black .`. So don't
+      # rely on this list to *find* lint targets if that misses a file; instead;
+      # use it to exclude files from linters when this can't be done by config.
+      #
+      # To check which files the linters examine, use:
+      #     black --verbose . 2>&1 | \grep -v ignored
+      #     isort --show-files .
+      #     flake8 --verbose .  # This isn't a great option
+      #     mypy has explicit config in mypy.ini; there is also mypy --verbose
       files=(
           "synapse" "docker" "tests"
-          # annoyingly, black doesn't find these so we have to list them
-          "scripts/export_signing_key"
-          "scripts/generate_config"
-          "scripts/generate_log_config"
-          "scripts/hash_password"
-          "scripts/register_new_matrix_user"
-          "scripts/synapse_port_db"
-          "scripts/update_synapse_database"
           "scripts-dev"
-          "scripts-dev/build_debian_packages"
-          "scripts-dev/sign_json"
-          "contrib" "synctl" "setup.py" "synmark" "stubs" ".ci"
+          "contrib" "synmark" "stubs" ".ci"
       )
   fi
 fi

scripts-dev/list_url_patterns.py

@@ -1,60 +0,0 @@
-#! /usr/bin/python
-
-import argparse
-import ast
-import os
-import sys
-
-import yaml
-
-PATTERNS_V1 = []
-PATTERNS_V2 = []
-
-RESULT = {"v1": PATTERNS_V1, "v2": PATTERNS_V2}
-
-
-class CallVisitor(ast.NodeVisitor):
-    def visit_Call(self, node):
-        if isinstance(node.func, ast.Name):
-            name = node.func.id
-        else:
-            return
-
-        if name == "client_patterns":
-            PATTERNS_V2.append(node.args[0].s)
-
-
-def find_patterns_in_code(input_code):
-    input_ast = ast.parse(input_code)
-    visitor = CallVisitor()
-    visitor.visit(input_ast)
-
-
-def find_patterns_in_file(filepath):
-    with open(filepath) as f:
-        find_patterns_in_code(f.read())
-
-
-parser = argparse.ArgumentParser(description="Find url patterns.")
-
-parser.add_argument(
-    "directories",
-    nargs="+",
-    metavar="DIR",
-    help="Directories to search for definitions",
-)
-
-args = parser.parse_args()
-
-
-for directory in args.directories:
-    for root, _, files in os.walk(directory):
-        for filename in files:
-            if filename.endswith(".py"):
-                filepath = os.path.join(root, filename)
-                find_patterns_in_file(filepath)
-
-PATTERNS_V1.sort()
-PATTERNS_V2.sort()
-
-yaml.dump(RESULT, sys.stdout, default_flow_style=False)

scripts-dev/make_full_schema.sh

@@ -147,7 +147,7 @@ python -m synapse.app.homeserver --generate-keys -c "$SQLITE_CONFIG"
 
 # Make sure the SQLite3 database is using the latest schema and has no pending background update.
 echo "Running db background jobs..."
-scripts/update_synapse_database --database-config --run-background-updates "$SQLITE_CONFIG"
+synapse/_scripts/update_synapse_database.py --database-config --run-background-updates "$SQLITE_CONFIG"
 
 # Create the PostgreSQL database.
 echo "Creating postgres database..."
@@ -156,10 +156,10 @@ createdb --lc-collate=C --lc-ctype=C --template=template0 "$POSTGRES_DB_NAME"
 echo "Copying data from SQLite3 to Postgres with synapse_port_db..."
 if [ -z "$COVERAGE" ]; then
   # No coverage needed
-  scripts/synapse_port_db --sqlite-database "$SQLITE_DB" --postgres-config "$POSTGRES_CONFIG"
+  synapse/_scripts/synapse_port_db.py --sqlite-database "$SQLITE_DB" --postgres-config "$POSTGRES_CONFIG"
 else
   # Coverage desired
-  coverage run scripts/synapse_port_db --sqlite-database "$SQLITE_DB" --postgres-config "$POSTGRES_CONFIG"
+  coverage run synapse/_scripts/synapse_port_db.py --sqlite-database "$SQLITE_DB" --postgres-config "$POSTGRES_CONFIG"
 fi
 
 # Delete schema_version, applied_schema_deltas and applied_module_schemas tables

scripts-dev/release.py

@@ -25,13 +25,12 @@ import sys
 import urllib.request
 from os import path
 from tempfile import TemporaryDirectory
-from typing import List, Optional, Tuple
+from typing import List, Optional
 
 import attr
 import click
 import commonmark
 import git
-import redbaron
 from click.exceptions import ClickException
 from github import Github
 from packaging import version
@@ -66,11 +65,16 @@ def cli():
 
         ./scripts-dev/release.py tag
 
-        # ... wait for asssets to build ...
+        # ... wait for assets to build ...
 
         ./scripts-dev/release.py publish
+
         ./scripts-dev/release.py upload
 
+        # Optional: generate some nice links for the announcement
+
+        ./scripts-dev/release.py announce
+
     If the env var GH_TOKEN (or GITHUB_TOKEN) is set, or passed into the
     `tag`/`publish` command, then a new draft release will be created/published.
     """
@@ -95,7 +99,7 @@ def prepare():
     repo.remote().fetch()
 
     # Get the current version and AST from root Synapse module.
-    current_version, parsed_synapse_ast, version_node = parse_version_from_module()
+    current_version = get_package_version()
 
     # Figure out what sort of release we're doing and calcuate the new version.
     rc = click.confirm("RC", default=True)
@@ -157,7 +161,7 @@ def prepare():
         click.get_current_context().abort()
 
     # Switch to the release branch.
-    parsed_new_version = version.parse(new_version)
+    parsed_new_version: version.Version = version.parse(new_version)
 
     # We assume for debian changelogs that we only do RCs or full releases.
     assert not parsed_new_version.is_devrelease
@@ -202,17 +206,15 @@ def prepare():
         # Create the new release branch
         release_branch = repo.create_head(release_branch_name, commit=base_branch)
 
-    # Switch to the release branch and ensure its up to date.
+    # Switch to the release branch and ensure it's up to date.
     repo.git.checkout(release_branch_name)
     update_branch(repo)
 
-    # Update the `__version__` variable and write it back to the file.
-    version_node.value = '"' + new_version + '"'
-    with open("synapse/__init__.py", "w") as f:
-        f.write(parsed_synapse_ast.dumps())
+    # Update the version specified in pyproject.toml.
+    subprocess.check_output(["poetry", "version", new_version])
 
     # Generate changelogs.
-    generate_and_write_changelog(current_version)
+    generate_and_write_changelog(current_version, new_version)
 
     # Generate debian changelogs
     if parsed_new_version.pre is not None:
@@ -225,7 +227,7 @@ def prepare():
         debian_version = new_version
 
     run_until_successful(
-        f'dch -M -v {debian_version} "New synapse release {debian_version}."',
+        f'dch -M -v {debian_version} "New Synapse release {new_version}."',
         shell=True,
     )
     run_until_successful('dch -M -r -D stable ""', shell=True)
@@ -279,7 +281,7 @@ def tag(gh_token: Optional[str]):
     repo.remote().fetch()
 
     # Find out the version and tag name.
-    current_version, _, _ = parse_version_from_module()
+    current_version = get_package_version()
     tag_name = f"v{current_version}"
 
     # Check we haven't released this version.
@@ -357,7 +359,7 @@ def publish(gh_token: str):
     if repo.is_dirty():
         raise click.ClickException("Uncommitted changes exist.")
 
-    current_version, _, _ = parse_version_from_module()
+    current_version = get_package_version()
     tag_name = f"v{current_version}"
 
     if not click.confirm(f"Publish {tag_name}?", default=True):
@@ -391,7 +393,7 @@ def publish(gh_token: str):
 def upload():
     """Upload release to pypi."""
 
-    current_version, _, _ = parse_version_from_module()
+    current_version = get_package_version()
     tag_name = f"v{current_version}"
 
     pypi_asset_names = [
@@ -415,37 +417,46 @@ def upload():
     )
 
 
-def parse_version_from_module() -> Tuple[
-    version.Version, redbaron.RedBaron, redbaron.Node
-]:
-    # Parse the AST and load the `__version__` node so that we can edit it
-    # later.
-    with open("synapse/__init__.py") as f:
-        red = redbaron.RedBaron(f.read())
+@cli.command()
+def announce():
+    """Generate markdown to announce the release."""
 
-    version_node = None
-    for node in red:
-        if node.type != "assignment":
-            continue
+    current_version = get_package_version()
+    tag_name = f"v{current_version}"
 
-        if node.target.type != "name":
-            continue
+    click.echo(
+        f"""
+Hi everyone. Synapse {current_version} has just been released.
 
-        if node.target.value != "__version__":
-            continue
+[notes](https://github.com/matrix-org/synapse/releases/tag/{tag_name}) |\
+[docker](https://hub.docker.com/r/matrixdotorg/synapse/tags?name={tag_name}) | \
+[debs](https://packages.matrix.org/debian/) | \
+[pypi](https://pypi.org/project/matrix-synapse/{current_version}/)"""
+    )
 
-        version_node = node
-        break
+    if "rc" in tag_name:
+        click.echo(
+            """
+Announce the RC in
+- #homeowners:matrix.org (Synapse Announcements)
+- #synapse-dev:matrix.org"""
+        )
+    else:
+        click.echo(
+            """
+Announce the release in
+- #homeowners:matrix.org (Synapse Announcements), bumping the version in the topic
+- #synapse:matrix.org (Synapse Admins), bumping the version in the topic
+- #synapse-dev:matrix.org
+- #synapse-package-maintainers:matrix.org"""
+        )
 
-    if not version_node:
-        print("Failed to find '__version__' definition in synapse/__init__.py")
-        sys.exit(1)
 
-    # Parse the current version.
-    current_version = version.parse(version_node.value.value.strip('"'))
-    assert isinstance(current_version, version.Version)
-
-    return current_version, red, version_node
+def get_package_version() -> version.Version:
+    version_string = subprocess.check_output(["poetry", "version", "--short"]).decode(
+        "utf-8"
+    )
+    return version.Version(version_string)
 
 
 def find_ref(repo: git.Repo, ref_name: str) -> Optional[git.HEAD]:
@@ -525,11 +536,13 @@ def get_changes_for_version(wanted_version: version.Version) -> str:
     return "\n".join(version_changelog)
 
 
-def generate_and_write_changelog(current_version: version.Version):
+def generate_and_write_changelog(current_version: version.Version, new_version: str):
     # We do this by getting a draft so that we can edit it before writing to the
     # changelog.
     result = run_until_successful(
-        "python3 -m towncrier --draft", shell=True, capture_output=True
+        f"python3 -m towncrier build --draft --version {new_version}",
+        shell=True,
+        capture_output=True,
     )
     new_changes = result.stdout.decode("utf-8")
     new_changes = new_changes.replace(