Add header margin change

to be more generic

.ci/latest_deps_build_failed_issue_template.md

@@ -1,4 +0,0 @@
----
-title: CI run against latest deps is failing
----
-See https://github.com/{{env.GITHUB_REPOSITORY}}/actions/runs/{{env.GITHUB_RUN_ID}}

.ci/patch_for_twisted_trunk.sh

@@ -0,0 +1,8 @@
+#!/bin/sh
+
+# replaces the dependency on Twisted in `python_dependencies` with trunk.
+
+set -e
+cd "$(dirname "$0")"/..
+
+sed -i -e 's#"Twisted.*"#"Twisted @ git+https://github.com/twisted/twisted"#' synapse/python_dependencies.py

.ci/scripts/record_available_doc_versions.py

@@ -1,28 +0,0 @@
-#!/usr/bin/env python3
-# This script will write a json file to $OUTPUT_FILE that contains the name of
-# each available Synapse version with documentation.
-#
-# This script assumes that any top-level directory in the "gh-pages" branch is
-# named after a documentation version and contains documentation website files.
-
-import os.path
-import json
-
-OUTPUT_FILE = "versions.json"
-
-# Determine the list of Synapse versions that have documentation.
-doc_versions = []
-for filepath in os.listdir():
-    if os.path.isdir(filepath):
-        doc_versions.append(filepath)
-
-# Record the documentation versions in a json file, such that the
-# frontend javascript is aware of what versions exist.
-to_write = {
-    "versions": doc_versions,
-    "default_version": "latest",
-}
-
-# Write the file.
-with open(OUTPUT_FILE, "w") as f:
-    f.write(json.dumps(to_write))

.ci/scripts/test_export_data_command.sh

@@ -2,24 +2,29 @@
 
 # Test for the export-data admin command against sqlite and postgres
 
-# Expects Synapse to have been already installed with `poetry install --extras postgres`.
-# Expects `poetry` to be available on the `PATH`.
-
 set -xe
 cd "$(dirname "$0")/../.."
 
+echo "--- Install dependencies"
+
+# Install dependencies for this test.
+pip install psycopg2
+
+# Install Synapse itself. This won't update any libraries.
+pip install -e .
+
 echo "--- Generate the signing key"
 
 # Generate the server's signing key.
-poetry run synapse_homeserver --generate-keys -c .ci/sqlite-config.yaml
+python -m synapse.app.homeserver --generate-keys -c .ci/sqlite-config.yaml
 
 echo "--- Prepare test database"
 
 # Make sure the SQLite3 database is using the latest schema and has no pending background update.
-poetry run update_synapse_database --database-config .ci/sqlite-config.yaml --run-background-updates
+update_synapse_database --database-config .ci/sqlite-config.yaml --run-background-updates
 
 # Run the export-data command on the sqlite test database
-poetry run python -m synapse.app.admin_cmd -c .ci/sqlite-config.yaml  export-data @anon-20191002_181700-832:localhost:8800 \
+python -m synapse.app.admin_cmd -c .ci/sqlite-config.yaml  export-data @anon-20191002_181700-832:localhost:8800 \
 --output-directory /tmp/export_data
 
 # Test that the output directory exists and contains the rooms directory
@@ -32,14 +37,14 @@ else
 fi
 
 # Create the PostgreSQL database.
-poetry run .ci/scripts/postgres_exec.py "CREATE DATABASE synapse"
+.ci/scripts/postgres_exec.py "CREATE DATABASE synapse"
 
 # Port the SQLite databse to postgres so we can check command works against postgres
 echo "+++ Port SQLite3 databse to postgres"
-poetry run synapse_port_db --sqlite-database .ci/test_db.db --postgres-config .ci/postgres-config.yaml
+synapse_port_db --sqlite-database .ci/test_db.db --postgres-config .ci/postgres-config.yaml
 
 # Run the export-data command on postgres database
-poetry run python -m synapse.app.admin_cmd -c .ci/postgres-config.yaml  export-data @anon-20191002_181700-832:localhost:8800 \
+python -m synapse.app.admin_cmd -c .ci/postgres-config.yaml  export-data @anon-20191002_181700-832:localhost:8800 \
 --output-directory /tmp/export_data2
 
 # Test that the output directory exists and contains the rooms directory

.ci/scripts/test_old_deps.sh

@@ -1,9 +1,6 @@
 #!/usr/bin/env bash
-# this script is run by GitHub Actions in a plain `focal` container; it
-# - installs the minimal system requirements, and poetry;
-# - patches the project definition file to refer to old versions only;
-# - creates a venv with these old versions using poetry; and finally
-# - invokes `trial` to run the tests with old deps.
+# this script is run by GitHub Actions in a plain `focal` container; it installs the
+# minimal requirements for tox and hands over to the py3-old tox environment.
 
 # Prevent tzdata from asking for user input
 export DEBIAN_FRONTEND=noninteractive
@@ -12,70 +9,12 @@ set -ex
 
 apt-get update
 apt-get install -y \
-        python3 python3-dev python3-pip python3-venv pipx \
-        libxml2-dev libxslt-dev xmlsec1 zlib1g-dev libjpeg-dev libwebp-dev
+        python3 python3-dev python3-pip python3-venv \
+        libxml2-dev libxslt-dev xmlsec1 zlib1g-dev tox libjpeg-dev libwebp-dev
 
 export LANG="C.UTF-8"
 
 # Prevent virtualenv from auto-updating pip to an incompatible version
 export VIRTUALENV_NO_DOWNLOAD=1
 
-# TODO: in the future, we could use an implementation of
-#   https://github.com/python-poetry/poetry/issues/3527
-#   https://github.com/pypa/pip/issues/8085
-# to select the lowest possible versions, rather than resorting to this sed script.
-
-# Patch the project definitions in-place:
-# - Replace all lower and tilde bounds with exact bounds
-# - Make the pyopenssl 17.0, which is the oldest version that works with
-#   a `cryptography` compiled against OpenSSL 1.1.
-# - Delete all lines referring to psycopg2 --- so no testing of postgres support.
-# - Omit systemd: we're not logging to journal here.
-
-# TODO: also replace caret bounds, see https://python-poetry.org/docs/dependency-specification/#version-constraints
-# We don't use these yet, but IIRC they are the default bound used when you `poetry add`.
-# The sed expression 's/\^/==/g' ought to do the trick. But it would also change
-# `python = "^3.7"` to `python = "==3.7", which would mean we fail because olddeps
-# runs on 3.8 (#12343).
-
-sed -i \
-   -e "s/[~>]=/==/g" \
-   -e "/psycopg2/d" \
-   -e 's/pyOpenSSL = "==16.0.0"/pyOpenSSL = "==17.0.0"/' \
-   -e '/systemd/d' \
-   pyproject.toml
-
-# Use poetry to do the installation. This ensures that the versions are all mutually
-# compatible (as far the package metadata declares, anyway); pip's package resolver
-# is more lax.
-#
-# Rather than `poetry install --no-dev`, we drop all dev dependencies from the
-# toml file. This means we don't have to ensure compatibility between old deps and
-# dev tools.
-
-pip install --user toml
-
-REMOVE_DEV_DEPENDENCIES="
-import toml
-with open('pyproject.toml', 'r') as f:
-    data = toml.loads(f.read())
-
-del data['tool']['poetry']['dev-dependencies']
-
-with open('pyproject.toml', 'w') as f:
-    toml.dump(data, f)
-"
-python3 -c "$REMOVE_DEV_DEPENDENCIES"
-
-pipx install poetry==1.1.12
-~/.local/bin/poetry lock
-
-echo "::group::Patched pyproject.toml"
-cat pyproject.toml
-echo "::endgroup::"
-echo "::group::Lockfile after patch"
-cat poetry.lock
-echo "::endgroup::"
-
-~/.local/bin/poetry install -E "all test"
-~/.local/bin/poetry run trial --jobs=2 tests
+exec tox -e py3-old

.ci/scripts/test_synapse_port_db.sh

@@ -1,37 +1,43 @@
 #!/usr/bin/env bash
 #
 # Test script for 'synapse_port_db'.
-#   - configures synapse and a postgres server.
+#   - sets up synapse and deps
 #   - runs the port script on a prepopulated test sqlite db
 #   - also runs it against an new sqlite db
-#
-# Expects Synapse to have been already installed with `poetry install --extras postgres`.
-# Expects `poetry` to be available on the `PATH`.
+
 
 set -xe
 cd "$(dirname "$0")/../.."
 
+echo "--- Install dependencies"
+
+# Install dependencies for this test.
+pip install psycopg2 coverage coverage-enable-subprocess
+
+# Install Synapse itself. This won't update any libraries.
+pip install -e .
+
 echo "--- Generate the signing key"
 
 # Generate the server's signing key.
-poetry run synapse_homeserver --generate-keys -c .ci/sqlite-config.yaml
+python -m synapse.app.homeserver --generate-keys -c .ci/sqlite-config.yaml
 
 echo "--- Prepare test database"
 
 # Make sure the SQLite3 database is using the latest schema and has no pending background update.
-poetry run update_synapse_database --database-config .ci/sqlite-config.yaml --run-background-updates
+update_synapse_database --database-config .ci/sqlite-config.yaml --run-background-updates
 
 # Create the PostgreSQL database.
-poetry run .ci/scripts/postgres_exec.py "CREATE DATABASE synapse"
+.ci/scripts/postgres_exec.py "CREATE DATABASE synapse"
 
 echo "+++ Run synapse_port_db against test database"
 # TODO: this invocation of synapse_port_db (and others below) used to be prepended with `coverage run`,
 # but coverage seems unable to find the entrypoints installed by `pip install -e .`.
-poetry run synapse_port_db --sqlite-database .ci/test_db.db --postgres-config .ci/postgres-config.yaml
+synapse_port_db --sqlite-database .ci/test_db.db --postgres-config .ci/postgres-config.yaml
 
 # We should be able to run twice against the same database.
 echo "+++ Run synapse_port_db a second time"
-poetry run synapse_port_db --sqlite-database .ci/test_db.db --postgres-config .ci/postgres-config.yaml
+synapse_port_db --sqlite-database .ci/test_db.db --postgres-config .ci/postgres-config.yaml
 
 #####
 
@@ -42,12 +48,12 @@ echo "--- Prepare empty SQLite database"
 # we do this by deleting the sqlite db, and then doing the same again.
 rm .ci/test_db.db
 
-poetry run update_synapse_database --database-config .ci/sqlite-config.yaml --run-background-updates
+update_synapse_database --database-config .ci/sqlite-config.yaml --run-background-updates
 
 # re-create the PostgreSQL database.
-poetry run .ci/scripts/postgres_exec.py \
+.ci/scripts/postgres_exec.py \
   "DROP DATABASE synapse" \
   "CREATE DATABASE synapse"
 
 echo "+++ Run synapse_port_db against empty database"
-poetry run synapse_port_db --sqlite-database .ci/test_db.db --postgres-config .ci/postgres-config.yaml
+synapse_port_db --sqlite-database .ci/test_db.db --postgres-config .ci/postgres-config.yaml

.dockerignore

@@ -4,8 +4,8 @@
 # things to include
 !docker
 !synapse
+!MANIFEST.in
 !README.rst
-!pyproject.toml
-!poetry.lock
+!setup.py
 
 **/__pycache__

.github/workflows/docs.yaml

@@ -14,7 +14,7 @@ on:
 
 jobs:
   pages:
-    name: Build and deploy docs
+    name: GitHub Pages
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
@@ -22,7 +22,7 @@ jobs:
       - name: Setup mdbook
         uses: peaceiris/actions-mdbook@4b5ef36b314c2599664ca107bb8c02412548d79d # v1.1.14
         with:
-          mdbook-version: '0.4.17'
+          mdbook-version: '0.4.9'
 
       - name: Build the documentation
         # mdbook will only create an index.html if we're including docs/README.md in SUMMARY.md.
@@ -63,29 +63,3 @@ jobs:
           github_token: ${{ secrets.GITHUB_TOKEN }}
           publish_dir: ./book
           destination_dir: ./${{ steps.vars.outputs.branch-version }}
-
-  list_available_versions:
-    needs: pages
-    runs-on: ubuntu-latest
-    steps:
-      # Check out the current branch
-      - uses: actions/checkout@v3
-        with:
-          persist-credentials: false
-
-      - name: Save the script
-        run: cp .ci/scripts/record_available_doc_versions.py /
-
-      - uses: actions/setup-python@v3
-
-      # Check out the gh-pages branch, which we'll be pushing the doc versions to
-      - uses: actions/checkout@v3
-        with:
-          persist-credentials: false
-          # Check out the gh-pages branch
-          ref: 'gh-pages'
-
-      - name: Record the available documentation versions
-        run: |
-          # Download the script
-          /record_available_doc_versions

.github/workflows/latest_deps.yml

@@ -1,159 +0,0 @@
-# People who are freshly `pip install`ing from PyPI will pull in the latest versions of
-# dependencies which match the broad requirements. Since most CI runs are against
-# the locked poetry environment, run specifically against the latest dependencies to
-# know if there's an upcoming breaking change.
-#
-# As an overview this workflow:
-# - checks out develop,
-# - installs from source, pulling in the dependencies like a fresh `pip install` would, and 
-# - runs mypy and test suites in that checkout.
-#
-# Based on the twisted trunk CI job.
-
-name: Latest dependencies
-
-on:
-  schedule:
-    - cron: 0 7 * * *
-  workflow_dispatch:
-
-concurrency:
-  group: ${{ github.workflow }}-${{ github.ref }}
-  cancel-in-progress: true
-
-jobs:
-  mypy:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
-      # The dev dependencies aren't exposed in the wheel metadata (at least with current
-      # poetry-core versions), so we install with poetry.
-      - uses: matrix-org/setup-python-poetry@v1
-        with:
-          python-version: "3.x"
-          poetry-version: "1.2.0b1"
-          extras: "all"
-      # Dump installed versions for debugging.
-      - run: poetry run pip list > before.txt
-      # Upgrade all runtime dependencies only. This is intended to mimic a fresh
-      # `pip install matrix-synapse[all]` as closely as possible.
-      - run: poetry update --no-dev
-      - run: poetry run pip list > after.txt && (diff -u before.txt after.txt || true)
-      - name: Remove warn_unused_ignores from mypy config
-        run: sed '/warn_unused_ignores = True/d' -i mypy.ini
-      - run: poetry run mypy
-  trial:
-    runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        include:
-          - database: "sqlite"
-          - database: "postgres"
-            postgres-version: "14"
-
-    steps:
-      - uses: actions/checkout@v2
-      - run: sudo apt-get -qq install xmlsec1
-      - name: Set up PostgreSQL ${{ matrix.postgres-version }}
-        if: ${{ matrix.postgres-version }}
-        run: |
-          docker run -d -p 5432:5432 \
-            -e POSTGRES_PASSWORD=postgres \
-            -e POSTGRES_INITDB_ARGS="--lc-collate C --lc-ctype C --encoding UTF8" \
-            postgres:${{ matrix.postgres-version }}
-      - uses: actions/setup-python@v2
-        with:
-          python-version: "3.x"
-      - run: pip install .[all,test]
-      - name: Await PostgreSQL
-        if: ${{ matrix.postgres-version }}
-        timeout-minutes: 2
-        run: until pg_isready -h localhost; do sleep 1; done
-      - run: python -m twisted.trial --jobs=2 tests
-        env:
-          SYNAPSE_POSTGRES: ${{ matrix.database == 'postgres' || '' }}
-          SYNAPSE_POSTGRES_HOST: localhost
-          SYNAPSE_POSTGRES_USER: postgres
-          SYNAPSE_POSTGRES_PASSWORD: postgres
-      - name: Dump logs
-        # Logs are most useful when the command fails, always include them.
-        if: ${{ always() }}
-        # Note: Dumps to workflow logs instead of using actions/upload-artifact
-        #       This keeps logs colocated with failing jobs
-        #       It also ignores find's exit code; this is a best effort affair
-        run: >-
-          find _trial_temp -name '*.log'
-          -exec echo "::group::{}" \;
-          -exec cat {} \;
-          -exec echo "::endgroup::" \;
-          || true
-
-
-  sytest:
-    runs-on: ubuntu-latest
-    container:
-      image: matrixdotorg/sytest-synapse:testing
-      volumes:
-        - ${{ github.workspace }}:/src
-    strategy:
-      fail-fast: false
-      matrix:
-        include:
-          - sytest-tag: focal
-
-          - sytest-tag: focal
-            postgres: postgres
-            workers: workers
-            redis: redis
-    env:
-      POSTGRES: ${{ matrix.postgres && 1}}
-      WORKERS: ${{ matrix.workers && 1 }}
-      REDIS: ${{ matrix.redis && 1 }}
-      BLACKLIST: ${{ matrix.workers && 'synapse-blacklist-with-workers' }}
-
-    steps:
-      - uses: actions/checkout@v2
-      - name: Ensure sytest runs `pip install`
-        # Delete the lockfile so sytest will `pip install` rather than `poetry install`
-        run: rm /src/poetry.lock
-        working-directory: /src
-      - name: Prepare test blacklist
-        run: cat sytest-blacklist .ci/worker-blacklist > synapse-blacklist-with-workers
-      - name: Run SyTest
-        run: /bootstrap.sh synapse
-        working-directory: /src
-      - name: Summarise results.tap
-        if: ${{ always() }}
-        run: /sytest/scripts/tap_to_gha.pl /logs/results.tap
-      - name: Upload SyTest logs
-        uses: actions/upload-artifact@v2
-        if: ${{ always() }}
-        with:
-          name: Sytest Logs - ${{ job.status }} - (${{ join(matrix.*, ', ') }})
-          path: |
-            /logs/results.tap
-            /logs/**/*.log*
-
-
-  # TODO: run complement (as with twisted trunk, see #12473).
-
-  # open an issue if the build fails, so we know about it.
-  open-issue:
-    if: failure()
-    needs:
-      # TODO: should mypy be included here? It feels more brittle than the other two.
-      - mypy
-      - trial
-      - sytest
-
-    runs-on: ubuntu-latest
-
-    steps:
-      - uses: actions/checkout@v2
-      - uses: JasonEtco/create-an-issue@5d9504915f79f9cc6d791934b8ef34f2353dd74d # v2.5.0, 2020-12-06
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-        with:
-          update_existing: true
-          filename: .ci/latest_deps_build_failed_issue_template.md
-

.github/workflows/tests.yml

@@ -15,14 +15,25 @@ jobs:
     steps:
       - uses: actions/checkout@v2
       - uses: actions/setup-python@v2
-      - run: pip install .
+      - run: pip install -e .
       - run: scripts-dev/generate_sample_config.sh --check
       - run: scripts-dev/config-lint.sh
 
   lint:
-    uses: "matrix-org/backend-meta/.github/workflows/python-poetry-ci.yml@v1"
-    with:
-      typechecking-extras: "all"
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        toxenv:
+          - "check_codestyle"
+          - "check_isort"
+          - "mypy"
+          - "packaging"
+
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/setup-python@v2
+      - run: pip install tox
+      - run: tox -e ${{ matrix.toxenv }}
 
   lint-crlf:
     runs-on: ubuntu-latest
@@ -61,23 +72,23 @@ jobs:
       matrix:
         python-version: ["3.7", "3.8", "3.9", "3.10"]
         database: ["sqlite"]
-        extras: ["all"]
+        toxenv: ["py"]
         include:
           # Newest Python without optional deps
           - python-version: "3.10"
-            extras: ""
+            toxenv: "py-noextras"
 
           # Oldest Python with PostgreSQL
           - python-version: "3.7"
             database: "postgres"
             postgres-version: "10"
-            extras: "all"
+            toxenv: "py"
 
           # Newest Python with newest PostgreSQL
           - python-version: "3.10"
             database: "postgres"
             postgres-version: "14"
-            extras: "all"
+            toxenv: "py"
 
     steps:
       - uses: actions/checkout@v2
@@ -89,16 +100,17 @@ jobs:
             -e POSTGRES_PASSWORD=postgres \
             -e POSTGRES_INITDB_ARGS="--lc-collate C --lc-ctype C --encoding UTF8" \
             postgres:${{ matrix.postgres-version }}
-      - uses: matrix-org/setup-python-poetry@v1
+      - uses: actions/setup-python@v2
         with:
           python-version: ${{ matrix.python-version }}
-          extras: ${{ matrix.extras }}
+      - run: pip install tox
       - name: Await PostgreSQL
         if: ${{ matrix.postgres-version }}
         timeout-minutes: 2
         run: until pg_isready -h localhost; do sleep 1; done
-      - run: poetry run trial --jobs=2 tests
+      - run: tox -e ${{ matrix.toxenv }}
         env:
+          TRIAL_FLAGS: "--jobs=2"
           SYNAPSE_POSTGRES: ${{ matrix.database == 'postgres' || '' }}
           SYNAPSE_POSTGRES_HOST: localhost
           SYNAPSE_POSTGRES_USER: postgres
@@ -117,7 +129,6 @@ jobs:
           || true
 
   trial-olddeps:
-    # Note: sqlite only; no postgres
     if: ${{ !cancelled() && !failure() }} # Allow previous steps to be skipped, but not fail
     needs: linting-done
     runs-on: ubuntu-latest
@@ -125,11 +136,11 @@ jobs:
       - uses: actions/checkout@v2
       - name: Test with old deps
         uses: docker://ubuntu:focal # For old python and sqlite
-        # Note: focal seems to be using 3.8, but the oldest is 3.7?
-        # See https://github.com/matrix-org/synapse/issues/12343
         with:
           workdir: /github/workspace
           entrypoint: .ci/scripts/test_old_deps.sh
+        env:
+          TRIAL_FLAGS: "--jobs=2"
       - name: Dump logs
         # Logs are most useful when the command fails, always include them.
         if: ${{ always() }}
@@ -145,24 +156,23 @@ jobs:
 
   trial-pypy:
     # Very slow; only run if the branch name includes 'pypy'
-    # Note: sqlite only; no postgres. Completely untested since poetry move.
     if: ${{ contains(github.ref, 'pypy') && !failure() && !cancelled() }}
     needs: linting-done
     runs-on: ubuntu-latest
     strategy:
       matrix:
         python-version: ["pypy-3.7"]
-        extras: ["all"]
 
     steps:
       - uses: actions/checkout@v2
-      # Install libs necessary for PyPy to build binary wheels for dependencies
       - run: sudo apt-get -qq install xmlsec1 libxml2-dev libxslt-dev
-      - uses: matrix-org/setup-python-poetry@v1
+      - uses: actions/setup-python@v2
         with:
           python-version: ${{ matrix.python-version }}
-          extras: ${{ matrix.extras }}
-      - run: poetry run trial --jobs=2 tests
+      - run: pip install tox
+      - run: tox -e py
+        env:
+          TRIAL_FLAGS: "--jobs=2"
       - name: Dump logs
         # Logs are most useful when the command fails, always include them.
         if: ${{ always() }}
@@ -261,10 +271,9 @@ jobs:
     steps:
       - uses: actions/checkout@v2
       - run: sudo apt-get -qq install xmlsec1
-      - uses: matrix-org/setup-python-poetry@v1
+      - uses: actions/setup-python@v2
         with:
-          python-version: ${{ matrix.python-version }}
-          extras: "postgres"
+          python-version: "3.9"
       - run: .ci/scripts/test_export_data_command.sh
 
   portdb:
@@ -299,10 +308,9 @@ jobs:
     steps:
       - uses: actions/checkout@v2
       - run: sudo apt-get -qq install xmlsec1
-      - uses: matrix-org/setup-python-poetry@v1
+      - uses: actions/setup-python@v2
         with:
           python-version: ${{ matrix.python-version }}
-          extras: "postgres"
       - run: .ci/scripts/test_synapse_port_db.sh
 
   complement:
@@ -354,11 +362,27 @@ jobs:
             (wget -O - "https://github.com/matrix-org/complement/archive/$BRANCH_NAME.tar.gz" | tar -xz --strip-components=1 -C complement) && break
           done
 
+      # Build initial Synapse image
+      - run: docker build -t matrixdotorg/synapse:latest -f docker/Dockerfile .
+        working-directory: synapse
+        env:
+          DOCKER_BUILDKIT: 1
+
+      # Build a ready-to-run Synapse image based on the initial image above.
+      # This new image includes a config file, keys for signing and TLS, and
+      # other settings to make it suitable for testing under Complement.
+      - run: docker build -t complement-synapse -f Synapse.Dockerfile .
+        working-directory: complement/dockerfiles
+
+      # Run Complement
       - run: |
           set -o pipefail
-          COMPLEMENT_DIR=`pwd`/complement synapse/scripts-dev/complement.sh -json 2>&1 | gotestfmt
+          go test -v -json -p 1 -tags synapse_blacklist,msc2403,msc2716,msc3030 ./tests/... 2>&1 | gotestfmt
         shell: bash
         name: Run Complement Tests
+        env:
+          COMPLEMENT_BASE_IMAGE: complement-synapse:latest
+        working-directory: complement
 
   # a job which marks all the other jobs as complete, thus allowing PRs to be merged.
   tests-done:

.github/workflows/twisted_trunk.yml

@@ -6,27 +6,16 @@ on:
 
   workflow_dispatch:
 
-concurrency:
-  group: ${{ github.workflow }}-${{ github.ref }}
-  cancel-in-progress: true
-
 jobs:
   mypy:
     runs-on: ubuntu-latest
 
     steps:
       - uses: actions/checkout@v2
-      - uses: matrix-org/setup-python-poetry@v1
-        with:
-          python-version: "3.x"
-          extras: "all"
-      - run: |
-          poetry remove twisted
-          poetry add --extras tls git+https://github.com/twisted/twisted.git#trunk
-          poetry install --no-interaction --extras "all test"
-      - name: Remove warn_unused_ignores from mypy config
-        run: sed '/warn_unused_ignores = True/d' -i mypy.ini
-      - run: poetry run mypy
+      - uses: actions/setup-python@v2
+      - run: .ci/patch_for_twisted_trunk.sh
+      - run: pip install tox
+      - run: tox -e mypy
 
   trial:
     runs-on: ubuntu-latest
@@ -34,15 +23,14 @@ jobs:
     steps:
       - uses: actions/checkout@v2
       - run: sudo apt-get -qq install xmlsec1
-      - uses: matrix-org/setup-python-poetry@v1
+      - uses: actions/setup-python@v2
         with:
-          python-version: "3.x"
-          extras: "all test"
-      - run: |
-          poetry remove twisted
-          poetry add --extras tls git+https://github.com/twisted/twisted.git#trunk
-          poetry install --no-interaction --extras "all test"
-      - run: poetry run trial --jobs 2 tests
+          python-version: 3.7
+      - run: .ci/patch_for_twisted_trunk.sh
+      - run: pip install tox
+      - run: tox -e py
+        env:
+          TRIAL_FLAGS: "--jobs=2"
 
       - name: Dump logs
         # Logs are most useful when the command fails, always include them.
@@ -67,23 +55,11 @@ jobs:
     steps:
       - uses: actions/checkout@v2
       - name: Patch dependencies
-        # Note: The poetry commands want to create a virtualenv in /src/.venv/,
-        #       but the sytest-synapse container expects it to be in /venv/.
-        #       We symlink it before running poetry so that poetry actually
-        #       ends up installing to `/venv`.
-        run: |
-          ln -s -T /venv /src/.venv
-          poetry remove twisted
-          poetry add --extras tls git+https://github.com/twisted/twisted.git#trunk
-          poetry install --no-interaction --extras "all test"
+        run: .ci/patch_for_twisted_trunk.sh
         working-directory: /src
       - name: Run SyTest
         run: /bootstrap.sh synapse
         working-directory: /src
-        env:
-          # Use offline mode to avoid reinstalling the pinned version of
-          # twisted.
-          OFFLINE: 1
       - name: Summarise results.tap
         if: ${{ always() }}
         run: /sytest/scripts/tap_to_gha.pl /logs/results.tap

.gitignore

@@ -15,9 +15,6 @@ _trial_temp*/
 .DS_Store
 __pycache__/
 
-# We do want the poetry lockfile.
-!poetry.lock
-
 # stuff that is likely to exist when you run a server locally
 /*.db
 /*.log
@@ -33,9 +30,6 @@ __pycache__/
 /media_store/
 /uploads
 
-# For direnv users
-/.envrc
-
 # IDEs
 /.idea/
 /.ropeproject/

MANIFEST.in

@@ -0,0 +1,54 @@
+include LICENSE
+include VERSION
+include *.rst
+include *.md
+include demo/README
+include demo/demo.tls.dh
+include demo/*.py
+include demo/*.sh
+
+include synapse/py.typed
+recursive-include synapse/storage *.sql
+recursive-include synapse/storage *.sql.postgres
+recursive-include synapse/storage *.sql.sqlite
+recursive-include synapse/storage *.py
+recursive-include synapse/storage *.txt
+recursive-include synapse/storage *.md
+
+recursive-include docs *
+recursive-include scripts-dev *
+recursive-include synapse *.pyi
+recursive-include tests *.py
+recursive-include tests *.pem
+recursive-include tests *.p8
+recursive-include tests *.crt
+recursive-include tests *.key
+
+recursive-include synapse/res *
+recursive-include synapse/static *.css
+recursive-include synapse/static *.gif
+recursive-include synapse/static *.html
+recursive-include synapse/static *.js
+
+exclude .codecov.yml
+exclude .coveragerc
+exclude .dockerignore
+exclude .editorconfig
+exclude Dockerfile
+exclude mypy.ini
+exclude sytest-blacklist
+exclude test_postgresql.sh
+
+include book.toml
+include pyproject.toml
+recursive-include changelog.d *
+
+include .flake8
+prune .circleci
+prune .github
+prune .ci
+prune contrib
+prune debian
+prune demo/etc
+prune docker
+prune stubs

README.rst

@@ -55,7 +55,7 @@ solutions. The hope is for Matrix to act as the building blocks for a new
 generation of fully open and interoperable messaging and VoIP apps for the
 internet.
 
-Synapse is a Matrix "homeserver" implementation developed by the matrix.org core
+Synapse is a Matrix "homeserver" implementation developed by the matrix.org core 
 team, written in Python 3/Twisted.
 
 In Matrix, every user runs one or more Matrix clients, which connect through to
@@ -293,42 +293,39 @@ directory of your choice::
     git clone https://github.com/matrix-org/synapse.git
     cd synapse
 
-Synapse has a number of external dependencies. We maintain a fixed development
-environment using `Poetry <https://python-poetry.org/>`_. First, install poetry. We recommend::
+Synapse has a number of external dependencies, that are easiest
+to install using pip and a virtualenv::
 
-    pip install --user pipx
-    pipx install poetry
-
-as described `here <https://python-poetry.org/docs/#installing-with-pipx>`_.
-(See `poetry's installation docs <https://python-poetry.org/docs/#installation>`_
-for other installation methods.) Then ask poetry to create a virtual environment
-from the project and install Synapse's dependencies::
-
-    poetry install --extras "all test"
+    python3 -m venv ./env
+    source ./env/bin/activate
+    pip install -e ".[all,dev]"
 
 This will run a process of downloading and installing all the needed
-dependencies into a virtual env.
+dependencies into a virtual env. If any dependencies fail to install,
+try installing the failing modules individually::
 
-We recommend using the demo which starts 3 federated instances running on ports `8080` - `8082`::
+    pip install -e "module-name"
 
-    poetry run ./demo/start.sh
+We recommend using the demo which starts 3 federated instances running on ports `8080` - `8082`
 
-(to stop, you can use ``poetry run ./demo/stop.sh``)
+    ./demo/start.sh
 
-See the `demo documentation <https://matrix-org.github.io/synapse/develop/development/demo.html>`_
+(to stop, you can use `./demo/stop.sh`)
+
+See the [demo documentation](https://matrix-org.github.io/synapse/develop/development/demo.html)
 for more information.
 
 If you just want to start a single instance of the app and run it directly::
 
     # Create the homeserver.yaml config once
-    poetry run synapse_homeserver \
+    python -m synapse.app.homeserver \
       --server-name my.domain.name \
       --config-path homeserver.yaml \
       --generate-config \
       --report-stats=[yes|no]
 
     # Start the app
-    poetry run synapse_homeserver --config-path homeserver.yaml
+    python -m synapse.app.homeserver --config-path homeserver.yaml
 
 
 Running the unit tests
@@ -337,7 +334,7 @@ Running the unit tests
 After getting up and running, you may wish to run Synapse's unit tests to
 check that everything is installed correctly::
 
-    poetry run trial tests
+    trial tests
 
 This should end with a 'PASSED' result (note that exact numbers will
 differ)::

book.toml

@@ -33,7 +33,7 @@ site-url = "/synapse/"
 additional-css = [
     "docs/website_files/table-of-contents.css",
     "docs/website_files/remove-nav-buttons.css",
-    "docs/website_files/indent-section-headers.css",
+    "docs/website_files/section-headers.css",
 ]
 additional-js = ["docs/website_files/table-of-contents.js"]
 theme = "docs/website_files/theme"

changelog.d/11700.removal

@@ -0,0 +1 @@
+Remove workaround introduced in Synapse 1.50.0 for Mjolnir compatibility. Breaks compatibility with Mjolnir 1.3.1 and earlier.

changelog.d/11915.misc

@@ -0,0 +1 @@
+Simplify the `ApplicationService` class' set of public methods related to interest checking.

changelog.d/11998.doc

@@ -0,0 +1 @@
+Fix complexity checking config example in [Resource Constrained Devices](https://matrix-org.github.io/synapse/v1.54/other/running_synapse_on_single_board_computers.html) docs page.

changelog.d/12028.feature

@@ -0,0 +1 @@
+Add third-party rules rules callbacks `check_can_shutdown_room` and `check_can_deactivate_user`.

changelog.d/12042.misc

@@ -0,0 +1 @@
+Correct type hints for txredis.

changelog.d/12090.bugfix

@@ -0,0 +1 @@
+Use the proper serialization format for bundled thread aggregations. The bug has existed since Synapse v1.48.0.

changelog.d/12101.misc

@@ -0,0 +1 @@
+Limit the size of `aggregation_key` on annotations.

changelog.d/12108.misc

@@ -0,0 +1 @@
+Add type hints to `tests/rest/client`.

changelog.d/12113.bugfix

@@ -0,0 +1 @@
+Fix a long-standing bug when redacting events with relations.

changelog.d/12118.misc

@@ -0,0 +1 @@
+Move scripts to Synapse package and expose as setuptools entry points.

changelog.d/12121.bugfix

@@ -0,0 +1 @@
+Fix a long-standing bug when redacting events with relations.

changelog.d/12128.misc

@@ -0,0 +1 @@
+Fix data validation to compare to lists, not sequences.

changelog.d/12130.bugfix

@@ -0,0 +1 @@
+Fix a long-standing bug when redacting events with relations.

changelog.d/12131.misc

@@ -0,0 +1 @@
+Fix CI not attaching source distributions and wheels to the GitHub releases.

changelog.d/12132.feature

@@ -0,0 +1 @@
+Improve performance of logging in for large accounts.

changelog.d/12135.feature

@@ -0,0 +1 @@
+Add experimental env var `SYNAPSE_ASYNC_IO_REACTOR` that causes Synapse to use the asyncio reactor for Twisted.

changelog.d/12136.misc

@@ -0,0 +1 @@
+Remove unused mocks from `test_typing`.

changelog.d/12137.misc

@@ -0,0 +1 @@
+Give `scripts-dev` scripts suffixes for neater CI config.

changelog.d/12138.removal

@@ -0,0 +1 @@
+Remove backwards compatibilty with pagination tokens from the `/relations` and `/aggregations` endpoints generated from Synapse < v1.52.0.

changelog.d/12140.misc

@@ -0,0 +1 @@
+Move `synctl` into `synapse._scripts` and expose as an entry point.

changelog.d/12142.misc

@@ -0,0 +1 @@
+Move the snapcraft configuration file to `contrib`.

changelog.d/12143.doc

@@ -0,0 +1 @@
+Improve documentation for demo scripts.

changelog.d/12144.misc

@@ -0,0 +1 @@
+Enable [MSC3030](https://github.com/matrix-org/matrix-doc/pull/3030) Complement tests in CI.

changelog.d/12145.misc

@@ -0,0 +1 @@
+Enable [MSC2716](https://github.com/matrix-org/matrix-doc/pull/2716) Complement tests in CI.

changelog.d/12146.misc

@@ -0,0 +1 @@
+Add type hints to `tests/rest`.

changelog.d/12149.misc

@@ -0,0 +1 @@
+Add test for `ObservableDeferred`'s cancellation behaviour.

changelog.d/12150.misc

@@ -0,0 +1 @@
+Use `ParamSpec` in type hints for `synapse.logging.context`.

changelog.d/12151.feature

@@ -0,0 +1 @@
+Support the stable identifiers from [MSC3440](https://github.com/matrix-org/matrix-doc/pull/3440): threads.

changelog.d/12152.misc

@@ -0,0 +1 @@
+Prune unused jobs from `tox` config.

changelog.d/12153.misc

@@ -0,0 +1 @@
+Move CI checks out of tox, to facilitate a move to using poetry.

changelog.d/12154.misc

@@ -0,0 +1 @@
+Avoid generating state groups for local out-of-band leaves.

changelog.d/12155.misc

@@ -0,0 +1 @@
+Avoid trying to calculate the state at outlier events.

changelog.d/12156.misc

@@ -0,0 +1 @@
+Fix some type annotations.

changelog.d/12157.bugfix

@@ -0,0 +1 @@
+Fix a bug introduced in #4864 whereby background updates are never run with the default background batch size.

changelog.d/12159.misc

@@ -0,0 +1 @@
+Add type hints for `ObservableDeferred` attributes.

changelog.d/12161.misc

@@ -0,0 +1 @@
+Use a prebuilt Action for the `tests-done` CI job.

changelog.d/12163.misc

@@ -0,0 +1 @@
+Reduce number of DB queries made during processing of `/sync`.

changelog.d/12173.misc

@@ -0,0 +1 @@
+Avoid trying to calculate the state at outlier events.

changelog.d/12175.bugfix

@@ -0,0 +1 @@
+Fix a bug where non-standard information was returned from the `/hierarchy` API. Introduced in Synapse v1.41.0.

changelog.d/12179.doc

@@ -0,0 +1 @@
+Updates to the Room DAG concepts development document.

changelog.d/12182.misc

@@ -0,0 +1 @@
+Retry HTTP replication failures, this should prevent 502's when restarting stateful workers (main, event persisters, stream writers). Contributed by Nick @ Beeper.

changelog.d/12187.misc

@@ -0,0 +1 @@
+Remove unused variables.

changelog.d/12189.bugfix

@@ -0,0 +1 @@
+Fix a long-standing bug when redacting events with relations.

changelog.d/12192.misc

@@ -0,0 +1 @@
+Rename `HomeServer.get_tcp_replication` to `get_replication_command_handler`.

changelog.d/12197.misc

@@ -0,0 +1 @@
+Remove some dead code.

changelog.d/12273.bugfix

@@ -1 +0,0 @@
-Fix a bug introduced in Synapse v1.48.0 where latest thread reply provided failed to include the proper bundled aggregations.

changelog.d/12356.misc

@@ -1 +0,0 @@
-Fix scripts-dev to pass typechecking.

changelog.d/12406.feature

@@ -1 +0,0 @@
-Add a module API to allow modules to change actions for existing push rules of local users.

changelog.d/12480.misc

@@ -1 +0,0 @@
-Use supervisord to supervise Postgres and Caddy in the Complement image to reduce restart time.

changelog.d/12485.misc

@@ -1 +0,0 @@
-Add some type hints to datastore.

changelog.d/12505.misc

@@ -1 +0,0 @@
-Use `make_awaitable` instead of `defer.succeed` for return values of mocks in tests.

changelog.d/12526.feature

@@ -1 +0,0 @@
-Add new `enable_registration_token_3pid_bypass` configuration option to allow registrations via token as an alternative to verifying a 3pid.

changelog.d/12531.misc

@@ -1 +0,0 @@
-Remove unused `# type: ignore`s.

changelog.d/12541.docker

@@ -1 +0,0 @@
-Explicitly opt-in to using [BuildKit-specific features](https://github.com/moby/buildkit/blob/master/frontend/dockerfile/docs/syntax.md) in the Dockerfile. This fixes issues with building images in some GitLab CI environments.

changelog.d/12544.bugfix

@@ -1 +0,0 @@
-Fix a bug where attempting to send a large amount of read receipts to an application service all at once would result in duplicate content and abnormally high memory usage. Contributed by Brad & Nick @ Beeper.

changelog.d/12556.misc

@@ -1 +0,0 @@
-Release script: confirm the commit to be tagged before tagging.

changelog.d/12564.misc

@@ -1 +0,0 @@
-Consistently check if an object is a `frozendict`.

changelog.d/12570.bugfix

@@ -1 +0,0 @@
-Fix a bug introduced in Synapse 1.57 which could cause `Failed to calculate hosts in room` errors to be logged for outbound federation.

changelog.d/12576.misc

@@ -1 +0,0 @@
-Allow unused `#type: ignore` comments in bleeding edge CI jobs.

changelog.d/12579.doc

@@ -1 +0,0 @@
-Add missing linebreak to pipx install instructions.

changelog.d/12580.bugfix

@@ -1 +0,0 @@
-Fix a long standing bug where status codes would almost always get logged as 200!, irrespective of the actual status code, when clients disconnect before a request has finished processing.

changelog.d/12581.misc

@@ -1 +0,0 @@
-Improve docstrings for the receipts store.

changelog.d/12582.misc

@@ -1 +0,0 @@
-Use constants for read-receipts in tests.

changelog.d/12587.misc

@@ -1 +0,0 @@
-Add `@cancellable` decorator, for use on endpoint methods that can be cancelled when clients disconnect.

changelog.d/12589.misc

@@ -1 +0,0 @@
-Remove special-case for `twisted` logger from default log config.

changelog.d/12594.bugfix

@@ -1 +0,0 @@
-Fix race when persisting an event and deleting a room that could lead to outbound federation breaking.

changelog.d/12596.removal

@@ -1 +0,0 @@
-Remove unstable identifiers from [MSC3069](https://github.com/matrix-org/matrix-doc/pull/3069).

changelog.d/12597.removal

@@ -1,2 +0,0 @@
-Remove the unspecified `m.login.jwt` login type and the unstable `uk.half-shot.msc2778.login.application_service` from 
-[MSC2778](https://github.com/matrix-org/matrix-doc/pull/2778).

changelog.d/12608.misc

@@ -1 +0,0 @@
-Remove redundant lines of config from `mypy.ini`.

changelog.d/12612.bugfix

@@ -1 +0,0 @@
-Fix a typo in the announcement text generated by the Synapse release development script.

changelog.d/12613.removal

@@ -1 +0,0 @@
-Synapse now requires at least Python 3.7.1 (up from 3.7.0), for compatibility with the latest Twisted trunk.

changelog.d/12614.misc

@@ -1 +0,0 @@
-Add extra debug logging to federation sender.

changelog.d/12620.misc

@@ -1 +0,0 @@
-Add a consistency check on events which we read from the database.

changelog.d/12624.misc

@@ -1 +0,0 @@
-Remove use of constantly library and switch to enums for EventRedactBehaviour. Contributed by @andrewdoh.

changelog.d/12627.doc

@@ -1 +0,0 @@
-Fixes to the formatting of README.rst.

contrib/jitsimeetbridge/jitsimeetbridge.py

@@ -193,15 +193,12 @@ class TrivialXmppClient:
         time.sleep(7)
         print("SSRC spammer started")
         while self.running:
-            ssrcMsg = (
-                "<presence to='%(tojid)s' xmlns='jabber:client'><x xmlns='http://jabber.org/protocol/muc'/><c xmlns='http://jabber.org/protocol/caps' hash='sha-1' node='http://jitsi.org/jitsimeet' ver='0WkSdhFnAUxrz4ImQQLdB80GFlE='/><nick xmlns='http://jabber.org/protocol/nick'>%(nick)s</nick><stats xmlns='http://jitsi.org/jitmeet/stats'><stat name='bitrate_download' value='175'/><stat name='bitrate_upload' value='176'/><stat name='packetLoss_total' value='0'/><stat name='packetLoss_download' value='0'/><stat name='packetLoss_upload' value='0'/></stats><media xmlns='http://estos.de/ns/mjs'><source type='audio' ssrc='%(assrc)s' direction='sendre'/><source type='video' ssrc='%(vssrc)s' direction='sendre'/></media></presence>"
-                % {
-                    "tojid": "%s@%s/%s" % (ROOMNAME, ROOMDOMAIN, self.shortJid),
-                    "nick": self.userId,
-                    "assrc": self.ssrcs["audio"],
-                    "vssrc": self.ssrcs["video"],
-                }
-            )
+            ssrcMsg = "<presence to='%(tojid)s' xmlns='jabber:client'><x xmlns='http://jabber.org/protocol/muc'/><c xmlns='http://jabber.org/protocol/caps' hash='sha-1' node='http://jitsi.org/jitsimeet' ver='0WkSdhFnAUxrz4ImQQLdB80GFlE='/><nick xmlns='http://jabber.org/protocol/nick'>%(nick)s</nick><stats xmlns='http://jitsi.org/jitmeet/stats'><stat name='bitrate_download' value='175'/><stat name='bitrate_upload' value='176'/><stat name='packetLoss_total' value='0'/><stat name='packetLoss_download' value='0'/><stat name='packetLoss_upload' value='0'/></stats><media xmlns='http://estos.de/ns/mjs'><source type='audio' ssrc='%(assrc)s' direction='sendre'/><source type='video' ssrc='%(vssrc)s' direction='sendre'/></media></presence>" % {
+                "tojid": "%s@%s/%s" % (ROOMNAME, ROOMDOMAIN, self.shortJid),
+                "nick": self.userId,
+                "assrc": self.ssrcs["audio"],
+                "vssrc": self.ssrcs["video"],
+            }
             res = self.sendIq(ssrcMsg)
             print("reply from ssrc announce: ", res)
             time.sleep(10)

debian/build_virtualenv

@@ -30,19 +30,9 @@ case $(dpkg-architecture -q DEB_HOST_ARCH) in
         ;;
 esac
 
-# Manually install Poetry and export a pip-compatible `requirements.txt`
-# We need a Poetry pre-release as the export command is buggy in < 1.2
-TEMP_VENV="$(mktemp -d)"
-python3 -m venv "$TEMP_VENV"
-source "$TEMP_VENV/bin/activate"
-pip install -U pip
-pip install poetry==1.2.0b1
-poetry export --extras all --extras test -o exported_requirements.txt
-deactivate
-rm -rf "$TEMP_VENV"
+# Use --builtin-venv to use the better `venv` module from CPython 3.4+ rather
+# than the 2/3 compatible `virtualenv`.
 
-# Use --no-deps to only install pinned versions in exported_requirements.txt,
-# and to avoid https://github.com/pypa/pip/issues/9644
 dh_virtualenv \
     --install-suffix "matrix-synapse" \
     --builtin-venv \
@@ -51,11 +41,9 @@ dh_virtualenv \
     --preinstall="lxml" \
     --preinstall="mock" \
     --preinstall="wheel" \
-    --extra-pip-arg="--no-deps" \
     --extra-pip-arg="--no-cache-dir" \
     --extra-pip-arg="--compile" \
-    --extras="all,systemd,test" \
-    --requirements="exported_requirements.txt"
+    --extras="all,systemd,test"
 
 PACKAGE_BUILD_DIR="debian/matrix-synapse-py3"
 VIRTUALENV_DIR="${PACKAGE_BUILD_DIR}${DH_VIRTUALENV_INSTALL_ROOT}/matrix-synapse"

debian/changelog

@@ -1,76 +1,3 @@
-matrix-synapse-py3 (1.58.0) stable; urgency=medium
-
-  * New Synapse release 1.58.0.
-
- -- Synapse Packaging team <packages@matrix.org>  Tue, 03 May 2022 10:52:58 +0100
-
-matrix-synapse-py3 (1.58.0~rc2) stable; urgency=medium
-
-  * New Synapse release 1.58.0rc2.
-
- -- Synapse Packaging team <packages@matrix.org>  Tue, 26 Apr 2022 17:14:56 +0100
-
-matrix-synapse-py3 (1.58.0~rc1) stable; urgency=medium
-
-  * Use poetry to manage the bundled virtualenv included with this package.
-  * New Synapse release 1.58.0rc1.
-
- -- Synapse Packaging team <packages@matrix.org>  Tue, 26 Apr 2022 11:15:20 +0100
-
-matrix-synapse-py3 (1.57.1) stable; urgency=medium
-
-  * New synapse release 1.57.1.
-
- -- Synapse Packaging team <packages@matrix.org>  Wed, 20 Apr 2022 15:27:21 +0100
-
-matrix-synapse-py3 (1.57.0) stable; urgency=medium
-
-  * New synapse release 1.57.0.
-
- -- Synapse Packaging team <packages@matrix.org>  Tue, 19 Apr 2022 10:58:42 +0100
-
-matrix-synapse-py3 (1.57.0~rc1) stable; urgency=medium
-
-  * New synapse release 1.57.0~rc1.
-
- -- Synapse Packaging team <packages@matrix.org>  Tue, 12 Apr 2022 13:36:25 +0100
-
-matrix-synapse-py3 (1.56.0) stable; urgency=medium
-
-  * New synapse release 1.56.0.
-
- -- Synapse Packaging team <packages@matrix.org>  Tue, 05 Apr 2022 12:38:39 +0100
-
-matrix-synapse-py3 (1.56.0~rc1) stable; urgency=medium
-
-  * New synapse release 1.56.0~rc1.
-
- -- Synapse Packaging team <packages@matrix.org>  Tue, 29 Mar 2022 10:40:50 +0100
-
-matrix-synapse-py3 (1.55.2) stable; urgency=medium
-
-  * New synapse release 1.55.2.
-
- -- Synapse Packaging team <packages@matrix.org>  Thu, 24 Mar 2022 19:07:11 +0000
-
-matrix-synapse-py3 (1.55.1) stable; urgency=medium
-
-  * New synapse release 1.55.1.
-
- -- Synapse Packaging team <packages@matrix.org>  Thu, 24 Mar 2022 17:44:23 +0000
-
-matrix-synapse-py3 (1.55.0) stable; urgency=medium
-
-  * New synapse release 1.55.0.
-
- -- Synapse Packaging team <packages@matrix.org>  Tue, 22 Mar 2022 13:59:26 +0000
-
-matrix-synapse-py3 (1.55.0~rc1) stable; urgency=medium
-
-  * New synapse release 1.55.0~rc1.
-
- -- Synapse Packaging team <packages@matrix.org>  Tue, 15 Mar 2022 10:59:31 +0000
-
 matrix-synapse-py3 (1.54.0) stable; urgency=medium
 
   * New synapse release 1.54.0.

debian/clean

@@ -1 +0,0 @@
-exported_requirements.txt

demo/start.sh

@@ -38,7 +38,6 @@ for port in 8080 8081 8082; do
             printf '\n\n# Customisation made by demo/start.sh\n\n'
             echo "public_baseurl: http://localhost:$port/"
             echo 'enable_registration: true'
-            echo 'enable_registration_without_verification: true'
             echo ''
 
 			# Warning, this heredoc depends on the interaction of tabs and spaces.

docker/Dockerfile

@@ -1,4 +1,3 @@
-# syntax=docker/dockerfile:1
 # Dockerfile to build the matrixdotorg/synapse docker images.
 #
 # Note that it uses features which are only available in BuildKit - see
@@ -15,61 +14,20 @@
 #    DOCKER_BUILDKIT=1 docker build -f docker/Dockerfile --build-arg PYTHON_VERSION=3.10 .
 #
 
-# Irritatingly, there is no blessed guide on how to distribute an application with its
-# poetry-managed environment in a docker image. We have opted for
-# `poetry export | pip install -r /dev/stdin`, but there are known bugs in
-# in `poetry export` whose fixes (scheduled for poetry 1.2) have yet to be released.
-# In case we get bitten by those bugs in the future, the recommendations here might
-# be useful:
-#     https://github.com/python-poetry/poetry/discussions/1879#discussioncomment-216865
-#     https://stackoverflow.com/questions/53835198/integrating-python-poetry-with-docker?answertab=scoredesc
-
-
-
 ARG PYTHON_VERSION=3.9
 
 ###
-### Stage 0: generate requirements.txt
-###
-FROM docker.io/python:${PYTHON_VERSION}-slim as requirements
-
-# RUN --mount is specific to buildkit and is documented at
-# https://github.com/moby/buildkit/blob/master/frontend/dockerfile/docs/syntax.md#build-mounts-run---mount.
-# Here we use it to set up a cache for apt (and below for pip), to improve
-# rebuild speeds on slow connections.
-RUN \
-   --mount=type=cache,target=/var/cache/apt,sharing=locked \
-   --mount=type=cache,target=/var/lib/apt,sharing=locked \
- apt-get update && apt-get install -y git \
-    && rm -rf /var/lib/apt/lists/*
-
-# We install poetry in its own build stage to avoid its dependencies conflicting with
-# synapse's dependencies.
-# We use a specific commit from poetry's master branch instead of our usual 1.1.12,
-# to incorporate fixes to some bugs in `poetry export`. This commit corresponds to
-#    https://github.com/python-poetry/poetry/pull/5156 and
-#    https://github.com/python-poetry/poetry/issues/5141 ;
-# without it, we generate a requirements.txt with incorrect environment markers,
-# which causes necessary packages to be omitted when we `pip install`.
-#
-# NB: In poetry 1.2 `poetry export` will be moved into a plugin; we'll need to also
-# pip install poetry-plugin-export (https://github.com/python-poetry/poetry-plugin-export).
-RUN --mount=type=cache,target=/root/.cache/pip \
-  pip install --user git+https://github.com/python-poetry/poetry.git@fb13b3a676f476177f7937ffa480ee5cff9a90a5
-
-WORKDIR /synapse
-
-# Copy just what we need to run `poetry export`...
-COPY pyproject.toml poetry.lock /synapse/
-
-RUN /root/.local/bin/poetry export --extras all -o /synapse/requirements.txt
-
-###
-### Stage 1: builder
+### Stage 0: builder
 ###
 FROM docker.io/python:${PYTHON_VERSION}-slim as builder
 
 # install the OS build deps
+#
+# RUN --mount is specific to buildkit and is documented at
+# https://github.com/moby/buildkit/blob/master/frontend/dockerfile/docs/syntax.md#build-mounts-run---mount.
+# Here we use it to set up a cache for apt, to improve rebuild speeds on
+# slow connections.
+#
 RUN \
    --mount=type=cache,target=/var/cache/apt,sharing=locked \
    --mount=type=cache,target=/var/lib/apt,sharing=locked \
@@ -87,25 +45,30 @@ RUN \
     zlib1g-dev \
     && rm -rf /var/lib/apt/lists/*
 
+# Copy just what we need to pip install
+COPY MANIFEST.in README.rst setup.py /synapse/
+COPY synapse/__init__.py /synapse/synapse/__init__.py
+COPY synapse/python_dependencies.py /synapse/synapse/python_dependencies.py
+
 # To speed up rebuilds, install all of the dependencies before we copy over
-# the whole synapse project, so that this layer in the Docker cache can be
+# the whole synapse project so that we this layer in the Docker cache can be
 # used while you develop on the source
 #
-# This is aiming at installing the `[tool.poetry.depdendencies]` from pyproject.toml.
-COPY --from=requirements /synapse/requirements.txt /synapse/
+# This is aiming at installing the `install_requires` and `extras_require` from `setup.py`
 RUN --mount=type=cache,target=/root/.cache/pip \
-  pip install --prefix="/install" --no-deps --no-warn-script-location -r /synapse/requirements.txt
+  pip install --prefix="/install" --no-warn-script-location \
+    /synapse[all]
 
-# Copy over the rest of the synapse source code.
+# Copy over the rest of the project
 COPY synapse /synapse/synapse/
-# ... and what we need to `pip install`.
-COPY pyproject.toml README.rst /synapse/
 
-# Install the synapse package itself.
+# Install the synapse package itself and all of its children packages.
+#
+# This is aiming at installing only the `packages=find_packages(...)` from `setup.py
 RUN pip install --prefix="/install" --no-deps --no-warn-script-location /synapse
 
 ###
-### Stage 2: runtime
+### Stage 1: runtime
 ###
 
 FROM docker.io/python:${PYTHON_VERSION}-slim

docker/Dockerfile-pgtests

@@ -0,0 +1,30 @@
+# Use the Sytest image that comes with a lot of the build dependencies
+# pre-installed
+FROM matrixdotorg/sytest:focal
+
+# The Sytest image doesn't come with python, so install that
+RUN apt-get update && apt-get -qq install -y python3 python3-dev python3-pip
+
+# We need tox to run the tests in run_pg_tests.sh
+RUN python3 -m pip install tox
+
+# Initialise the db
+RUN su -c '/usr/lib/postgresql/10/bin/initdb -D /var/lib/postgresql/data -E "UTF-8" --lc-collate="C.UTF-8" --lc-ctype="C.UTF-8" --username=postgres' postgres
+
+# Add a user with our UID and GID so that files get created on the host owned
+# by us, not root.
+ARG UID
+ARG GID
+RUN groupadd --gid $GID user
+RUN useradd --uid $UID --gid $GID --groups sudo --no-create-home user
+
+# Ensure we can start postgres by sudo-ing as the postgres user.
+RUN apt-get update && apt-get -qq install -y sudo
+RUN echo "user ALL=(ALL) NOPASSWD: ALL" >> /etc/sudoers
+
+ADD run_pg_tests.sh /run_pg_tests.sh
+# Use the "exec form" of ENTRYPOINT (https://docs.docker.com/engine/reference/builder/#entrypoint)
+# so that we can `docker run` this container and pass arguments to pg_tests.sh
+ENTRYPOINT ["/run_pg_tests.sh"]
+
+USER user

docker/Dockerfile-workers

@@ -2,36 +2,25 @@
 FROM matrixdotorg/synapse
 
 # Install deps
-RUN \
-   --mount=type=cache,target=/var/cache/apt,sharing=locked \
-   --mount=type=cache,target=/var/lib/apt,sharing=locked \
-  apt-get update && \
-  DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
-     redis-server nginx-light
+RUN apt-get update
+RUN apt-get install -y supervisor redis nginx
 
-# Install supervisord with pip instead of apt, to avoid installing a second
-# copy of python.
-RUN --mount=type=cache,target=/root/.cache/pip \
-    pip install supervisor~=4.2
-
-# Disable the default nginx sites
+# Remove the default nginx sites
 RUN rm /etc/nginx/sites-enabled/default
 
 # Copy Synapse worker, nginx and supervisord configuration template files
 COPY ./docker/conf-workers/* /conf/
 
-# Copy a script to prefix log lines with the supervisor program name
-COPY ./docker/prefix-log /usr/local/bin/
-
 # Expose nginx listener port
 EXPOSE 8080/tcp
 
+# Volume for user-editable config files, logs etc.
+VOLUME ["/data"]
+
 # A script to read environment variables and create the necessary
 # files to run the desired worker configuration. Will start supervisord.
 COPY ./docker/configure_workers_and_start.py /configure_workers_and_start.py
 ENTRYPOINT ["/configure_workers_and_start.py"]
 
-# Replace the healthcheck with one which checks *all* the workers. The script
-# is generated by configure_workers_and_start.py.
 HEALTHCHECK --start-period=5s --interval=15s --timeout=5s \
     CMD /bin/sh /healthcheck.sh

docker/README-testing.md

@@ -10,10 +10,10 @@ Note that running Synapse's unit tests from within the docker image is not suppo
 
 ## Testing with SQLite and single-process Synapse
 
-> Note that `scripts-dev/complement.sh` is a script that will automatically build
+> Note that `scripts-dev/complement.sh` is a script that will automatically build 
 > and run an SQLite-based, single-process of Synapse against Complement.
 
-The instructions below will set up Complement testing for a single-process,
+The instructions below will set up Complement testing for a single-process, 
 SQLite-based Synapse deployment.
 
 Start by building the base Synapse docker image. If you wish to run tests with the latest
@@ -26,22 +26,23 @@ docker build -t matrixdotorg/synapse -f docker/Dockerfile .
 
 This will build an image with the tag `matrixdotorg/synapse`.
 
-Next, build the Synapse image for Complement.
+Next, build the Synapse image for Complement. You will need a local checkout 
+of Complement. Change to the root of your Complement checkout and run:
 
 ```sh
-docker build -t complement-synapse -f "docker/complement/Dockerfile" docker/complement
+docker build -t complement-synapse -f "dockerfiles/Synapse.Dockerfile" dockerfiles
 ```
 
-This will build an image with the tag `complement-synapse`, which can be handed to
-Complement for testing via the `COMPLEMENT_BASE_IMAGE` environment variable. Refer to
-[Complement's documentation](https://github.com/matrix-org/complement/#running) for
+This will build an image with the tag `complement-synapse`, which can be handed to 
+Complement for testing via the `COMPLEMENT_BASE_IMAGE` environment variable. Refer to 
+[Complement's documentation](https://github.com/matrix-org/complement/#running) for 
 how to run the tests, as well as the various available command line flags.
 
 ## Testing with PostgreSQL and single or multi-process Synapse
 
-The above docker image only supports running Synapse with SQLite and in a
-single-process topology. The following instructions are used to build a Synapse image for
-Complement that supports either single or multi-process topology with a PostgreSQL
+The above docker image only supports running Synapse with SQLite and in a 
+single-process topology. The following instructions are used to build a Synapse image for 
+Complement that supports either single or multi-process topology with a PostgreSQL 
 database backend.
 
 As with the single-process image, build the base Synapse docker image. If you wish to run
@@ -54,7 +55,7 @@ docker build -t matrixdotorg/synapse -f docker/Dockerfile .
 
 This will build an image with the tag `matrixdotorg/synapse`.
 
-Next, we build a new image with worker support based on `matrixdotorg/synapse:latest`.
+Next, we build a new image with worker support based on `matrixdotorg/synapse:latest`. 
 Again, from the root of the repository:
 
 ```sh
@@ -63,20 +64,21 @@ docker build -t matrixdotorg/synapse-workers -f docker/Dockerfile-workers .
 
 This will build an image with the tag` matrixdotorg/synapse-workers`.
 
-It's worth noting at this point that this image is fully functional, and
-can be used for testing against locally. See instructions for using the container
+It's worth noting at this point that this image is fully functional, and 
+can be used for testing against locally. See instructions for using the container 
 under
 [Running the Dockerfile-worker image standalone](#running-the-dockerfile-worker-image-standalone)
 below.
 
 Finally, build the Synapse image for Complement, which is based on
-`matrixdotorg/synapse-workers`.
+`matrixdotorg/synapse-workers`. You will need a local checkout of Complement. Change to
+the root of your Complement checkout and run:
 
 ```sh
-docker build -t matrixdotorg/complement-synapse-workers -f docker/complement/SynapseWorkers.Dockerfile docker/complement
+docker build -t matrixdotorg/complement-synapse-workers -f dockerfiles/SynapseWorkers.Dockerfile dockerfiles
 ```
 
-This will build an image with the tag `complement-synapse-workers`, which can be handed to
+This will build an image with the tag `complement-synapse`, which can be handed to
 Complement for testing via the `COMPLEMENT_BASE_IMAGE` environment variable. Refer to
 [Complement's documentation](https://github.com/matrix-org/complement/#running) for
 how to run the tests, as well as the various available command line flags.
@@ -89,10 +91,10 @@ bundling all necessary components together for a workerised homeserver instance.
 
 This includes any desired Synapse worker processes, a nginx to route traffic accordingly,
 a redis for worker communication and a supervisord instance to start up and monitor all
-processes. You will need to provide your own postgres container to connect to, and TLS
+processes. You will need to provide your own postgres container to connect to, and TLS 
 is not handled by the container.
 
-Once you've built the image using the above instructions, you can run it. Be sure
+Once you've built the image using the above instructions, you can run it. Be sure 
 you've set up a volume according to the [usual Synapse docker instructions](README.md).
 Then run something along the lines of:
 
@@ -110,7 +112,7 @@ docker run -d --name synapse \
     matrixdotorg/synapse-workers
 ```
 
-...substituting `POSTGRES*` variables for those that match a postgres host you have
+...substituting `POSTGRES*` variables for those that match a postgres host you have 
 available (usually a running postgres docker container).
 
 The `SYNAPSE_WORKER_TYPES` environment variable is a comma-separated list of workers to
@@ -128,11 +130,11 @@ Otherwise, `SYNAPSE_WORKER_TYPES` can either be left empty or unset to spawn no
 (leaving only the main process). The container is configured to use redis-based worker
 mode.
 
-Logs for workers and the main process are logged to stdout and can be viewed with
-standard `docker logs` tooling. Worker logs contain their worker name
+Logs for workers and the main process are logged to stdout and can be viewed with 
+standard `docker logs` tooling. Worker logs contain their worker name 
 after the timestamp.
 
 Setting `SYNAPSE_WORKERS_WRITE_LOGS_TO_DISK=1` will cause worker logs to be written to
 `<data_dir>/logs/<worker_name>.log`. Logs are kept for 1 week and rotate every day at 00:
-00, according to the container's clock. Logging for the main process must still be
+00, according to the container's clock. Logging for the main process must still be 
 configured by modifying the homeserver's log config in your Synapse data volume.