Version picker added for v1.65 docs

.ci/complement_package.gotpl

@@ -27,10 +27,10 @@ which is under the Unlicense licence.
         {{- . -}}{{- "\n" -}}
     {{- end -}}
     {{- with .TestCases -}}
-        {{- /* Passing tests are first */ -}}
+        {{- /* Failing tests are first */ -}}
         {{- range . -}}
-            {{- if eq .Result "PASS" -}}
-                ::group::{{ "\033" }}[0;32m✅{{ " " }}{{- .Name -}}
+            {{- if and (ne .Result "PASS") (ne .Result "SKIP") -}}
+                ::group::{{ "\033" }}[0;31m❌{{ " " }}{{- .Name -}}
                 {{- "\033" -}}[0;37m ({{if $settings.ShowTestStatus}}{{.Result}}; {{end}}{{ .Duration -}}
                 {{- with .Coverage -}}
                     , coverage: {{ . }}%
@@ -47,6 +47,7 @@ which is under the Unlicense licence.
             {{- end -}}
         {{- end -}}
 
+
         {{- /* Then skipped tests are second */ -}}
         {{- range . -}}
             {{- if eq .Result "SKIP" -}}
@@ -67,10 +68,11 @@ which is under the Unlicense licence.
             {{- end -}}
         {{- end -}}
 
-        {{- /* and failing tests are last */ -}}
+
+        {{- /* Then passing tests are last */ -}}
         {{- range . -}}
-            {{- if and (ne .Result "PASS") (ne .Result "SKIP") -}}
-                ::group::{{ "\033" }}[0;31m❌{{ " " }}{{- .Name -}}
+            {{- if eq .Result "PASS" -}}
+                ::group::{{ "\033" }}[0;32m✅{{ " " }}{{- .Name -}}
                 {{- "\033" -}}[0;37m ({{if $settings.ShowTestStatus}}{{.Result}}; {{end}}{{ .Duration -}}
                 {{- with .Coverage -}}
                     , coverage: {{ . }}%

.ci/scripts/auditwheel_wrapper.py

@@ -1,141 +0,0 @@
-#!/usr/bin/env python
-# Copyright 2022 The Matrix.org Foundation C.I.C.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# Wraps `auditwheel repair` to first check if we're repairing a potentially abi3
-# compatible wheel, if so rename the wheel before repairing it.
-
-import argparse
-import os
-import subprocess
-from typing import Optional
-from zipfile import ZipFile
-
-from packaging.tags import Tag
-from packaging.utils import parse_wheel_filename
-from packaging.version import Version
-
-
-def check_is_abi3_compatible(wheel_file: str) -> None:
-    """Check the contents of the built wheel for any `.so` files that are *not*
-    abi3 compatible.
-    """
-
-    with ZipFile(wheel_file, "r") as wheel:
-        for file in wheel.namelist():
-            if not file.endswith(".so"):
-                continue
-
-            if not file.endswith(".abi3.so"):
-                raise Exception(f"Found non-abi3 lib: {file}")
-
-
-def cpython(wheel_file: str, name: str, version: Version, tag: Tag) -> str:
-    """Replaces the cpython wheel file with a ABI3 compatible wheel"""
-
-    if tag.abi == "abi3":
-        # Nothing to do.
-        return wheel_file
-
-    check_is_abi3_compatible(wheel_file)
-
-    # HACK: it seems that some older versions of pip will consider a wheel marked
-    # as macosx_11_0 as incompatible with Big Sur. I haven't done the full archaeology
-    # here; there are some clues in
-    #     https://github.com/pantsbuild/pants/pull/12857
-    #     https://github.com/pypa/pip/issues/9138
-    #     https://github.com/pypa/packaging/pull/319
-    # Empirically this seems to work, note that macOS 11 and 10.16 are the same,
-    # both versions are valid for backwards compatibility.
-    platform = tag.platform.replace("macosx_11_0", "macosx_10_16")
-    abi3_tag = Tag(tag.interpreter, "abi3", platform)
-
-    dirname = os.path.dirname(wheel_file)
-    new_wheel_file = os.path.join(
-        dirname,
-        f"{name}-{version}-{abi3_tag}.whl",
-    )
-
-    os.rename(wheel_file, new_wheel_file)
-
-    print("Renamed wheel to", new_wheel_file)
-
-    return new_wheel_file
-
-
-def main(wheel_file: str, dest_dir: str, archs: Optional[str]) -> None:
-    """Entry point"""
-
-    # Parse the wheel file name into its parts. Note that `parse_wheel_filename`
-    # normalizes the package name (i.e. it converts matrix_synapse ->
-    # matrix-synapse), which is not what we want.
-    _, version, build, tags = parse_wheel_filename(os.path.basename(wheel_file))
-    name = os.path.basename(wheel_file).split("-")[0]
-
-    if len(tags) != 1:
-        # We expect only a wheel file with only a single tag
-        raise Exception(f"Unexpectedly found multiple tags: {tags}")
-
-    tag = next(iter(tags))
-
-    if build:
-        # We don't use build tags in Synapse
-        raise Exception(f"Unexpected build tag: {build}")
-
-    # If the wheel is for cpython then convert it into an abi3 wheel.
-    if tag.interpreter.startswith("cp"):
-        wheel_file = cpython(wheel_file, name, version, tag)
-
-    # Finally, repair the wheel.
-    if archs is not None:
-        # If we are given archs then we are on macos and need to use
-        # `delocate-listdeps`.
-        subprocess.run(["delocate-listdeps", wheel_file], check=True)
-        subprocess.run(
-            ["delocate-wheel", "--require-archs", archs, "-w", dest_dir, wheel_file],
-            check=True,
-        )
-    else:
-        subprocess.run(["auditwheel", "repair", "-w", dest_dir, wheel_file], check=True)
-
-
-if __name__ == "__main__":
-    parser = argparse.ArgumentParser(description="Tag wheel as abi3 and repair it.")
-
-    parser.add_argument(
-        "--wheel-dir",
-        "-w",
-        metavar="WHEEL_DIR",
-        help="Directory to store delocated wheels",
-        required=True,
-    )
-
-    parser.add_argument(
-        "--require-archs",
-        metavar="archs",
-        default=None,
-    )
-
-    parser.add_argument(
-        "wheel_file",
-        metavar="WHEEL_FILE",
-    )
-
-    args = parser.parse_args()
-
-    wheel_file = args.wheel_file
-    wheel_dir = args.wheel_dir
-    archs = args.require_archs
-
-    main(wheel_file, wheel_dir, archs)

.ci/scripts/calculate_jobs.py

@@ -1,150 +0,0 @@
-#!/usr/bin/env python
-# Copyright 2022 The Matrix.org Foundation C.I.C.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# Calculate the trial jobs to run based on if we're in a PR or not.
-
-import json
-import os
-
-
-def set_output(key: str, value: str):
-    # See https://docs.github.com/en/actions/using-workflows/workflow-commands-for-github-actions#setting-an-output-parameter
-    with open(os.environ["GITHUB_OUTPUT"], "at") as f:
-        print(f"{key}={value}", file=f)
-
-
-IS_PR = os.environ["GITHUB_REF"].startswith("refs/pull/")
-
-# First calculate the various trial jobs.
-#
-# For each type of test we only run on Py3.7 on PRs
-
-trial_sqlite_tests = [
-    {
-        "python-version": "3.7",
-        "database": "sqlite",
-        "extras": "all",
-    }
-]
-
-if not IS_PR:
-    trial_sqlite_tests.extend(
-        {
-            "python-version": version,
-            "database": "sqlite",
-            "extras": "all",
-        }
-        for version in ("3.8", "3.9", "3.10", "3.11")
-    )
-
-
-trial_postgres_tests = [
-    {
-        "python-version": "3.7",
-        "database": "postgres",
-        "postgres-version": "11",
-        "extras": "all",
-    }
-]
-
-if not IS_PR:
-    trial_postgres_tests.append(
-        {
-            "python-version": "3.11",
-            "database": "postgres",
-            "postgres-version": "15",
-            "extras": "all",
-        }
-    )
-
-trial_no_extra_tests = [
-    {
-        "python-version": "3.7",
-        "database": "sqlite",
-        "extras": "",
-    }
-]
-
-print("::group::Calculated trial jobs")
-print(
-    json.dumps(
-        trial_sqlite_tests + trial_postgres_tests + trial_no_extra_tests, indent=4
-    )
-)
-print("::endgroup::")
-
-test_matrix = json.dumps(
-    trial_sqlite_tests + trial_postgres_tests + trial_no_extra_tests
-)
-set_output("trial_test_matrix", test_matrix)
-
-
-# First calculate the various sytest jobs.
-#
-# For each type of test we only run on focal on PRs
-
-
-sytest_tests = [
-    {
-        "sytest-tag": "focal",
-    },
-    {
-        "sytest-tag": "focal",
-        "postgres": "postgres",
-    },
-    {
-        "sytest-tag": "focal",
-        "postgres": "multi-postgres",
-        "workers": "workers",
-    },
-    {
-        "sytest-tag": "focal",
-        "postgres": "multi-postgres",
-        "workers": "workers",
-        "reactor": "asyncio",
-    },
-]
-
-if not IS_PR:
-    sytest_tests.extend(
-        [
-            {
-                "sytest-tag": "focal",
-                "reactor": "asyncio",
-            },
-            {
-                "sytest-tag": "focal",
-                "postgres": "postgres",
-                "reactor": "asyncio",
-            },
-            {
-                "sytest-tag": "testing",
-                "postgres": "postgres",
-            },
-            {
-                "sytest-tag": "buster",
-                "postgres": "multi-postgres",
-                "workers": "workers",
-            },
-        ]
-    )
-
-
-print("::group::Calculated sytest jobs")
-print(json.dumps(sytest_tests, indent=4))
-print("::endgroup::")
-
-test_matrix = json.dumps(sytest_tests)
-set_output("sytest_test_matrix", test_matrix)

.ci/scripts/check_lockfile.py

@@ -1,23 +0,0 @@
-#! /usr/bin/env python
-import sys
-
-if sys.version_info < (3, 11):
-    raise RuntimeError("Requires at least Python 3.11, to import tomllib")
-
-import tomllib
-
-with open("poetry.lock", "rb") as f:
-    lockfile = tomllib.load(f)
-
-try:
-    lock_version = lockfile["metadata"]["lock-version"]
-    assert lock_version == "2.0"
-except Exception:
-    print(
-        """\
-    Lockfile is not version 2.0. You probably need to upgrade poetry on your local box
-    and re-run `poetry lock --no-update`. See the Poetry cheat sheet at
-    https://matrix-org.github.io/synapse/develop/development/dependencies.html
-    """
-    )
-    raise

.ci/scripts/gotestfmt

@@ -1,21 +0,0 @@
-#!/bin/bash
-#
-# wraps `gotestfmt`, hiding output from successful packages unless
-# all tests passed.
-
-set -o pipefail
-set -e
-
-# tee the test results to a log, whilst also piping them into gotestfmt,
-# telling it to hide successful results, so that we can clearly see
-# unsuccessful results.
-tee complement.log | gotestfmt -hide successful-packages
-
-# gotestfmt will exit non-zero if there were any failures, so if we got to this
-# point, we must have had a successful result.
-echo "All tests successful; showing all test results"
-
-# Pipe the test results back through gotestfmt, showing all results.
-# The log file consists of JSON lines giving the test results, interspersed
-# with regular stdout lines (including reports of downloaded packages).
-grep '^{"Time":' complement.log | gotestfmt

.ci/scripts/postgres_exec.py

@@ -0,0 +1,31 @@
+#!/usr/bin/env python
+# Copyright 2019 The Matrix.org Foundation C.I.C.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import sys
+
+import psycopg2
+
+# a very simple replacment for `psql`, to make up for the lack of the postgres client
+# libraries in the synapse docker image.
+
+# We use "postgres" as a database because it's bound to exist and the "synapse" one
+# doesn't exist yet.
+db_conn = psycopg2.connect(
+    user="postgres", host="localhost", password="postgres", dbname="postgres"
+)
+db_conn.autocommit = True
+cur = db_conn.cursor()
+for c in sys.argv[1:]:
+    cur.execute(c)

.ci/scripts/prepare_old_deps.sh

@@ -1,36 +0,0 @@
-#!/usr/bin/env bash
-# this script is run by GitHub Actions in a plain `focal` container; it
-# - installs the minimal system requirements, and poetry;
-# - patches the project definition file to refer to old versions only;
-# - creates a venv with these old versions using poetry; and finally
-# - invokes `trial` to run the tests with old deps.
-
-set -ex
-
-# Prevent virtualenv from auto-updating pip to an incompatible version
-export VIRTUALENV_NO_DOWNLOAD=1
-
-# TODO: in the future, we could use an implementation of
-#   https://github.com/python-poetry/poetry/issues/3527
-#   https://github.com/pypa/pip/issues/8085
-# to select the lowest possible versions, rather than resorting to this sed script.
-
-# Patch the project definitions in-place:
-# - Replace all lower and tilde bounds with exact bounds
-# - Replace all caret bounds---but not the one that defines the supported Python version!
-# - Delete all lines referring to psycopg2 --- so no testing of postgres support.
-# - Use pyopenssl 17.0, which is the oldest version that works with
-#   a `cryptography` compiled against OpenSSL 1.1.
-# - Omit systemd: we're not logging to journal here.
-
-sed -i \
-   -e "s/[~>]=/==/g" \
-   -e '/^python = "^/!s/\^/==/g' \
-   -e "/psycopg2/d" \
-   -e 's/pyOpenSSL = "==16.0.0"/pyOpenSSL = "==17.0.0"/' \
-   -e '/systemd/d' \
-   pyproject.toml
-
-echo "::group::Patched pyproject.toml"
-cat pyproject.toml
-echo "::endgroup::"

.ci/scripts/setup_complement_prerequisites.sh

@@ -9,9 +9,19 @@ set -eu
 alias block='{ set +x; } 2>/dev/null; func() { echo "::group::$*"; set -x; }; func'
 alias endblock='{ set +x; } 2>/dev/null; func() { echo "::endgroup::"; set -x; }; func'
 
+block Set Go Version
+  # The path is set via a file given by $GITHUB_PATH. We need both Go 1.17 and GOPATH on the path to run Complement.
+  # See https://docs.github.com/en/actions/using-workflows/workflow-commands-for-github-actions#adding-a-system-path
+
+  # Add Go 1.17 to the PATH: see https://github.com/actions/virtual-environments/blob/main/images/linux/Ubuntu2004-Readme.md#environment-variables-2
+  echo "$GOROOT_1_17_X64/bin" >> $GITHUB_PATH
+  # Add the Go path to the PATH: We need this so we can call gotestfmt
+  echo "~/go/bin" >> $GITHUB_PATH
+endblock
+
 block Install Complement Dependencies
   sudo apt-get -qq update && sudo apt-get install -qqy libolm3 libolm-dev
-  go install -v github.com/gotesttools/gotestfmt/v2/cmd/gotestfmt@latest
+  go get -v github.com/haveyoudebuggedit/gotestfmt/v2/cmd/gotestfmt@latest
 endblock
 
 block Install custom gotestfmt template

.ci/scripts/test_export_data_command.sh

@@ -23,9 +23,8 @@ poetry run python -m synapse.app.admin_cmd -c .ci/sqlite-config.yaml  export-dat
 --output-directory /tmp/export_data
 
 # Test that the output directory exists and contains the rooms directory
-dir_r="/tmp/export_data/rooms"
-dir_u="/tmp/export_data/user_data"
-if [ -d "$dir_r" ] && [ -d "$dir_u" ]; then
+dir="/tmp/export_data/rooms"
+if [ -d "$dir" ]; then
   echo "Command successful, this test passes"
 else
   echo "No output directories found, the command fails against a sqlite database."
@@ -33,7 +32,7 @@ else
 fi
 
 # Create the PostgreSQL database.
-psql -c "CREATE DATABASE synapse"
+poetry run .ci/scripts/postgres_exec.py "CREATE DATABASE synapse"
 
 # Port the SQLite databse to postgres so we can check command works against postgres
 echo "+++ Port SQLite3 databse to postgres"
@@ -44,9 +43,8 @@ poetry run python -m synapse.app.admin_cmd -c .ci/postgres-config.yaml  export-d
 --output-directory /tmp/export_data2
 
 # Test that the output directory exists and contains the rooms directory
-dir_r2="/tmp/export_data2/rooms"
-dir_u2="/tmp/export_data2/user_data"
-if [ -d "$dir_r2" ] && [ -d "$dir_u2" ]; then
+dir2="/tmp/export_data2/rooms"
+if [ -d "$dir2" ]; then
   echo "Command successful, this test passes"
 else
   echo "No output directories found, the command fails against a postgres database."

.ci/scripts/test_old_deps.sh

@@ -0,0 +1,83 @@
+#!/usr/bin/env bash
+# this script is run by GitHub Actions in a plain `focal` container; it
+# - installs the minimal system requirements, and poetry;
+# - patches the project definition file to refer to old versions only;
+# - creates a venv with these old versions using poetry; and finally
+# - invokes `trial` to run the tests with old deps.
+
+# Prevent tzdata from asking for user input
+export DEBIAN_FRONTEND=noninteractive
+
+set -ex
+
+apt-get update
+apt-get install -y \
+        python3 python3-dev python3-pip python3-venv pipx \
+        libxml2-dev libxslt-dev xmlsec1 zlib1g-dev libjpeg-dev libwebp-dev
+
+export LANG="C.UTF-8"
+
+# Prevent virtualenv from auto-updating pip to an incompatible version
+export VIRTUALENV_NO_DOWNLOAD=1
+
+# TODO: in the future, we could use an implementation of
+#   https://github.com/python-poetry/poetry/issues/3527
+#   https://github.com/pypa/pip/issues/8085
+# to select the lowest possible versions, rather than resorting to this sed script.
+
+# Patch the project definitions in-place:
+# - Replace all lower and tilde bounds with exact bounds
+# - Replace all caret bounds---but not the one that defines the supported Python version!
+# - Delete all lines referring to psycopg2 --- so no testing of postgres support.
+# - Use pyopenssl 17.0, which is the oldest version that works with
+#   a `cryptography` compiled against OpenSSL 1.1.
+# - Omit systemd: we're not logging to journal here.
+
+# TODO: also replace caret bounds, see https://python-poetry.org/docs/dependency-specification/#version-constraints
+# We don't use these yet, but IIRC they are the default bound used when you `poetry add`.
+# The sed expression 's/\^/==/g' ought to do the trick. But it would also change
+# `python = "^3.7"` to `python = "==3.7", which would mean we fail because olddeps
+# runs on 3.8 (#12343).
+
+sed -i \
+   -e "s/[~>]=/==/g" \
+   -e '/^python = "^/!s/\^/==/g' \
+   -e "/psycopg2/d" \
+   -e 's/pyOpenSSL = "==16.0.0"/pyOpenSSL = "==17.0.0"/' \
+   -e '/systemd/d' \
+   pyproject.toml
+
+# Use poetry to do the installation. This ensures that the versions are all mutually
+# compatible (as far the package metadata declares, anyway); pip's package resolver
+# is more lax.
+#
+# Rather than `poetry install --no-dev`, we drop all dev dependencies from the
+# toml file. This means we don't have to ensure compatibility between old deps and
+# dev tools.
+
+pip install --user toml
+
+REMOVE_DEV_DEPENDENCIES="
+import toml
+with open('pyproject.toml', 'r') as f:
+    data = toml.loads(f.read())
+
+del data['tool']['poetry']['dev-dependencies']
+
+with open('pyproject.toml', 'w') as f:
+    toml.dump(data, f)
+"
+python3 -c "$REMOVE_DEV_DEPENDENCIES"
+
+pipx install poetry==1.1.14
+~/.local/bin/poetry lock
+
+echo "::group::Patched pyproject.toml"
+cat pyproject.toml
+echo "::endgroup::"
+echo "::group::Lockfile after patch"
+cat poetry.lock
+echo "::endgroup::"
+
+~/.local/bin/poetry install -E "all test"
+~/.local/bin/poetry run trial --jobs=2 tests

.ci/scripts/test_synapse_port_db.sh

@@ -2,27 +2,27 @@
 #
 # Test script for 'synapse_port_db'.
 #   - configures synapse and a postgres server.
-#   - runs the port script on a prepopulated test sqlite db. Checks that the
-#     return code is zero.
-#   - reruns the port script on the same sqlite db, targetting the same postgres db.
-#     Checks that the return code is zero.
-#   - runs the port script against a new sqlite db. Checks the return code is zero.
+#   - runs the port script on a prepopulated test sqlite db
+#   - also runs it against an new sqlite db
 #
 # Expects Synapse to have been already installed with `poetry install --extras postgres`.
 # Expects `poetry` to be available on the `PATH`.
 
-set -xe -o pipefail
+set -xe
 cd "$(dirname "$0")/../.."
 
 echo "--- Generate the signing key"
+
+# Generate the server's signing key.
 poetry run synapse_homeserver --generate-keys -c .ci/sqlite-config.yaml
 
 echo "--- Prepare test database"
-# Make sure the SQLite3 database is using the latest schema and has no pending background updates.
+
+# Make sure the SQLite3 database is using the latest schema and has no pending background update.
 poetry run update_synapse_database --database-config .ci/sqlite-config.yaml --run-background-updates
 
 # Create the PostgreSQL database.
-psql -c "CREATE DATABASE synapse"
+poetry run .ci/scripts/postgres_exec.py "CREATE DATABASE synapse"
 
 echo "+++ Run synapse_port_db against test database"
 # TODO: this invocation of synapse_port_db (and others below) used to be prepended with `coverage run`,
@@ -45,23 +45,9 @@ rm .ci/test_db.db
 poetry run update_synapse_database --database-config .ci/sqlite-config.yaml --run-background-updates
 
 # re-create the PostgreSQL database.
-psql \
-  -c "DROP DATABASE synapse" \
-  -c "CREATE DATABASE synapse"
+poetry run .ci/scripts/postgres_exec.py \
+  "DROP DATABASE synapse" \
+  "CREATE DATABASE synapse"
 
 echo "+++ Run synapse_port_db against empty database"
 poetry run synapse_port_db --sqlite-database .ci/test_db.db --postgres-config .ci/postgres-config.yaml
-
-echo "--- Create a brand new postgres database from schema"
-cp .ci/postgres-config.yaml .ci/postgres-config-unported.yaml
-sed -i -e 's/database: synapse/database: synapse_unported/' .ci/postgres-config-unported.yaml
-psql -c "CREATE DATABASE synapse_unported"
-poetry run update_synapse_database --database-config .ci/postgres-config-unported.yaml --run-background-updates
-
-echo "+++ Comparing ported schema with unported schema"
-# Ignore the tables that portdb creates. (Should it tidy them up when the porting is completed?)
-psql synapse -c "DROP TABLE port_from_sqlite3;"
-pg_dump --format=plain --schema-only --no-tablespaces --no-acl --no-owner synapse_unported > unported.sql
-pg_dump --format=plain --schema-only --no-tablespaces --no-acl --no-owner synapse          >   ported.sql
-# By default, `diff` returns zero if there are no changes and nonzero otherwise
-diff -u unported.sql ported.sql | tee schema_diff

.dockerignore

@@ -4,15 +4,8 @@
 # things to include
 !docker
 !synapse
-!rust
 !README.rst
 !pyproject.toml
 !poetry.lock
-!Cargo.lock
-!Cargo.toml
-!build_rust.py
-
-rust/target
-synapse/*.so
 
 **/__pycache__

.editorconfig

@@ -4,7 +4,7 @@
 root = true
 
 # 4 space indentation
-[*.{py,pyi}]
+[*.py]
 indent_style = space
 indent_size = 4
 max_line_length = 88

.flake8

@@ -0,0 +1,11 @@
+# TODO: incorporate this into pyproject.toml if flake8 supports it in the future.
+# See https://github.com/PyCQA/flake8/issues/234
+[flake8]
+# see https://pycodestyle.readthedocs.io/en/latest/intro.html#error-codes
+# for error codes. The ones we ignore are:
+#  W503: line break before binary operator
+#  W504: line break after binary operator
+#  E203: whitespace before ':' (which is contrary to pep8?)
+#  E731: do not assign a lambda expression, use a def
+#  E501: Line too long (black enforces this for us)
+ignore=W503,W504,E203,E731,E501

.git-blame-ignore-revs

@@ -21,8 +21,4 @@ aff1eb7c671b0a3813407321d2702ec46c71fa56
 0a00b7ff14890987f09112a2ae696c61001e6cf1
 
 # Convert tests/rest/admin/test_room.py to unix file endings (#7953).
-c4268e3da64f1abb5b31deaeb5769adb6510c0a7
-
-# Update black to 23.1.0 (#15103)
-9bb2eac71962970d02842bca441f4bcdbbf93a11
-
+c4268e3da64f1abb5b31deaeb5769adb6510c0a7

.github/ISSUE_TEMPLATE/BUG_REPORT.yml

@@ -74,36 +74,6 @@ body:
         - Debian packages from packages.matrix.org
         - pip (from PyPI)
         - Other (please mention below)
-        - I don't know
-    validations:
-      required: true
-  - type: input
-    id: database
-    attributes:
-      label: Database
-      description: |
-        Are you using SQLite or PostgreSQL? What's the version of your database?
-
-        If PostgreSQL, please also answer the following:
-         - are you using a single PostgreSQL server
-        or [separate servers for `main` and `state`](https://matrix-org.github.io/synapse/latest/usage/configuration/config_documentation.html#databases)?
-         - have you previously ported from SQLite using the Synapse "portdb" script?
-         - have you previously restored from a backup?
-    validations:
-      required: true
-  - type: dropdown
-    id: workers
-    attributes:
-      label: Workers
-      description: |
-        Are you running a single Synapse process, or are you running
-        [2 or more workers](https://matrix-org.github.io/synapse/latest/workers.html)?
-      options:
-        - Single process
-        - Multiple workers
-        - I don't know
-    validations:
-      required: true
   - type: textarea
     id: platform
     attributes:
@@ -113,28 +83,17 @@ body:
         e.g. distro, hardware, if it's running in a vm/container, etc.
     validations:
       required: true
-  - type: textarea
-    id: config
-    attributes:
-      label: Configuration
-      description: |
-        Do you have any unusual config options turned on? If so, please provide details.
-
-        - Experimental or undocumented features
-        - [Presence](https://matrix-org.github.io/synapse/latest/usage/configuration/config_documentation.html#presence)
-        - [Message retention](https://matrix-org.github.io/synapse/latest/message_retention_policies.html)
-        - [Synapse modules](https://matrix-org.github.io/synapse/latest/modules/index.html)
   - type: textarea
     id: logs
     attributes:
       label: Relevant log output
       description: |
-        Please copy and paste any relevant log output as text (not images), ideally at INFO or DEBUG log level.
-        This will be automatically formatted into code, so there is no need for backticks (`\``).
+        Please copy and paste any relevant log output, ideally at INFO or DEBUG log level.
+        This will be automatically formatted into code, so there is no need for backticks.
 
         Please be careful to remove any personal or private data.
 
-        **Bug reports are usually impossible to diagnose without logging.**
+        **Bug reports are usually very difficult to diagnose without logging.**
       render: shell
     validations:
       required: true

.github/dependabot.yml

@@ -1,23 +0,0 @@
-version: 2
-updates:
-  - # "pip" is the correct setting for poetry, per https://docs.github.com/en/code-security/dependabot/dependabot-version-updates/configuration-options-for-the-dependabot.yml-file#package-ecosystem
-    package-ecosystem: "pip"
-    directory: "/"
-    schedule:
-      interval: "weekly"
-
-  - package-ecosystem: "docker"
-    directory: "/docker"
-    schedule:
-      interval: "weekly"
-
-  - package-ecosystem: "github-actions"
-    directory: "/"
-    schedule:
-      interval: "weekly"
-
-  - package-ecosystem: "cargo"
-    directory: "/"
-    versioning-strategy: "lockfile-only"
-    schedule:
-      interval: "weekly"

.github/workflows/docker.yml

@@ -10,7 +10,6 @@ on:
 
 permissions:
   contents: read
-  packages: write
 
 jobs:
   build:
@@ -18,37 +17,28 @@ jobs:
     steps:
       - name: Set up QEMU
         id: qemu
-        uses: docker/setup-qemu-action@v2
+        uses: docker/setup-qemu-action@v1
         with:
           platforms: arm64
 
       - name: Set up Docker Buildx
         id: buildx
-        uses: docker/setup-buildx-action@v2
+        uses: docker/setup-buildx-action@v1
 
       - name: Inspect builder
         run: docker buildx inspect
-
+          
       - name: Log in to DockerHub
-        uses: docker/login-action@v2
+        uses: docker/login-action@v1
         with:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_TOKEN }}
 
-      - name: Log in to GHCR
-        uses: docker/login-action@v2
-        with:
-          registry: ghcr.io
-          username: ${{ github.repository_owner }}
-          password: ${{ secrets.GITHUB_TOKEN }}
-
       - name: Calculate docker image tag
         id: set-tag
         uses: docker/metadata-action@master
         with:
-          images: |
-            docker.io/matrixdotorg/synapse
-            ghcr.io/matrix-org/synapse
+          images: matrixdotorg/synapse
           flavor: |
             latest=false
           tags: |
@@ -58,15 +48,10 @@ jobs:
             type=pep440,pattern={{raw}}
 
       - name: Build and push all platforms
-        uses: docker/build-push-action@v4
+        uses: docker/build-push-action@v2
         with:
           push: true
           labels: "gitsha1=${{ github.sha }}"
           tags: "${{ steps.set-tag.outputs.tags }}"
           file: "docker/Dockerfile"
           platforms: linux/amd64,linux/arm64
-
-          # arm64 builds OOM without the git fetch setting. c.f.
-          # https://github.com/rust-lang/cargo/issues/10583
-          build-args: |
-            CARGO_NET_GIT_FETCH_WITH_CLI=true

.github/workflows/docs-pr-netlify.yaml

@@ -1,34 +0,0 @@
-name: Deploy documentation PR preview
-
-on:
-  workflow_run:
-    workflows: [ "Prepare documentation PR preview" ]
-    types:
-      - completed
-
-jobs:
-  netlify:
-    if: github.event.workflow_run.conclusion == 'success' && github.event.workflow_run.event == 'pull_request'
-    runs-on: ubuntu-latest
-    steps:
-      # There's a 'download artifact' action, but it hasn't been updated for the workflow_run action
-      # (https://github.com/actions/download-artifact/issues/60) so instead we get this mess:
-      - name: 📥 Download artifact
-        uses: dawidd6/action-download-artifact@246dbf436b23d7c49e21a7ab8204ca9ecd1fe615 # v2.27.0
-        with:
-          workflow: docs-pr.yaml
-          run_id: ${{ github.event.workflow_run.id }}
-          name: book
-          path: book
-
-      - name: 📤 Deploy to Netlify
-        uses: matrix-org/netlify-pr-preview@v2
-        with:
-          path: book
-          owner: ${{ github.event.workflow_run.head_repository.owner.login }}
-          branch: ${{ github.event.workflow_run.head_branch }}
-          revision: ${{ github.event.workflow_run.head_sha }}
-          token: ${{ secrets.NETLIFY_AUTH_TOKEN }}
-          site_id: ${{ secrets.NETLIFY_SITE_ID }}
-          desc: Documentation preview
-          deployment_env: PR Documentation Preview

.github/workflows/docs-pr.yaml

@@ -1,60 +0,0 @@
-name: Prepare documentation PR preview
-
-on:
-  pull_request:
-    paths:
-      - docs/**
-      - book.toml
-      - .github/workflows/docs-pr.yaml
-
-jobs:
-  pages:
-    name: GitHub Pages
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v3
-
-      - name: Setup mdbook
-        uses: peaceiris/actions-mdbook@adeb05db28a0c0004681db83893d56c0388ea9ea # v1.2.0
-        with:
-          mdbook-version: '0.4.17'
-
-      - name: Build the documentation
-        # mdbook will only create an index.html if we're including docs/README.md in SUMMARY.md.
-        # However, we're using docs/README.md for other purposes and need to pick a new page
-        # as the default. Let's opt for the welcome page instead.
-        run: |
-          mdbook build
-          cp book/welcome_and_overview.html book/index.html
-
-      - name: Upload Artifact
-        uses: actions/upload-artifact@v3
-        with:
-          name: book
-          path: book
-          # We'll only use this in a workflow_run, then we're done with it
-          retention-days: 1
-
-  link-check:
-    name: Check links in documentation
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v3
-
-      - name: Setup mdbook
-        uses: peaceiris/actions-mdbook@adeb05db28a0c0004681db83893d56c0388ea9ea # v1.2.0
-        with:
-          mdbook-version: '0.4.17'
-
-      - name: Setup htmltest
-        run: |
-          wget https://github.com/wjdp/htmltest/releases/download/v0.17.0/htmltest_0.17.0_linux_amd64.tar.gz
-          echo '775c597ee74899d6002cd2d93076f897f4ba68686bceabe2e5d72e84c57bc0fb  htmltest_0.17.0_linux_amd64.tar.gz' | sha256sum -c
-          tar zxf htmltest_0.17.0_linux_amd64.tar.gz
-
-      - name: Test links with htmltest
-        # Build the book with `./` as the site URL (to make checks on 404.html possible)
-        # Then run htmltest (without checking external links since that involves the network and is slow).
-        run: |
-          MDBOOK_OUTPUT__HTML__SITE_URL="./" mdbook build
-          ./htmltest book --skip-external

.github/workflows/docs.yaml

@@ -13,10 +13,25 @@ on:
   workflow_dispatch:
 
 jobs:
-  pre:
-    name: Calculate variables for GitHub Pages deployment
+  pages:
+    name: GitHub Pages
     runs-on: ubuntu-latest
     steps:
+      - uses: actions/checkout@v2
+
+      - name: Setup mdbook
+        uses: peaceiris/actions-mdbook@4b5ef36b314c2599664ca107bb8c02412548d79d # v1.1.14
+        with:
+          mdbook-version: '0.4.17'
+
+      - name: Build the documentation
+        # mdbook will only create an index.html if we're including docs/README.md in SUMMARY.md.
+        # However, we're using docs/README.md for other purposes and need to pick a new page
+        # as the default. Let's opt for the welcome page instead.
+        run: |
+          mdbook build
+          cp book/welcome_and_overview.html book/index.html
+
       # Figure out the target directory.
       #
       # The target directory depends on the name of the branch
@@ -39,66 +54,12 @@ jobs:
           esac
 
           # finally, set the 'branch-version' var.
-          echo "branch-version=$branch" >> "$GITHUB_OUTPUT"
-    outputs:
-      branch-version: ${{ steps.vars.outputs.branch-version }}
-
-################################################################################
-  pages-docs:
-    name: GitHub Pages
-    runs-on: ubuntu-latest
-    needs:
-      - pre
-    steps:
-      - uses: actions/checkout@v3
-
-      - name: Setup mdbook
-        uses: peaceiris/actions-mdbook@adeb05db28a0c0004681db83893d56c0388ea9ea # v1.2.0
-        with:
-          mdbook-version: '0.4.17'
-
-      - name: Build the documentation
-        # mdbook will only create an index.html if we're including docs/README.md in SUMMARY.md.
-        # However, we're using docs/README.md for other purposes and need to pick a new page
-        # as the default. Let's opt for the welcome page instead.
-        run: |
-          mdbook build
-          cp book/welcome_and_overview.html book/index.html
-
+          echo "::set-output name=branch-version::$branch"
+          
       # Deploy to the target directory.
       - name: Deploy to gh pages
-        uses: peaceiris/actions-gh-pages@373f7f263a76c20808c831209c920827a82a2847 # v3.9.3
+        uses: peaceiris/actions-gh-pages@068dc23d9710f1ba62e86896f84735d869951305 # v3.8.0
         with:
           github_token: ${{ secrets.GITHUB_TOKEN }}
           publish_dir: ./book
-          destination_dir: ./${{ needs.pre.outputs.branch-version }}
-
-################################################################################
-  pages-devdocs:
-    name: GitHub Pages (developer docs)
-    runs-on: ubuntu-latest
-    needs:
-      - pre
-    steps:
-      - uses: actions/checkout@v3
-
-      - name: "Set up Sphinx"
-        uses: matrix-org/setup-python-poetry@v1
-        with:
-          python-version: "3.x"
-          poetry-version: "1.3.2"
-          groups: "dev-docs"
-          extras: ""
-
-      - name: Build the documentation
-        run: |
-          cd dev-docs
-          poetry run make html
-
-      # Deploy to the target directory.
-      - name: Deploy to gh pages
-        uses: peaceiris/actions-gh-pages@373f7f263a76c20808c831209c920827a82a2847 # v3.9.3
-        with:
-          github_token: ${{ secrets.GITHUB_TOKEN }}
-          publish_dir: ./dev-docs/_build/html
-          destination_dir: ./dev-docs/${{ needs.pre.outputs.branch-version }}
+          destination_dir: ./${{ steps.vars.outputs.branch-version }}

.github/workflows/latest_deps.yml

@@ -5,7 +5,7 @@
 #
 # As an overview this workflow:
 # - checks out develop,
-# - installs from source, pulling in the dependencies like a fresh `pip install` would, and
+# - installs from source, pulling in the dependencies like a fresh `pip install` would, and 
 # - runs mypy and test suites in that checkout.
 #
 # Based on the twisted trunk CI job.
@@ -22,34 +22,16 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
-  check_repo:
-    # Prevent this workflow from running on any fork of Synapse other than matrix-org/synapse, as it is
-    # only useful to the Synapse core team.
-    # All other workflow steps depend on this one, thus if 'should_run_workflow' is not 'true', the rest
-    # of the workflow will be skipped as well.
-    runs-on: ubuntu-latest
-    outputs:
-      should_run_workflow: ${{ steps.check_condition.outputs.should_run_workflow }}
-    steps:
-      - id: check_condition
-        run: echo "should_run_workflow=${{ github.repository == 'matrix-org/synapse' }}" >> "$GITHUB_OUTPUT"
-
   mypy:
-    needs: check_repo
-    if: needs.check_repo.outputs.should_run_workflow == 'true'
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
-      - name: Install Rust
-        uses: dtolnay/rust-toolchain@stable
-      - uses: Swatinem/rust-cache@v2
-
+      - uses: actions/checkout@v2
       # The dev dependencies aren't exposed in the wheel metadata (at least with current
       # poetry-core versions), so we install with poetry.
       - uses: matrix-org/setup-python-poetry@v1
         with:
           python-version: "3.x"
-          poetry-version: "1.3.2"
+          poetry-version: "1.2.0b1"
           extras: "all"
       # Dump installed versions for debugging.
       - run: poetry run pip list > before.txt
@@ -61,8 +43,6 @@ jobs:
         run: sed '/warn_unused_ignores = True/d' -i mypy.ini
       - run: poetry run mypy
   trial:
-    needs: check_repo
-    if: needs.check_repo.outputs.should_run_workflow == 'true'
     runs-on: ubuntu-latest
     strategy:
       matrix:
@@ -72,12 +52,7 @@ jobs:
             postgres-version: "14"
 
     steps:
-      - uses: actions/checkout@v3
-
-      - name: Install Rust
-        uses: dtolnay/rust-toolchain@stable
-      - uses: Swatinem/rust-cache@v2
-
+      - uses: actions/checkout@v2
       - run: sudo apt-get -qq install xmlsec1
       - name: Set up PostgreSQL ${{ matrix.postgres-version }}
         if: ${{ matrix.postgres-version }}
@@ -86,7 +61,7 @@ jobs:
             -e POSTGRES_PASSWORD=postgres \
             -e POSTGRES_INITDB_ARGS="--lc-collate C --lc-ctype C --encoding UTF8" \
             postgres:${{ matrix.postgres-version }}
-      - uses: actions/setup-python@v4
+      - uses: actions/setup-python@v2
         with:
           python-version: "3.x"
       - run: pip install .[all,test]
@@ -94,12 +69,6 @@ jobs:
         if: ${{ matrix.postgres-version }}
         timeout-minutes: 2
         run: until pg_isready -h localhost; do sleep 1; done
-
-      # We nuke the local copy, as we've installed synapse into the virtualenv
-      # (rather than use an editable install, which we no longer support). If we
-      # don't do this then python can't find the native lib.
-      - run: rm -rf synapse/
-
       - run: python -m twisted.trial --jobs=2 tests
         env:
           SYNAPSE_POSTGRES: ${{ matrix.database == 'postgres' || '' }}
@@ -121,8 +90,6 @@ jobs:
 
 
   sytest:
-    needs: check_repo
-    if: needs.check_repo.outputs.should_run_workflow == 'true'
     runs-on: ubuntu-latest
     container:
       image: matrixdotorg/sytest-synapse:testing
@@ -145,12 +112,7 @@ jobs:
       BLACKLIST: ${{ matrix.workers && 'synapse-blacklist-with-workers' }}
 
     steps:
-      - uses: actions/checkout@v3
-
-      - name: Install Rust
-        uses: dtolnay/rust-toolchain@stable
-      - uses: Swatinem/rust-cache@v2
-
+      - uses: actions/checkout@v2
       - name: Ensure sytest runs `pip install`
         # Delete the lockfile so sytest will `pip install` rather than `poetry install`
         run: rm /src/poetry.lock
@@ -164,7 +126,7 @@ jobs:
         if: ${{ always() }}
         run: /sytest/scripts/tap_to_gha.pl /logs/results.tap
       - name: Upload SyTest logs
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v2
         if: ${{ always() }}
         with:
           name: Sytest Logs - ${{ job.status }} - (${{ join(matrix.*, ', ') }})
@@ -174,8 +136,7 @@ jobs:
 
 
   complement:
-    needs: check_repo
-    if: "!failure() && !cancelled() && needs.check_repo.outputs.should_run_workflow == 'true'"
+    if: "${{ !failure() && !cancelled() }}"
     runs-on: ubuntu-latest
 
     strategy:
@@ -192,40 +153,38 @@ jobs:
             database: Postgres
 
     steps:
-      - name: Run actions/checkout@v3 for synapse
-        uses: actions/checkout@v3
+      - name: Run actions/checkout@v2 for synapse
+        uses: actions/checkout@v2
         with:
           path: synapse
 
-      - uses: actions/setup-go@v4
-
       - name: Prepare Complement's Prerequisites
         run: synapse/.ci/scripts/setup_complement_prerequisites.sh
 
       - run: |
           set -o pipefail
-          TEST_ONLY_IGNORE_POETRY_LOCKFILE=1 POSTGRES=${{ (matrix.database == 'Postgres') && 1 || '' }} WORKERS=${{ (matrix.arrangement == 'workers') && 1 || '' }} COMPLEMENT_DIR=`pwd`/complement synapse/scripts-dev/complement.sh -json 2>&1 | synapse/.ci/scripts/gotestfmt
+          TEST_ONLY_IGNORE_POETRY_LOCKFILE=1 POSTGRES=${{ (matrix.database == 'Postgres') && 1 || '' }} WORKERS=${{ (matrix.arrangement == 'workers') && 1 || '' }} COMPLEMENT_DIR=`pwd`/complement synapse/scripts-dev/complement.sh -json 2>&1 | gotestfmt
         shell: bash
         name: Run Complement Tests
 
   # Open an issue if the build fails, so we know about it.
   # Only do this if we're not experimenting with this action in a PR.
   open-issue:
-    if: "failure() && github.event_name != 'push' && github.event_name != 'pull_request' && needs.check_repo.outputs.should_run_workflow == 'true'"
+    if: "failure() && github.event_name != 'push' && github.event_name != 'pull_request'"
     needs:
-      # TODO: should mypy be included here? It feels more brittle than the others.
+      # TODO: should mypy be included here? It feels more brittle than the other two.
       - mypy
       - trial
       - sytest
-      - complement
 
     runs-on: ubuntu-latest
 
     steps:
-      - uses: actions/checkout@v3
-      - uses: JasonEtco/create-an-issue@e27dddc79c92bc6e4562f268fffa5ed752639abd # v2.9.1
+      - uses: actions/checkout@v2
+      - uses: JasonEtco/create-an-issue@5d9504915f79f9cc6d791934b8ef34f2353dd74d # v2.5.0, 2020-12-06
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         with:
           update_existing: true
           filename: .ci/latest_deps_build_failed_issue_template.md
+

.github/workflows/poetry_lockfile.yaml

@@ -1,24 +0,0 @@
-on:
-  push:
-    branches: ["develop", "release-*"]
-    paths:
-      - poetry.lock
-  pull_request:
-    paths:
-      - poetry.lock
-
-concurrency:
-  group: ${{ github.workflow }}-${{ github.ref }}
-  cancel-in-progress: true
-
-jobs:
-  check-sdists:
-    name: "Check locked dependencies have sdists"
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v3
-      - uses: actions/setup-python@v4
-        with:
-          python-version: '3.x'
-      - run: pip install tomli
-      - run: ./scripts-dev/check_locked_deps_have_sdists.py

.github/workflows/push_complement_image.yml

@@ -1,74 +0,0 @@
-# This task does not run complement tests, see tests.yaml instead.
-# This task does not build docker images for synapse for use on docker hub, see docker.yaml instead
-
-name: Store complement-synapse image in ghcr.io
-on:
-  push:
-    branches: [ "master" ]
-  schedule:
-    - cron: '0 5 * * *'
-  workflow_dispatch:
-    inputs:
-      branch:
-        required: true
-        default: 'develop'
-        type: choice
-        options:
-          - develop
-          - master
-
-# Only run this action once per pull request/branch; restart if a new commit arrives.
-# C.f. https://docs.github.com/en/actions/reference/workflow-syntax-for-github-actions#concurrency
-# and https://docs.github.com/en/actions/reference/context-and-expression-syntax-for-github-actions#github-context
-concurrency:
-  group: ${{ github.workflow }}-${{ github.ref }}
-  cancel-in-progress: true
-
-jobs:
-  build:
-    name: Build and push complement image
-    runs-on: ubuntu-latest
-    permissions:
-      contents: read
-      packages: write
-    steps:
-      - name: Checkout specific branch (debug build)
-        uses: actions/checkout@v3
-        if: github.event_name == 'workflow_dispatch'
-        with:
-          ref: ${{ inputs.branch }}
-      - name: Checkout clean copy of develop (scheduled build)
-        uses: actions/checkout@v3
-        if: github.event_name == 'schedule'
-        with:
-          ref: develop
-      - name: Checkout clean copy of master (on-push)
-        uses: actions/checkout@v3
-        if: github.event_name == 'push'
-        with:
-          ref: master
-      - name: Login to registry
-        uses: docker/login-action@v2
-        with:
-          registry: ghcr.io
-          username: ${{ github.actor }}
-          password: ${{ secrets.GITHUB_TOKEN }}
-      - name: Work out labels for complement image
-        id: meta
-        uses: docker/metadata-action@v4
-        with:
-          images: ghcr.io/${{ github.repository }}/complement-synapse
-          tags: |
-            type=schedule,pattern=nightly,enable=${{ github.event_name == 'schedule'}}
-            type=raw,value=develop,enable=${{ github.event_name == 'schedule' || inputs.branch == 'develop' }}
-            type=raw,value=latest,enable=${{ github.event_name == 'push' || inputs.branch == 'master' }}
-            type=sha,format=long
-      - name: Run scripts-dev/complement.sh to generate complement-synapse:latest image.
-        run: scripts-dev/complement.sh --build-only
-      - name: Tag and push generated image
-        run: |
-          for TAG in ${{ join(fromJson(steps.meta.outputs.json).tags, ' ') }}; do 
-            echo "tag and push $TAG"
-            docker tag complement-synapse $TAG
-            docker push $TAG
-          done

.github/workflows/release-artifacts.yml

@@ -4,21 +4,18 @@ name: Build release artifacts
 
 on:
   # we build on PRs and develop to (hopefully) get early warning
-  # of things breaking (but only build one set of debs). PRs skip
-  # building wheels on macOS & ARM.
+  # of things breaking (but only build one set of debs)
   pull_request:
   push:
     branches: ["develop", "release-*"]
 
     # we do the full build on tags.
     tags: ["v*"]
-  merge_group:
-  workflow_dispatch:
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}
   cancel-in-progress: true
-
+  
 permissions:
   contents: write
 
@@ -27,19 +24,16 @@ jobs:
     name: "Calculate list of debian distros"
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
-      - uses: actions/setup-python@v4
-        with:
-          python-version: '3.x'
+      - uses: actions/checkout@v2
+      - uses: actions/setup-python@v2
       - id: set-distros
         run: |
           # if we're running from a tag, get the full list of distros; otherwise just use debian:sid
-          # NOTE: inside the actual Dockerfile-dhvirtualenv, the image name is expanded into its full image path
           dists='["debian:sid"]'
           if [[ $GITHUB_REF == refs/tags/* ]]; then
               dists=$(scripts-dev/build_debian_packages.py --show-dists-json)
           fi
-          echo "distros=$dists" >> "$GITHUB_OUTPUT"
+          echo "::set-output name=distros::$dists"
     # map the step outputs to job outputs
     outputs:
       distros: ${{ steps.set-distros.outputs.distros }}
@@ -55,18 +49,18 @@ jobs:
 
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v2
         with:
           path: src
 
       - name: Set up Docker Buildx
         id: buildx
-        uses: docker/setup-buildx-action@v2
+        uses: docker/setup-buildx-action@v1
         with:
           install: true
 
       - name: Set up docker layer caching
-        uses: actions/cache@v3
+        uses: actions/cache@v2
         with:
           path: /tmp/.buildx-cache
           key: ${{ runner.os }}-buildx-${{ github.sha }}
@@ -74,9 +68,7 @@ jobs:
             ${{ runner.os }}-buildx-
 
       - name: Set up python
-        uses: actions/setup-python@v4
-        with:
-          python-version: '3.x'
+        uses: actions/setup-python@v2
 
       - name: Build the packages
         # see https://github.com/docker/build-push-action/issues/252
@@ -92,96 +84,14 @@ jobs:
           mv /tmp/.buildx-cache-new /tmp/.buildx-cache
 
       - name: Upload debs as artifacts
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v2
         with:
           name: debs
           path: debs/*
 
-  build-wheels:
-    name: Build wheels on ${{ matrix.os }} for ${{ matrix.arch }}
-    runs-on: ${{ matrix.os }}
-    strategy:
-      matrix:
-        os: [ubuntu-20.04, macos-11]
-        arch: [x86_64, aarch64]
-        # is_pr is a flag used to exclude certain jobs from the matrix on PRs.
-        # It is not read by the rest of the workflow.
-        is_pr:
-          - ${{ startsWith(github.ref, 'refs/pull/') }}
-
-        exclude:
-          # Don't build macos wheels on PR CI.
-          - is_pr: true
-            os: "macos-11"
-          # Don't build aarch64 wheels on mac.
-          - os: "macos-11"
-            arch: aarch64
-          # Don't build aarch64 wheels on PR CI.
-          - is_pr: true
-            arch: aarch64
-
-    steps:
-      - uses: actions/checkout@v3
-
-      - uses: actions/setup-python@v4
-        with:
-          # setup-python@v4 doesn't impose a default python version. Need to use 3.x
-          # here, because `python` on osx points to Python 2.7.
-          python-version: "3.x"
-
-      - name: Install cibuildwheel
-        run: python -m pip install cibuildwheel==2.9.0
-
-      - name: Set up QEMU to emulate aarch64
-        if: matrix.arch == 'aarch64'
-        uses: docker/setup-qemu-action@v2
-        with:
-          platforms: arm64
-
-      - name: Build aarch64 wheels
-        if: matrix.arch == 'aarch64'
-        run: echo 'CIBW_ARCHS_LINUX=aarch64' >> $GITHUB_ENV
-
-      - name: Only build a single wheel on PR
-        if: startsWith(github.ref, 'refs/pull/')
-        run: echo "CIBW_BUILD="cp37-manylinux_${{ matrix.arch }}"" >> $GITHUB_ENV
-
-      - name: Build wheels
-        run: python -m cibuildwheel --output-dir wheelhouse
-        env:
-          # Skip testing for platforms which various libraries don't have wheels
-          # for, and so need extra build deps.
-          CIBW_TEST_SKIP: pp3*-* *i686* *musl*
-          # Fix Rust OOM errors on emulated aarch64: https://github.com/rust-lang/cargo/issues/10583
-          CARGO_NET_GIT_FETCH_WITH_CLI: true
-          CIBW_ENVIRONMENT_PASS_LINUX: CARGO_NET_GIT_FETCH_WITH_CLI
-
-      - uses: actions/upload-artifact@v3
-        with:
-          name: Wheel
-          path: ./wheelhouse/*.whl
-
   build-sdist:
-    name: Build sdist
-    runs-on: ubuntu-latest
-    if: ${{ !startsWith(github.ref, 'refs/pull/') }}
-
-    steps:
-      - uses: actions/checkout@v3
-      - uses: actions/setup-python@v4
-        with:
-          python-version: '3.10'
-
-      - run: pip install build
-
-      - name: Build sdist
-        run: python -m build --sdist
-
-      - uses: actions/upload-artifact@v3
-        with:
-          name: Sdist
-          path: dist/*.tar.gz
-
+    name: "Build pypi distribution files"
+    uses: "matrix-org/backend-meta/.github/workflows/packaging.yml@v1"
 
   # if it's a tag, create a release and attach the artifacts to it
   attach-assets:
@@ -189,12 +99,11 @@ jobs:
     if: ${{ !failure() && !cancelled() && startsWith(github.ref, 'refs/tags/') }}
     needs:
       - build-debs
-      - build-wheels
       - build-sdist
     runs-on: ubuntu-latest
     steps:
       - name: Download all workflow run artifacts
-        uses: actions/download-artifact@v3
+        uses: actions/download-artifact@v2
       - name: Build a tarball for the debs
         run: tar -cvJf debs.tar.xz debs
       - name: Attach to release

.github/workflows/tests.yml

@@ -4,286 +4,111 @@ on:
   push:
     branches: ["develop", "release-*"]
   pull_request:
-  merge_group:
-  workflow_dispatch:
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}
   cancel-in-progress: true
 
 jobs:
-  # Job to detect what has changed so we don't run e.g. Rust checks on PRs that
-  # don't modify Rust code.
-  changes:
-    runs-on: ubuntu-latest
-    outputs:
-      rust: ${{ !startsWith(github.ref, 'refs/pull/') || steps.filter.outputs.rust }}
-    steps:
-    - uses: dorny/paths-filter@v2
-      id: filter
-      # We only check on PRs
-      if: startsWith(github.ref, 'refs/pull/')
-      with:
-        filters: |
-          rust:
-            - 'rust/**'
-            - 'Cargo.toml'
-            - 'Cargo.lock'
-
   check-sampleconfig:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
-      - name: Install Rust
-        uses: dtolnay/rust-toolchain@1.60.0
-      - uses: Swatinem/rust-cache@v2
-      - uses: matrix-org/setup-python-poetry@v1
-        with:
-          python-version: "3.x"
-          poetry-version: "1.3.2"
-          extras: "all"
-      - run: poetry run scripts-dev/generate_sample_config.sh --check
-      - run: poetry run scripts-dev/config-lint.sh
+      - uses: actions/checkout@v2
+      - uses: actions/setup-python@v2
+      - run: pip install .
+      - run: scripts-dev/generate_sample_config.sh --check
+      - run: scripts-dev/config-lint.sh
 
   check-schema-delta:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
-      - uses: actions/setup-python@v4
-        with:
-          python-version: "3.x"
+      - uses: actions/checkout@v2
+      - uses: actions/setup-python@v2
       - run: "pip install 'click==8.1.1' 'GitPython>=3.1.20'"
       - run: scripts-dev/check_schema_delta.py --force-colors
 
-  check-lockfile:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v3
-      - uses: actions/setup-python@v4
-        with:
-          python-version: "3.x"
-      - run: .ci/scripts/check_lockfile.py
-
   lint:
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout repository
-        uses: actions/checkout@v3
-
-      - name: Setup Poetry
-        uses: matrix-org/setup-python-poetry@v1
-        with:
-          install-project: "false"
-
-      - name: Import order (isort)
-        run: poetry run isort --check --diff .
-
-      - name: Code style (black)
-        run: poetry run black --check --diff .
-
-      - name: Semantic checks (ruff)
-        # --quiet suppresses the update check.
-        run: poetry run ruff --quiet .
-
-  lint-mypy:
-    runs-on: ubuntu-latest
-    name: Typechecking
-    steps:
-      - name: Checkout repository
-        uses: actions/checkout@v3
-
-      - name: Install Rust
-        uses: dtolnay/rust-toolchain@1.60.0
-      - uses: Swatinem/rust-cache@v2
-
-      - name: Setup Poetry
-        uses: matrix-org/setup-python-poetry@v1
-        with:
-          # We want to make use of type hints in optional dependencies too.
-          extras: all
-          # We have seen odd mypy failures that were resolved when we started
-          # installing the project again:
-          # https://github.com/matrix-org/synapse/pull/15376#issuecomment-1498983775
-          # To make CI green, err towards caution and install the project.
-          install-project: "true"
-
-      # Cribbed from
-      # https://github.com/AustinScola/mypy-cache-github-action/blob/85ea4f2972abed39b33bd02c36e341b28ca59213/src/restore.ts#L10-L17
-      - name: Restore/persist mypy's cache
-        uses: actions/cache@v3
-        with:
-          path: |
-            .mypy_cache
-          key: mypy-cache-${{ github.context.sha }}
-          restore-keys: mypy-cache-
-
-      - name: Run mypy
-        run: poetry run mypy
+    uses: "matrix-org/backend-meta/.github/workflows/python-poetry-ci.yml@v1"
+    with:
+      typechecking-extras: "all"
 
   lint-crlf:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v2
       - name: Check line endings
         run: scripts-dev/check_line_terminators.sh
 
   lint-newsfile:
-    if: ${{ (github.base_ref == 'develop'  || contains(github.base_ref, 'release-')) && github.actor != 'dependabot[bot]' }}
+    if: ${{ github.base_ref == 'develop'  || contains(github.base_ref, 'release-') }}
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v2
         with:
           ref: ${{ github.event.pull_request.head.sha }}
           fetch-depth: 0
-      - uses: actions/setup-python@v4
-        with:
-          python-version: "3.x"
+      - uses: actions/setup-python@v2
       - run: "pip install 'towncrier>=18.6.0rc1'"
       - run: scripts-dev/check-newsfragment.sh
         env:
           PULL_REQUEST_NUMBER: ${{ github.event.number }}
 
-  lint-pydantic:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v3
-        with:
-          ref: ${{ github.event.pull_request.head.sha }}
-      - name: Install Rust
-        uses: dtolnay/rust-toolchain@1.60.0
-      - uses: Swatinem/rust-cache@v2
-      - uses: matrix-org/setup-python-poetry@v1
-        with:
-          poetry-version: "1.3.2"
-          extras: "all"
-      - run: poetry run scripts-dev/check_pydantic_models.py
-
-  lint-clippy:
-    runs-on: ubuntu-latest
-    needs: changes
-    if: ${{ needs.changes.outputs.rust == 'true' }}
-
-    steps:
-      - uses: actions/checkout@v3
-
-      - name: Install Rust
-        uses: dtolnay/rust-toolchain@1.60.0
-        with:
-            components: clippy
-      - uses: Swatinem/rust-cache@v2
-
-      - run: cargo clippy -- -D warnings
-
-  # We also lint against a nightly rustc so that we can lint the benchmark
-  # suite, which requires a nightly compiler.
-  lint-clippy-nightly:
-    runs-on: ubuntu-latest
-    needs: changes
-    if: ${{ needs.changes.outputs.rust == 'true' }}
-
-    steps:
-      - uses: actions/checkout@v3
-
-      - name: Install Rust
-        uses: dtolnay/rust-toolchain@master
-        with:
-            toolchain: nightly-2022-12-01
-            components: clippy
-      - uses: Swatinem/rust-cache@v2
-
-      - run: cargo clippy --all-features -- -D warnings
-
-  lint-rustfmt:
-    runs-on: ubuntu-latest
-    needs: changes
-    if: ${{ needs.changes.outputs.rust == 'true' }}
-
-    steps:
-      - uses: actions/checkout@v3
-
-      - name: Install Rust
-        uses: dtolnay/rust-toolchain@master
-        with:
-          # We use nightly so that it correctly groups together imports
-          toolchain: nightly-2022-12-01
-          components: rustfmt
-      - uses: Swatinem/rust-cache@v2
-
-      - run: cargo fmt --check
-
   # Dummy step to gate other tests on without repeating the whole list
   linting-done:
     if: ${{ !cancelled() }} # Run this even if prior jobs were skipped
-    needs:
-      - lint
-      - lint-mypy
-      - lint-crlf
-      - lint-newsfile
-      - lint-pydantic
-      - check-sampleconfig
-      - check-schema-delta
-      - check-lockfile
-      - lint-clippy
-      - lint-rustfmt
+    needs: [lint, lint-crlf, lint-newsfile, check-sampleconfig, check-schema-delta]
     runs-on: ubuntu-latest
     steps:
       - run: "true"
 
-  calculate-test-jobs:
+  trial:
     if: ${{ !cancelled() && !failure() }} # Allow previous steps to be skipped, but not fail
     needs: linting-done
     runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v3
-      - uses: actions/setup-python@v4
-        with:
-          python-version: "3.x"
-      - id: get-matrix
-        run: .ci/scripts/calculate_jobs.py
-    outputs:
-      trial_test_matrix: ${{ steps.get-matrix.outputs.trial_test_matrix }}
-      sytest_test_matrix: ${{ steps.get-matrix.outputs.sytest_test_matrix }}
-
-  trial:
-    if: ${{ !cancelled() && !failure() }} # Allow previous steps to be skipped, but not fail
-    needs: calculate-test-jobs
-    runs-on: ubuntu-latest
     strategy:
       matrix:
-        job:  ${{ fromJson(needs.calculate-test-jobs.outputs.trial_test_matrix) }}
+        python-version: ["3.7", "3.8", "3.9", "3.10"]
+        database: ["sqlite"]
+        extras: ["all"]
+        include:
+          # Newest Python without optional deps
+          - python-version: "3.10"
+            extras: ""
+
+          # Oldest Python with PostgreSQL
+          - python-version: "3.7"
+            database: "postgres"
+            postgres-version: "10"
+            extras: "all"
+
+          # Newest Python with newest PostgreSQL
+          - python-version: "3.10"
+            database: "postgres"
+            postgres-version: "14"
+            extras: "all"
 
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v2
       - run: sudo apt-get -qq install xmlsec1
-      - name: Set up PostgreSQL ${{ matrix.job.postgres-version }}
-        if: ${{ matrix.job.postgres-version }}
-        # 1. Mount postgres data files onto a tmpfs in-memory filesystem to reduce overhead of docker's overlayfs layer.
-        # 2. Expose the unix socket for postgres. This removes latency of using docker-proxy for connections.
+      - name: Set up PostgreSQL ${{ matrix.postgres-version }}
+        if: ${{ matrix.postgres-version }}
         run: |
           docker run -d -p 5432:5432 \
-            --tmpfs /var/lib/postgres:rw,size=6144m \
-            --mount 'type=bind,src=/var/run/postgresql,dst=/var/run/postgresql' \
             -e POSTGRES_PASSWORD=postgres \
             -e POSTGRES_INITDB_ARGS="--lc-collate C --lc-ctype C --encoding UTF8" \
-            postgres:${{ matrix.job.postgres-version }}
-
-      - name: Install Rust
-        uses: dtolnay/rust-toolchain@1.60.0
-      - uses: Swatinem/rust-cache@v2
-
+            postgres:${{ matrix.postgres-version }}
       - uses: matrix-org/setup-python-poetry@v1
         with:
-          python-version: ${{ matrix.job.python-version }}
-          poetry-version: "1.3.2"
-          extras: ${{ matrix.job.extras }}
+          python-version: ${{ matrix.python-version }}
+          extras: ${{ matrix.extras }}
       - name: Await PostgreSQL
-        if: ${{ matrix.job.postgres-version }}
+        if: ${{ matrix.postgres-version }}
         timeout-minutes: 2
         run: until pg_isready -h localhost; do sleep 1; done
-      - run: poetry run trial --jobs=6 tests
+      - run: poetry run trial --jobs=2 tests
         env:
-          SYNAPSE_POSTGRES: ${{ matrix.job.database == 'postgres' || '' }}
-          SYNAPSE_POSTGRES_HOST: /var/run/postgresql
+          SYNAPSE_POSTGRES: ${{ matrix.database == 'postgres' || '' }}
+          SYNAPSE_POSTGRES_HOST: localhost
           SYNAPSE_POSTGRES_USER: postgres
           SYNAPSE_POSTGRES_PASSWORD: postgres
       - name: Dump logs
@@ -303,44 +128,16 @@ jobs:
     # Note: sqlite only; no postgres
     if: ${{ !cancelled() && !failure() }} # Allow previous steps to be skipped, but not fail
     needs: linting-done
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
-
-      - name: Install Rust
-        uses: dtolnay/rust-toolchain@1.60.0
-      - uses: Swatinem/rust-cache@v2
-
-      # There aren't wheels for some of the older deps, so we need to install
-      # their build dependencies
-      - run: |
-          sudo apt-get -qq update
-          sudo apt-get -qq install build-essential libffi-dev python-dev \
-            libxml2-dev libxslt-dev xmlsec1 zlib1g-dev libjpeg-dev libwebp-dev
-
-      - uses: actions/setup-python@v4
+      - uses: actions/checkout@v2
+      - name: Test with old deps
+        uses: docker://ubuntu:focal # For old python and sqlite
+        # Note: focal seems to be using 3.8, but the oldest is 3.7?
+        # See https://github.com/matrix-org/synapse/issues/12343
         with:
-          python-version: '3.7'
-
-      - name: Prepare old deps
-        if: steps.cache-poetry-old-deps.outputs.cache-hit != 'true'
-        run: .ci/scripts/prepare_old_deps.sh
-
-      # Note: we install using `pip` here, not poetry. `poetry install` ignores the
-      # build-system section (https://github.com/python-poetry/poetry/issues/6154), but
-      # we explicitly want to test that you can `pip install` using the oldest version
-      # of poetry-core and setuptools-rust.
-      - run: pip install .[all,test]
-
-      # We nuke the local copy, as we've installed synapse into the virtualenv
-      # (rather than use an editable install, which we no longer support). If we
-      # don't do this then python can't find the native lib.
-      - run: rm -rf synapse/
-
-      # Sanity check we can import/run Synapse
-      - run: python -m synapse.app.homeserver --help
-
-      - run: python -m twisted.trial -j6 tests
+          workdir: /github/workspace
+          entrypoint: .ci/scripts/test_old_deps.sh
       - name: Dump logs
         # Logs are most useful when the command fails, always include them.
         if: ${{ always() }}
@@ -366,13 +163,12 @@ jobs:
         extras: ["all"]
 
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v2
       # Install libs necessary for PyPy to build binary wheels for dependencies
       - run: sudo apt-get -qq install xmlsec1 libxml2-dev libxslt-dev
       - uses: matrix-org/setup-python-poetry@v1
         with:
           python-version: ${{ matrix.python-version }}
-          poetry-version: "1.3.2"
           extras: ${{ matrix.extras }}
       - run: poetry run trial --jobs=2 tests
       - name: Dump logs
@@ -390,35 +186,50 @@ jobs:
 
   sytest:
     if: ${{ !failure() && !cancelled() }}
-    needs: calculate-test-jobs
+    needs: linting-done
     runs-on: ubuntu-latest
     container:
-      image: matrixdotorg/sytest-synapse:${{ matrix.job.sytest-tag }}
+      image: matrixdotorg/sytest-synapse:${{ matrix.sytest-tag }}
       volumes:
         - ${{ github.workspace }}:/src
       env:
         SYTEST_BRANCH: ${{ github.head_ref }}
-        POSTGRES: ${{ matrix.job.postgres && 1}}
-        MULTI_POSTGRES: ${{ (matrix.job.postgres == 'multi-postgres') && 1}}
-        ASYNCIO_REACTOR: ${{ (matrix.job.reactor == 'asyncio') && 1 }}
-        WORKERS: ${{ matrix.job.workers && 1 }}
-        BLACKLIST: ${{ matrix.job.workers && 'synapse-blacklist-with-workers' }}
+        POSTGRES: ${{ matrix.postgres && 1}}
+        MULTI_POSTGRES: ${{ (matrix.postgres == 'multi-postgres') && 1}}
+        WORKERS: ${{ matrix.workers && 1 }}
+        REDIS: ${{ matrix.redis && 1 }}
+        BLACKLIST: ${{ matrix.workers && 'synapse-blacklist-with-workers' }}
         TOP: ${{ github.workspace }}
 
     strategy:
       fail-fast: false
       matrix:
-        job: ${{ fromJson(needs.calculate-test-jobs.outputs.sytest_test_matrix) }}
+        include:
+          - sytest-tag: focal
+
+          - sytest-tag: focal
+            postgres: postgres
+
+          - sytest-tag: testing
+            postgres: postgres
+
+          - sytest-tag: focal
+            postgres: multi-postgres
+            workers: workers
+
+          - sytest-tag: buster
+            postgres: multi-postgres
+            workers: workers
+
+          - sytest-tag: buster
+            postgres: postgres
+            workers: workers
+            redis: redis
 
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v2
       - name: Prepare test blacklist
         run: cat sytest-blacklist .ci/worker-blacklist > synapse-blacklist-with-workers
-
-      - name: Install Rust
-        uses: dtolnay/rust-toolchain@1.60.0
-      - uses: Swatinem/rust-cache@v2
-
       - name: Run SyTest
         run: /bootstrap.sh synapse
         working-directory: /src
@@ -426,10 +237,10 @@ jobs:
         if: ${{ always() }}
         run: /sytest/scripts/tap_to_gha.pl /logs/results.tap
       - name: Upload SyTest logs
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v2
         if: ${{ always() }}
         with:
-          name: Sytest Logs - ${{ job.status }} - (${{ join(matrix.job.*, ', ') }})
+          name: Sytest Logs - ${{ job.status }} - (${{ join(matrix.*, ', ') }})
           path: |
             /logs/results.tap
             /logs/**/*.log*
@@ -456,32 +267,28 @@ jobs:
           --health-retries 5
 
     steps:
-      - uses: actions/checkout@v3
-      - run: sudo apt-get -qq install xmlsec1 postgresql-client
+      - uses: actions/checkout@v2
+      - run: sudo apt-get -qq install xmlsec1
       - uses: matrix-org/setup-python-poetry@v1
         with:
-          poetry-version: "1.3.2"
+          python-version: ${{ matrix.python-version }}
           extras: "postgres"
       - run: .ci/scripts/test_export_data_command.sh
-        env:
-          PGHOST: localhost
-          PGUSER: postgres
-          PGPASSWORD: postgres
-          PGDATABASE: postgres
-
 
   portdb:
     if: ${{ !failure() && !cancelled() }} # Allow previous steps to be skipped, but not fail
     needs: linting-done
     runs-on: ubuntu-latest
+    env:
+      TOP: ${{ github.workspace }}
     strategy:
       matrix:
         include:
           - python-version: "3.7"
-            postgres-version: "11"
+            postgres-version: "10"
 
-          - python-version: "3.11"
-            postgres-version: "15"
+          - python-version: "3.10"
+            postgres-version: "14"
 
     services:
       postgres:
@@ -498,38 +305,13 @@ jobs:
           --health-retries 5
 
     steps:
-      - uses: actions/checkout@v3
-      - name: Add PostgreSQL apt repository
-        # We need a version of pg_dump that can handle the version of
-        # PostgreSQL being tested against. The Ubuntu package repository lags
-        # behind new releases, so we have to use the PostreSQL apt repository.
-        # Steps taken from https://www.postgresql.org/download/linux/ubuntu/
-        run: |
-          sudo sh -c 'echo "deb http://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main" > /etc/apt/sources.list.d/pgdg.list'
-          wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | sudo apt-key add -
-          sudo apt-get update
-      - run: sudo apt-get -qq install xmlsec1 postgresql-client
+      - uses: actions/checkout@v2
+      - run: sudo apt-get -qq install xmlsec1
       - uses: matrix-org/setup-python-poetry@v1
         with:
           python-version: ${{ matrix.python-version }}
-          poetry-version: "1.3.2"
           extras: "postgres"
       - run: .ci/scripts/test_synapse_port_db.sh
-        id: run_tester_script
-        env:
-          PGHOST: localhost
-          PGUSER: postgres
-          PGPASSWORD: postgres
-          PGDATABASE: postgres
-      - name: "Upload schema differences"
-        uses: actions/upload-artifact@v3
-        if: ${{ failure() && !cancelled() && steps.run_tester_script.outcome == 'failure' }}
-        with:
-          name: Schema dumps
-          path: |
-            unported.sql
-            ported.sql
-            schema_diff
 
   complement:
     if: "${{ !failure() && !cancelled() }}"
@@ -550,77 +332,34 @@ jobs:
             database: Postgres
 
     steps:
-      - name: Run actions/checkout@v3 for synapse
-        uses: actions/checkout@v3
+      - name: Run actions/checkout@v2 for synapse
+        uses: actions/checkout@v2
         with:
           path: synapse
 
-      - name: Install Rust
-        uses: dtolnay/rust-toolchain@1.60.0
-      - uses: Swatinem/rust-cache@v2
-
-      - uses: actions/setup-go@v4
-
       - name: Prepare Complement's Prerequisites
         run: synapse/.ci/scripts/setup_complement_prerequisites.sh
 
       - run: |
           set -o pipefail
-          COMPLEMENT_DIR=`pwd`/complement synapse/scripts-dev/complement.sh -json 2>&1 | synapse/.ci/scripts/gotestfmt
+          POSTGRES=${{ (matrix.database == 'Postgres') && 1 || '' }} WORKERS=${{ (matrix.arrangement == 'workers') && 1 || '' }} COMPLEMENT_DIR=`pwd`/complement synapse/scripts-dev/complement.sh -json 2>&1 | gotestfmt
         shell: bash
-        env:
-          POSTGRES: ${{ (matrix.database == 'Postgres') && 1 || '' }}
-          WORKERS: ${{ (matrix.arrangement == 'workers') && 1 || '' }}
         name: Run Complement Tests
 
-  cargo-test:
-    if: ${{ needs.changes.outputs.rust == 'true' }}
-    runs-on: ubuntu-latest
-    needs:
-      - linting-done
-      - changes
-
-    steps:
-      - uses: actions/checkout@v3
-
-      - name: Install Rust
-        uses: dtolnay/rust-toolchain@1.60.0
-      - uses: Swatinem/rust-cache@v2
-
-      - run: cargo test
-
-  # We want to ensure that the cargo benchmarks still compile, which requires a
-  # nightly compiler.
-  cargo-bench:
-    if: ${{ needs.changes.outputs.rust == 'true' }}
-    runs-on: ubuntu-latest
-    needs:
-      - linting-done
-      - changes
-
-    steps:
-      - uses: actions/checkout@v3
-
-      - name: Install Rust
-        uses: dtolnay/rust-toolchain@master
-        with:
-            toolchain: nightly-2022-12-01
-      - uses: Swatinem/rust-cache@v2
-
-      - run: cargo bench --no-run
-
   # a job which marks all the other jobs as complete, thus allowing PRs to be merged.
   tests-done:
     if: ${{ always() }}
     needs:
+      - check-sampleconfig
+      - lint
+      - lint-crlf
+      - lint-newsfile
       - trial
       - trial-olddeps
       - sytest
       - export-data
       - portdb
       - complement
-      - cargo-test
-      - cargo-bench
     runs-on: ubuntu-latest
     steps:
       - uses: matrix-org/done-action@v2
@@ -628,8 +367,5 @@ jobs:
           needs: ${{ toJSON(needs) }}
 
           # The newsfile lint may be skipped on non PR builds
-          # Cargo test is skipped if there is no changes on Rust code
-          skippable: |
+          skippable:
             lint-newsfile
-            cargo-test
-            cargo-bench

.github/workflows/triage-incoming.yml

@@ -1,15 +0,0 @@
-name: Move new issues into the issue triage board
-
-on:
-  issues:
-    types: [ opened ]
-
-jobs:
-  triage:
-    uses: matrix-org/backend-meta/.github/workflows/triage-incoming.yml@v2
-    with: 
-      project_id: 'PVT_kwDOAIB0Bs4AFDdZ'
-      content_id: ${{ github.event.issue.node_id }}
-    secrets: 
-      github_access_token: ${{ secrets.ELEMENT_BOT_TOKEN }}
-          

.github/workflows/triage_labelled.yml

@@ -1,44 +0,0 @@
-name: Move labelled issues to correct projects
-
-on:
-  issues:
-    types: [ labeled ]
-
-jobs:
-  move_needs_info:
-    name: Move X-Needs-Info on the triage board
-    runs-on: ubuntu-latest
-    if: >
-      contains(github.event.issue.labels.*.name, 'X-Needs-Info')
-    steps:
-      - uses: actions/add-to-project@main
-        id: add_project
-        with:
-          project-url: "https://github.com/orgs/matrix-org/projects/67"
-          github-token: ${{ secrets.ELEMENT_BOT_TOKEN }}
-      - name: Set status
-        env:
-          GITHUB_TOKEN: ${{ secrets.ELEMENT_BOT_TOKEN }}
-        run: |
-          gh api graphql -f query='
-          mutation(
-              $project: ID!
-              $item: ID!
-              $fieldid: ID!
-              $columnid: String!
-            )  {
-            updateProjectV2ItemFieldValue(
-              input: {
-               projectId: $project
-                itemId: $item
-                fieldId: $fieldid
-                value: { 
-                  singleSelectOptionId: $columnid
-                  }
-              }
-            ) {
-              projectV2Item {
-                id
-              }
-            }
-          }' -f project="PVT_kwDOAIB0Bs4AFDdZ" -f item=${{ steps.add_project.outputs.itemId }} -f fieldid="PVTSSF_lADOAIB0Bs4AFDdZzgC6ZA4" -f columnid=ba22e43c --silent

.github/workflows/twisted_trunk.yml

@@ -5,69 +5,35 @@ on:
     - cron: 0 8 * * *
 
   workflow_dispatch:
-    inputs:
-      twisted_ref:
-        description: Commit, branch or tag to checkout from upstream Twisted.
-        required: false
-        default: 'trunk'
-        type: string
-
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}
   cancel-in-progress: true
 
 jobs:
-  check_repo:
-    # Prevent this workflow from running on any fork of Synapse other than matrix-org/synapse, as it is
-    # only useful to the Synapse core team.
-    # All other workflow steps depend on this one, thus if 'should_run_workflow' is not 'true', the rest
-    # of the workflow will be skipped as well.
-    if: github.repository == 'matrix-org/synapse'
-    runs-on: ubuntu-latest
-    outputs:
-      should_run_workflow: ${{ steps.check_condition.outputs.should_run_workflow }}
-    steps:
-      - id: check_condition
-        run: echo "should_run_workflow=${{ github.repository == 'matrix-org/synapse' }}" >> "$GITHUB_OUTPUT"
-
   mypy:
-    needs: check_repo
-    if: needs.check_repo.outputs.should_run_workflow == 'true'
     runs-on: ubuntu-latest
 
     steps:
-      - uses: actions/checkout@v3
-
-      - name: Install Rust
-        uses: dtolnay/rust-toolchain@stable
-      - uses: Swatinem/rust-cache@v2
-
+      - uses: actions/checkout@v2
       - uses: matrix-org/setup-python-poetry@v1
         with:
           python-version: "3.x"
           extras: "all"
       - run: |
           poetry remove twisted
-          poetry add --extras tls git+https://github.com/twisted/twisted.git#${{ inputs.twisted_ref }}
+          poetry add --extras tls git+https://github.com/twisted/twisted.git#trunk
           poetry install --no-interaction --extras "all test"
       - name: Remove warn_unused_ignores from mypy config
         run: sed '/warn_unused_ignores = True/d' -i mypy.ini
       - run: poetry run mypy
 
   trial:
-    needs: check_repo
-    if: needs.check_repo.outputs.should_run_workflow == 'true'
     runs-on: ubuntu-latest
 
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v2
       - run: sudo apt-get -qq install xmlsec1
-
-      - name: Install Rust
-        uses: dtolnay/rust-toolchain@stable
-      - uses: Swatinem/rust-cache@v2
-
       - uses: matrix-org/setup-python-poetry@v1
         with:
           python-version: "3.x"
@@ -92,8 +58,6 @@ jobs:
           || true
 
   sytest:
-    needs: check_repo
-    if: needs.check_repo.outputs.should_run_workflow == 'true'
     runs-on: ubuntu-latest
     container:
       image: matrixdotorg/sytest-synapse:buster
@@ -101,12 +65,7 @@ jobs:
         - ${{ github.workspace }}:/src
 
     steps:
-      - uses: actions/checkout@v3
-
-      - name: Install Rust
-        uses: dtolnay/rust-toolchain@stable
-      - uses: Swatinem/rust-cache@v2
-
+      - uses: actions/checkout@v2
       - name: Patch dependencies
         # Note: The poetry commands want to create a virtualenv in /src/.venv/,
         #       but the sytest-synapse container expects it to be in /venv/.
@@ -129,7 +88,7 @@ jobs:
         if: ${{ always() }}
         run: /sytest/scripts/tap_to_gha.pl /logs/results.tap
       - name: Upload SyTest logs
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v2
         if: ${{ always() }}
         with:
           name: Sytest Logs - ${{ job.status }} - (${{ join(matrix.*, ', ') }})
@@ -138,8 +97,7 @@ jobs:
             /logs/**/*.log*
 
   complement:
-    needs: check_repo
-    if: "!failure() && !cancelled() && needs.check_repo.outputs.should_run_workflow == 'true'"
+    if: "${{ !failure() && !cancelled() }}"
     runs-on: ubuntu-latest
 
     strategy:
@@ -156,13 +114,11 @@ jobs:
             database: Postgres
 
     steps:
-      - name: Run actions/checkout@v3 for synapse
-        uses: actions/checkout@v3
+      - name: Run actions/checkout@v2 for synapse
+        uses: actions/checkout@v2
         with:
           path: synapse
 
-      - uses: actions/setup-go@v4
-
       - name: Prepare Complement's Prerequisites
         run: synapse/.ci/scripts/setup_complement_prerequisites.sh
 
@@ -171,22 +127,23 @@ jobs:
         run: |
           set -x
           DEBIAN_FRONTEND=noninteractive sudo apt-get install -yqq python3 pipx
-          pipx install poetry==1.3.2
+          pipx install poetry==1.1.14
 
           poetry remove -n twisted
           poetry add -n --extras tls git+https://github.com/twisted/twisted.git#trunk
           poetry lock --no-update
+          # NOT IN 1.1.14 poetry lock --check
         working-directory: synapse
 
       - run: |
           set -o pipefail
-          TEST_ONLY_SKIP_DEP_HASH_VERIFICATION=1 POSTGRES=${{ (matrix.database == 'Postgres') && 1 || '' }} WORKERS=${{ (matrix.arrangement == 'workers') && 1 || '' }} COMPLEMENT_DIR=`pwd`/complement synapse/scripts-dev/complement.sh -json 2>&1 | synapse/.ci/scripts/gotestfmt
+          TEST_ONLY_SKIP_DEP_HASH_VERIFICATION=1 POSTGRES=${{ (matrix.database == 'Postgres') && 1 || '' }} WORKERS=${{ (matrix.arrangement == 'workers') && 1 || '' }} COMPLEMENT_DIR=`pwd`/complement synapse/scripts-dev/complement.sh -json 2>&1 | gotestfmt
         shell: bash
         name: Run Complement Tests
 
   # open an issue if the build fails, so we know about it.
   open-issue:
-    if: failure() && needs.check_repo.outputs.should_run_workflow == 'true'
+    if: failure()
     needs:
       - mypy
       - trial
@@ -196,8 +153,8 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-      - uses: actions/checkout@v3
-      - uses: JasonEtco/create-an-issue@e27dddc79c92bc6e4562f268fffa5ed752639abd # v2.9.1
+      - uses: actions/checkout@v2
+      - uses: JasonEtco/create-an-issue@5d9504915f79f9cc6d791934b8ef34f2353dd74d # v2.5.0, 2020-12-06
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         with:

.gitignore

@@ -15,10 +15,8 @@ _trial_temp*/
 .DS_Store
 __pycache__/
 
-# We do want poetry, cargo and flake lockfiles.
+# We do want the poetry lockfile.
 !poetry.lock
-!Cargo.lock
-!flake.lock
 
 # stuff that is likely to exist when you run a server locally
 /*.db
@@ -37,10 +35,6 @@ __pycache__/
 
 # For direnv users
 /.envrc
-.direnv/
-
-# For nix/devenv users
-.devenv/
 
 # IDEs
 /.idea/
@@ -57,7 +51,6 @@ __pycache__/
 /coverage.*
 /dist/
 /docs/build/
-/dev-docs/_build/
 /htmlcov
 /pip-wheel-metadata/
 
@@ -66,14 +59,4 @@ book/
 
 # complement
 /complement-*
-/main.tar.gz
-
-# rust
-/target/
-/synapse/*.so
-
-# Poetry will create a setup.py, which we don't want to include.
-/setup.py
-
-# Don't include users' poetry configs
-/poetry.toml
+/master.tar.gz

.rustfmt.toml

@@ -1 +0,0 @@
-group_imports = "StdExternalCrate"

Cargo.lock

@@ -1,474 +0,0 @@
-# This file is automatically @generated by Cargo.
-# It is not intended for manual editing.
-version = 3
-
-[[package]]
-name = "aho-corasick"
-version = "1.0.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "43f6cb1bf222025340178f382c426f13757b2960e89779dfcb319c32542a5a41"
-dependencies = [
- "memchr",
-]
-
-[[package]]
-name = "anyhow"
-version = "1.0.71"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9c7d0618f0e0b7e8ff11427422b64564d5fb0be1940354bfe2e0529b18a9d9b8"
-
-[[package]]
-name = "arc-swap"
-version = "1.5.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "983cd8b9d4b02a6dc6ffa557262eb5858a27a0038ffffe21a0f133eaa819a164"
-
-[[package]]
-name = "autocfg"
-version = "1.1.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa"
-
-[[package]]
-name = "bitflags"
-version = "1.3.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a"
-
-[[package]]
-name = "blake2"
-version = "0.10.6"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "46502ad458c9a52b69d4d4d32775c788b7a1b85e8bc9d482d92250fc0e3f8efe"
-dependencies = [
- "digest",
-]
-
-[[package]]
-name = "block-buffer"
-version = "0.10.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "69cce20737498f97b993470a6e536b8523f0af7892a4f928cceb1ac5e52ebe7e"
-dependencies = [
- "generic-array",
-]
-
-[[package]]
-name = "cfg-if"
-version = "1.0.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
-
-[[package]]
-name = "crypto-common"
-version = "0.1.6"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3"
-dependencies = [
- "generic-array",
- "typenum",
-]
-
-[[package]]
-name = "digest"
-version = "0.10.5"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "adfbc57365a37acbd2ebf2b64d7e69bb766e2fea813521ed536f5d0520dcf86c"
-dependencies = [
- "block-buffer",
- "crypto-common",
- "subtle",
-]
-
-[[package]]
-name = "generic-array"
-version = "0.14.6"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bff49e947297f3312447abdca79f45f4738097cc82b06e72054d2223f601f1b9"
-dependencies = [
- "typenum",
- "version_check",
-]
-
-[[package]]
-name = "hex"
-version = "0.4.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70"
-
-[[package]]
-name = "indoc"
-version = "1.0.7"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "adab1eaa3408fb7f0c777a73e7465fd5656136fc93b670eb6df3c88c2c1344e3"
-
-[[package]]
-name = "itoa"
-version = "1.0.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4217ad341ebadf8d8e724e264f13e593e0648f5b3e94b3896a5df283be015ecc"
-
-[[package]]
-name = "lazy_static"
-version = "1.4.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646"
-
-[[package]]
-name = "libc"
-version = "0.2.135"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "68783febc7782c6c5cb401fbda4de5a9898be1762314da0bb2c10ced61f18b0c"
-
-[[package]]
-name = "lock_api"
-version = "0.4.9"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "435011366fe56583b16cf956f9df0095b405b82d76425bc8981c0e22e60ec4df"
-dependencies = [
- "autocfg",
- "scopeguard",
-]
-
-[[package]]
-name = "log"
-version = "0.4.19"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b06a4cde4c0f271a446782e3eff8de789548ce57dbc8eca9292c27f4a42004b4"
-
-[[package]]
-name = "memchr"
-version = "2.5.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d"
-
-[[package]]
-name = "memoffset"
-version = "0.6.5"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5aa361d4faea93603064a027415f07bd8e1d5c88c9fbf68bf56a285428fd79ce"
-dependencies = [
- "autocfg",
-]
-
-[[package]]
-name = "once_cell"
-version = "1.15.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e82dad04139b71a90c080c8463fe0dc7902db5192d939bd0950f074d014339e1"
-
-[[package]]
-name = "parking_lot"
-version = "0.12.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3742b2c103b9f06bc9fff0a37ff4912935851bee6d36f3c02bcc755bcfec228f"
-dependencies = [
- "lock_api",
- "parking_lot_core",
-]
-
-[[package]]
-name = "parking_lot_core"
-version = "0.9.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "09a279cbf25cb0757810394fbc1e359949b59e348145c643a939a525692e6929"
-dependencies = [
- "cfg-if",
- "libc",
- "redox_syscall",
- "smallvec",
- "windows-sys",
-]
-
-[[package]]
-name = "proc-macro2"
-version = "1.0.52"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1d0e1ae9e836cc3beddd63db0df682593d7e2d3d891ae8c9083d2113e1744224"
-dependencies = [
- "unicode-ident",
-]
-
-[[package]]
-name = "pyo3"
-version = "0.17.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "268be0c73583c183f2b14052337465768c07726936a260f480f0857cb95ba543"
-dependencies = [
- "anyhow",
- "cfg-if",
- "indoc",
- "libc",
- "memoffset",
- "parking_lot",
- "pyo3-build-config",
- "pyo3-ffi",
- "pyo3-macros",
- "unindent",
-]
-
-[[package]]
-name = "pyo3-build-config"
-version = "0.17.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "28fcd1e73f06ec85bf3280c48c67e731d8290ad3d730f8be9dc07946923005c8"
-dependencies = [
- "once_cell",
- "target-lexicon",
-]
-
-[[package]]
-name = "pyo3-ffi"
-version = "0.17.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0f6cb136e222e49115b3c51c32792886defbfb0adead26a688142b346a0b9ffc"
-dependencies = [
- "libc",
- "pyo3-build-config",
-]
-
-[[package]]
-name = "pyo3-log"
-version = "0.8.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c94ff6535a6bae58d7d0b85e60d4c53f7f84d0d0aa35d6a28c3f3e70bfe51444"
-dependencies = [
- "arc-swap",
- "log",
- "pyo3",
-]
-
-[[package]]
-name = "pyo3-macros"
-version = "0.17.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "94144a1266e236b1c932682136dc35a9dee8d3589728f68130c7c3861ef96b28"
-dependencies = [
- "proc-macro2",
- "pyo3-macros-backend",
- "quote",
- "syn 1.0.104",
-]
-
-[[package]]
-name = "pyo3-macros-backend"
-version = "0.17.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c8df9be978a2d2f0cdebabb03206ed73b11314701a5bfe71b0d753b81997777f"
-dependencies = [
- "proc-macro2",
- "quote",
- "syn 1.0.104",
-]
-
-[[package]]
-name = "pythonize"
-version = "0.17.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0f7f0c136f5fbc01868185eef462800e49659eb23acca83b9e884367a006acb6"
-dependencies = [
- "pyo3",
- "serde",
-]
-
-[[package]]
-name = "quote"
-version = "1.0.26"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4424af4bf778aae2051a77b60283332f386554255d722233d09fbfc7e30da2fc"
-dependencies = [
- "proc-macro2",
-]
-
-[[package]]
-name = "redox_syscall"
-version = "0.2.16"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fb5a58c1855b4b6819d59012155603f0b22ad30cad752600aadfcb695265519a"
-dependencies = [
- "bitflags",
-]
-
-[[package]]
-name = "regex"
-version = "1.8.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d0ab3ca65655bb1e41f2a8c8cd662eb4fb035e67c3f78da1d61dffe89d07300f"
-dependencies = [
- "aho-corasick",
- "memchr",
- "regex-syntax",
-]
-
-[[package]]
-name = "regex-syntax"
-version = "0.7.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "436b050e76ed2903236f032a59761c1eb99e1b0aead2c257922771dab1fc8c78"
-
-[[package]]
-name = "ryu"
-version = "1.0.11"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4501abdff3ae82a1c1b477a17252eb69cee9e66eb915c1abaa4f44d873df9f09"
-
-[[package]]
-name = "scopeguard"
-version = "1.1.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd"
-
-[[package]]
-name = "serde"
-version = "1.0.164"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9e8c8cf938e98f769bc164923b06dce91cea1751522f46f8466461af04c9027d"
-dependencies = [
- "serde_derive",
-]
-
-[[package]]
-name = "serde_derive"
-version = "1.0.164"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d9735b638ccc51c28bf6914d90a2e9725b377144fc612c49a611fddd1b631d68"
-dependencies = [
- "proc-macro2",
- "quote",
- "syn 2.0.10",
-]
-
-[[package]]
-name = "serde_json"
-version = "1.0.97"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bdf3bf93142acad5821c99197022e170842cdbc1c30482b98750c688c640842a"
-dependencies = [
- "itoa",
- "ryu",
- "serde",
-]
-
-[[package]]
-name = "smallvec"
-version = "1.10.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a507befe795404456341dfab10cef66ead4c041f62b8b11bbb92bffe5d0953e0"
-
-[[package]]
-name = "subtle"
-version = "2.4.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6bdef32e8150c2a081110b42772ffe7d7c9032b606bc226c8260fd97e0976601"
-
-[[package]]
-name = "syn"
-version = "1.0.104"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4ae548ec36cf198c0ef7710d3c230987c2d6d7bd98ad6edc0274462724c585ce"
-dependencies = [
- "proc-macro2",
- "quote",
- "unicode-ident",
-]
-
-[[package]]
-name = "syn"
-version = "2.0.10"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5aad1363ed6d37b84299588d62d3a7d95b5a5c2d9aad5c85609fda12afaa1f40"
-dependencies = [
- "proc-macro2",
- "quote",
- "unicode-ident",
-]
-
-[[package]]
-name = "synapse"
-version = "0.1.0"
-dependencies = [
- "anyhow",
- "blake2",
- "hex",
- "lazy_static",
- "log",
- "pyo3",
- "pyo3-log",
- "pythonize",
- "regex",
- "serde",
- "serde_json",
-]
-
-[[package]]
-name = "target-lexicon"
-version = "0.12.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c02424087780c9b71cc96799eaeddff35af2bc513278cda5c99fc1f5d026d3c1"
-
-[[package]]
-name = "typenum"
-version = "1.15.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "dcf81ac59edc17cc8697ff311e8f5ef2d99fcbd9817b34cec66f90b6c3dfd987"
-
-[[package]]
-name = "unicode-ident"
-version = "1.0.5"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6ceab39d59e4c9499d4e5a8ee0e2735b891bb7308ac83dfb4e80cad195c9f6f3"
-
-[[package]]
-name = "unindent"
-version = "0.1.10"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "58ee9362deb4a96cef4d437d1ad49cffc9b9e92d202b6995674e928ce684f112"
-
-[[package]]
-name = "version_check"
-version = "0.9.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f"
-
-[[package]]
-name = "windows-sys"
-version = "0.36.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ea04155a16a59f9eab786fe12a4a450e75cdb175f9e0d80da1e17db09f55b8d2"
-dependencies = [
- "windows_aarch64_msvc",
- "windows_i686_gnu",
- "windows_i686_msvc",
- "windows_x86_64_gnu",
- "windows_x86_64_msvc",
-]
-
-[[package]]
-name = "windows_aarch64_msvc"
-version = "0.36.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9bb8c3fd39ade2d67e9874ac4f3db21f0d710bee00fe7cab16949ec184eeaa47"
-
-[[package]]
-name = "windows_i686_gnu"
-version = "0.36.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "180e6ccf01daf4c426b846dfc66db1fc518f074baa793aa7d9b9aaeffad6a3b6"
-
-[[package]]
-name = "windows_i686_msvc"
-version = "0.36.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e2e7917148b2812d1eeafaeb22a97e4813dfa60a3f8f78ebe204bcc88f12f024"
-
-[[package]]
-name = "windows_x86_64_gnu"
-version = "0.36.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4dcd171b8776c41b97521e5da127a2d86ad280114807d0b2ab1e462bc764d9e1"
-
-[[package]]
-name = "windows_x86_64_msvc"
-version = "0.36.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c811ca4a8c853ef420abd8592ba53ddbbac90410fab6903b3e79972a631f7680"

Cargo.toml

@@ -1,5 +0,0 @@
-# We make the whole Synapse folder a workspace so that we can run `cargo`
-# commands from the root (rather than having to cd into rust/).
-
-[workspace]
-members = ["rust"]

README.rst

@@ -2,70 +2,152 @@
 Synapse |support| |development| |documentation| |license| |pypi| |python|
 =========================================================================
 
-Synapse is an open-source `Matrix <https://matrix.org/>`_ homeserver written and
-maintained by the Matrix.org Foundation. We began rapid development in 2014,
-reaching v1.0.0 in 2019. Development on Synapse and the Matrix protocol itself continues
-in earnest today.
-
-Briefly, Matrix is an open standard for communications on the internet, supporting
-federation, encryption and VoIP. Matrix.org has more to say about the `goals of the
-Matrix project <https://matrix.org/docs/guides/introduction>`_, and the `formal specification
-<https://spec.matrix.org/>`_ describes the technical details.
-
 .. contents::
 
-Installing and configuration
-============================
+Introduction
+============
 
-The Synapse documentation describes `how to install Synapse <https://matrix-org.github.io/synapse/latest/setup/installation.html>`_. We recommend using
-`Docker images <https://matrix-org.github.io/synapse/latest/setup/installation.html#docker-images-and-ansible-playbooks>`_ or `Debian packages from Matrix.org
-<https://matrix-org.github.io/synapse/latest/setup/installation.html#matrixorg-packages>`_.
+Matrix is an ambitious new ecosystem for open federated Instant Messaging and
+VoIP.  The basics you need to know to get up and running are:
+
+- Everything in Matrix happens in a room.  Rooms are distributed and do not
+  exist on any single server.  Rooms can be located using convenience aliases
+  like ``#matrix:matrix.org`` or ``#test:localhost:8448``.
+
+- Matrix user IDs look like ``@matthew:matrix.org`` (although in the future
+  you will normally refer to yourself and others using a third party identifier
+  (3PID): email address, phone number, etc rather than manipulating Matrix user IDs)
+
+The overall architecture is::
+
+      client <----> homeserver <=====================> homeserver <----> client
+             https://somewhere.org/_matrix      https://elsewhere.net/_matrix
+
+``#matrix:matrix.org`` is the official support room for Matrix, and can be
+accessed by any client from https://matrix.org/docs/projects/try-matrix-now.html or
+via IRC bridge at irc://irc.libera.chat/matrix.
+
+Synapse is currently in rapid development, but as of version 0.5 we believe it
+is sufficiently stable to be run as an internet-facing service for real usage!
+
+About Matrix
+============
+
+Matrix specifies a set of pragmatic RESTful HTTP JSON APIs as an open standard,
+which handle:
+
+- Creating and managing fully distributed chat rooms with no
+  single points of control or failure
+- Eventually-consistent cryptographically secure synchronisation of room
+  state across a global open network of federated servers and services
+- Sending and receiving extensible messages in a room with (optional)
+  end-to-end encryption
+- Inviting, joining, leaving, kicking, banning room members
+- Managing user accounts (registration, login, logout)
+- Using 3rd Party IDs (3PIDs) such as email addresses, phone numbers,
+  Facebook accounts to authenticate, identify and discover users on Matrix.
+- Placing 1:1 VoIP and Video calls
+
+These APIs are intended to be implemented on a wide range of servers, services
+and clients, letting developers build messaging and VoIP functionality on top
+of the entirely open Matrix ecosystem rather than using closed or proprietary
+solutions. The hope is for Matrix to act as the building blocks for a new
+generation of fully open and interoperable messaging and VoIP apps for the
+internet.
+
+Synapse is a Matrix "homeserver" implementation developed by the matrix.org core
+team, written in Python 3/Twisted.
+
+In Matrix, every user runs one or more Matrix clients, which connect through to
+a Matrix homeserver. The homeserver stores all their personal chat history and
+user account information - much as a mail client connects through to an
+IMAP/SMTP server. Just like email, you can either run your own Matrix
+homeserver and control and own your own communications and history or use one
+hosted by someone else (e.g. matrix.org) - there is no single point of control
+or mandatory service provider in Matrix, unlike WhatsApp, Facebook, Hangouts,
+etc.
+
+We'd like to invite you to join #matrix:matrix.org (via
+https://matrix.org/docs/projects/try-matrix-now.html), run a homeserver, take a look
+at the `Matrix spec <https://matrix.org/docs/spec>`_, and experiment with the
+`APIs <https://matrix.org/docs/api>`_ and `Client SDKs
+<https://matrix.org/docs/projects/try-matrix-now.html#client-sdks>`_.
+
+Thanks for using Matrix!
+
+Support
+=======
+
+For support installing or managing Synapse, please join |room|_ (from a matrix.org
+account if necessary) and ask questions there. We do not use GitHub issues for
+support requests, only for bug reports and feature requests.
+
+Synapse's documentation is `nicely rendered on GitHub Pages <https://matrix-org.github.io/synapse>`_,
+with its source available in |docs|_.
+
+.. |room| replace:: ``#synapse:matrix.org``
+.. _room: https://matrix.to/#/#synapse:matrix.org
+
+.. |docs| replace:: ``docs``
+.. _docs: docs
+
+Synapse Installation
+====================
 
 .. _federation:
 
-Synapse has a variety of `config options
-<https://matrix-org.github.io/synapse/latest/usage/configuration/config_documentation.html>`_
-which can be used to customise its behaviour after installation.
-There are additional details on how to `configure Synapse for federation here
-<https://matrix-org.github.io/synapse/latest/federate.html>`_.
-
-.. _reverse-proxy:
-
-Using a reverse proxy with Synapse
-----------------------------------
-
-It is recommended to put a reverse proxy such as
-`nginx <https://nginx.org/en/docs/http/ngx_http_proxy_module.html>`_,
-`Apache <https://httpd.apache.org/docs/current/mod/mod_proxy_http.html>`_,
-`Caddy <https://caddyserver.com/docs/quick-starts/reverse-proxy>`_,
-`HAProxy <https://www.haproxy.org/>`_ or
-`relayd <https://man.openbsd.org/relayd.8>`_ in front of Synapse. One advantage of
-doing so is that it means that you can expose the default https port (443) to
-Matrix clients without needing to run Synapse with root privileges.
-For information on configuring one, see `the reverse proxy docs
-<https://matrix-org.github.io/synapse/latest/reverse_proxy.html>`_.
-
-Upgrading an existing Synapse
------------------------------
-
-The instructions for upgrading Synapse are in `the upgrade notes`_.
-Please check these instructions as upgrading may require extra steps for some
-versions of Synapse.
-
-.. _the upgrade notes: https://matrix-org.github.io/synapse/develop/upgrade.html
+* For details on how to install synapse, see
+  `Installation Instructions <https://matrix-org.github.io/synapse/latest/setup/installation.html>`_.
+* For specific details on how to configure Synapse for federation see `docs/federate.md <docs/federate.md>`_
 
 
-Platform dependencies
----------------------
+Connecting to Synapse from a client
+===================================
 
-Synapse uses a number of platform dependencies such as Python and PostgreSQL,
-and aims to follow supported upstream versions. See the
-`deprecation policy <https://matrix-org.github.io/synapse/latest/deprecation_policy.html>`_
-for more details.
+The easiest way to try out your new Synapse installation is by connecting to it
+from a web client.
 
+Unless you are running a test instance of Synapse on your local machine, in
+general, you will need to enable TLS support before you can successfully
+connect from a client: see
+`TLS certificates <https://matrix-org.github.io/synapse/latest/setup/installation.html#tls-certificates>`_.
+
+An easy way to get started is to login or register via Element at
+https://app.element.io/#/login or https://app.element.io/#/register respectively.
+You will need to change the server you are logging into from ``matrix.org``
+and instead specify a Homeserver URL of ``https://<server_name>:8448``
+(or just ``https://<server_name>`` if you are using a reverse proxy).
+If you prefer to use another client, refer to our
+`client breakdown <https://matrix.org/docs/projects/clients-matrix>`_.
+
+If all goes well you should at least be able to log in, create a room, and
+start sending messages.
+
+.. _`client-user-reg`:
+
+Registering a new user from a client
+------------------------------------
+
+By default, registration of new users via Matrix clients is disabled. To enable
+it, specify ``enable_registration: true`` in ``homeserver.yaml``. (It is then
+recommended to also set up CAPTCHA - see `<docs/CAPTCHA_SETUP.md>`_.)
+
+Once ``enable_registration`` is set to ``true``, it is possible to register a
+user via a Matrix client.
+
+Your new user name will be formed partly from the ``server_name``, and partly
+from a localpart you specify when you create the account. Your name will take
+the form of::
+
+    @localpart:my.domain.name
+
+(pronounced "at localpart on my dot domain dot name").
+
+As when logging in, you will need to specify a "Custom server".  Specify your
+desired ``localpart`` in the 'User name' box.
 
 Security note
--------------
+=============
 
 Matrix serves raw, user-supplied data in some APIs -- specifically the `content
 repository endpoints`_.
@@ -105,76 +187,30 @@ Following this advice ensures that even if an XSS is found in Synapse, the
 impact to other applications will be minimal.
 
 
-Testing a new installation
-==========================
+Upgrading an existing Synapse
+=============================
 
-The easiest way to try out your new Synapse installation is by connecting to it
-from a web client.
+The instructions for upgrading synapse are in `the upgrade notes`_.
+Please check these instructions as upgrading may require extra steps for some
+versions of synapse.
 
-Unless you are running a test instance of Synapse on your local machine, in
-general, you will need to enable TLS support before you can successfully
-connect from a client: see
-`TLS certificates <https://matrix-org.github.io/synapse/latest/setup/installation.html#tls-certificates>`_.
+.. _the upgrade notes: https://matrix-org.github.io/synapse/develop/upgrade.html
 
-An easy way to get started is to login or register via Element at
-https://app.element.io/#/login or https://app.element.io/#/register respectively.
-You will need to change the server you are logging into from ``matrix.org``
-and instead specify a Homeserver URL of ``https://<server_name>:8448``
-(or just ``https://<server_name>`` if you are using a reverse proxy).
-If you prefer to use another client, refer to our
-`client breakdown <https://matrix.org/docs/projects/clients-matrix>`_.
+.. _reverse-proxy:
 
-If all goes well you should at least be able to log in, create a room, and
-start sending messages.
+Using a reverse proxy with Synapse
+==================================
 
-.. _`client-user-reg`:
+It is recommended to put a reverse proxy such as
+`nginx <https://nginx.org/en/docs/http/ngx_http_proxy_module.html>`_,
+`Apache <https://httpd.apache.org/docs/current/mod/mod_proxy_http.html>`_,
+`Caddy <https://caddyserver.com/docs/quick-starts/reverse-proxy>`_,
+`HAProxy <https://www.haproxy.org/>`_ or
+`relayd <https://man.openbsd.org/relayd.8>`_ in front of Synapse. One advantage of
+doing so is that it means that you can expose the default https port (443) to
+Matrix clients without needing to run Synapse with root privileges.
 
-Registering a new user from a client
-------------------------------------
-
-By default, registration of new users via Matrix clients is disabled. To enable
-it:
-
-1. In the
-   `registration config section <https://matrix-org.github.io/synapse/latest/usage/configuration/config_documentation.html#registration>`_
-   set ``enable_registration: true`` in ``homeserver.yaml``.
-2. Then **either**:
-
-   a. set up a `CAPTCHA <https://matrix-org.github.io/synapse/latest/CAPTCHA_SETUP.html>`_, or
-   b. set ``enable_registration_without_verification: true`` in ``homeserver.yaml``.
-
-We **strongly** recommend using a CAPTCHA, particularly if your homeserver is exposed to
-the public internet. Without it, anyone can freely register accounts on your homeserver.
-This can be exploited by attackers to create spambots targetting the rest of the Matrix
-federation.
-
-Your new user name will be formed partly from the ``server_name``, and partly
-from a localpart you specify when you create the account. Your name will take
-the form of::
-
-    @localpart:my.domain.name
-
-(pronounced "at localpart on my dot domain dot name").
-
-As when logging in, you will need to specify a "Custom server".  Specify your
-desired ``localpart`` in the 'User name' box.
-
-Troubleshooting and support
-===========================
-
-The `Admin FAQ <https://matrix-org.github.io/synapse/latest/usage/administration/admin_faq.html>`_
-includes tips on dealing with some common problems. For more details, see
-`Synapse's wider documentation <https://matrix-org.github.io/synapse/latest/>`_.
-
-For additional support installing or managing Synapse, please ask in the community
-support room |room|_ (from a matrix.org account if necessary). We do not use GitHub
-issues for support requests, only for bug reports and feature requests.
-
-.. |room| replace:: ``#synapse:matrix.org``
-.. _room: https://matrix.to/#/#synapse:matrix.org
-
-.. |docs| replace:: ``docs``
-.. _docs: docs
+For information on configuring one, see `<docs/reverse_proxy.md>`_.
 
 Identity Servers
 ================
@@ -206,15 +242,34 @@ an email address with your account, or send an invite to another user via their
 email address.
 
 
-Development
-===========
+Password reset
+==============
+
+Users can reset their password through their client. Alternatively, a server admin
+can reset a users password using the `admin API <docs/admin_api/user_admin_api.md#reset-password>`_
+or by directly editing the database as shown below.
+
+First calculate the hash of the new password::
+
+    $ ~/synapse/env/bin/hash_password
+    Password:
+    Confirm password:
+    $2a$12$xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
+
+Then update the ``users`` table in the database::
+
+    UPDATE users SET password_hash='$2a$12$xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx'
+        WHERE name='@test:test.com';
+
+
+Synapse Development
+===================
 
-We welcome contributions to Synapse from the community!
 The best place to get started is our
 `guide for contributors <https://matrix-org.github.io/synapse/latest/development/contributing_guide.html>`_.
 This is part of our larger `documentation <https://matrix-org.github.io/synapse/latest>`_, which includes
+information for synapse developers as well as synapse administrators.
 
-information for Synapse developers as well as Synapse administrators.
 Developers might be particularly interested in:
 
 * `Synapse's database schema <https://matrix-org.github.io/synapse/latest/development/database_schema.html>`_,
@@ -225,6 +280,187 @@ Alongside all that, join our developer community on Matrix:
 `#synapse-dev:matrix.org <https://matrix.to/#/#synapse-dev:matrix.org>`_, featuring real humans!
 
 
+Quick start
+-----------
+
+Before setting up a development environment for synapse, make sure you have the
+system dependencies (such as the python header files) installed - see
+`Platform-specific prerequisites <https://matrix-org.github.io/synapse/latest/setup/installation.html#platform-specific-prerequisites>`_.
+
+To check out a synapse for development, clone the git repo into a working
+directory of your choice::
+
+    git clone https://github.com/matrix-org/synapse.git
+    cd synapse
+
+Synapse has a number of external dependencies. We maintain a fixed development
+environment using `Poetry <https://python-poetry.org/>`_. First, install poetry. We recommend::
+
+    pip install --user pipx
+    pipx install poetry
+
+as described `here <https://python-poetry.org/docs/#installing-with-pipx>`_.
+(See `poetry's installation docs <https://python-poetry.org/docs/#installation>`_
+for other installation methods.) Then ask poetry to create a virtual environment
+from the project and install Synapse's dependencies::
+
+    poetry install --extras "all test"
+
+This will run a process of downloading and installing all the needed
+dependencies into a virtual env.
+
+We recommend using the demo which starts 3 federated instances running on ports `8080` - `8082`::
+
+    poetry run ./demo/start.sh
+
+(to stop, you can use ``poetry run ./demo/stop.sh``)
+
+See the `demo documentation <https://matrix-org.github.io/synapse/develop/development/demo.html>`_
+for more information.
+
+If you just want to start a single instance of the app and run it directly::
+
+    # Create the homeserver.yaml config once
+    poetry run synapse_homeserver \
+      --server-name my.domain.name \
+      --config-path homeserver.yaml \
+      --generate-config \
+      --report-stats=[yes|no]
+
+    # Start the app
+    poetry run synapse_homeserver --config-path homeserver.yaml
+
+
+Running the unit tests
+----------------------
+
+After getting up and running, you may wish to run Synapse's unit tests to
+check that everything is installed correctly::
+
+    poetry run trial tests
+
+This should end with a 'PASSED' result (note that exact numbers will
+differ)::
+
+    Ran 1337 tests in 716.064s
+
+    PASSED (skips=15, successes=1322)
+
+For more tips on running the unit tests, like running a specific test or
+to see the logging output, see the `CONTRIBUTING doc <CONTRIBUTING.md#run-the-unit-tests>`_.
+
+
+Running the Integration Tests
+-----------------------------
+
+Synapse is accompanied by `SyTest <https://github.com/matrix-org/sytest>`_,
+a Matrix homeserver integration testing suite, which uses HTTP requests to
+access the API as a Matrix client would. It is able to run Synapse directly from
+the source tree, so installation of the server is not required.
+
+Testing with SyTest is recommended for verifying that changes related to the
+Client-Server API are functioning correctly. See the `SyTest installation
+instructions <https://github.com/matrix-org/sytest#installing>`_ for details.
+
+
+Platform dependencies
+=====================
+
+Synapse uses a number of platform dependencies such as Python and PostgreSQL,
+and aims to follow supported upstream versions. See the
+`<docs/deprecation_policy.md>`_ document for more details.
+
+
+Troubleshooting
+===============
+
+Need help? Join our community support room on Matrix:
+`#synapse:matrix.org <https://matrix.to/#/#synapse:matrix.org>`_
+
+Running out of File Handles
+---------------------------
+
+If synapse runs out of file handles, it typically fails badly - live-locking
+at 100% CPU, and/or failing to accept new TCP connections (blocking the
+connecting client).  Matrix currently can legitimately use a lot of file handles,
+thanks to busy rooms like #matrix:matrix.org containing hundreds of participating
+servers.  The first time a server talks in a room it will try to connect
+simultaneously to all participating servers, which could exhaust the available
+file descriptors between DNS queries & HTTPS sockets, especially if DNS is slow
+to respond. (We need to improve the routing algorithm used to be better than
+full mesh, but as of March 2019 this hasn't happened yet).
+
+If you hit this failure mode, we recommend increasing the maximum number of
+open file handles to be at least 4096 (assuming a default of 1024 or 256).
+This is typically done by editing ``/etc/security/limits.conf``
+
+Separately, Synapse may leak file handles if inbound HTTP requests get stuck
+during processing - e.g. blocked behind a lock or talking to a remote server etc.
+This is best diagnosed by matching up the 'Received request' and 'Processed request'
+log lines and looking for any 'Processed request' lines which take more than
+a few seconds to execute. Please let us know at #synapse:matrix.org if
+you see this failure mode so we can help debug it, however.
+
+Help!! Synapse is slow and eats all my RAM/CPU!
+-----------------------------------------------
+
+First, ensure you are running the latest version of Synapse, using Python 3
+with a PostgreSQL database.
+
+Synapse's architecture is quite RAM hungry currently - we deliberately
+cache a lot of recent room data and metadata in RAM in order to speed up
+common requests. We'll improve this in the future, but for now the easiest
+way to either reduce the RAM usage (at the risk of slowing things down)
+is to set the almost-undocumented ``SYNAPSE_CACHE_FACTOR`` environment
+variable. The default is 0.5, which can be decreased to reduce RAM usage
+in memory constrained enviroments, or increased if performance starts to
+degrade.
+
+However, degraded performance due to a low cache factor, common on
+machines with slow disks, often leads to explosions in memory use due
+backlogged requests. In this case, reducing the cache factor will make
+things worse. Instead, try increasing it drastically. 2.0 is a good
+starting value.
+
+Using `libjemalloc <http://jemalloc.net/>`_ can also yield a significant
+improvement in overall memory use, and especially in terms of giving back
+RAM to the OS. To use it, the library must simply be put in the
+LD_PRELOAD environment variable when launching Synapse. On Debian, this
+can be done by installing the ``libjemalloc1`` package and adding this
+line to ``/etc/default/matrix-synapse``::
+
+    LD_PRELOAD=/usr/lib/x86_64-linux-gnu/libjemalloc.so.1
+
+This can make a significant difference on Python 2.7 - it's unclear how
+much of an improvement it provides on Python 3.x.
+
+If you're encountering high CPU use by the Synapse process itself, you
+may be affected by a bug with presence tracking that leads to a
+massive excess of outgoing federation requests (see `discussion
+<https://github.com/matrix-org/synapse/issues/3971>`_). If metrics
+indicate that your server is also issuing far more outgoing federation
+requests than can be accounted for by your users' activity, this is a
+likely cause. The misbehavior can be worked around by setting
+the following in the Synapse config file:
+
+.. code-block:: yaml
+
+   presence:
+       enabled: false
+
+People can't accept room invitations from me
+--------------------------------------------
+
+The typical failure mode here is that you send an invitation to someone
+to join a room or direct chat, but when they go to accept it, they get an
+error (typically along the lines of "Invalid signature"). They might see
+something like the following in their logs::
+
+    2019-09-11 19:32:04,271 - synapse.federation.transport.server - 288 - WARNING - GET-11752 - authenticate_request failed: 401: Invalid signature for server <server> with key ed25519:a_EqML: Unable to verify signature for <server>
+
+This is normally caused by a misconfiguration in your reverse-proxy. See
+`<docs/reverse_proxy.md>`_ and double-check that your settings are correct.
+
 .. |support| image:: https://img.shields.io/matrix/synapse:matrix.org?label=support&logo=matrix
   :alt: (get support on #synapse:matrix.org)
   :target: https://matrix.to/#/#synapse:matrix.org

book.toml

@@ -34,6 +34,14 @@ additional-css = [
     "docs/website_files/table-of-contents.css",
     "docs/website_files/remove-nav-buttons.css",
     "docs/website_files/indent-section-headers.css",
+    "docs/website_files/version-picker.css",
 ]
-additional-js = ["docs/website_files/table-of-contents.js"]
-theme = "docs/website_files/theme"
+additional-js = [
+    "docs/website_files/table-of-contents.js",
+    "docs/website_files/version-picker.js",
+    "docs/website_files/version.js",
+]
+theme = "docs/website_files/theme"
+
+[preprocessor.schema_versions]
+command = "./scripts-dev/schema_versions.py"

build_rust.py

@@ -1,23 +0,0 @@
-# A build script for poetry that adds the rust extension.
-
-import os
-from typing import Any, Dict
-
-from setuptools_rust import Binding, RustExtension
-
-
-def build(setup_kwargs: Dict[str, Any]) -> None:
-    original_project_dir = os.path.dirname(os.path.realpath(__file__))
-    cargo_toml_path = os.path.join(original_project_dir, "rust", "Cargo.toml")
-
-    extension = RustExtension(
-        target="synapse.synapse_rust",
-        path=cargo_toml_path,
-        binding=Binding.PyO3,
-        py_limited_api=True,
-        # We force always building in release mode, as we can't tell the
-        # difference between using `poetry` in development vs production.
-        debug=False,
-    )
-    setup_kwargs.setdefault("rust_extensions", []).append(extension)
-    setup_kwargs["zip_safe"] = False

changelog.d/15233.misc

@@ -1 +0,0 @@
-Replace `EventContext` fields `prev_group` and `delta_ids` with field `state_group_deltas`.

changelog.d/15680.bugfix

@@ -1 +0,0 @@
-Fix a long-standing bug where media files were served in an unsafe manner. Contributed by @joshqou.

changelog.d/15737.feature

@@ -1 +0,0 @@
-Improve `/messages` response time by avoiding backfill when we already have messages to return.

changelog.d/15743.misc

@@ -1 +0,0 @@
-Regularly try to send transactions to other servers after they failed instead of waiting for a new event to be available before trying.

changelog.d/15748.removal

@@ -1 +0,0 @@
-Remove experimental [MSC2716](https://github.com/matrix-org/matrix-spec-proposals/pull/2716) implementation to incrementally import history into existing rooms.

changelog.d/15755.misc

@@ -1 +0,0 @@
-Fix requesting multiple keys at once over federation, related to [MSC3983](https://github.com/matrix-org/matrix-spec-proposals/pull/3983).

changelog.d/15758.bugfix

@@ -1 +0,0 @@
-Avoid invalidating a cache that was just prefilled.

changelog.d/15770.bugfix

@@ -1 +0,0 @@
-Fix requesting multiple keys at once over federation, related to [MSC3983](https://github.com/matrix-org/matrix-spec-proposals/pull/3983).

changelog.d/15772.doc

@@ -1 +0,0 @@
-Document `looping_call()` functionality that will wait for the given function to finish before scheduling another.

changelog.d/15776.bugfix

@@ -1 +0,0 @@
-Fix joining rooms through aliases where the alias server isn't a real homeserver. Contributed by @tulir @ Beeper.

changelog.d/15781.bugfix

@@ -1 +0,0 @@
-Fix a bug in push rules handling leading to an invalid (per spec) `is_user_mention` rule sent to clients. Also fix wrong rule names for `is_user_mention` and `is_room_mention`.

changelog.d/15788.bugfix

@@ -1 +0,0 @@
-Fix a bug introduced in 1.57.0 where the wrong table would be locked on updating database rows when using SQLite as the database backend.

changelog.d/15805.doc

@@ -1 +0,0 @@
-Fix a typo in the [Admin API](https://matrix-org.github.io/synapse/latest/usage/administration/admin_api/index.html).

changelog.d/15806.misc

@@ -1 +0,0 @@
-Switch from `matrix://` to `matrix-federation://` scheme for internal Synapse routing of outbound federation traffic.

changelog.d/15808.misc

@@ -1 +0,0 @@
-Ignore key request if the device inbox is already big.

contrib/datagrip/README.md

@@ -1,28 +0,0 @@
-# Schema symlinks
-
-This directory contains symlinks to the latest dump of the postgres full schema. This is useful to have, as it allows IDEs to understand our schema and provide autocomplete, linters, inspections, etc.
-
-In particular, the DataGrip functionality in IntelliJ's products seems to only consider files called `*.sql` when defining a schema from DDL; `*.sql.postgres` will be ignored. To get around this we symlink those files to ones ending in `.sql`. We've chosen to ignore the `.sql.sqlite` schema dumps here, as they're not intended for production use (and are much quicker to test against).
-
-## Example
-![](datagrip-aware-of-schema.png)
-
-## Caveats
-
-- Doesn't include temporary tables created ad-hoc by Synapse.
-- Postgres only. IDEs will likely be confused by SQLite-specific queries.
-- Will not include migrations created after the latest schema dump.
-- Symlinks might confuse checkouts on Windows systems.
-
-## Instructions
-
-### Jetbrains IDEs with DataGrip plugin
-
-- View -> Tool Windows -> Database
-- `+` Icon -> DDL Data Source
-- Pick a name, e.g. `Synapse schema dump`
-- Under sources, click `+`.
-- Add an entry with Path pointing to this directory, and dialect set to PostgreSQL.
-- OK, and OK.
-- IDE should now be aware of the schema.
-- Try control-clicking on a table name in a bit of SQL e.g. in `_get_forgotten_rooms_for_user_txn`.

contrib/datagrip/common.sql

@@ -1 +0,0 @@
-../../synapse/storage/schema/common/full_schemas/72/full.sql.postgres

contrib/datagrip/main.sql

@@ -1 +0,0 @@
-../../synapse/storage/schema/main/full_schemas/72/full.sql.postgres

contrib/datagrip/schema_version.sql

@@ -1 +0,0 @@
-../../synapse/storage/schema/common/schema_version.sql

contrib/datagrip/state.sql

@@ -1 +0,0 @@
-../../synapse/storage/schema/state/full_schemas/72/full.sql.postgres

contrib/docker_compose_workers/README.md

@@ -68,12 +68,7 @@ redis:
   enabled: true
   host: redis
   port: 6379
-  # dbid:  <redis_logical_db_id>
   # password: <secret_password>  
-  # use_tls: True
-  # certificate_file: <path_to_certificate>
-  # private_key_file: <path_to_private_key>
-  # ca_file: <path_to_ca_certificate>
 ```
 
 This assumes that your Redis service is called `redis` in your Docker Compose file.
@@ -99,6 +94,20 @@ worker_replication_host: synapse
 worker_replication_http_port: 9093
 ```
 
+### Add Workers to `instance_map`
+
+Locate the `instance_map` section of your `homeserver.yaml` and populate it with your workers:
+
+```yaml
+instance_map:
+  synapse-generic-worker-1:        # The worker_name setting in your worker configuration file
+    host: synapse-generic-worker-1 # The name of the worker service in your Docker Compose file
+    port: 8034                     # The port assigned to the replication listener in your worker config file
+  synapse-federation-sender-1:
+    host: synapse-federation-sender-1
+    port: 8034
+```
+
 ### Configure Federation Senders
 
 This section is applicable if you are using Federation senders (synapse.app.federation_sender). Locate the `send_federation` and `federation_sender_instances` settings in your `homeserver.yaml` and configure them:
@@ -113,4 +122,4 @@ federation_sender_instances:
 
 ## Other Worker types
 
-Using the concepts shown here it is possible to create other worker types in Docker Compose. See the [Workers](https://matrix-org.github.io/synapse/latest/workers.html#available-worker-applications) documentation for a list of available workers.
+Using the concepts shown here it is possible to create other worker types in Docker Compose. See the [Workers](https://matrix-org.github.io/synapse/latest/workers.html#available-worker-applications) documentation for a list of available workers.

contrib/docker_compose_workers/workers/synapse-federation-sender-1.yaml

@@ -5,4 +5,10 @@ worker_name: synapse-federation-sender-1
 worker_replication_host: synapse
 worker_replication_http_port: 9093
 
+worker_listeners:
+  - type: http
+    port: 8034
+    resources:
+      - names: [replication]
+
 worker_log_config: /data/federation_sender.log.config

contrib/docker_compose_workers/workers/synapse-generic-worker-1.yaml

@@ -6,6 +6,10 @@ worker_replication_host: synapse
 worker_replication_http_port: 9093
 
 worker_listeners:
+  - type: http
+    port: 8034
+    resources:
+      - names: [replication]
   - type: http
     port: 8081
     x_forwarded: true

contrib/lnav/README.md

@@ -1,47 +0,0 @@
-# `lnav` config for Synapse logs
-
-[lnav](https://lnav.org/) is a log-viewing tool. It is particularly useful when 
-you need to interleave multiple log files, or for exploring a large log file
-with regex filters. The downside is that it is not as ubiquitous as tools like
-`less`, `grep`, etc.
-
-This directory contains an `lnav` [log format definition](
-    https://docs.lnav.org/en/v0.10.1/formats.html#defining-a-new-format
-) for Synapse logs as
-emitted by Synapse with the default [logging configuration](
-    https://matrix-org.github.io/synapse/latest/usage/configuration/config_documentation.html#log_config
-). It supports lnav 0.10.1 because that's what's packaged by my distribution.
-
-This should allow lnav:
-
-- to interpret timestamps, allowing log interleaving;
-- to interpret log severity levels, allowing colouring by log level(!!!);
-- to interpret request IDs, allowing you to skip through a specific request; and
-- to highlight room, event and user IDs in logs.
-
-See also https://gist.github.com/benje/e2ab750b0a81d11920d83af637d289f7 for a
- similar example.
-
-## Example
-
-[![asciicast](https://asciinema.org/a/556133.svg)](https://asciinema.org/a/556133)
-
-## Tips
-
-- `lnav -i /path/to/synapse/checkout/contrib/lnav/synapse-log-format.json`
-- `lnav my_synapse_log_file` or `lnav synapse_log_files.*`, etc.
-- `lnav --help` for CLI help.
-
-Within lnav itself:
-
-- `?` for help within lnav itself.
-- `q` to quit.
-- `/` to search a-la `less` and `vim`, then `n` and `N` to continue searching 
-  down and up.
-- Use `o` and `O` to skip through logs based on the request ID (`POST-1234`, or
-  else the value of the [`request_id_header`](
-    https://matrix-org.github.io/synapse/latest/usage/configuration/config_documentation.html?highlight=request_id_header#listeners
-  ) header). This may get confused if the same request ID is repeated among 
-  multiple files or process restarts.
-- ???
-- Profit

contrib/lnav/synapse-log-format.json

@@ -1,67 +0,0 @@
-{
-  "$schema": "https://lnav.org/schemas/format-v1.schema.json",
-  "synapse": {
-    "title": "Synapse logs",
-    "description": "Logs output by Synapse, a Matrix homesever, under its default logging config.",
-    "regex": {
-      "log": {
-        "pattern": ".*(?<timestamp>\\d{4}-\\d{2}-\\d{2} \\d{2}:\\d{2}:\\d{2},\\d{3}) - (?<logger>.+) - (?<lineno>\\d+) - (?<level>\\w+) - (?<context>.+) - (?<body>.*)"
-      }
-    },
-    "json": false,
-    "timestamp-field": "timestamp",
-    "timestamp-format": [
-      "%Y-%m-%d %H:%M:%S,%L"
-    ],
-    "level-field": "level",
-    "body-field": "body",
-    "opid-field": "context",
-    "level": {
-      "critical": "CRITICAL",
-      "error": "ERROR",
-      "warning": "WARNING",
-      "info": "INFO",
-      "debug": "DEBUG"
-    },
-    "sample": [
-      {
-        "line": "my-matrix-server-generic-worker-4 | 2023-01-27 09:47:09,818 - synapse.replication.tcp.client - 381 - ERROR - PUT-32992 - Timed out waiting for stream receipts",
-        "level": "error"
-      },
-      {
-        "line": "my-matrix-server-federation-sender-1 | 2023-01-25 20:56:20,995 - synapse.http.matrixfederationclient - 709 - WARNING - federation_transaction_transmission_loop-3 - {PUT-O-3} [example.com] Request failed: PUT matrix-federation://example.com/_matrix/federation/v1/send/1674680155797: HttpResponseException('403: Forbidden')",
-        "level": "warning"
-      },
-      {
-        "line": "my-matrix-server  | 2023-01-25 20:55:54,433 - synapse.storage.databases - 66 - INFO - main - [database config 'master']: Checking database server",
-        "level": "info"
-      },
-      {
-        "line": "my-matrix-server  | 2023-01-26 15:08:40,447 - synapse.access.http.8008 - 460 - INFO - PUT-74929 - 0.0.0.0 - 8008 - {@alice:example.com} Processed request: 0.011sec/0.000sec (0.000sec, 0.000sec) (0.001sec/0.008sec/3) 2B 200 \"PUT /_matrix/client/r0/user/%40alice%3Atexample.com/account_data/im.vector.setting.breadcrumbs HTTP/1.0\" \"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Element/1.11.20 Chrome/108.0.5359.179 Electron/22.0.3 Safari/537.36\" [0 dbevts]",
-        "level": "info"
-      }
-    ],
-    "highlights": {
-      "user_id": {
-        "pattern": "(@|%40)[^:% ]+(:|%3A)[\\[\\]0-9a-zA-Z.\\-:]+(:\\d{1,5})?(?<!:)",
-        "underline": true
-      },
-      "room_id": {
-        "pattern": "(!|%21)[^:% ]+(:|%3A)[\\[\\]0-9a-zA-Z.\\-:]+(:\\d{1,5})?(?<!:)",
-        "underline": true
-      },
-      "room_alias": {
-        "pattern": "(#|%23)[^:% ]+(:|%3A)[\\[\\]0-9a-zA-Z.\\-:]+(:\\d{1,5})?(?<!:)",
-        "underline": true
-      },
-      "event_id_v1_v2": {
-        "pattern": "(\\$|%25)[^:% ]+(:|%3A)[\\[\\]0-9a-zA-Z.\\-:]+(:\\d{1,5})?(?<!:)",
-        "underline": true
-      },
-      "event_id_v3_plus": {
-        "pattern": "(\\$|%25)([A-Za-z0-9+/_]|-){43}",
-        "underline": true
-      }
-    }
-  }
-}

contrib/prometheus/synapse-v1.rules

@@ -0,0 +1,21 @@
+synapse_federation_transaction_queue_pendingEdus:total = sum(synapse_federation_transaction_queue_pendingEdus or absent(synapse_federation_transaction_queue_pendingEdus)*0)
+synapse_federation_transaction_queue_pendingPdus:total = sum(synapse_federation_transaction_queue_pendingPdus or absent(synapse_federation_transaction_queue_pendingPdus)*0)
+
+synapse_http_server_request_count:method{servlet=""} = sum(synapse_http_server_request_count) by (method)
+synapse_http_server_request_count:servlet{method=""} = sum(synapse_http_server_request_count) by (servlet)
+
+synapse_http_server_request_count:total{servlet=""} = sum(synapse_http_server_request_count:by_method) by (servlet)
+
+synapse_cache:hit_ratio_5m = rate(synapse_util_caches_cache:hits[5m]) / rate(synapse_util_caches_cache:total[5m])
+synapse_cache:hit_ratio_30s = rate(synapse_util_caches_cache:hits[30s]) / rate(synapse_util_caches_cache:total[30s])
+
+synapse_federation_client_sent{type="EDU"} = synapse_federation_client_sent_edus + 0
+synapse_federation_client_sent{type="PDU"} = synapse_federation_client_sent_pdu_destinations:count + 0
+synapse_federation_client_sent{type="Query"} = sum(synapse_federation_client_sent_queries) by (job)
+
+synapse_federation_server_received{type="EDU"} = synapse_federation_server_received_edus + 0
+synapse_federation_server_received{type="PDU"} = synapse_federation_server_received_pdus + 0
+synapse_federation_server_received{type="Query"} = sum(synapse_federation_server_received_queries) by (job)
+
+synapse_federation_transaction_queue_pending{type="EDU"} = synapse_federation_transaction_queue_pending_edus + 0
+synapse_federation_transaction_queue_pending{type="PDU"} = synapse_federation_transaction_queue_pending_pdus + 0

contrib/prometheus/synapse-v2.rules

@@ -1,20 +1,37 @@
 groups:
 - name: synapse
   rules:
+  - record: "synapse_federation_transaction_queue_pendingEdus:total"
+    expr: "sum(synapse_federation_transaction_queue_pendingEdus or absent(synapse_federation_transaction_queue_pendingEdus)*0)"
+  - record: "synapse_federation_transaction_queue_pendingPdus:total"
+    expr:   "sum(synapse_federation_transaction_queue_pendingPdus or absent(synapse_federation_transaction_queue_pendingPdus)*0)"
+  - record: 'synapse_http_server_request_count:method'
+    labels:
+      servlet: ""
+    expr: "sum(synapse_http_server_request_count) by (method)"
+  - record: 'synapse_http_server_request_count:servlet'
+    labels:
+      method: ""
+    expr: 'sum(synapse_http_server_request_count) by (servlet)'
+
+  - record: 'synapse_http_server_request_count:total'
+    labels:
+      servlet: ""
+    expr: 'sum(synapse_http_server_request_count:by_method) by (servlet)'
+
+  - record: 'synapse_cache:hit_ratio_5m'
+    expr: 'rate(synapse_util_caches_cache:hits[5m]) / rate(synapse_util_caches_cache:total[5m])'
+  - record: 'synapse_cache:hit_ratio_30s'
+    expr: 'rate(synapse_util_caches_cache:hits[30s]) / rate(synapse_util_caches_cache:total[30s])'
 
-  ###
-  ### Prometheus Console Only
-  ### The following rules are only needed if you use the Prometheus Console
-  ### in contrib/prometheus/consoles/synapse.html
-  ###
   - record: 'synapse_federation_client_sent'
     labels:
       type: "EDU"
-    expr: 'synapse_federation_client_sent_edus_total + 0'
+    expr: 'synapse_federation_client_sent_edus + 0'
   - record: 'synapse_federation_client_sent'
     labels:
       type: "PDU"
-    expr: 'synapse_federation_client_sent_pdu_destinations_count_total + 0'
+    expr: 'synapse_federation_client_sent_pdu_destinations:count + 0'
   - record: 'synapse_federation_client_sent'
     labels:
       type: "Query"
@@ -23,11 +40,11 @@ groups:
   - record: 'synapse_federation_server_received'
     labels:
       type: "EDU"
-    expr: 'synapse_federation_server_received_edus_total + 0'
+    expr: 'synapse_federation_server_received_edus + 0'
   - record: 'synapse_federation_server_received'
     labels:
       type: "PDU"
-    expr: 'synapse_federation_server_received_pdus_total + 0'
+    expr: 'synapse_federation_server_received_pdus + 0'
   - record: 'synapse_federation_server_received'
     labels:
       type: "Query"
@@ -41,34 +58,21 @@ groups:
     labels:
       type: "PDU"
     expr: 'synapse_federation_transaction_queue_pending_pdus + 0'
-  ###
-  ### End of 'Prometheus Console Only' rules block
-  ###
 
-
-  ###
-  ### Grafana Only
-  ### The following rules are only needed if you use the Grafana dashboard
-  ### in contrib/grafana/synapse.json
-  ###
   - record: synapse_storage_events_persisted_by_source_type
-    expr: sum without(type, origin_type, origin_entity) (synapse_storage_events_persisted_events_sep_total{origin_type="remote"})
+    expr: sum without(type, origin_type, origin_entity) (synapse_storage_events_persisted_events_sep{origin_type="remote"})
     labels:
       type: remote
   - record: synapse_storage_events_persisted_by_source_type
-    expr: sum without(type, origin_type, origin_entity) (synapse_storage_events_persisted_events_sep_total{origin_entity="*client*",origin_type="local"})
+    expr: sum without(type, origin_type, origin_entity) (synapse_storage_events_persisted_events_sep{origin_entity="*client*",origin_type="local"})
     labels:
       type: local
   - record: synapse_storage_events_persisted_by_source_type
-    expr: sum without(type, origin_type, origin_entity) (synapse_storage_events_persisted_events_sep_total{origin_entity!="*client*",origin_type="local"})
+    expr: sum without(type, origin_type, origin_entity) (synapse_storage_events_persisted_events_sep{origin_entity!="*client*",origin_type="local"})
     labels:
       type: bridges
-
   - record: synapse_storage_events_persisted_by_event_type
-    expr: sum without(origin_entity, origin_type) (synapse_storage_events_persisted_events_sep_total)
-
+    expr: sum without(origin_entity, origin_type) (synapse_storage_events_persisted_events_sep)
   - record: synapse_storage_events_persisted_by_origin
-    expr: sum without(type) (synapse_storage_events_persisted_events_sep_total)
-  ###
-  ### End of 'Grafana Only' rules block
-  ###
+    expr: sum without(type) (synapse_storage_events_persisted_events_sep)
+

contrib/workers-bash-scripts/create-multiple-generic-workers.md

@@ -7,7 +7,7 @@ You can alternatively create multiple worker configuration files with a simple `
 #!/bin/bash
 for i in {1..5}
 do
-cat << EOF > generic_worker$i.yaml
+cat << EOF >> generic_worker$i.yaml
 worker_app: synapse.app.generic_worker
 worker_name: generic_worker$i
 
@@ -18,16 +18,14 @@ worker_replication_http_port: 9093
 worker_listeners:
   - type: http
     port: 808$i
-    x_forwarded: true
     resources:
       - names: [client, federation]
 
 worker_log_config: /etc/matrix-synapse/generic-worker-log.yaml
-#worker_pid_file: DATADIR/generic_worker$i.pid
 EOF
 done
 ```
 
 This would create five generic workers with a unique `worker_name` field in each file and listening on ports 8081-8085.
 
-Customise the script to your needs. Note that `worker_pid_file` is required if `worker_daemonize` is `true`. Uncomment and/or modify the line if needed.
+Customise the script to your needs.

contrib/workers-bash-scripts/create-multiple-stream-writers.md

@@ -8,9 +8,7 @@ It also prints out the example lines for Synapse main configuration file.
 
 Remember to route necessary endpoints directly to a worker associated with it.
 
-If you run the script as-is, it will create workers with the replication listener starting from port 8034 and another, regular http listener starting from 8044. If you don't need all of the stream writers listed in the script, just remove them from the ```STREAM_WRITERS``` array. 
-
-Hint: Note that `worker_pid_file` is required if `worker_daemonize` is `true`. Uncomment and/or modify the line if needed.
+If you run the script as-is, it will create workers with the replication listener starting from port 8034 and another, regular http listener starting from 8044. If you don't need all of the stream writers listed in the script, just remove them from the ```STREAM_WRITERS``` array.
 
 ```sh
 #!/bin/bash
@@ -48,11 +46,9 @@ worker_listeners:
 
   - type: http
     port: $(expr $HTTP_START_PORT + $i)
-    x_forwarded: true
     resources:
       - names: [client]
 
-#worker_pid_file: DATADIR/${STREAM_WRITERS[$i]}.pid
 worker_log_config: /etc/matrix-synapse/stream-writer-log.yaml
 EOF
 HOMESERVER_YAML_INSTANCE_MAP+=$"  ${STREAM_WRITERS[$i]}_stream_writer:
@@ -95,9 +91,7 @@ Simply run the script to create YAML files in the current folder and print out t
 
 ```console
 $ ./create_stream_writers.sh
-```
-You should receive an output similar to the following:
-```console
+
 # Add these lines to your homeserver.yaml.
 # Don't forget to configure your reverse proxy and
 # necessary endpoints to their respective worker.

debian/build_virtualenv

@@ -31,11 +31,12 @@ case $(dpkg-architecture -q DEB_HOST_ARCH) in
 esac
 
 # Manually install Poetry and export a pip-compatible `requirements.txt`
+# We need a Poetry pre-release as the export command is buggy in < 1.2
 TEMP_VENV="$(mktemp -d)"
 python3 -m venv "$TEMP_VENV"
 source "$TEMP_VENV/bin/activate"
 pip install -U pip
-pip install poetry==1.3.2
+pip install poetry==1.2.0b1
 poetry export \
     --extras all \
     --extras test \
@@ -60,7 +61,7 @@ dh_virtualenv \
     --extras="all,systemd,test" \
     --requirements="exported_requirements.txt"
 
-PACKAGE_BUILD_DIR="$(pwd)/debian/matrix-synapse-py3"
+PACKAGE_BUILD_DIR="debian/matrix-synapse-py3"
 VIRTUALENV_DIR="${PACKAGE_BUILD_DIR}${DH_VIRTUALENV_INSTALL_ROOT}/matrix-synapse"
 TARGET_PYTHON="${VIRTUALENV_DIR}/bin/python"
 
@@ -77,14 +78,9 @@ case "$DEB_BUILD_OPTIONS" in
 
         cp -r tests "$tmpdir"
 
-        # To avoid pulling in the unbuilt Synapse in the local directory
-        pushd /
-
         PYTHONPATH="$tmpdir" \
             "${TARGET_PYTHON}" -m twisted.trial --reporter=text -j2 tests
 
-        popd
-
         ;;
 esac
 

debian/changelog

@@ -1,392 +1,3 @@
-matrix-synapse-py3 (1.86.0~rc2) stable; urgency=medium
-
-  * New Synapse release 1.86.0rc2.
-
- -- Synapse Packaging team <packages@matrix.org>  Wed, 14 Jun 2023 12:16:27 +0200
-
-matrix-synapse-py3 (1.86.0~rc1) stable; urgency=medium
-
-  * New Synapse release 1.86.0rc1.
-
- -- Synapse Packaging team <packages@matrix.org>  Tue, 13 Jun 2023 14:30:45 +0200
-
-matrix-synapse-py3 (1.85.2) stable; urgency=medium
-
-  * New Synapse release 1.85.2.
-
- -- Synapse Packaging team <packages@matrix.org>  Thu, 08 Jun 2023 13:04:18 +0100
-
-matrix-synapse-py3 (1.85.1) stable; urgency=medium
-
-  * New Synapse release 1.85.1.
-
- -- Synapse Packaging team <packages@matrix.org>  Wed, 07 Jun 2023 10:51:12 +0100
-
-matrix-synapse-py3 (1.85.0) stable; urgency=medium
-
-  * New Synapse release 1.85.0.
-
- -- Synapse Packaging team <packages@matrix.org>  Tue, 06 Jun 2023 09:39:29 +0100
-
-matrix-synapse-py3 (1.85.0~rc2) stable; urgency=medium
-
-  * New Synapse release 1.85.0rc2.
-
- -- Synapse Packaging team <packages@matrix.org>  Thu, 01 Jun 2023 09:16:18 -0700
-
-matrix-synapse-py3 (1.85.0~rc1) stable; urgency=medium
-
-  * New Synapse release 1.85.0rc1.
-
- -- Synapse Packaging team <packages@matrix.org>  Tue, 30 May 2023 13:56:54 +0100
-
-matrix-synapse-py3 (1.84.1) stable; urgency=medium
-
-  * New Synapse release 1.84.1.
-
- -- Synapse Packaging team <packages@matrix.org>  Fri, 26 May 2023 16:15:30 +0100
-
-matrix-synapse-py3 (1.84.0) stable; urgency=medium
-
-  * New Synapse release 1.84.0.
-
- -- Synapse Packaging team <packages@matrix.org>  Tue, 23 May 2023 10:57:22 +0100
-
-matrix-synapse-py3 (1.84.0~rc1) stable; urgency=medium
-
-  * New Synapse release 1.84.0rc1.
-
- -- Synapse Packaging team <packages@matrix.org>  Tue, 16 May 2023 11:12:02 +0100
-
-matrix-synapse-py3 (1.83.0) stable; urgency=medium
-
-  * New Synapse release 1.83.0.
-
- -- Synapse Packaging team <packages@matrix.org>  Tue, 09 May 2023 18:13:37 +0200
-
-matrix-synapse-py3 (1.83.0~rc1) stable; urgency=medium
-
-  * New Synapse release 1.83.0rc1.
-
- -- Synapse Packaging team <packages@matrix.org>  Tue, 02 May 2023 15:56:38 +0100
-
-matrix-synapse-py3 (1.82.0) stable; urgency=medium
-
-  * New Synapse release 1.82.0.
-
- -- Synapse Packaging team <packages@matrix.org>  Tue, 25 Apr 2023 11:56:06 +0100
-
-matrix-synapse-py3 (1.82.0~rc1) stable; urgency=medium
-
-  * New Synapse release 1.82.0rc1.
-
- -- Synapse Packaging team <packages@matrix.org>  Tue, 18 Apr 2023 09:47:30 +0100
-
-matrix-synapse-py3 (1.81.0) stable; urgency=medium
-
-  * New Synapse release 1.81.0.
-
- -- Synapse Packaging team <packages@matrix.org>  Tue, 11 Apr 2023 14:18:35 +0100
-
-matrix-synapse-py3 (1.81.0~rc2) stable; urgency=medium
-
-  * New Synapse release 1.81.0rc2.
-
- -- Synapse Packaging team <packages@matrix.org>  Thu, 06 Apr 2023 16:07:54 +0100
-
-matrix-synapse-py3 (1.81.0~rc1) stable; urgency=medium
-
-  * New Synapse release 1.81.0rc1.
-
- -- Synapse Packaging team <packages@matrix.org>  Tue, 04 Apr 2023 14:29:03 +0100
-
-matrix-synapse-py3 (1.80.0) stable; urgency=medium
-
-  * New Synapse release 1.80.0.
-
- -- Synapse Packaging team <packages@matrix.org>  Tue, 28 Mar 2023 11:10:33 +0100
-
-matrix-synapse-py3 (1.80.0~rc2) stable; urgency=medium
-
-  * New Synapse release 1.80.0rc2.
-
- -- Synapse Packaging team <packages@matrix.org>  Wed, 22 Mar 2023 08:30:16 -0700
-
-matrix-synapse-py3 (1.80.0~rc1) stable; urgency=medium
-
-  * New Synapse release 1.80.0rc1.
-
- -- Synapse Packaging team <packages@matrix.org>  Tue, 21 Mar 2023 10:56:08 -0700
-
-matrix-synapse-py3 (1.79.0) stable; urgency=medium
-
-  * New Synapse release 1.79.0.
-
- -- Synapse Packaging team <packages@matrix.org>  Tue, 14 Mar 2023 16:14:50 +0100
-
-matrix-synapse-py3 (1.79.0~rc2) stable; urgency=medium
-
-  * New Synapse release 1.79.0rc2.
-
- -- Synapse Packaging team <packages@matrix.org>  Mon, 13 Mar 2023 12:54:21 +0000
-
-matrix-synapse-py3 (1.79.0~rc1) stable; urgency=medium
-
-  * New Synapse release 1.79.0rc1.
-
- -- Synapse Packaging team <packages@matrix.org>  Tue, 07 Mar 2023 12:03:49 +0000
-
-matrix-synapse-py3 (1.78.0) stable; urgency=medium
-
-  * New Synapse release 1.78.0.
-
- -- Synapse Packaging team <packages@matrix.org>  Tue, 28 Feb 2023 08:56:03 -0800
-
-matrix-synapse-py3 (1.78.0~rc1) stable; urgency=medium
-
-  * Add `matrix-org-archive-keyring` package as recommended.
-  * New Synapse release 1.78.0rc1.
-
- -- Synapse Packaging team <packages@matrix.org>  Tue, 21 Feb 2023 14:29:19 +0000
-
-matrix-synapse-py3 (1.77.0) stable; urgency=medium
-
-  * New Synapse release 1.77.0.
-
- -- Synapse Packaging team <packages@matrix.org>  Tue, 14 Feb 2023 12:59:02 +0100
-
-matrix-synapse-py3 (1.77.0~rc2) stable; urgency=medium
-
-  * New Synapse release 1.77.0rc2.
-
- -- Synapse Packaging team <packages@matrix.org>  Fri, 10 Feb 2023 12:44:21 +0000
-
-matrix-synapse-py3 (1.77.0~rc1) stable; urgency=medium
-
-  * New Synapse release 1.77.0rc1.
-
- -- Synapse Packaging team <packages@matrix.org>  Tue, 07 Feb 2023 13:45:14 +0000
-
-matrix-synapse-py3 (1.76.0) stable; urgency=medium
-
-  * New Synapse release 1.76.0.
-
- -- Synapse Packaging team <packages@matrix.org>  Tue, 31 Jan 2023 08:21:47 -0800
-
-matrix-synapse-py3 (1.76.0~rc2) stable; urgency=medium
-
-  * New Synapse release 1.76.0rc2.
-
- -- Synapse Packaging team <packages@matrix.org>  Fri, 27 Jan 2023 11:17:57 +0000
-
-matrix-synapse-py3 (1.76.0~rc1) stable; urgency=medium
-
-  * Use Poetry 1.3.2 to manage the bundled virtualenv included with this package.
-  * New Synapse release 1.76.0rc1.
-
- -- Synapse Packaging team <packages@matrix.org>  Wed, 25 Jan 2023 16:21:16 +0000
-
-matrix-synapse-py3 (1.75.0) stable; urgency=medium
-
-  * New Synapse release 1.75.0.
-
- -- Synapse Packaging team <packages@matrix.org>  Tue, 17 Jan 2023 11:36:02 +0000
-
-matrix-synapse-py3 (1.75.0~rc2) stable; urgency=medium
-
-  * New Synapse release 1.75.0rc2.
-
- -- Synapse Packaging team <packages@matrix.org>  Thu, 12 Jan 2023 10:30:15 -0800
-
-matrix-synapse-py3 (1.75.0~rc1) stable; urgency=medium
-
-  * New Synapse release 1.75.0rc1.
-
- -- Synapse Packaging team <packages@matrix.org>  Tue, 10 Jan 2023 12:18:27 +0000
-
-matrix-synapse-py3 (1.74.0) stable; urgency=medium
-
-  * New Synapse release 1.74.0.
-
- -- Synapse Packaging team <packages@matrix.org>  Tue, 20 Dec 2022 16:07:38 +0000
-
-matrix-synapse-py3 (1.74.0~rc1) stable; urgency=medium
-
-  * New dependency on libicu-dev to provide improved results for user
-    search.
-  * New Synapse release 1.74.0rc1.
-
- -- Synapse Packaging team <packages@matrix.org>  Tue, 13 Dec 2022 13:30:01 +0000
-
-matrix-synapse-py3 (1.73.0) stable; urgency=medium
-
-  * New Synapse release 1.73.0.
-
- -- Synapse Packaging team <packages@matrix.org>  Tue, 06 Dec 2022 11:48:56 +0000
-
-matrix-synapse-py3 (1.73.0~rc2) stable; urgency=medium
-
-  * New Synapse release 1.73.0rc2.
-
- -- Synapse Packaging team <packages@matrix.org>  Thu, 01 Dec 2022 10:02:19 +0000
-
-matrix-synapse-py3 (1.73.0~rc1) stable; urgency=medium
-
-  * New Synapse release 1.73.0rc1.
-
- -- Synapse Packaging team <packages@matrix.org>  Tue, 29 Nov 2022 12:28:13 +0000
-
-matrix-synapse-py3 (1.72.0) stable; urgency=medium
-
-  * New Synapse release 1.72.0.
-
- -- Synapse Packaging team <packages@matrix.org>  Tue, 22 Nov 2022 10:57:30 +0000
-
-matrix-synapse-py3 (1.72.0~rc1) stable; urgency=medium
-
-  * New Synapse release 1.72.0rc1.
-
- -- Synapse Packaging team <packages@matrix.org>  Wed, 16 Nov 2022 15:10:59 +0000
-
-matrix-synapse-py3 (1.71.0) stable; urgency=medium
-
-  * New Synapse release 1.71.0.
-
- -- Synapse Packaging team <packages@matrix.org>  Tue, 08 Nov 2022 10:38:10 +0000
-
-matrix-synapse-py3 (1.71.0~rc2) stable; urgency=medium
-
-  * New Synapse release 1.71.0rc2.
-
- -- Synapse Packaging team <packages@matrix.org>  Fri, 04 Nov 2022 12:00:33 +0000
-
-matrix-synapse-py3 (1.71.0~rc1) stable; urgency=medium
-
-  * New Synapse release 1.71.0rc1.
-
- -- Synapse Packaging team <packages@matrix.org>  Tue, 01 Nov 2022 12:10:17 +0000
-
-matrix-synapse-py3 (1.70.1) stable; urgency=medium
-
-  * New Synapse release 1.70.1.
-
- -- Synapse Packaging team <packages@matrix.org>  Fri, 28 Oct 2022 12:10:21 +0100
-
-matrix-synapse-py3 (1.70.0) stable; urgency=medium
-
-  * New Synapse release 1.70.0.
-
- -- Synapse Packaging team <packages@matrix.org>  Wed, 26 Oct 2022 11:11:50 +0100
-
-matrix-synapse-py3 (1.70.0~rc2) stable; urgency=medium
-
-  * New Synapse release 1.70.0rc2.
-
- -- Synapse Packaging team <packages@matrix.org>  Tue, 25 Oct 2022 10:59:47 +0100
-
-matrix-synapse-py3 (1.70.0~rc1) stable; urgency=medium
-
-  * New Synapse release 1.70.0rc1.
-
- -- Synapse Packaging team <packages@matrix.org>  Wed, 19 Oct 2022 14:11:57 +0100
-
-matrix-synapse-py3 (1.69.0) stable; urgency=medium
-
-  * New Synapse release 1.69.0.
-
- -- Synapse Packaging team <packages@matrix.org>  Mon, 17 Oct 2022 11:31:03 +0100
-
-matrix-synapse-py3 (1.69.0~rc4) stable; urgency=medium
-
-  * New Synapse release 1.69.0rc4.
-
- -- Synapse Packaging team <packages@matrix.org>  Fri, 14 Oct 2022 15:04:47 +0100
-
-matrix-synapse-py3 (1.69.0~rc3) stable; urgency=medium
-
-  * New Synapse release 1.69.0rc3.
-
- -- Synapse Packaging team <packages@matrix.org>  Wed, 12 Oct 2022 13:24:04 +0100
-
-matrix-synapse-py3 (1.69.0~rc2) stable; urgency=medium
-
-  * New Synapse release 1.69.0rc2.
-
- -- Synapse Packaging team <packages@matrix.org>  Thu, 06 Oct 2022 14:45:00 +0100
-
-matrix-synapse-py3 (1.69.0~rc1) stable; urgency=medium
-
-  * The man page for the hash_password script has been updated to reflect
-    the correct default value of 'bcrypt_rounds'.
-  * New Synapse release 1.69.0rc1.
-
- -- Synapse Packaging team <packages@matrix.org>  Tue, 04 Oct 2022 11:17:16 +0100
-
-matrix-synapse-py3 (1.68.0) stable; urgency=medium
-
-  * New Synapse release 1.68.0.
-
- -- Synapse Packaging team <packages@matrix.org>  Tue, 27 Sep 2022 12:02:09 +0100
-
-matrix-synapse-py3 (1.68.0~rc2) stable; urgency=medium
-
-  * New Synapse release 1.68.0rc2.
-
- -- Synapse Packaging team <packages@matrix.org>  Fri, 23 Sep 2022 09:40:10 +0100
-
-matrix-synapse-py3 (1.68.0~rc1) stable; urgency=medium
-
-  * New Synapse release 1.68.0rc1.
-
- -- Synapse Packaging team <packages@matrix.org>  Tue, 20 Sep 2022 11:18:20 +0100
-
-matrix-synapse-py3 (1.67.0) stable; urgency=medium
-
-  * New Synapse release 1.67.0.
-
- -- Synapse Packaging team <packages@matrix.org>  Tue, 13 Sep 2022 09:19:56 +0100
-
-matrix-synapse-py3 (1.67.0~rc1) stable; urgency=medium
-
-  [ Erik Johnston ]
-  * Use stable poetry 1.2.0 version, rather than a prerelease.
-
-  [ Synapse Packaging team ]
-  * New Synapse release 1.67.0rc1.
-
- -- Synapse Packaging team <packages@matrix.org>  Tue, 06 Sep 2022 09:01:06 +0100
-
-matrix-synapse-py3 (1.66.0) stable; urgency=medium
-
-  * New Synapse release 1.66.0.
-
- -- Synapse Packaging team <packages@matrix.org>  Wed, 31 Aug 2022 11:20:17 +0100
-
-matrix-synapse-py3 (1.66.0~rc2+nmu1) UNRELEASED; urgency=medium
-
-  [ Jörg Behrmann ]
-  * Update debhelper to compatibility level 12.
-  * Drop the preinst script stopping synapse.
-  * Allocate a group for the system user.
-  * Change dpkg-statoverride to --force-statoverride-add.
-
-  [ Erik Johnston ]
-  * Disable `dh_auto_configure` as it broke during Rust build.
-
- -- Jörg Behrmann <behrmann@physik.fu-berlin.de>  Tue, 23 Aug 2022 17:17:00 +0100
-
-matrix-synapse-py3 (1.66.0~rc2) stable; urgency=medium
-
-  * New Synapse release 1.66.0rc2.
-
- -- Synapse Packaging team <packages@matrix.org>  Tue, 30 Aug 2022 12:25:19 +0100
-
-matrix-synapse-py3 (1.66.0~rc1) stable; urgency=medium
-
-  * New Synapse release 1.66.0rc1.
-
- -- Synapse Packaging team <packages@matrix.org>  Tue, 23 Aug 2022 09:48:55 +0100
-
 matrix-synapse-py3 (1.65.0) stable; urgency=medium
 
   * New Synapse release 1.65.0.

debian/compat

@@ -0,0 +1 @@
+10

debian/control

@@ -4,12 +4,10 @@ Priority: extra
 Maintainer: Synapse Packaging team <packages@matrix.org>
 # keep this list in sync with the build dependencies in docker/Dockerfile-dhvirtualenv.
 Build-Depends:
- debhelper-compat (= 12),
+ debhelper (>= 10),
  dh-virtualenv (>= 1.1),
  libsystemd-dev,
  libpq-dev,
- libicu-dev,
- pkg-config,
  lsb-release,
  python3-dev,
  python3,
@@ -37,7 +35,6 @@ Depends:
 # so we put perl:Depends in Suggests rather than Depends.
 Recommends:
  ${shlibs1:Recommends},
- matrix-org-archive-keyring,
 Suggests:
  sqlite3,
  ${perl:Depends},

debian/hash_password.1

@@ -10,7 +10,7 @@
 .P
 \fBhash_password\fR takes a password as an parameter either on the command line or the \fBSTDIN\fR if not supplied\.
 .P
-It accepts an YAML file which can be used to specify parameters like the number of rounds for bcrypt and password_config section having the pepper value used for the hashing\. By default \fBbcrypt_rounds\fR is set to \fB12\fR\.
+It accepts an YAML file which can be used to specify parameters like the number of rounds for bcrypt and password_config section having the pepper value used for the hashing\. By default \fBbcrypt_rounds\fR is set to \fB10\fR\.
 .P
 The hashed password is written on the \fBSTDOUT\fR\.
 .SH "FILES"

debian/hash_password.ronn

@@ -14,7 +14,7 @@ or the `STDIN` if not supplied.
 
 It accepts an YAML file which can be used to specify parameters like the
 number of rounds for bcrypt and password_config section having the pepper
-value used for the hashing. By default `bcrypt_rounds` is set to **12**.
+value used for the hashing. By default `bcrypt_rounds` is set to **10**.
 
 The hashed password is written on the `STDOUT`.
 

debian/matrix-synapse-py3.postinst

@@ -40,12 +40,12 @@ EOF
     /opt/venvs/matrix-synapse/lib/manage_debconf.pl update
 
     if ! getent passwd $USER >/dev/null; then
-      adduser --quiet --system --group --no-create-home --home /var/lib/matrix-synapse $USER
+      adduser --quiet --system --no-create-home --home /var/lib/matrix-synapse $USER
     fi
 
     for DIR in /var/lib/matrix-synapse /var/log/matrix-synapse /etc/matrix-synapse; do
       if ! dpkg-statoverride --list --quiet $DIR >/dev/null; then
-        dpkg-statoverride --force-statoverride-add --quiet --update --add $USER "$(id -gn $USER)" 0755 $DIR
+        dpkg-statoverride --force --quiet --update --add $USER nogroup 0755 $DIR
       fi
     done
 

debian/matrix-synapse-py3.preinst

@@ -0,0 +1,31 @@
+#!/bin/sh -e
+
+# Attempt to undo some of the braindamage caused by
+# https://github.com/matrix-org/package-synapse-debian/issues/18.
+#
+# Due to reasons [1], the old python2 matrix-synapse package will not stop the
+# service when the package is uninstalled. Our maintainer scripts will do the
+# right thing in terms of ensuring the service is enabled and unmasked, but
+# then do a `systemctl start matrix-synapse`, which of course does nothing -
+# leaving the old (py2) service running.
+#
+# There should normally be no reason for the service to be running during our
+# preinst, so we assume that if it *is* running, it's due to that situation,
+# and stop it.
+#
+# [1] dh_systemd_start doesn't do anything because it sees that there is an
+#     init.d script with the same name, so leaves it to dh_installinit.
+#
+#     dh_installinit doesn't do anything because somebody gave it a --no-start
+#     for unknown reasons.
+
+if [ -x /bin/systemctl ]; then
+    if /bin/systemctl --quiet is-active -- matrix-synapse; then
+        echo >&2 "stopping existing matrix-synapse service"
+        /bin/systemctl stop matrix-synapse || true
+    fi
+fi
+
+#DEBHELPER#
+
+exit 0

debian/matrix-synapse.default

@@ -0,0 +1,2 @@
+# Specify environment variables used when running Synapse
+# SYNAPSE_CACHE_FACTOR=0.5 (default)

debian/matrix-synapse.service

@@ -5,6 +5,7 @@ Description=Synapse Matrix homeserver
 Type=notify
 User=matrix-synapse
 WorkingDirectory=/var/lib/matrix-synapse
+EnvironmentFile=-/etc/default/matrix-synapse
 ExecStartPre=/opt/venvs/matrix-synapse/bin/python -m synapse.app.homeserver --config-path=/etc/matrix-synapse/homeserver.yaml --config-path=/etc/matrix-synapse/conf.d/ --generate-keys
 ExecStart=/opt/venvs/matrix-synapse/bin/python -m synapse.app.homeserver --config-path=/etc/matrix-synapse/homeserver.yaml --config-path=/etc/matrix-synapse/conf.d/
 ExecReload=/bin/kill -HUP $MAINPID
@@ -12,10 +13,5 @@ Restart=always
 RestartSec=3
 SyslogIdentifier=matrix-synapse
 
-# The environment file is not shipped by default anymore and the below directive
-# is for backwards compatibility only. Please use your homeserver.yaml if
-# possible.
-EnvironmentFile=-/etc/default/matrix-synapse
-
 [Install]
 WantedBy=multi-user.target

debian/rules

@@ -6,19 +6,15 @@
 # assume we only have one package
 PACKAGE_NAME:=`dh_listpackages`
 
-override_dh_installsystemd:
-	dh_installsystemd --name=matrix-synapse
+override_dh_systemd_enable:
+	dh_systemd_enable --name=matrix-synapse
+
+override_dh_installinit:
+	dh_installinit --name=matrix-synapse
 
 # we don't really want to strip the symbols from our object files.
 override_dh_strip:
 
-override_dh_auto_configure:
-
-# many libraries pulled from PyPI have allocatable sections after
-# non-allocatable ones on which dwz errors out. For those without the issue the
-# gains are only marginal
-override_dh_dwz:
-
 # dh_shlibdeps calls dpkg-shlibdeps, which finds all the binary files
 # (executables and shared libs) in the package, and looks for the shared
 # libraries that they depend on. It then adds a dependency on the package that

demo/start.sh

@@ -46,7 +46,7 @@ for port in 8080 8081 8082; do
             echo ''
 
 			# Warning, this heredoc depends on the interaction of tabs and spaces.
-			# Please don't accidentally bork me with your fancy settings.
+			# Please don't accidentaly bork me with your fancy settings.
 			listeners=$(cat <<-PORTLISTENERS
 			# Configure server to listen on both $https_port and $port
 			# This overides some of the default settings above
@@ -80,8 +80,12 @@ for port in 8080 8081 8082; do
             echo "tls_certificate_path: \"$DIR/$port/localhost:$port.tls.crt\""
             echo "tls_private_key_path: \"$DIR/$port/localhost:$port.tls.key\""
 
-            # Request keys directly from servers contacted over federation
-            echo 'trusted_key_servers: []'
+            # Ignore keys from the trusted keys server
+            echo '# Ignore keys from the trusted keys server'
+            echo 'trusted_key_servers:'
+            echo '  - server_name: "matrix.org"'
+            echo '    accept_keys_insecurely: true'
+            echo ''
 
 			# Allow the servers to communicate over localhost.
 			allow_list=$(cat <<-ALLOW_LIST

dev-docs/Makefile

@@ -1,20 +0,0 @@
-# Minimal makefile for Sphinx documentation
-#
-
-# You can set these variables from the command line, and also
-# from the environment for the first two.
-SPHINXOPTS    ?=
-SPHINXBUILD   ?= sphinx-build
-SOURCEDIR     = .
-BUILDDIR      = _build
-
-# Put it first so that "make" without argument is like "make help".
-help:
-	@$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
-
-.PHONY: help Makefile
-
-# Catch-all target: route all unknown targets to Sphinx using the new
-# "make mode" option.  $(O) is meant as a shortcut for $(SPHINXOPTS).
-%: Makefile
-	@$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)

dev-docs/conf.py

@@ -1,50 +0,0 @@
-# Configuration file for the Sphinx documentation builder.
-#
-# For the full list of built-in configuration values, see the documentation:
-# https://www.sphinx-doc.org/en/master/usage/configuration.html
-
-# -- Project information -----------------------------------------------------
-# https://www.sphinx-doc.org/en/master/usage/configuration.html#project-information
-
-project = "Synapse development"
-copyright = "2023, The Matrix.org Foundation C.I.C."
-author = "The Synapse Maintainers and Community"
-
-# -- General configuration ---------------------------------------------------
-# https://www.sphinx-doc.org/en/master/usage/configuration.html#general-configuration
-
-extensions = [
-    "autodoc2",
-    "myst_parser",
-]
-
-templates_path = ["_templates"]
-exclude_patterns = ["_build", "Thumbs.db", ".DS_Store"]
-
-
-# -- Options for Autodoc2 ----------------------------------------------------
-
-autodoc2_docstring_parser_regexes = [
-    # this will render all docstrings as 'MyST' Markdown
-    (r".*", "myst"),
-]
-
-autodoc2_packages = [
-    {
-        "path": "../synapse",
-        # Don't render documentation for everything as a matter of course
-        "auto_mode": False,
-    },
-]
-
-
-# -- Options for MyST (Markdown) ---------------------------------------------
-
-# myst_heading_anchors = 2
-
-
-# -- Options for HTML output -------------------------------------------------
-# https://www.sphinx-doc.org/en/master/usage/configuration.html#options-for-html-output
-
-html_theme = "furo"
-html_static_path = ["_static"]

dev-docs/index.rst

@@ -1,22 +0,0 @@
-.. Synapse Developer Documentation documentation master file, created by
-   sphinx-quickstart on Mon Mar 13 08:59:51 2023.
-   You can adapt this file completely to your liking, but it should at least
-   contain the root `toctree` directive.
-
-Welcome to the Synapse Developer Documentation!
-===========================================================
-
-.. toctree::
-   :maxdepth: 2
-   :caption: Contents:
-
-   modules/federation_sender
-
-
-
-Indices and tables
-==================
-
-* :ref:`genindex`
-* :ref:`modindex`
-* :ref:`search`

dev-docs/modules/federation_sender.md

@@ -1,5 +0,0 @@
-Federation Sender
-=================
-
-```{autodoc2-docstring} synapse.federation.sender
-```

docker/Dockerfile

@@ -17,48 +17,46 @@
 
 # Irritatingly, there is no blessed guide on how to distribute an application with its
 # poetry-managed environment in a docker image. We have opted for
-# `poetry export | pip install -r /dev/stdin`, but beware: we have experienced bugs in
-# in `poetry export` in the past.
+# `poetry export | pip install -r /dev/stdin`, but there are known bugs in
+# in `poetry export` whose fixes (scheduled for poetry 1.2) have yet to be released.
+# In case we get bitten by those bugs in the future, the recommendations here might
+# be useful:
+#     https://github.com/python-poetry/poetry/discussions/1879#discussioncomment-216865
+#     https://stackoverflow.com/questions/53835198/integrating-python-poetry-with-docker?answertab=scoredesc
 
-ARG PYTHON_VERSION=3.11
+
+
+ARG PYTHON_VERSION=3.9
 
 ###
 ### Stage 0: generate requirements.txt
 ###
-# We hardcode the use of Debian bullseye here because this could change upstream
-# and other Dockerfiles used for testing are expecting bullseye.
-FROM docker.io/library/python:${PYTHON_VERSION}-slim-bullseye as requirements
+FROM docker.io/python:${PYTHON_VERSION}-slim as requirements
 
 # RUN --mount is specific to buildkit and is documented at
 # https://github.com/moby/buildkit/blob/master/frontend/dockerfile/docs/syntax.md#build-mounts-run---mount.
 # Here we use it to set up a cache for apt (and below for pip), to improve
 # rebuild speeds on slow connections.
 RUN \
-  --mount=type=cache,target=/var/cache/apt,sharing=locked \
-  --mount=type=cache,target=/var/lib/apt,sharing=locked \
-  apt-get update -qq && apt-get install -yqq \
-  build-essential curl git libffi-dev libssl-dev pkg-config \
-  && rm -rf /var/lib/apt/lists/*
-
-# Install rust and ensure its in the PATH.
-# (Rust may be needed to compile `cryptography`---which is one of poetry's
-# dependencies---on platforms that don't have a `cryptography` wheel.
-ENV RUSTUP_HOME=/rust
-ENV CARGO_HOME=/cargo
-ENV PATH=/cargo/bin:/rust/bin:$PATH
-RUN mkdir /rust /cargo
-
-RUN curl -sSf https://sh.rustup.rs | sh -s -- -y --no-modify-path --default-toolchain stable --profile minimal
-
-# arm64 builds consume a lot of memory if `CARGO_NET_GIT_FETCH_WITH_CLI` is not
-# set to true, so we expose it as a build-arg.
-ARG CARGO_NET_GIT_FETCH_WITH_CLI=false
-ENV CARGO_NET_GIT_FETCH_WITH_CLI=$CARGO_NET_GIT_FETCH_WITH_CLI
+   --mount=type=cache,target=/var/cache/apt,sharing=locked \
+   --mount=type=cache,target=/var/lib/apt,sharing=locked \
+    apt-get update -qq && apt-get install -yqq \
+      build-essential cargo git libffi-dev libssl-dev \
+    && rm -rf /var/lib/apt/lists/*
 
 # We install poetry in its own build stage to avoid its dependencies conflicting with
 # synapse's dependencies.
+# We use a specific commit from poetry's master branch instead of our usual 1.1.14,
+# to incorporate fixes to some bugs in `poetry export`. This commit corresponds to
+#    https://github.com/python-poetry/poetry/pull/5156 and
+#    https://github.com/python-poetry/poetry/issues/5141 ;
+# without it, we generate a requirements.txt with incorrect environment markers,
+# which causes necessary packages to be omitted when we `pip install`.
+#
+# NB: In poetry 1.2 `poetry export` will be moved into a plugin; we'll need to also
+# pip install poetry-plugin-export (https://github.com/python-poetry/poetry-plugin-export).
 RUN --mount=type=cache,target=/root/.cache/pip \
-  pip install --user "poetry==1.3.2"
+  pip install --user "poetry-core==1.1.0a7" "git+https://github.com/python-poetry/poetry.git@fb13b3a676f476177f7937ffa480ee5cff9a90a5"
 
 WORKDIR /synapse
 
@@ -79,51 +77,34 @@ ARG TEST_ONLY_IGNORE_POETRY_LOCKFILE
 # Otherwise, just create an empty requirements file so that the Dockerfile can
 # proceed.
 RUN if [ -z "$TEST_ONLY_IGNORE_POETRY_LOCKFILE" ]; then \
-  /root/.local/bin/poetry export --extras all -o /synapse/requirements.txt ${TEST_ONLY_SKIP_DEP_HASH_VERIFICATION:+--without-hashes}; \
+    /root/.local/bin/poetry export --extras all -o /synapse/requirements.txt ${TEST_ONLY_SKIP_DEP_HASH_VERIFICATION:+--without-hashes}; \
   else \
-  touch /synapse/requirements.txt; \
+    touch /synapse/requirements.txt; \
   fi
 
 ###
 ### Stage 1: builder
 ###
-FROM docker.io/library/python:${PYTHON_VERSION}-slim-bullseye as builder
+FROM docker.io/python:${PYTHON_VERSION}-slim as builder
 
 # install the OS build deps
 RUN \
-  --mount=type=cache,target=/var/cache/apt,sharing=locked \
-  --mount=type=cache,target=/var/lib/apt,sharing=locked \
-  apt-get update -qq && apt-get install -yqq \
-  build-essential \
-  libffi-dev \
-  libjpeg-dev \
-  libpq-dev \
-  libssl-dev \
-  libwebp-dev \
-  libxml++2.6-dev \
-  libxslt1-dev \
-  openssl \
-  zlib1g-dev \
-  git \
-  curl \
-  libicu-dev \
-  pkg-config \
-  && rm -rf /var/lib/apt/lists/*
-
-
-# Install rust and ensure its in the PATH
-ENV RUSTUP_HOME=/rust
-ENV CARGO_HOME=/cargo
-ENV PATH=/cargo/bin:/rust/bin:$PATH
-RUN mkdir /rust /cargo
-
-RUN curl -sSf https://sh.rustup.rs | sh -s -- -y --no-modify-path --default-toolchain stable --profile minimal
-
-
-# arm64 builds consume a lot of memory if `CARGO_NET_GIT_FETCH_WITH_CLI` is not
-# set to true, so we expose it as a build-arg.
-ARG CARGO_NET_GIT_FETCH_WITH_CLI=false
-ENV CARGO_NET_GIT_FETCH_WITH_CLI=$CARGO_NET_GIT_FETCH_WITH_CLI
+   --mount=type=cache,target=/var/cache/apt,sharing=locked \
+   --mount=type=cache,target=/var/lib/apt,sharing=locked \
+ apt-get update -qq && apt-get install -yqq \
+    build-essential \
+    libffi-dev \
+    libjpeg-dev \
+    libpq-dev \
+    libssl-dev \
+    libwebp-dev \
+    libxml++2.6-dev \
+    libxslt1-dev \
+    openssl \
+    rustc \
+    zlib1g-dev \
+    git \
+    && rm -rf /var/lib/apt/lists/*
 
 # To speed up rebuilds, install all of the dependencies before we copy over
 # the whole synapse project, so that this layer in the Docker cache can be
@@ -136,9 +117,8 @@ RUN --mount=type=cache,target=/root/.cache/pip \
 
 # Copy over the rest of the synapse source code.
 COPY synapse /synapse/synapse/
-COPY rust /synapse/rust/
 # ... and what we need to `pip install`.
-COPY pyproject.toml README.rst build_rust.py Cargo.toml Cargo.lock /synapse/
+COPY pyproject.toml README.rst /synapse/
 
 # Repeat of earlier build argument declaration, as this is a new build stage.
 ARG TEST_ONLY_IGNORE_POETRY_LOCKFILE
@@ -146,19 +126,17 @@ ARG TEST_ONLY_IGNORE_POETRY_LOCKFILE
 # Install the synapse package itself.
 # If we have populated requirements.txt, we don't install any dependencies
 # as we should already have those from the previous `pip install` step.
-RUN --mount=type=cache,target=/synapse/target,sharing=locked \
-  --mount=type=cache,target=${CARGO_HOME}/registry,sharing=locked \
-  if [ -z "$TEST_ONLY_IGNORE_POETRY_LOCKFILE" ]; then \
-  pip install --prefix="/install" --no-deps --no-warn-script-location /synapse[all]; \
+RUN if [ -z "$TEST_ONLY_IGNORE_POETRY_LOCKFILE" ]; then \
+    pip install --prefix="/install" --no-deps --no-warn-script-location /synapse[all]; \
   else \
-  pip install --prefix="/install" --no-warn-script-location /synapse[all]; \
+    pip install --prefix="/install" --no-warn-script-location /synapse[all]; \
   fi
 
 ###
 ### Stage 2: runtime
 ###
 
-FROM docker.io/library/python:${PYTHON_VERSION}-slim-bullseye
+FROM docker.io/python:${PYTHON_VERSION}-slim
 
 LABEL org.opencontainers.image.url='https://matrix.org/docs/projects/server/synapse'
 LABEL org.opencontainers.image.documentation='https://github.com/matrix-org/synapse/blob/master/docker/README.md'
@@ -166,20 +144,19 @@ LABEL org.opencontainers.image.source='https://github.com/matrix-org/synapse.git
 LABEL org.opencontainers.image.licenses='Apache-2.0'
 
 RUN \
-  --mount=type=cache,target=/var/cache/apt,sharing=locked \
-  --mount=type=cache,target=/var/lib/apt,sharing=locked \
+   --mount=type=cache,target=/var/cache/apt,sharing=locked \
+   --mount=type=cache,target=/var/lib/apt,sharing=locked \
   apt-get update -qq && apt-get install -yqq \
-  curl \
-  gosu \
-  libjpeg62-turbo \
-  libpq5 \
-  libwebp6 \
-  xmlsec1 \
-  libjemalloc2 \
-  libicu67 \
-  libssl-dev \
-  openssl \
-  && rm -rf /var/lib/apt/lists/*
+    curl \
+    gosu \
+    libjpeg62-turbo \
+    libpq5 \
+    libwebp6 \
+    xmlsec1 \
+    libjemalloc2 \
+    libssl-dev \
+    openssl \
+    && rm -rf /var/lib/apt/lists/*
 
 COPY --from=builder /install /usr/local
 COPY ./docker/start.py /start.py
@@ -190,4 +167,4 @@ EXPOSE 8008/tcp 8009/tcp 8448/tcp
 ENTRYPOINT ["/start.py"]
 
 HEALTHCHECK --start-period=5s --interval=15s --timeout=5s \
-  CMD curl -fSs http://localhost:8008/health || exit 1
+    CMD curl -fSs http://localhost:8008/health || exit 1

docker/Dockerfile-dhvirtualenv

@@ -24,7 +24,7 @@ ARG distro=""
 # https://launchpad.net/~jyrki-pulliainen/+archive/ubuntu/dh-virtualenv, but
 # it's not obviously easier to use that than to build our own.)
 
-FROM docker.io/library/${distro} as builder
+FROM ${distro} as builder
 
 RUN apt-get update -qq -o Acquire::Languages=none
 RUN env DEBIAN_FRONTEND=noninteractive apt-get install \
@@ -36,10 +36,8 @@ RUN env DEBIAN_FRONTEND=noninteractive apt-get install \
         wget
 
 # fetch and unpack the package
-# We are temporarily using a fork of dh-virtualenv due to an incompatibility with Python 3.11, which ships with
-# Debian sid. TODO: Switch back to upstream once https://github.com/spotify/dh-virtualenv/pull/354 has merged.
 RUN mkdir /dh-virtualenv
-RUN wget -q -O /dh-virtualenv.tar.gz https://github.com/matrix-org/dh-virtualenv/archive/refs/tags/matrixorg-2023010302.tar.gz
+RUN wget -q -O /dh-virtualenv.tar.gz https://github.com/spotify/dh-virtualenv/archive/refs/tags/1.2.2.tar.gz
 RUN tar -xv --strip-components=1 -C /dh-virtualenv -f /dh-virtualenv.tar.gz
 
 # install its build deps. We do another apt-cache-update here, because we might
@@ -55,7 +53,7 @@ RUN cd /dh-virtualenv && DEB_BUILD_OPTIONS=nodoc dpkg-buildpackage -us -uc -b
 ###
 ### Stage 1
 ###
-FROM docker.io/library/${distro}
+FROM ${distro}
 
 # Get the distro we want to pull from as a dynamic build variable
 # (We need to define it in each build stage)
@@ -74,7 +72,6 @@ RUN apt-get update -qq -o Acquire::Languages=none \
     && env DEBIAN_FRONTEND=noninteractive apt-get install \
         -yqq --no-install-recommends -o Dpkg::Options::=--force-unsafe-io \
         build-essential \
-        curl \
         debhelper \
         devscripts \
         libsystemd-dev \
@@ -86,19 +83,8 @@ RUN apt-get update -qq -o Acquire::Languages=none \
         python3-venv \
         sqlite3 \
         libpq-dev \
-        libicu-dev \
-        pkg-config \
         xmlsec1
 
-# Install rust and ensure it's in the PATH
-ENV RUSTUP_HOME=/rust
-ENV CARGO_HOME=/cargo
-ENV PATH=/cargo/bin:/rust/bin:$PATH
-RUN mkdir /rust /cargo
-
-RUN curl -sSf https://sh.rustup.rs | sh -s -- -y --no-modify-path --default-toolchain stable --profile minimal
-
-
 COPY --from=builder /dh-virtualenv_1.2.2-1_all.deb /
 
 # install dhvirtualenv. Update the apt cache again first, in case we got a

docker/Dockerfile-workers

@@ -1,13 +1,12 @@
 # syntax=docker/dockerfile:1
 
 ARG SYNAPSE_VERSION=latest
-ARG FROM=matrixdotorg/synapse:$SYNAPSE_VERSION
 
 # first of all, we create a base image with an nginx which we can copy into the
 # target image. For repeated rebuilds, this is much faster than apt installing
 # each time.
 
-FROM docker.io/library/debian:bullseye-slim AS deps_base
+FROM debian:bullseye-slim AS deps_base
     RUN \
        --mount=type=cache,target=/var/cache/apt,sharing=locked \
        --mount=type=cache,target=/var/lib/apt,sharing=locked \
@@ -21,10 +20,10 @@ FROM docker.io/library/debian:bullseye-slim AS deps_base
 # which makes it much easier to copy (but we need to make sure we use an image
 # based on the same debian version as the synapse image, to make sure we get
 # the expected version of libc.
-FROM docker.io/library/redis:7-bullseye AS redis_base
+FROM redis:6-bullseye AS redis_base
 
 # now build the final image, based on the the regular Synapse docker image
-FROM $FROM
+FROM matrixdotorg/synapse:$SYNAPSE_VERSION
 
     # Install supervisord with pip instead of apt, to avoid installing a second
     # copy of python.
@@ -41,11 +40,7 @@ FROM $FROM
     COPY --from=deps_base /etc/nginx /etc/nginx
     RUN rm /etc/nginx/sites-enabled/default
     RUN mkdir /var/log/nginx /var/lib/nginx
-    RUN chown www-data /var/lib/nginx
-
-    # have nginx log to stderr/out
-    RUN ln -sf /dev/stdout /var/log/nginx/access.log
-    RUN ln -sf /dev/stderr /var/log/nginx/error.log
+    RUN chown www-data /var/log/nginx /var/lib/nginx
 
     # Copy Synapse worker, nginx and supervisord configuration template files
     COPY ./docker/conf-workers/* /conf/

docker/README.md

@@ -73,8 +73,7 @@ The following environment variables are supported in `generate` mode:
   will log sensitive information such as access tokens.
   This should not be needed unless you are a developer attempting to debug something
   particularly tricky.
-* `SYNAPSE_LOG_TESTING`: if set, Synapse will log additional information useful
-  for testing.
+
 
 ## Postgres
 
@@ -192,7 +191,7 @@ If you need to build the image from a Synapse checkout, use the following `docke
  build` command from the repo's root:
 
 ```
-DOCKER_BUILDKIT=1 docker build -t matrixdotorg/synapse -f docker/Dockerfile .
+docker build -t matrixdotorg/synapse -f docker/Dockerfile .
 ```
 
 You can choose to build a different docker image by changing the value of the `-f` flag to
@@ -242,4 +241,4 @@ healthcheck:
 
 Jemalloc is embedded in the image and will be used instead of the default allocator.
 You can read about jemalloc by reading the Synapse
-[Admin FAQ](https://matrix-org.github.io/synapse/latest/usage/administration/admin_faq.html#help-synapse-is-slow-and-eats-all-my-ramcpu).
+[README](https://github.com/matrix-org/synapse/blob/HEAD/README.rst#help-synapse-is-slow-and-eats-all-my-ram-cpu).

docker/complement/Dockerfile

@@ -7,32 +7,36 @@
 # https://github.com/matrix-org/synapse/blob/develop/docker/README-testing.md#testing-with-postgresql-and-single-or-multi-process-synapse
 
 ARG SYNAPSE_VERSION=latest
-# This is an intermediate image, to be built locally (not pulled from a registry).
-ARG FROM=matrixdotorg/synapse-workers:$SYNAPSE_VERSION
 
-FROM $FROM
-    # First of all, we copy postgres server from the official postgres image,
-    # since for repeated rebuilds, this is much faster than apt installing
-    # postgres each time.
+# first of all, we create a base image with a postgres server and database,
+# which we can copy into the target image. For repeated rebuilds, this is
+# much faster than apt installing postgres each time.
+#
+# This trick only works because (a) the Synapse image happens to have all the
+# shared libraries that postgres wants, (b) we use a postgres image based on
+# the same debian version as Synapse's docker image (so the versions of the
+# shared libraries match).
 
-    # This trick only works because (a) the Synapse image happens to have all the
-    # shared libraries that postgres wants, (b) we use a postgres image based on
-    # the same debian version as Synapse's docker image (so the versions of the
-    # shared libraries match).
-    RUN adduser --system --uid 999 postgres --home /var/lib/postgresql
-    COPY --from=docker.io/library/postgres:13-bullseye /usr/lib/postgresql /usr/lib/postgresql
-    COPY --from=docker.io/library/postgres:13-bullseye /usr/share/postgresql /usr/share/postgresql
-    RUN mkdir /var/run/postgresql && chown postgres /var/run/postgresql
-    ENV PATH="${PATH}:/usr/lib/postgresql/13/bin"
-    ENV PGDATA=/var/lib/postgresql/data
-
-    # We also initialize the database at build time, rather than runtime, so that it's faster to spin up the image.
+FROM postgres:13-bullseye AS postgres_base
+    # initialise the database cluster in /var/lib/postgresql
     RUN gosu postgres initdb --locale=C --encoding=UTF-8 --auth-host password
 
     # Configure a password and create a database for Synapse
     RUN echo "ALTER USER postgres PASSWORD 'somesecret'" | gosu postgres postgres --single
     RUN echo "CREATE DATABASE synapse" | gosu postgres postgres --single
 
+# now build the final image, based on the Synapse image.
+
+FROM matrixdotorg/synapse-workers:$SYNAPSE_VERSION
+    # copy the postgres installation over from the image we built above
+    RUN adduser --system --uid 999 postgres --home /var/lib/postgresql
+    COPY --from=postgres_base /var/lib/postgresql /var/lib/postgresql
+    COPY --from=postgres_base /usr/lib/postgresql /usr/lib/postgresql
+    COPY --from=postgres_base /usr/share/postgresql /usr/share/postgresql
+    RUN mkdir /var/run/postgresql && chown postgres /var/run/postgresql
+    ENV PATH="${PATH}:/usr/lib/postgresql/13/bin"
+    ENV PGDATA=/var/lib/postgresql/data
+
     # Extend the shared homeserver config to disable rate-limiting,
     # set Complement's static shared secret, enable registration, amongst other
     # tweaks to get Synapse ready for testing.

docker/complement/conf/start_for_complement.sh

@@ -6,7 +6,7 @@ set -e
 
 echo "Complement Synapse launcher"
 echo "  Args: $@"
-echo "  Env: SYNAPSE_COMPLEMENT_DATABASE=$SYNAPSE_COMPLEMENT_DATABASE SYNAPSE_COMPLEMENT_USE_WORKERS=$SYNAPSE_COMPLEMENT_USE_WORKERS SYNAPSE_COMPLEMENT_USE_ASYNCIO_REACTOR=$SYNAPSE_COMPLEMENT_USE_ASYNCIO_REACTOR"
+echo "  Env: SYNAPSE_COMPLEMENT_DATABASE=$SYNAPSE_COMPLEMENT_DATABASE SYNAPSE_COMPLEMENT_USE_WORKERS=$SYNAPSE_COMPLEMENT_USE_WORKERS"
 
 function log {
     d=$(date +"%Y-%m-%d %H:%M:%S,%3N")
@@ -45,13 +45,9 @@ esac
 
 if [[ -n "$SYNAPSE_COMPLEMENT_USE_WORKERS" ]]; then
   # Specify the workers to test with
-  # Allow overriding by explicitly setting SYNAPSE_WORKER_TYPES outside, while still
-  # utilizing WORKERS=1 for backwards compatibility.
-  # -n True if the length of string is non-zero.
-  # -z True if the length of string is zero.
-  if [[ -z "$SYNAPSE_WORKER_TYPES" ]]; then
-    export SYNAPSE_WORKER_TYPES="\
-      event_persister:2, \
+  export SYNAPSE_WORKER_TYPES="\
+      event_persister, \
+      event_persister, \
       background_worker, \
       frontend_proxy, \
       event_creator, \
@@ -61,13 +57,9 @@ if [[ -n "$SYNAPSE_COMPLEMENT_USE_WORKERS" ]]; then
       federation_reader, \
       federation_sender, \
       synchrotron, \
-      client_reader, \
       appservice, \
-      pusher, \
-      stream_writers=account_data+presence+receipts+to_device+typing"
+      pusher"
 
-  fi
-  log "Workers requested: $SYNAPSE_WORKER_TYPES"
   # Improve startup times by using a launcher based on fork()
   export SYNAPSE_USE_EXPERIMENTAL_FORKING_LAUNCHER=1
 else
@@ -76,17 +68,6 @@ else
 fi
 
 
-if [[ -n "$SYNAPSE_COMPLEMENT_USE_ASYNCIO_REACTOR" ]]; then
-  if [[ -n "$SYNAPSE_USE_EXPERIMENTAL_FORKING_LAUNCHER" ]]; then
-    export SYNAPSE_COMPLEMENT_FORKING_LAUNCHER_ASYNC_IO_REACTOR="1"
-  else
-    export SYNAPSE_ASYNC_IO_REACTOR="1"
-  fi
-else
-  export SYNAPSE_ASYNC_IO_REACTOR="0"
-fi
-
-
 # Add Complement's appservice registration directory, if there is one
 # (It can be absent when there are no application services in this test!)
 if [ -d /complement/appservice ]; then

docker/complement/conf/workers-shared-extra.yaml.j2

@@ -12,8 +12,6 @@ trusted_key_servers: []
 enable_registration: true
 enable_registration_without_verification: true
 bcrypt_rounds: 4
-url_preview_enabled: true
-url_preview_ip_range_blacklist: []
 
 ## Registration ##
 
@@ -92,16 +90,18 @@ allow_device_name_lookup_over_federation: true
 ## Experimental Features ##
 
 experimental_features:
+  # Enable spaces support
+  spaces_enabled: true
+  # Enable history backfilling support
+  msc2716_enabled: true
+  # server-side support for partial state in /send_join responses
+  msc3706_enabled: true
+  {% if not workers_in_use %}
   # client-side support for partial state in /send_join responses
   faster_joins: true
-  # Enable support for polls
-  msc3381_polls_enabled: true
-  # Enable deleting device-specific notification settings stored in account data
-  msc3890_enabled: true
-  # Enable removing account data support
-  msc3391_enabled: true
-  # Filtering /messages by relation type.
-  msc3874_enabled: true
+  {% endif %}
+  # Enable jump to date endpoint
+  msc3030_enabled: true
 
 server_notices:
   system_mxid_localpart: _server

docker/conf-workers/worker.yaml.j2

@@ -6,6 +6,10 @@
 worker_app: "{{ app }}"
 worker_name: "{{ name }}"
 
+# The replication listener on the main synapse process.
+worker_replication_host: 127.0.0.1
+worker_replication_http_port: 9093
+
 worker_listeners:
   - type: http
     port: {{ port }}

docker/conf/log.config

@@ -49,35 +49,17 @@ handlers:
     class: logging.StreamHandler
     formatter: precise
 
+{% if not SYNAPSE_LOG_SENSITIVE %}
+{#
+  If SYNAPSE_LOG_SENSITIVE is unset, then override synapse.storage.SQL to INFO
+  so that DEBUG entries (containing sensitive information) are not emitted.
+#}
 loggers:
-    # This is just here so we can leave `loggers` in the config regardless of whether
-    # we configure other loggers below (avoid empty yaml dict error).
-    _placeholder:
-        level: "INFO"
-
-    {% if not SYNAPSE_LOG_SENSITIVE %}
-    {#
-      If SYNAPSE_LOG_SENSITIVE is unset, then override synapse.storage.SQL to INFO
-      so that DEBUG entries (containing sensitive information) are not emitted.
-    #}
     synapse.storage.SQL:
         # beware: increasing this to DEBUG will make synapse log sensitive
         # information such as access tokens.
         level: INFO
-    {% endif %}
-
-    {% if SYNAPSE_LOG_TESTING %}
-    {#
-      If Synapse is under test, log a few more useful things for a developer
-      attempting to debug something particularly tricky.
-
-      With `synapse.visibility.filtered_event_debug`, it logs when events are (maybe
-      unexpectedly) filtered out of responses in tests. It's just nice to be able to
-      look at the CI log and figure out why an event isn't being returned.
-    #}
-    synapse.visibility.filtered_event_debug:
-        level: DEBUG
-    {% endif %}
+{% endif %}
 
 root:
     level: {{ SYNAPSE_LOG_LEVEL or "INFO" }}

docker/configure_workers_and_start.py

@@ -19,15 +19,8 @@
 # The environment variables it reads are:
 #   * SYNAPSE_SERVER_NAME: The desired server_name of the homeserver.
 #   * SYNAPSE_REPORT_STATS: Whether to report stats.
-#   * SYNAPSE_WORKER_TYPES: A comma separated list of worker names as specified in WORKERS_CONFIG
-#         below. Leave empty for no workers. Add a ':' and a number at the end to
-#         multiply that worker. Append multiple worker types with '+' to merge the
-#         worker types into a single worker. Add a name and a '=' to the front of a
-#         worker type to give this instance a name in logs and nginx.
-#         Examples:
-#         SYNAPSE_WORKER_TYPES='event_persister, federation_sender, client_reader'
-#         SYNAPSE_WORKER_TYPES='event_persister:2, federation_sender:2, client_reader'
-#         SYNAPSE_WORKER_TYPES='stream_writers=account_data+presence+typing'
+#   * SYNAPSE_WORKER_TYPES: A comma separated list of worker names as specified in WORKER_CONFIG
+#         below. Leave empty for no workers, or set to '*' for all possible workers.
 #   * SYNAPSE_AS_REGISTRATION_DIR: If specified, a directory in which .yaml and .yml files
 #         will be treated as Application Service registration files.
 #   * SYNAPSE_TLS_CERT: Path to a TLS certificate in PEM format.
@@ -40,57 +33,29 @@
 #         log level. INFO is the default.
 #   * SYNAPSE_LOG_SENSITIVE: If unset, SQL and SQL values won't be logged,
 #         regardless of the SYNAPSE_LOG_LEVEL setting.
-#   * SYNAPSE_LOG_TESTING: if set, Synapse will log additional information useful
-#     for testing.
 #
 # NOTE: According to Complement's ENTRYPOINT expectations for a homeserver image (as defined
 # in the project's README), this script may be run multiple times, and functionality should
 # continue to work if so.
 
 import os
-import platform
-import re
 import subprocess
 import sys
-from collections import defaultdict
-from itertools import chain
 from pathlib import Path
-from typing import (
-    Any,
-    Dict,
-    List,
-    Mapping,
-    MutableMapping,
-    NoReturn,
-    Optional,
-    Set,
-    SupportsIndex,
-)
+from typing import Any, Dict, List, Mapping, MutableMapping, NoReturn, Optional, Set
 
 import yaml
 from jinja2 import Environment, FileSystemLoader
 
 MAIN_PROCESS_HTTP_LISTENER_PORT = 8080
-MAIN_PROCESS_INSTANCE_NAME = "main"
-MAIN_PROCESS_LOCALHOST_ADDRESS = "127.0.0.1"
-MAIN_PROCESS_REPLICATION_PORT = 9093
 
-# A simple name used as a placeholder in the WORKERS_CONFIG below. This will be replaced
-# during processing with the name of the worker.
-WORKER_PLACEHOLDER_NAME = "placeholder_name"
 
-# Workers with exposed endpoints needs either "client", "federation", or "media" listener_resources
-# Watching /_matrix/client needs a "client" listener
-# Watching /_matrix/federation needs a "federation" listener
-# Watching /_matrix/media and related needs a "media" listener
-# Stream Writers require "client" and "replication" listeners because they
-#   have to attach by instance_map to the master process and have client endpoints.
 WORKERS_CONFIG: Dict[str, Dict[str, Any]] = {
     "pusher": {
-        "app": "synapse.app.generic_worker",
+        "app": "synapse.app.pusher",
         "listener_resources": [],
         "endpoint_patterns": [],
-        "shared_extra_conf": {},
+        "shared_extra_conf": {"start_pushers": False},
         "worker_extra_conf": "",
     },
     "user_dir": {
@@ -99,13 +64,11 @@ WORKERS_CONFIG: Dict[str, Dict[str, Any]] = {
         "endpoint_patterns": [
             "^/_matrix/client/(api/v1|r0|v3|unstable)/user_directory/search$"
         ],
-        "shared_extra_conf": {
-            "update_user_directory_from_worker": WORKER_PLACEHOLDER_NAME
-        },
+        "shared_extra_conf": {"update_user_directory_from_worker": "user_dir1"},
         "worker_extra_conf": "",
     },
     "media_repository": {
-        "app": "synapse.app.generic_worker",
+        "app": "synapse.app.media_repository",
         "listener_resources": ["media"],
         "endpoint_patterns": [
             "^/_matrix/media/",
@@ -115,27 +78,21 @@ WORKERS_CONFIG: Dict[str, Dict[str, Any]] = {
             "^/_synapse/admin/v1/media/.*$",
             "^/_synapse/admin/v1/quarantine_media/.*$",
         ],
-        # The first configured media worker will run the media background jobs
-        "shared_extra_conf": {
-            "enable_media_repo": False,
-            "media_instance_running_background_jobs": WORKER_PLACEHOLDER_NAME,
-        },
+        "shared_extra_conf": {"enable_media_repo": False},
         "worker_extra_conf": "enable_media_repo: true",
     },
     "appservice": {
         "app": "synapse.app.generic_worker",
         "listener_resources": [],
         "endpoint_patterns": [],
-        "shared_extra_conf": {
-            "notify_appservices_from_worker": WORKER_PLACEHOLDER_NAME
-        },
+        "shared_extra_conf": {"notify_appservices_from_worker": "appservice1"},
         "worker_extra_conf": "",
     },
     "federation_sender": {
-        "app": "synapse.app.generic_worker",
+        "app": "synapse.app.federation_sender",
         "listener_resources": [],
         "endpoint_patterns": [],
-        "shared_extra_conf": {},
+        "shared_extra_conf": {"send_federation": False},
         "worker_extra_conf": "",
     },
     "synchrotron": {
@@ -150,40 +107,6 @@ WORKERS_CONFIG: Dict[str, Dict[str, Any]] = {
         "shared_extra_conf": {},
         "worker_extra_conf": "",
     },
-    "client_reader": {
-        "app": "synapse.app.generic_worker",
-        "listener_resources": ["client"],
-        "endpoint_patterns": [
-            "^/_matrix/client/(api/v1|r0|v3|unstable)/publicRooms$",
-            "^/_matrix/client/(api/v1|r0|v3|unstable)/rooms/.*/joined_members$",
-            "^/_matrix/client/(api/v1|r0|v3|unstable)/rooms/.*/context/.*$",
-            "^/_matrix/client/(api/v1|r0|v3|unstable)/rooms/.*/members$",
-            "^/_matrix/client/(api/v1|r0|v3|unstable)/rooms/.*/state$",
-            "^/_matrix/client/v1/rooms/.*/hierarchy$",
-            "^/_matrix/client/(v1|unstable)/rooms/.*/relations/",
-            "^/_matrix/client/v1/rooms/.*/threads$",
-            "^/_matrix/client/(api/v1|r0|v3|unstable)/login$",
-            "^/_matrix/client/(api/v1|r0|v3|unstable)/account/3pid$",
-            "^/_matrix/client/(api/v1|r0|v3|unstable)/account/whoami$",
-            "^/_matrix/client/versions$",
-            "^/_matrix/client/(api/v1|r0|v3|unstable)/voip/turnServer$",
-            "^/_matrix/client/(r0|v3|unstable)/register$",
-            "^/_matrix/client/(r0|v3|unstable)/register/available$",
-            "^/_matrix/client/(r0|v3|unstable)/auth/.*/fallback/web$",
-            "^/_matrix/client/(api/v1|r0|v3|unstable)/rooms/.*/messages$",
-            "^/_matrix/client/(api/v1|r0|v3|unstable)/rooms/.*/event",
-            "^/_matrix/client/(api/v1|r0|v3|unstable)/joined_rooms",
-            "^/_matrix/client/(api/v1|r0|v3|unstable/.*)/rooms/.*/aliases",
-            "^/_matrix/client/v1/rooms/.*/timestamp_to_event$",
-            "^/_matrix/client/(api/v1|r0|v3|unstable)/search",
-            "^/_matrix/client/(r0|v3|unstable)/user/.*/filter(/|$)",
-            "^/_matrix/client/(r0|v3|unstable)/password_policy$",
-            "^/_matrix/client/(api/v1|r0|v3|unstable)/directory/room/.*$",
-            "^/_matrix/client/(r0|v3|unstable)/capabilities$",
-        ],
-        "shared_extra_conf": {},
-        "worker_extra_conf": "",
-    },
     "federation_reader": {
         "app": "synapse.app.generic_worker",
         "listener_resources": ["federation"],
@@ -202,7 +125,6 @@ WORKERS_CONFIG: Dict[str, Dict[str, Any]] = {
             "^/_matrix/federation/(v1|v2)/invite/",
             "^/_matrix/federation/(v1|v2)/query_auth/",
             "^/_matrix/federation/(v1|v2)/event_auth/",
-            "^/_matrix/federation/v1/timestamp_to_event/",
             "^/_matrix/federation/(v1|v2)/exchange_third_party_invite/",
             "^/_matrix/federation/(v1|v2)/user/devices/",
             "^/_matrix/federation/(v1|v2)/get_groups_publicised$",
@@ -229,9 +151,9 @@ WORKERS_CONFIG: Dict[str, Dict[str, Any]] = {
         "app": "synapse.app.generic_worker",
         "listener_resources": [],
         "endpoint_patterns": [],
-        # This worker cannot be sharded. Therefore, there should only ever be one
-        # background worker. This is enforced for the safety of your database.
-        "shared_extra_conf": {"run_background_tasks_on": WORKER_PLACEHOLDER_NAME},
+        # This worker cannot be sharded. Therefore there should only ever be one background
+        # worker, and it should be named background_worker1
+        "shared_extra_conf": {"run_background_tasks_on": "background_worker1"},
         "worker_extra_conf": "",
     },
     "event_creator": {
@@ -242,61 +164,21 @@ WORKERS_CONFIG: Dict[str, Dict[str, Any]] = {
             "^/_matrix/client/(api/v1|r0|v3|unstable)/rooms/.*/send",
             "^/_matrix/client/(api/v1|r0|v3|unstable)/rooms/.*/(join|invite|leave|ban|unban|kick)$",
             "^/_matrix/client/(api/v1|r0|v3|unstable)/join/",
-            "^/_matrix/client/(api/v1|r0|v3|unstable)/knock/",
             "^/_matrix/client/(api/v1|r0|v3|unstable)/profile/",
+            "^/_matrix/client/(v1|unstable/org.matrix.msc2716)/rooms/.*/batch_send",
         ],
         "shared_extra_conf": {},
         "worker_extra_conf": "",
     },
     "frontend_proxy": {
-        "app": "synapse.app.generic_worker",
+        "app": "synapse.app.frontend_proxy",
         "listener_resources": ["client", "replication"],
         "endpoint_patterns": ["^/_matrix/client/(api/v1|r0|v3|unstable)/keys/upload"],
         "shared_extra_conf": {},
-        "worker_extra_conf": "",
-    },
-    "account_data": {
-        "app": "synapse.app.generic_worker",
-        "listener_resources": ["client", "replication"],
-        "endpoint_patterns": [
-            "^/_matrix/client/(r0|v3|unstable)/.*/tags",
-            "^/_matrix/client/(r0|v3|unstable)/.*/account_data",
-        ],
-        "shared_extra_conf": {},
-        "worker_extra_conf": "",
-    },
-    "presence": {
-        "app": "synapse.app.generic_worker",
-        "listener_resources": ["client", "replication"],
-        "endpoint_patterns": ["^/_matrix/client/(api/v1|r0|v3|unstable)/presence/"],
-        "shared_extra_conf": {},
-        "worker_extra_conf": "",
-    },
-    "receipts": {
-        "app": "synapse.app.generic_worker",
-        "listener_resources": ["client", "replication"],
-        "endpoint_patterns": [
-            "^/_matrix/client/(r0|v3|unstable)/rooms/.*/receipt",
-            "^/_matrix/client/(r0|v3|unstable)/rooms/.*/read_markers",
-        ],
-        "shared_extra_conf": {},
-        "worker_extra_conf": "",
-    },
-    "to_device": {
-        "app": "synapse.app.generic_worker",
-        "listener_resources": ["client", "replication"],
-        "endpoint_patterns": ["^/_matrix/client/(r0|v3|unstable)/sendToDevice/"],
-        "shared_extra_conf": {},
-        "worker_extra_conf": "",
-    },
-    "typing": {
-        "app": "synapse.app.generic_worker",
-        "listener_resources": ["client", "replication"],
-        "endpoint_patterns": [
-            "^/_matrix/client/(api/v1|r0|v3|unstable)/rooms/.*/typing"
-        ],
-        "shared_extra_conf": {},
-        "worker_extra_conf": "",
+        "worker_extra_conf": (
+            "worker_main_http_uri: http://127.0.0.1:%d"
+            % (MAIN_PROCESS_HTTP_LISTENER_PORT,)
+        ),
     },
 }
 
@@ -311,7 +193,7 @@ NGINX_LOCATION_CONFIG_BLOCK = """
 """
 
 NGINX_UPSTREAM_CONFIG_BLOCK = """
-upstream {upstream_worker_base_name} {{
+upstream {upstream_worker_type} {{
 {body}
 }}
 """
@@ -319,19 +201,24 @@ upstream {upstream_worker_base_name} {{
 
 # Utility functions
 def log(txt: str) -> None:
+    """Log something to the stdout.
+
+    Args:
+        txt: The text to log.
+    """
     print(txt)
 
 
 def error(txt: str) -> NoReturn:
-    print(txt, file=sys.stderr)
+    """Log something and exit with an error code.
+
+    Args:
+        txt: The text to log in error.
+    """
+    log(txt)
     sys.exit(2)
 
 
-def flush_buffers() -> None:
-    sys.stdout.flush()
-    sys.stderr.flush()
-
-
 def convert(src: str, dst: str, **template_vars: object) -> None:
     """Generate a file from a template
 
@@ -360,46 +247,32 @@ def convert(src: str, dst: str, **template_vars: object) -> None:
         outfile.write(rendered)
 
 
-def add_worker_roles_to_shared_config(
+def add_sharding_to_shared_config(
     shared_config: dict,
-    worker_types_set: Set[str],
+    worker_type: str,
     worker_name: str,
     worker_port: int,
 ) -> None:
     """Given a dictionary representing a config file shared across all workers,
-    append appropriate worker information to it for the current worker_type instance.
+    append sharded worker information to it for the current worker_type instance.
 
     Args:
-        shared_config: The config dict that all worker instances share (after being
-            converted to YAML)
-        worker_types_set: The type of worker (one of those defined in WORKERS_CONFIG).
-            This list can be a single worker type or multiple.
+        shared_config: The config dict that all worker instances share (after being converted to YAML)
+        worker_type: The type of worker (one of those defined in WORKERS_CONFIG).
         worker_name: The name of the worker instance.
         worker_port: The HTTP replication port that the worker instance is listening on.
     """
-    # The instance_map config field marks the workers that write to various replication
-    # streams
+    # The instance_map config field marks the workers that write to various replication streams
     instance_map = shared_config.setdefault("instance_map", {})
 
-    # This is a list of the stream_writers that there can be only one of. Events can be
-    # sharded, and therefore doesn't belong here.
-    singular_stream_writers = [
-        "account_data",
-        "presence",
-        "receipts",
-        "to_device",
-        "typing",
-    ]
-
-    # Worker-type specific sharding config. Now a single worker can fulfill multiple
-    # roles, check each.
-    if "pusher" in worker_types_set:
+    # Worker-type specific sharding config
+    if worker_type == "pusher":
         shared_config.setdefault("pusher_instances", []).append(worker_name)
 
-    if "federation_sender" in worker_types_set:
+    elif worker_type == "federation_sender":
         shared_config.setdefault("federation_sender_instances", []).append(worker_name)
 
-    if "event_persister" in worker_types_set:
+    elif worker_type == "event_persister":
         # Event persisters write to the events stream, so we need to update
         # the list of event stream writers
         shared_config.setdefault("stream_writers", {}).setdefault("events", []).append(
@@ -412,154 +285,9 @@ def add_worker_roles_to_shared_config(
             "port": worker_port,
         }
 
-    # Update the list of stream writers. It's convenient that the name of the worker
-    # type is the same as the stream to write. Iterate over the whole list in case there
-    # is more than one.
-    for worker in worker_types_set:
-        if worker in singular_stream_writers:
-            shared_config.setdefault("stream_writers", {}).setdefault(
-                worker, []
-            ).append(worker_name)
-
-            # Map of stream writer instance names to host/ports combos
-            # For now, all stream writers need http replication ports
-            instance_map[worker_name] = {
-                "host": "localhost",
-                "port": worker_port,
-            }
-
-
-def merge_worker_template_configs(
-    existing_dict: Optional[Dict[str, Any]],
-    to_be_merged_dict: Dict[str, Any],
-) -> Dict[str, Any]:
-    """When given an existing dict of worker template configuration consisting with both
-        dicts and lists, merge new template data from WORKERS_CONFIG(or create) and
-        return new dict.
-
-    Args:
-        existing_dict: Either an existing worker template or a fresh blank one.
-        to_be_merged_dict: The template from WORKERS_CONFIGS to be merged into
-            existing_dict.
-    Returns: The newly merged together dict values.
-    """
-    new_dict: Dict[str, Any] = {}
-    if not existing_dict:
-        # It doesn't exist yet, just use the new dict(but take a copy not a reference)
-        new_dict = to_be_merged_dict.copy()
-    else:
-        for i in to_be_merged_dict.keys():
-            if (i == "endpoint_patterns") or (i == "listener_resources"):
-                # merge the two lists, remove duplicates
-                new_dict[i] = list(set(existing_dict[i] + to_be_merged_dict[i]))
-            elif i == "shared_extra_conf":
-                # merge dictionary's, the worker name will be replaced later
-                new_dict[i] = {**existing_dict[i], **to_be_merged_dict[i]}
-            elif i == "worker_extra_conf":
-                # There is only one worker type that has a 'worker_extra_conf' and it is
-                # the media_repo. Since duplicate worker types on the same worker don't
-                # work, this is fine.
-                new_dict[i] = existing_dict[i] + to_be_merged_dict[i]
-            else:
-                # Everything else should be identical, like "app", which only works
-                # because all apps are now generic_workers.
-                new_dict[i] = to_be_merged_dict[i]
-    return new_dict
-
-
-def insert_worker_name_for_worker_config(
-    existing_dict: Dict[str, Any], worker_name: str
-) -> Dict[str, Any]:
-    """Insert a given worker name into the worker's configuration dict.
-
-    Args:
-        existing_dict: The worker_config dict that is imported into shared_config.
-        worker_name: The name of the worker to insert.
-    Returns: Copy of the dict with newly inserted worker name
-    """
-    dict_to_edit = existing_dict.copy()
-    for k, v in dict_to_edit["shared_extra_conf"].items():
-        # Only proceed if it's the placeholder name string
-        if v == WORKER_PLACEHOLDER_NAME:
-            dict_to_edit["shared_extra_conf"][k] = worker_name
-    return dict_to_edit
-
-
-def apply_requested_multiplier_for_worker(worker_types: List[str]) -> List[str]:
-    """
-    Apply multiplier(if found) by returning a new expanded list with some basic error
-    checking.
-
-    Args:
-        worker_types: The unprocessed List of requested workers
-    Returns:
-        A new list with all requested workers expanded.
-    """
-    # Checking performed:
-    # 1. if worker:2 or more is declared, it will create additional workers up to number
-    # 2. if worker:1, it will create a single copy of this worker as if no number was
-    #   given
-    # 3. if worker:0 is declared, this worker will be ignored. This is to allow for
-    #   scripting and automated expansion and is intended behaviour.
-    # 4. if worker:NaN or is a negative number, it will error and log it.
-    new_worker_types = []
-    for worker_type in worker_types:
-        if ":" in worker_type:
-            worker_type_components = split_and_strip_string(worker_type, ":", 1)
-            worker_count = 0
-            # Should only be 2 components, a type of worker(s) and an integer as a
-            # string. Cast the number as an int then it can be used as a counter.
-            try:
-                worker_count = int(worker_type_components[1])
-            except ValueError:
-                error(
-                    f"Bad number in worker count for '{worker_type}': "
-                    f"'{worker_type_components[1]}' is not an integer"
-                )
-
-            # As long as there are more than 0, we add one to the list to make below.
-            for _ in range(worker_count):
-                new_worker_types.append(worker_type_components[0])
-
-        else:
-            # If it's not a real worker_type, it will error out later.
-            new_worker_types.append(worker_type)
-    return new_worker_types
-
-
-def is_sharding_allowed_for_worker_type(worker_type: str) -> bool:
-    """Helper to check to make sure worker types that cannot have multiples do not.
-
-    Args:
-        worker_type: The type of worker to check against.
-    Returns: True if allowed, False if not
-    """
-    return worker_type not in [
-        "background_worker",
-        "account_data",
-        "presence",
-        "receipts",
-        "typing",
-        "to_device",
-    ]
-
-
-def split_and_strip_string(
-    given_string: str, split_char: str, max_split: SupportsIndex = -1
-) -> List[str]:
-    """
-    Helper to split a string on split_char and strip whitespace from each end of each
-        element.
-    Args:
-        given_string: The string to split
-        split_char: The character to split the string on
-        max_split: kwarg for split() to limit how many times the split() happens
-    Returns:
-        A List of strings
-    """
-    # Removes whitespace from ends of result strings before adding to list. Allow for
-    # overriding 'maxsplit' kwarg, default being -1 to signify no maximum.
-    return [x.strip() for x in given_string.split(split_char, maxsplit=max_split)]
+    elif worker_type == "media_repository":
+        # The first configured media worker will run the media background jobs
+        shared_config.setdefault("media_instance_running_background_jobs", worker_name)
 
 
 def generate_base_homeserver_config() -> None:
@@ -571,160 +299,36 @@ def generate_base_homeserver_config() -> None:
     # start.py already does this for us, so just call that.
     # note that this script is copied in in the official, monolith dockerfile
     os.environ["SYNAPSE_HTTP_PORT"] = str(MAIN_PROCESS_HTTP_LISTENER_PORT)
-    subprocess.run(["/usr/local/bin/python", "/start.py", "migrate_config"], check=True)
-
-
-def parse_worker_types(
-    requested_worker_types: List[str],
-) -> Dict[str, Set[str]]:
-    """Read the desired list of requested workers and prepare the data for use in
-        generating worker config files while also checking for potential gotchas.
-
-    Args:
-        requested_worker_types: The list formed from the split environment variable
-            containing the unprocessed requests for workers.
-
-    Returns: A dict of worker names to set of worker types. Format:
-        {'worker_name':
-            {'worker_type', 'worker_type2'}
-        }
-    """
-    # A counter of worker_base_name -> int. Used for determining the name for a given
-    # worker when generating its config file, as each worker's name is just
-    # worker_base_name followed by instance number
-    worker_base_name_counter: Dict[str, int] = defaultdict(int)
-
-    # Similar to above, but more finely grained. This is used to determine we don't have
-    # more than a single worker for cases where multiples would be bad(e.g. presence).
-    worker_type_shard_counter: Dict[str, int] = defaultdict(int)
-
-    # The final result of all this processing
-    dict_to_return: Dict[str, Set[str]] = {}
-
-    # Handle any multipliers requested for given workers.
-    multiple_processed_worker_types = apply_requested_multiplier_for_worker(
-        requested_worker_types
-    )
-
-    # Process each worker_type_string
-    # Examples of expected formats:
-    #  - requested_name=type1+type2+type3
-    #  - synchrotron
-    #  - event_creator+event_persister
-    for worker_type_string in multiple_processed_worker_types:
-        # First, if a name is requested, use that — otherwise generate one.
-        worker_base_name: str = ""
-        if "=" in worker_type_string:
-            # Split on "=", remove extra whitespace from ends then make list
-            worker_type_split = split_and_strip_string(worker_type_string, "=")
-            if len(worker_type_split) > 2:
-                error(
-                    "There should only be one '=' in the worker type string. "
-                    f"Please fix: {worker_type_string}"
-                )
-
-            # Assign the name
-            worker_base_name = worker_type_split[0]
-
-            if not re.match(r"^[a-zA-Z0-9_+-]*[a-zA-Z_+-]$", worker_base_name):
-                # Apply a fairly narrow regex to the worker names. Some characters
-                # aren't safe for use in file paths or nginx configurations.
-                # Don't allow to end with a number because we'll add a number
-                # ourselves in a moment.
-                error(
-                    "Invalid worker name; please choose a name consisting of "
-                    "alphanumeric letters, _ + -, but not ending with a digit: "
-                    f"{worker_base_name!r}"
-                )
-
-            # Continue processing the remainder of the worker_type string
-            # with the name override removed.
-            worker_type_string = worker_type_split[1]
-
-        # Split the worker_type_string on "+", remove whitespace from ends then make
-        # the list a set so it's deduplicated.
-        worker_types_set: Set[str] = set(
-            split_and_strip_string(worker_type_string, "+")
-        )
-
-        if not worker_base_name:
-            # No base name specified: generate one deterministically from set of
-            # types
-            worker_base_name = "+".join(sorted(worker_types_set))
-
-        # At this point, we have:
-        #   worker_base_name which is the name for the worker, without counter.
-        #   worker_types_set which is the set of worker types for this worker.
-
-        # Validate worker_type and make sure we don't allow sharding for a worker type
-        # that doesn't support it. Will error and stop if it is a problem,
-        # e.g. 'background_worker'.
-        for worker_type in worker_types_set:
-            # Verify this is a real defined worker type. If it's not, stop everything so
-            # it can be fixed.
-            if worker_type not in WORKERS_CONFIG:
-                error(
-                    f"{worker_type} is an unknown worker type! Was found in "
-                    f"'{worker_type_string}'. Please fix!"
-                )
-
-            if worker_type in worker_type_shard_counter:
-                if not is_sharding_allowed_for_worker_type(worker_type):
-                    error(
-                        f"There can be only a single worker with {worker_type} "
-                        "type. Please recount and remove."
-                    )
-            # Not in shard counter, must not have seen it yet, add it.
-            worker_type_shard_counter[worker_type] += 1
-
-        # Generate the number for the worker using incrementing counter
-        worker_base_name_counter[worker_base_name] += 1
-        worker_number = worker_base_name_counter[worker_base_name]
-        worker_name = f"{worker_base_name}{worker_number}"
-
-        if worker_number > 1:
-            # If this isn't the first worker, check that we don't have a confusing
-            # mixture of worker types with the same base name.
-            first_worker_with_base_name = dict_to_return[f"{worker_base_name}1"]
-            if first_worker_with_base_name != worker_types_set:
-                error(
-                    f"Can not use worker_name: '{worker_name}' for worker_type(s): "
-                    f"{worker_types_set!r}. It is already in use by "
-                    f"worker_type(s): {first_worker_with_base_name!r}"
-                )
-
-        dict_to_return[worker_name] = worker_types_set
-
-    return dict_to_return
+    subprocess.check_output(["/usr/local/bin/python", "/start.py", "migrate_config"])
 
 
 def generate_worker_files(
-    environ: Mapping[str, str],
-    config_path: str,
-    data_dir: str,
-    requested_worker_types: Dict[str, Set[str]],
+    environ: Mapping[str, str], config_path: str, data_dir: str
 ) -> None:
-    """Read the desired workers(if any) that is passed in and generate shared
-        homeserver, nginx and supervisord configs.
+    """Read the desired list of workers from environment variables and generate
+    shared homeserver, nginx and supervisord configs.
 
     Args:
         environ: os.environ instance.
         config_path: The location of the generated Synapse main worker config file.
         data_dir: The location of the synapse data directory. Where log and
             user-facing config files live.
-        requested_worker_types: A Dict containing requested workers in the format of
-            {'worker_name1': {'worker_type', ...}}
     """
     # Note that yaml cares about indentation, so care should be taken to insert lines
     # into files at the correct indentation below.
 
-    # First read the original config file and extract the listeners block. Then we'll
-    # add another listener for replication. Later we'll write out the result to the
-    # shared config file.
+    # shared_config is the contents of a Synapse config file that will be shared amongst
+    # the main Synapse process as well as all workers.
+    # It is intended mainly for disabling functionality when certain workers are spun up,
+    # and adding a replication listener.
+
+    # First read the original config file and extract the listeners block. Then we'll add
+    # another listener for replication. Later we'll write out the result to the shared
+    # config file.
     listeners = [
         {
-            "port": MAIN_PROCESS_REPLICATION_PORT,
-            "bind_address": MAIN_PROCESS_LOCALHOST_ADDRESS,
+            "port": 9093,
+            "bind_address": "127.0.0.1",
             "type": "http",
             "resources": [{"names": ["replication"]}],
         }
@@ -736,9 +340,9 @@ def generate_worker_files(
             listeners += original_listeners
 
     # The shared homeserver config. The contents of which will be inserted into the
-    # base shared worker jinja2 template. This config file will be passed to all
-    # workers, included Synapse's main process. It is intended mainly for disabling
-    # functionality when certain workers are spun up, and adding a replication listener.
+    # base shared worker jinja2 template.
+    #
+    # This config file will be passed to all workers, included Synapse's main process.
     shared_config: Dict[str, Any] = {"listeners": listeners}
 
     # List of dicts that describe workers.
@@ -746,20 +350,31 @@ def generate_worker_files(
     # program blocks.
     worker_descriptors: List[Dict[str, Any]] = []
 
-    # Upstreams for load-balancing purposes. This dict takes the form of the worker
-    # type to the ports of each worker. For example:
+    # Upstreams for load-balancing purposes. This dict takes the form of a worker type to the
+    # ports of each worker. For example:
     # {
     #   worker_type: {1234, 1235, ...}}
     # }
     # and will be used to construct 'upstream' nginx directives.
     nginx_upstreams: Dict[str, Set[int]] = {}
 
-    # A map of: {"endpoint": "upstream"}, where "upstream" is a str representing what
-    # will be placed after the proxy_pass directive. The main benefit to representing
-    # this data as a dict over a str is that we can easily deduplicate endpoints
-    # across multiple instances of the same worker. The final rendering will be combined
-    # with nginx_upstreams and placed in /etc/nginx/conf.d.
-    nginx_locations: Dict[str, str] = {}
+    # A map of: {"endpoint": "upstream"}, where "upstream" is a str representing what will be
+    # placed after the proxy_pass directive. The main benefit to representing this data as a
+    # dict over a str is that we can easily deduplicate endpoints across multiple instances
+    # of the same worker.
+    #
+    # An nginx site config that will be amended to depending on the workers that are
+    # spun up. To be placed in /etc/nginx/conf.d.
+    nginx_locations = {}
+
+    # Read the desired worker configuration from the environment
+    worker_types_env = environ.get("SYNAPSE_WORKER_TYPES", "").strip()
+    if not worker_types_env:
+        # No workers, just the main process
+        worker_types = []
+    else:
+        # Split type names by comma
+        worker_types = worker_types_env.split(",")
 
     # Create the worker configuration directory if it doesn't already exist
     os.makedirs("/conf/workers", exist_ok=True)
@@ -767,57 +382,69 @@ def generate_worker_files(
     # Start worker ports from this arbitrary port
     worker_port = 18009
 
+    # A counter of worker_type -> int. Used for determining the name for a given
+    # worker type when generating its config file, as each worker's name is just
+    # worker_type + instance #
+    worker_type_counter: Dict[str, int] = {}
+
     # A list of internal endpoints to healthcheck, starting with the main process
     # which exists even if no workers do.
     healthcheck_urls = ["http://localhost:8080/health"]
 
-    # Get the set of all worker types that we have configured
-    all_worker_types_in_use = set(chain(*requested_worker_types.values()))
-    # Map locations to upstreams (corresponding to worker types) in Nginx
-    # but only if we use the appropriate worker type
-    for worker_type in all_worker_types_in_use:
-        for endpoint_pattern in WORKERS_CONFIG[worker_type]["endpoint_patterns"]:
-            nginx_locations[endpoint_pattern] = f"http://{worker_type}"
+    # For each worker type specified by the user, create config values
+    for worker_type in worker_types:
+        worker_type = worker_type.strip()
 
-    # For each worker type specified by the user, create config values and write it's
-    # yaml config file
-    for worker_name, worker_types_set in requested_worker_types.items():
-        # The collected and processed data will live here.
-        worker_config: Dict[str, Any] = {}
+        worker_config = WORKERS_CONFIG.get(worker_type)
+        if worker_config:
+            worker_config = worker_config.copy()
+        else:
+            log(worker_type + " is an unknown worker type! It will be ignored")
+            continue
 
-        # Merge all worker config templates for this worker into a single config
-        for worker_type in worker_types_set:
-            copy_of_template_config = WORKERS_CONFIG[worker_type].copy()
-
-            # Merge worker type template configuration data. It's a combination of lists
-            # and dicts, so use this helper.
-            worker_config = merge_worker_template_configs(
-                worker_config, copy_of_template_config
-            )
-
-        # Replace placeholder names in the config template with the actual worker name.
-        worker_config = insert_worker_name_for_worker_config(worker_config, worker_name)
+        new_worker_count = worker_type_counter.setdefault(worker_type, 0) + 1
+        worker_type_counter[worker_type] = new_worker_count
 
+        # Name workers by their type concatenated with an incrementing number
+        # e.g. federation_reader1
+        worker_name = worker_type + str(new_worker_count)
         worker_config.update(
             {"name": worker_name, "port": str(worker_port), "config_path": config_path}
         )
 
-        # Update the shared config with any worker_type specific options. The first of a
-        # given worker_type needs to stay assigned and not be replaced.
-        worker_config["shared_extra_conf"].update(shared_config)
-        shared_config = worker_config["shared_extra_conf"]
+        # Update the shared config with any worker-type specific options
+        shared_config.update(worker_config["shared_extra_conf"])
 
         healthcheck_urls.append("http://localhost:%d/health" % (worker_port,))
 
-        # Update the shared config with sharding-related options if necessary
-        add_worker_roles_to_shared_config(
-            shared_config, worker_types_set, worker_name, worker_port
-        )
+        # Check if more than one instance of this worker type has been specified
+        worker_type_total_count = worker_types.count(worker_type)
+        if worker_type_total_count > 1:
+            # Update the shared config with sharding-related options if necessary
+            add_sharding_to_shared_config(
+                shared_config, worker_type, worker_name, worker_port
+            )
 
         # Enable the worker in supervisord
         worker_descriptors.append(worker_config)
 
+        # Add nginx location blocks for this worker's endpoints (if any are defined)
+        for pattern in worker_config["endpoint_patterns"]:
+            # Determine whether we need to load-balance this worker
+            if worker_type_total_count > 1:
+                # Create or add to a load-balanced upstream for this worker
+                nginx_upstreams.setdefault(worker_type, set()).add(worker_port)
+
+                # Upstreams are named after the worker_type
+                upstream = "http://" + worker_type
+            else:
+                upstream = "http://localhost:%d" % (worker_port,)
+
+            # Note that this endpoint should proxy to this upstream
+            nginx_locations[pattern] = upstream
+
         # Write out the worker's logging config file
+
         log_config_filepath = generate_worker_log_config(environ, worker_name, data_dir)
 
         # Then a worker config file
@@ -828,10 +455,6 @@ def generate_worker_files(
             worker_log_config_filepath=log_config_filepath,
         )
 
-        # Save this worker's port number to the correct nginx upstreams
-        for worker_type in worker_types_set:
-            nginx_upstreams.setdefault(worker_type, set()).add(worker_port)
-
         worker_port += 1
 
     # Build the nginx location config blocks
@@ -844,14 +467,15 @@ def generate_worker_files(
 
     # Determine the load-balancing upstreams to configure
     nginx_upstream_config = ""
-    for upstream_worker_base_name, upstream_worker_ports in nginx_upstreams.items():
+
+    for upstream_worker_type, upstream_worker_ports in nginx_upstreams.items():
         body = ""
         for port in upstream_worker_ports:
-            body += f"    server localhost:{port};\n"
+            body += "    server localhost:%d;\n" % (port,)
 
         # Add to the list of configured upstreams
         nginx_upstream_config += NGINX_UPSTREAM_CONFIG_BLOCK.format(
-            upstream_worker_base_name=upstream_worker_base_name,
+            upstream_worker_type=upstream_worker_type,
             body=body,
         )
 
@@ -872,15 +496,7 @@ def generate_worker_files(
             if reg_path.suffix.lower() in (".yaml", ".yml")
         ]
 
-    workers_in_use = len(requested_worker_types) > 0
-
-    # If there are workers, add the main process to the instance_map too.
-    if workers_in_use:
-        instance_map = shared_config.setdefault("instance_map", {})
-        instance_map[MAIN_PROCESS_INSTANCE_NAME] = {
-            "host": MAIN_PROCESS_LOCALHOST_ADDRESS,
-            "port": MAIN_PROCESS_REPLICATION_PORT,
-        }
+    workers_in_use = len(worker_types) > 0
 
     # Shared homeserver config
     convert(
@@ -948,7 +564,6 @@ def generate_worker_log_config(
     extra_log_template_args["SYNAPSE_LOG_SENSITIVE"] = environ.get(
         "SYNAPSE_LOG_SENSITIVE"
     )
-    extra_log_template_args["SYNAPSE_LOG_TESTING"] = environ.get("SYNAPSE_LOG_TESTING")
 
     # Render and write the file
     log_config_filepath = f"/conf/workers/{worker_name}.log.config"
@@ -977,53 +592,26 @@ def main(args: List[str], environ: MutableMapping[str, str]) -> None:
     if not os.path.exists(config_path):
         log("Generating base homeserver config")
         generate_base_homeserver_config()
-    else:
-        log("Base homeserver config exists—not regenerating")
-    # This script may be run multiple times (mostly by Complement, see note at top of
-    # file). Don't re-configure workers in this instance.
+
+    # This script may be run multiple times (mostly by Complement, see note at top of file).
+    # Don't re-configure workers in this instance.
     mark_filepath = "/conf/workers_have_been_configured"
     if not os.path.exists(mark_filepath):
-        # Collect and validate worker_type requests
-        # Read the desired worker configuration from the environment
-        worker_types_env = environ.get("SYNAPSE_WORKER_TYPES", "").strip()
-        # Only process worker_types if they exist
-        if not worker_types_env:
-            # No workers, just the main process
-            worker_types = []
-            requested_worker_types: Dict[str, Any] = {}
-        else:
-            # Split type names by comma, ignoring whitespace.
-            worker_types = split_and_strip_string(worker_types_env, ",")
-            requested_worker_types = parse_worker_types(worker_types)
-
         # Always regenerate all other config files
-        log("Generating worker config files")
-        generate_worker_files(environ, config_path, data_dir, requested_worker_types)
+        generate_worker_files(environ, config_path, data_dir)
 
         # Mark workers as being configured
         with open(mark_filepath, "w") as f:
             f.write("")
-    else:
-        log("Worker config exists—not regenerating")
-
-    # Lifted right out of start.py
-    jemallocpath = "/usr/lib/%s-linux-gnu/libjemalloc.so.2" % (platform.machine(),)
-
-    if os.path.isfile(jemallocpath):
-        environ["LD_PRELOAD"] = jemallocpath
-    else:
-        log("Could not find %s, will not use" % (jemallocpath,))
 
     # Start supervisord, which will start Synapse, all of the configured worker
     # processes, redis, nginx etc. according to the config we created above.
     log("Starting supervisord")
-    flush_buffers()
-    os.execle(
+    os.execl(
         "/usr/local/bin/supervisord",
         "supervisord",
         "-c",
         "/etc/supervisor/supervisord.conf",
-        environ,
     )
 
 

docker/editable.Dockerfile

@@ -1,75 +0,0 @@
-# syntax=docker/dockerfile:1
-# This dockerfile builds an editable install of Synapse.
-#
-# Used by `complement.sh`. Not suitable for production use.
-
-ARG PYTHON_VERSION=3.9
-
-###
-### Stage 0: generate requirements.txt
-###
-# We hardcode the use of Debian bullseye here because this could change upstream
-# and other Dockerfiles used for testing are expecting bullseye.
-FROM docker.io/library/python:${PYTHON_VERSION}-slim-bullseye
-
-# Install Rust and other dependencies (stolen from normal Dockerfile)
-# install the OS build deps
-RUN \
-   --mount=type=cache,target=/var/cache/apt,sharing=locked \
-   --mount=type=cache,target=/var/lib/apt,sharing=locked \
- apt-get update -qq && apt-get install -yqq \
-    build-essential \
-    libffi-dev \
-    libjpeg-dev \
-    libpq-dev \
-    libssl-dev \
-    libwebp-dev \
-    libxml++2.6-dev \
-    libxslt1-dev \
-    openssl \
-    zlib1g-dev \
-    git \
-    curl \
-    gosu \
-    libjpeg62-turbo \
-    libpq5 \
-    libwebp6 \
-    xmlsec1 \
-    libjemalloc2 \
-    && rm -rf /var/lib/apt/lists/*
-ENV RUSTUP_HOME=/rust
-ENV CARGO_HOME=/cargo
-ENV PATH=/cargo/bin:/rust/bin:$PATH
-RUN mkdir /rust /cargo
-RUN curl -sSf https://sh.rustup.rs | sh -s -- -y --no-modify-path --default-toolchain stable --profile minimal
-
-
-# Make a base copy of the editable source tree, so that we have something to
-# install and build now — even though it's going to be covered up by a mount
-# at runtime.
-COPY synapse /editable-src/synapse/
-COPY rust /editable-src/rust/
-# ... and what we need to `pip install`.
-COPY pyproject.toml poetry.lock README.rst build_rust.py Cargo.toml Cargo.lock /editable-src/
-
-RUN pip install poetry
-RUN poetry config virtualenvs.create false
-RUN cd /editable-src && poetry install --extras all
-
-# Make copies of useful things for inspection:
-# - the Rust module (must be copied to the editable source tree before startup)
-# - poetry.lock is useful for checking if dependencies have changed.
-RUN cp /editable-src/synapse/synapse_rust.abi3.so /synapse_rust.abi3.so.bak
-RUN cp /editable-src/poetry.lock /poetry.lock.bak
-
-
-### Extra setup from original Dockerfile
-COPY ./docker/start.py /start.py
-COPY ./docker/conf /conf
-
-EXPOSE 8008/tcp 8009/tcp 8448/tcp
-
-ENTRYPOINT ["/start.py"]
-
-HEALTHCHECK --start-period=5s --interval=15s --timeout=5s \
-    CMD curl -fSs http://localhost:8008/health || exit 1

docker/start.py

@@ -13,19 +13,14 @@ import jinja2
 
 # Utility functions
 def log(txt: str) -> None:
-    print(txt)
+    print(txt, file=sys.stderr)
 
 
 def error(txt: str) -> NoReturn:
-    print(txt, file=sys.stderr)
+    log(txt)
     sys.exit(2)
 
 
-def flush_buffers() -> None:
-    sys.stdout.flush()
-    sys.stderr.flush()
-
-
 def convert(src: str, dst: str, environ: Mapping[str, object]) -> None:
     """Generate a file from a template
 
@@ -136,10 +131,10 @@ def generate_config_from_template(
 
     if ownership is not None:
         log(f"Setting ownership on /data to {ownership}")
-        subprocess.run(["chown", "-R", ownership, "/data"], check=True)
+        subprocess.check_output(["chown", "-R", ownership, "/data"])
         args = ["gosu", ownership] + args
 
-    subprocess.run(args, check=True)
+    subprocess.check_output(args)
 
 
 def run_generate_config(environ: Mapping[str, str], ownership: Optional[str]) -> None:
@@ -163,7 +158,7 @@ def run_generate_config(environ: Mapping[str, str], ownership: Optional[str]) ->
     if ownership is not None:
         # make sure that synapse has perms to write to the data dir.
         log(f"Setting ownership on {data_dir} to {ownership}")
-        subprocess.run(["chown", ownership, data_dir], check=True)
+        subprocess.check_output(["chown", ownership, data_dir])
 
     # create a suitable log config from our template
     log_config_file = "%s/%s.log.config" % (config_dir, server_name)
@@ -190,7 +185,6 @@ def run_generate_config(environ: Mapping[str, str], ownership: Optional[str]) ->
         "--open-private-ports",
     ]
     # log("running %s" % (args, ))
-    flush_buffers()
     os.execv(sys.executable, args)
 
 
@@ -273,10 +267,8 @@ running with 'migrate_config'. See the README for more details.
     args = [sys.executable] + args
     if ownership is not None:
         args = ["gosu", ownership] + args
-        flush_buffers()
         os.execve("/usr/sbin/gosu", args, environ)
     else:
-        flush_buffers()
         os.execve(sys.executable, args, environ)
 
 

docs/SUMMARY.md

@@ -9,8 +9,6 @@
   - [Configuring a Reverse Proxy](reverse_proxy.md)
   - [Configuring a Forward/Outbound Proxy](setup/forward_proxy.md)
   - [Configuring a Turn Server](turn-howto.md)
-    - [coturn TURN server](setup/turn/coturn.md)
-    - [eturnal TURN server](setup/turn/eturnal.md)
   - [Delegation](delegate.md)
 
 # Upgrading
@@ -57,7 +55,6 @@
       - [Account Validity](admin_api/account_validity.md)
       - [Background Updates](usage/administration/admin_api/background_updates.md)
       - [Event Reports](admin_api/event_reports.md)
-      - [Experimental Features](admin_api/experimental_features.md)
       - [Media](admin_api/media_admin_api.md)
       - [Purge History](admin_api/purge_history_api.md)
       - [Register Users](admin_api/register_api.md)
@@ -72,7 +69,6 @@
     - [Manhole](manhole.md)
     - [Monitoring](metrics-howto.md)
       - [Reporting Homeserver Usage Statistics](usage/administration/monitoring/reporting_homeserver_usage_statistics.md)
-    - [Monthly Active Users](usage/administration/monthly_active_users.md)
     - [Understanding Synapse Through Grafana Graphs](usage/administration/understanding_synapse_through_grafana_graphs.md)
     - [Useful SQL for Admins](usage/administration/useful_sql_for_admins.md)
     - [Database Maintenance Tools](usage/administration/database_maintenance_tools.md)
@@ -98,7 +94,6 @@
     - [Log Contexts](log_contexts.md)
     - [Replication](replication.md)
     - [TCP Replication](tcp_replication.md)
-    - [Faster remote joins](development/synapse_architecture/faster_joins.md)
   - [Internal Documentation](development/internal_documentation/README.md)
     - [Single Sign-On]()
       - [SAML](development/saml.md)

docs/admin_api/account_validity.md

@@ -5,7 +5,7 @@ use it, you must enable the account validity feature (under
 `account_validity`) in Synapse's configuration.
 
 To use it, you will need to authenticate by providing an `access_token`
-for a server admin: see [Admin API](../usage/administration/admin_api/).
+for a server admin: see [Admin API](../usage/administration/admin_api).
 
 ## Renew account
 

docs/admin_api/event_reports.md

@@ -3,7 +3,7 @@
 This API returns information about reported events.
 
 To use it, you will need to authenticate by providing an `access_token`
-for a server admin: see [Admin API](../usage/administration/admin_api/).
+for a server admin: see [Admin API](../usage/administration/admin_api).
 
 The api is:
 ```
@@ -169,17 +169,3 @@ The following fields are returned in the JSON response body:
 * `canonical_alias`: string - The canonical alias of the room. `null` if the room does not
   have a canonical alias set.
 * `event_json`: object - Details of the original event that was reported.
-
-# Delete a specific event report
-
-This API deletes a specific event report. If the request is successful, the response body
-will be an empty JSON object.
-
-The api is:
-```
-DELETE /_synapse/admin/v1/event_reports/<report_id>
-```
-
-**URL parameters:**
-
-* `report_id`: string - The ID of the event report.

docs/admin_api/experimental_features.md

@@ -1,55 +0,0 @@
-# Experimental Features API
-
-This API allows a server administrator to enable or disable some experimental features on a per-user
-basis. The currently supported features are: 
-- [MSC3026](https://github.com/matrix-org/matrix-spec-proposals/pull/3026): busy 
-presence state enabled
-- [MSC3881](https://github.com/matrix-org/matrix-spec-proposals/pull/3881): enable remotely toggling push notifications 
-for another client 
-- [MSC3967](https://github.com/matrix-org/matrix-spec-proposals/pull/3967): do not require
-UIA when first uploading cross-signing keys. 
-
-
-To use it, you will need to authenticate by providing an `access_token`
-for a server admin: see [Admin API](../usage/administration/admin_api/).
-
-## Enabling/Disabling Features
-
-This API allows a server administrator to enable experimental features for a given user. The request must 
-provide a body containing the user id and listing the features to enable/disable in the following format:
-```json
-{
-   "features": {
-      "msc3026":true,
-      "msc3881":true
-   }
-}
-```
-where true is  used to enable the feature, and false is used to disable the feature.
-
-
-The API is:
-
-```
-PUT /_synapse/admin/v1/experimental_features/<user_id>
-```
-
-## Listing Enabled Features
- 
-To list which features are enabled/disabled for a given user send a request to the following API:
-
-```
-GET /_synapse/admin/v1/experimental_features/<user_id>
-```
-
-It will return a list of possible features and indicate whether they are enabled or disabled for the
-user like so:
-```json
-{
-   "features": {
-      "msc3026": true,
-      "msc3881": false,
-      "msc3967": false
-   }
-}
-```