CCCC

BBBB
AAAA#
2022-07-07 16:05:22 +01:00 · 2022-07-07 16:05:22 +01:00 · 2022-07-07 16:05:22 +01:00 · 2022-07-07 16:05:22 +01:00 · 2022-07-07 16:05:22 +01:00 · 2022-07-07 16:05:22 +01:00
284 changed files with 6206 additions and 8486 deletions
@@ -0,0 +1,93 @@
+{{- /*gotype: github.com/haveyoudebuggedit/gotestfmt/parser.Package*/ -}}
+{{- /*
+This template contains the format for an individual package. GitHub actions does not currently support nested groups so
+we are creating a stylized header for each package.
+
+This template is based on https://github.com/haveyoudebuggedit/gotestfmt/blob/f179b0e462a9dcf7101515d87eec4e4d7e58b92a/.gotestfmt/github/package.gotpl
+which is under the Unlicense licence.
+*/ -}}
+{{- $settings := .Settings -}}
+{{- if and (or (not $settings.HideSuccessfulPackages) (ne .Result "PASS")) (or (not $settings.HideEmptyPackages) (ne .Result "SKIP") (ne (len .TestCases) 0)) -}}
+    {{- if eq .Result "PASS" -}}
+        {{ "\033" }}[0;32m
+    {{- else if eq .Result "SKIP" -}}
+        {{ "\033" }}[0;33m
+    {{- else -}}
+        {{ "\033" }}[0;31m
+    {{- end -}}
+    📦 {{ .Name }}{{- "\033" }}[0m
+    {{- with .Coverage -}}
+       {{- "\033" -}}[0;37m ({{ . }}% coverage){{- "\033" -}}[0m
+    {{- end -}}
+    {{- "\n" -}}
+    {{- with .Reason -}}
+        {{- "  " -}}🛑 {{ . -}}{{- "\n" -}}
+    {{- end -}}
+    {{- with .Output -}}
+        {{- . -}}{{- "\n" -}}
+    {{- end -}}
+    {{- with .TestCases -}}
+        {{- /* Failing tests are first */ -}}
+        {{- range . -}}
+            {{- if and (ne .Result "PASS") (ne .Result "SKIP") -}}
+                ::group::{{ "\033" }}[0;31m❌{{ " " }}{{- .Name -}}
+                {{- "\033" -}}[0;37m ({{if $settings.ShowTestStatus}}{{.Result}}; {{end}}{{ .Duration -}}
+                {{- with .Coverage -}}
+                    , coverage: {{ . }}%
+                {{- end -}})
+                {{- "\033" -}}[0m
+                {{- "\n" -}}
+
+                {{- with .Output -}}
+                    {{- formatTestOutput . $settings -}}
+                    {{- "\n" -}}
+                {{- end -}}
+
+                ::endgroup::{{- "\n" -}}
+            {{- end -}}
+        {{- end -}}
+
+
+        {{- /* Then skipped tests are second */ -}}
+        {{- range . -}}
+            {{- if eq .Result "SKIP" -}}
+                ::group::{{ "\033" }}[0;33m🚧{{ " " }}{{- .Name -}}
+                {{- "\033" -}}[0;37m ({{if $settings.ShowTestStatus}}{{.Result}}; {{end}}{{ .Duration -}}
+                {{- with .Coverage -}}
+                    , coverage: {{ . }}%
+                {{- end -}})
+                {{- "\033" -}}[0m
+                {{- "\n" -}}
+
+                {{- with .Output -}}
+                    {{- formatTestOutput . $settings -}}
+                    {{- "\n" -}}
+                {{- end -}}
+
+                ::endgroup::{{- "\n" -}}
+            {{- end -}}
+        {{- end -}}
+
+
+        {{- /* Then passing tests are last */ -}}
+        {{- range . -}}
+            {{- if eq .Result "PASS" -}}
+                ::group::{{ "\033" }}[0;32m✅{{ " " }}{{- .Name -}}
+                {{- "\033" -}}[0;37m ({{if $settings.ShowTestStatus}}{{.Result}}; {{end}}{{ .Duration -}}
+                {{- with .Coverage -}}
+                    , coverage: {{ . }}%
+                {{- end -}})
+                {{- "\033" -}}[0m
+                {{- "\n" -}}
+
+                {{- with .Output -}}
+                    {{- formatTestOutput . $settings -}}
+                    {{- "\n" -}}
+                {{- end -}}
+
+                ::endgroup::{{- "\n" -}}
+            {{- end -}}
+        {{- end -}}
+    {{- end -}}
+    {{- "\n" -}}
+{{- end -}}
@@ -0,0 +1,36 @@
+#!/bin/sh
+#
+# Common commands to set up Complement's prerequisites in a GitHub Actions CI run.
+#
+# Must be called after Synapse has been checked out to `synapse/`.
+#
+set -eu
+
+alias block='{ set +x; } 2>/dev/null; func() { echo "::group::$*"; set -x; }; func'
+alias endblock='{ set +x; } 2>/dev/null; func() { echo "::endgroup::"; set -x; }; func'
+
+block Set Go Version
+  # The path is set via a file given by $GITHUB_PATH. We need both Go 1.17 and GOPATH on the path to run Complement.
+  # See https://docs.github.com/en/actions/using-workflows/workflow-commands-for-github-actions#adding-a-system-path
+
+  # Add Go 1.17 to the PATH: see https://github.com/actions/virtual-environments/blob/main/images/linux/Ubuntu2004-Readme.md#environment-variables-2
+  echo "$GOROOT_1_17_X64/bin" >> $GITHUB_PATH
+  # Add the Go path to the PATH: We need this so we can call gotestfmt
+  echo "~/go/bin" >> $GITHUB_PATH
+endblock
+
+block Install Complement Dependencies
+  sudo apt-get -qq update && sudo apt-get install -qqy libolm3 libolm-dev
+  go get -v github.com/haveyoudebuggedit/gotestfmt/v2/cmd/gotestfmt@latest
+endblock
+
+block Install custom gotestfmt template
+  mkdir .gotestfmt/github -p
+  cp synapse/.ci/complement_package.gotpl .gotestfmt/github/package.gotpl
+endblock
+
+block Check out Complement
+  # Attempt to check out the same branch of Complement as the PR. If it
+  # doesn't exist, fallback to HEAD.
+  synapse/.ci/scripts/checkout_complement.sh
+endblock
@@ -27,9 +27,10 @@ export VIRTUALENV_NO_DOWNLOAD=1

 # Patch the project definitions in-place:
 # - Replace all lower and tilde bounds with exact bounds
-# - Make the pyopenssl 17.0, which is the oldest version that works with
-#   a `cryptography` compiled against OpenSSL 1.1.
+# - Replace all caret bounds---but not the one that defines the supported Python version!
 # - Delete all lines referring to psycopg2 --- so no testing of postgres support.
+# - Use pyopenssl 17.0, which is the oldest version that works with
+#   a `cryptography` compiled against OpenSSL 1.1.
 # - Omit systemd: we're not logging to journal here.

 # TODO: also replace caret bounds, see https://python-poetry.org/docs/dependency-specification/#version-constraints
@@ -40,6 +41,7 @@ export VIRTUALENV_NO_DOWNLOAD=1

 sed -i \
   -e "s/[~>]=/==/g" \
+   -e '/^python = "^/!s/\^/==/g' \
   -e "/psycopg2/d" \
   -e 's/pyOpenSSL = "==16.0.0"/pyOpenSSL = "==17.0.0"/' \
   -e '/systemd/d' \
@@ -10,383 +10,37 @@ concurrency:
  cancel-in-progress: true

 jobs:
-  check-sampleconfig:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
-      - uses: actions/setup-python@v2
-      - run: pip install .
-      - run: scripts-dev/generate_sample_config.sh --check
-      - run: scripts-dev/config-lint.sh
-
-  lint:
-    uses: "matrix-org/backend-meta/.github/workflows/python-poetry-ci.yml@v1"
-    with:
-      typechecking-extras: "all"
-
-  lint-crlf:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
-      - name: Check line endings
-        run: scripts-dev/check_line_terminators.sh
-
-  lint-newsfile:
-    if: ${{ github.base_ref == 'develop'  || contains(github.base_ref, 'release-') }}
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
-        with:
-          ref: ${{ github.event.pull_request.head.sha }}
-          fetch-depth: 0
-      - uses: actions/setup-python@v2
-      - run: "pip install 'towncrier>=18.6.0rc1'"
-      - run: scripts-dev/check-newsfragment.sh
-        env:
-          PULL_REQUEST_NUMBER: ${{ github.event.number }}
-
-  # Dummy step to gate other tests on without repeating the whole list
-  linting-done:
-    if: ${{ !cancelled() }} # Run this even if prior jobs were skipped
-    needs: [lint, lint-crlf, lint-newsfile, check-sampleconfig]
-    runs-on: ubuntu-latest
-    steps:
-      - run: "true"
-
-  trial:
-    if: ${{ !cancelled() && !failure() }} # Allow previous steps to be skipped, but not fail
-    needs: linting-done
-    runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        python-version: ["3.7", "3.8", "3.9", "3.10"]
-        database: ["sqlite"]
-        extras: ["all"]
-        include:
-          # Newest Python without optional deps
-          - python-version: "3.10"
-            extras: ""
-
-          # Oldest Python with PostgreSQL
-          - python-version: "3.7"
-            database: "postgres"
-            postgres-version: "10"
-            extras: "all"
-
-          # Newest Python with newest PostgreSQL
-          - python-version: "3.10"
-            database: "postgres"
-            postgres-version: "14"
-            extras: "all"
-
-    steps:
-      - uses: actions/checkout@v2
-      - run: sudo apt-get -qq install xmlsec1
-      - name: Set up PostgreSQL ${{ matrix.postgres-version }}
-        if: ${{ matrix.postgres-version }}
-        run: |
-          docker run -d -p 5432:5432 \
-            -e POSTGRES_PASSWORD=postgres \
-            -e POSTGRES_INITDB_ARGS="--lc-collate C --lc-ctype C --encoding UTF8" \
-            postgres:${{ matrix.postgres-version }}
-      - uses: matrix-org/setup-python-poetry@v1
-        with:
-          python-version: ${{ matrix.python-version }}
-          extras: ${{ matrix.extras }}
-      - name: Await PostgreSQL
-        if: ${{ matrix.postgres-version }}
-        timeout-minutes: 2
-        run: until pg_isready -h localhost; do sleep 1; done
-      - run: poetry run trial --jobs=2 tests
-        env:
-          SYNAPSE_POSTGRES: ${{ matrix.database == 'postgres' || '' }}
-          SYNAPSE_POSTGRES_HOST: localhost
-          SYNAPSE_POSTGRES_USER: postgres
-          SYNAPSE_POSTGRES_PASSWORD: postgres
-      - name: Dump logs
-        # Logs are most useful when the command fails, always include them.
-        if: ${{ always() }}
-        # Note: Dumps to workflow logs instead of using actions/upload-artifact
-        #       This keeps logs colocated with failing jobs
-        #       It also ignores find's exit code; this is a best effort affair
-        run: >-
-          find _trial_temp -name '*.log'
-          -exec echo "::group::{}" \;
-          -exec cat {} \;
-          -exec echo "::endgroup::" \;
-          || true
-
-  trial-olddeps:
-    # Note: sqlite only; no postgres
-    if: ${{ !cancelled() && !failure() }} # Allow previous steps to be skipped, but not fail
-    needs: linting-done
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
-      - name: Test with old deps
-        uses: docker://ubuntu:focal # For old python and sqlite
-        # Note: focal seems to be using 3.8, but the oldest is 3.7?
-        # See https://github.com/matrix-org/synapse/issues/12343
-        with:
-          workdir: /github/workspace
-          entrypoint: .ci/scripts/test_old_deps.sh
-      - name: Dump logs
-        # Logs are most useful when the command fails, always include them.
-        if: ${{ always() }}
-        # Note: Dumps to workflow logs instead of using actions/upload-artifact
-        #       This keeps logs colocated with failing jobs
-        #       It also ignores find's exit code; this is a best effort affair
-        run: >-
-          find _trial_temp -name '*.log'
-          -exec echo "::group::{}" \;
-          -exec cat {} \;
-          -exec echo "::endgroup::" \;
-          || true
-
-  trial-pypy:
-    # Very slow; only run if the branch name includes 'pypy'
-    # Note: sqlite only; no postgres. Completely untested since poetry move.
-    if: ${{ contains(github.ref, 'pypy') && !failure() && !cancelled() }}
-    needs: linting-done
-    runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        python-version: ["pypy-3.7"]
-        extras: ["all"]
-
-    steps:
-      - uses: actions/checkout@v2
-      # Install libs necessary for PyPy to build binary wheels for dependencies
-      - run: sudo apt-get -qq install xmlsec1 libxml2-dev libxslt-dev
-      - uses: matrix-org/setup-python-poetry@v1
-        with:
-          python-version: ${{ matrix.python-version }}
-          extras: ${{ matrix.extras }}
-      - run: poetry run trial --jobs=2 tests
-      - name: Dump logs
-        # Logs are most useful when the command fails, always include them.
-        if: ${{ always() }}
-        # Note: Dumps to workflow logs instead of using actions/upload-artifact
-        #       This keeps logs colocated with failing jobs
-        #       It also ignores find's exit code; this is a best effort affair
-        run: >-
-          find _trial_temp -name '*.log'
-          -exec echo "::group::{}" \;
-          -exec cat {} \;
-          -exec echo "::endgroup::" \;
-          || true
-
-  sytest:
-    if: ${{ !failure() && !cancelled() }}
-    needs: linting-done
-    runs-on: ubuntu-latest
-    container:
-      image: matrixdotorg/sytest-synapse:${{ matrix.sytest-tag }}
-      volumes:
-        - ${{ github.workspace }}:/src
-      env:
-        SYTEST_BRANCH: ${{ github.head_ref }}
-        POSTGRES: ${{ matrix.postgres && 1}}
-        MULTI_POSTGRES: ${{ (matrix.postgres == 'multi-postgres') && 1}}
-        WORKERS: ${{ matrix.workers && 1 }}
-        REDIS: ${{ matrix.redis && 1 }}
-        BLACKLIST: ${{ matrix.workers && 'synapse-blacklist-with-workers' }}
-        TOP: ${{ github.workspace }}
-
-    strategy:
-      fail-fast: false
-      matrix:
-        include:
-          - sytest-tag: focal
-
-          - sytest-tag: focal
-            postgres: postgres
-
-          - sytest-tag: testing
-            postgres: postgres
-
-          - sytest-tag: focal
-            postgres: multi-postgres
-            workers: workers
-
-          - sytest-tag: buster
-            postgres: multi-postgres
-            workers: workers
-
-          - sytest-tag: buster
-            postgres: postgres
-            workers: workers
-            redis: redis
-
-    steps:
-      - uses: actions/checkout@v2
-      - name: Prepare test blacklist
-        run: cat sytest-blacklist .ci/worker-blacklist > synapse-blacklist-with-workers
-      - name: Run SyTest
-        run: /bootstrap.sh synapse
-        working-directory: /src
-      - name: Summarise results.tap
-        if: ${{ always() }}
-        run: /sytest/scripts/tap_to_gha.pl /logs/results.tap
-      - name: Upload SyTest logs
-        uses: actions/upload-artifact@v2
-        if: ${{ always() }}
-        with:
-          name: Sytest Logs - ${{ job.status }} - (${{ join(matrix.*, ', ') }})
-          path: |
-            /logs/results.tap
-            /logs/**/*.log*
-
-  export-data:
-    if: ${{ !failure() && !cancelled() }} # Allow previous steps to be skipped, but not fail
-    needs: [linting-done, portdb]
-    runs-on: ubuntu-latest
-    env:
-      TOP: ${{ github.workspace }}
-
-    services:
-      postgres:
-        image: postgres
-        ports:
-          - 5432:5432
-        env:
-          POSTGRES_PASSWORD: "postgres"
-          POSTGRES_INITDB_ARGS: "--lc-collate C --lc-ctype C --encoding UTF8"
-        options: >-
-          --health-cmd pg_isready
-          --health-interval 10s
-          --health-timeout 5s
-          --health-retries 5
-
-    steps:
-      - uses: actions/checkout@v2
-      - run: sudo apt-get -qq install xmlsec1
-      - uses: matrix-org/setup-python-poetry@v1
-        with:
-          python-version: ${{ matrix.python-version }}
-          extras: "postgres"
-      - run: .ci/scripts/test_export_data_command.sh
-
-  portdb:
-    if: ${{ !failure() && !cancelled() }} # Allow previous steps to be skipped, but not fail
-    needs: linting-done
-    runs-on: ubuntu-latest
-    env:
-      TOP: ${{ github.workspace }}
-    strategy:
-      matrix:
-        include:
-          - python-version: "3.7"
-            postgres-version: "10"
-
-          - python-version: "3.10"
-            postgres-version: "14"
-
-    services:
-      postgres:
-        image: postgres:${{ matrix.postgres-version }}
-        ports:
-          - 5432:5432
-        env:
-          POSTGRES_PASSWORD: "postgres"
-          POSTGRES_INITDB_ARGS: "--lc-collate C --lc-ctype C --encoding UTF8"
-        options: >-
-          --health-cmd pg_isready
-          --health-interval 10s
-          --health-timeout 5s
-          --health-retries 5
-
-    steps:
-      - uses: actions/checkout@v2
-      - run: sudo apt-get -qq install xmlsec1
-      - uses: matrix-org/setup-python-poetry@v1
-        with:
-          python-version: ${{ matrix.python-version }}
-          extras: "postgres"
-      - run: .ci/scripts/test_synapse_port_db.sh

  complement:
    if: "${{ !failure() && !cancelled() }}"
-    needs: linting-done
    runs-on: ubuntu-latest

    strategy:
      fail-fast: false
      matrix:
        include:
-          - arrangement: monolith
-            database: SQLite
-
-          - arrangement: monolith
+          - arrangement: workers
            database: Postgres

    steps:
-      # The path is set via a file given by $GITHUB_PATH. We need both Go 1.17 and GOPATH on the path to run Complement.
-      # See https://docs.github.com/en/actions/using-workflows/workflow-commands-for-github-actions#adding-a-system-path
-      - name: "Set Go Version"
-        run: |
-          # Add Go 1.17 to the PATH: see https://github.com/actions/virtual-environments/blob/main/images/linux/Ubuntu2004-Readme.md#environment-variables-2
-          echo "$GOROOT_1_17_X64/bin" >> $GITHUB_PATH
-          # Add the Go path to the PATH: We need this so we can call gotestfmt
-          echo "~/go/bin" >> $GITHUB_PATH
-
-      - name: "Install Complement Dependencies"
-        run: |
-          sudo apt-get update && sudo apt-get install -y libolm3 libolm-dev
-          go get -v github.com/haveyoudebuggedit/gotestfmt/v2/cmd/gotestfmt@latest
-
      - name: Run actions/checkout@v2 for synapse
        uses: actions/checkout@v2
        with:
          path: synapse

-      # Attempt to check out the same branch of Complement as the PR. If it
-      # doesn't exist, fallback to HEAD.
-      - name: Checkout complement
-        run: synapse/.ci/scripts/checkout_complement.sh
+      - name: Prepare Complement's Prerequisites
+        run: synapse/.ci/scripts/setup_complement_prerequisites.sh

      - run: |
          set -o pipefail
-          POSTGRES=${{ (matrix.database == 'Postgres') && 1 }} COMPLEMENT_DIR=`pwd`/complement synapse/scripts-dev/complement.sh -json 2>&1 | gotestfmt
-        shell: bash
-        name: Run Complement Tests
+          synapse/scripts-dev/complement.sh --build-only
+          while :; do
+            POSTGRES=${{ (matrix.database == 'Postgres') && 1 || '' }} WORKERS=${{ (matrix.arrangement == 'workers') && 1 || '' }} SYNAPSE_TEST_LOG_LEVEL=DEBUG COMPLEMENT_DIR=`pwd`/complement synapse/scripts-dev/complement.sh -f -run TestSendJoinPartialStateResponse -json 2>&1 | gotestfmt | tee /tmp/xxx
+            if grep ❌ /tmp/xxx; then
+              break
+            fi
+          done

-  # We only run the workers tests on `develop` for now, because they're too slow to wait for on PRs.
-  # Sadly, you can't have an `if` condition on the value of a matrix, so this is a temporary, separate job for now.
-  # GitHub Actions doesn't support YAML anchors, so it's full-on duplication for now.
-  complement-developonly:
-    if: "${{ !failure() && !cancelled() && (github.ref == 'refs/heads/develop') }}"
-    needs: linting-done
-    runs-on: ubuntu-latest
-
-    steps:
-      # The path is set via a file given by $GITHUB_PATH. We need both Go 1.17 and GOPATH on the path to run Complement.
-      # See https://docs.github.com/en/actions/using-workflows/workflow-commands-for-github-actions#adding-a-system-path
-      - name: "Set Go Version"
-        run: |
-          # Add Go 1.17 to the PATH: see https://github.com/actions/virtual-environments/blob/main/images/linux/Ubuntu2004-Readme.md#environment-variables-2
-          echo "$GOROOT_1_17_X64/bin" >> $GITHUB_PATH
-          # Add the Go path to the PATH: We need this so we can call gotestfmt
-          echo "~/go/bin" >> $GITHUB_PATH
-
-      - name: "Install Complement Dependencies"
-        run: |
-          sudo apt-get update && sudo apt-get install -y libolm3 libolm-dev
-          go get -v github.com/haveyoudebuggedit/gotestfmt/v2/cmd/gotestfmt@latest
-
-      - name: Run actions/checkout@v2 for synapse
-        uses: actions/checkout@v2
-        with:
-          path: synapse
-
-      # Attempt to check out the same branch of Complement as the PR. If it
-      # doesn't exist, fallback to HEAD.
-      - name: Checkout complement
-        run: synapse/.ci/scripts/checkout_complement.sh
-
-      - run: |
-          set -o pipefail
-          WORKERS=1 COMPLEMENT_DIR=`pwd`/complement synapse/scripts-dev/complement.sh -json 2>&1 | gotestfmt
        shell: bash
        name: Run Complement Tests

@@ -394,15 +48,6 @@ jobs:
  tests-done:
    if: ${{ always() }}
    needs:
-      - check-sampleconfig
-      - lint
-      - lint-crlf
-      - lint-newsfile
-      - trial
-      - trial-olddeps
-      - sytest
-      - export-data
-      - portdb
      - complement
    runs-on: ubuntu-latest
    steps:
@@ -96,6 +96,51 @@ jobs:
            /logs/results.tap
            /logs/**/*.log*

+  complement:
+    if: "${{ !failure() && !cancelled() }}"
+    runs-on: ubuntu-latest
+
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - arrangement: monolith
+            database: SQLite
+
+          - arrangement: monolith
+            database: Postgres
+
+          - arrangement: workers
+            database: Postgres
+
+    steps:
+      - name: Run actions/checkout@v2 for synapse
+        uses: actions/checkout@v2
+        with:
+          path: synapse
+
+      - name: Prepare Complement's Prerequisites
+        run: synapse/.ci/scripts/setup_complement_prerequisites.sh
+
+      # This step is specific to the 'Twisted trunk' test run:
+      - name: Patch dependencies
+        run: |
+          set -x
+          DEBIAN_FRONTEND=noninteractive sudo apt-get install -yqq python3 pipx
+          pipx install poetry==1.1.12
+
+          poetry remove -n twisted
+          poetry add -n --extras tls git+https://github.com/twisted/twisted.git#trunk
+          poetry lock --no-update
+          # NOT IN 1.1.12 poetry lock --check
+        working-directory: synapse
+
+      - run: |
+          set -o pipefail
+          TEST_ONLY_SKIP_DEP_HASH_VERIFICATION=1 POSTGRES=${{ (matrix.database == 'Postgres') && 1 || '' }} WORKERS=${{ (matrix.arrangement == 'workers') && 1 || '' }} COMPLEMENT_DIR=`pwd`/complement synapse/scripts-dev/complement.sh -json 2>&1 | gotestfmt
+        shell: bash
+        name: Run Complement Tests
+
  # open an issue if the build fails, so we know about it.
  open-issue:
    if: failure()
@@ -103,6 +148,7 @@ jobs:
      - mypy
      - trial
      - sytest
+      - complement

    runs-on: ubuntu-latest

@@ -1,8 +1,164 @@
-Synapse 1.61.0rc1 (2022-06-07)
+Synapse 1.62.0 (2022-07-05)
+===========================
+
+No significant changes since 1.62.0rc3.
+
+Authors of spam-checker plugins should consult the [upgrade notes](https://github.com/matrix-org/synapse/blob/release-v1.62/docs/upgrade.md#upgrading-to-v1620) to learn about the enriched signatures for spam checker callbacks, which are supported with this release of Synapse.
+
+
+Synapse 1.62.0rc3 (2022-07-04)
 ==============================

+Bugfixes
+--------
+
+- Update the version of the [ldap3 plugin](https://github.com/matrix-org/matrix-synapse-ldap3/) included in the `matrixdotorg/synapse` DockerHub images and the Debian packages hosted on `packages.matrix.org` to 0.2.1. This fixes [a bug](https://github.com/matrix-org/matrix-synapse-ldap3/pull/163) with usernames containing uppercase characters. ([\#13156](https://github.com/matrix-org/synapse/issues/13156))
+- Fix a bug introduced in Synapse 1.62.0rc1 affecting unread counts for users on small servers. ([\#13168](https://github.com/matrix-org/synapse/issues/13168))
+
+
+Synapse 1.62.0rc2 (2022-07-01)
+==============================
+
+Bugfixes
+--------
+
+- Fix unread counts for users on large servers. Introduced in v1.62.0rc1. ([\#13140](https://github.com/matrix-org/synapse/issues/13140))
+- Fix DB performance when deleting old push notifications. Introduced in v1.62.0rc1. ([\#13141](https://github.com/matrix-org/synapse/issues/13141))
+
+
+Synapse 1.62.0rc1 (2022-06-28)
+==============================
+
+Features
+--------
+
+- Port the spam-checker API callbacks to a new, richer API. This is part of an ongoing change to let spam-checker modules inform users of the reason their event or operation is rejected. ([\#12857](https://github.com/matrix-org/synapse/issues/12857), [\#13047](https://github.com/matrix-org/synapse/issues/13047))
+- Allow server admins to customise the response of the `/.well-known/matrix/client` endpoint. ([\#13035](https://github.com/matrix-org/synapse/issues/13035))
+- Add metrics measuring the CPU and DB time spent in state resolution. ([\#13036](https://github.com/matrix-org/synapse/issues/13036))
+- Speed up fetching of device list changes in `/sync` and `/keys/changes`. ([\#13045](https://github.com/matrix-org/synapse/issues/13045), [\#13098](https://github.com/matrix-org/synapse/issues/13098))
+- Improve URL previews for sites which only provide Twitter Card metadata, e.g. LWN.net. ([\#13056](https://github.com/matrix-org/synapse/issues/13056))
+
+
+Bugfixes
+--------
+
+- Update [MSC3786](https://github.com/matrix-org/matrix-spec-proposals/pull/3786) implementation to check `state_key`. ([\#12939](https://github.com/matrix-org/synapse/issues/12939))
+- Fix a bug introduced in Synapse 1.58 where Synapse would not report full version information when installed from a git checkout. This is a best-effort affair and not guaranteed to be stable. ([\#12973](https://github.com/matrix-org/synapse/issues/12973))
+- Fix a bug introduced in Synapse 1.60 where Synapse would fail to start if the `sqlite3` module was not available. ([\#12979](https://github.com/matrix-org/synapse/issues/12979))
+- Fix a bug where non-standard information was required when requesting the `/hierarchy` API over federation. Introduced 
+  in Synapse v1.41.0. ([\#12991](https://github.com/matrix-org/synapse/issues/12991))
+- Fix a long-standing bug which meant that rate limiting was not restrictive enough in some cases. ([\#13018](https://github.com/matrix-org/synapse/issues/13018))
+- Fix a bug introduced in Synapse 1.58 where profile requests for a malformed user ID would ccause an internal error. Synapse now returns 400 Bad Request in this situation. ([\#13041](https://github.com/matrix-org/synapse/issues/13041))
+- Fix some inconsistencies in the event authentication code. ([\#13087](https://github.com/matrix-org/synapse/issues/13087), [\#13088](https://github.com/matrix-org/synapse/issues/13088))
+- Fix a long-standing bug where room directory requests would cause an internal server error if given a malformed room alias. ([\#13106](https://github.com/matrix-org/synapse/issues/13106))
+
+
+Improved Documentation
+----------------------
+
+- Add documentation for how to configure Synapse with Workers using Docker Compose. Includes example worker config and docker-compose.yaml. Contributed by @Thumbscrew. ([\#12737](https://github.com/matrix-org/synapse/issues/12737))
+- Ensure the [Poetry cheat sheet](https://matrix-org.github.io/synapse/develop/development/dependencies.html) is available in the online documentation. ([\#13022](https://github.com/matrix-org/synapse/issues/13022))
+- Mention removed community/group worker endpoints in upgrade.md. Contributed by @olmari. ([\#13023](https://github.com/matrix-org/synapse/issues/13023))
+- Add instructions for running Complement with `gotestfmt`-formatted output locally. ([\#13073](https://github.com/matrix-org/synapse/issues/13073))
+- Update OpenTracing docs to reference the configuration manual rather than the configuration file. ([\#13076](https://github.com/matrix-org/synapse/issues/13076))
+- Update information on downstream Debian packages. ([\#13095](https://github.com/matrix-org/synapse/issues/13095))
+- Remove documentation for the Delete Group Admin API which no longer exists. ([\#13112](https://github.com/matrix-org/synapse/issues/13112))
+
+
+Deprecations and Removals
+-------------------------
+
+- Remove the unspecced `DELETE /directory/list/room/{roomId}` endpoint, which hid rooms from the [public room directory](https://spec.matrix.org/v1.3/client-server-api/#listing-rooms). Instead, `PUT` to the same URL with a visibility of `"private"`. ([\#13123](https://github.com/matrix-org/synapse/issues/13123))
+
+
+Internal Changes
+----------------
+
+- Add tests for cancellation of `GET /rooms/$room_id/members` and `GET /rooms/$room_id/state` requests. ([\#12674](https://github.com/matrix-org/synapse/issues/12674))
+- Report login failures due to unknown third party identifiers in the same way as failures due to invalid passwords. This prevents an attacker from using the error response to determine if the identifier exists. Contributed by Daniel Aloni. ([\#12738](https://github.com/matrix-org/synapse/issues/12738))
+- Merge the Complement testing Docker images into a single, multi-purpose image. ([\#12881](https://github.com/matrix-org/synapse/issues/12881), [\#13075](https://github.com/matrix-org/synapse/issues/13075))
+- Simplify the database schema for `event_edges`. ([\#12893](https://github.com/matrix-org/synapse/issues/12893))
+- Clean up the test code for client disconnection. ([\#12929](https://github.com/matrix-org/synapse/issues/12929))
+- Remove code generating comments in configuration. ([\#12941](https://github.com/matrix-org/synapse/issues/12941))
+- Add `Cross-Origin-Resource-Policy: cross-origin` header to content repository's thumbnail and download endpoints. ([\#12944](https://github.com/matrix-org/synapse/issues/12944))
+- Replace noop background updates with `DELETE` delta. ([\#12954](https://github.com/matrix-org/synapse/issues/12954), [\#13050](https://github.com/matrix-org/synapse/issues/13050))
+- Use lower isolation level when inserting read receipts to avoid serialization errors. Contributed by Nick @ Beeper. ([\#12957](https://github.com/matrix-org/synapse/issues/12957))
+- Reduce the amount of state we pull from the DB. ([\#12963](https://github.com/matrix-org/synapse/issues/12963))
+- Enable testing against PostgreSQL databases in Complement CI. ([\#12965](https://github.com/matrix-org/synapse/issues/12965), [\#13034](https://github.com/matrix-org/synapse/issues/13034))
+- Fix an inaccurate comment. ([\#12969](https://github.com/matrix-org/synapse/issues/12969))
+- Remove the `delete_device` method and always call `delete_devices`. ([\#12970](https://github.com/matrix-org/synapse/issues/12970))
+- Use a GitHub form for issues rather than a hard-to-read, easy-to-ignore template. ([\#12982](https://github.com/matrix-org/synapse/issues/12982))
+- Move [MSC3715](https://github.com/matrix-org/matrix-spec-proposals/pull/3715) behind an experimental config flag. ([\#12984](https://github.com/matrix-org/synapse/issues/12984))
+- Add type hints to tests. ([\#12985](https://github.com/matrix-org/synapse/issues/12985), [\#13099](https://github.com/matrix-org/synapse/issues/13099))
+- Refactor macaroon tokens generation and move the unsubscribe link in notification emails to `/_synapse/client/unsubscribe`. ([\#12986](https://github.com/matrix-org/synapse/issues/12986))
+- Fix documentation for running complement tests. ([\#12990](https://github.com/matrix-org/synapse/issues/12990))
+- Faster joins: add issue links to the TODO comments in the code. ([\#13004](https://github.com/matrix-org/synapse/issues/13004))
+- Reduce DB usage of `/sync` when a large number of unread messages have recently been sent in a room. ([\#13005](https://github.com/matrix-org/synapse/issues/13005), [\#13096](https://github.com/matrix-org/synapse/issues/13096), [\#13118](https://github.com/matrix-org/synapse/issues/13118))
+- Replaced usage of PyJWT with methods from Authlib in `org.matrix.login.jwt`. Contributed by Hannes Lerchl. ([\#13011](https://github.com/matrix-org/synapse/issues/13011))
+- Modernize the `contrib/graph/` scripts. ([\#13013](https://github.com/matrix-org/synapse/issues/13013))
+- Remove redundant `room_version` parameters from event auth functions. ([\#13017](https://github.com/matrix-org/synapse/issues/13017))
+- Decouple `synapse.api.auth_blocking.AuthBlocking` from `synapse.api.auth.Auth`. ([\#13021](https://github.com/matrix-org/synapse/issues/13021))
+- Add type annotations to `synapse.storage.databases.main.devices`. ([\#13025](https://github.com/matrix-org/synapse/issues/13025))
+- Set default `sync_response_cache_duration` to two minutes. ([\#13042](https://github.com/matrix-org/synapse/issues/13042))
+- Rename CI test runs. ([\#13046](https://github.com/matrix-org/synapse/issues/13046))
+- Increase timeout of complement CI test runs. ([\#13048](https://github.com/matrix-org/synapse/issues/13048))
+- Refactor entry points so that they all have a `main` function. ([\#13052](https://github.com/matrix-org/synapse/issues/13052))
+- Refactor the Dockerfile-workers configuration script to use Jinja2 templates in Synapse workers' Supervisord blocks. ([\#13054](https://github.com/matrix-org/synapse/issues/13054))
+- Add headers to individual options in config documentation to allow for linking. ([\#13055](https://github.com/matrix-org/synapse/issues/13055))
+- Make Complement CI logs easier to read. ([\#13057](https://github.com/matrix-org/synapse/issues/13057), [\#13058](https://github.com/matrix-org/synapse/issues/13058), [\#13069](https://github.com/matrix-org/synapse/issues/13069))
+- Don't instantiate modules with keyword arguments. ([\#13060](https://github.com/matrix-org/synapse/issues/13060))
+- Fix type checking errors against Twisted trunk. ([\#13061](https://github.com/matrix-org/synapse/issues/13061))
+- Allow MSC3030 `timestamp_to_event` calls from anyone on world-readable rooms. ([\#13062](https://github.com/matrix-org/synapse/issues/13062))
+- Add a CI job to check that schema deltas are in the correct folder. ([\#13063](https://github.com/matrix-org/synapse/issues/13063))
+- Avoid rechecking event auth rules which are independent of room state. ([\#13065](https://github.com/matrix-org/synapse/issues/13065))
+- Reduce the duplication of code that invokes the rate limiter. ([\#13070](https://github.com/matrix-org/synapse/issues/13070))
+- Add a Subject Alternative Name to the certificate generated for Complement tests. ([\#13071](https://github.com/matrix-org/synapse/issues/13071))
+- Add more tests for room upgrades. ([\#13074](https://github.com/matrix-org/synapse/issues/13074))
+- Pin dependencies maintained by matrix.org to [semantic version](https://semver.org/) bounds. ([\#13082](https://github.com/matrix-org/synapse/issues/13082))
+- Correctly report prometheus DB stats for `get_earliest_token_for_stats`. ([\#13085](https://github.com/matrix-org/synapse/issues/13085))
+- Fix a long-standing bug where a finished logging context would be re-started when Synapse failed to persist an event from federation. ([\#13089](https://github.com/matrix-org/synapse/issues/13089))
+- Simplify the alias deletion logic as an application service. ([\#13093](https://github.com/matrix-org/synapse/issues/13093))
+- Add type annotations to `tests.test_server`. ([\#13124](https://github.com/matrix-org/synapse/issues/13124))
+
+
+Synapse 1.61.1 (2022-06-28)
+===========================
+
+This patch release fixes a security issue regarding URL previews, affecting all prior versions of Synapse. Server administrators are encouraged to update Synapse as soon as possible. We are not aware of these vulnerabilities being exploited in the wild.
+
+Server administrators who are unable to update Synapse may use the workarounds described in the linked GitHub Security Advisory below.
+
+## Security advisory
+
+The following issue is fixed in 1.61.1.
+
+* [GHSA-22p3-qrh9-cx32](https://github.com/matrix-org/synapse/security/advisories/GHSA-22p3-qrh9-cx32) / [CVE-2022-31052](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2022-31052)
+
+  Synapse instances with the [`url_preview_enabled`](https://matrix-org.github.io/synapse/v1.61/usage/configuration/config_documentation.html#media-store) homeserver config option set to `true` are affected. URL previews of some web pages can lead to unbounded recursion, causing the request to either fail, or in some cases crash the running Synapse process.
+
+  Requesting URL previews requires authentication. Nevertheless, it is possible to exploit this maliciously, either by malicious users on the homeserver, or by remote users sending URLs that a local user's client may automatically request a URL preview for.
+
+  Homeservers with the `url_preview_enabled` configuration option set to `false` (the default) are unaffected. Instances with the `enable_media_repo` configuration option set to `false` are also unaffected, as this also disables URL preview functionality.
+
+  Fixed by [fa1308061802ac7b7d20e954ba7372c5ac292333](https://github.com/matrix-org/synapse/commit/fa1308061802ac7b7d20e954ba7372c5ac292333).
+
+Synapse 1.61.0 (2022-06-14)
+===========================
+
 This release removes support for the non-standard feature known both as 'groups' and as 'communities', which have been superseded by *Spaces*.

+See [the upgrade notes](https://github.com/matrix-org/synapse/blob/develop/docs/upgrade.md#upgrading-to-v1610)
+for more details.
+
+Improved Documentation
+----------------------
+
+- Mention removed community/group worker endpoints in [the upgrade notes](https://github.com/matrix-org/synapse/blob/develop/docs/upgrade.md#upgrading-to-v1610). Contributed by @olmari. ([\#13023](https://github.com/matrix-org/synapse/issues/13023))
+
+
+Synapse 1.61.0rc1 (2022-06-07)
+==============================
+
 Features
 --------

@@ -1 +0,0 @@
-Add tests for cancellation of `GET /rooms/$room_id/members` and `GET /rooms/$room_id/state` requests.
@@ -1 +0,0 @@
-Add documentation for how to configure Synapse with Workers using Docker Compose. Includes example worker config and docker-compose.yaml. Contributed by @Thumbscrew.
@@ -1 +0,0 @@
-Report login failures due to unknown third party identifiers in the same way as failures due to invalid passwords. This prevents an attacker from using the error response to determine if the identifier exists. Contributed by Daniel Aloni.
@@ -1 +0,0 @@
-Merge the Complement testing Docker images into a single, multi-purpose image.
@@ -1 +0,0 @@
-Clean up the test code for client disconnection.
@@ -1 +0,0 @@
-Use lower isolation level when inserting read receipts to avoid serialization errors. Contributed by Nick @ Beeper.
@@ -1 +0,0 @@
-Reduce the amount of state we pull from the DB.
@@ -1 +0,0 @@
-Enable testing against PostgreSQL databases in Complement CI.
@@ -1 +0,0 @@
-Fix an inaccurate comment.
@@ -1 +0,0 @@
-Remove the `delete_device` method and always call `delete_devices`.
@@ -1 +0,0 @@
-Fix a bug introduced in Synapse 1.58 where Synapse would not report full version information when installed from a git checkout. This is a best-effort affair and not guaranteed to be stable.
@@ -1 +0,0 @@
-Fix a bug introduced in Synapse 1.60 where Synapse would fail to start if the `sqlite3` module was not available.
@@ -1 +0,0 @@
-Use a GitHub form for issues rather than a hard-to-read, easy-to-ignore template.
@@ -1 +0,0 @@
-Move [MSC3715](https://github.com/matrix-org/matrix-spec-proposals/pull/3715) behind an experimental config flag.
@@ -1 +0,0 @@
-Add type annotations to `tests.state.test_v2`.
@@ -1 +0,0 @@
-Fix documentation for running complement tests.
@@ -1,2 +0,0 @@
-Fix a bug where non-standard information was required when requesting the `/hierarchy` API over federation. Introduced 
-in Synapse v1.41.0.
@@ -1 +0,0 @@
-Faster joins: add issue links to the TODO comments in the code.
@@ -1 +0,0 @@
-Modernize the `contrib/graph/` scripts.
@@ -1 +0,0 @@
-Remove redundant `room_version` parameters from event auth functions.
@@ -1 +0,0 @@
-Ensure the [Poetry cheat sheet](https://matrix-org.github.io/synapse/develop/development/dependencies.html) is available in the online documentation.
@@ -0,0 +1 @@
+Add type annotations to `tests.utils`.
@@ -0,0 +1 @@
+Add an explanation of the `--report-stats` argument to the docs.
@@ -0,0 +1 @@
+Implement [MSC3827](https://github.com/matrix-org/matrix-spec-proposals/pull/3827): Filtering of /publicRooms by room type.
@@ -0,0 +1,3 @@
+Clean up references to sample configuration and redirect users to the configuration manual instead.
+
+
@@ -0,0 +1 @@
+Enable Complement testing in the 'Twisted Trunk' CI runs.
@@ -0,0 +1 @@
+Add documentation for anonymised homeserver statistics collection.
@@ -0,0 +1 @@
+Faster room joins: Handle race between persisting an event and un-partial stating a room.
@@ -0,0 +1 @@
+Add missing type hints to `synapse.logging`.
@@ -0,0 +1 @@
+Raise a `DependencyError` on missing dependencies instead of a `ConfigError`.
@@ -0,0 +1 @@
+Fix wrong section header for `allow_public_rooms_over_federation` in the homeserver config documentation.
@@ -0,0 +1 @@
+Reduce DB usage of `/sync` when a large number of unread messages have recently been sent in a room.
@@ -0,0 +1 @@
+Add a rate limit for local users sending invites.
@@ -0,0 +1 @@
+Improve startup times in Complement test runs against workers, particularly in CPU-constrained environments.
@@ -0,0 +1 @@
+Only one-line SQL statements for logging and tracing.
@@ -0,0 +1 @@
+Fix application service not being able to join remote federated room without a profile set.
@@ -0,0 +1 @@
+Document how the Synapse team does reviews.
@@ -0,0 +1 @@
+Apply ratelimiting earlier in processing of /send request.
@@ -0,0 +1 @@
+Enforce type annotations for `tests.test_server`.
@@ -0,0 +1 @@
+Add type annotations to `tests.server`.
@@ -0,0 +1 @@
+Add a link to the configuration manual from the homeserver sample config documentation.
@@ -0,0 +1 @@
+Add support to `complement.sh` for skipping the docker build.
@@ -0,0 +1 @@
+Faster joins: skip waiting for full state when processing incoming events over federation.
@@ -0,0 +1 @@
+Improve exception handling when processing events received over federation.
@@ -0,0 +1 @@
+Improve validation logic in Synapse's REST endpoints.
@@ -0,0 +1 @@
+Faster room joins: fix race in recalculation of current room state.
@@ -0,0 +1 @@
+Add the ability to set the log level using the `SYNAPSE_TEST_LOG_LEVEL` environment when using `complement.sh`.
@@ -0,0 +1 @@
+Reduce DB usage of `/sync` when a large number of unread messages have recently been sent in a room.
@@ -0,0 +1 @@
+Enable Complement testing in the 'Twisted Trunk' CI runs.
@@ -0,0 +1 @@
+Add support to `complement.sh` for skipping the docker build.
@@ -0,0 +1 @@
+Improve and fix type hints.
@@ -0,0 +1 @@
+Add missing links to config options.
@@ -0,0 +1 @@
+Update config used by Complement to allow device name lookup over federation.
@@ -0,0 +1 @@
+Make use of the more robust `get_current_state` in `_get_state_map_for_room` to avoid breakages.
@@ -0,0 +1 @@
+Fix bug where rows were not deleted from `event_push_actions` table on large servers. Introduced in v1.62.0.
@@ -0,0 +1 @@
+Check that `auto_vacuum` is disabled when porting a SQLite database to Postgres, as `VACUUM`s must not be performed between runs of the script.
@@ -0,0 +1 @@
+Remove obsolete and for 8 years unused `RoomEventsStoreTestCase`. Contributed by @arkamar.
@@ -0,0 +1 @@
+Bump the version of `lxml` in matrix.org Docker images Debian packages from 4.8.0 to 4.9.1.
@@ -0,0 +1 @@
+Reduce number of queries used to get profile information. Contributed by Nick @ Beeper (@fizzadar).
@@ -1,3 +1,39 @@
+matrix-synapse-py3 (1.62.0) stable; urgency=medium
+
+  * New Synapse release 1.62.0.
+
+ -- Synapse Packaging team <packages@matrix.org>  Tue, 05 Jul 2022 11:14:15 +0100
+
+matrix-synapse-py3 (1.62.0~rc3) stable; urgency=medium
+
+  * New Synapse release 1.62.0rc3.
+
+ -- Synapse Packaging team <packages@matrix.org>  Mon, 04 Jul 2022 16:07:01 +0100
+
+matrix-synapse-py3 (1.62.0~rc2) stable; urgency=medium
+
+  * New Synapse release 1.62.0rc2.
+
+ -- Synapse Packaging team <packages@matrix.org>  Fri, 01 Jul 2022 11:42:41 +0100
+
+matrix-synapse-py3 (1.62.0~rc1) stable; urgency=medium
+
+  * New Synapse release 1.62.0rc1.
+
+ -- Synapse Packaging team <packages@matrix.org>  Tue, 28 Jun 2022 16:34:57 +0100
+
+matrix-synapse-py3 (1.61.1) stable; urgency=medium
+
+  * New Synapse release 1.61.1.
+
+ -- Synapse Packaging team <packages@matrix.org>  Tue, 28 Jun 2022 14:33:46 +0100
+
+matrix-synapse-py3 (1.61.0) stable; urgency=medium
+
+  * New Synapse release 1.61.0.
+
+ -- Synapse Packaging team <packages@matrix.org>  Tue, 14 Jun 2022 11:44:19 +0100
+
 matrix-synapse-py3 (1.61.0~rc1) stable; urgency=medium

  * Remove unused `jitsimeetbridge` experiment from `contrib` directory.
@@ -40,7 +40,7 @@ FROM docker.io/python:${PYTHON_VERSION}-slim as requirements
 RUN \
   --mount=type=cache,target=/var/cache/apt,sharing=locked \
   --mount=type=cache,target=/var/lib/apt,sharing=locked \
- apt-get update && apt-get install -y git \
+ apt-get update -qq && apt-get install -yqq git \
    && rm -rf /var/lib/apt/lists/*

 # We install poetry in its own build stage to avoid its dependencies conflicting with
@@ -62,7 +62,13 @@ WORKDIR /synapse
 # Copy just what we need to run `poetry export`...
 COPY pyproject.toml poetry.lock /synapse/

-RUN /root/.local/bin/poetry export --extras all -o /synapse/requirements.txt
+
+# If specified, we won't verify the hashes of dependencies.
+# This is only needed if the hashes of dependencies cannot be checked for some
+# reason, such as when a git repository is used directly as a dependency.
+ARG TEST_ONLY_SKIP_DEP_HASH_VERIFICATION
+
+RUN /root/.local/bin/poetry export --extras all -o /synapse/requirements.txt ${TEST_ONLY_SKIP_DEP_HASH_VERIFICATION:+--without-hashes}

 ###
 ### Stage 1: builder
@@ -73,7 +79,7 @@ FROM docker.io/python:${PYTHON_VERSION}-slim as builder
 RUN \
   --mount=type=cache,target=/var/cache/apt,sharing=locked \
   --mount=type=cache,target=/var/lib/apt,sharing=locked \
- apt-get update && apt-get install -y \
+ apt-get update -qq && apt-get install -yqq \
    build-essential \
    libffi-dev \
    libjpeg-dev \
@@ -85,6 +91,7 @@ RUN \
    openssl \
    rustc \
    zlib1g-dev \
+    git \
    && rm -rf /var/lib/apt/lists/*

 # To speed up rebuilds, install all of the dependencies before we copy over
@@ -118,7 +125,7 @@ LABEL org.opencontainers.image.licenses='Apache-2.0'
 RUN \
   --mount=type=cache,target=/var/cache/apt,sharing=locked \
   --mount=type=cache,target=/var/lib/apt,sharing=locked \
-  apt-get update && apt-get install -y \
+  apt-get update -qq && apt-get install -yqq \
    curl \
    gosu \
    libjpeg62-turbo \
@@ -6,8 +6,8 @@ FROM matrixdotorg/synapse:$SYNAPSE_VERSION
 RUN \
   --mount=type=cache,target=/var/cache/apt,sharing=locked \
   --mount=type=cache,target=/var/lib/apt,sharing=locked \
-  apt-get update && \
-  DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
+  apt-get update -qq && \
+  DEBIAN_FRONTEND=noninteractive apt-get install -yqq --no-install-recommends \
     redis-server nginx-light

 # Install supervisord with pip instead of apt, to avoid installing a second
@@ -67,6 +67,13 @@ The following environment variables are supported in `generate` mode:
 * `UID`, `GID`: the user id and group id to use for creating the data
  directories. If unset, and no user is set via `docker run --user`, defaults
  to `991`, `991`.
+* `SYNAPSE_LOG_LEVEL`: the log level to use (one of `DEBUG`, `INFO`, `WARNING` or `ERROR`).
+  Defaults to `INFO`.
+* `SYNAPSE_LOG_SENSITIVE`: if set and the log level is set to `DEBUG`, Synapse
+  will log sensitive information such as access tokens.
+  This should not be needed unless you are a developer attempting to debug something
+  particularly tricky.
+

 ## Postgres

@@ -9,7 +9,7 @@ FROM matrixdotorg/synapse-workers:$SYNAPSE_VERSION

 # Install postgresql
 RUN apt-get update && \
-  DEBIAN_FRONTEND=noninteractive apt-get install --no-install-recommends -y postgresql-13
+  DEBIAN_FRONTEND=noninteractive apt-get install --no-install-recommends -yqq postgresql-13

 # Configure a user and create a database for Synapse
 RUN pg_ctlcluster 13 main start &&  su postgres -c "echo \
@@ -7,7 +7,7 @@ so **please don't use this image for a production server**.
 This multi-purpose image is built on top of `Dockerfile-workers` in the parent directory
 and can be switched using environment variables between the following configurations:

- Monolithic Synapse with SQLite (`SYNAPSE_COMPLEMENT_DATABASE=sqlite`)
+- Monolithic Synapse with SQLite (default, or `SYNAPSE_COMPLEMENT_DATABASE=sqlite`)
 - Monolithic Synapse with Postgres (`SYNAPSE_COMPLEMENT_DATABASE=postgres`)
 - Workerised Synapse with Postgres (`SYNAPSE_COMPLEMENT_DATABASE=postgres` and `SYNAPSE_COMPLEMENT_USE_WORKERS=true`)

@@ -31,7 +31,7 @@ case "$SYNAPSE_COMPLEMENT_DATABASE" in
    export START_POSTGRES=true
    ;;

-  sqlite)
+  sqlite|"")
    # Configure supervisord not to start Postgres, as we don't need it
    export START_POSTGRES=false
    ;;
@@ -59,6 +59,9 @@ if [[ -n "$SYNAPSE_COMPLEMENT_USE_WORKERS" ]]; then
      synchrotron, \
      appservice, \
      pusher"
+
+  # Improve startup times by using a launcher based on fork()
+  export SYNAPSE_USE_EXPERIMENTAL_FORKING_LAUNCHER=1
 else
  # Empty string here means 'main process only'
  export SYNAPSE_WORKER_TYPES=""
@@ -73,14 +76,30 @@ fi

 # Generate a TLS key, then generate a certificate by having Complement's CA sign it
 # Note that both the key and certificate are in PEM format (not DER).
+
+# First generate a configuration file to set up a Subject Alternative Name.
+cat > /conf/server.tls.conf <<EOF
+.include /etc/ssl/openssl.cnf
+
+[SAN]
+subjectAltName=DNS:${SERVER_NAME}
+EOF
+
+# Generate an RSA key
 openssl genrsa -out /conf/server.tls.key 2048

-openssl req -new -key /conf/server.tls.key -out /conf/server.tls.csr \
-  -subj "/CN=${SERVER_NAME}"
+# Generate a certificate signing request
+openssl req -new -config /conf/server.tls.conf -key /conf/server.tls.key -out /conf/server.tls.csr \
+  -subj "/CN=${SERVER_NAME}" -reqexts SAN

+# Make the Complement Certificate Authority sign and generate a certificate.
 openssl x509 -req -in /conf/server.tls.csr \
  -CA /complement/ca/ca.crt -CAkey /complement/ca/ca.key -set_serial 1 \
-  -out /conf/server.tls.crt
+  -out /conf/server.tls.crt -extfile /conf/server.tls.conf -extensions SAN
+
+# Assert that we have a Subject Alternative Name in the certificate.
+# (grep will exit with 1 here if there isn't a SAN in the certificate.)
+openssl x509 -in /conf/server.tls.crt -noout -text | grep DNS:

 export SYNAPSE_TLS_CERT=/conf/server.tls.crt
 export SYNAPSE_TLS_KEY=/conf/server.tls.key
@@ -81,6 +81,8 @@ rc_invites:

 federation_rr_transactions_per_room_per_second: 9999

+allow_device_name_lookup_over_federation: true
+
 ## Experimental Features ##

 experimental_features:
@@ -103,4 +105,10 @@ server_notices:
  system_mxid_avatar_url: ""
  room_name: "Server Alert"

+
+# Disable sync cache so that initial `/sync` requests are up-to-date.
+caches:
+  sync_response_cache_duration: 0
+
+
 {% include "shared-orig.yaml.j2" %}
@@ -31,17 +31,3 @@ autorestart=true
 # Redis can be disabled if the image is being used without workers
 autostart={{ enable_redis }}

-[program:synapse_main]
-command=/usr/local/bin/prefix-log /usr/local/bin/python -m synapse.app.homeserver --config-path="{{ main_config_path }}" --config-path=/conf/workers/shared.yaml
-priority=10
-# Log startup failures to supervisord's stdout/err
-# Regular synapse logs will still go in the configured data directory
-stdout_logfile=/dev/stdout
-stdout_logfile_maxbytes=0
-stderr_logfile=/dev/stderr
-stderr_logfile_maxbytes=0
-autorestart=unexpected
-exitcodes=0
-
-# Additional process blocks
-{{ worker_config }}
@@ -0,0 +1,52 @@
+{% if use_forking_launcher %}
+[program:synapse_fork]
+command=/usr/local/bin/python -m synapse.app.complement_fork_starter
+  {{ main_config_path }}
+  synapse.app.homeserver
+  --config-path="{{ main_config_path }}"
+  --config-path=/conf/workers/shared.yaml
+  {%- for worker in workers %}
+    -- {{ worker.app }}
+    --config-path="{{ main_config_path }}"
+    --config-path=/conf/workers/shared.yaml
+    --config-path=/conf/workers/{{ worker.name }}.yaml
+  {%- endfor %}
+stdout_logfile=/dev/stdout
+stdout_logfile_maxbytes=0
+stderr_logfile=/dev/stderr
+stderr_logfile_maxbytes=0
+autorestart=unexpected
+exitcodes=0
+
+{% else %}
+[program:synapse_main]
+command=/usr/local/bin/prefix-log /usr/local/bin/python -m synapse.app.homeserver
+  --config-path="{{ main_config_path }}"
+  --config-path=/conf/workers/shared.yaml
+priority=10
+# Log startup failures to supervisord's stdout/err
+# Regular synapse logs will still go in the configured data directory
+stdout_logfile=/dev/stdout
+stdout_logfile_maxbytes=0
+stderr_logfile=/dev/stderr
+stderr_logfile_maxbytes=0
+autorestart=unexpected
+exitcodes=0
+
+
+  {% for worker in workers %}
+[program:synapse_{{ worker.name }}]
+command=/usr/local/bin/prefix-log /usr/local/bin/python -m {{ worker.app }}
+  --config-path="{{ main_config_path }}"
+  --config-path=/conf/workers/shared.yaml
+  --config-path=/conf/workers/{{ worker.name }}.yaml
+autorestart=unexpected
+priority=500
+exitcodes=0
+stdout_logfile=/dev/stdout
+stdout_logfile_maxbytes=0
+stderr_logfile=/dev/stderr
+stderr_logfile_maxbytes=0
+
+  {% endfor %}
+{% endif %}
@@ -2,7 +2,11 @@ version: 1

 formatters:
  precise:
+    {% if include_worker_name_in_log_line %}
+    format: '{{ worker_name }} | %(asctime)s - %(name)s - %(lineno)d - %(levelname)s - %(request)s - %(message)s'
+    {% else %}
    format: '%(asctime)s - %(name)s - %(lineno)d - %(levelname)s - %(request)s - %(message)s'
+    {% endif %}

 handlers:
 {% if LOG_FILE_PATH %}
@@ -45,11 +49,17 @@ handlers:
    class: logging.StreamHandler
    formatter: precise

+{% if not SYNAPSE_LOG_SENSITIVE %}
+{#
+  If SYNAPSE_LOG_SENSITIVE is unset, then override synapse.storage.SQL to INFO
+  so that DEBUG entries (containing sensitive information) are not emitted.
+#}
 loggers:
    synapse.storage.SQL:
        # beware: increasing this to DEBUG will make synapse log sensitive
        # information such as access tokens.
        level: INFO
+{% endif %}

 root:
    level: {{ SYNAPSE_LOG_LEVEL or "INFO" }}
@@ -26,6 +26,13 @@
 #   * SYNAPSE_TLS_CERT: Path to a TLS certificate in PEM format.
 #   * SYNAPSE_TLS_KEY: Path to a TLS key. If this and SYNAPSE_TLS_CERT are specified,
 #         Nginx will be configured to serve TLS on port 8448.
+#   * SYNAPSE_USE_EXPERIMENTAL_FORKING_LAUNCHER: Whether to use the forking launcher,
+#         only intended for usage in Complement at the moment.
+#         No stability guarantees are provided.
+#   * SYNAPSE_LOG_LEVEL: Set this to DEBUG, INFO, WARNING or ERROR to change the
+#         log level. INFO is the default.
+#   * SYNAPSE_LOG_SENSITIVE: If unset, SQL and SQL values won't be logged,
+#         regardless of the SYNAPSE_LOG_LEVEL setting.
 #
 # NOTE: According to Complement's ENTRYPOINT expectations for a homeserver image (as defined
 # in the project's README), this script may be run multiple times, and functionality should
@@ -35,7 +42,7 @@ import os
 import subprocess
 import sys
 from pathlib import Path
-from typing import Any, Dict, List, Mapping, MutableMapping, NoReturn, Set
+from typing import Any, Dict, List, Mapping, MutableMapping, NoReturn, Optional, Set

 import yaml
 from jinja2 import Environment, FileSystemLoader
@@ -52,12 +59,12 @@ WORKERS_CONFIG: Dict[str, Dict[str, Any]] = {
        "worker_extra_conf": "",
    },
    "user_dir": {
-        "app": "synapse.app.user_dir",
+        "app": "synapse.app.generic_worker",
        "listener_resources": ["client"],
        "endpoint_patterns": [
            "^/_matrix/client/(api/v1|r0|v3|unstable)/user_directory/search$"
        ],
-        "shared_extra_conf": {"update_user_directory": False},
+        "shared_extra_conf": {"update_user_directory_from_worker": "user_dir1"},
        "worker_extra_conf": "",
    },
    "media_repository": {
@@ -78,7 +85,7 @@ WORKERS_CONFIG: Dict[str, Dict[str, Any]] = {
        "app": "synapse.app.generic_worker",
        "listener_resources": [],
        "endpoint_patterns": [],
-        "shared_extra_conf": {"notify_appservices_from_worker": "appservice"},
+        "shared_extra_conf": {"notify_appservices_from_worker": "appservice1"},
        "worker_extra_conf": "",
    },
    "federation_sender": {
@@ -176,21 +183,6 @@ WORKERS_CONFIG: Dict[str, Dict[str, Any]] = {
 }

 # Templates for sections that may be inserted multiple times in config files
-SUPERVISORD_PROCESS_CONFIG_BLOCK = """
-[program:synapse_{name}]
-command=/usr/local/bin/prefix-log /usr/local/bin/python -m {app} \
-    --config-path="{config_path}" \
-    --config-path=/conf/workers/shared.yaml \
-    --config-path=/conf/workers/{name}.yaml
-autorestart=unexpected
-priority=500
-exitcodes=0
-stdout_logfile=/dev/stdout
-stdout_logfile_maxbytes=0
-stderr_logfile=/dev/stderr
-stderr_logfile_maxbytes=0
-"""
-
 NGINX_LOCATION_CONFIG_BLOCK = """
    location ~* {endpoint} {{
        proxy_pass {upstream};
@@ -353,13 +345,10 @@ def generate_worker_files(
    # This config file will be passed to all workers, included Synapse's main process.
    shared_config: Dict[str, Any] = {"listeners": listeners}

-    # The supervisord config. The contents of which will be inserted into the
-    # base supervisord jinja2 template.
-    #
-    # Supervisord will be in charge of running everything, from redis to nginx to Synapse
-    # and all of its worker processes. Load the config template, which defines a few
-    # services that are necessary to run.
-    supervisord_config = ""
+    # List of dicts that describe workers.
+    # We pass this to the Supervisor template later to generate the appropriate
+    # program blocks.
+    worker_descriptors: List[Dict[str, Any]] = []

    # Upstreams for load-balancing purposes. This dict takes the form of a worker type to the
    # ports of each worker. For example:
@@ -437,7 +426,7 @@ def generate_worker_files(
            )

        # Enable the worker in supervisord
-        supervisord_config += SUPERVISORD_PROCESS_CONFIG_BLOCK.format_map(worker_config)
+        worker_descriptors.append(worker_config)

        # Add nginx location blocks for this worker's endpoints (if any are defined)
        for pattern in worker_config["endpoint_patterns"]:
@@ -535,10 +524,17 @@ def generate_worker_files(
        "/conf/supervisord.conf.j2",
        "/etc/supervisor/supervisord.conf",
        main_config_path=config_path,
-        worker_config=supervisord_config,
        enable_redis=workers_in_use,
    )

+    convert(
+        "/conf/synapse.supervisord.conf.j2",
+        "/etc/supervisor/conf.d/synapse.conf",
+        workers=worker_descriptors,
+        main_config_path=config_path,
+        use_forking_launcher=environ.get("SYNAPSE_USE_EXPERIMENTAL_FORKING_LAUNCHER"),
+    )
+
    # healthcheck config
    convert(
        "/conf/healthcheck.sh.j2",
@@ -560,18 +556,25 @@ def generate_worker_log_config(
    Returns: the path to the generated file
    """
    # Check whether we should write worker logs to disk, in addition to the console
-    extra_log_template_args = {}
+    extra_log_template_args: Dict[str, Optional[str]] = {}
    if environ.get("SYNAPSE_WORKERS_WRITE_LOGS_TO_DISK"):
-        extra_log_template_args["LOG_FILE_PATH"] = "{dir}/logs/{name}.log".format(
-            dir=data_dir, name=worker_name
-        )
+        extra_log_template_args["LOG_FILE_PATH"] = f"{data_dir}/logs/{worker_name}.log"
+
+    extra_log_template_args["SYNAPSE_LOG_LEVEL"] = environ.get("SYNAPSE_LOG_LEVEL")
+    extra_log_template_args["SYNAPSE_LOG_SENSITIVE"] = environ.get(
+        "SYNAPSE_LOG_SENSITIVE"
+    )
+
    # Render and write the file
-    log_config_filepath = "/conf/workers/{name}.log.config".format(name=worker_name)
+    log_config_filepath = f"/conf/workers/{worker_name}.log.config"
    convert(
        "/conf/log.config",
        log_config_filepath,
        worker_name=worker_name,
        **extra_log_template_args,
+        include_worker_name_in_log_line=environ.get(
+            "SYNAPSE_USE_EXPERIMENTAL_FORKING_LAUNCHER"
+        ),
    )
    return log_config_filepath

@@ -110,7 +110,11 @@ def generate_config_from_template(

    log_config_file = environ["SYNAPSE_LOG_CONFIG"]
    log("Generating log config file " + log_config_file)
-    convert("/conf/log.config", log_config_file, environ)
+    convert(
+        "/conf/log.config",
+        log_config_file,
+        {**environ, "include_worker_name_in_log_line": False},
+    )

    # Hopefully we already have a signing key, but generate one if not.
    args = [
@@ -1,26 +1,12 @@
 # This file is maintained as an up-to-date snapshot of the default
-# homeserver.yaml configuration generated by Synapse.
-#
-# It is intended to act as a reference for the default configuration,
-# helping admins keep track of new options and other changes, and compare
-# their configs with the current default.  As such, many of the actual
-# config values shown are placeholders.
+# homeserver.yaml configuration generated by Synapse. You can find a
+# complete accounting of possible configuration options at
+# https://matrix-org.github.io/synapse/latest/usage/configuration/config_documentation.html
 #
 # It is *not* intended to be copied and used as the basis for a real
 # homeserver.yaml. Instead, if you are starting from scratch, please generate
 # a fresh config using Synapse by following the instructions in
 # https://matrix-org.github.io/synapse/latest/setup/installation.html.
-
-# Configuration options that take a time period can be set using a number
-# followed by a letter. Letters have the following meanings:
-# s = second
-# m = minute
-# h = hour
-# d = day
-# w = week
-# y = year
-# For example, setting redaction_retention_period: 5m would remove redacted
-# messages from the database after 5 minutes, rather than 5 months.
-
+#
 ################################################################################

@@ -55,7 +55,6 @@
    - [Admin API](usage/administration/admin_api/README.md)
      - [Account Validity](admin_api/account_validity.md)
      - [Background Updates](usage/administration/admin_api/background_updates.md)
-      - [Delete Group](admin_api/delete_group.md)
      - [Event Reports](admin_api/event_reports.md)
      - [Media](admin_api/media_admin_api.md)
      - [Purge History](admin_api/purge_history_api.md)
@@ -70,6 +69,7 @@
      - [Federation](usage/administration/admin_api/federation.md)
    - [Manhole](manhole.md)
    - [Monitoring](metrics-howto.md)
+      - [Reporting Anonymised Statistics](usage/administration/monitoring/reporting_anonymised_statistics.md)
    - [Understanding Synapse Through Grafana Graphs](usage/administration/understanding_synapse_through_grafana_graphs.md)
    - [Useful SQL for Admins](usage/administration/useful_sql_for_admins.md)
    - [Database Maintenance Tools](usage/administration/database_maintenance_tools.md)
@@ -81,6 +81,7 @@
 # Development
  - [Contributing Guide](development/contributing_guide.md)
  - [Code Style](code_style.md)
+  - [Reviewing Code](development/reviews.md)
  - [Release Cycle](development/releases.md)
  - [Git Usage](development/git.md)
  - [Testing]()
@@ -1,14 +0,0 @@
-# Delete a local group
-
-This API lets a server admin delete a local group. Doing so will kick all
-users out of the group so that their clients will correctly handle the group
-being deleted.
-
-To use it, you will need to authenticate by providing an `access_token`
-for a server admin: see [Admin API](../usage/administration/admin_api).
-
-The API is:
-
-```
-POST /_synapse/admin/v1/delete_group/<group_id>
-```
@@ -124,9 +124,8 @@ Body parameters:
  - `address` - string. Value of third-party ID.
  belonging to a user.
 - `external_ids` - array, optional. Allow setting the identifier of the external identity
-  provider for SSO (Single sign-on). Details in
-  [Sample Configuration File](../usage/configuration/homeserver_sample_config.html)
-  section `sso` and `oidc_providers`.
+  provider for SSO (Single sign-on). Details in the configuration manual under the
+  sections [sso](../usage/configuration/config_documentation.md#sso) and [oidc_providers](../usage/configuration/config_documentation.md#oidc_providers).
  - `auth_provider` - string. ID of the external identity provider. Value of `idp_id`
    in the homeserver configuration. Note that no error is raised if the provided
    value is not in the homeserver configuration.
@@ -70,82 +70,61 @@ on save as they take a while and can be very resource intensive.
    -   Avoid wildcard imports (`from synapse.types import *`) and
        relative imports (`from .types import UserID`).

-## Configuration file format
+## Configuration code and documentation format

-The [sample configuration file](./sample_config.yaml) acts as a
+When adding a configuration option to the code, if several settings are grouped into a single dict, ensure that your code
+correctly handles the top-level option being set to `None` (as it will be if no sub-options are enabled).
+
+The [configuration manual](usage/configuration/config_documentation.md) acts as a
 reference to Synapse's configuration options for server administrators.
 Remember that many readers will be unfamiliar with YAML and server
-administration in general, so that it is important that the file be as
-easy to understand as possible, which includes following a consistent
-format.
+administration in general, so it is important that when you add
+a configuration option the documentation be as easy to understand as possible, which 
+includes following a consistent format.

 Some guidelines follow:

-   Sections should be separated with a heading consisting of a single
-    line prefixed and suffixed with `##`. There should be **two** blank
-    lines before the section header, and **one** after.
-   Each option should be listed in the file with the following format:
-    -   A comment describing the setting. Each line of this comment
-        should be prefixed with a hash (`#`) and a space.
+- Each option should be listed in the config manual with the following format:
+      
+    - The name of the option, prefixed by `###`. 

-        The comment should describe the default behaviour (ie, what
+    - A comment which describes the default behaviour (i.e. what
        happens if the setting is omitted), as well as what the effect
        will be if the setting is changed.
-
-        Often, the comment end with something like "uncomment the
-        following to <do action>".
-
-    -   A line consisting of only `#`.
-    -   A commented-out example setting, prefixed with only `#`.
+    - An example setting, using backticks to define the code block

        For boolean (on/off) options, convention is that this example
-        should be the *opposite* to the default (so the comment will end
-        with "Uncomment the following to enable [or disable]
-        <feature>." For other options, the example should give some
-        non-default value which is likely to be useful to the reader.
+        should be the *opposite* to the default. For other options, the example should give
+        some non-default value which is likely to be useful to the reader.

-   There should be a blank line between each option.
-   Where several settings are grouped into a single dict, *avoid* the
-    convention where the whole block is commented out, resulting in
-    comment lines starting `# #`, as this is hard to read and confusing
-    to edit. Instead, leave the top-level config option uncommented, and
-    follow the conventions above for sub-options. Ensure that your code
-    correctly handles the top-level option being set to `None` (as it
-    will be if no sub-options are enabled).
-   Lines should be wrapped at 80 characters.
-   Use two-space indents.
-   `true` and `false` are spelt thus (as opposed to `True`, etc.)
-   Use single quotes (`'`) rather than double-quotes (`"`) or backticks
-    (`` ` ``) to refer to configuration options.
+- There should be a horizontal rule between each option, which can be achieved by adding `---` before and
+  after the option.
+- `true` and `false` are spelt thus (as opposed to `True`, etc.)

 Example:

+---
+### `modules`
+
+Use the `module` sub-option to add a module under `modules` to extend functionality. 
+The `module` setting then has a sub-option, `config`, which can be used to define some configuration
+for the `module`.
+
+Defaults to none.
+
+Example configuration:
 ```yaml
-## Frobnication ##
-
-# The frobnicator will ensure that all requests are fully frobnicated.
-# To enable it, uncomment the following.
-#
-#frobnicator_enabled: true
-
-# By default, the frobnicator will frobnicate with the default frobber.
-# The following will make it use an alternative frobber.
-#
-#frobincator_frobber: special_frobber
-
-# Settings for the frobber
-#
-frobber:
-  # frobbing speed. Defaults to 1.
-  #
-  #speed: 10
-
-  # frobbing distance. Defaults to 1000.
-  #
-  #distance: 100
+modules:
+  - module: my_super_module.MySuperClass
+    config:
+      do_thing: true
+  - module: my_other_super_module.SomeClass
+    config: {}
 ```
+---

 Note that the sample configuration is generated from the synapse code
 and is maintained by a script, `scripts-dev/generate_sample_config.sh`.
 Making sure that the output from this script matches the desired format
 is left as an exercise for the reader!
+
@@ -309,6 +309,24 @@ The above will run a monolithic (single-process) Synapse with SQLite as the data
 - Passing `POSTGRES=1` as an environment variable to use the Postgres database instead.
 - Passing `WORKERS=1` as an environment variable to use a workerised setup instead. This option implies the use of Postgres.

+To increase the log level for the tests, set `SYNAPSE_TEST_LOG_LEVEL`, e.g:
+```sh
+SYNAPSE_TEST_LOG_LEVEL=DEBUG COMPLEMENT_DIR=../complement ./scripts-dev/complement.sh -run TestImportHistoricalMessages
+```
+
+### Prettier formatting with `gotestfmt`
+
+If you want to format the output of the tests the same way as it looks in CI,
+install [gotestfmt](https://github.com/haveyoudebuggedit/gotestfmt).
+
+You can then use this incantation to format the tests appropriately:
+
+```sh
+COMPLEMENT_DIR=../complement ./scripts-dev/complement.sh -json | gotestfmt -hide successful-tests
+```
+
+(Remove `-hide successful-tests` if you don't want to hide successful tests.)
+

 ### Access database for homeserver after Complement test runs.

@@ -333,7 +351,7 @@ To prepare a Pull Request, please:
 3. `git push` your commit to your fork of Synapse;
 4. on GitHub, [create the Pull Request](https://docs.github.com/en/github/collaborating-with-issues-and-pull-requests/creating-a-pull-request);
 5. add a [changelog entry](#changelog) and push it to your Pull Request;
-6. for most contributors, that's all - however, if you are a member of the organization `matrix-org`, on GitHub, please request a review from `matrix.org / Synapse Core`.
+6. that's it for now, a non-draft pull request will automatically request review from the team;
 7. if you need to update your PR, please avoid rebasing and just add new commits to your branch.


@@ -509,10 +527,13 @@ From this point, you should:
 1. Look at the results of the CI pipeline.
    - If there is any error, fix the error.
 2. If a developer has requested changes, make these changes and let us know if it is ready for a developer to review again.
+   - A pull request is a conversation, if you disagree with the suggestions, please respond and discuss it.
 3. Create a new commit with the changes.
    - Please do NOT overwrite the history. New commits make the reviewer's life easier.
    - Push this commits to your Pull Request.
 4. Back to 1.
+5. Once the pull request is ready for review again please re-request review from whichever developer did your initial
+  review (or leave a comment in the pull request that you believe all required changes have been done).

 Once both the CI and the developers are happy, the patch will be merged into Synapse and released shortly!

@@ -0,0 +1,41 @@
+Some notes on how we do reviews
+===============================
+
+The Synapse team works off a shared review queue -- any new pull requests for
+Synapse (or related projects) has a review requested from the entire team. Team
+members should process this queue using the following rules:
+
+* Any high urgency pull requests (e.g. fixes for broken continuous integration
+  or fixes for release blockers);
+* Follow-up reviews for pull requests which have previously received reviews;
+* Any remaining pull requests.
+
+For the latter two categories above, older pull requests should be prioritised.
+
+It is explicit that there is no priority given to pull requests from the team
+(vs from the community). If a pull request requires a quick turn around, please
+explicitly communicate this via [#synapse-dev:matrix.org](https://matrix.to/#/#synapse-dev:matrix.org)
+or as a comment on the pull request.
+
+Once an initial review has been completed and the author has made additional changes,
+follow-up reviews should go back to the same reviewer. This helps build a shared
+context and conversation between author and reviewer.
+
+As a team we aim to keep the number of inflight pull requests to a minimum to ensure
+that ongoing work is finished before starting new work.
+
+Performing a review
+-------------------
+
+To communicate to the rest of the team the status of each pull request, team
+members should do the following:
+
+* Assign themselves to the pull request (they should be left assigned to the
+  pull request until it is merged, closed, or are no longer the reviewer);
+* Review the pull request by leaving comments, questions, and suggestions;
+* Mark the pull request appropriately (as needing changes or accepted).
+
+If you are unsure about a particular part of the pull request (or are not confident
+in your understanding of part of the code) then ask questions or request review
+from the team again. When requesting review from the team be sure to leave a comment
+with the rationale on why you're putting it back in the queue.
@@ -37,27 +37,26 @@ As with other login types, there are additional fields (e.g. `device_id` and
 ## Preparing Synapse

 The JSON Web Token integration in Synapse uses the
-[`PyJWT`](https://pypi.org/project/pyjwt/) library, which must be installed
+[`Authlib`](https://docs.authlib.org/en/latest/index.html) library, which must be installed
 as follows:

- * The relevant libraries are included in the Docker images and Debian packages
-   provided by `matrix.org` so no further action is needed.
+* The relevant libraries are included in the Docker images and Debian packages
+  provided by `matrix.org` so no further action is needed.

- * If you installed Synapse into a virtualenv, run `/path/to/env/bin/pip
-   install synapse[pyjwt]` to install the necessary dependencies.
+* If you installed Synapse into a virtualenv, run `/path/to/env/bin/pip
+  install synapse[jwt]` to install the necessary dependencies.

- * For other installation mechanisms, see the documentation provided by the
-   maintainer.
+* For other installation mechanisms, see the documentation provided by the
+  maintainer.

-To enable the JSON web token integration, you should then add an `jwt_config` section
-to your configuration file (or uncomment the `enabled: true` line in the
-existing section). See [sample_config.yaml](./sample_config.yaml) for some
+To enable the JSON web token integration, you should then add a `jwt_config` option
+to your configuration file. See the [configuration manual](usage/configuration/config_documentation.md#jwt_config) for some
 sample settings.

 ## How to test JWT as a developer

 Although JSON Web Tokens are typically generated from an external server, the
-examples below use [PyJWT](https://pyjwt.readthedocs.io/en/latest/) directly.
+example below uses a locally generated JWT.

 1.  Configure Synapse with JWT logins, note that this example uses a pre-shared
    secret and an algorithm of HS256:
@@ -70,10 +69,21 @@ examples below use [PyJWT](https://pyjwt.readthedocs.io/en/latest/) directly.
    ```
 2.  Generate a JSON web token:

-    ```bash
-    $ pyjwt --key=my-secret-token --alg=HS256 encode sub=test-user
-    eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJzdWIiOiJ0ZXN0LXVzZXIifQ.Ag71GT8v01UO3w80aqRPTeuVPBIBZkYhNTJJ-_-zQIc
+    You can use the following short Python snippet to generate a JWT
+    protected by an HMAC.
+    Take care that the `secret` and the algorithm given in the `header` match
+    the entries from `jwt_config` above.
+
+    ```python
+    from authlib.jose import jwt
+
+    header = {"alg": "HS256"}
+    payload = {"sub": "user1", "aud": ["audience"]}
+    secret = "my-secret-token"
+    result = jwt.encode(header, payload, secret)
+    print(result.decode("ascii"))
    ```
+
 3.  Query for the login types and ensure `org.matrix.login.jwt` is there:

    ```bash
@@ -13,8 +13,10 @@ environments where untrusted users have shell access.

 ## Configuring the manhole

-To enable it, first uncomment the `manhole` listener configuration in
-`homeserver.yaml`. The configuration is slightly different if you're using docker.
+To enable it, first add the `manhole` listener configuration in your
+`homeserver.yaml`. You can find information on how to do that 
+in the [configuration manual](usage/configuration/config_documentation.md#manhole_settings).
+The configuration is slightly different if you're using docker.

 #### Docker config

@@ -49,9 +49,9 @@ clients.

 ## Server configuration

-Support for this feature can be enabled and configured in the
-`retention` section of the Synapse configuration file (see the
-[sample file](https://github.com/matrix-org/synapse/blob/v1.36.0/docs/sample_config.yaml#L451-L518)).
+Support for this feature can be enabled and configured by adding a the
+`retention` in the Synapse configuration file (see
+[configuration manual](usage/configuration/config_documentation.md#retention)).

 To enable support for message retention policies, set the setting
 `enabled` in this section to `true`.
@@ -65,8 +65,8 @@ message retention policy configured in its state. This allows server
 admins to ensure that messages are never kept indefinitely in a server's
 database. 

-A default policy can be defined as such, in the `retention` section of
-the configuration file:
+A default policy can be defined as such, by adding the `retention` option in
+the configuration file and adding these sub-options:

 ```yaml
 default_policy:
@@ -86,8 +86,8 @@ Purge jobs are the jobs that Synapse runs in the background to purge
 expired events from the database. They are only run if support for
 message retention policies is enabled in the server's configuration. If
 no configuration for purge jobs is configured by the server admin,
-Synapse will use a default configuration, which is described in the
-[sample configuration file](https://github.com/matrix-org/synapse/blob/v1.36.0/docs/sample_config.yaml#L451-L518).
+Synapse will use a default configuration, which is described here in the
+[configuration manual](usage/configuration/config_documentation.md#retention).

 Some server admins might want a finer control on when events are removed
 depending on an event's room's policy. This can be done by setting the
@@ -137,8 +137,8 @@ the server's database.
 ### Lifetime limits

 Server admins can set limits on the values of `max_lifetime` to use when
-purging old events in a room. These limits can be defined as such in the
-`retention` section of the configuration file:
+purging old events in a room. These limits can be defined under the
+`retention` option in the configuration file:

 ```yaml
 allowed_lifetime_min: 1d
@@ -38,15 +38,13 @@ this callback.

 _First introduced in Synapse v1.37.0_

+_Changed in Synapse v1.61.0: `synapse.module_api.NOT_SPAM` and `synapse.module_api.errors.Codes` can be returned by this callback. Returning a boolean is now deprecated._ 
+
 ```python
-async def user_may_join_room(user: str, room: str, is_invited: bool) -> bool
+async def user_may_join_room(user: str, room: str, is_invited: bool) -> Union["synapse.module_api.NOT_SPAM", "synapse.module_api.errors.Codes", bool]
 ```

-Called when a user is trying to join a room. The module must return a `bool` to indicate
-whether the user can join the room. Return `False` to prevent the user from joining the
-room; otherwise return `True` to permit the joining.
-
-The user is represented by their Matrix user ID (e.g.
+Called when a user is trying to join a room. The user is represented by their Matrix user ID (e.g.
 `@alice:example.com`) and the room is represented by its Matrix ID (e.g.
 `!room:example.com`). The module is also given a boolean to indicate whether the user
 currently has a pending invite in the room.
@@ -54,46 +52,67 @@ currently has a pending invite in the room.
 This callback isn't called if the join is performed by a server administrator, or in the
 context of a room creation.

+The callback must return one of:
+  - `synapse.module_api.NOT_SPAM`, to allow the operation. Other callbacks may still 
+    decide to reject it.
+  - `synapse.module_api.errors.Codes` to reject the operation with an error code. In case
+    of doubt, `synapse.module_api.errors.Codes.FORBIDDEN` is a good error code.
+  - (deprecated) `False`, which is the same as returning `synapse.module_api.NOT_SPAM`.
+  - (deprecated) `True`, which is the same as returning `synapse.module_api.errors.Codes.FORBIDDEN`.
+
 If multiple modules implement this callback, they will be considered in order. If a
-callback returns `True`, Synapse falls through to the next one. The value of the first
-callback that does not return `True` will be used. If this happens, Synapse will not call
-any of the subsequent implementations of this callback.
+callback returns `synapse.module_api.NOT_SPAM`, Synapse falls through to the next one.
+The value of the first callback that does not return `synapse.module_api.NOT_SPAM` will
+be used. If this happens, Synapse will not call any of the subsequent implementations of
+this callback.

 ### `user_may_invite`

 _First introduced in Synapse v1.37.0_

+_Changed in Synapse v1.62.0: `synapse.module_api.NOT_SPAM` and `synapse.module_api.errors.Codes` can be returned by this callback. Returning a boolean is now deprecated._ 
+
 ```python
-async def user_may_invite(inviter: str, invitee: str, room_id: str) -> bool
+async def user_may_invite(inviter: str, invitee: str, room_id: str) -> Union["synapse.module_api.NOT_SPAM", "synapse.module_api.errors.Codes", bool]
 ```

-Called when processing an invitation. The module must return a `bool` indicating whether
-the inviter can invite the invitee to the given room. Both inviter and invitee are
-represented by their Matrix user ID (e.g. `@alice:example.com`). Return `False` to prevent
-the invitation; otherwise return `True` to permit it.
+Called when processing an invitation. Both inviter and invitee are
+represented by their Matrix user ID (e.g. `@alice:example.com`).
+
+
+The callback must return one of:
+  - `synapse.module_api.NOT_SPAM`, to allow the operation. Other callbacks may still 
+    decide to reject it.
+  - `synapse.module_api.errors.Codes` to reject the operation with an error code. In case
+    of doubt, `synapse.module_api.errors.Codes.FORBIDDEN` is a good error code.
+
+  - (deprecated) `False`, which is the same as returning `synapse.module_api.NOT_SPAM`.
+  - (deprecated) `True`, which is the same as returning `synapse.module_api.errors.Codes.FORBIDDEN`.

 If multiple modules implement this callback, they will be considered in order. If a
-callback returns `True`, Synapse falls through to the next one. The value of the first
-callback that does not return `True` will be used. If this happens, Synapse will not call
-any of the subsequent implementations of this callback.
+callback returns `synapse.module_api.NOT_SPAM`, Synapse falls through to the next one.
+The value of the first callback that does not return `synapse.module_api.NOT_SPAM` will
+be used. If this happens, Synapse will not call any of the subsequent implementations of
+this callback.
+

 ### `user_may_send_3pid_invite`

 _First introduced in Synapse v1.45.0_

+_Changed in Synapse v1.62.0: `synapse.module_api.NOT_SPAM` and `synapse.module_api.errors.Codes` can be returned by this callback. Returning a boolean is now deprecated._ 
+
 ```python
 async def user_may_send_3pid_invite(
    inviter: str,
    medium: str,
    address: str,
    room_id: str,
-) -> bool
+) -> Union["synapse.module_api.NOT_SPAM", "synapse.module_api.errors.Codes", bool]
 ```

 Called when processing an invitation using a third-party identifier (also called a 3PID,
-e.g. an email address or a phone number). The module must return a `bool` indicating
-whether the inviter can invite the invitee to the given room. Return `False` to prevent
-the invitation; otherwise return `True` to permit it.
+e.g. an email address or a phone number). 

 The inviter is represented by their Matrix user ID (e.g. `@alice:example.com`), and the
 invitee is represented by its medium (e.g. "email") and its address
@@ -115,63 +134,108 @@ await user_may_send_3pid_invite(
 **Note**: If the third-party identifier is already associated with a matrix user ID,
 [`user_may_invite`](#user_may_invite) will be used instead.

+The callback must return one of:
+  - `synapse.module_api.NOT_SPAM`, to allow the operation. Other callbacks may still 
+    decide to reject it.
+  - `synapse.module_api.errors.Codes` to reject the operation with an error code. In case
+    of doubt, `synapse.module_api.errors.Codes.FORBIDDEN` is a good error code.
+
+  - (deprecated) `False`, which is the same as returning `synapse.module_api.NOT_SPAM`.
+  - (deprecated) `True`, which is the same as returning `synapse.module_api.errors.Codes.FORBIDDEN`.
+
 If multiple modules implement this callback, they will be considered in order. If a
-callback returns `True`, Synapse falls through to the next one. The value of the first
-callback that does not return `True` will be used. If this happens, Synapse will not call
-any of the subsequent implementations of this callback.
+callback returns `synapse.module_api.NOT_SPAM`, Synapse falls through to the next one.
+The value of the first callback that does not return `synapse.module_api.NOT_SPAM` will
+be used. If this happens, Synapse will not call any of the subsequent implementations of
+this callback.
+

 ### `user_may_create_room`

 _First introduced in Synapse v1.37.0_

+_Changed in Synapse v1.62.0: `synapse.module_api.NOT_SPAM` and `synapse.module_api.errors.Codes` can be returned by this callback. Returning a boolean is now deprecated._ 
+
 ```python
-async def user_may_create_room(user: str) -> bool
+async def user_may_create_room(user_id: str) -> Union["synapse.module_api.NOT_SPAM", "synapse.module_api.errors.Codes", bool]
 ```

-Called when processing a room creation request. The module must return a `bool` indicating
-whether the given user (represented by their Matrix user ID) is allowed to create a room.
-Return `False` to prevent room creation; otherwise return `True` to permit it.
+Called when processing a room creation request.
+
+The callback must return one of:
+  - `synapse.module_api.NOT_SPAM`, to allow the operation. Other callbacks may still 
+    decide to reject it.
+  - `synapse.module_api.errors.Codes` to reject the operation with an error code. In case
+    of doubt, `synapse.module_api.errors.Codes.FORBIDDEN` is a good error code.
+
+  - (deprecated) `False`, which is the same as returning `synapse.module_api.NOT_SPAM`.
+  - (deprecated) `True`, which is the same as returning `synapse.module_api.errors.Codes.FORBIDDEN`.

 If multiple modules implement this callback, they will be considered in order. If a
-callback returns `True`, Synapse falls through to the next one. The value of the first
-callback that does not return `True` will be used. If this happens, Synapse will not call
-any of the subsequent implementations of this callback.
+callback returns `synapse.module_api.NOT_SPAM`, Synapse falls through to the next one.
+The value of the first callback that does not return `synapse.module_api.NOT_SPAM` will
+be used. If this happens, Synapse will not call any of the subsequent implementations of
+this callback.
+
+

 ### `user_may_create_room_alias`

 _First introduced in Synapse v1.37.0_

+_Changed in Synapse v1.62.0: `synapse.module_api.NOT_SPAM` and `synapse.module_api.errors.Codes` can be returned by this callback. Returning a boolean is now deprecated._ 
+
 ```python
-async def user_may_create_room_alias(user: str, room_alias: "synapse.types.RoomAlias") -> bool
+async def user_may_create_room_alias(user_id: str, room_alias: "synapse.module_api.RoomAlias") -> Union["synapse.module_api.NOT_SPAM", "synapse.module_api.errors.Codes", bool]
 ```

-Called when trying to associate an alias with an existing room. The module must return a
-`bool` indicating whether the given user (represented by their Matrix user ID) is allowed
-to set the given alias. Return `False` to prevent the alias creation; otherwise return 
-`True` to permit it.
+Called when trying to associate an alias with an existing room.
+
+The callback must return one of:
+  - `synapse.module_api.NOT_SPAM`, to allow the operation. Other callbacks may still 
+    decide to reject it.
+  - `synapse.module_api.errors.Codes` to reject the operation with an error code. In case
+    of doubt, `synapse.module_api.errors.Codes.FORBIDDEN` is a good error code.
+
+  - (deprecated) `False`, which is the same as returning `synapse.module_api.NOT_SPAM`.
+  - (deprecated) `True`, which is the same as returning `synapse.module_api.errors.Codes.FORBIDDEN`.

 If multiple modules implement this callback, they will be considered in order. If a
-callback returns `True`, Synapse falls through to the next one. The value of the first
-callback that does not return `True` will be used. If this happens, Synapse will not call
-any of the subsequent implementations of this callback.
+callback returns `synapse.module_api.NOT_SPAM`, Synapse falls through to the next one.
+The value of the first callback that does not return `synapse.module_api.NOT_SPAM` will
+be used. If this happens, Synapse will not call any of the subsequent implementations of
+this callback.
+
+

 ### `user_may_publish_room`

 _First introduced in Synapse v1.37.0_

+_Changed in Synapse v1.62.0: `synapse.module_api.NOT_SPAM` and `synapse.module_api.errors.Codes` can be returned by this callback. Returning a boolean is now deprecated._ 
+
 ```python
-async def user_may_publish_room(user: str, room_id: str) -> bool
+async def user_may_publish_room(user_id: str, room_id: str) -> Union["synapse.module_api.NOT_SPAM", "synapse.module_api.errors.Codes", bool]
 ```

-Called when trying to publish a room to the homeserver's public rooms directory. The
-module must return a `bool` indicating whether the given user (represented by their
-Matrix user ID) is allowed to publish the given room. Return `False` to prevent the
-room from being published; otherwise return `True` to permit its publication.
+Called when trying to publish a room to the homeserver's public rooms directory.
+
+The callback must return one of:
+  - `synapse.module_api.NOT_SPAM`, to allow the operation. Other callbacks may still 
+    decide to reject it.
+  - `synapse.module_api.errors.Codes` to reject the operation with an error code. In case
+    of doubt, `synapse.module_api.errors.Codes.FORBIDDEN` is a good error code.
+
+  - (deprecated) `False`, which is the same as returning `synapse.module_api.NOT_SPAM`.
+  - (deprecated) `True`, which is the same as returning `synapse.module_api.errors.Codes.FORBIDDEN`.

 If multiple modules implement this callback, they will be considered in order. If a
-callback returns `True`, Synapse falls through to the next one. The value of the first
-callback that does not return `True` will be used. If this happens, Synapse will not call
-any of the subsequent implementations of this callback.
+callback returns `synapse.module_api.NOT_SPAM`, Synapse falls through to the next one.
+The value of the first callback that does not return `synapse.module_api.NOT_SPAM` will
+be used. If this happens, Synapse will not call any of the subsequent implementations of
+this callback.
+
+

 ### `check_username_for_spam`

@@ -239,21 +303,32 @@ this callback.

 _First introduced in Synapse v1.37.0_

+_Changed in Synapse v1.62.0: `synapse.module_api.NOT_SPAM` and `synapse.module_api.errors.Codes` can be returned by this callback. Returning a boolean is now deprecated._ 
+
 ```python
 async def check_media_file_for_spam(
    file_wrapper: "synapse.rest.media.v1.media_storage.ReadableFileWrapper",
    file_info: "synapse.rest.media.v1._base.FileInfo",
-) -> bool
+) -> Union["synapse.module_api.NOT_SPAM", "synapse.module_api.errors.Codes", bool]
 ```

-Called when storing a local or remote file. The module must return a `bool` indicating
-whether the given file should be excluded from the homeserver's media store. Return
-`True` to prevent this file from being stored; otherwise return `False`.
+Called when storing a local or remote file.
+
+The callback must return one of:
+  - `synapse.module_api.NOT_SPAM`, to allow the operation. Other callbacks may still 
+    decide to reject it.
+  - `synapse.module_api.errors.Codes` to reject the operation with an error code. In case
+    of doubt, `synapse.module_api.errors.Codes.FORBIDDEN` is a good error code.
+
+  - (deprecated) `False`, which is the same as returning `synapse.module_api.NOT_SPAM`.
+  - (deprecated) `True`, which is the same as returning `synapse.module_api.errors.Codes.FORBIDDEN`.

 If multiple modules implement this callback, they will be considered in order. If a
-callback returns `False`, Synapse falls through to the next one. The value of the first
-callback that does not return `False` will be used. If this happens, Synapse will not call
-any of the subsequent implementations of this callback.
+callback returns `synapse.module_api.NOT_SPAM`, Synapse falls through to the next one.
+The value of the first callback that does not return `synapse.module_api.NOT_SPAM` will
+be used. If this happens, Synapse will not call any of the subsequent implementations of
+this callback.
+

 ### `should_drop_federated_event`

@@ -316,6 +391,9 @@ class ListSpamChecker:
            resource=IsUserEvilResource(config),
        )

-    async def check_event_for_spam(self, event: "synapse.events.EventBase") -> Union[bool, str]:
-        return event.sender not in self.evil_users
+    async def check_event_for_spam(self, event: "synapse.events.EventBase") -> Union[Literal["NOT_SPAM"], Codes]:
+        if event.sender in self.evil_users:
+          return Codes.FORBIDDEN
+        else:
+          return synapse.module_api.NOT_SPAM
 ```
@@ -45,8 +45,8 @@ as follows:
   maintainer.

 To enable the OpenID integration, you should then add a section to the `oidc_providers`
-setting in your configuration file (or uncomment one of the existing examples).
-See [sample_config.yaml](./sample_config.yaml) for some sample settings, as well as
+setting in your configuration file.
+See the [configuration manual](usage/configuration/config_documentation.md#oidc_providers) for some sample settings, as well as
 the text below for example configurations for specific providers.

 ## Sample configs
@@ -57,8 +57,9 @@ https://www.jaegertracing.io/docs/latest/getting-started.
 ## Enable OpenTracing in Synapse

 OpenTracing is not enabled by default. It must be enabled in the
-homeserver config by uncommenting the config options under `opentracing`
-as shown in the [sample config](./sample_config.yaml). For example:
+homeserver config by adding the `opentracing` option to your config file. You can find 
+documentation about how to do this in the [config manual under the header 'Opentracing'](usage/configuration/config_documentation.md#opentracing).
+See below for an example Opentracing configuration: 

 ```yaml
 opentracing:
@@ -143,6 +143,14 @@ to do step 2.

 It is safe to at any time kill the port script and restart it.

+However, under no circumstances should the SQLite database be `VACUUM`ed between
+multiple runs of the script. Doing so can lead to an inconsistent copy of your database
+into Postgres.
+To avoid accidental error, the script will check that SQLite's `auto_vacuum` mechanism
+is disabled, but the script is not able to protect against a manual `VACUUM` operation
+performed either by the administrator or by any automated task that the administrator
+may have configured.
+
 Note that the database may take up significantly more (25% - 100% more)
 space on disk after porting to Postgres.

@@ -66,8 +66,8 @@ in Synapse can be deactivated.

 **NOTE**: This has an impact on security and is for testing purposes only!

-To deactivate the certificate validation, the following setting must be made in
-[homserver.yaml](../usage/configuration/homeserver_sample_config.md).
+To deactivate the certificate validation, the following setting must be added to
+your [homserver.yaml](../usage/configuration/homeserver_sample_config.md).

 ```yaml
 use_insecure_ssl_client_just_for_testing_do_not_use: true
@@ -84,20 +84,19 @@ file when you upgrade the Debian package to a later version.

 ##### Downstream Debian packages

-We do not recommend using the packages from the default Debian `buster`
-repository at this time, as they are old and suffer from known security
-vulnerabilities. You can install the latest version of Synapse from
-[our repository](#matrixorg-packages) or from `buster-backports`. Please
-see the [Debian documentation](https://backports.debian.org/Instructions/)
-for information on how to use backports.
-
-If you are using Debian `sid` or testing, Synapse is available in the default
-repositories and it should be possible to install it simply with:
+Andrej Shadura maintains a `matrix-synapse` package in the Debian repositories.
+For `bookworm` and `sid`, it can be installed simply with:

 ```sh
 sudo apt install matrix-synapse
 ```

+Synapse is also avaliable in `bullseye-backports`.  Please
+see the [Debian documentation](https://backports.debian.org/Instructions/)
+for information on how to use backports.
+
+`matrix-synapse` is no longer maintained for `buster` and older.
+
 ##### Downstream Ubuntu packages

 We do not recommend using the packages in the default Ubuntu repository
@@ -233,7 +232,9 @@ python -m synapse.app.homeserver \
    --report-stats=[yes|no]
 ```

-... substituting an appropriate value for `--server-name`.
+... substituting an appropriate value for `--server-name` and choosing whether
+or not to report usage statistics (hostname, Synapse version, uptime, total
+users, etc.) to the developers via the `--report-stats` argument.

 This command will generate you a config file that you can then customise, but it will
 also generate a set of keys for you. These keys will allow your homeserver to
@@ -406,11 +407,11 @@ The recommended way to do so is to set up a reverse proxy on port
 Alternatively, you can configure Synapse to expose an HTTPS port. To do
 so, you will need to edit `homeserver.yaml`, as follows:

- First, under the `listeners` section, uncomment the configuration for the
-  TLS-enabled listener. (Remove the hash sign (`#`) at the start of
-  each line). The relevant lines are like this:
+- First, under the `listeners` option, add the configuration for the
+  TLS-enabled listener like so:

 ```yaml
+listeners:
  - port: 8448
    type: http
    tls: true
@@ -418,9 +419,11 @@ so, you will need to edit `homeserver.yaml`, as follows:
      - names: [client, federation]
  ```

- You will also need to uncomment the `tls_certificate_path` and
-  `tls_private_key_path` lines under the `TLS` section. You will need to manage
-  provisioning of these certificates yourself.
+- You will also need to add the options `tls_certificate_path` and
+  `tls_private_key_path`. to your configuration file. You will need to manage provisioning of 
+   these certificates yourself.
+- You can find more information about these options as well as how to configure synapse in the 
+  [configuration manual](../usage/configuration/config_documentation.md).

  If you are using your own certificate, be sure to use a `.pem` file that
  includes the full certificate chain including any intermediate certificates
@@ -89,6 +89,65 @@ process, for example:
    dpkg -i matrix-synapse-py3_1.3.0+stretch1_amd64.deb
    ```

+# Upgrading to v1.62.0
+
+## New signatures for spam checker callbacks
+
+As a followup to changes in v1.60.0, the following spam-checker callbacks have changed signature:
+
+- `user_may_join_room`
+- `user_may_invite`
+- `user_may_send_3pid_invite`
+- `user_may_create_room`
+- `user_may_create_room_alias`
+- `user_may_publish_room`
+- `check_media_file_for_spam`
+
+For each of these methods, the previous callback signature has been deprecated.
+
+Whereas callbacks used to return `bool`, they should now return `Union["synapse.module_api.NOT_SPAM", "synapse.module_api.errors.Codes"]`.
+
+For instance, if your module implements `user_may_join_room` as follows:
+
+```python
+async def user_may_join_room(self, user_id: str, room_id: str, is_invited: bool)
+    if ...:
+        # Request is spam
+        return False
+    # Request is not spam
+    return True
+```
+
+you should rewrite it as follows:
+
+```python
+async def user_may_join_room(self, user_id: str, room_id: str, is_invited: bool)
+    if ...:
+        # Request is spam, mark it as forbidden (you may use some more precise error
+        # code if it is useful).
+        return synapse.module_api.errors.Codes.FORBIDDEN
+    # Request is not spam, mark it as such.
+    return synapse.module_api.NOT_SPAM
+```
+
+# Upgrading to v1.61.0
+
+## Removal of deprecated community/groups
+
+This release of Synapse will remove deprecated community/groups from codebase.
+
+### Worker endpoints
+
+For those who have deployed workers, following worker endpoints will no longer
+exist and they can be removed from the reverse proxy configuration:
+
+-   `^/_matrix/federation/v1/get_groups_publicised$`
+-   `^/_matrix/client/(r0|v3|unstable)/joined_groups$`
+-   `^/_matrix/client/(r0|v3|unstable)/publicised_groups$`
+-   `^/_matrix/client/(r0|v3|unstable)/publicised_groups/`
+-   `^/_matrix/federation/v1/groups/`
+-   `^/_matrix/client/(r0|v3|unstable)/groups/`
+
 # Upgrading to v1.60.0

 ## Adding a new unique index to `state_group_edges` could fail if your database is corrupted
@@ -0,0 +1,81 @@
+# Reporting Anonymised Statistics
+
+When generating your Synapse configuration file, you are asked whether you
+would like to report anonymised statistics to Matrix.org. These statistics
+provide the foundation a glimpse into the number of Synapse homeservers
+participating in the network, as well as statistics such as the number of
+rooms being created and messages being sent. This feature is sometimes
+affectionately called "phone-home" stats. Reporting
+[is optional](../../configuration/config_documentation.md#report_stats)
+and the reporting endpoint
+[can be configured](../../configuration/config_documentation.md#report_stats_endpoint),
+in case you would like to instead report statistics from a set of homeservers
+to your own infrastructure.
+
+This documentation aims to define the statistics available and the
+homeserver configuration options that exist to tweak it.
+
+## Available Statistics
+
+The following statistics are sent to the configured reporting endpoint:
+
+| Statistic Name             | Type   | Description                                                                                                                                                                                                                                                                                     |
+|----------------------------|--------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
+| `memory_rss`               | int    | The memory usage of the process (in kilobytes on Unix-based systems, bytes on MacOS).                                                                                                                                                                                                           |
+| `cpu_average`              | int    | CPU time in % of a single core (not % of all cores).                                                                                                                                                                                                                                            |              
+| `homeserver`               | string | The homeserver's server name.                                                                                                                                                                                                                                                                   |
+| `server_context`           | string | An arbitrary string used to group statistics from a set of homeservers.                                                                                                                                                                                                                         |
+| `timestamp`                | int    | The current time, represented as the number of seconds since the epoch.                                                                                                                                                                                                                         |                 
+| `uptime_seconds`           | int    | The number of seconds since the homeserver was last started.                                                                                                                                                                                                                                    |
+| `python_version`           | string | The Python version number in use (e.g "3.7.1"). Taken from `sys.version_info`.                                                                                                                                                                                                                  |
+| `total_users`              | int    | The number of registered users on the homeserver.                                                                                                                                                                                                                                               |
+| `total_nonbridged_users`   | int    | The number of users, excluding those created by an Application Service.                                                                                                                                                                                                                         |
+| `daily_user_type_native`   | int    | The number of native users created in the last 24 hours.                                                                                                                                                                                                                                        |
+| `daily_user_type_guest`    | int    | The number of guest users created in the last 24 hours.                                                                                                                                                                                                                                         |
+| `daily_user_type_bridged`  | int    | The number of users created by Application Services in the last 24 hours.                                                                                                                                                                                                                       |
+| `total_room_count`         | int    | The total number of rooms present on the homeserver.                                                                                                                                                                                                                                            |
+| `daily_active_users`       | int    | The number of unique users[^1] that have used the homeserver in the last 24 hours.                                                                                                                                                                                                              |
+| `monthly_active_users`     | int    | The number of unique users[^1] that have used the homeserver in the last 30 days.                                                                                                                                                                                                               |
+| `daily_active_rooms`       | int    | The number of rooms that have had a (state) event with the type `m.room.message` sent in them in the last 24 hours.                                                                                                                                                                             |
+| `daily_active_e2ee_rooms`  | int    | The number of rooms that have had a (state) event with the type `m.room.encrypted` sent in them in the last 24 hours.                                                                                                                                                                           |
+| `daily_messages`           | int    | The number of (state) events with the type `m.room.message` seen in the last 24 hours.                                                                                                                                                                                                          |
+| `daily_e2ee_messages`      | int    | The number of (state) events with the type `m.room.encrypted` seen in the last 24 hours.                                                                                                                                                                                                        |
+| `daily_sent_messages`      | int    | The number of (state) events sent by a local user with the type `m.room.message` seen in the last 24 hours.                                                                                                                                                                                     |
+| `daily_sent_e2ee_messages` | int    | The number of (state) events sent by a local user with the type `m.room.encrypted` seen in the last 24 hours.                                                                                                                                                                                   |
+| `r30_users_all`            | int    | The number of 30 day retained users, defined as users who have created their accounts more than 30 days ago, where they were last seen at most 30 days ago and where those two timestamps are over 30 days apart. Includes clients that do not fit into the below r30 client types.             |
+| `r30_users_android`        | int    | The number of 30 day retained users, as defined above. Filtered only to clients with "Android" in the user agent string.                                                                                                                                                                        |
+| `r30_users_ios`            | int    | The number of 30 day retained users, as defined above. Filtered only to clients with "iOS" in the user agent string.                                                                                                                                                                            |
+| `r30_users_electron`       | int    | The number of 30 day retained users, as defined above. Filtered only to clients with "Electron" in the user agent string.                                                                                                                                                                       |
+| `r30_users_web`            | int    | The number of 30 day retained users, as defined above. Filtered only to clients with "Mozilla" or "Gecko" in the user agent string.                                                                                                                                                             |
+| `r30v2_users_all`          | int    | The number of 30 day retained users, with a revised algorithm. Defined as users that appear more than once in the past 60 days, and have more than 30 days between the most and least recent appearances in the past 60 days. Includes clients that do not fit into the below r30 client types. |
+| `r30v2_users_android`      | int    | The number of 30 day retained users, as defined above. Filtered only to clients with ("riot" or "element") and "android" (case-insensitive) in the user agent string.                                                                                                                           |
+| `r30v2_users_ios`          | int    | The number of 30 day retained users, as defined above. Filtered only to clients with ("riot" or "element") and "ios" (case-insensitive) in the user agent string.                                                                                                                               |
+| `r30v2_users_electron`     | int    | The number of 30 day retained users, as defined above. Filtered only to clients with ("riot" or "element") and "electron" (case-insensitive) in the user agent string.                                                                                                                          |
+| `r30v2_users_web`          | int    | The number of 30 day retained users, as defined above. Filtered only to clients with "mozilla" or "gecko" (case-insensitive) in the user agent string.                                                                                                                                          |
+| `cache_factor`             | int    | The configured [`global factor`](../../configuration/config_documentation.md#caching) value for caching.                                                                                                                                                                                        |
+| `event_cache_size`         | int    | The configured [`event_cache_size`](../../configuration/config_documentation.md#caching) value for caching.                                                                                                                                                                                     |
+| `database_engine`          | string | The database engine that is in use. Either "psycopg2" meaning PostgreSQL is in use, or "sqlite3" for SQLite3.                                                                                                                                                                                   |
+| `database_server_version` | string | The version of the database server. Examples being "10.10" for PostgreSQL server version 10.0, and "3.38.5" for SQLite 3.38.5 installed on the system.                                                                                                                                          |
+| `log_level` | string | The log level in use. Examples are "INFO", "WARNING", "ERROR", "DEBUG", etc.                                                                                                                                                                                                                    |
+
+
+[^1]: Native matrix users and guests are always counted. If the
+[`track_puppeted_user_ips`](../../configuration/config_documentation.md#track_puppeted_user_ips)
+option is set to `true`, "puppeted" users (users that an Application Service have performed
+[an action on behalf of](https://spec.matrix.org/v1.3/application-service-api/#identity-assertion))
+will also be counted. Note that an Application Service can "puppet" any user in their
+[user namespace](https://spec.matrix.org/v1.3/application-service-api/#registration),
+not only users that the Application Service has created. If this happens, the Application Service
+will additionally be counted as a user (irrespective of `track_puppeted_user_ips`).
+
+## Using a Custom Statistics Collection Server
+
+If statistics reporting is enabled, the endpoint that Synapse sends metrics to is configured by the
+[`report_stats_endpoint`](../../configuration/config_documentation.md#report_stats_endpoint) config
+option. By default, statistics are sent to Matrix.org.
+
+If you would like to set up your own statistics collection server and send metrics there, you may
+consider using one of the following known implementations:
+
+* [Matrix.org's Panopticon](https://github.com/matrix-org/panopticon)
+* [Famedly's Barad-dûr](https://gitlab.com/famedly/company/devops/services/barad-dur)
@@ -9,6 +9,9 @@ a real homeserver.yaml. Instead, if you are starting from scratch, please genera
 a fresh config using Synapse by following the instructions in
 [Installation](../../setup/installation.md).

+Documentation for all configuration options can be found in the
+[Configuration Manual](./config_documentation.md).
+
 ```yaml
 {{#include ../../sample_config.yaml}}
 ```
@@ -4,5 +4,5 @@ Synapse supports authenticating users via the [Central Authentication
 Service protocol](https://en.wikipedia.org/wiki/Central_Authentication_Service)
 (CAS) natively.

-Please see the `cas_config` and `sso` sections of the [Synapse configuration
-file](../../../configuration/homeserver_sample_config.md) for more details.
+Please see the [cas_config](../../../configuration/config_documentation.md#cas_config) and [sso](../../../configuration/config_documentation.md#sso)
+sections of the configuration manual for more details.
--- a/Show More
+++ b/Show More
				`@@ -1 +0,0 @@`
				Add tests for cancellation of `GET /rooms/$room_id/members` and `GET /rooms/$room_id/state` requests.
				`@@ -1 +0,0 @@`
				`Add documentation for how to configure Synapse with Workers using Docker Compose. Includes example worker config and docker-compose.yaml. Contributed by @Thumbscrew.`
				`@@ -1 +0,0 @@`
				`Report login failures due to unknown third party identifiers in the same way as failures due to invalid passwords. This prevents an attacker from using the error response to determine if the identifier exists. Contributed by Daniel Aloni.`
				`@@ -1 +0,0 @@`
				`Merge the Complement testing Docker images into a single, multi-purpose image.`
				`@@ -1 +0,0 @@`
				`Clean up the test code for client disconnection.`
				`@@ -1 +0,0 @@`
				`Use lower isolation level when inserting read receipts to avoid serialization errors. Contributed by Nick @ Beeper.`
				`@@ -1 +0,0 @@`
				`Reduce the amount of state we pull from the DB.`
				`@@ -1 +0,0 @@`
				`Enable testing against PostgreSQL databases in Complement CI.`
				`@@ -1 +0,0 @@`
				Remove the `delete_device` method and always call `delete_devices`.
				`@@ -1 +0,0 @@`
				`Fix a bug introduced in Synapse 1.58 where Synapse would not report full version information when installed from a git checkout. This is a best-effort affair and not guaranteed to be stable.`