AAAA#

DBG cache all
DBG cache
2022-07-05 17:26:12 +01:00 · 2022-07-05 12:40:26 +01:00 · 2022-07-05 12:31:50 +01:00 · 2022-07-05 12:18:26 +01:00 · 2022-07-05 12:18:07 +01:00 · 2022-07-05 11:59:48 +01:00
248 changed files with 5315 additions and 8065 deletions
@@ -0,0 +1,93 @@
+{{- /*gotype: github.com/haveyoudebuggedit/gotestfmt/parser.Package*/ -}}
+{{- /*
+This template contains the format for an individual package. GitHub actions does not currently support nested groups so
+we are creating a stylized header for each package.
+
+This template is based on https://github.com/haveyoudebuggedit/gotestfmt/blob/f179b0e462a9dcf7101515d87eec4e4d7e58b92a/.gotestfmt/github/package.gotpl
+which is under the Unlicense licence.
+*/ -}}
+{{- $settings := .Settings -}}
+{{- if and (or (not $settings.HideSuccessfulPackages) (ne .Result "PASS")) (or (not $settings.HideEmptyPackages) (ne .Result "SKIP") (ne (len .TestCases) 0)) -}}
+    {{- if eq .Result "PASS" -}}
+        {{ "\033" }}[0;32m
+    {{- else if eq .Result "SKIP" -}}
+        {{ "\033" }}[0;33m
+    {{- else -}}
+        {{ "\033" }}[0;31m
+    {{- end -}}
+    📦 {{ .Name }}{{- "\033" }}[0m
+    {{- with .Coverage -}}
+       {{- "\033" -}}[0;37m ({{ . }}% coverage){{- "\033" -}}[0m
+    {{- end -}}
+    {{- "\n" -}}
+    {{- with .Reason -}}
+        {{- "  " -}}🛑 {{ . -}}{{- "\n" -}}
+    {{- end -}}
+    {{- with .Output -}}
+        {{- . -}}{{- "\n" -}}
+    {{- end -}}
+    {{- with .TestCases -}}
+        {{- /* Failing tests are first */ -}}
+        {{- range . -}}
+            {{- if and (ne .Result "PASS") (ne .Result "SKIP") -}}
+                ::group::{{ "\033" }}[0;31m❌{{ " " }}{{- .Name -}}
+                {{- "\033" -}}[0;37m ({{if $settings.ShowTestStatus}}{{.Result}}; {{end}}{{ .Duration -}}
+                {{- with .Coverage -}}
+                    , coverage: {{ . }}%
+                {{- end -}})
+                {{- "\033" -}}[0m
+                {{- "\n" -}}
+
+                {{- with .Output -}}
+                    {{- formatTestOutput . $settings -}}
+                    {{- "\n" -}}
+                {{- end -}}
+
+                ::endgroup::{{- "\n" -}}
+            {{- end -}}
+        {{- end -}}
+
+
+        {{- /* Then skipped tests are second */ -}}
+        {{- range . -}}
+            {{- if eq .Result "SKIP" -}}
+                ::group::{{ "\033" }}[0;33m🚧{{ " " }}{{- .Name -}}
+                {{- "\033" -}}[0;37m ({{if $settings.ShowTestStatus}}{{.Result}}; {{end}}{{ .Duration -}}
+                {{- with .Coverage -}}
+                    , coverage: {{ . }}%
+                {{- end -}})
+                {{- "\033" -}}[0m
+                {{- "\n" -}}
+
+                {{- with .Output -}}
+                    {{- formatTestOutput . $settings -}}
+                    {{- "\n" -}}
+                {{- end -}}
+
+                ::endgroup::{{- "\n" -}}
+            {{- end -}}
+        {{- end -}}
+
+
+        {{- /* Then passing tests are last */ -}}
+        {{- range . -}}
+            {{- if eq .Result "PASS" -}}
+                ::group::{{ "\033" }}[0;32m✅{{ " " }}{{- .Name -}}
+                {{- "\033" -}}[0;37m ({{if $settings.ShowTestStatus}}{{.Result}}; {{end}}{{ .Duration -}}
+                {{- with .Coverage -}}
+                    , coverage: {{ . }}%
+                {{- end -}})
+                {{- "\033" -}}[0m
+                {{- "\n" -}}
+
+                {{- with .Output -}}
+                    {{- formatTestOutput . $settings -}}
+                    {{- "\n" -}}
+                {{- end -}}
+
+                ::endgroup::{{- "\n" -}}
+            {{- end -}}
+        {{- end -}}
+    {{- end -}}
+    {{- "\n" -}}
+{{- end -}}
@@ -0,0 +1,36 @@
+#!/bin/sh
+#
+# Common commands to set up Complement's prerequisites in a GitHub Actions CI run.
+#
+# Must be called after Synapse has been checked out to `synapse/`.
+#
+set -eu
+
+alias block='{ set +x; } 2>/dev/null; func() { echo "::group::$*"; set -x; }; func'
+alias endblock='{ set +x; } 2>/dev/null; func() { echo "::endgroup::"; set -x; }; func'
+
+block Set Go Version
+  # The path is set via a file given by $GITHUB_PATH. We need both Go 1.17 and GOPATH on the path to run Complement.
+  # See https://docs.github.com/en/actions/using-workflows/workflow-commands-for-github-actions#adding-a-system-path
+
+  # Add Go 1.17 to the PATH: see https://github.com/actions/virtual-environments/blob/main/images/linux/Ubuntu2004-Readme.md#environment-variables-2
+  echo "$GOROOT_1_17_X64/bin" >> $GITHUB_PATH
+  # Add the Go path to the PATH: We need this so we can call gotestfmt
+  echo "~/go/bin" >> $GITHUB_PATH
+endblock
+
+block Install Complement Dependencies
+  sudo apt-get -qq update && sudo apt-get install -qqy libolm3 libolm-dev
+  go get -v github.com/haveyoudebuggedit/gotestfmt/v2/cmd/gotestfmt@latest
+endblock
+
+block Install custom gotestfmt template
+  mkdir .gotestfmt/github -p
+  cp synapse/.ci/complement_package.gotpl .gotestfmt/github/package.gotpl
+endblock
+
+block Check out Complement
+  # Attempt to check out the same branch of Complement as the PR. If it
+  # doesn't exist, fallback to HEAD.
+  synapse/.ci/scripts/checkout_complement.sh
+endblock
@@ -27,9 +27,10 @@ export VIRTUALENV_NO_DOWNLOAD=1

 # Patch the project definitions in-place:
 # - Replace all lower and tilde bounds with exact bounds
-# - Make the pyopenssl 17.0, which is the oldest version that works with
-#   a `cryptography` compiled against OpenSSL 1.1.
+# - Replace all caret bounds---but not the one that defines the supported Python version!
 # - Delete all lines referring to psycopg2 --- so no testing of postgres support.
+# - Use pyopenssl 17.0, which is the oldest version that works with
+#   a `cryptography` compiled against OpenSSL 1.1.
 # - Omit systemd: we're not logging to journal here.

 # TODO: also replace caret bounds, see https://python-poetry.org/docs/dependency-specification/#version-constraints
@@ -40,6 +41,7 @@ export VIRTUALENV_NO_DOWNLOAD=1

 sed -i \
   -e "s/[~>]=/==/g" \
+   -e '/^python = "^/!s/\^/==/g' \
   -e "/psycopg2/d" \
   -e 's/pyOpenSSL = "==16.0.0"/pyOpenSSL = "==17.0.0"/' \
   -e '/systemd/d' \
@@ -10,383 +10,37 @@ concurrency:
  cancel-in-progress: true

 jobs:
-  check-sampleconfig:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
-      - uses: actions/setup-python@v2
-      - run: pip install .
-      - run: scripts-dev/generate_sample_config.sh --check
-      - run: scripts-dev/config-lint.sh
-
-  lint:
-    uses: "matrix-org/backend-meta/.github/workflows/python-poetry-ci.yml@v1"
-    with:
-      typechecking-extras: "all"
-
-  lint-crlf:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
-      - name: Check line endings
-        run: scripts-dev/check_line_terminators.sh
-
-  lint-newsfile:
-    if: ${{ github.base_ref == 'develop'  || contains(github.base_ref, 'release-') }}
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
-        with:
-          ref: ${{ github.event.pull_request.head.sha }}
-          fetch-depth: 0
-      - uses: actions/setup-python@v2
-      - run: "pip install 'towncrier>=18.6.0rc1'"
-      - run: scripts-dev/check-newsfragment.sh
-        env:
-          PULL_REQUEST_NUMBER: ${{ github.event.number }}
-
-  # Dummy step to gate other tests on without repeating the whole list
-  linting-done:
-    if: ${{ !cancelled() }} # Run this even if prior jobs were skipped
-    needs: [lint, lint-crlf, lint-newsfile, check-sampleconfig]
-    runs-on: ubuntu-latest
-    steps:
-      - run: "true"
-
-  trial:
-    if: ${{ !cancelled() && !failure() }} # Allow previous steps to be skipped, but not fail
-    needs: linting-done
-    runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        python-version: ["3.7", "3.8", "3.9", "3.10"]
-        database: ["sqlite"]
-        extras: ["all"]
-        include:
-          # Newest Python without optional deps
-          - python-version: "3.10"
-            extras: ""
-
-          # Oldest Python with PostgreSQL
-          - python-version: "3.7"
-            database: "postgres"
-            postgres-version: "10"
-            extras: "all"
-
-          # Newest Python with newest PostgreSQL
-          - python-version: "3.10"
-            database: "postgres"
-            postgres-version: "14"
-            extras: "all"
-
-    steps:
-      - uses: actions/checkout@v2
-      - run: sudo apt-get -qq install xmlsec1
-      - name: Set up PostgreSQL ${{ matrix.postgres-version }}
-        if: ${{ matrix.postgres-version }}
-        run: |
-          docker run -d -p 5432:5432 \
-            -e POSTGRES_PASSWORD=postgres \
-            -e POSTGRES_INITDB_ARGS="--lc-collate C --lc-ctype C --encoding UTF8" \
-            postgres:${{ matrix.postgres-version }}
-      - uses: matrix-org/setup-python-poetry@v1
-        with:
-          python-version: ${{ matrix.python-version }}
-          extras: ${{ matrix.extras }}
-      - name: Await PostgreSQL
-        if: ${{ matrix.postgres-version }}
-        timeout-minutes: 2
-        run: until pg_isready -h localhost; do sleep 1; done
-      - run: poetry run trial --jobs=2 tests
-        env:
-          SYNAPSE_POSTGRES: ${{ matrix.database == 'postgres' || '' }}
-          SYNAPSE_POSTGRES_HOST: localhost
-          SYNAPSE_POSTGRES_USER: postgres
-          SYNAPSE_POSTGRES_PASSWORD: postgres
-      - name: Dump logs
-        # Logs are most useful when the command fails, always include them.
-        if: ${{ always() }}
-        # Note: Dumps to workflow logs instead of using actions/upload-artifact
-        #       This keeps logs colocated with failing jobs
-        #       It also ignores find's exit code; this is a best effort affair
-        run: >-
-          find _trial_temp -name '*.log'
-          -exec echo "::group::{}" \;
-          -exec cat {} \;
-          -exec echo "::endgroup::" \;
-          || true
-
-  trial-olddeps:
-    # Note: sqlite only; no postgres
-    if: ${{ !cancelled() && !failure() }} # Allow previous steps to be skipped, but not fail
-    needs: linting-done
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
-      - name: Test with old deps
-        uses: docker://ubuntu:focal # For old python and sqlite
-        # Note: focal seems to be using 3.8, but the oldest is 3.7?
-        # See https://github.com/matrix-org/synapse/issues/12343
-        with:
-          workdir: /github/workspace
-          entrypoint: .ci/scripts/test_old_deps.sh
-      - name: Dump logs
-        # Logs are most useful when the command fails, always include them.
-        if: ${{ always() }}
-        # Note: Dumps to workflow logs instead of using actions/upload-artifact
-        #       This keeps logs colocated with failing jobs
-        #       It also ignores find's exit code; this is a best effort affair
-        run: >-
-          find _trial_temp -name '*.log'
-          -exec echo "::group::{}" \;
-          -exec cat {} \;
-          -exec echo "::endgroup::" \;
-          || true
-
-  trial-pypy:
-    # Very slow; only run if the branch name includes 'pypy'
-    # Note: sqlite only; no postgres. Completely untested since poetry move.
-    if: ${{ contains(github.ref, 'pypy') && !failure() && !cancelled() }}
-    needs: linting-done
-    runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        python-version: ["pypy-3.7"]
-        extras: ["all"]
-
-    steps:
-      - uses: actions/checkout@v2
-      # Install libs necessary for PyPy to build binary wheels for dependencies
-      - run: sudo apt-get -qq install xmlsec1 libxml2-dev libxslt-dev
-      - uses: matrix-org/setup-python-poetry@v1
-        with:
-          python-version: ${{ matrix.python-version }}
-          extras: ${{ matrix.extras }}
-      - run: poetry run trial --jobs=2 tests
-      - name: Dump logs
-        # Logs are most useful when the command fails, always include them.
-        if: ${{ always() }}
-        # Note: Dumps to workflow logs instead of using actions/upload-artifact
-        #       This keeps logs colocated with failing jobs
-        #       It also ignores find's exit code; this is a best effort affair
-        run: >-
-          find _trial_temp -name '*.log'
-          -exec echo "::group::{}" \;
-          -exec cat {} \;
-          -exec echo "::endgroup::" \;
-          || true
-
-  sytest:
-    if: ${{ !failure() && !cancelled() }}
-    needs: linting-done
-    runs-on: ubuntu-latest
-    container:
-      image: matrixdotorg/sytest-synapse:${{ matrix.sytest-tag }}
-      volumes:
-        - ${{ github.workspace }}:/src
-      env:
-        SYTEST_BRANCH: ${{ github.head_ref }}
-        POSTGRES: ${{ matrix.postgres && 1}}
-        MULTI_POSTGRES: ${{ (matrix.postgres == 'multi-postgres') && 1}}
-        WORKERS: ${{ matrix.workers && 1 }}
-        REDIS: ${{ matrix.redis && 1 }}
-        BLACKLIST: ${{ matrix.workers && 'synapse-blacklist-with-workers' }}
-        TOP: ${{ github.workspace }}
-
-    strategy:
-      fail-fast: false
-      matrix:
-        include:
-          - sytest-tag: focal
-
-          - sytest-tag: focal
-            postgres: postgres
-
-          - sytest-tag: testing
-            postgres: postgres
-
-          - sytest-tag: focal
-            postgres: multi-postgres
-            workers: workers
-
-          - sytest-tag: buster
-            postgres: multi-postgres
-            workers: workers
-
-          - sytest-tag: buster
-            postgres: postgres
-            workers: workers
-            redis: redis
-
-    steps:
-      - uses: actions/checkout@v2
-      - name: Prepare test blacklist
-        run: cat sytest-blacklist .ci/worker-blacklist > synapse-blacklist-with-workers
-      - name: Run SyTest
-        run: /bootstrap.sh synapse
-        working-directory: /src
-      - name: Summarise results.tap
-        if: ${{ always() }}
-        run: /sytest/scripts/tap_to_gha.pl /logs/results.tap
-      - name: Upload SyTest logs
-        uses: actions/upload-artifact@v2
-        if: ${{ always() }}
-        with:
-          name: Sytest Logs - ${{ job.status }} - (${{ join(matrix.*, ', ') }})
-          path: |
-            /logs/results.tap
-            /logs/**/*.log*
-
-  export-data:
-    if: ${{ !failure() && !cancelled() }} # Allow previous steps to be skipped, but not fail
-    needs: [linting-done, portdb]
-    runs-on: ubuntu-latest
-    env:
-      TOP: ${{ github.workspace }}
-
-    services:
-      postgres:
-        image: postgres
-        ports:
-          - 5432:5432
-        env:
-          POSTGRES_PASSWORD: "postgres"
-          POSTGRES_INITDB_ARGS: "--lc-collate C --lc-ctype C --encoding UTF8"
-        options: >-
-          --health-cmd pg_isready
-          --health-interval 10s
-          --health-timeout 5s
-          --health-retries 5
-
-    steps:
-      - uses: actions/checkout@v2
-      - run: sudo apt-get -qq install xmlsec1
-      - uses: matrix-org/setup-python-poetry@v1
-        with:
-          python-version: ${{ matrix.python-version }}
-          extras: "postgres"
-      - run: .ci/scripts/test_export_data_command.sh
-
-  portdb:
-    if: ${{ !failure() && !cancelled() }} # Allow previous steps to be skipped, but not fail
-    needs: linting-done
-    runs-on: ubuntu-latest
-    env:
-      TOP: ${{ github.workspace }}
-    strategy:
-      matrix:
-        include:
-          - python-version: "3.7"
-            postgres-version: "10"
-
-          - python-version: "3.10"
-            postgres-version: "14"
-
-    services:
-      postgres:
-        image: postgres:${{ matrix.postgres-version }}
-        ports:
-          - 5432:5432
-        env:
-          POSTGRES_PASSWORD: "postgres"
-          POSTGRES_INITDB_ARGS: "--lc-collate C --lc-ctype C --encoding UTF8"
-        options: >-
-          --health-cmd pg_isready
-          --health-interval 10s
-          --health-timeout 5s
-          --health-retries 5
-
-    steps:
-      - uses: actions/checkout@v2
-      - run: sudo apt-get -qq install xmlsec1
-      - uses: matrix-org/setup-python-poetry@v1
-        with:
-          python-version: ${{ matrix.python-version }}
-          extras: "postgres"
-      - run: .ci/scripts/test_synapse_port_db.sh

  complement:
    if: "${{ !failure() && !cancelled() }}"
-    needs: linting-done
    runs-on: ubuntu-latest

    strategy:
      fail-fast: false
      matrix:
        include:
-          - arrangement: monolith
-            database: SQLite
-
-          - arrangement: monolith
+          - arrangement: workers
            database: Postgres

    steps:
-      # The path is set via a file given by $GITHUB_PATH. We need both Go 1.17 and GOPATH on the path to run Complement.
-      # See https://docs.github.com/en/actions/using-workflows/workflow-commands-for-github-actions#adding-a-system-path
-      - name: "Set Go Version"
-        run: |
-          # Add Go 1.17 to the PATH: see https://github.com/actions/virtual-environments/blob/main/images/linux/Ubuntu2004-Readme.md#environment-variables-2
-          echo "$GOROOT_1_17_X64/bin" >> $GITHUB_PATH
-          # Add the Go path to the PATH: We need this so we can call gotestfmt
-          echo "~/go/bin" >> $GITHUB_PATH
-
-      - name: "Install Complement Dependencies"
-        run: |
-          sudo apt-get update && sudo apt-get install -y libolm3 libolm-dev
-          go get -v github.com/haveyoudebuggedit/gotestfmt/v2/cmd/gotestfmt@latest
-
      - name: Run actions/checkout@v2 for synapse
        uses: actions/checkout@v2
        with:
          path: synapse

-      # Attempt to check out the same branch of Complement as the PR. If it
-      # doesn't exist, fallback to HEAD.
-      - name: Checkout complement
-        run: synapse/.ci/scripts/checkout_complement.sh
+      - name: Prepare Complement's Prerequisites
+        run: synapse/.ci/scripts/setup_complement_prerequisites.sh

      - run: |
          set -o pipefail
-          POSTGRES=${{ (matrix.database == 'Postgres') && 1 }} COMPLEMENT_DIR=`pwd`/complement synapse/scripts-dev/complement.sh -json 2>&1 | gotestfmt
-        shell: bash
-        name: Run Complement Tests
+          synapse/scripts-dev/complement.sh --build-only
+          while :; do
+            POSTGRES=${{ (matrix.database == 'Postgres') && 1 || '' }} WORKERS=${{ (matrix.arrangement == 'workers') && 1 || '' }} SYNAPSE_TEST_LOG_LEVEL=DEBUG COMPLEMENT_DIR=`pwd`/complement synapse/scripts-dev/complement.sh -f -run TestMembersLocal -json 2>&1 | gotestfmt | tee /tmp/xxx
+            if grep FAIL /tmp/xxx; then
+              break
+            fi
+          done

-  # We only run the workers tests on `develop` for now, because they're too slow to wait for on PRs.
-  # Sadly, you can't have an `if` condition on the value of a matrix, so this is a temporary, separate job for now.
-  # GitHub Actions doesn't support YAML anchors, so it's full-on duplication for now.
-  complement-developonly:
-    if: "${{ !failure() && !cancelled() && (github.ref == 'refs/heads/develop') }}"
-    needs: linting-done
-    runs-on: ubuntu-latest
-
-    steps:
-      # The path is set via a file given by $GITHUB_PATH. We need both Go 1.17 and GOPATH on the path to run Complement.
-      # See https://docs.github.com/en/actions/using-workflows/workflow-commands-for-github-actions#adding-a-system-path
-      - name: "Set Go Version"
-        run: |
-          # Add Go 1.17 to the PATH: see https://github.com/actions/virtual-environments/blob/main/images/linux/Ubuntu2004-Readme.md#environment-variables-2
-          echo "$GOROOT_1_17_X64/bin" >> $GITHUB_PATH
-          # Add the Go path to the PATH: We need this so we can call gotestfmt
-          echo "~/go/bin" >> $GITHUB_PATH
-
-      - name: "Install Complement Dependencies"
-        run: |
-          sudo apt-get update && sudo apt-get install -y libolm3 libolm-dev
-          go get -v github.com/haveyoudebuggedit/gotestfmt/v2/cmd/gotestfmt@latest
-
-      - name: Run actions/checkout@v2 for synapse
-        uses: actions/checkout@v2
-        with:
-          path: synapse
-
-      # Attempt to check out the same branch of Complement as the PR. If it
-      # doesn't exist, fallback to HEAD.
-      - name: Checkout complement
-        run: synapse/.ci/scripts/checkout_complement.sh
-
-      - run: |
-          set -o pipefail
-          WORKERS=1 COMPLEMENT_DIR=`pwd`/complement synapse/scripts-dev/complement.sh -json 2>&1 | gotestfmt
        shell: bash
        name: Run Complement Tests

@@ -394,15 +48,6 @@ jobs:
  tests-done:
    if: ${{ always() }}
    needs:
-      - check-sampleconfig
-      - lint
-      - lint-crlf
-      - lint-newsfile
-      - trial
-      - trial-olddeps
-      - sytest
-      - export-data
-      - portdb
      - complement
    runs-on: ubuntu-latest
    steps:
@@ -96,6 +96,51 @@ jobs:
            /logs/results.tap
            /logs/**/*.log*

+  complement:
+    if: "${{ !failure() && !cancelled() }}"
+    runs-on: ubuntu-latest
+
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - arrangement: monolith
+            database: SQLite
+
+          - arrangement: monolith
+            database: Postgres
+
+          - arrangement: workers
+            database: Postgres
+
+    steps:
+      - name: Run actions/checkout@v2 for synapse
+        uses: actions/checkout@v2
+        with:
+          path: synapse
+
+      - name: Prepare Complement's Prerequisites
+        run: synapse/.ci/scripts/setup_complement_prerequisites.sh
+
+      # This step is specific to the 'Twisted trunk' test run:
+      - name: Patch dependencies
+        run: |
+          set -x
+          DEBIAN_FRONTEND=noninteractive sudo apt-get install -yqq python3 pipx
+          pipx install poetry==1.1.12
+
+          poetry remove -n twisted
+          poetry add -n --extras tls git+https://github.com/twisted/twisted.git#trunk
+          poetry lock --no-update
+          # NOT IN 1.1.12 poetry lock --check
+        working-directory: synapse
+
+      - run: |
+          set -o pipefail
+          TEST_ONLY_SKIP_DEP_HASH_VERIFICATION=1 POSTGRES=${{ (matrix.database == 'Postgres') && 1 || '' }} WORKERS=${{ (matrix.arrangement == 'workers') && 1 || '' }} COMPLEMENT_DIR=`pwd`/complement synapse/scripts-dev/complement.sh -json 2>&1 | gotestfmt
+        shell: bash
+        name: Run Complement Tests
+
  # open an issue if the build fails, so we know about it.
  open-issue:
    if: failure()
@@ -103,6 +148,7 @@ jobs:
      - mypy
      - trial
      - sytest
+      - complement

    runs-on: ubuntu-latest

@@ -1,8 +1,146 @@
-Synapse 1.61.0rc1 (2022-06-07)
+Synapse 1.62.0rc2 (2022-07-01)
 ==============================

+Bugfixes
+--------
+
+- Fix unread counts for users on large servers. Introduced in v1.62.0rc1. ([\#13140](https://github.com/matrix-org/synapse/issues/13140))
+- Fix DB performance when deleting old push notifications. Introduced in v1.62.0rc1. ([\#13141](https://github.com/matrix-org/synapse/issues/13141))
+
+
+Synapse 1.62.0rc1 (2022-06-28)
+==============================
+
+Features
+--------
+
+- Port the spam-checker API callbacks to a new, richer API. This is part of an ongoing change to let spam-checker modules inform users of the reason their event or operation is rejected. ([\#12857](https://github.com/matrix-org/synapse/issues/12857), [\#13047](https://github.com/matrix-org/synapse/issues/13047))
+- Allow server admins to customise the response of the `/.well-known/matrix/client` endpoint. ([\#13035](https://github.com/matrix-org/synapse/issues/13035))
+- Add metrics measuring the CPU and DB time spent in state resolution. ([\#13036](https://github.com/matrix-org/synapse/issues/13036))
+- Speed up fetching of device list changes in `/sync` and `/keys/changes`. ([\#13045](https://github.com/matrix-org/synapse/issues/13045), [\#13098](https://github.com/matrix-org/synapse/issues/13098))
+- Improve URL previews for sites which only provide Twitter Card metadata, e.g. LWN.net. ([\#13056](https://github.com/matrix-org/synapse/issues/13056))
+
+
+Bugfixes
+--------
+
+- Update [MSC3786](https://github.com/matrix-org/matrix-spec-proposals/pull/3786) implementation to check `state_key`. ([\#12939](https://github.com/matrix-org/synapse/issues/12939))
+- Fix a bug introduced in Synapse 1.58 where Synapse would not report full version information when installed from a git checkout. This is a best-effort affair and not guaranteed to be stable. ([\#12973](https://github.com/matrix-org/synapse/issues/12973))
+- Fix a bug introduced in Synapse 1.60 where Synapse would fail to start if the `sqlite3` module was not available. ([\#12979](https://github.com/matrix-org/synapse/issues/12979))
+- Fix a bug where non-standard information was required when requesting the `/hierarchy` API over federation. Introduced 
+  in Synapse v1.41.0. ([\#12991](https://github.com/matrix-org/synapse/issues/12991))
+- Fix a long-standing bug which meant that rate limiting was not restrictive enough in some cases. ([\#13018](https://github.com/matrix-org/synapse/issues/13018))
+- Fix a bug introduced in Synapse 1.58 where profile requests for a malformed user ID would ccause an internal error. Synapse now returns 400 Bad Request in this situation. ([\#13041](https://github.com/matrix-org/synapse/issues/13041))
+- Fix some inconsistencies in the event authentication code. ([\#13087](https://github.com/matrix-org/synapse/issues/13087), [\#13088](https://github.com/matrix-org/synapse/issues/13088))
+- Fix a long-standing bug where room directory requests would cause an internal server error if given a malformed room alias. ([\#13106](https://github.com/matrix-org/synapse/issues/13106))
+
+
+Improved Documentation
+----------------------
+
+- Add documentation for how to configure Synapse with Workers using Docker Compose. Includes example worker config and docker-compose.yaml. Contributed by @Thumbscrew. ([\#12737](https://github.com/matrix-org/synapse/issues/12737))
+- Ensure the [Poetry cheat sheet](https://matrix-org.github.io/synapse/develop/development/dependencies.html) is available in the online documentation. ([\#13022](https://github.com/matrix-org/synapse/issues/13022))
+- Mention removed community/group worker endpoints in upgrade.md. Contributed by @olmari. ([\#13023](https://github.com/matrix-org/synapse/issues/13023))
+- Add instructions for running Complement with `gotestfmt`-formatted output locally. ([\#13073](https://github.com/matrix-org/synapse/issues/13073))
+- Update OpenTracing docs to reference the configuration manual rather than the configuration file. ([\#13076](https://github.com/matrix-org/synapse/issues/13076))
+- Update information on downstream Debian packages. ([\#13095](https://github.com/matrix-org/synapse/issues/13095))
+- Remove documentation for the Delete Group Admin API which no longer exists. ([\#13112](https://github.com/matrix-org/synapse/issues/13112))
+
+
+Deprecations and Removals
+-------------------------
+
+- Remove the unspecced `DELETE /directory/list/room/{roomId}` endpoint, which hid rooms from the [public room directory](https://spec.matrix.org/v1.3/client-server-api/#listing-rooms). Instead, `PUT` to the same URL with a visibility of `"private"`. ([\#13123](https://github.com/matrix-org/synapse/issues/13123))
+
+
+Internal Changes
+----------------
+
+- Add tests for cancellation of `GET /rooms/$room_id/members` and `GET /rooms/$room_id/state` requests. ([\#12674](https://github.com/matrix-org/synapse/issues/12674))
+- Report login failures due to unknown third party identifiers in the same way as failures due to invalid passwords. This prevents an attacker from using the error response to determine if the identifier exists. Contributed by Daniel Aloni. ([\#12738](https://github.com/matrix-org/synapse/issues/12738))
+- Merge the Complement testing Docker images into a single, multi-purpose image. ([\#12881](https://github.com/matrix-org/synapse/issues/12881), [\#13075](https://github.com/matrix-org/synapse/issues/13075))
+- Simplify the database schema for `event_edges`. ([\#12893](https://github.com/matrix-org/synapse/issues/12893))
+- Clean up the test code for client disconnection. ([\#12929](https://github.com/matrix-org/synapse/issues/12929))
+- Remove code generating comments in configuration. ([\#12941](https://github.com/matrix-org/synapse/issues/12941))
+- Add `Cross-Origin-Resource-Policy: cross-origin` header to content repository's thumbnail and download endpoints. ([\#12944](https://github.com/matrix-org/synapse/issues/12944))
+- Replace noop background updates with `DELETE` delta. ([\#12954](https://github.com/matrix-org/synapse/issues/12954), [\#13050](https://github.com/matrix-org/synapse/issues/13050))
+- Use lower isolation level when inserting read receipts to avoid serialization errors. Contributed by Nick @ Beeper. ([\#12957](https://github.com/matrix-org/synapse/issues/12957))
+- Reduce the amount of state we pull from the DB. ([\#12963](https://github.com/matrix-org/synapse/issues/12963))
+- Enable testing against PostgreSQL databases in Complement CI. ([\#12965](https://github.com/matrix-org/synapse/issues/12965), [\#13034](https://github.com/matrix-org/synapse/issues/13034))
+- Fix an inaccurate comment. ([\#12969](https://github.com/matrix-org/synapse/issues/12969))
+- Remove the `delete_device` method and always call `delete_devices`. ([\#12970](https://github.com/matrix-org/synapse/issues/12970))
+- Use a GitHub form for issues rather than a hard-to-read, easy-to-ignore template. ([\#12982](https://github.com/matrix-org/synapse/issues/12982))
+- Move [MSC3715](https://github.com/matrix-org/matrix-spec-proposals/pull/3715) behind an experimental config flag. ([\#12984](https://github.com/matrix-org/synapse/issues/12984))
+- Add type hints to tests. ([\#12985](https://github.com/matrix-org/synapse/issues/12985), [\#13099](https://github.com/matrix-org/synapse/issues/13099))
+- Refactor macaroon tokens generation and move the unsubscribe link in notification emails to `/_synapse/client/unsubscribe`. ([\#12986](https://github.com/matrix-org/synapse/issues/12986))
+- Fix documentation for running complement tests. ([\#12990](https://github.com/matrix-org/synapse/issues/12990))
+- Faster joins: add issue links to the TODO comments in the code. ([\#13004](https://github.com/matrix-org/synapse/issues/13004))
+- Reduce DB usage of `/sync` when a large number of unread messages have recently been sent in a room. ([\#13005](https://github.com/matrix-org/synapse/issues/13005), [\#13096](https://github.com/matrix-org/synapse/issues/13096), [\#13118](https://github.com/matrix-org/synapse/issues/13118))
+- Replaced usage of PyJWT with methods from Authlib in `org.matrix.login.jwt`. Contributed by Hannes Lerchl. ([\#13011](https://github.com/matrix-org/synapse/issues/13011))
+- Modernize the `contrib/graph/` scripts. ([\#13013](https://github.com/matrix-org/synapse/issues/13013))
+- Remove redundant `room_version` parameters from event auth functions. ([\#13017](https://github.com/matrix-org/synapse/issues/13017))
+- Decouple `synapse.api.auth_blocking.AuthBlocking` from `synapse.api.auth.Auth`. ([\#13021](https://github.com/matrix-org/synapse/issues/13021))
+- Add type annotations to `synapse.storage.databases.main.devices`. ([\#13025](https://github.com/matrix-org/synapse/issues/13025))
+- Set default `sync_response_cache_duration` to two minutes. ([\#13042](https://github.com/matrix-org/synapse/issues/13042))
+- Rename CI test runs. ([\#13046](https://github.com/matrix-org/synapse/issues/13046))
+- Increase timeout of complement CI test runs. ([\#13048](https://github.com/matrix-org/synapse/issues/13048))
+- Refactor entry points so that they all have a `main` function. ([\#13052](https://github.com/matrix-org/synapse/issues/13052))
+- Refactor the Dockerfile-workers configuration script to use Jinja2 templates in Synapse workers' Supervisord blocks. ([\#13054](https://github.com/matrix-org/synapse/issues/13054))
+- Add headers to individual options in config documentation to allow for linking. ([\#13055](https://github.com/matrix-org/synapse/issues/13055))
+- Make Complement CI logs easier to read. ([\#13057](https://github.com/matrix-org/synapse/issues/13057), [\#13058](https://github.com/matrix-org/synapse/issues/13058), [\#13069](https://github.com/matrix-org/synapse/issues/13069))
+- Don't instantiate modules with keyword arguments. ([\#13060](https://github.com/matrix-org/synapse/issues/13060))
+- Fix type checking errors against Twisted trunk. ([\#13061](https://github.com/matrix-org/synapse/issues/13061))
+- Allow MSC3030 `timestamp_to_event` calls from anyone on world-readable rooms. ([\#13062](https://github.com/matrix-org/synapse/issues/13062))
+- Add a CI job to check that schema deltas are in the correct folder. ([\#13063](https://github.com/matrix-org/synapse/issues/13063))
+- Avoid rechecking event auth rules which are independent of room state. ([\#13065](https://github.com/matrix-org/synapse/issues/13065))
+- Reduce the duplication of code that invokes the rate limiter. ([\#13070](https://github.com/matrix-org/synapse/issues/13070))
+- Add a Subject Alternative Name to the certificate generated for Complement tests. ([\#13071](https://github.com/matrix-org/synapse/issues/13071))
+- Add more tests for room upgrades. ([\#13074](https://github.com/matrix-org/synapse/issues/13074))
+- Pin dependencies maintained by matrix.org to [semantic version](https://semver.org/) bounds. ([\#13082](https://github.com/matrix-org/synapse/issues/13082))
+- Correctly report prometheus DB stats for `get_earliest_token_for_stats`. ([\#13085](https://github.com/matrix-org/synapse/issues/13085))
+- Fix a long-standing bug where a finished logging context would be re-started when Synapse failed to persist an event from federation. ([\#13089](https://github.com/matrix-org/synapse/issues/13089))
+- Simplify the alias deletion logic as an application service. ([\#13093](https://github.com/matrix-org/synapse/issues/13093))
+- Add type annotations to `tests.test_server`. ([\#13124](https://github.com/matrix-org/synapse/issues/13124))
+
+
+Synapse 1.61.1 (2022-06-28)
+===========================
+
+This patch release fixes a security issue regarding URL previews, affecting all prior versions of Synapse. Server administrators are encouraged to update Synapse as soon as possible. We are not aware of these vulnerabilities being exploited in the wild.
+
+Server administrators who are unable to update Synapse may use the workarounds described in the linked GitHub Security Advisory below.
+
+## Security advisory
+
+The following issue is fixed in 1.61.1.
+
+* [GHSA-22p3-qrh9-cx32](https://github.com/matrix-org/synapse/security/advisories/GHSA-22p3-qrh9-cx32) / [CVE-2022-31052](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2022-31052)
+
+  Synapse instances with the [`url_preview_enabled`](https://matrix-org.github.io/synapse/v1.61/usage/configuration/config_documentation.html#media-store) homeserver config option set to `true` are affected. URL previews of some web pages can lead to unbounded recursion, causing the request to either fail, or in some cases crash the running Synapse process.
+
+  Requesting URL previews requires authentication. Nevertheless, it is possible to exploit this maliciously, either by malicious users on the homeserver, or by remote users sending URLs that a local user's client may automatically request a URL preview for.
+
+  Homeservers with the `url_preview_enabled` configuration option set to `false` (the default) are unaffected. Instances with the `enable_media_repo` configuration option set to `false` are also unaffected, as this also disables URL preview functionality.
+
+  Fixed by [fa1308061802ac7b7d20e954ba7372c5ac292333](https://github.com/matrix-org/synapse/commit/fa1308061802ac7b7d20e954ba7372c5ac292333).
+
+Synapse 1.61.0 (2022-06-14)
+===========================
+
 This release removes support for the non-standard feature known both as 'groups' and as 'communities', which have been superseded by *Spaces*.

+See [the upgrade notes](https://github.com/matrix-org/synapse/blob/develop/docs/upgrade.md#upgrading-to-v1610)
+for more details.
+
+Improved Documentation
+----------------------
+
+- Mention removed community/group worker endpoints in [the upgrade notes](https://github.com/matrix-org/synapse/blob/develop/docs/upgrade.md#upgrading-to-v1610). Contributed by @olmari. ([\#13023](https://github.com/matrix-org/synapse/issues/13023))
+
+
+Synapse 1.61.0rc1 (2022-06-07)
+==============================
+
 Features
 --------

@@ -1 +0,0 @@
-Add tests for cancellation of `GET /rooms/$room_id/members` and `GET /rooms/$room_id/state` requests.
@@ -1 +0,0 @@
-Add documentation for how to configure Synapse with Workers using Docker Compose. Includes example worker config and docker-compose.yaml. Contributed by @Thumbscrew.
@@ -1 +0,0 @@
-Report login failures due to unknown third party identifiers in the same way as failures due to invalid passwords. This prevents an attacker from using the error response to determine if the identifier exists. Contributed by Daniel Aloni.
@@ -1 +0,0 @@
-Merge the Complement testing Docker images into a single, multi-purpose image.
@@ -1 +0,0 @@
-Clean up the test code for client disconnection.
@@ -1 +0,0 @@
-Use lower isolation level when inserting read receipts to avoid serialization errors. Contributed by Nick @ Beeper.
@@ -1 +0,0 @@
-Reduce the amount of state we pull from the DB.
@@ -1 +0,0 @@
-Enable testing against PostgreSQL databases in Complement CI.
@@ -1 +0,0 @@
-Fix an inaccurate comment.
@@ -1 +0,0 @@
-Remove the `delete_device` method and always call `delete_devices`.
@@ -1 +0,0 @@
-Fix a bug introduced in Synapse 1.58 where Synapse would not report full version information when installed from a git checkout. This is a best-effort affair and not guaranteed to be stable.
@@ -1 +0,0 @@
-Fix a bug introduced in Synapse 1.60 where Synapse would fail to start if the `sqlite3` module was not available.
@@ -1 +0,0 @@
-Use a GitHub form for issues rather than a hard-to-read, easy-to-ignore template.
@@ -1 +0,0 @@
-Move [MSC3715](https://github.com/matrix-org/matrix-spec-proposals/pull/3715) behind an experimental config flag.
@@ -1 +0,0 @@
-Add type annotations to `tests.state.test_v2`.
@@ -1 +0,0 @@
-Fix documentation for running complement tests.
@@ -1,2 +0,0 @@
-Fix a bug where non-standard information was required when requesting the `/hierarchy` API over federation. Introduced 
-in Synapse v1.41.0.
@@ -1 +0,0 @@
-Faster joins: add issue links to the TODO comments in the code.
@@ -1 +0,0 @@
-Modernize the `contrib/graph/` scripts.
@@ -1 +0,0 @@
-Remove redundant `room_version` parameters from event auth functions.
@@ -1 +0,0 @@
-Ensure the [Poetry cheat sheet](https://matrix-org.github.io/synapse/develop/development/dependencies.html) is available in the online documentation.
@@ -0,0 +1 @@
+Add an explanation of the `--report-stats` argument to the docs.
@@ -0,0 +1 @@
+Implement [MSC3827](https://github.com/matrix-org/matrix-spec-proposals/pull/3827): Filtering of /publicRooms by room type.
@@ -0,0 +1,3 @@
+Clean up references to sample configuration and redirect users to the configuration manual instead.
+
+
@@ -0,0 +1 @@
+Enable Complement testing in the 'Twisted Trunk' CI runs.
@@ -0,0 +1 @@
+Add documentation for anonymised homeserver statistics collection.
@@ -0,0 +1 @@
+Add missing type hints to `synapse.logging`.
@@ -0,0 +1 @@
+Raise a `DependencyError` on missing dependencies instead of a `ConfigError`.
@@ -0,0 +1 @@
+Fix wrong section header for `allow_public_rooms_over_federation` in the homeserver config documentation.
@@ -0,0 +1 @@
+Reduce DB usage of `/sync` when a large number of unread messages have recently been sent in a room.
@@ -0,0 +1 @@
+Add a rate limit for local users sending invites.
@@ -0,0 +1 @@
+Improve startup times in Complement test runs against workers, particularly in CPU-constrained environments.
@@ -0,0 +1 @@
+Only one-line SQL statements for logging and tracing.
@@ -0,0 +1 @@
+Apply ratelimiting earlier in processing of /send request.
@@ -0,0 +1 @@
+Enforce type annotations for `tests.test_server`.
@@ -0,0 +1 @@
+Add a link to the configuration manual from the homeserver sample config documentation.
@@ -0,0 +1 @@
+Add support to `complement.sh` for skipping the docker build.
@@ -0,0 +1 @@
+Faster joins: skip waiting for full state when processing incoming events over federation.
@@ -0,0 +1 @@
+Improve exception handling when processing events received over federation.
@@ -0,0 +1 @@
+Improve validation logic in Synapse's REST endpoints.
@@ -0,0 +1 @@
+Enable Complement testing in the 'Twisted Trunk' CI runs.
@@ -0,0 +1 @@
+Add support to `complement.sh` for skipping the docker build.
@@ -1,3 +1,27 @@
+matrix-synapse-py3 (1.62.0~rc2) stable; urgency=medium
+
+  * New Synapse release 1.62.0rc2.
+
+ -- Synapse Packaging team <packages@matrix.org>  Fri, 01 Jul 2022 11:42:41 +0100
+
+matrix-synapse-py3 (1.62.0~rc1) stable; urgency=medium
+
+  * New Synapse release 1.62.0rc1.
+
+ -- Synapse Packaging team <packages@matrix.org>  Tue, 28 Jun 2022 16:34:57 +0100
+
+matrix-synapse-py3 (1.61.1) stable; urgency=medium
+
+  * New Synapse release 1.61.1.
+
+ -- Synapse Packaging team <packages@matrix.org>  Tue, 28 Jun 2022 14:33:46 +0100
+
+matrix-synapse-py3 (1.61.0) stable; urgency=medium
+
+  * New Synapse release 1.61.0.
+
+ -- Synapse Packaging team <packages@matrix.org>  Tue, 14 Jun 2022 11:44:19 +0100
+
 matrix-synapse-py3 (1.61.0~rc1) stable; urgency=medium

  * Remove unused `jitsimeetbridge` experiment from `contrib` directory.
@@ -40,7 +40,7 @@ FROM docker.io/python:${PYTHON_VERSION}-slim as requirements
 RUN \
   --mount=type=cache,target=/var/cache/apt,sharing=locked \
   --mount=type=cache,target=/var/lib/apt,sharing=locked \
- apt-get update && apt-get install -y git \
+ apt-get update -qq && apt-get install -yqq git \
    && rm -rf /var/lib/apt/lists/*

 # We install poetry in its own build stage to avoid its dependencies conflicting with
@@ -62,7 +62,13 @@ WORKDIR /synapse
 # Copy just what we need to run `poetry export`...
 COPY pyproject.toml poetry.lock /synapse/

-RUN /root/.local/bin/poetry export --extras all -o /synapse/requirements.txt
+
+# If specified, we won't verify the hashes of dependencies.
+# This is only needed if the hashes of dependencies cannot be checked for some
+# reason, such as when a git repository is used directly as a dependency.
+ARG TEST_ONLY_SKIP_DEP_HASH_VERIFICATION
+
+RUN /root/.local/bin/poetry export --extras all -o /synapse/requirements.txt ${TEST_ONLY_SKIP_DEP_HASH_VERIFICATION:+--without-hashes}

 ###
 ### Stage 1: builder
@@ -73,7 +79,7 @@ FROM docker.io/python:${PYTHON_VERSION}-slim as builder
 RUN \
   --mount=type=cache,target=/var/cache/apt,sharing=locked \
   --mount=type=cache,target=/var/lib/apt,sharing=locked \
- apt-get update && apt-get install -y \
+ apt-get update -qq && apt-get install -yqq \
    build-essential \
    libffi-dev \
    libjpeg-dev \
@@ -85,6 +91,7 @@ RUN \
    openssl \
    rustc \
    zlib1g-dev \
+    git \
    && rm -rf /var/lib/apt/lists/*

 # To speed up rebuilds, install all of the dependencies before we copy over
@@ -118,7 +125,7 @@ LABEL org.opencontainers.image.licenses='Apache-2.0'
 RUN \
   --mount=type=cache,target=/var/cache/apt,sharing=locked \
   --mount=type=cache,target=/var/lib/apt,sharing=locked \
-  apt-get update && apt-get install -y \
+  apt-get update -qq && apt-get install -yqq \
    curl \
    gosu \
    libjpeg62-turbo \
@@ -6,8 +6,8 @@ FROM matrixdotorg/synapse:$SYNAPSE_VERSION
 RUN \
   --mount=type=cache,target=/var/cache/apt,sharing=locked \
   --mount=type=cache,target=/var/lib/apt,sharing=locked \
-  apt-get update && \
-  DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
+  apt-get update -qq && \
+  DEBIAN_FRONTEND=noninteractive apt-get install -yqq --no-install-recommends \
     redis-server nginx-light

 # Install supervisord with pip instead of apt, to avoid installing a second
@@ -9,7 +9,7 @@ FROM matrixdotorg/synapse-workers:$SYNAPSE_VERSION

 # Install postgresql
 RUN apt-get update && \
-  DEBIAN_FRONTEND=noninteractive apt-get install --no-install-recommends -y postgresql-13
+  DEBIAN_FRONTEND=noninteractive apt-get install --no-install-recommends -yqq postgresql-13

 # Configure a user and create a database for Synapse
 RUN pg_ctlcluster 13 main start &&  su postgres -c "echo \
@@ -7,7 +7,7 @@ so **please don't use this image for a production server**.
 This multi-purpose image is built on top of `Dockerfile-workers` in the parent directory
 and can be switched using environment variables between the following configurations:

- Monolithic Synapse with SQLite (`SYNAPSE_COMPLEMENT_DATABASE=sqlite`)
+- Monolithic Synapse with SQLite (default, or `SYNAPSE_COMPLEMENT_DATABASE=sqlite`)
 - Monolithic Synapse with Postgres (`SYNAPSE_COMPLEMENT_DATABASE=postgres`)
 - Workerised Synapse with Postgres (`SYNAPSE_COMPLEMENT_DATABASE=postgres` and `SYNAPSE_COMPLEMENT_USE_WORKERS=true`)

@@ -31,7 +31,7 @@ case "$SYNAPSE_COMPLEMENT_DATABASE" in
    export START_POSTGRES=true
    ;;

-  sqlite)
+  sqlite|"")
    # Configure supervisord not to start Postgres, as we don't need it
    export START_POSTGRES=false
    ;;
@@ -59,6 +59,9 @@ if [[ -n "$SYNAPSE_COMPLEMENT_USE_WORKERS" ]]; then
      synchrotron, \
      appservice, \
      pusher"
+
+  # Improve startup times by using a launcher based on fork()
+  export SYNAPSE_USE_EXPERIMENTAL_FORKING_LAUNCHER=1
 else
  # Empty string here means 'main process only'
  export SYNAPSE_WORKER_TYPES=""
@@ -73,14 +76,30 @@ fi

 # Generate a TLS key, then generate a certificate by having Complement's CA sign it
 # Note that both the key and certificate are in PEM format (not DER).
+
+# First generate a configuration file to set up a Subject Alternative Name.
+cat > /conf/server.tls.conf <<EOF
+.include /etc/ssl/openssl.cnf
+
+[SAN]
+subjectAltName=DNS:${SERVER_NAME}
+EOF
+
+# Generate an RSA key
 openssl genrsa -out /conf/server.tls.key 2048

-openssl req -new -key /conf/server.tls.key -out /conf/server.tls.csr \
-  -subj "/CN=${SERVER_NAME}"
+# Generate a certificate signing request
+openssl req -new -config /conf/server.tls.conf -key /conf/server.tls.key -out /conf/server.tls.csr \
+  -subj "/CN=${SERVER_NAME}" -reqexts SAN

+# Make the Complement Certificate Authority sign and generate a certificate.
 openssl x509 -req -in /conf/server.tls.csr \
  -CA /complement/ca/ca.crt -CAkey /complement/ca/ca.key -set_serial 1 \
-  -out /conf/server.tls.crt
+  -out /conf/server.tls.crt -extfile /conf/server.tls.conf -extensions SAN
+
+# Assert that we have a Subject Alternative Name in the certificate.
+# (grep will exit with 1 here if there isn't a SAN in the certificate.)
+openssl x509 -in /conf/server.tls.crt -noout -text | grep DNS:

 export SYNAPSE_TLS_CERT=/conf/server.tls.crt
 export SYNAPSE_TLS_KEY=/conf/server.tls.key
@@ -103,4 +103,10 @@ server_notices:
  system_mxid_avatar_url: ""
  room_name: "Server Alert"

+
+# Disable sync cache so that initial `/sync` requests are up-to-date.
+caches:
+  sync_response_cache_duration: 0
+
+
 {% include "shared-orig.yaml.j2" %}
@@ -31,17 +31,3 @@ autorestart=true
 # Redis can be disabled if the image is being used without workers
 autostart={{ enable_redis }}

-[program:synapse_main]
-command=/usr/local/bin/prefix-log /usr/local/bin/python -m synapse.app.homeserver --config-path="{{ main_config_path }}" --config-path=/conf/workers/shared.yaml
-priority=10
-# Log startup failures to supervisord's stdout/err
-# Regular synapse logs will still go in the configured data directory
-stdout_logfile=/dev/stdout
-stdout_logfile_maxbytes=0
-stderr_logfile=/dev/stderr
-stderr_logfile_maxbytes=0
-autorestart=unexpected
-exitcodes=0
-
-# Additional process blocks
-{{ worker_config }}
@@ -0,0 +1,52 @@
+{% if use_forking_launcher %}
+[program:synapse_fork]
+command=/usr/local/bin/python -m synapse.app.complement_fork_starter
+  {{ main_config_path }}
+  synapse.app.homeserver
+  --config-path="{{ main_config_path }}"
+  --config-path=/conf/workers/shared.yaml
+  {%- for worker in workers %}
+    -- {{ worker.app }}
+    --config-path="{{ main_config_path }}"
+    --config-path=/conf/workers/shared.yaml
+    --config-path=/conf/workers/{{ worker.name }}.yaml
+  {%- endfor %}
+stdout_logfile=/dev/stdout
+stdout_logfile_maxbytes=0
+stderr_logfile=/dev/stderr
+stderr_logfile_maxbytes=0
+autorestart=unexpected
+exitcodes=0
+
+{% else %}
+[program:synapse_main]
+command=/usr/local/bin/prefix-log /usr/local/bin/python -m synapse.app.homeserver
+  --config-path="{{ main_config_path }}"
+  --config-path=/conf/workers/shared.yaml
+priority=10
+# Log startup failures to supervisord's stdout/err
+# Regular synapse logs will still go in the configured data directory
+stdout_logfile=/dev/stdout
+stdout_logfile_maxbytes=0
+stderr_logfile=/dev/stderr
+stderr_logfile_maxbytes=0
+autorestart=unexpected
+exitcodes=0
+
+
+  {% for worker in workers %}
+[program:synapse_{{ worker.name }}]
+command=/usr/local/bin/prefix-log /usr/local/bin/python -m {{ worker.app }}
+  --config-path="{{ main_config_path }}"
+  --config-path=/conf/workers/shared.yaml
+  --config-path=/conf/workers/{{ worker.name }}.yaml
+autorestart=unexpected
+priority=500
+exitcodes=0
+stdout_logfile=/dev/stdout
+stdout_logfile_maxbytes=0
+stderr_logfile=/dev/stderr
+stderr_logfile_maxbytes=0
+
+  {% endfor %}
+{% endif %}
@@ -2,7 +2,11 @@ version: 1

 formatters:
  precise:
+    {% if include_worker_name_in_log_line %}
+    format: '{{ worker_name }} | %(asctime)s - %(name)s - %(lineno)d - %(levelname)s - %(request)s - %(message)s'
+    {% else %}
    format: '%(asctime)s - %(name)s - %(lineno)d - %(levelname)s - %(request)s - %(message)s'
+    {% endif %}

 handlers:
 {% if LOG_FILE_PATH %}
@@ -26,6 +26,9 @@
 #   * SYNAPSE_TLS_CERT: Path to a TLS certificate in PEM format.
 #   * SYNAPSE_TLS_KEY: Path to a TLS key. If this and SYNAPSE_TLS_CERT are specified,
 #         Nginx will be configured to serve TLS on port 8448.
+#   * SYNAPSE_USE_EXPERIMENTAL_FORKING_LAUNCHER: Whether to use the forking launcher,
+#         only intended for usage in Complement at the moment.
+#         No stability guarantees are provided.
 #
 # NOTE: According to Complement's ENTRYPOINT expectations for a homeserver image (as defined
 # in the project's README), this script may be run multiple times, and functionality should
@@ -52,12 +55,12 @@ WORKERS_CONFIG: Dict[str, Dict[str, Any]] = {
        "worker_extra_conf": "",
    },
    "user_dir": {
-        "app": "synapse.app.user_dir",
+        "app": "synapse.app.generic_worker",
        "listener_resources": ["client"],
        "endpoint_patterns": [
            "^/_matrix/client/(api/v1|r0|v3|unstable)/user_directory/search$"
        ],
-        "shared_extra_conf": {"update_user_directory": False},
+        "shared_extra_conf": {"update_user_directory_from_worker": "user_dir1"},
        "worker_extra_conf": "",
    },
    "media_repository": {
@@ -78,7 +81,7 @@ WORKERS_CONFIG: Dict[str, Dict[str, Any]] = {
        "app": "synapse.app.generic_worker",
        "listener_resources": [],
        "endpoint_patterns": [],
-        "shared_extra_conf": {"notify_appservices_from_worker": "appservice"},
+        "shared_extra_conf": {"notify_appservices_from_worker": "appservice1"},
        "worker_extra_conf": "",
    },
    "federation_sender": {
@@ -176,21 +179,6 @@ WORKERS_CONFIG: Dict[str, Dict[str, Any]] = {
 }

 # Templates for sections that may be inserted multiple times in config files
-SUPERVISORD_PROCESS_CONFIG_BLOCK = """
-[program:synapse_{name}]
-command=/usr/local/bin/prefix-log /usr/local/bin/python -m {app} \
-    --config-path="{config_path}" \
-    --config-path=/conf/workers/shared.yaml \
-    --config-path=/conf/workers/{name}.yaml
-autorestart=unexpected
-priority=500
-exitcodes=0
-stdout_logfile=/dev/stdout
-stdout_logfile_maxbytes=0
-stderr_logfile=/dev/stderr
-stderr_logfile_maxbytes=0
-"""
-
 NGINX_LOCATION_CONFIG_BLOCK = """
    location ~* {endpoint} {{
        proxy_pass {upstream};
@@ -353,13 +341,10 @@ def generate_worker_files(
    # This config file will be passed to all workers, included Synapse's main process.
    shared_config: Dict[str, Any] = {"listeners": listeners}

-    # The supervisord config. The contents of which will be inserted into the
-    # base supervisord jinja2 template.
-    #
-    # Supervisord will be in charge of running everything, from redis to nginx to Synapse
-    # and all of its worker processes. Load the config template, which defines a few
-    # services that are necessary to run.
-    supervisord_config = ""
+    # List of dicts that describe workers.
+    # We pass this to the Supervisor template later to generate the appropriate
+    # program blocks.
+    worker_descriptors: List[Dict[str, Any]] = []

    # Upstreams for load-balancing purposes. This dict takes the form of a worker type to the
    # ports of each worker. For example:
@@ -437,7 +422,7 @@ def generate_worker_files(
            )

        # Enable the worker in supervisord
-        supervisord_config += SUPERVISORD_PROCESS_CONFIG_BLOCK.format_map(worker_config)
+        worker_descriptors.append(worker_config)

        # Add nginx location blocks for this worker's endpoints (if any are defined)
        for pattern in worker_config["endpoint_patterns"]:
@@ -535,10 +520,17 @@ def generate_worker_files(
        "/conf/supervisord.conf.j2",
        "/etc/supervisor/supervisord.conf",
        main_config_path=config_path,
-        worker_config=supervisord_config,
        enable_redis=workers_in_use,
    )

+    convert(
+        "/conf/synapse.supervisord.conf.j2",
+        "/etc/supervisor/conf.d/synapse.conf",
+        workers=worker_descriptors,
+        main_config_path=config_path,
+        use_forking_launcher=environ.get("SYNAPSE_USE_EXPERIMENTAL_FORKING_LAUNCHER"),
+    )
+
    # healthcheck config
    convert(
        "/conf/healthcheck.sh.j2",
@@ -572,6 +564,9 @@ def generate_worker_log_config(
        log_config_filepath,
        worker_name=worker_name,
        **extra_log_template_args,
+        include_worker_name_in_log_line=environ.get(
+            "SYNAPSE_USE_EXPERIMENTAL_FORKING_LAUNCHER"
+        ),
    )
    return log_config_filepath

@@ -110,7 +110,11 @@ def generate_config_from_template(

    log_config_file = environ["SYNAPSE_LOG_CONFIG"]
    log("Generating log config file " + log_config_file)
-    convert("/conf/log.config", log_config_file, environ)
+    convert(
+        "/conf/log.config",
+        log_config_file,
+        {**environ, "include_worker_name_in_log_line": False},
+    )

    # Hopefully we already have a signing key, but generate one if not.
    args = [
@@ -1,26 +1,12 @@
 # This file is maintained as an up-to-date snapshot of the default
-# homeserver.yaml configuration generated by Synapse.
-#
-# It is intended to act as a reference for the default configuration,
-# helping admins keep track of new options and other changes, and compare
-# their configs with the current default.  As such, many of the actual
-# config values shown are placeholders.
+# homeserver.yaml configuration generated by Synapse. You can find a
+# complete accounting of possible configuration options at
+# https://matrix-org.github.io/synapse/latest/usage/configuration/config_documentation.html
 #
 # It is *not* intended to be copied and used as the basis for a real
 # homeserver.yaml. Instead, if you are starting from scratch, please generate
 # a fresh config using Synapse by following the instructions in
 # https://matrix-org.github.io/synapse/latest/setup/installation.html.
-
-# Configuration options that take a time period can be set using a number
-# followed by a letter. Letters have the following meanings:
-# s = second
-# m = minute
-# h = hour
-# d = day
-# w = week
-# y = year
-# For example, setting redaction_retention_period: 5m would remove redacted
-# messages from the database after 5 minutes, rather than 5 months.
-
+#
 ################################################################################

@@ -55,7 +55,6 @@
    - [Admin API](usage/administration/admin_api/README.md)
      - [Account Validity](admin_api/account_validity.md)
      - [Background Updates](usage/administration/admin_api/background_updates.md)
-      - [Delete Group](admin_api/delete_group.md)
      - [Event Reports](admin_api/event_reports.md)
      - [Media](admin_api/media_admin_api.md)
      - [Purge History](admin_api/purge_history_api.md)
@@ -70,6 +69,7 @@
      - [Federation](usage/administration/admin_api/federation.md)
    - [Manhole](manhole.md)
    - [Monitoring](metrics-howto.md)
+      - [Reporting Anonymised Statistics](usage/administration/monitoring/reporting_anonymised_statistics.md)
    - [Understanding Synapse Through Grafana Graphs](usage/administration/understanding_synapse_through_grafana_graphs.md)
    - [Useful SQL for Admins](usage/administration/useful_sql_for_admins.md)
    - [Database Maintenance Tools](usage/administration/database_maintenance_tools.md)
@@ -1,14 +0,0 @@
-# Delete a local group
-
-This API lets a server admin delete a local group. Doing so will kick all
-users out of the group so that their clients will correctly handle the group
-being deleted.
-
-To use it, you will need to authenticate by providing an `access_token`
-for a server admin: see [Admin API](../usage/administration/admin_api).
-
-The API is:
-
-```
-POST /_synapse/admin/v1/delete_group/<group_id>
-```
@@ -124,9 +124,8 @@ Body parameters:
  - `address` - string. Value of third-party ID.
  belonging to a user.
 - `external_ids` - array, optional. Allow setting the identifier of the external identity
-  provider for SSO (Single sign-on). Details in
-  [Sample Configuration File](../usage/configuration/homeserver_sample_config.html)
-  section `sso` and `oidc_providers`.
+  provider for SSO (Single sign-on). Details in the configuration manual under the
+  sections [sso](../usage/configuration/config_documentation.md#sso) and [oidc_providers](../usage/configuration/config_documentation.md#oidc_providers).
  - `auth_provider` - string. ID of the external identity provider. Value of `idp_id`
    in the homeserver configuration. Note that no error is raised if the provided
    value is not in the homeserver configuration.
@@ -70,82 +70,61 @@ on save as they take a while and can be very resource intensive.
    -   Avoid wildcard imports (`from synapse.types import *`) and
        relative imports (`from .types import UserID`).

-## Configuration file format
+## Configuration code and documentation format

-The [sample configuration file](./sample_config.yaml) acts as a
+When adding a configuration option to the code, if several settings are grouped into a single dict, ensure that your code
+correctly handles the top-level option being set to `None` (as it will be if no sub-options are enabled).
+
+The [configuration manual](usage/configuration/config_documentation.md) acts as a
 reference to Synapse's configuration options for server administrators.
 Remember that many readers will be unfamiliar with YAML and server
-administration in general, so that it is important that the file be as
-easy to understand as possible, which includes following a consistent
-format.
+administration in general, so it is important that when you add
+a configuration option the documentation be as easy to understand as possible, which 
+includes following a consistent format.

 Some guidelines follow:

-   Sections should be separated with a heading consisting of a single
-    line prefixed and suffixed with `##`. There should be **two** blank
-    lines before the section header, and **one** after.
-   Each option should be listed in the file with the following format:
-    -   A comment describing the setting. Each line of this comment
-        should be prefixed with a hash (`#`) and a space.
+- Each option should be listed in the config manual with the following format:
+      
+    - The name of the option, prefixed by `###`. 

-        The comment should describe the default behaviour (ie, what
+    - A comment which describes the default behaviour (i.e. what
        happens if the setting is omitted), as well as what the effect
        will be if the setting is changed.
-
-        Often, the comment end with something like "uncomment the
-        following to <do action>".
-
-    -   A line consisting of only `#`.
-    -   A commented-out example setting, prefixed with only `#`.
+    - An example setting, using backticks to define the code block

        For boolean (on/off) options, convention is that this example
-        should be the *opposite* to the default (so the comment will end
-        with "Uncomment the following to enable [or disable]
-        <feature>." For other options, the example should give some
-        non-default value which is likely to be useful to the reader.
+        should be the *opposite* to the default. For other options, the example should give
+        some non-default value which is likely to be useful to the reader.

-   There should be a blank line between each option.
-   Where several settings are grouped into a single dict, *avoid* the
-    convention where the whole block is commented out, resulting in
-    comment lines starting `# #`, as this is hard to read and confusing
-    to edit. Instead, leave the top-level config option uncommented, and
-    follow the conventions above for sub-options. Ensure that your code
-    correctly handles the top-level option being set to `None` (as it
-    will be if no sub-options are enabled).
-   Lines should be wrapped at 80 characters.
-   Use two-space indents.
-   `true` and `false` are spelt thus (as opposed to `True`, etc.)
-   Use single quotes (`'`) rather than double-quotes (`"`) or backticks
-    (`` ` ``) to refer to configuration options.
+- There should be a horizontal rule between each option, which can be achieved by adding `---` before and
+  after the option.
+- `true` and `false` are spelt thus (as opposed to `True`, etc.)

 Example:

+---
+### `modules`
+
+Use the `module` sub-option to add a module under `modules` to extend functionality. 
+The `module` setting then has a sub-option, `config`, which can be used to define some configuration
+for the `module`.
+
+Defaults to none.
+
+Example configuration:
 ```yaml
-## Frobnication ##
-
-# The frobnicator will ensure that all requests are fully frobnicated.
-# To enable it, uncomment the following.
-#
-#frobnicator_enabled: true
-
-# By default, the frobnicator will frobnicate with the default frobber.
-# The following will make it use an alternative frobber.
-#
-#frobincator_frobber: special_frobber
-
-# Settings for the frobber
-#
-frobber:
-  # frobbing speed. Defaults to 1.
-  #
-  #speed: 10
-
-  # frobbing distance. Defaults to 1000.
-  #
-  #distance: 100
+modules:
+  - module: my_super_module.MySuperClass
+    config:
+      do_thing: true
+  - module: my_other_super_module.SomeClass
+    config: {}
 ```
+---

 Note that the sample configuration is generated from the synapse code
 and is maintained by a script, `scripts-dev/generate_sample_config.sh`.
 Making sure that the output from this script matches the desired format
 is left as an exercise for the reader!
+
@@ -310,6 +310,20 @@ The above will run a monolithic (single-process) Synapse with SQLite as the data
 - Passing `WORKERS=1` as an environment variable to use a workerised setup instead. This option implies the use of Postgres.


+### Prettier formatting with `gotestfmt`
+
+If you want to format the output of the tests the same way as it looks in CI,
+install [gotestfmt](https://github.com/haveyoudebuggedit/gotestfmt).
+
+You can then use this incantation to format the tests appropriately:
+
+```sh
+COMPLEMENT_DIR=../complement ./scripts-dev/complement.sh -json | gotestfmt -hide successful-tests
+```
+
+(Remove `-hide successful-tests` if you don't want to hide successful tests.)
+
+
 ### Access database for homeserver after Complement test runs.

 If you're curious what the database looks like after you run some tests, here are some steps to get you going in Synapse:
@@ -37,27 +37,26 @@ As with other login types, there are additional fields (e.g. `device_id` and
 ## Preparing Synapse

 The JSON Web Token integration in Synapse uses the
-[`PyJWT`](https://pypi.org/project/pyjwt/) library, which must be installed
+[`Authlib`](https://docs.authlib.org/en/latest/index.html) library, which must be installed
 as follows:

- * The relevant libraries are included in the Docker images and Debian packages
-   provided by `matrix.org` so no further action is needed.
+* The relevant libraries are included in the Docker images and Debian packages
+  provided by `matrix.org` so no further action is needed.

- * If you installed Synapse into a virtualenv, run `/path/to/env/bin/pip
-   install synapse[pyjwt]` to install the necessary dependencies.
+* If you installed Synapse into a virtualenv, run `/path/to/env/bin/pip
+  install synapse[jwt]` to install the necessary dependencies.

- * For other installation mechanisms, see the documentation provided by the
-   maintainer.
+* For other installation mechanisms, see the documentation provided by the
+  maintainer.

-To enable the JSON web token integration, you should then add an `jwt_config` section
-to your configuration file (or uncomment the `enabled: true` line in the
-existing section). See [sample_config.yaml](./sample_config.yaml) for some
+To enable the JSON web token integration, you should then add a `jwt_config` option
+to your configuration file. See the [configuration manual](usage/configuration/config_documentation.md#jwt_config) for some
 sample settings.

 ## How to test JWT as a developer

 Although JSON Web Tokens are typically generated from an external server, the
-examples below use [PyJWT](https://pyjwt.readthedocs.io/en/latest/) directly.
+example below uses a locally generated JWT.

 1.  Configure Synapse with JWT logins, note that this example uses a pre-shared
    secret and an algorithm of HS256:
@@ -70,10 +69,21 @@ examples below use [PyJWT](https://pyjwt.readthedocs.io/en/latest/) directly.
    ```
 2.  Generate a JSON web token:

-    ```bash
-    $ pyjwt --key=my-secret-token --alg=HS256 encode sub=test-user
-    eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJzdWIiOiJ0ZXN0LXVzZXIifQ.Ag71GT8v01UO3w80aqRPTeuVPBIBZkYhNTJJ-_-zQIc
+    You can use the following short Python snippet to generate a JWT
+    protected by an HMAC.
+    Take care that the `secret` and the algorithm given in the `header` match
+    the entries from `jwt_config` above.
+
+    ```python
+    from authlib.jose import jwt
+
+    header = {"alg": "HS256"}
+    payload = {"sub": "user1", "aud": ["audience"]}
+    secret = "my-secret-token"
+    result = jwt.encode(header, payload, secret)
+    print(result.decode("ascii"))
    ```
+
 3.  Query for the login types and ensure `org.matrix.login.jwt` is there:

    ```bash
@@ -13,8 +13,10 @@ environments where untrusted users have shell access.

 ## Configuring the manhole

-To enable it, first uncomment the `manhole` listener configuration in
-`homeserver.yaml`. The configuration is slightly different if you're using docker.
+To enable it, first add the `manhole` listener configuration in your
+`homeserver.yaml`. You can find information on how to do that 
+in the [configuration manual](usage/configuration/config_documentation.md#manhole_settings).
+The configuration is slightly different if you're using docker.

 #### Docker config

@@ -49,9 +49,9 @@ clients.

 ## Server configuration

-Support for this feature can be enabled and configured in the
-`retention` section of the Synapse configuration file (see the
-[sample file](https://github.com/matrix-org/synapse/blob/v1.36.0/docs/sample_config.yaml#L451-L518)).
+Support for this feature can be enabled and configured by adding a the
+`retention` in the Synapse configuration file (see
+[configuration manual](usage/configuration/config_documentation.md#retention)).

 To enable support for message retention policies, set the setting
 `enabled` in this section to `true`.
@@ -65,8 +65,8 @@ message retention policy configured in its state. This allows server
 admins to ensure that messages are never kept indefinitely in a server's
 database. 

-A default policy can be defined as such, in the `retention` section of
-the configuration file:
+A default policy can be defined as such, by adding the `retention` option in
+the configuration file and adding these sub-options:

 ```yaml
 default_policy:
@@ -86,8 +86,8 @@ Purge jobs are the jobs that Synapse runs in the background to purge
 expired events from the database. They are only run if support for
 message retention policies is enabled in the server's configuration. If
 no configuration for purge jobs is configured by the server admin,
-Synapse will use a default configuration, which is described in the
-[sample configuration file](https://github.com/matrix-org/synapse/blob/v1.36.0/docs/sample_config.yaml#L451-L518).
+Synapse will use a default configuration, which is described here in the
+[configuration manual](usage/configuration/config_documentation.md#retention).

 Some server admins might want a finer control on when events are removed
 depending on an event's room's policy. This can be done by setting the
@@ -137,8 +137,8 @@ the server's database.
 ### Lifetime limits

 Server admins can set limits on the values of `max_lifetime` to use when
-purging old events in a room. These limits can be defined as such in the
-`retention` section of the configuration file:
+purging old events in a room. These limits can be defined under the
+`retention` option in the configuration file:

 ```yaml
 allowed_lifetime_min: 1d
@@ -38,15 +38,13 @@ this callback.

 _First introduced in Synapse v1.37.0_

+_Changed in Synapse v1.61.0: `synapse.module_api.NOT_SPAM` and `synapse.module_api.errors.Codes` can be returned by this callback. Returning a boolean is now deprecated._ 
+
 ```python
-async def user_may_join_room(user: str, room: str, is_invited: bool) -> bool
+async def user_may_join_room(user: str, room: str, is_invited: bool) -> Union["synapse.module_api.NOT_SPAM", "synapse.module_api.errors.Codes", bool]
 ```

-Called when a user is trying to join a room. The module must return a `bool` to indicate
-whether the user can join the room. Return `False` to prevent the user from joining the
-room; otherwise return `True` to permit the joining.
-
-The user is represented by their Matrix user ID (e.g.
+Called when a user is trying to join a room. The user is represented by their Matrix user ID (e.g.
 `@alice:example.com`) and the room is represented by its Matrix ID (e.g.
 `!room:example.com`). The module is also given a boolean to indicate whether the user
 currently has a pending invite in the room.
@@ -54,46 +52,67 @@ currently has a pending invite in the room.
 This callback isn't called if the join is performed by a server administrator, or in the
 context of a room creation.

+The callback must return one of:
+  - `synapse.module_api.NOT_SPAM`, to allow the operation. Other callbacks may still 
+    decide to reject it.
+  - `synapse.module_api.errors.Codes` to reject the operation with an error code. In case
+    of doubt, `synapse.module_api.errors.Codes.FORBIDDEN` is a good error code.
+  - (deprecated) `False`, which is the same as returning `synapse.module_api.NOT_SPAM`.
+  - (deprecated) `True`, which is the same as returning `synapse.module_api.errors.Codes.FORBIDDEN`.
+
 If multiple modules implement this callback, they will be considered in order. If a
-callback returns `True`, Synapse falls through to the next one. The value of the first
-callback that does not return `True` will be used. If this happens, Synapse will not call
-any of the subsequent implementations of this callback.
+callback returns `synapse.module_api.NOT_SPAM`, Synapse falls through to the next one.
+The value of the first callback that does not return `synapse.module_api.NOT_SPAM` will
+be used. If this happens, Synapse will not call any of the subsequent implementations of
+this callback.

 ### `user_may_invite`

 _First introduced in Synapse v1.37.0_

+_Changed in Synapse v1.62.0: `synapse.module_api.NOT_SPAM` and `synapse.module_api.errors.Codes` can be returned by this callback. Returning a boolean is now deprecated._ 
+
 ```python
-async def user_may_invite(inviter: str, invitee: str, room_id: str) -> bool
+async def user_may_invite(inviter: str, invitee: str, room_id: str) -> Union["synapse.module_api.NOT_SPAM", "synapse.module_api.errors.Codes", bool]
 ```

-Called when processing an invitation. The module must return a `bool` indicating whether
-the inviter can invite the invitee to the given room. Both inviter and invitee are
-represented by their Matrix user ID (e.g. `@alice:example.com`). Return `False` to prevent
-the invitation; otherwise return `True` to permit it.
+Called when processing an invitation. Both inviter and invitee are
+represented by their Matrix user ID (e.g. `@alice:example.com`).
+
+
+The callback must return one of:
+  - `synapse.module_api.NOT_SPAM`, to allow the operation. Other callbacks may still 
+    decide to reject it.
+  - `synapse.module_api.errors.Codes` to reject the operation with an error code. In case
+    of doubt, `synapse.module_api.errors.Codes.FORBIDDEN` is a good error code.
+
+  - (deprecated) `False`, which is the same as returning `synapse.module_api.NOT_SPAM`.
+  - (deprecated) `True`, which is the same as returning `synapse.module_api.errors.Codes.FORBIDDEN`.

 If multiple modules implement this callback, they will be considered in order. If a
-callback returns `True`, Synapse falls through to the next one. The value of the first
-callback that does not return `True` will be used. If this happens, Synapse will not call
-any of the subsequent implementations of this callback.
+callback returns `synapse.module_api.NOT_SPAM`, Synapse falls through to the next one.
+The value of the first callback that does not return `synapse.module_api.NOT_SPAM` will
+be used. If this happens, Synapse will not call any of the subsequent implementations of
+this callback.
+

 ### `user_may_send_3pid_invite`

 _First introduced in Synapse v1.45.0_

+_Changed in Synapse v1.62.0: `synapse.module_api.NOT_SPAM` and `synapse.module_api.errors.Codes` can be returned by this callback. Returning a boolean is now deprecated._ 
+
 ```python
 async def user_may_send_3pid_invite(
    inviter: str,
    medium: str,
    address: str,
    room_id: str,
-) -> bool
+) -> Union["synapse.module_api.NOT_SPAM", "synapse.module_api.errors.Codes", bool]
 ```

 Called when processing an invitation using a third-party identifier (also called a 3PID,
-e.g. an email address or a phone number). The module must return a `bool` indicating
-whether the inviter can invite the invitee to the given room. Return `False` to prevent
-the invitation; otherwise return `True` to permit it.
+e.g. an email address or a phone number). 

 The inviter is represented by their Matrix user ID (e.g. `@alice:example.com`), and the
 invitee is represented by its medium (e.g. "email") and its address
@@ -115,63 +134,108 @@ await user_may_send_3pid_invite(
 **Note**: If the third-party identifier is already associated with a matrix user ID,
 [`user_may_invite`](#user_may_invite) will be used instead.

+The callback must return one of:
+  - `synapse.module_api.NOT_SPAM`, to allow the operation. Other callbacks may still 
+    decide to reject it.
+  - `synapse.module_api.errors.Codes` to reject the operation with an error code. In case
+    of doubt, `synapse.module_api.errors.Codes.FORBIDDEN` is a good error code.
+
+  - (deprecated) `False`, which is the same as returning `synapse.module_api.NOT_SPAM`.
+  - (deprecated) `True`, which is the same as returning `synapse.module_api.errors.Codes.FORBIDDEN`.
+
 If multiple modules implement this callback, they will be considered in order. If a
-callback returns `True`, Synapse falls through to the next one. The value of the first
-callback that does not return `True` will be used. If this happens, Synapse will not call
-any of the subsequent implementations of this callback.
+callback returns `synapse.module_api.NOT_SPAM`, Synapse falls through to the next one.
+The value of the first callback that does not return `synapse.module_api.NOT_SPAM` will
+be used. If this happens, Synapse will not call any of the subsequent implementations of
+this callback.
+

 ### `user_may_create_room`

 _First introduced in Synapse v1.37.0_

+_Changed in Synapse v1.62.0: `synapse.module_api.NOT_SPAM` and `synapse.module_api.errors.Codes` can be returned by this callback. Returning a boolean is now deprecated._ 
+
 ```python
-async def user_may_create_room(user: str) -> bool
+async def user_may_create_room(user_id: str) -> Union["synapse.module_api.NOT_SPAM", "synapse.module_api.errors.Codes", bool]
 ```

-Called when processing a room creation request. The module must return a `bool` indicating
-whether the given user (represented by their Matrix user ID) is allowed to create a room.
-Return `False` to prevent room creation; otherwise return `True` to permit it.
+Called when processing a room creation request.
+
+The callback must return one of:
+  - `synapse.module_api.NOT_SPAM`, to allow the operation. Other callbacks may still 
+    decide to reject it.
+  - `synapse.module_api.errors.Codes` to reject the operation with an error code. In case
+    of doubt, `synapse.module_api.errors.Codes.FORBIDDEN` is a good error code.
+
+  - (deprecated) `False`, which is the same as returning `synapse.module_api.NOT_SPAM`.
+  - (deprecated) `True`, which is the same as returning `synapse.module_api.errors.Codes.FORBIDDEN`.

 If multiple modules implement this callback, they will be considered in order. If a
-callback returns `True`, Synapse falls through to the next one. The value of the first
-callback that does not return `True` will be used. If this happens, Synapse will not call
-any of the subsequent implementations of this callback.
+callback returns `synapse.module_api.NOT_SPAM`, Synapse falls through to the next one.
+The value of the first callback that does not return `synapse.module_api.NOT_SPAM` will
+be used. If this happens, Synapse will not call any of the subsequent implementations of
+this callback.
+
+

 ### `user_may_create_room_alias`

 _First introduced in Synapse v1.37.0_

+_Changed in Synapse v1.62.0: `synapse.module_api.NOT_SPAM` and `synapse.module_api.errors.Codes` can be returned by this callback. Returning a boolean is now deprecated._ 
+
 ```python
-async def user_may_create_room_alias(user: str, room_alias: "synapse.types.RoomAlias") -> bool
+async def user_may_create_room_alias(user_id: str, room_alias: "synapse.module_api.RoomAlias") -> Union["synapse.module_api.NOT_SPAM", "synapse.module_api.errors.Codes", bool]
 ```

-Called when trying to associate an alias with an existing room. The module must return a
-`bool` indicating whether the given user (represented by their Matrix user ID) is allowed
-to set the given alias. Return `False` to prevent the alias creation; otherwise return 
-`True` to permit it.
+Called when trying to associate an alias with an existing room.
+
+The callback must return one of:
+  - `synapse.module_api.NOT_SPAM`, to allow the operation. Other callbacks may still 
+    decide to reject it.
+  - `synapse.module_api.errors.Codes` to reject the operation with an error code. In case
+    of doubt, `synapse.module_api.errors.Codes.FORBIDDEN` is a good error code.
+
+  - (deprecated) `False`, which is the same as returning `synapse.module_api.NOT_SPAM`.
+  - (deprecated) `True`, which is the same as returning `synapse.module_api.errors.Codes.FORBIDDEN`.

 If multiple modules implement this callback, they will be considered in order. If a
-callback returns `True`, Synapse falls through to the next one. The value of the first
-callback that does not return `True` will be used. If this happens, Synapse will not call
-any of the subsequent implementations of this callback.
+callback returns `synapse.module_api.NOT_SPAM`, Synapse falls through to the next one.
+The value of the first callback that does not return `synapse.module_api.NOT_SPAM` will
+be used. If this happens, Synapse will not call any of the subsequent implementations of
+this callback.
+
+

 ### `user_may_publish_room`

 _First introduced in Synapse v1.37.0_

+_Changed in Synapse v1.62.0: `synapse.module_api.NOT_SPAM` and `synapse.module_api.errors.Codes` can be returned by this callback. Returning a boolean is now deprecated._ 
+
 ```python
-async def user_may_publish_room(user: str, room_id: str) -> bool
+async def user_may_publish_room(user_id: str, room_id: str) -> Union["synapse.module_api.NOT_SPAM", "synapse.module_api.errors.Codes", bool]
 ```

-Called when trying to publish a room to the homeserver's public rooms directory. The
-module must return a `bool` indicating whether the given user (represented by their
-Matrix user ID) is allowed to publish the given room. Return `False` to prevent the
-room from being published; otherwise return `True` to permit its publication.
+Called when trying to publish a room to the homeserver's public rooms directory.
+
+The callback must return one of:
+  - `synapse.module_api.NOT_SPAM`, to allow the operation. Other callbacks may still 
+    decide to reject it.
+  - `synapse.module_api.errors.Codes` to reject the operation with an error code. In case
+    of doubt, `synapse.module_api.errors.Codes.FORBIDDEN` is a good error code.
+
+  - (deprecated) `False`, which is the same as returning `synapse.module_api.NOT_SPAM`.
+  - (deprecated) `True`, which is the same as returning `synapse.module_api.errors.Codes.FORBIDDEN`.

 If multiple modules implement this callback, they will be considered in order. If a
-callback returns `True`, Synapse falls through to the next one. The value of the first
-callback that does not return `True` will be used. If this happens, Synapse will not call
-any of the subsequent implementations of this callback.
+callback returns `synapse.module_api.NOT_SPAM`, Synapse falls through to the next one.
+The value of the first callback that does not return `synapse.module_api.NOT_SPAM` will
+be used. If this happens, Synapse will not call any of the subsequent implementations of
+this callback.
+
+

 ### `check_username_for_spam`

@@ -239,21 +303,32 @@ this callback.

 _First introduced in Synapse v1.37.0_

+_Changed in Synapse v1.62.0: `synapse.module_api.NOT_SPAM` and `synapse.module_api.errors.Codes` can be returned by this callback. Returning a boolean is now deprecated._ 
+
 ```python
 async def check_media_file_for_spam(
    file_wrapper: "synapse.rest.media.v1.media_storage.ReadableFileWrapper",
    file_info: "synapse.rest.media.v1._base.FileInfo",
-) -> bool
+) -> Union["synapse.module_api.NOT_SPAM", "synapse.module_api.errors.Codes", bool]
 ```

-Called when storing a local or remote file. The module must return a `bool` indicating
-whether the given file should be excluded from the homeserver's media store. Return
-`True` to prevent this file from being stored; otherwise return `False`.
+Called when storing a local or remote file.
+
+The callback must return one of:
+  - `synapse.module_api.NOT_SPAM`, to allow the operation. Other callbacks may still 
+    decide to reject it.
+  - `synapse.module_api.errors.Codes` to reject the operation with an error code. In case
+    of doubt, `synapse.module_api.errors.Codes.FORBIDDEN` is a good error code.
+
+  - (deprecated) `False`, which is the same as returning `synapse.module_api.NOT_SPAM`.
+  - (deprecated) `True`, which is the same as returning `synapse.module_api.errors.Codes.FORBIDDEN`.

 If multiple modules implement this callback, they will be considered in order. If a
-callback returns `False`, Synapse falls through to the next one. The value of the first
-callback that does not return `False` will be used. If this happens, Synapse will not call
-any of the subsequent implementations of this callback.
+callback returns `synapse.module_api.NOT_SPAM`, Synapse falls through to the next one.
+The value of the first callback that does not return `synapse.module_api.NOT_SPAM` will
+be used. If this happens, Synapse will not call any of the subsequent implementations of
+this callback.
+

 ### `should_drop_federated_event`

@@ -316,6 +391,9 @@ class ListSpamChecker:
            resource=IsUserEvilResource(config),
        )

-    async def check_event_for_spam(self, event: "synapse.events.EventBase") -> Union[bool, str]:
-        return event.sender not in self.evil_users
+    async def check_event_for_spam(self, event: "synapse.events.EventBase") -> Union[Literal["NOT_SPAM"], Codes]:
+        if event.sender in self.evil_users:
+          return Codes.FORBIDDEN
+        else:
+          return synapse.module_api.NOT_SPAM
 ```
@@ -45,8 +45,8 @@ as follows:
   maintainer.

 To enable the OpenID integration, you should then add a section to the `oidc_providers`
-setting in your configuration file (or uncomment one of the existing examples).
-See [sample_config.yaml](./sample_config.yaml) for some sample settings, as well as
+setting in your configuration file.
+See the [configuration manual](usage/configuration/config_documentation.md#oidc_providers) for some sample settings, as well as
 the text below for example configurations for specific providers.

 ## Sample configs
@@ -57,8 +57,9 @@ https://www.jaegertracing.io/docs/latest/getting-started.
 ## Enable OpenTracing in Synapse

 OpenTracing is not enabled by default. It must be enabled in the
-homeserver config by uncommenting the config options under `opentracing`
-as shown in the [sample config](./sample_config.yaml). For example:
+homeserver config by adding the `opentracing` option to your config file. You can find 
+documentation about how to do this in the [config manual under the header 'Opentracing'](usage/configuration/config_documentation.md#opentracing).
+See below for an example Opentracing configuration: 

 ```yaml
 opentracing:
@@ -66,8 +66,8 @@ in Synapse can be deactivated.

 **NOTE**: This has an impact on security and is for testing purposes only!

-To deactivate the certificate validation, the following setting must be made in
-[homserver.yaml](../usage/configuration/homeserver_sample_config.md).
+To deactivate the certificate validation, the following setting must be added to
+your [homserver.yaml](../usage/configuration/homeserver_sample_config.md).

 ```yaml
 use_insecure_ssl_client_just_for_testing_do_not_use: true
@@ -84,20 +84,19 @@ file when you upgrade the Debian package to a later version.

 ##### Downstream Debian packages

-We do not recommend using the packages from the default Debian `buster`
-repository at this time, as they are old and suffer from known security
-vulnerabilities. You can install the latest version of Synapse from
-[our repository](#matrixorg-packages) or from `buster-backports`. Please
-see the [Debian documentation](https://backports.debian.org/Instructions/)
-for information on how to use backports.
-
-If you are using Debian `sid` or testing, Synapse is available in the default
-repositories and it should be possible to install it simply with:
+Andrej Shadura maintains a `matrix-synapse` package in the Debian repositories.
+For `bookworm` and `sid`, it can be installed simply with:

 ```sh
 sudo apt install matrix-synapse
 ```

+Synapse is also avaliable in `bullseye-backports`.  Please
+see the [Debian documentation](https://backports.debian.org/Instructions/)
+for information on how to use backports.
+
+`matrix-synapse` is no longer maintained for `buster` and older.
+
 ##### Downstream Ubuntu packages

 We do not recommend using the packages in the default Ubuntu repository
@@ -233,7 +232,9 @@ python -m synapse.app.homeserver \
    --report-stats=[yes|no]
 ```

-... substituting an appropriate value for `--server-name`.
+... substituting an appropriate value for `--server-name` and choosing whether
+or not to report usage statistics (hostname, Synapse version, uptime, total
+users, etc.) to the developers via the `--report-stats` argument.

 This command will generate you a config file that you can then customise, but it will
 also generate a set of keys for you. These keys will allow your homeserver to
@@ -406,11 +407,11 @@ The recommended way to do so is to set up a reverse proxy on port
 Alternatively, you can configure Synapse to expose an HTTPS port. To do
 so, you will need to edit `homeserver.yaml`, as follows:

- First, under the `listeners` section, uncomment the configuration for the
-  TLS-enabled listener. (Remove the hash sign (`#`) at the start of
-  each line). The relevant lines are like this:
+- First, under the `listeners` option, add the configuration for the
+  TLS-enabled listener like so:

 ```yaml
+listeners:
  - port: 8448
    type: http
    tls: true
@@ -418,9 +419,11 @@ so, you will need to edit `homeserver.yaml`, as follows:
      - names: [client, federation]
  ```

- You will also need to uncomment the `tls_certificate_path` and
-  `tls_private_key_path` lines under the `TLS` section. You will need to manage
-  provisioning of these certificates yourself.
+- You will also need to add the options `tls_certificate_path` and
+  `tls_private_key_path`. to your configuration file. You will need to manage provisioning of 
+   these certificates yourself.
+- You can find more information about these options as well as how to configure synapse in the 
+  [configuration manual](../usage/configuration/config_documentation.md).

  If you are using your own certificate, be sure to use a `.pem` file that
  includes the full certificate chain including any intermediate certificates
@@ -89,6 +89,65 @@ process, for example:
    dpkg -i matrix-synapse-py3_1.3.0+stretch1_amd64.deb
    ```

+# Upgrading to v1.62.0
+
+## New signatures for spam checker callbacks
+
+As a followup to changes in v1.60.0, the following spam-checker callbacks have changed signature:
+
+- `user_may_join_room`
+- `user_may_invite`
+- `user_may_send_3pid_invite`
+- `user_may_create_room`
+- `user_may_create_room_alias`
+- `user_may_publish_room`
+- `check_media_file_for_spam`
+
+For each of these methods, the previous callback signature has been deprecated.
+
+Whereas callbacks used to return `bool`, they should now return `Union["synapse.module_api.NOT_SPAM", "synapse.module_api.errors.Codes"]`.
+
+For instance, if your module implements `user_may_join_room` as follows:
+
+```python
+async def user_may_join_room(self, user_id: str, room_id: str, is_invited: bool)
+    if ...:
+        # Request is spam
+        return False
+    # Request is not spam
+    return True
+```
+
+you should rewrite it as follows:
+
+```python
+async def user_may_join_room(self, user_id: str, room_id: str, is_invited: bool)
+    if ...:
+        # Request is spam, mark it as forbidden (you may use some more precise error
+        # code if it is useful).
+        return synapse.module_api.errors.Codes.FORBIDDEN
+    # Request is not spam, mark it as such.
+    return synapse.module_api.NOT_SPAM
+```
+
+# Upgrading to v1.61.0
+
+## Removal of deprecated community/groups
+
+This release of Synapse will remove deprecated community/groups from codebase.
+
+### Worker endpoints
+
+For those who have deployed workers, following worker endpoints will no longer
+exist and they can be removed from the reverse proxy configuration:
+
+-   `^/_matrix/federation/v1/get_groups_publicised$`
+-   `^/_matrix/client/(r0|v3|unstable)/joined_groups$`
+-   `^/_matrix/client/(r0|v3|unstable)/publicised_groups$`
+-   `^/_matrix/client/(r0|v3|unstable)/publicised_groups/`
+-   `^/_matrix/federation/v1/groups/`
+-   `^/_matrix/client/(r0|v3|unstable)/groups/`
+
 # Upgrading to v1.60.0

 ## Adding a new unique index to `state_group_edges` could fail if your database is corrupted
@@ -0,0 +1,81 @@
+# Reporting Anonymised Statistics
+
+When generating your Synapse configuration file, you are asked whether you
+would like to report anonymised statistics to Matrix.org. These statistics
+provide the foundation a glimpse into the number of Synapse homeservers
+participating in the network, as well as statistics such as the number of
+rooms being created and messages being sent. This feature is sometimes
+affectionately called "phone-home" stats. Reporting
+[is optional](../../configuration/config_documentation.md#report_stats)
+and the reporting endpoint
+[can be configured](../../configuration/config_documentation.md#report_stats_endpoint),
+in case you would like to instead report statistics from a set of homeservers
+to your own infrastructure.
+
+This documentation aims to define the statistics available and the
+homeserver configuration options that exist to tweak it.
+
+## Available Statistics
+
+The following statistics are sent to the configured reporting endpoint:
+
+| Statistic Name             | Type   | Description                                                                                                                                                                                                                                                                                     |
+|----------------------------|--------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
+| `memory_rss`               | int    | The memory usage of the process (in kilobytes on Unix-based systems, bytes on MacOS).                                                                                                                                                                                                           |
+| `cpu_average`              | int    | CPU time in % of a single core (not % of all cores).                                                                                                                                                                                                                                            |              
+| `homeserver`               | string | The homeserver's server name.                                                                                                                                                                                                                                                                   |
+| `server_context`           | string | An arbitrary string used to group statistics from a set of homeservers.                                                                                                                                                                                                                         |
+| `timestamp`                | int    | The current time, represented as the number of seconds since the epoch.                                                                                                                                                                                                                         |                 
+| `uptime_seconds`           | int    | The number of seconds since the homeserver was last started.                                                                                                                                                                                                                                    |
+| `python_version`           | string | The Python version number in use (e.g "3.7.1"). Taken from `sys.version_info`.                                                                                                                                                                                                                  |
+| `total_users`              | int    | The number of registered users on the homeserver.                                                                                                                                                                                                                                               |
+| `total_nonbridged_users`   | int    | The number of users, excluding those created by an Application Service.                                                                                                                                                                                                                         |
+| `daily_user_type_native`   | int    | The number of native users created in the last 24 hours.                                                                                                                                                                                                                                        |
+| `daily_user_type_guest`    | int    | The number of guest users created in the last 24 hours.                                                                                                                                                                                                                                         |
+| `daily_user_type_bridged`  | int    | The number of users created by Application Services in the last 24 hours.                                                                                                                                                                                                                       |
+| `total_room_count`         | int    | The total number of rooms present on the homeserver.                                                                                                                                                                                                                                            |
+| `daily_active_users`       | int    | The number of unique users[^1] that have used the homeserver in the last 24 hours.                                                                                                                                                                                                              |
+| `monthly_active_users`     | int    | The number of unique users[^1] that have used the homeserver in the last 30 days.                                                                                                                                                                                                               |
+| `daily_active_rooms`       | int    | The number of rooms that have had a (state) event with the type `m.room.message` sent in them in the last 24 hours.                                                                                                                                                                             |
+| `daily_active_e2ee_rooms`  | int    | The number of rooms that have had a (state) event with the type `m.room.encrypted` sent in them in the last 24 hours.                                                                                                                                                                           |
+| `daily_messages`           | int    | The number of (state) events with the type `m.room.message` seen in the last 24 hours.                                                                                                                                                                                                          |
+| `daily_e2ee_messages`      | int    | The number of (state) events with the type `m.room.encrypted` seen in the last 24 hours.                                                                                                                                                                                                        |
+| `daily_sent_messages`      | int    | The number of (state) events sent by a local user with the type `m.room.message` seen in the last 24 hours.                                                                                                                                                                                     |
+| `daily_sent_e2ee_messages` | int    | The number of (state) events sent by a local user with the type `m.room.encrypted` seen in the last 24 hours.                                                                                                                                                                                   |
+| `r30_users_all`            | int    | The number of 30 day retained users, defined as users who have created their accounts more than 30 days ago, where they were last seen at most 30 days ago and where those two timestamps are over 30 days apart. Includes clients that do not fit into the below r30 client types.             |
+| `r30_users_android`        | int    | The number of 30 day retained users, as defined above. Filtered only to clients with "Android" in the user agent string.                                                                                                                                                                        |
+| `r30_users_ios`            | int    | The number of 30 day retained users, as defined above. Filtered only to clients with "iOS" in the user agent string.                                                                                                                                                                            |
+| `r30_users_electron`       | int    | The number of 30 day retained users, as defined above. Filtered only to clients with "Electron" in the user agent string.                                                                                                                                                                       |
+| `r30_users_web`            | int    | The number of 30 day retained users, as defined above. Filtered only to clients with "Mozilla" or "Gecko" in the user agent string.                                                                                                                                                             |
+| `r30v2_users_all`          | int    | The number of 30 day retained users, with a revised algorithm. Defined as users that appear more than once in the past 60 days, and have more than 30 days between the most and least recent appearances in the past 60 days. Includes clients that do not fit into the below r30 client types. |
+| `r30v2_users_android`      | int    | The number of 30 day retained users, as defined above. Filtered only to clients with ("riot" or "element") and "android" (case-insensitive) in the user agent string.                                                                                                                           |
+| `r30v2_users_ios`          | int    | The number of 30 day retained users, as defined above. Filtered only to clients with ("riot" or "element") and "ios" (case-insensitive) in the user agent string.                                                                                                                               |
+| `r30v2_users_electron`     | int    | The number of 30 day retained users, as defined above. Filtered only to clients with ("riot" or "element") and "electron" (case-insensitive) in the user agent string.                                                                                                                          |
+| `r30v2_users_web`          | int    | The number of 30 day retained users, as defined above. Filtered only to clients with "mozilla" or "gecko" (case-insensitive) in the user agent string.                                                                                                                                          |
+| `cache_factor`             | int    | The configured [`global factor`](../../configuration/config_documentation.md#caching) value for caching.                                                                                                                                                                                        |
+| `event_cache_size`         | int    | The configured [`event_cache_size`](../../configuration/config_documentation.md#caching) value for caching.                                                                                                                                                                                     |
+| `database_engine`          | string | The database engine that is in use. Either "psycopg2" meaning PostgreSQL is in use, or "sqlite3" for SQLite3.                                                                                                                                                                                   |
+| `database_server_version` | string | The version of the database server. Examples being "10.10" for PostgreSQL server version 10.0, and "3.38.5" for SQLite 3.38.5 installed on the system.                                                                                                                                          |
+| `log_level` | string | The log level in use. Examples are "INFO", "WARNING", "ERROR", "DEBUG", etc.                                                                                                                                                                                                                    |
+
+
+[^1]: Native matrix users and guests are always counted. If the
+[`track_puppeted_user_ips`](../../configuration/config_documentation.md#track_puppeted_user_ips)
+option is set to `true`, "puppeted" users (users that an Application Service have performed
+[an action on behalf of](https://spec.matrix.org/v1.3/application-service-api/#identity-assertion))
+will also be counted. Note that an Application Service can "puppet" any user in their
+[user namespace](https://spec.matrix.org/v1.3/application-service-api/#registration),
+not only users that the Application Service has created. If this happens, the Application Service
+will additionally be counted as a user (irrespective of `track_puppeted_user_ips`).
+
+## Using a Custom Statistics Collection Server
+
+If statistics reporting is enabled, the endpoint that Synapse sends metrics to is configured by the
+[`report_stats_endpoint`](../../configuration/config_documentation.md#report_stats_endpoint) config
+option. By default, statistics are sent to Matrix.org.
+
+If you would like to set up your own statistics collection server and send metrics there, you may
+consider using one of the following known implementations:
+
+* [Matrix.org's Panopticon](https://github.com/matrix-org/panopticon)
+* [Famedly's Barad-dûr](https://gitlab.com/famedly/company/devops/services/barad-dur)
@@ -9,6 +9,9 @@ a real homeserver.yaml. Instead, if you are starting from scratch, please genera
 a fresh config using Synapse by following the instructions in
 [Installation](../../setup/installation.md).

+Documentation for all configuration options can be found in the
+[Configuration Manual](./config_documentation.md).
+
 ```yaml
 {{#include ../../sample_config.yaml}}
 ```
@@ -4,5 +4,5 @@ Synapse supports authenticating users via the [Central Authentication
 Service protocol](https://en.wikipedia.org/wiki/Central_Authentication_Service)
 (CAS) natively.

-Please see the `cas_config` and `sso` sections of the [Synapse configuration
-file](../../../configuration/homeserver_sample_config.md) for more details.
+Please see the [cas_config](../../../configuration/config_documentation.md#cas_config) and [sso](../../../configuration/config_documentation.md#sso)
+sections of the configuration manual for more details.
@@ -27,7 +27,6 @@ exclude = (?x)
  ^(
   |synapse/storage/databases/__init__.py
   |synapse/storage/databases/main/cache.py
-   |synapse/storage/databases/main/devices.py
   |synapse/storage/schema/

   |tests/api/test_auth.py
@@ -57,7 +56,6 @@ exclude = (?x)
   |tests/server.py
   |tests/server_notices/test_resource_limits_server_notices.py
   |tests/test_metrics.py
-   |tests/test_server.py
   |tests/test_state.py
   |tests/test_terms_auth.py
   |tests/util/caches/test_cached_call.py
@@ -90,9 +88,6 @@ disallow_untyped_defs = False
 [mypy-synapse.logging.opentracing]
 disallow_untyped_defs = False

-[mypy-synapse.logging.scopecontextmanager]
-disallow_untyped_defs = False
-
 [mypy-synapse.metrics._reactor_metrics]
 disallow_untyped_defs = False
 # This module imports select.epoll. That exists on Linux, but doesn't on macOS.
@@ -114,6 +109,9 @@ disallow_untyped_defs = False
 [mypy-tests.handlers.test_user_directory]
 disallow_untyped_defs = True

+[mypy-tests.test_server]
+disallow_untyped_defs = True
+
 [mypy-tests.state.test_profile]
 disallow_untyped_defs = True

@@ -139,7 +139,7 @@ unicode_backport = ["unicodedata2"]

 [[package]]
 name = "click"
-version = "8.1.0"
+version = "8.1.1"
 description = "Composable command line interface toolkit"
 category = "dev"
 optional = false
@@ -815,7 +815,7 @@ python-versions = ">=3.5"
 name = "pyjwt"
 version = "2.4.0"
 description = "JSON Web Token implementation in Python"
-category = "main"
+category = "dev"
 optional = false
 python-versions = ">=3.6"

@@ -1546,9 +1546,9 @@ docs = ["sphinx", "repoze.sphinx.autointerface"]
 test = ["zope.i18nmessageid", "zope.testing", "zope.testrunner"]

 [extras]
-all = ["matrix-synapse-ldap3", "psycopg2", "psycopg2cffi", "psycopg2cffi-compat", "pysaml2", "authlib", "lxml", "sentry-sdk", "jaeger-client", "opentracing", "pyjwt", "txredisapi", "hiredis", "Pympler"]
+all = ["matrix-synapse-ldap3", "psycopg2", "psycopg2cffi", "psycopg2cffi-compat", "pysaml2", "authlib", "lxml", "sentry-sdk", "jaeger-client", "opentracing", "txredisapi", "hiredis", "Pympler"]
 cache_memory = ["Pympler"]
-jwt = ["pyjwt"]
+jwt = ["authlib"]
 matrix-synapse-ldap3 = ["matrix-synapse-ldap3"]
 oidc = ["authlib"]
 opentracing = ["jaeger-client", "opentracing"]
@@ -1563,7 +1563,7 @@ url_preview = ["lxml"]
 [metadata]
 lock-version = "1.1"
 python-versions = "^3.7.1"
-content-hash = "c1bb4dabba1e87517e25ca7bf778e8082fbc960a51d83819aec3a154110a374f"
+content-hash = "e96625923122e29b6ea5964379828e321b6cede2b020fc32c6f86c09d86d1ae8"

 [metadata.files]
 attrs = [
@@ -1684,8 +1684,8 @@ charset-normalizer = [
    {file = "charset_normalizer-2.0.12-py3-none-any.whl", hash = "sha256:6881edbebdb17b39b4eaaa821b438bf6eddffb4468cf344f09f89def34a8b1df"},
 ]
 click = [
-    {file = "click-8.1.0-py3-none-any.whl", hash = "sha256:19a4baa64da924c5e0cd889aba8e947f280309f1a2ce0947a3e3a7bcb7cc72d6"},
-    {file = "click-8.1.0.tar.gz", hash = "sha256:977c213473c7665d3aa092b41ff12063227751c41d7b17165013e10069cc5cd2"},
+    {file = "click-8.1.1-py3-none-any.whl", hash = "sha256:5e0d195c2067da3136efb897449ec1e9e6c98282fbf30d7f9e164af9be901a6b"},
+    {file = "click-8.1.1.tar.gz", hash = "sha256:7ab900e38149c9872376e8f9b5986ddcaf68c0f413cf73678a0bca5547e6f976"},
 ]
 click-default-group = [
    {file = "click-default-group-1.2.2.tar.gz", hash = "sha256:d9560e8e8dfa44b3562fbc9425042a0fd6d21956fcc2db0077f63f34253ab904"},
@@ -54,7 +54,7 @@ skip_gitignore = true

 [tool.poetry]
 name = "matrix-synapse"
-version = "1.61.0rc1"
+version = "1.62.0rc2"
 description = "Homeserver for the Matrix decentralised comms protocol"
 authors = ["Matrix.org Team and Contributors <packages@matrix.org>"]
 license = "Apache-2.0"
@@ -110,9 +110,9 @@ jsonschema = ">=3.0.0"
 frozendict = ">=1,!=2.1.2"
 # We require 2.1.0 or higher for type hints. Previous guard was >= 1.1.0
 unpaddedbase64 = ">=2.1.0"
-canonicaljson = ">=1.4.0"
+canonicaljson = "^1.4.0"
 # we use the type definitions added in signedjson 1.1.
-signedjson = ">=1.1.0"
+signedjson = "^1.1.0"
 # validating SSL certs for IP addresses requires service_identity 18.1.
 service-identity = ">=18.1.0"
 # Twisted 18.9 introduces some logger improvements that the structured
@@ -150,7 +150,7 @@ typing-extensions = ">=3.10.0.1"
 cryptography = ">=3.4.7"
 # ijson 3.1.4 fixes a bug with "." in property names
 ijson = ">=3.1.4"
-matrix-common = "~=1.2.1"
+matrix-common = "^1.2.1"
 # We need packaging.requirements.Requirement, added in 16.1.
 packaging = ">=16.1"
 # At the time of writing, we only use functions from the version `importlib.metadata`
@@ -175,7 +175,6 @@ lxml = { version = ">=4.2.0", optional = true }
 sentry-sdk = { version = ">=0.7.2", optional = true }
 opentracing = { version = ">=2.2.0", optional = true }
 jaeger-client = { version = ">=4.0.0", optional = true }
-pyjwt = { version = ">=1.6.4", optional = true }
 txredisapi = { version = ">=1.4.7", optional = true }
 hiredis = { version = "*", optional = true }
 Pympler = { version = "*", optional = true }
@@ -196,7 +195,7 @@ systemd = ["systemd-python"]
 url_preview = ["lxml"]
 sentry = ["sentry-sdk"]
 opentracing = ["jaeger-client", "opentracing"]
-jwt = ["pyjwt"]
+jwt = ["authlib"]
 # hiredis is not a *strict* dependency, but it makes things much faster.
 # (if it is not installed, we fall back to slow code.)
 redis = ["txredisapi", "hiredis"]
@@ -222,7 +221,7 @@ all = [
    "psycopg2", "psycopg2cffi", "psycopg2cffi-compat",
    # saml2
    "pysaml2",
-    # oidc
+    # oidc and jwt
    "authlib",
    # url_preview
    "lxml",
@@ -230,8 +229,6 @@ all = [
    "sentry-sdk",
    # opentracing
    "jaeger-client", "opentracing",
-    # jwt
-    "pyjwt",
    # redis
    "txredisapi", "hiredis",
    # cache_memory
@@ -272,7 +269,7 @@ parameterized = ">=0.7.4"
 idna = ">=2.5"

 # The following are used by the release script
-click = "==8.1.0"
+click = "==8.1.1"
 # GitPython was == 3.1.14; bumped to 3.1.20, the first release with type hints.
 GitPython = ">=3.1.20"
 commonmark = "==0.9.1"
@@ -0,0 +1,111 @@
+#!/usr/bin/env python3
+
+# Check that no schema deltas have been added to the wrong version.
+
+import re
+from typing import Any, Dict, List
+
+import click
+import git
+
+SCHEMA_FILE_REGEX = re.compile(r"^synapse/storage/schema/(.*)/delta/(.*)/(.*)$")
+
+
+@click.command()
+@click.option(
+    "--force-colors",
+    is_flag=True,
+    flag_value=True,
+    default=None,
+    help="Always output ANSI colours",
+)
+def main(force_colors: bool) -> None:
+    click.secho(
+        "+++ Checking schema deltas are in the right folder",
+        fg="green",
+        bold=True,
+        color=force_colors,
+    )
+
+    click.secho("Updating repo...")
+
+    repo = git.Repo()
+    repo.remote().fetch()
+
+    click.secho("Getting current schema version...")
+
+    r = repo.git.show("origin/develop:synapse/storage/schema/__init__.py")
+
+    locals: Dict[str, Any] = {}
+    exec(r, locals)
+    current_schema_version = locals["SCHEMA_VERSION"]
+
+    click.secho(f"Current schema version: {current_schema_version}")
+
+    diffs: List[git.Diff] = repo.remote().refs.develop.commit.diff(None)
+
+    seen_deltas = False
+    bad_files = []
+    for diff in diffs:
+        if not diff.new_file or diff.b_path is None:
+            continue
+
+        match = SCHEMA_FILE_REGEX.match(diff.b_path)
+        if not match:
+            continue
+
+        seen_deltas = True
+
+        _, delta_version, _ = match.groups()
+
+        if delta_version != str(current_schema_version):
+            bad_files.append(diff.b_path)
+
+    if not seen_deltas:
+        click.secho(
+            "No deltas found.",
+            fg="green",
+            bold=True,
+            color=force_colors,
+        )
+        return
+
+    if not bad_files:
+        click.secho(
+            f"All deltas are in the correct folder: {current_schema_version}!",
+            fg="green",
+            bold=True,
+            color=force_colors,
+        )
+        return
+
+    bad_files.sort()
+
+    click.secho(
+        "Found deltas in the wrong folder!",
+        fg="red",
+        bold=True,
+        color=force_colors,
+    )
+
+    for f in bad_files:
+        click.secho(
+            f"\t{f}",
+            fg="red",
+            bold=True,
+            color=force_colors,
+        )
+
+    click.secho()
+    click.secho(
+        f"Please move these files to delta/{current_schema_version}/",
+        fg="red",
+        bold=True,
+        color=force_colors,
+    )
+
+    click.get_current_context().exit(1)
+
+
+if __name__ == "__main__":
+    main()
@@ -14,16 +14,70 @@
 # By default Synapse is run in monolith mode. This can be overridden by
 # setting the WORKERS environment variable.
 #
-# A regular expression of test method names can be supplied as the first
-# argument to the script. Complement will then only run those tests. If
-# no regex is supplied, all tests are run. For example;
+# You can optionally give a "-f" argument (for "fast") before any to skip
+# rebuilding the docker images, if you just want to rerun the tests.
+#
+# Remaining commandline arguments are passed through to `go test`. For example,
+# you can supply a regular expression of test method names via the "-run"
+# argument:
 #
 # ./complement.sh -run "TestOutboundFederation(Profile|Send)"
 #
+# Specifying TEST_ONLY_SKIP_DEP_HASH_VERIFICATION=1 will cause `poetry export`
+# to not emit any hashes when building the Docker image. This then means that
+# you can use 'unverifiable' sources such as git repositories as dependencies.

 # Exit if a line returns a non-zero exit code
 set -e

+
+# Helper to emit annotations that collapse portions of the log in GitHub Actions
+echo_if_github() {
+  if [[ -n "$GITHUB_WORKFLOW" ]]; then
+    echo $*
+  fi
+}
+
+# Helper to print out the usage instructions
+usage() {
+    cat >&2 <<EOF
+Usage: $0 [-f] <go test arguments>...
+Run the complement test suite on Synapse.
+
+  -f, --fast
+        Skip rebuilding the docker images, and just use the most recent
+        'complement-synapse:latest' image
+
+  --build-only
+        Only build the Docker images. Don't actually run Complement.
+
+For help on arguments to 'go test', run 'go help testflag'.
+EOF
+}
+
+# parse our arguments
+skip_docker_build=""
+skip_complement_run=""
+while [ $# -ge 1 ]; do
+    arg=$1
+    case "$arg" in
+        "-h")
+            usage
+            exit 1
+            ;;
+        "-f"|"--fast")
+            skip_docker_build=1
+            ;;
+        "--build-only")
+            skip_complement_run=1
+            ;;
+        *)
+            # unknown arg: presumably an argument to gotest. break the loop.
+            break
+    esac
+    shift
+done
+
 # enable buildkit for the docker builds
 export DOCKER_BUILDKIT=1

@@ -40,15 +94,30 @@ if [[ -z "$COMPLEMENT_DIR" ]]; then
  echo "Checkout available at 'complement-${COMPLEMENT_REF}'"
 fi

-# Build the base Synapse image from the local checkout
-docker build -t matrixdotorg/synapse -f "docker/Dockerfile" .
+if [ -z "$skip_docker_build" ]; then
+    # Build the base Synapse image from the local checkout
+    echo_if_github "::group::Build Docker image: matrixdotorg/synapse"
+    docker build -t matrixdotorg/synapse \
+      --build-arg TEST_ONLY_SKIP_DEP_HASH_VERIFICATION \
+      -f "docker/Dockerfile" .
+    echo_if_github "::endgroup::"

-# Build the workers docker image (from the base Synapse image we just built).
-docker build -t matrixdotorg/synapse-workers -f "docker/Dockerfile-workers" .
+    # Build the workers docker image (from the base Synapse image we just built).
+    echo_if_github "::group::Build Docker image: matrixdotorg/synapse-workers"
+    docker build -t matrixdotorg/synapse-workers -f "docker/Dockerfile-workers" .
+    echo_if_github "::endgroup::"

-# Build the unified Complement image (from the worker Synapse image we just built).
-docker build -t complement-synapse \
-  -f "docker/complement/Dockerfile" "docker/complement"
+    # Build the unified Complement image (from the worker Synapse image we just built).
+    echo_if_github "::group::Build Docker image: complement/Dockerfile"
+    docker build -t complement-synapse \
+           -f "docker/complement/Dockerfile" "docker/complement"
+    echo_if_github "::endgroup::"
+fi
+
+if [ -n "$skip_complement_run" ]; then
+    echo "Skipping Complement run as requested."
+    exit
+fi

 export COMPLEMENT_BASE_IMAGE=complement-synapse

@@ -60,6 +129,9 @@ test_tags="synapse_blacklist,msc2716,msc3030,msc3787"
 # (The prefix is stripped off before reaching the container.)
 export COMPLEMENT_SHARE_ENV_PREFIX=PASS_

+# It takes longer than 10m to run the whole suite.
+extra_test_args+=("-timeout=60m")
+
 if [[ -n "$WORKERS" ]]; then
  # Use workers.
  export PASS_SYNAPSE_COMPLEMENT_USE_WORKERS=true
@@ -73,9 +145,6 @@ if [[ -n "$WORKERS" ]]; then
  # time (the main problem is that we start 14 python processes for each test,
  # and complement likes to do two of them in parallel).
  export COMPLEMENT_SPAWN_HS_TIMEOUT_SECS=120
-
-  # ... and it takes longer than 10m to run the whole suite.
-  extra_test_args+=("-timeout=60m")
 else
  export PASS_SYNAPSE_COMPLEMENT_USE_WORKERS=
  if [[ -n "$POSTGRES" ]]; then
@@ -58,10 +58,10 @@ from synapse.storage.databases.main.client_ips import ClientIpBackgroundUpdateSt
 from synapse.storage.databases.main.deviceinbox import DeviceInboxBackgroundUpdateStore
 from synapse.storage.databases.main.devices import DeviceBackgroundUpdateStore
 from synapse.storage.databases.main.end_to_end_keys import EndToEndKeyBackgroundStore
+from synapse.storage.databases.main.event_push_actions import EventPushActionsStore
 from synapse.storage.databases.main.events_bg_updates import (
    EventsBackgroundUpdatesStore,
 )
-from synapse.storage.databases.main.group_server import GroupServerStore
 from synapse.storage.databases.main.media_repository import (
    MediaRepositoryBackgroundUpdateStore,
 )
@@ -200,6 +200,7 @@ R = TypeVar("R")


 class Store(
+    EventPushActionsStore,
    ClientIpBackgroundUpdateStore,
    DeviceInboxBackgroundUpdateStore,
    DeviceBackgroundUpdateStore,
@@ -218,7 +219,6 @@ class Store(
    PushRuleStore,
    PusherWorkerStore,
    PresenceBackgroundUpdateStore,
-    GroupServerStore,
 ):
    def execute(self, f: Callable[..., R], *args: Any, **kwargs: Any) -> Awaitable[R]:
        return self.db_pool.runInteraction(f.__name__, f, *args, **kwargs)
@@ -268,6 +268,9 @@ class MockHomeserver:
    def get_instance_name(self) -> str:
        return "master"

+    def should_send_federation(self) -> bool:
+        return False
+

 class Porter:
    def __init__(
@@ -20,7 +20,6 @@ from netaddr import IPAddress
 from twisted.web.server import Request

 from synapse import event_auth
-from synapse.api.auth_blocking import AuthBlocking
 from synapse.api.constants import EventTypes, HistoryVisibility, Membership
 from synapse.api.errors import (
    AuthError,
@@ -34,8 +33,6 @@ from synapse.http.site import SynapseRequest
 from synapse.logging.opentracing import active_span, force_tracing, start_active_span
 from synapse.storage.databases.main.registration import TokenLookupResult
 from synapse.types import Requester, UserID, create_requester
-from synapse.util.caches.lrucache import LruCache
-from synapse.util.macaroons import get_value_from_macaroon, satisfy_expiry

 if TYPE_CHECKING:
    from synapse.server import HomeServer
@@ -47,10 +44,6 @@ logger = logging.getLogger(__name__)
 GUEST_DEVICE_ID = "guest_device"


-class _InvalidMacaroonException(Exception):
-    pass
-
-
 class Auth:
    """
    This class contains functions for authenticating users of our client-server API.
@@ -62,16 +55,10 @@ class Auth:
        self.store = hs.get_datastores().main
        self._account_validity_handler = hs.get_account_validity_handler()
        self._storage_controllers = hs.get_storage_controllers()
-
-        self.token_cache: LruCache[str, Tuple[str, bool]] = LruCache(
-            10000, "token_cache"
-        )
-
-        self._auth_blocking = AuthBlocking(self.hs)
+        self._macaroon_generator = hs.get_macaroon_generator()

        self._track_appservice_user_ips = hs.config.appservice.track_appservice_user_ips
        self._track_puppeted_user_ips = hs.config.api.track_puppeted_user_ips
-        self._macaroon_secret_key = hs.config.key.macaroon_secret_key
        self._force_tracing_for_users = hs.config.tracing.force_tracing_for_users

    async def check_user_in_room(
@@ -126,7 +113,6 @@ class Auth:
        self,
        request: SynapseRequest,
        allow_guest: bool = False,
-        rights: str = "access",
        allow_expired: bool = False,
    ) -> Requester:
        """Get a registered user's ID.
@@ -135,7 +121,6 @@ class Auth:
            request: An HTTP request with an access_token query parameter.
            allow_guest: If False, will raise an AuthError if the user making the
                request is a guest.
-            rights: The operation being performed; the access token must allow this
            allow_expired: If True, allow the request through even if the account
                is expired, or session token lifetime has ended. Note that
                /login will deliver access tokens regardless of expiration.
@@ -150,7 +135,7 @@ class Auth:
        parent_span = active_span()
        with start_active_span("get_user_by_req"):
            requester = await self._wrapped_get_user_by_req(
-                request, allow_guest, rights, allow_expired
+                request, allow_guest, allow_expired
            )

            if parent_span:
@@ -176,7 +161,6 @@ class Auth:
        self,
        request: SynapseRequest,
        allow_guest: bool,
-        rights: str,
        allow_expired: bool,
    ) -> Requester:
        """Helper for get_user_by_req
@@ -214,7 +198,7 @@ class Auth:
                return requester

            user_info = await self.get_user_by_access_token(
-                access_token, rights, allow_expired=allow_expired
+                access_token, allow_expired=allow_expired
            )
            token_id = user_info.token_id
            is_guest = user_info.is_guest
@@ -394,15 +378,12 @@ class Auth:
    async def get_user_by_access_token(
        self,
        token: str,
-        rights: str = "access",
        allow_expired: bool = False,
    ) -> TokenLookupResult:
        """Validate access token and get user_id from it

        Args:
            token: The access token to get the user by
-            rights: The operation being performed; the access token must
-                allow this
            allow_expired: If False, raises an InvalidClientTokenError
                if the token is expired

@@ -413,70 +394,55 @@ class Auth:
                is invalid
        """

-        if rights == "access":
-            # First look in the database to see if the access token is present
-            # as an opaque token.
-            r = await self.store.get_user_by_access_token(token)
-            if r:
-                valid_until_ms = r.valid_until_ms
-                if (
-                    not allow_expired
-                    and valid_until_ms is not None
-                    and valid_until_ms < self.clock.time_msec()
-                ):
-                    # there was a valid access token, but it has expired.
-                    # soft-logout the user.
-                    raise InvalidClientTokenError(
-                        msg="Access token has expired", soft_logout=True
-                    )
+        # First look in the database to see if the access token is present
+        # as an opaque token.
+        r = await self.store.get_user_by_access_token(token)
+        if r:
+            valid_until_ms = r.valid_until_ms
+            if (
+                not allow_expired
+                and valid_until_ms is not None
+                and valid_until_ms < self.clock.time_msec()
+            ):
+                # there was a valid access token, but it has expired.
+                # soft-logout the user.
+                raise InvalidClientTokenError(
+                    msg="Access token has expired", soft_logout=True
+                )

-                return r
+            return r

        # If the token isn't found in the database, then it could still be a
-        # macaroon, so we check that here.
+        # macaroon for a guest, so we check that here.
        try:
-            user_id, guest = self._parse_and_validate_macaroon(token, rights)
+            user_id = self._macaroon_generator.verify_guest_token(token)

-            if rights == "access":
-                if not guest:
-                    # non-guest access tokens must be in the database
-                    logger.warning("Unrecognised access token - not in store.")
-                    raise InvalidClientTokenError()
-
-                # Guest access tokens are not stored in the database (there can
-                # only be one access token per guest, anyway).
-                #
-                # In order to prevent guest access tokens being used as regular
-                # user access tokens (and hence getting around the invalidation
-                # process), we look up the user id and check that it is indeed
-                # a guest user.
-                #
-                # It would of course be much easier to store guest access
-                # tokens in the database as well, but that would break existing
-                # guest tokens.
-                stored_user = await self.store.get_user_by_id(user_id)
-                if not stored_user:
-                    raise InvalidClientTokenError("Unknown user_id %s" % user_id)
-                if not stored_user["is_guest"]:
-                    raise InvalidClientTokenError(
-                        "Guest access token used for regular user"
-                    )
-
-                ret = TokenLookupResult(
-                    user_id=user_id,
-                    is_guest=True,
-                    # all guests get the same device id
-                    device_id=GUEST_DEVICE_ID,
+            # Guest access tokens are not stored in the database (there can
+            # only be one access token per guest, anyway).
+            #
+            # In order to prevent guest access tokens being used as regular
+            # user access tokens (and hence getting around the invalidation
+            # process), we look up the user id and check that it is indeed
+            # a guest user.
+            #
+            # It would of course be much easier to store guest access
+            # tokens in the database as well, but that would break existing
+            # guest tokens.
+            stored_user = await self.store.get_user_by_id(user_id)
+            if not stored_user:
+                raise InvalidClientTokenError("Unknown user_id %s" % user_id)
+            if not stored_user["is_guest"]:
+                raise InvalidClientTokenError(
+                    "Guest access token used for regular user"
                )
-            elif rights == "delete_pusher":
-                # We don't store these tokens in the database

-                ret = TokenLookupResult(user_id=user_id, is_guest=False)
-            else:
-                raise RuntimeError("Unknown rights setting %s", rights)
-            return ret
+            return TokenLookupResult(
+                user_id=user_id,
+                is_guest=True,
+                # all guests get the same device id
+                device_id=GUEST_DEVICE_ID,
+            )
        except (
-            _InvalidMacaroonException,
            pymacaroons.exceptions.MacaroonException,
            TypeError,
            ValueError,
@@ -488,78 +454,6 @@ class Auth:
            )
            raise InvalidClientTokenError("Invalid access token passed.")

-    def _parse_and_validate_macaroon(
-        self, token: str, rights: str = "access"
-    ) -> Tuple[str, bool]:
-        """Takes a macaroon and tries to parse and validate it. This is cached
-        if and only if rights == access and there isn't an expiry.
-
-        On invalid macaroon raises _InvalidMacaroonException
-
-        Returns:
-            (user_id, is_guest)
-        """
-        if rights == "access":
-            cached = self.token_cache.get(token, None)
-            if cached:
-                return cached
-
-        try:
-            macaroon = pymacaroons.Macaroon.deserialize(token)
-        except Exception:  # deserialize can throw more-or-less anything
-            # The access token doesn't look like a macaroon.
-            raise _InvalidMacaroonException()
-
-        try:
-            user_id = get_value_from_macaroon(macaroon, "user_id")
-
-            guest = False
-            for caveat in macaroon.caveats:
-                if caveat.caveat_id == "guest = true":
-                    guest = True
-
-            self.validate_macaroon(macaroon, rights, user_id=user_id)
-        except (
-            pymacaroons.exceptions.MacaroonException,
-            KeyError,
-            TypeError,
-            ValueError,
-        ):
-            raise InvalidClientTokenError("Invalid macaroon passed.")
-
-        if rights == "access":
-            self.token_cache[token] = (user_id, guest)
-
-        return user_id, guest
-
-    def validate_macaroon(
-        self, macaroon: pymacaroons.Macaroon, type_string: str, user_id: str
-    ) -> None:
-        """
-        validate that a Macaroon is understood by and was signed by this server.
-
-        Args:
-            macaroon: The macaroon to validate
-            type_string: The kind of token required (e.g. "access", "delete_pusher")
-            user_id: The user_id required
-        """
-        v = pymacaroons.Verifier()
-
-        # the verifier runs a test for every caveat on the macaroon, to check
-        # that it is met for the current request. Each caveat must match at
-        # least one of the predicates specified by satisfy_exact or
-        # specify_general.
-        v.satisfy_exact("gen = 1")
-        v.satisfy_exact("type = " + type_string)
-        v.satisfy_exact("user_id = %s" % user_id)
-        v.satisfy_exact("guest = true")
-        satisfy_expiry(v, self.clock.time_msec)
-
-        # access_tokens include a nonce for uniqueness: any value is acceptable
-        v.satisfy_general(lambda c: c.startswith("nonce = "))
-
-        v.verify(macaroon, self._macaroon_secret_key)
-
    def get_appservice_by_req(self, request: SynapseRequest) -> ApplicationService:
        token = self.get_access_token_from_request(request)
        service = self.store.get_app_service_by_token(token)
@@ -711,14 +605,3 @@ class Auth:
                "User %s not in room %s, and room previews are disabled"
                % (user_id, room_id),
            )
-
-    async def check_auth_blocking(
-        self,
-        user_id: Optional[str] = None,
-        threepid: Optional[dict] = None,
-        user_type: Optional[str] = None,
-        requester: Optional[Requester] = None,
-    ) -> None:
-        await self._auth_blocking.check_auth_blocking(
-            user_id=user_id, threepid=threepid, user_type=user_type, requester=requester
-        )
@@ -259,3 +259,13 @@ class ReceiptTypes:
    READ: Final = "m.read"
    READ_PRIVATE: Final = "org.matrix.msc2285.read.private"
    FULLY_READ: Final = "m.fully_read"
+
+
+class PublicRoomsFilterFields:
+    """Fields in the search filter for `/publicRooms` that we understand.
+
+    As defined in https://spec.matrix.org/v1.3/client-server-api/#post_matrixclientv3publicrooms
+    """
+
+    GENERIC_SEARCH_TERM: Final = "generic_search_term"
+    ROOM_TYPES: Final = "org.matrix.msc3827.room_types"
@@ -128,6 +128,9 @@ class Ratelimiter:
        performed_count = action_count - time_delta * rate_hz
        if performed_count < 0:
            performed_count = 0
+
+            # Reset the start time and forgive all actions
+            action_count = 0
            time_start = time_now_s

        # This check would be easier read as performed_count + n_actions > burst_count,
@@ -140,7 +143,7 @@ class Ratelimiter:
        else:
            # We haven't reached our limit yet
            allowed = True
-            action_count = performed_count + n_actions
+            action_count = action_count + n_actions

        if update:
            self.actions[key] = (action_count, time_start, rate_hz)
@@ -106,7 +106,9 @@ def register_sighup(func: Callable[P, None], *args: P.args, **kwargs: P.kwargs)
 def start_worker_reactor(
    appname: str,
    config: HomeServerConfig,
-    run_command: Callable[[], None] = reactor.run,
+    # Use a lambda to avoid binding to a given reactor at import time.
+    # (needed when synapse.app.complement_fork_starter is being used)
+    run_command: Callable[[], None] = lambda: reactor.run(),
 ) -> None:
    """Run the reactor in the main process

@@ -141,7 +143,9 @@ def start_reactor(
    daemonize: bool,
    print_pidfile: bool,
    logger: logging.Logger,
-    run_command: Callable[[], None] = reactor.run,
+    # Use a lambda to avoid binding to a given reactor at import time.
+    # (needed when synapse.app.complement_fork_starter is being used)
+    run_command: Callable[[], None] = lambda: reactor.run(),
 ) -> None:
    """Run the reactor in the main process

@@ -450,7 +454,7 @@ async def start(hs: "HomeServer") -> None:
    # before we start the listeners.
    module_api = hs.get_module_api()
    for module, config in hs.config.modules.loaded_modules:
-        m = module(config=config, api=module_api)
+        m = module(config, module_api)
        logger.info("Loaded module %s", m)

    load_legacy_spam_checkers(hs)
@@ -17,6 +17,11 @@ import sys
 from synapse.app.generic_worker import start
 from synapse.util.logcontext import LoggingContext

-if __name__ == "__main__":
+
+def main() -> None:
    with LoggingContext("main"):
        start(sys.argv[1:])
+
+
+if __name__ == "__main__":
+    main()
@@ -17,6 +17,11 @@ import sys
 from synapse.app.generic_worker import start
 from synapse.util.logcontext import LoggingContext

-if __name__ == "__main__":
+
+def main() -> None:
    with LoggingContext("main"):
        start(sys.argv[1:])
+
+
+if __name__ == "__main__":
+    main()
@@ -0,0 +1,190 @@
+# Copyright 2022 The Matrix.org Foundation C.I.C.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# ## What this script does
+#
+# This script spawns multiple workers, whilst only going through the code loading
+# process once. The net effect is that start-up time for a swarm of workers is
+# reduced, particularly in CPU-constrained environments.
+#
+# Before the workers are spawned, the database is prepared in order to avoid the
+# workers racing.
+#
+# ## Stability
+#
+# This script is only intended for use within the Synapse images for the
+# Complement test suite.
+# There are currently no stability guarantees whatsoever; especially not about:
+# - whether it will continue to exist in future versions;
+# - the format of its command-line arguments; or
+# - any details about its behaviour or principles of operation.
+#
+# ## Usage
+#
+# The first argument should be the path to the database configuration, used to
+# set up the database. The rest of the arguments are used as follows:
+# Each worker is specified as an argument group (each argument group is
+# separated by '--').
+# The first argument in each argument group is the Python module name of the application
+# to start. Further arguments are then passed to that module as-is.
+#
+# ## Example
+#
+#   python -m synapse.app.complement_fork_starter path_to_db_config.yaml \
+#     synapse.app.homeserver [args..] -- \
+#     synapse.app.generic_worker [args..] -- \
+#   ...
+#     synapse.app.generic_worker [args..]
+#
+import argparse
+import importlib
+import itertools
+import multiprocessing
+import sys
+from typing import Any, Callable, List
+
+from twisted.internet.main import installReactor
+
+
+class ProxiedReactor:
+    """
+    Twisted tracks the 'installed' reactor as a global variable.
+    (Actually, it does some module trickery, but the effect is similar.)
+
+    The default EpollReactor is buggy if it's created before a process is
+    forked, then used in the child.
+    See https://twistedmatrix.com/trac/ticket/4759#comment:17.
+
+    However, importing certain Twisted modules will automatically create and
+    install a reactor if one hasn't already been installed.
+    It's not normally possible to re-install a reactor.
+
+    Given the goal of launching workers with fork() to only import the code once,
+    this presents a conflict.
+    Our work around is to 'install' this ProxiedReactor which prevents Twisted
+    from creating and installing one, but which lets us replace the actual reactor
+    in use later on.
+    """
+
+    def __init__(self) -> None:
+        self.___reactor_target: Any = None
+
+    def _install_real_reactor(self, new_reactor: Any) -> None:
+        """
+        Install a real reactor for this ProxiedReactor to forward lookups onto.
+
+        This method is specific to our ProxiedReactor and should not clash with
+        any names used on an actual Twisted reactor.
+        """
+        self.___reactor_target = new_reactor
+
+    def __getattr__(self, attr_name: str) -> Any:
+        return getattr(self.___reactor_target, attr_name)
+
+
+def _worker_entrypoint(
+    func: Callable[[], None], proxy_reactor: ProxiedReactor, args: List[str]
+) -> None:
+    """
+    Entrypoint for a forked worker process.
+
+    We just need to set up the command-line arguments, create our real reactor
+    and then kick off the worker's main() function.
+    """
+
+    sys.argv = args
+
+    from twisted.internet.epollreactor import EPollReactor
+
+    proxy_reactor._install_real_reactor(EPollReactor())
+    func()
+
+
+def main() -> None:
+    """
+    Entrypoint for the forking launcher.
+    """
+    parser = argparse.ArgumentParser()
+    parser.add_argument("db_config", help="Path to database config file")
+    parser.add_argument(
+        "args",
+        nargs="...",
+        help="Argument groups separated by `--`. "
+        "The first argument of each group is a Synapse app name. "
+        "Subsequent arguments are passed through.",
+    )
+    ns = parser.parse_args()
+
+    # Split up the subsequent arguments into each workers' arguments;
+    # `--` is our delimiter of choice.
+    args_by_worker: List[List[str]] = [
+        list(args)
+        for cond, args in itertools.groupby(ns.args, lambda ele: ele != "--")
+        if cond and args
+    ]
+
+    # Prevent Twisted from installing a shared reactor that all the workers will
+    # inherit when we fork(), by installing our own beforehand.
+    proxy_reactor = ProxiedReactor()
+    installReactor(proxy_reactor)
+
+    # Import the entrypoints for all the workers.
+    worker_functions = []
+    for worker_args in args_by_worker:
+        worker_module = importlib.import_module(worker_args[0])
+        worker_functions.append(worker_module.main)
+
+    # We need to prepare the database first as otherwise all the workers will
+    # try to create a schema version table and some will crash out.
+    from synapse._scripts import update_synapse_database
+
+    update_proc = multiprocessing.Process(
+        target=_worker_entrypoint,
+        args=(
+            update_synapse_database.main,
+            proxy_reactor,
+            [
+                "update_synapse_database",
+                "--database-config",
+                ns.db_config,
+                "--run-background-updates",
+            ],
+        ),
+    )
+    print("===== PREPARING DATABASE =====", file=sys.stderr)
+    update_proc.start()
+    update_proc.join()
+    print("===== PREPARED DATABASE =====", file=sys.stderr)
+
+    # At this point, we've imported all the main entrypoints for all the workers.
+    # Now we basically just fork() out to create the workers we need.
+    # Because we're using fork(), all the workers get a clone of this launcher's
+    # memory space and don't need to repeat the work of loading the code!
+    # Instead of using fork() directly, we use the multiprocessing library,
+    # which uses fork() on Unix platforms.
+    processes = []
+    for (func, worker_args) in zip(worker_functions, args_by_worker):
+        process = multiprocessing.Process(
+            target=_worker_entrypoint, args=(func, proxy_reactor, worker_args)
+        )
+        process.start()
+        processes.append(process)
+
+    # Be a good parent and wait for our children to die before exiting.
+    for process in processes:
+        process.join()
+
+
+if __name__ == "__main__":
+    main()
@@ -17,6 +17,11 @@ import sys
 from synapse.app.generic_worker import start
 from synapse.util.logcontext import LoggingContext

-if __name__ == "__main__":
+
+def main() -> None:
    with LoggingContext("main"):
        start(sys.argv[1:])
+
+
+if __name__ == "__main__":
+    main()
@@ -17,6 +17,11 @@ import sys
 from synapse.app.generic_worker import start
 from synapse.util.logcontext import LoggingContext

-if __name__ == "__main__":
+
+def main() -> None:
    with LoggingContext("main"):
        start(sys.argv[1:])
+
+
+if __name__ == "__main__":
+    main()
@@ -17,6 +17,11 @@ import sys
 from synapse.app.generic_worker import start
 from synapse.util.logcontext import LoggingContext

-if __name__ == "__main__":
+
+def main() -> None:
    with LoggingContext("main"):
        start(sys.argv[1:])
+
+
+if __name__ == "__main__":
+    main()
@@ -17,6 +17,11 @@ import sys
 from synapse.app.generic_worker import start
 from synapse.util.logcontext import LoggingContext

-if __name__ == "__main__":
+
+def main() -> None:
    with LoggingContext("main"):
        start(sys.argv[1:])
+
+
+if __name__ == "__main__":
+    main()
@@ -17,6 +17,11 @@ import sys
 from synapse.app.generic_worker import start
 from synapse.util.logcontext import LoggingContext

-if __name__ == "__main__":
+
+def main() -> None:
    with LoggingContext("main"):
        start(sys.argv[1:])
+
+
+if __name__ == "__main__":
+    main()
@@ -17,6 +17,11 @@ import sys
 from synapse.app.generic_worker import start
 from synapse.util.logcontext import LoggingContext

-if __name__ == "__main__":
+
+def main() -> None:
    with LoggingContext("main"):
        start(sys.argv[1:])
+
+
+if __name__ == "__main__":
+    main()
@@ -17,6 +17,11 @@ import sys
 from synapse.app.generic_worker import start
 from synapse.util.logcontext import LoggingContext

-if __name__ == "__main__":
+
+def main() -> None:
    with LoggingContext("main"):
        start(sys.argv[1:])
+
+
+if __name__ == "__main__":
+    main()
--- a/Show More
+++ b/Show More
Author	SHA1	Message	Date
Olivier Wilkinson (reivilibre)	c747b4b37d	AAAA#	2022-07-05 17:26:12 +01:00
Olivier Wilkinson (reivilibre)	7094bcf251	DBG cache all	2022-07-05 12:40:26 +01:00
Olivier Wilkinson (reivilibre)	5af3ce8bf8	DBG cache	2022-07-05 12:31:50 +01:00
Olivier Wilkinson (reivilibre)	7fd218ad8b	DBG retry until fail	2022-07-05 12:18:26 +01:00
Olivier Wilkinson (reivilibre)	30627554d4	DBG set debug	2022-07-05 12:18:07 +01:00
Olivier Wilkinson (reivilibre)	3c41609a9e	DBG limit	2022-07-05 11:59:48 +01:00
Olivier Wilkinson (reivilibre)	18695c7631	DBG	2022-07-05 11:37:14 +01:00
Olivier Wilkinson (reivilibre)	c76da0d948	TMP flake debug code	2022-07-05 11:25:57 +01:00
Olivier Wilkinson (reivilibre)	3f203e1b10	Newsfile Signed-off-by: Olivier Wilkinson (reivilibre) <oliverw@matrix.org>	2022-07-04 12:01:09 +01:00
Olivier Wilkinson (reivilibre)	ae7f8cccc2	Give -f a long option --fast as it's confusing otherwise (and I might remember it better)	2022-07-04 12:01:09 +01:00
Olivier Wilkinson (reivilibre)	37112f76f7	Add a --build-only argument to complement.sh	2022-07-04 12:01:09 +01:00
Olivier Wilkinson (reivilibre)	f61931fb8a	Newsfile Signed-off-by: Olivier Wilkinson (reivilibre) <oliverw@matrix.org>	2022-07-01 16:55:14 +01:00
Olivier Wilkinson (reivilibre)	ae1712dd71	Factor out Complement checking-out	2022-07-01 16:55:14 +01:00
Olivier Wilkinson (reivilibre)	c1229e0218	Factor out some common commands into a script	2022-07-01 16:55:14 +01:00
reivilibre	c04e25789e	Enable Complement testing in the 'Twisted Trunk' CI runs. (#13079 )	2022-07-01 15:42:49 +00:00
Richard van der Hoff	fe910fb10e	complement.sh: Permit skipping docker build (#13143 ) Add a `-f` argument to `complement.sh` to skip the docker build	2022-07-01 12:33:59 +00:00
Andrew Morgan	5296c09473	Merge tag 'v1.62.0rc2' into develop Synapse 1.62.0rc2 (2022-07-01) ============================== Bugfixes -------- - Fix unread counts for users on large servers. Introduced in v1.62.0rc1. ([\#13140](https://github.com/matrix-org/synapse/issues/13140)) - Fix DB performance when deleting old push notifications. Introduced in v1.62.0rc1. ([\#13141](https://github.com/matrix-org/synapse/issues/13141))	2022-07-01 12:29:23 +01:00
Andrew Morgan	d40b2708cf	1.62.0rc2	2022-07-01 11:42:57 +01:00
David Robertson	d70ff5cc35	Extra validation for rest/client/account_data (#13148 ) * Extra validation for rest/client/account_data This is a fairly simple endpoint and we did pretty well here. * Changelog	2022-07-01 11:04:56 +01:00
Richard van der Hoff	6da861ae69	`_process_received_pdu`: Improve exception handling (#13145 ) `_check_event_auth` is expected to raise `AuthError`s, so no need to log it again.	2022-07-01 10:52:10 +01:00
Richard van der Hoff	8c2825276f	Skip waiting for full state for incoming events (#13144 ) When we receive an event over federation during a faster join, there is no need to wait for full state, since we have a whole reconciliation process designed to take the partial state into account.	2022-07-01 10:19:27 +01:00
Andrew Morgan	c0efc689cb	Add documentation for phone home stats (#13086 )	2022-06-30 22:12:28 +01:00
Jacek Kuśnierz	50f0e4028b	Allow dependency errors to pass through (#13113 ) Signed-off-by: Jacek Kusnierz <jacek.kusnierz@tum.de> Co-authored-by: Brendan Abolivier <babolivier@matrix.org>	2022-06-30 19:48:04 +02:00
Patrick Cloke	b0366853ca	Merge remote-tracking branch 'origin/release-v1.62' into develop	2022-06-30 13:27:24 -04:00
Shay	046a6513bc	Don't process /send requests for users who have hit their ratelimit (#13134 )	2022-06-30 09:22:40 -07:00
Shay	8330fc9953	Cleanup references to sample config in the docs and redirect users to configuration manual (#13077 )	2022-06-30 09:21:39 -07:00
Andrew Morgan	0ceb3af10b	Add a link to the configuration manual from the homeserver sample config documentation page (#13139 )	2022-06-30 15:59:11 +01:00
Erik Johnston	dbce28b2f1	Fix unread counts on large servers (#13140 )	2022-06-30 15:08:40 +01:00
Erik Johnston	a3a05c812d	Add index to help delete old push actions (#13141 )	2022-06-30 14:05:49 +00:00
Patrick Cloke	6ad012ef89	More type hints for `synapse.logging` (#13103 ) Completes type hints for synapse.logging.scopecontextmanager and (partially) for synapse.logging.opentracing.	2022-06-30 13:05:06 +00:00
reivilibre	9667bad55d	Improve startup times in Complement test runs against workers, particularly in CPU-constrained environments. (#13127 ) Co-authored-by: Richard van der Hoff <1389908+richvdh@users.noreply.github.com>	2022-06-30 11:58:12 +00:00
David Robertson	09f6e43025	Actually typecheck `tests.test_server` (#13135 )	2022-06-30 10:45:47 +01:00
David Teller	80c7a06777	Rate limiting invites per issuer (#13125 ) Co-authored-by: reivilibre <oliverw@matrix.org>	2022-06-30 09:44:47 +00:00
Brendan Abolivier	4d3b8fb23f	Don't actually one-line the SQL statements we send to the DB (#13129 )	2022-06-30 10:43:24 +02:00
Šimon Brandner	13e359aec8	Implement MSC3827: Filtering of `/publicRooms` by room type (#13031 ) Signed-off-by: Šimon Brandner <simon.bra.ag@gmail.com>	2022-06-29 17:12:45 +00:00
Moritz Stückler	e714b8a057	Fix documentation header for `allow_public_rooms_over_federation` (#13116 ) Signed-off-by: Moritz Stückler <moritz.stueckler@gmail.com> Co-authored-by: Patrick Cloke <clokep@users.noreply.github.com>	2022-06-29 18:41:39 +02:00
Erik Johnston	92a0c18ef0	Improve performance of getting unread counts in rooms (#13119 )	2022-06-29 10:32:38 +00:00
jejo86	cdc0259449	Document the `--report-stats` argument (#13029 ) Signed-off-by: jejo86 <28619134+jejo86@users.noreply.github.com>	2022-06-29 10:24:10 +01:00
Andrew Morgan	79c6b9e12b	Merge branch 'develop' into release-v1.62	2022-06-28 16:47:21 +01:00
Andrew Morgan	bc9b0912cc	fix linting error from the 1.61.1 main -> develop merge	2022-06-28 16:47:04 +01:00
Andrew Morgan	b210146fd9	1.62.0rc1	2022-06-28 16:42:44 +01:00
Andrew Morgan	6cba6a51af	Merge branch 'master' into develop	2022-06-28 15:19:48 +01:00
Andrew Morgan	09d89ddc1f	Linkify GHSA commit	2022-06-28 14:41:06 +01:00
Andrew Morgan	ea10cdbea7	1.61.1	2022-06-28 14:37:35 +01:00
reivilibre	fa13080618	Merge pull request from GHSA-22p3-qrh9-cx32 * Make _iterate_over_text easier to read by using simple data structures * Prefer a set of tags to ignore In my tests, it's 4x faster to check for containment in a set of this size * Add a stack size limit to _iterate_over_text * Continue accepting the case where there is no body element * Use an early return instead for None Co-authored-by: Richard van der Hoff <richard@matrix.org>	2022-06-28 14:29:08 +01:00
Erik Johnston	7469824d58	Fix serialization errors when rotating notifications (#13118 )	2022-06-28 13:13:44 +01:00
David Robertson	f1145563f6	Extra type annotations in `test_server` (#13124 )	2022-06-28 12:12:17 +00:00
santhoshivan23	6b99a66fe0	Remove unspecced DELETE endpoint that modifies room visibility (#13123 )	2022-06-28 11:22:59 +00:00
Šimon Brandner	1017f09c18	Update MSC3786 implementation: Check the `state_key` (#12939 ) Signed-off-by: Šimon Brandner <simon.bra.ag@gmail.com>	2022-06-27 20:28:34 +01:00
Robert Long	9b683ea80f	Add Cross-Origin-Resource-Policy header to thumbnail and download media endpoints (#12944 )	2022-06-27 14:44:05 +01:00
reivilibre	3c5549e74a	Refactor the Dockerfile-workers configuration script to use Jinja2 templates in Synapse workers' Supervisord blocks. (#13054 ) Co-authored-by: Richard van der Hoff <1389908+richvdh@users.noreply.github.com>	2022-06-27 11:43:20 +01:00
Aaron Raimist	3ceaf1462d	Remove docs for Delete Group Admin API (#13112 ) This API no longer exists. Signed-off-by: Aaron Raimist <aaron@raim.ist>	2022-06-27 11:15:25 +01:00
santhoshivan23	d54909956e	validate room alias before interacting with the room directory (#13106 )	2022-06-22 15:32:18 +01:00
David Robertson	f33356e8f8	Use caret (semver bounds) for matrix.org packages (#13082 )	2022-06-17 19:07:04 +01:00
Shay	3d94d07db3	Update opentracing docs to reference the configuration manual rather than the configuation file. (#13076 )	2022-06-17 10:47:38 -07:00
Richard van der Hoff	d4b1c0d800	Fix inconsistencies in event validation (#13088 )	2022-06-17 16:30:59 +01:00
Richard van der Hoff	e16ea87d0f	Fix inconsistencies in event validation for `m.room.create` events (#13087 ) * Extend the auth rule checks for `m.room.create` events ... and move them up to the top of the function. Since the no auth_events are allowed for m.room.create events, we may as well get the m.room.create event checks out of the way first. * Add a test for create events with prev_events	2022-06-17 13:56:46 +00:00
Patrick Cloke	d3d84685ce	Add type hints to event push actions tests. (#13099 )	2022-06-17 12:38:13 +00:00
reivilibre	b26cbe3d45	Fix type error that made its way onto develop (#13098 ) * Fix type error introduced accidentally by #13045 * Newsfile Signed-off-by: Olivier Wilkinson (reivilibre) <oliverw@matrix.org>	2022-06-17 13:05:27 +01:00
Richard van der Hoff	5d6f55959e	Update info on downstream debs (#13095 )	2022-06-17 12:47:22 +01:00
Quentin Gliech	73af10f419	Simplify the alias deletion logic as an application service. (#13093 )	2022-06-17 12:19:22 +01:00
Erik Johnston	5ef05c70c3	Rotate notifications more frequently (#13096 )	2022-06-17 10:58:00 +00:00
Erik Johnston	5099b5ecc7	Use new `device_list_changes_in_room` table when getting device list changes (#13045 )	2022-06-17 11:42:03 +01:00
Quentin Gliech	c6d6176411	Allow MSC3030 'timestamp_to_event' calls from anyone on world-readable rooms. (#13062 ) Signed-off-by: Quentin Gliech <quenting@element.io>	2022-06-17 11:39:26 +01:00
Sean Quah	9372f6f842	Fix logging context misuse when we fail to persist a federation event (#13089 ) When we fail to persist a federation event, we kick off a task to remove its push actions in the background, using the current logging context. Since we don't `await` that task, we may finish our logging context before the task finishes. There's no reason to not `await` the task, so let's do that. Signed-off-by: Sean Quah <seanq@matrix.org>	2022-06-17 10:22:50 +01:00
Erik Johnston	8ceed5e6b5	Add desc to `get_earliest_token_for_stats` (#13085 )	2022-06-16 17:50:46 +00:00
reivilibre	90cadcd403	Add a Subject Alternative Name to the certificate generated for Complement tests. (#13071 )	2022-06-16 12:43:21 +01:00
Patrick Cloke	0fcc0ae37c	Improve URL previews for sites with only Twitter card information. (#13056 ) Pull out `twitter:` meta tags when generating a preview and use it to augment any `og:` meta tags. Prefers Open Graph information over Twitter card information.	2022-06-16 07:41:57 -04:00
reivilibre	7552615247	Reduce the duplication of code that invokes the rate limiter. (#13070 )	2022-06-16 12:40:29 +01:00
Richard van der Hoff	1e0044e8f9	Complement: use SQLite by default (#13075 ) If no database is configured explicitly, use sqlite. This means that you don't have to pass any variables into the image.	2022-06-16 12:12:26 +01:00
Jacek Kuśnierz	0ef1307619	Add custom well-known (#13035 ) Co-authored-by: David Robertson <david.m.robertson1@gmail.com>	2022-06-16 11:48:18 +01:00
reivilibre	ffe2464836	Add instructions for running Complement with `gotestfmt`-formatted output locally. (#13073 )	2022-06-16 09:31:10 +00:00
Richard van der Hoff	8ecf6be1e1	Move some event auth checks out to a different method (#13065 ) * Add auth events to events used in tests * Move some event auth checks out to a different method Some of the event auth checks apply to an event's auth_events, rather than the state at the event - which means they can play no part in state resolution. Move them out to a separate method. * Rename check_auth_rules_for_event Now it only checks the state-dependent auth rules, it needs a better name.	2022-06-15 19:48:22 +01:00
Shay	cba1c5cbc2	Add headers to individual options in config documentation to allow for linking. (#13055 )	2022-06-15 11:31:46 -07:00
Sean Quah	99d3931974	Add more tests for room upgrades (#13074 ) Signed-off-by: Sean Quah <seanq@element.io>	2022-06-15 18:58:23 +01:00
Erik Johnston	c95b04bb0e	Change default `sync_response_cache_duration` (#13042 )	2022-06-15 16:55:20 +00:00
Hannes Lerchl	7d99414edf	Replace pyjwt with authlib in `org.matrix.login.jwt` (#13011 )	2022-06-15 16:45:16 +00:00
reivilibre	e12ff697a4	Sort failing jobs in Complement CI to the top of the logs to make them easier to read. (#13057 )	2022-06-15 16:13:36 +00:00
Erik Johnston	de334ac183	Add a CI job to check that schema deltas are in the correct folder. (#13063 )	2022-06-15 16:27:18 +01:00
David Robertson	97e9fbe1b2	Type annotations in `synapse.databases.main.devices` (#13025 ) Co-authored-by: Patrick Cloke <clokep@users.noreply.github.com>	2022-06-15 15:20:04 +00:00
Erik Johnston	0d1d3e0708	Speed up `get_unread_event_push_actions_by_room` (#13005 ) Fixes #11887 hopefully. The core change here is that `event_push_summary` now holds a summary of counts up until a much more recent point, meaning that the range of rows we need to count in `event_push_actions` is much smaller. This needs two major changes: 1. When we get a receipt we need to recalculate `event_push_summary` rather than just delete it 2. The logic for deleting `event_push_actions` is now divorced from calculating `event_push_summary`. In future it would be good to calculate `event_push_summary` while we persist a new event (it should just be a case of adding one to the relevant rows in `event_push_summary`), as that will further simplify the get counts logic and remove the need for us to periodically update `event_push_summary` in a background job.	2022-06-15 15:17:14 +00:00
Erik Johnston	9ad2197fa7	Rename complement-developonly (#13046 )	2022-06-15 15:11:42 +00:00
reivilibre	212be2edc1	Use updated `update_user_directory_from_worker` options in Complement tests. (#13069 )	2022-06-15 15:54:32 +01:00
reivilibre	538044ac01	Collapse Docker build commands in Complement CI runs to make the logs easier to read. (#13058 )	2022-06-15 14:42:27 +00:00
David Robertson	941dc3db13	Track a histogram of state res durations (#13036 )	2022-06-15 15:19:49 +01:00
reivilibre	0dbdc39940	Fix a long-standing bug which meant that rate limiting was not restrictive enough in some cases. (#13018 )	2022-06-15 14:11:55 +00:00
Brendan Abolivier	417f4cf40b	Don't use keyword arguments when initialising modules (#13060 )	2022-06-15 15:36:16 +02:00
Richard van der Hoff	75fb10ee45	Clean up schema for `event_edges` (#12893 ) * Remove redundant references to `event_edges.room_id` We don't need to care about the room_id here, because we are already checking the event id. * Clean up the event_edges table We make a number of changes to `event_edges`: * We give the `room_id` and `is_state` columns defaults (null and false respectively) so that we can stop populating them. * We drop any rows that have `is_state` set true - they should no longer exist. * We drop any rows that do not exist in `events` - these should not exist either. * We drop the old unique constraint on all the colums, which wasn't much use. * We create a new unique index on `(event_id, prev_event_id)`. * We add a foreign key constraint to `events`. These happen rather differently depending on whether we are on Postgres or SQLite. For SQLite, we just rebuild the whole table, copying only the rows we want to keep. For Postgres, we try to do things in the background as much as possible. * Stop populating `event_edges.room_id` and `is_state` We can just rely on the defaults.	2022-06-15 12:29:42 +01:00
David Robertson	a4ae1406d1	Fix typechecks against twisted trunk (#13061 )	2022-06-15 11:49:58 +01:00
Patrick Cloke	bd03332a1d	Merge branch 'master' into develop	2022-06-14 14:27:53 -04:00
Patrick Cloke	21e6c0ed64	Fix incorrect link in changelog.	2022-06-14 14:27:17 -04:00
reivilibre	5b645ae2ad	Refactor entry points so that they all have a `main` function. (#13052 )	2022-06-14 17:41:06 +00:00
David Robertson	c99b511db9	Fix `destination_is` errors seen in sentry. (#13041 ) * Rename test_fedclient to match its source file * Require at least one destination to be truthy * Explicitly validate user ID in profile endpoint GETs Co-authored-by: Patrick Cloke <clokep@users.noreply.github.com>	2022-06-14 18:28:26 +01:00
Erik Johnston	aef398457f	Up complement time outs (#13048 )	2022-06-14 17:59:06 +01:00
Brendan Abolivier	bdb6628dcf	Fix version number in spam checker callbacks doc (#13047 )	2022-06-14 17:24:25 +02:00
Shay	493c2fc44a	Remove code generating comments in configuration file (#12941 )	2022-06-14 07:53:42 -07:00
Patrick Cloke	5f4ecf759d	Rename delta to apply in the proper schema version. (#13050 )	2022-06-14 14:34:04 +00:00
Quentin Gliech	fe1daad672	Move the "email unsubscribe" resource, refactor the macaroon generator & simplify the access token verification logic. (#12986 ) This simplifies the access token verification logic by removing the `rights` parameter which was only ever used for the unsubscribe link in email notifications. The latter has been moved under the `/_synapse` namespace, since it is not a standard API. This also makes the email verification link more secure, by embedding the app_id and pushkey in the macaroon and verifying it. This prevents the user from tampering the query parameters of that unsubscribe link. Macaroon generation is refactored: - Centralised all macaroon generation and verification logic to the `MacaroonGenerator` - Moved to `synapse.utils` - Changed the constructor to require only a `Clock`, hostname, and a secret key (instead of a full `Homeserver`). - Added tests for all methods.	2022-06-14 09:12:08 -04:00
reivilibre	09a3c5ce0b	Fix Complement runs always being Postgres (#13034 ) * Fix Complement runs always being Postgres * Newsfile Signed-off-by: Olivier Wilkinson (reivilibre) <oliverw@matrix.org>	2022-06-14 13:13:35 +01:00
Erik Johnston	5d139f578d	Merge branch 'release-v1.61' into develop	2022-06-14 12:00:02 +01:00
Erik Johnston	b8bf61230c	Fixup upgrades	2022-06-14 11:56:45 +01:00
Erik Johnston	e87355f201	Update changelog	2022-06-14 11:49:33 +01:00
Erik Johnston	d580014e22	1.61.0	2022-06-14 11:44:27 +01:00
Sami Olmari	7b54badd31	Mention removed community/group worker endpoints in upgrade.md (#13023 )	2022-06-14 11:40:02 +01:00
Sami Olmari	a542a92c57	Mention removed community/group worker endpoints in upgrade.md (#13023 )	2022-06-14 11:35:22 +01:00
Quentin Gliech	92103cb2c8	Decouple `synapse.api.auth_blocking.AuthBlocking` from `synapse.api.auth.Auth`. (#13021 )	2022-06-14 09:51:15 +01:00
David Teller	a164a46038	Uniformize spam-checker API, part 4: port other spam-checker callbacks to return `Union[Allow, Codes]`. (#12857 ) Co-authored-by: Brendan Abolivier <babolivier@matrix.org>	2022-06-13 18:16:16 +00:00
Patrick Cloke	53b77b203a	Replace noop background updates with DELETE. (#12954 ) Removes the `register_noop_background_update` and deletes the background updates directly in a delta file.	2022-06-13 14:06:27 -04:00
				`@@ -1 +0,0 @@`
				Add tests for cancellation of `GET /rooms/$room_id/members` and `GET /rooms/$room_id/state` requests.
				`@@ -1 +0,0 @@`
				`Add documentation for how to configure Synapse with Workers using Docker Compose. Includes example worker config and docker-compose.yaml. Contributed by @Thumbscrew.`
				`@@ -1 +0,0 @@`
				`Report login failures due to unknown third party identifiers in the same way as failures due to invalid passwords. This prevents an attacker from using the error response to determine if the identifier exists. Contributed by Daniel Aloni.`
				`@@ -1 +0,0 @@`
				`Merge the Complement testing Docker images into a single, multi-purpose image.`
				`@@ -1 +0,0 @@`
				`Clean up the test code for client disconnection.`
				`@@ -1 +0,0 @@`
				`Use lower isolation level when inserting read receipts to avoid serialization errors. Contributed by Nick @ Beeper.`
				`@@ -1 +0,0 @@`
				`Reduce the amount of state we pull from the DB.`
				`@@ -1 +0,0 @@`
				`Enable testing against PostgreSQL databases in Complement CI.`
				`@@ -1 +0,0 @@`
				Remove the `delete_device` method and always call `delete_devices`.