CCCC

.ci/scripts/test_old_deps.sh

@@ -69,7 +69,7 @@ with open('pyproject.toml', 'w') as f:
 "
 python3 -c "$REMOVE_DEV_DEPENDENCIES"
 
-pipx install poetry==1.1.14
+pipx install poetry==1.1.12
 ~/.local/bin/poetry lock
 
 echo "::group::Patched pyproject.toml"

.editorconfig

@@ -7,4 +7,3 @@ root = true
 [*.py]
 indent_style = space
 indent_size = 4
-max_line_length = 88

.git-blame-ignore-revs

@@ -1,16 +1,3 @@
-# Commits in this file will be removed from GitHub blame results.
-#
-# To use this file locally, use:
-#   git blame --ignore-revs-file="path/to/.git-blame-ignore-revs" <files>
-#
-# or configure the `blame.ignoreRevsFile` option in your git config.
-#
-# If ignoring a pull request that was not squash merged, only the merge
-# commit needs to be put here. Child commits will be resolved from it.
-
-# Run black (#3679).
-8b3d9b6b199abb87246f982d5db356f1966db925
-
 # Black reformatting (#5482).
 32e7c9e7f20b57dd081023ac42d6931a8da9b3a3
 

.github/workflows/latest_deps.yml

@@ -135,42 +135,11 @@ jobs:
             /logs/**/*.log*
 
 
-  complement:
-    if: "${{ !failure() && !cancelled() }}"
-    runs-on: ubuntu-latest
+  # TODO: run complement (as with twisted trunk, see #12473).
 
-    strategy:
-      fail-fast: false
-      matrix:
-        include:
-          - arrangement: monolith
-            database: SQLite
-
-          - arrangement: monolith
-            database: Postgres
-
-          - arrangement: workers
-            database: Postgres
-
-    steps:
-      - name: Run actions/checkout@v2 for synapse
-        uses: actions/checkout@v2
-        with:
-          path: synapse
-
-      - name: Prepare Complement's Prerequisites
-        run: synapse/.ci/scripts/setup_complement_prerequisites.sh
-
-      - run: |
-          set -o pipefail
-          TEST_ONLY_IGNORE_POETRY_LOCKFILE=1 POSTGRES=${{ (matrix.database == 'Postgres') && 1 || '' }} WORKERS=${{ (matrix.arrangement == 'workers') && 1 || '' }} COMPLEMENT_DIR=`pwd`/complement synapse/scripts-dev/complement.sh -json 2>&1 | gotestfmt
-        shell: bash
-        name: Run Complement Tests
-
-  # Open an issue if the build fails, so we know about it.
-  # Only do this if we're not experimenting with this action in a PR.
+  # open an issue if the build fails, so we know about it.
   open-issue:
-    if: "failure() && github.event_name != 'push' && github.event_name != 'pull_request'"
+    if: failure()
     needs:
       # TODO: should mypy be included here? It feels more brittle than the other two.
       - mypy

.github/workflows/tests.yml

@@ -10,324 +10,15 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
-  check-sampleconfig:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
-      - uses: actions/setup-python@v2
-      - run: pip install .
-      - run: scripts-dev/generate_sample_config.sh --check
-      - run: scripts-dev/config-lint.sh
-
-  check-schema-delta:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
-      - uses: actions/setup-python@v2
-      - run: "pip install 'click==8.1.1' 'GitPython>=3.1.20'"
-      - run: scripts-dev/check_schema_delta.py --force-colors
-
-  lint:
-    uses: "matrix-org/backend-meta/.github/workflows/python-poetry-ci.yml@v1"
-    with:
-      typechecking-extras: "all"
-
-  lint-crlf:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
-      - name: Check line endings
-        run: scripts-dev/check_line_terminators.sh
-
-  lint-newsfile:
-    if: ${{ github.base_ref == 'develop'  || contains(github.base_ref, 'release-') }}
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
-        with:
-          ref: ${{ github.event.pull_request.head.sha }}
-          fetch-depth: 0
-      - uses: actions/setup-python@v2
-      - run: "pip install 'towncrier>=18.6.0rc1'"
-      - run: scripts-dev/check-newsfragment.sh
-        env:
-          PULL_REQUEST_NUMBER: ${{ github.event.number }}
-
-  # Dummy step to gate other tests on without repeating the whole list
-  linting-done:
-    if: ${{ !cancelled() }} # Run this even if prior jobs were skipped
-    needs: [lint, lint-crlf, lint-newsfile, check-sampleconfig, check-schema-delta]
-    runs-on: ubuntu-latest
-    steps:
-      - run: "true"
-
-  trial:
-    if: ${{ !cancelled() && !failure() }} # Allow previous steps to be skipped, but not fail
-    needs: linting-done
-    runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        python-version: ["3.7", "3.8", "3.9", "3.10"]
-        database: ["sqlite"]
-        extras: ["all"]
-        include:
-          # Newest Python without optional deps
-          - python-version: "3.10"
-            extras: ""
-
-          # Oldest Python with PostgreSQL
-          - python-version: "3.7"
-            database: "postgres"
-            postgres-version: "10"
-            extras: "all"
-
-          # Newest Python with newest PostgreSQL
-          - python-version: "3.10"
-            database: "postgres"
-            postgres-version: "14"
-            extras: "all"
-
-    steps:
-      - uses: actions/checkout@v2
-      - run: sudo apt-get -qq install xmlsec1
-      - name: Set up PostgreSQL ${{ matrix.postgres-version }}
-        if: ${{ matrix.postgres-version }}
-        run: |
-          docker run -d -p 5432:5432 \
-            -e POSTGRES_PASSWORD=postgres \
-            -e POSTGRES_INITDB_ARGS="--lc-collate C --lc-ctype C --encoding UTF8" \
-            postgres:${{ matrix.postgres-version }}
-      - uses: matrix-org/setup-python-poetry@v1
-        with:
-          python-version: ${{ matrix.python-version }}
-          extras: ${{ matrix.extras }}
-      - name: Await PostgreSQL
-        if: ${{ matrix.postgres-version }}
-        timeout-minutes: 2
-        run: until pg_isready -h localhost; do sleep 1; done
-      - run: poetry run trial --jobs=2 tests
-        env:
-          SYNAPSE_POSTGRES: ${{ matrix.database == 'postgres' || '' }}
-          SYNAPSE_POSTGRES_HOST: localhost
-          SYNAPSE_POSTGRES_USER: postgres
-          SYNAPSE_POSTGRES_PASSWORD: postgres
-      - name: Dump logs
-        # Logs are most useful when the command fails, always include them.
-        if: ${{ always() }}
-        # Note: Dumps to workflow logs instead of using actions/upload-artifact
-        #       This keeps logs colocated with failing jobs
-        #       It also ignores find's exit code; this is a best effort affair
-        run: >-
-          find _trial_temp -name '*.log'
-          -exec echo "::group::{}" \;
-          -exec cat {} \;
-          -exec echo "::endgroup::" \;
-          || true
-
-  trial-olddeps:
-    # Note: sqlite only; no postgres
-    if: ${{ !cancelled() && !failure() }} # Allow previous steps to be skipped, but not fail
-    needs: linting-done
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
-      - name: Test with old deps
-        uses: docker://ubuntu:focal # For old python and sqlite
-        # Note: focal seems to be using 3.8, but the oldest is 3.7?
-        # See https://github.com/matrix-org/synapse/issues/12343
-        with:
-          workdir: /github/workspace
-          entrypoint: .ci/scripts/test_old_deps.sh
-      - name: Dump logs
-        # Logs are most useful when the command fails, always include them.
-        if: ${{ always() }}
-        # Note: Dumps to workflow logs instead of using actions/upload-artifact
-        #       This keeps logs colocated with failing jobs
-        #       It also ignores find's exit code; this is a best effort affair
-        run: >-
-          find _trial_temp -name '*.log'
-          -exec echo "::group::{}" \;
-          -exec cat {} \;
-          -exec echo "::endgroup::" \;
-          || true
-
-  trial-pypy:
-    # Very slow; only run if the branch name includes 'pypy'
-    # Note: sqlite only; no postgres. Completely untested since poetry move.
-    if: ${{ contains(github.ref, 'pypy') && !failure() && !cancelled() }}
-    needs: linting-done
-    runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        python-version: ["pypy-3.7"]
-        extras: ["all"]
-
-    steps:
-      - uses: actions/checkout@v2
-      # Install libs necessary for PyPy to build binary wheels for dependencies
-      - run: sudo apt-get -qq install xmlsec1 libxml2-dev libxslt-dev
-      - uses: matrix-org/setup-python-poetry@v1
-        with:
-          python-version: ${{ matrix.python-version }}
-          extras: ${{ matrix.extras }}
-      - run: poetry run trial --jobs=2 tests
-      - name: Dump logs
-        # Logs are most useful when the command fails, always include them.
-        if: ${{ always() }}
-        # Note: Dumps to workflow logs instead of using actions/upload-artifact
-        #       This keeps logs colocated with failing jobs
-        #       It also ignores find's exit code; this is a best effort affair
-        run: >-
-          find _trial_temp -name '*.log'
-          -exec echo "::group::{}" \;
-          -exec cat {} \;
-          -exec echo "::endgroup::" \;
-          || true
-
-  sytest:
-    if: ${{ !failure() && !cancelled() }}
-    needs: linting-done
-    runs-on: ubuntu-latest
-    container:
-      image: matrixdotorg/sytest-synapse:${{ matrix.sytest-tag }}
-      volumes:
-        - ${{ github.workspace }}:/src
-      env:
-        SYTEST_BRANCH: ${{ github.head_ref }}
-        POSTGRES: ${{ matrix.postgres && 1}}
-        MULTI_POSTGRES: ${{ (matrix.postgres == 'multi-postgres') && 1}}
-        WORKERS: ${{ matrix.workers && 1 }}
-        REDIS: ${{ matrix.redis && 1 }}
-        BLACKLIST: ${{ matrix.workers && 'synapse-blacklist-with-workers' }}
-        TOP: ${{ github.workspace }}
-
-    strategy:
-      fail-fast: false
-      matrix:
-        include:
-          - sytest-tag: focal
-
-          - sytest-tag: focal
-            postgres: postgres
-
-          - sytest-tag: testing
-            postgres: postgres
-
-          - sytest-tag: focal
-            postgres: multi-postgres
-            workers: workers
-
-          - sytest-tag: buster
-            postgres: multi-postgres
-            workers: workers
-
-          - sytest-tag: buster
-            postgres: postgres
-            workers: workers
-            redis: redis
-
-    steps:
-      - uses: actions/checkout@v2
-      - name: Prepare test blacklist
-        run: cat sytest-blacklist .ci/worker-blacklist > synapse-blacklist-with-workers
-      - name: Run SyTest
-        run: /bootstrap.sh synapse
-        working-directory: /src
-      - name: Summarise results.tap
-        if: ${{ always() }}
-        run: /sytest/scripts/tap_to_gha.pl /logs/results.tap
-      - name: Upload SyTest logs
-        uses: actions/upload-artifact@v2
-        if: ${{ always() }}
-        with:
-          name: Sytest Logs - ${{ job.status }} - (${{ join(matrix.*, ', ') }})
-          path: |
-            /logs/results.tap
-            /logs/**/*.log*
-
-  export-data:
-    if: ${{ !failure() && !cancelled() }} # Allow previous steps to be skipped, but not fail
-    needs: [linting-done, portdb]
-    runs-on: ubuntu-latest
-    env:
-      TOP: ${{ github.workspace }}
-
-    services:
-      postgres:
-        image: postgres
-        ports:
-          - 5432:5432
-        env:
-          POSTGRES_PASSWORD: "postgres"
-          POSTGRES_INITDB_ARGS: "--lc-collate C --lc-ctype C --encoding UTF8"
-        options: >-
-          --health-cmd pg_isready
-          --health-interval 10s
-          --health-timeout 5s
-          --health-retries 5
-
-    steps:
-      - uses: actions/checkout@v2
-      - run: sudo apt-get -qq install xmlsec1
-      - uses: matrix-org/setup-python-poetry@v1
-        with:
-          python-version: ${{ matrix.python-version }}
-          extras: "postgres"
-      - run: .ci/scripts/test_export_data_command.sh
-
-  portdb:
-    if: ${{ !failure() && !cancelled() }} # Allow previous steps to be skipped, but not fail
-    needs: linting-done
-    runs-on: ubuntu-latest
-    env:
-      TOP: ${{ github.workspace }}
-    strategy:
-      matrix:
-        include:
-          - python-version: "3.7"
-            postgres-version: "10"
-
-          - python-version: "3.10"
-            postgres-version: "14"
-
-    services:
-      postgres:
-        image: postgres:${{ matrix.postgres-version }}
-        ports:
-          - 5432:5432
-        env:
-          POSTGRES_PASSWORD: "postgres"
-          POSTGRES_INITDB_ARGS: "--lc-collate C --lc-ctype C --encoding UTF8"
-        options: >-
-          --health-cmd pg_isready
-          --health-interval 10s
-          --health-timeout 5s
-          --health-retries 5
-
-    steps:
-      - uses: actions/checkout@v2
-      - run: sudo apt-get -qq install xmlsec1
-      - uses: matrix-org/setup-python-poetry@v1
-        with:
-          python-version: ${{ matrix.python-version }}
-          extras: "postgres"
-      - run: .ci/scripts/test_synapse_port_db.sh
 
   complement:
     if: "${{ !failure() && !cancelled() }}"
-    needs: linting-done
     runs-on: ubuntu-latest
 
     strategy:
       fail-fast: false
       matrix:
         include:
-          - arrangement: monolith
-            database: SQLite
-
-          - arrangement: monolith
-            database: Postgres
-
           - arrangement: workers
             database: Postgres
 
@@ -342,7 +33,14 @@ jobs:
 
       - run: |
           set -o pipefail
-          POSTGRES=${{ (matrix.database == 'Postgres') && 1 || '' }} WORKERS=${{ (matrix.arrangement == 'workers') && 1 || '' }} COMPLEMENT_DIR=`pwd`/complement synapse/scripts-dev/complement.sh -json 2>&1 | gotestfmt
+          synapse/scripts-dev/complement.sh --build-only
+          while :; do
+            POSTGRES=${{ (matrix.database == 'Postgres') && 1 || '' }} WORKERS=${{ (matrix.arrangement == 'workers') && 1 || '' }} SYNAPSE_TEST_LOG_LEVEL=DEBUG COMPLEMENT_DIR=`pwd`/complement synapse/scripts-dev/complement.sh -f -run TestSendJoinPartialStateResponse -json 2>&1 | gotestfmt | tee /tmp/xxx
+            if grep ❌ /tmp/xxx; then
+              break
+            fi
+          done
+
         shell: bash
         name: Run Complement Tests
 
@@ -350,15 +48,6 @@ jobs:
   tests-done:
     if: ${{ always() }}
     needs:
-      - check-sampleconfig
-      - lint
-      - lint-crlf
-      - lint-newsfile
-      - trial
-      - trial-olddeps
-      - sytest
-      - export-data
-      - portdb
       - complement
     runs-on: ubuntu-latest
     steps:

.github/workflows/twisted_trunk.yml

@@ -127,12 +127,12 @@ jobs:
         run: |
           set -x
           DEBIAN_FRONTEND=noninteractive sudo apt-get install -yqq python3 pipx
-          pipx install poetry==1.1.14
+          pipx install poetry==1.1.12
 
           poetry remove -n twisted
           poetry add -n --extras tls git+https://github.com/twisted/twisted.git#trunk
           poetry lock --no-update
-          # NOT IN 1.1.14 poetry lock --check
+          # NOT IN 1.1.12 poetry lock --check
         working-directory: synapse
 
       - run: |

CHANGES.md

@@ -1,283 +1,3 @@
-Synapse 1.65.0rc1 (2022-08-09)
-==============================
-
-Features
---------
-
-- Add support for stable prefixes for [MSC2285 (private read receipts)](https://github.com/matrix-org/matrix-spec-proposals/pull/2285). ([\#13273](https://github.com/matrix-org/synapse/issues/13273))
-- Add new unstable error codes `ORG.MATRIX.MSC3848.ALREADY_JOINED`, `ORG.MATRIX.MSC3848.NOT_JOINED`, and `ORG.MATRIX.MSC3848.INSUFFICIENT_POWER` described in [MSC3848](https://github.com/matrix-org/matrix-spec-proposals/pull/3848). ([\#13343](https://github.com/matrix-org/synapse/issues/13343))
-- Use stable prefixes for [MSC3827](https://github.com/matrix-org/matrix-spec-proposals/pull/3827). ([\#13370](https://github.com/matrix-org/synapse/issues/13370))
-- Add a new module API method to translate a room alias into a room ID. ([\#13428](https://github.com/matrix-org/synapse/issues/13428))
-- Add a new module API method to create a room. ([\#13429](https://github.com/matrix-org/synapse/issues/13429))
-- Add remote join capability to the module API's `update_room_membership` method (in a backwards compatible manner). ([\#13441](https://github.com/matrix-org/synapse/issues/13441))
-
-
-Bugfixes
---------
-
-- Update the version of the LDAP3 auth provider module included in the `matrixdotorg/synapse` DockerHub images and the Debian packages hosted on packages.matrix.org to 0.2.2. This version fixes a regression in the module. ([\#13470](https://github.com/matrix-org/synapse/issues/13470))
-- Fix a bug introduced in Synapse v1.41.0 where the `/hierarchy` API returned non-standard information (a `room_id` field under each entry in `children_state`). ([\#13365](https://github.com/matrix-org/synapse/issues/13365))
-- Fix a bug introduced in Synapse 0.24.0 that would respond with the wrong error status code to `/joined_members` requests when the requester is not a current member of the room. Contributed by @andrewdoh. ([\#13374](https://github.com/matrix-org/synapse/issues/13374))
-- Fix bug in handling of typing events for appservices. Contributed by Nick @ Beeper (@fizzadar). ([\#13392](https://github.com/matrix-org/synapse/issues/13392))
-- Fix a bug introduced in Synapse 1.57.0 where rooms listed in `exclude_rooms_from_sync` in the configuration file would not be properly excluded from incremental syncs. ([\#13408](https://github.com/matrix-org/synapse/issues/13408))
-- Fix a bug in the experimental faster-room-joins support which could cause it to get stuck in an infinite loop. ([\#13353](https://github.com/matrix-org/synapse/issues/13353))
-- Faster room joins: fix a bug which caused rejected events to become un-rejected during state syncing. ([\#13413](https://github.com/matrix-org/synapse/issues/13413))
-- Faster room joins: fix error when running out of servers to sync partial state with, so that Synapse raises the intended error instead. ([\#13432](https://github.com/matrix-org/synapse/issues/13432))
-
-
-Updates to the Docker image
----------------------------
-
-- Make Docker images build on armv7 by installing cryptography dependencies in the 'requirements' stage. Contributed by Jasper Spaans. ([\#13372](https://github.com/matrix-org/synapse/issues/13372))
-
-
-Improved Documentation
-----------------------
-
-- Update the 'registration tokens' page to acknowledge that the relevant MSC was merged into version 1.2 of the Matrix specification. Contributed by @moan0s. ([\#11897](https://github.com/matrix-org/synapse/issues/11897))
-- Document which HTTP resources support gzip compression. ([\#13221](https://github.com/matrix-org/synapse/issues/13221))
-- Add steps describing how to elevate an existing user to administrator by manipulating the database. ([\#13230](https://github.com/matrix-org/synapse/issues/13230))
-- Fix wrong headline for `url_preview_accept_language` in documentation. ([\#13437](https://github.com/matrix-org/synapse/issues/13437))
-- Remove redundant 'Contents' section from the Configuration Manual. Contributed by @dklimpel. ([\#13438](https://github.com/matrix-org/synapse/issues/13438))
-- Update documentation for config setting `macaroon_secret_key`. ([\#13443](https://github.com/matrix-org/synapse/issues/13443))
-- Update outdated information on `sso_mapping_providers` documentation. ([\#13449](https://github.com/matrix-org/synapse/issues/13449))
-- Fix example code in module documentation of `password_auth_provider_callbacks`. ([\#13450](https://github.com/matrix-org/synapse/issues/13450))
-- Make the configuration for the cache clearer. ([\#13481](https://github.com/matrix-org/synapse/issues/13481))
-
-
-Internal Changes
-----------------
-
-- Extend the release script to automatically push a new SyTest branch, rather than having that be a manual process. ([\#12978](https://github.com/matrix-org/synapse/issues/12978))
-- Make minor clarifications to the error messages given when we fail to join a room via any server. ([\#13160](https://github.com/matrix-org/synapse/issues/13160))
-- Enable Complement CI tests in the 'latest deps' test run. ([\#13213](https://github.com/matrix-org/synapse/issues/13213))
-- Fix long-standing bugged logic which was never hit in `get_pdu` asking every remote destination even after it finds an event. ([\#13346](https://github.com/matrix-org/synapse/issues/13346))
-- Faster room joins: avoid blocking when pulling events with partially missing prev events. ([\#13355](https://github.com/matrix-org/synapse/issues/13355))
-- Instrument `/messages` for understandable traces in Jaeger. ([\#13368](https://github.com/matrix-org/synapse/issues/13368))
-- Remove an unused argument to `get_relations_for_event`. ([\#13383](https://github.com/matrix-org/synapse/issues/13383))
-- Add a `merge-back` command to the release script, which automates merging the correct branches after a release. ([\#13393](https://github.com/matrix-org/synapse/issues/13393))
-- Adding missing type hints to tests. ([\#13397](https://github.com/matrix-org/synapse/issues/13397))
-- Faster Room Joins: don't leave a stuck room partial state flag if the join fails. ([\#13403](https://github.com/matrix-org/synapse/issues/13403))
-- Refactor `_resolve_state_at_missing_prevs` to compute an `EventContext` instead. ([\#13404](https://github.com/matrix-org/synapse/issues/13404), [\#13431](https://github.com/matrix-org/synapse/issues/13431))
-- Faster Room Joins: prevent Synapse from answering federated join requests for a room which it has not fully joined yet. ([\#13416](https://github.com/matrix-org/synapse/issues/13416))
-- Re-enable running Complement tests against Synapse with workers. ([\#13420](https://github.com/matrix-org/synapse/issues/13420))
-- Prevent unnecessary lookups to any external `get_event` cache. Contributed by Nick @ Beeper (@fizzadar). ([\#13435](https://github.com/matrix-org/synapse/issues/13435))
-- Add some tracing to give more insight into local room joins. ([\#13439](https://github.com/matrix-org/synapse/issues/13439))
-- Rename class `RateLimitConfig` to `RatelimitSettings` and `FederationRateLimitConfig` to `FederationRatelimitSettings`. ([\#13442](https://github.com/matrix-org/synapse/issues/13442))
-- Add some comments about how event push actions are stored. ([\#13445](https://github.com/matrix-org/synapse/issues/13445), [\#13455](https://github.com/matrix-org/synapse/issues/13455))
-- Improve rebuild speed for the "synapse-workers" docker image. ([\#13447](https://github.com/matrix-org/synapse/issues/13447))
-- Fix `@tag_args` being off-by-one with the arguments when tagging a span (tracing). ([\#13452](https://github.com/matrix-org/synapse/issues/13452))
-- Update type of `EventContext.rejected`. ([\#13460](https://github.com/matrix-org/synapse/issues/13460))
-- Use literals in place of `HTTPStatus` constants in tests. ([\#13463](https://github.com/matrix-org/synapse/issues/13463), [\#13469](https://github.com/matrix-org/synapse/issues/13469))
-- Correct a misnamed argument in state res v2 internals. ([\#13467](https://github.com/matrix-org/synapse/issues/13467))
-
-
-Synapse 1.64.0 (2022-08-02)
-===========================
-
-No significant changes since 1.64.0rc2.
-
-
-Deprecation Warning
--------------------
-
-Synapse v1.66.0 will remove the ability to delegate the tasks of verifying email address ownership, and password reset confirmation, to an identity server.
-
-If you require your homeserver to verify e-mail addresses or to support password resets via e-mail, please configure your homeserver with SMTP access so that it can send e-mails on its own behalf.
-[Consult the configuration documentation for more information.](https://matrix-org.github.io/synapse/latest/usage/configuration/config_documentation.html#email)
-
-
-Synapse 1.64.0rc2 (2022-07-29)
-==============================
-
-This RC reintroduces support for `account_threepid_delegates.email`, which was removed in 1.64.0rc1. It remains deprecated and will be removed altogether in Synapse v1.66.0. ([\#13406](https://github.com/matrix-org/synapse/issues/13406))
-
-
-Synapse 1.64.0rc1 (2022-07-26)
-==============================
-
-This RC removed the ability to delegate the tasks of verifying email address ownership, and password reset confirmation, to an identity server.
-
-We have also stopped building `.deb` packages for Ubuntu 21.10 as it is no longer an active version of Ubuntu.
-
-
-Features
---------
-
-- Improve error messages when media thumbnails cannot be served. ([\#13038](https://github.com/matrix-org/synapse/issues/13038))
-- Allow pagination from remote event after discovering it from [MSC3030](https://github.com/matrix-org/matrix-spec-proposals/pull/3030) `/timestamp_to_event`. ([\#13205](https://github.com/matrix-org/synapse/issues/13205))
-- Add a `room_type` field in the responses for the list room and room details admin APIs. Contributed by @andrewdoh. ([\#13208](https://github.com/matrix-org/synapse/issues/13208))
-- Add support for room version 10. ([\#13220](https://github.com/matrix-org/synapse/issues/13220))
-- Add per-room rate limiting for room joins. For each room, Synapse now monitors the rate of join events in that room, and throttles additional joins if that rate grows too large. ([\#13253](https://github.com/matrix-org/synapse/issues/13253), [\#13254](https://github.com/matrix-org/synapse/issues/13254), [\#13255](https://github.com/matrix-org/synapse/issues/13255), [\#13276](https://github.com/matrix-org/synapse/issues/13276))
-- Support Implicit TLS (TLS without using a STARTTLS upgrade, typically on port 465) for sending emails, enabled by the new option `force_tls`. Contributed by Jan Schär. ([\#13317](https://github.com/matrix-org/synapse/issues/13317))
-
-
-Bugfixes
---------
-
-- Fix a bug introduced in Synapse 1.15.0 where adding a user through the Synapse Admin API with a phone number would fail if the `enable_email_notifs` and `email_notifs_for_new_users` options were enabled. Contributed by @thomasweston12. ([\#13263](https://github.com/matrix-org/synapse/issues/13263))
-- Fix a bug introduced in Synapse 1.40.0 where a user invited to a restricted room would be briefly unable to join. ([\#13270](https://github.com/matrix-org/synapse/issues/13270))
-- Fix a long-standing bug where, in rare instances, Synapse could store the incorrect state for a room after a state resolution. ([\#13278](https://github.com/matrix-org/synapse/issues/13278))
-- Fix a bug introduced in v1.18.0 where the `synapse_pushers` metric would overcount pushers when they are replaced. ([\#13296](https://github.com/matrix-org/synapse/issues/13296))
-- Disable autocorrection and autocapitalisation on the username text field shown during registration when using SSO. ([\#13350](https://github.com/matrix-org/synapse/issues/13350))
-- Update locked version of `frozendict` to 2.3.3, which has fixes for memory leaks affecting `/sync`. ([\#13284](https://github.com/matrix-org/synapse/issues/13284), [\#13352](https://github.com/matrix-org/synapse/issues/13352))
-
-
-Improved Documentation
-----------------------
-
-- Provide an example of using the Admin API. Contributed by @jejo86. ([\#13231](https://github.com/matrix-org/synapse/issues/13231))
-- Move the documentation for how URL previews work to the URL preview module. ([\#13233](https://github.com/matrix-org/synapse/issues/13233), [\#13261](https://github.com/matrix-org/synapse/issues/13261))
-- Add another `contrib` script to help set up worker processes. Contributed by @villepeh. ([\#13271](https://github.com/matrix-org/synapse/issues/13271))
-- Document that certain config options were added or changed in Synapse 1.62. Contributed by @behrmann. ([\#13314](https://github.com/matrix-org/synapse/issues/13314))
-- Document the new `rc_invites.per_issuer` throttling option added in Synapse 1.63. ([\#13333](https://github.com/matrix-org/synapse/issues/13333))
-- Mention that BuildKit is needed when building Docker images for tests. ([\#13338](https://github.com/matrix-org/synapse/issues/13338))
-- Improve Caddy reverse proxy documentation. ([\#13344](https://github.com/matrix-org/synapse/issues/13344))
-
-
-Deprecations and Removals
--------------------------
-
-- Drop tables that were formerly used for groups/communities. ([\#12967](https://github.com/matrix-org/synapse/issues/12967))
-- Drop support for delegating email verification to an external server. ([\#13192](https://github.com/matrix-org/synapse/issues/13192))
-- Drop support for calling `/_matrix/client/v3/account/3pid/bind` without an `id_access_token`, which was not permitted by the spec. Contributed by @Vetchu. ([\#13239](https://github.com/matrix-org/synapse/issues/13239))
-- Stop building `.deb` packages for Ubuntu 21.10 (Impish Indri), which has reached end of life. ([\#13326](https://github.com/matrix-org/synapse/issues/13326))
-
-
-Internal Changes
-----------------
-
-- Use lower transaction isolation level when purging rooms to avoid serialization errors. Contributed by Nick @ Beeper. ([\#12942](https://github.com/matrix-org/synapse/issues/12942))
-- Remove code which incorrectly attempted to reconcile state with remote servers when processing incoming events. ([\#12943](https://github.com/matrix-org/synapse/issues/12943))
-- Make the AS login method call `Auth.get_user_by_req` for checking the AS token. ([\#13094](https://github.com/matrix-org/synapse/issues/13094))
-- Always use a version of canonicaljson that supports the C implementation of frozendict. ([\#13172](https://github.com/matrix-org/synapse/issues/13172))
-- Add prometheus counters for ephemeral events and to device messages pushed to app services. Contributed by Brad @ Beeper. ([\#13175](https://github.com/matrix-org/synapse/issues/13175))
-- Refactor receipts servlet logic to avoid duplicated code. ([\#13198](https://github.com/matrix-org/synapse/issues/13198))
-- Preparation for database schema simplifications: populate `state_key` and `rejection_reason` for existing rows in the `events` table. ([\#13215](https://github.com/matrix-org/synapse/issues/13215))
-- Remove unused database table `event_reference_hashes`. ([\#13218](https://github.com/matrix-org/synapse/issues/13218))
-- Further reduce queries used sending events when creating new rooms. Contributed by Nick @ Beeper (@fizzadar). ([\#13224](https://github.com/matrix-org/synapse/issues/13224))
-- Call the v2 identity service `/3pid/unbind` endpoint, rather than v1. Contributed by @Vetchu. ([\#13240](https://github.com/matrix-org/synapse/issues/13240))
-- Use an asynchronous cache wrapper for the get event cache. Contributed by Nick @ Beeper (@fizzadar). ([\#13242](https://github.com/matrix-org/synapse/issues/13242), [\#13308](https://github.com/matrix-org/synapse/issues/13308))
-- Optimise federation sender and appservice pusher event stream processing queries. Contributed by Nick @ Beeper (@fizzadar). ([\#13251](https://github.com/matrix-org/synapse/issues/13251))
-- Log the stack when waiting for an entire room to be un-partial stated. ([\#13257](https://github.com/matrix-org/synapse/issues/13257))
-- Fix spurious warning when fetching state after a missing prev event. ([\#13258](https://github.com/matrix-org/synapse/issues/13258))
-- Clean-up tests for notifications. ([\#13260](https://github.com/matrix-org/synapse/issues/13260))
-- Do not fail build if complement with workers fails. ([\#13266](https://github.com/matrix-org/synapse/issues/13266))
-- Don't pull out state in `compute_event_context` for unconflicted state. ([\#13267](https://github.com/matrix-org/synapse/issues/13267), [\#13274](https://github.com/matrix-org/synapse/issues/13274))
-- Reduce the rebuild time for the complement-synapse docker image. ([\#13279](https://github.com/matrix-org/synapse/issues/13279))
-- Don't pull out the full state when creating an event. ([\#13281](https://github.com/matrix-org/synapse/issues/13281), [\#13307](https://github.com/matrix-org/synapse/issues/13307))
-- Upgrade from Poetry 1.1.12 to 1.1.14, to fix bugs when locking packages. ([\#13285](https://github.com/matrix-org/synapse/issues/13285))
-- Make `DictionaryCache` expire full entries if they haven't been queried in a while, even if specific keys have been queried recently. ([\#13292](https://github.com/matrix-org/synapse/issues/13292))
-- Use `HTTPStatus` constants in place of literals in tests. ([\#13297](https://github.com/matrix-org/synapse/issues/13297))
-- Improve performance of query  `_get_subset_users_in_room_with_profiles`. ([\#13299](https://github.com/matrix-org/synapse/issues/13299))
-- Up batch size of `bulk_get_push_rules` and `_get_joined_profiles_from_event_ids`. ([\#13300](https://github.com/matrix-org/synapse/issues/13300))
-- Remove unnecessary `json.dumps` from tests. ([\#13303](https://github.com/matrix-org/synapse/issues/13303))
-- Reduce memory usage of sending dummy events. ([\#13310](https://github.com/matrix-org/synapse/issues/13310))
-- Prevent formatting changes of [#3679](https://github.com/matrix-org/synapse/pull/3679) from appearing in `git blame`. ([\#13311](https://github.com/matrix-org/synapse/issues/13311))
-- Change `get_users_in_room` and `get_rooms_for_user` caches to enable pruning of old entries. ([\#13313](https://github.com/matrix-org/synapse/issues/13313))
-- Validate federation destinations and log an error if a destination is invalid. ([\#13318](https://github.com/matrix-org/synapse/issues/13318))
-- Fix `FederationClient.get_pdu()` returning events from the cache as `outliers` instead of original events we saw over federation. ([\#13320](https://github.com/matrix-org/synapse/issues/13320))
-- Reduce memory usage of state caches. ([\#13323](https://github.com/matrix-org/synapse/issues/13323))
-- Reduce the amount of state we store in the `state_cache`. ([\#13324](https://github.com/matrix-org/synapse/issues/13324))
-- Add missing type hints to open tracing module. ([\#13328](https://github.com/matrix-org/synapse/issues/13328), [\#13345](https://github.com/matrix-org/synapse/issues/13345), [\#13362](https://github.com/matrix-org/synapse/issues/13362))
-- Remove old base slaved store and de-duplicate cache ID generators. Contributed by Nick @ Beeper (@fizzadar). ([\#13329](https://github.com/matrix-org/synapse/issues/13329), [\#13349](https://github.com/matrix-org/synapse/issues/13349))
-- When reporting metrics is enabled, use ~8x less data to describe DB transaction metrics. ([\#13342](https://github.com/matrix-org/synapse/issues/13342))
-- Faster room joins: skip soft fail checks while Synapse only has partial room state, since the current membership of event senders may not be accurately known. ([\#13354](https://github.com/matrix-org/synapse/issues/13354))
-
-
-Synapse 1.63.1 (2022-07-20)
-===========================
-
-Bugfixes
---------
-
-- Fix a bug introduced in Synapse 1.63.0 where push actions were incorrectly calculated for appservice users. This caused performance issues on servers with large numbers of appservices. ([\#13332](https://github.com/matrix-org/synapse/issues/13332))
-
-
-Synapse 1.63.0 (2022-07-19)
-===========================
-
-Improved Documentation
-----------------------
-
-- Clarify that homeserver server names are included in the reported data when the `report_stats` config option is enabled. ([\#13321](https://github.com/matrix-org/synapse/issues/13321))
-
-
-Synapse 1.63.0rc1 (2022-07-12)
-==============================
-
-Features
---------
-
-- Add a rate limit for local users sending invites. ([\#13125](https://github.com/matrix-org/synapse/issues/13125))
-- Implement [MSC3827](https://github.com/matrix-org/matrix-spec-proposals/pull/3827): Filtering of `/publicRooms` by room type. ([\#13031](https://github.com/matrix-org/synapse/issues/13031))
-- Improve validation logic in the account data REST endpoints. ([\#13148](https://github.com/matrix-org/synapse/issues/13148))
-
-
-Bugfixes
---------
-
-- Fix a long-standing bug where application services were not able to join remote federated rooms without a profile. ([\#13131](https://github.com/matrix-org/synapse/issues/13131))
-- Fix a long-standing bug where `_get_state_map_for_room` might raise errors when third party event rules callbacks are present. ([\#13174](https://github.com/matrix-org/synapse/issues/13174))
-- Fix a long-standing bug where the `synapse_port_db` script could fail to copy rows with negative row ids. ([\#13226](https://github.com/matrix-org/synapse/issues/13226))
-- Fix a bug introduced in 1.54.0 where appservices would not receive room-less EDUs, like presence, when both [MSC2409](https://github.com/matrix-org/matrix-spec-proposals/pull/2409) and [MSC3202](https://github.com/matrix-org/matrix-spec-proposals/pull/3202) are enabled. ([\#13236](https://github.com/matrix-org/synapse/issues/13236))
-- Fix a bug introduced in 1.62.0 where rows were not deleted from `event_push_actions` table on large servers. ([\#13194](https://github.com/matrix-org/synapse/issues/13194))
-- Fix a bug introduced in 1.62.0 where notification counts would get stuck after a highlighted message. ([\#13223](https://github.com/matrix-org/synapse/issues/13223))
-- Fix exception when using experimental [MSC3030](https://github.com/matrix-org/matrix-spec-proposals/pull/3030) `/timestamp_to_event` endpoint to look for remote federated imported events before room creation. ([\#13197](https://github.com/matrix-org/synapse/issues/13197))
-- Fix [MSC3202](https://github.com/matrix-org/matrix-spec-proposals/pull/3202)-enabled appservices not receiving to-device messages, preventing messages from being decrypted. ([\#13235](https://github.com/matrix-org/synapse/issues/13235))
-
-
-Updates to the Docker image
----------------------------
-
-- Bump the version of `lxml` in matrix.org Docker images Debian packages from 4.8.0 to 4.9.1. ([\#13207](https://github.com/matrix-org/synapse/issues/13207))
-
-
-Improved Documentation
-----------------------
-
-- Add an explanation of the `--report-stats` argument to the docs. ([\#13029](https://github.com/matrix-org/synapse/issues/13029))
-- Add a helpful example bash script to the contrib directory for creating multiple worker configuration files of the same type. Contributed by @villepeh. ([\#13032](https://github.com/matrix-org/synapse/issues/13032))
-- Add missing links to config options. ([\#13166](https://github.com/matrix-org/synapse/issues/13166))
-- Add documentation for homeserver usage statistics collection. ([\#13086](https://github.com/matrix-org/synapse/issues/13086))
-- Add documentation for the existing `databases` option in the homeserver configuration manual. ([\#13212](https://github.com/matrix-org/synapse/issues/13212))
-- Clean up references to sample configuration and redirect users to the configuration manual instead. ([\#13077](https://github.com/matrix-org/synapse/issues/13077), [\#13139](https://github.com/matrix-org/synapse/issues/13139))
-- Document how the Synapse team does reviews. ([\#13132](https://github.com/matrix-org/synapse/issues/13132))
-- Fix wrong section header for `allow_public_rooms_over_federation` in the homeserver config documentation. ([\#13116](https://github.com/matrix-org/synapse/issues/13116))
-
-
-Deprecations and Removals
--------------------------
-
-- Remove obsolete and for 8 years unused `RoomEventsStoreTestCase`. Contributed by @arkamar. ([\#13200](https://github.com/matrix-org/synapse/issues/13200))
-
-
-Internal Changes
-----------------
-
-- Add type annotations to `synapse.logging`, `tests.server` and `tests.utils`. ([\#13028](https://github.com/matrix-org/synapse/issues/13028), [\#13103](https://github.com/matrix-org/synapse/issues/13103), [\#13159](https://github.com/matrix-org/synapse/issues/13159), [\#13136](https://github.com/matrix-org/synapse/issues/13136))
-- Enforce type annotations for `tests.test_server`. ([\#13135](https://github.com/matrix-org/synapse/issues/13135))
-- Support temporary experimental return values for spam checker module callbacks. ([\#13044](https://github.com/matrix-org/synapse/issues/13044))
-- Add support to `complement.sh` for skipping the docker build. ([\#13143](https://github.com/matrix-org/synapse/issues/13143), [\#13158](https://github.com/matrix-org/synapse/issues/13158))
-- Add support to `complement.sh` for setting the log level using the `SYNAPSE_TEST_LOG_LEVEL` environment variable. ([\#13152](https://github.com/matrix-org/synapse/issues/13152))
-- Enable Complement testing in the 'Twisted Trunk' CI runs. ([\#13079](https://github.com/matrix-org/synapse/issues/13079), [\#13157](https://github.com/matrix-org/synapse/issues/13157))
-- Improve startup times in Complement test runs against workers, particularly in CPU-constrained environments. ([\#13127](https://github.com/matrix-org/synapse/issues/13127))
-- Update config used by Complement to allow device name lookup over federation. ([\#13167](https://github.com/matrix-org/synapse/issues/13167))
-- Faster room joins: handle race between persisting an event and un-partial stating a room. ([\#13100](https://github.com/matrix-org/synapse/issues/13100))
-- Faster room joins: fix race in recalculation of current room state. ([\#13151](https://github.com/matrix-org/synapse/issues/13151))
-- Faster room joins: skip waiting for full state when processing incoming events over federation. ([\#13144](https://github.com/matrix-org/synapse/issues/13144))
-- Raise a `DependencyError` on missing dependencies instead of a `ConfigError`. ([\#13113](https://github.com/matrix-org/synapse/issues/13113))
-- Avoid stripping line breaks from SQL sent to the database. ([\#13129](https://github.com/matrix-org/synapse/issues/13129))
-- Apply ratelimiting earlier in processing of `/send` requests. ([\#13134](https://github.com/matrix-org/synapse/issues/13134))
-- Improve exception handling when processing events received over federation. ([\#13145](https://github.com/matrix-org/synapse/issues/13145))
-- Check that `auto_vacuum` is disabled when porting a SQLite database to Postgres, as `VACUUM`s must not be performed between runs of the script. ([\#13195](https://github.com/matrix-org/synapse/issues/13195))
-- Reduce DB usage of `/sync` when a large number of unread messages have recently been sent in a room. ([\#13119](https://github.com/matrix-org/synapse/issues/13119), [\#13153](https://github.com/matrix-org/synapse/issues/13153))
-- Reduce memory consumption when processing incoming events in large rooms. ([\#13078](https://github.com/matrix-org/synapse/issues/13078), [\#13222](https://github.com/matrix-org/synapse/issues/13222))
-- Reduce number of queries used to get profile information. Contributed by Nick @ Beeper (@fizzadar). ([\#13209](https://github.com/matrix-org/synapse/issues/13209))
-- Reduce number of events queried during room creation. Contributed by Nick @ Beeper (@fizzadar). ([\#13210](https://github.com/matrix-org/synapse/issues/13210))
-- More aggressively rotate push actions. ([\#13211](https://github.com/matrix-org/synapse/issues/13211))
-- Add `max_line_length` setting for Python files to the `.editorconfig`. Contributed by @sumnerevans @ Beeper. ([\#13228](https://github.com/matrix-org/synapse/issues/13228))
-
 Synapse 1.62.0 (2022-07-05)
 ===========================
 
@@ -285,6 +5,7 @@ No significant changes since 1.62.0rc3.
 
 Authors of spam-checker plugins should consult the [upgrade notes](https://github.com/matrix-org/synapse/blob/release-v1.62/docs/upgrade.md#upgrading-to-v1620) to learn about the enriched signatures for spam checker callbacks, which are supported with this release of Synapse.
 
+
 Synapse 1.62.0rc3 (2022-07-04)
 ==============================
 
@@ -324,7 +45,7 @@ Bugfixes
 - Update [MSC3786](https://github.com/matrix-org/matrix-spec-proposals/pull/3786) implementation to check `state_key`. ([\#12939](https://github.com/matrix-org/synapse/issues/12939))
 - Fix a bug introduced in Synapse 1.58 where Synapse would not report full version information when installed from a git checkout. This is a best-effort affair and not guaranteed to be stable. ([\#12973](https://github.com/matrix-org/synapse/issues/12973))
 - Fix a bug introduced in Synapse 1.60 where Synapse would fail to start if the `sqlite3` module was not available. ([\#12979](https://github.com/matrix-org/synapse/issues/12979))
-- Fix a bug where non-standard information was required when requesting the `/hierarchy` API over federation. Introduced
+- Fix a bug where non-standard information was required when requesting the `/hierarchy` API over federation. Introduced 
   in Synapse v1.41.0. ([\#12991](https://github.com/matrix-org/synapse/issues/12991))
 - Fix a long-standing bug which meant that rate limiting was not restrictive enough in some cases. ([\#13018](https://github.com/matrix-org/synapse/issues/13018))
 - Fix a bug introduced in Synapse 1.58 where profile requests for a malformed user ID would ccause an internal error. Synapse now returns 400 Bad Request in this situation. ([\#13041](https://github.com/matrix-org/synapse/issues/13041))

changelog.d/13028.misc

@@ -0,0 +1 @@
+Add type annotations to `tests.utils`.

changelog.d/13029.doc

@@ -0,0 +1 @@
+Add an explanation of the `--report-stats` argument to the docs.

changelog.d/13031.feature

@@ -0,0 +1 @@
+Implement [MSC3827](https://github.com/matrix-org/matrix-spec-proposals/pull/3827): Filtering of /publicRooms by room type.

changelog.d/13077.doc

@@ -0,0 +1,3 @@
+Clean up references to sample configuration and redirect users to the configuration manual instead.
+
+

changelog.d/13079.misc

@@ -0,0 +1 @@
+Enable Complement testing in the 'Twisted Trunk' CI runs.

changelog.d/13086.doc

@@ -0,0 +1 @@
+Add documentation for anonymised homeserver statistics collection.

changelog.d/13100.misc

@@ -0,0 +1 @@
+Faster room joins: Handle race between persisting an event and un-partial stating a room.

changelog.d/13103.misc

@@ -0,0 +1 @@
+Add missing type hints to `synapse.logging`.

changelog.d/13113.misc

@@ -0,0 +1 @@
+Raise a `DependencyError` on missing dependencies instead of a `ConfigError`.

changelog.d/13116.doc

@@ -0,0 +1 @@
+Fix wrong section header for `allow_public_rooms_over_federation` in the homeserver config documentation.

changelog.d/13119.misc

@@ -0,0 +1 @@
+Reduce DB usage of `/sync` when a large number of unread messages have recently been sent in a room.

changelog.d/13125.feature

@@ -0,0 +1 @@
+Add a rate limit for local users sending invites.

changelog.d/13127.misc

@@ -0,0 +1 @@
+Improve startup times in Complement test runs against workers, particularly in CPU-constrained environments.

changelog.d/13129.misc

@@ -0,0 +1 @@
+Only one-line SQL statements for logging and tracing.

changelog.d/13131.bugfix

@@ -0,0 +1 @@
+Fix application service not being able to join remote federated room without a profile set.

changelog.d/13132.doc

@@ -0,0 +1 @@
+Document how the Synapse team does reviews.

changelog.d/13134.misc

@@ -0,0 +1 @@
+Apply ratelimiting earlier in processing of /send request.

changelog.d/13135.misc

@@ -0,0 +1 @@
+Enforce type annotations for `tests.test_server`.

changelog.d/13136.misc

@@ -0,0 +1 @@
+Add type annotations to `tests.server`.

changelog.d/13139.doc

@@ -0,0 +1 @@
+Add a link to the configuration manual from the homeserver sample config documentation.

changelog.d/13143.misc

@@ -0,0 +1 @@
+Add support to `complement.sh` for skipping the docker build.

changelog.d/13144.misc

@@ -0,0 +1 @@
+Faster joins: skip waiting for full state when processing incoming events over federation.

changelog.d/13145.misc

@@ -0,0 +1 @@
+Improve exception handling when processing events received over federation.

changelog.d/13148.feature

@@ -0,0 +1 @@
+Improve validation logic in Synapse's REST endpoints.

changelog.d/13151.misc

@@ -0,0 +1 @@
+Faster room joins: fix race in recalculation of current room state.

changelog.d/13152.misc

@@ -0,0 +1 @@
+Add the ability to set the log level using the `SYNAPSE_TEST_LOG_LEVEL` environment when using `complement.sh`.

changelog.d/13153.misc

@@ -0,0 +1 @@
+Reduce DB usage of `/sync` when a large number of unread messages have recently been sent in a room.

changelog.d/13157.misc

@@ -0,0 +1 @@
+Enable Complement testing in the 'Twisted Trunk' CI runs.

changelog.d/13158.misc

@@ -0,0 +1 @@
+Add support to `complement.sh` for skipping the docker build.

changelog.d/13159.misc

@@ -0,0 +1 @@
+Improve and fix type hints.

changelog.d/13166.doc

@@ -0,0 +1 @@
+Add missing links to config options.

changelog.d/13167.misc

@@ -0,0 +1 @@
+Update config used by Complement to allow device name lookup over federation.

changelog.d/13174.bugfix

@@ -0,0 +1 @@
+Make use of the more robust `get_current_state` in `_get_state_map_for_room` to avoid breakages.

changelog.d/13194.bugfix

@@ -0,0 +1 @@
+Fix bug where rows were not deleted from `event_push_actions` table on large servers. Introduced in v1.62.0.

changelog.d/13195.misc

@@ -0,0 +1 @@
+Check that `auto_vacuum` is disabled when porting a SQLite database to Postgres, as `VACUUM`s must not be performed between runs of the script.

changelog.d/13200.removal

@@ -0,0 +1 @@
+Remove obsolete and for 8 years unused `RoomEventsStoreTestCase`. Contributed by @arkamar.

changelog.d/13207.docker

@@ -0,0 +1 @@
+Bump the version of `lxml` in matrix.org Docker images Debian packages from 4.8.0 to 4.9.1.

changelog.d/13209.misc

@@ -0,0 +1 @@
+Reduce number of queries used to get profile information. Contributed by Nick @ Beeper (@fizzadar).

changelog.d/13453.misc

@@ -1 +0,0 @@
-Allow use of both `@trace` and `@tag_args` stacked on the same function (tracing).

changelog.d/13471.misc

@@ -1 +0,0 @@
-Clean-up tests for notifications.

changelog.d/13474.misc

@@ -1 +0,0 @@
-Add some miscellaneous comments to document sync, especially around `compute_state_delta`.

changelog.d/13479.misc

@@ -1 +0,0 @@
-Use literals in place of `HTTPStatus` constants in tests.

contrib/workers-bash-scripts/create-multiple-generic-workers.md

@@ -1,31 +0,0 @@
-# Creating multiple generic workers with a bash script
-
-Setting up multiple worker configuration files manually can be time-consuming.
-You can alternatively create multiple worker configuration files with a simple `bash` script. For example:
-
-```sh
-#!/bin/bash
-for i in {1..5}
-do
-cat << EOF >> generic_worker$i.yaml
-worker_app: synapse.app.generic_worker
-worker_name: generic_worker$i
-
-# The replication listener on the main synapse process.
-worker_replication_host: 127.0.0.1
-worker_replication_http_port: 9093
-
-worker_listeners:
-  - type: http
-    port: 808$i
-    resources:
-      - names: [client, federation]
-
-worker_log_config: /etc/matrix-synapse/generic-worker-log.yaml
-EOF
-done
-```
-
-This would create five generic workers with a unique `worker_name` field in each file and listening on ports 8081-8085.
-
-Customise the script to your needs.

contrib/workers-bash-scripts/create-multiple-stream-writers.md

@@ -1,145 +0,0 @@
-# Creating multiple stream writers with a bash script
-
-This script creates multiple [stream writer](https://github.com/matrix-org/synapse/blob/develop/docs/workers.md#stream-writers) workers.
-
-Stream writers require both replication and HTTP listeners.
-
-It also prints out the example lines for Synapse main configuration file.
-
-Remember to route necessary endpoints directly to a worker associated with it.
-
-If you run the script as-is, it will create workers with the replication listener starting from port 8034 and another, regular http listener starting from 8044. If you don't need all of the stream writers listed in the script, just remove them from the ```STREAM_WRITERS``` array.
-
-```sh
-#!/bin/bash
-
-# Start with these replication and http ports.
-# The script loop starts with the exact port and then increments it by one.
-REP_START_PORT=8034
-HTTP_START_PORT=8044
-
-# Stream writer workers to generate. Feel free to add or remove them as you wish.
-# Event persister ("events") isn't included here as it does not require its
-# own HTTP listener.
-
-STREAM_WRITERS+=( "presence" "typing" "receipts" "to_device" "account_data" )
-
-NUM_WRITERS=$(expr ${#STREAM_WRITERS[@]})
-
-i=0
-
-while [ $i -lt "$NUM_WRITERS" ]
-do
-cat << EOF > ${STREAM_WRITERS[$i]}_stream_writer.yaml
-worker_app: synapse.app.generic_worker
-worker_name: ${STREAM_WRITERS[$i]}_stream_writer
-
-# The replication listener on the main synapse process.
-worker_replication_host: 127.0.0.1
-worker_replication_http_port: 9093
-
-worker_listeners:
-  - type: http
-    port: $(expr $REP_START_PORT + $i)
-    resources:
-      - names: [replication]
-
-  - type: http
-    port: $(expr $HTTP_START_PORT + $i)
-    resources:
-      - names: [client]
-
-worker_log_config: /etc/matrix-synapse/stream-writer-log.yaml
-EOF
-HOMESERVER_YAML_INSTANCE_MAP+=$"  ${STREAM_WRITERS[$i]}_stream_writer:
-    host: 127.0.0.1
-    port: $(expr $REP_START_PORT + $i)
-"
-
-HOMESERVER_YAML_STREAM_WRITERS+=$"  ${STREAM_WRITERS[$i]}: ${STREAM_WRITERS[$i]}_stream_writer
-"
-
-((i++))
-done
-
-cat << EXAMPLECONFIG
-# Add these lines to your homeserver.yaml.
-# Don't forget to configure your reverse proxy and
-# necessary endpoints to their respective worker.
-
-# See https://github.com/matrix-org/synapse/blob/develop/docs/workers.md
-# for more information.
-
-# Remember: Under NO circumstances should the replication
-# listener be exposed to the public internet;
-# it has no authentication and is unencrypted.
-
-instance_map:
-$HOMESERVER_YAML_INSTANCE_MAP
-stream_writers:
-$HOMESERVER_YAML_STREAM_WRITERS
-EXAMPLECONFIG
-```
-
-Copy the code above save it to a file ```create_stream_writers.sh``` (for example).
-
-Make the script executable by running ```chmod +x create_stream_writers.sh```.
-
-## Run the script to create workers and print out a sample configuration
-
-Simply run the script to create YAML files in the current folder and print out the required configuration for ```homeserver.yaml```.
-
-```console
-$ ./create_stream_writers.sh
-
-# Add these lines to your homeserver.yaml.
-# Don't forget to configure your reverse proxy and
-# necessary endpoints to their respective worker.
-
-# See https://github.com/matrix-org/synapse/blob/develop/docs/workers.md
-# for more information
-
-# Remember: Under NO circumstances should the replication
-# listener be exposed to the public internet;
-# it has no authentication and is unencrypted.
-
-instance_map:
-  presence_stream_writer:
-    host: 127.0.0.1
-    port: 8034
-  typing_stream_writer:
-    host: 127.0.0.1
-    port: 8035
-  receipts_stream_writer:
-    host: 127.0.0.1
-    port: 8036
-  to_device_stream_writer:
-    host: 127.0.0.1
-    port: 8037
-  account_data_stream_writer:
-    host: 127.0.0.1
-    port: 8038
-
-stream_writers:
-  presence: presence_stream_writer
-  typing: typing_stream_writer
-  receipts: receipts_stream_writer
-  to_device: to_device_stream_writer
-  account_data: account_data_stream_writer
-```
-
-Simply copy-and-paste the output to an appropriate place in your Synapse main configuration file.
-
-## Write directly to Synapse configuration file
-
-You could also write the output directly to homeserver main configuration file. **This, however, is not recommended** as even a small typo (such as replacing >> with >) can erase the entire ```homeserver.yaml```. 
-
-If you do this, back up your original configuration file first:
-
-```console
-# Back up homeserver.yaml first
-cp /etc/matrix-synapse/homeserver.yaml /etc/matrix-synapse/homeserver.yaml.bak 
-
-# Create workers and write output to your homeserver.yaml
-./create_stream_writers.sh >> /etc/matrix-synapse/homeserver.yaml 
-```

debian/changelog

@@ -1,47 +1,3 @@
-matrix-synapse-py3 (1.65.0~rc1) stable; urgency=medium
-
-  * New Synapse release 1.65.0rc1.
-
- -- Synapse Packaging team <packages@matrix.org>  Tue, 09 Aug 2022 11:39:29 +0100
-
-matrix-synapse-py3 (1.64.0) stable; urgency=medium
-
-  * New Synapse release 1.64.0.
-
- -- Synapse Packaging team <packages@matrix.org>  Tue, 02 Aug 2022 10:32:30 +0100
-
-matrix-synapse-py3 (1.64.0~rc2) stable; urgency=medium
-
-  * New Synapse release 1.64.0rc2.
-
- -- Synapse Packaging team <packages@matrix.org>  Fri, 29 Jul 2022 12:22:53 +0100
-
-matrix-synapse-py3 (1.64.0~rc1) stable; urgency=medium
-
-  * New Synapse release 1.64.0rc1.
-
- -- Synapse Packaging team <packages@matrix.org>  Tue, 26 Jul 2022 12:11:49 +0100
-
-matrix-synapse-py3 (1.63.1) stable; urgency=medium
-
-  * New Synapse release 1.63.1.
-
- -- Synapse Packaging team <packages@matrix.org>  Wed, 20 Jul 2022 13:36:52 +0100
-
-matrix-synapse-py3 (1.63.0) stable; urgency=medium
-
-  * Clarify that homeserver server names are included in the data reported
-    by opt-in server stats reporting (`report_stats` homeserver config option).
-  * New Synapse release 1.63.0.
-
- -- Synapse Packaging team <packages@matrix.org>  Tue, 19 Jul 2022 14:42:24 +0200
-
-matrix-synapse-py3 (1.63.0~rc1) stable; urgency=medium
-
-  * New Synapse release 1.63.0rc1.
-
- -- Synapse Packaging team <packages@matrix.org>  Tue, 12 Jul 2022 11:26:02 +0100
-
 matrix-synapse-py3 (1.62.0) stable; urgency=medium
 
   * New Synapse release 1.62.0.

debian/matrix-synapse-py3.postinst

@@ -31,7 +31,7 @@ EOF
 # This file is autogenerated, and will be recreated on upgrade if it is deleted.
 # Any changes you make will be preserved.
 
-# Whether to report homeserver usage statistics.
+# Whether to report anonymized homeserver usage statistics.
 report_stats: false
 EOF
     fi

debian/po/templates.pot

@@ -37,7 +37,7 @@ msgstr ""
 #. Type: boolean
 #. Description
 #: ../templates:2001
-msgid "Report homeserver usage statistics?"
+msgid "Report anonymous statistics?"
 msgstr ""
 
 #. Type: boolean
@@ -45,11 +45,11 @@ msgstr ""
 #: ../templates:2001
 msgid ""
 "Developers of Matrix and Synapse really appreciate helping the project out "
-"by reporting homeserver usage statistics from this homeserver. Your "
-"homeserver's server name, along with very basic aggregate data (e.g. "
-"number of users) will be reported. But it helps track the growth of the "
-"Matrix community, and helps in making Matrix a success, as well as to "
-"convince other networks that they should peer with Matrix."
+"by reporting anonymized usage statistics from this homeserver. Only very "
+"basic aggregate data (e.g. number of users) will be reported, but it helps "
+"track the growth of the Matrix community, and helps in making Matrix a "
+"success, as well as to convince other networks that they should peer with "
+"Matrix."
 msgstr ""
 
 #. Type: boolean

debian/templates

@@ -10,13 +10,12 @@ _Description: Name of the server:
 Template: matrix-synapse/report-stats
 Type: boolean
 Default: false
-_Description: Report homeserver usage statistics?
+_Description: Report anonymous statistics?
  Developers of Matrix and Synapse really appreciate helping the
- project out by reporting homeserver usage statistics from this
- homeserver. Your homeserver's server name, along with very basic
- aggregate data (e.g. number of users) will be reported. But it
- helps track the growth of the Matrix community, and helps in
- making Matrix a success, as well as to convince other networks
- that they should peer with Matrix.
+ project out by reporting anonymized usage statistics from this
+ homeserver. Only very basic aggregate data (e.g. number of users)
+ will be reported, but it helps track the growth of the Matrix
+ community, and helps in making Matrix a success, as well as to
+ convince other networks that they should peer with Matrix.
  .
  Thank you.

docker/Dockerfile

@@ -40,13 +40,12 @@ FROM docker.io/python:${PYTHON_VERSION}-slim as requirements
 RUN \
    --mount=type=cache,target=/var/cache/apt,sharing=locked \
    --mount=type=cache,target=/var/lib/apt,sharing=locked \
-    apt-get update -qq && apt-get install -yqq \
-      build-essential cargo git libffi-dev libssl-dev \
+ apt-get update -qq && apt-get install -yqq git \
     && rm -rf /var/lib/apt/lists/*
 
 # We install poetry in its own build stage to avoid its dependencies conflicting with
 # synapse's dependencies.
-# We use a specific commit from poetry's master branch instead of our usual 1.1.14,
+# We use a specific commit from poetry's master branch instead of our usual 1.1.12,
 # to incorporate fixes to some bugs in `poetry export`. This commit corresponds to
 #    https://github.com/python-poetry/poetry/pull/5156 and
 #    https://github.com/python-poetry/poetry/issues/5141 ;
@@ -69,18 +68,7 @@ COPY pyproject.toml poetry.lock /synapse/
 # reason, such as when a git repository is used directly as a dependency.
 ARG TEST_ONLY_SKIP_DEP_HASH_VERIFICATION
 
-# If specified, we won't use the Poetry lockfile.
-# Instead, we'll just install what a regular `pip install` would from PyPI.
-ARG TEST_ONLY_IGNORE_POETRY_LOCKFILE
-
-# Export the dependencies, but only if we're actually going to use the Poetry lockfile.
-# Otherwise, just create an empty requirements file so that the Dockerfile can
-# proceed.
-RUN if [ -z "$TEST_ONLY_IGNORE_POETRY_LOCKFILE" ]; then \
-    /root/.local/bin/poetry export --extras all -o /synapse/requirements.txt ${TEST_ONLY_SKIP_DEP_HASH_VERIFICATION:+--without-hashes}; \
-  else \
-    touch /synapse/requirements.txt; \
-  fi
+RUN /root/.local/bin/poetry export --extras all -o /synapse/requirements.txt ${TEST_ONLY_SKIP_DEP_HASH_VERIFICATION:+--without-hashes}
 
 ###
 ### Stage 1: builder
@@ -120,17 +108,8 @@ COPY synapse /synapse/synapse/
 # ... and what we need to `pip install`.
 COPY pyproject.toml README.rst /synapse/
 
-# Repeat of earlier build argument declaration, as this is a new build stage.
-ARG TEST_ONLY_IGNORE_POETRY_LOCKFILE
-
 # Install the synapse package itself.
-# If we have populated requirements.txt, we don't install any dependencies
-# as we should already have those from the previous `pip install` step.
-RUN if [ -z "$TEST_ONLY_IGNORE_POETRY_LOCKFILE" ]; then \
-    pip install --prefix="/install" --no-deps --no-warn-script-location /synapse[all]; \
-  else \
-    pip install --prefix="/install" --no-warn-script-location /synapse[all]; \
-  fi
+RUN pip install --prefix="/install" --no-deps --no-warn-script-location /synapse
 
 ###
 ### Stage 2: runtime

docker/Dockerfile-workers

@@ -1,62 +1,38 @@
-# syntax=docker/dockerfile:1
-
+# Inherit from the official Synapse docker image
 ARG SYNAPSE_VERSION=latest
-
-# first of all, we create a base image with an nginx which we can copy into the
-# target image. For repeated rebuilds, this is much faster than apt installing
-# each time.
-
-FROM debian:bullseye-slim AS deps_base
-    RUN \
-       --mount=type=cache,target=/var/cache/apt,sharing=locked \
-       --mount=type=cache,target=/var/lib/apt,sharing=locked \
-      apt-get update -qq && \
-      DEBIAN_FRONTEND=noninteractive apt-get install -yqq --no-install-recommends \
-          redis-server nginx-light
-
-# Similarly, a base to copy the redis server from.
-#
-# The redis docker image has fewer dynamic libraries than the debian package,
-# which makes it much easier to copy (but we need to make sure we use an image
-# based on the same debian version as the synapse image, to make sure we get
-# the expected version of libc.
-FROM redis:6-bullseye AS redis_base
-
-# now build the final image, based on the the regular Synapse docker image
 FROM matrixdotorg/synapse:$SYNAPSE_VERSION
 
-    # Install supervisord with pip instead of apt, to avoid installing a second
-    # copy of python.
-    RUN --mount=type=cache,target=/root/.cache/pip \
-        pip install supervisor~=4.2
-    RUN mkdir -p /etc/supervisor/conf.d
+# Install deps
+RUN \
+   --mount=type=cache,target=/var/cache/apt,sharing=locked \
+   --mount=type=cache,target=/var/lib/apt,sharing=locked \
+  apt-get update -qq && \
+  DEBIAN_FRONTEND=noninteractive apt-get install -yqq --no-install-recommends \
+     redis-server nginx-light
 
-    # Copy over redis and nginx
-    COPY --from=redis_base /usr/local/bin/redis-server /usr/local/bin
+# Install supervisord with pip instead of apt, to avoid installing a second
+# copy of python.
+RUN --mount=type=cache,target=/root/.cache/pip \
+    pip install supervisor~=4.2
 
-    COPY --from=deps_base /usr/sbin/nginx /usr/sbin
-    COPY --from=deps_base /usr/share/nginx /usr/share/nginx
-    COPY --from=deps_base /usr/lib/nginx /usr/lib/nginx
-    COPY --from=deps_base /etc/nginx /etc/nginx
-    RUN rm /etc/nginx/sites-enabled/default
-    RUN mkdir /var/log/nginx /var/lib/nginx
-    RUN chown www-data /var/log/nginx /var/lib/nginx
+# Disable the default nginx sites
+RUN rm /etc/nginx/sites-enabled/default
 
-    # Copy Synapse worker, nginx and supervisord configuration template files
-    COPY ./docker/conf-workers/* /conf/
+# Copy Synapse worker, nginx and supervisord configuration template files
+COPY ./docker/conf-workers/* /conf/
 
-    # Copy a script to prefix log lines with the supervisor program name
-    COPY ./docker/prefix-log /usr/local/bin/
+# Copy a script to prefix log lines with the supervisor program name
+COPY ./docker/prefix-log /usr/local/bin/
 
-    # Expose nginx listener port
-    EXPOSE 8080/tcp
+# Expose nginx listener port
+EXPOSE 8080/tcp
 
-    # A script to read environment variables and create the necessary
-    # files to run the desired worker configuration. Will start supervisord.
-    COPY ./docker/configure_workers_and_start.py /configure_workers_and_start.py
-    ENTRYPOINT ["/configure_workers_and_start.py"]
+# A script to read environment variables and create the necessary
+# files to run the desired worker configuration. Will start supervisord.
+COPY ./docker/configure_workers_and_start.py /configure_workers_and_start.py
+ENTRYPOINT ["/configure_workers_and_start.py"]
 
-    # Replace the healthcheck with one which checks *all* the workers. The script
-    # is generated by configure_workers_and_start.py.
-    HEALTHCHECK --start-period=5s --interval=15s --timeout=5s \
-        CMD /bin/sh /healthcheck.sh
+# Replace the healthcheck with one which checks *all* the workers. The script
+# is generated by configure_workers_and_start.py.
+HEALTHCHECK --start-period=5s --interval=15s --timeout=5s \
+    CMD /bin/sh /healthcheck.sh

docker/README-testing.md

@@ -22,10 +22,6 @@ Consult the [contributing guide][guideComplementSh] for instructions on how to u
 Under some circumstances, you may wish to build the images manually.
 The instructions below will lead you to doing that.
 
-Note that these images can only be built using [BuildKit](https://docs.docker.com/develop/develop-images/build_enhancements/),
-therefore BuildKit needs to be enabled when calling `docker build`. This can be done by
-setting `DOCKER_BUILDKIT=1` in your environment.
-
 Start by building the base Synapse docker image. If you wish to run tests with the latest
 release of Synapse, instead of your current checkout, you can skip this step. From the
 root of the repository:

docker/complement/Dockerfile

@@ -1,62 +1,45 @@
-# syntax=docker/dockerfile:1
 # This dockerfile builds on top of 'docker/Dockerfile-workers' in matrix-org/synapse
 # by including a built-in postgres instance, as well as setting up the homeserver so
 # that it is ready for testing via Complement.
 #
 # Instructions for building this image from those it depends on is detailed in this guide:
 # https://github.com/matrix-org/synapse/blob/develop/docker/README-testing.md#testing-with-postgresql-and-single-or-multi-process-synapse
-
 ARG SYNAPSE_VERSION=latest
-
-# first of all, we create a base image with a postgres server and database,
-# which we can copy into the target image. For repeated rebuilds, this is
-# much faster than apt installing postgres each time.
-#
-# This trick only works because (a) the Synapse image happens to have all the
-# shared libraries that postgres wants, (b) we use a postgres image based on
-# the same debian version as Synapse's docker image (so the versions of the
-# shared libraries match).
-
-FROM postgres:13-bullseye AS postgres_base
-    # initialise the database cluster in /var/lib/postgresql
-    RUN gosu postgres initdb --locale=C --encoding=UTF-8 --auth-host password
-
-    # Configure a password and create a database for Synapse
-    RUN echo "ALTER USER postgres PASSWORD 'somesecret'" | gosu postgres postgres --single
-    RUN echo "CREATE DATABASE synapse" | gosu postgres postgres --single
-
-# now build the final image, based on the Synapse image.
-
 FROM matrixdotorg/synapse-workers:$SYNAPSE_VERSION
-    # copy the postgres installation over from the image we built above
-    RUN adduser --system --uid 999 postgres --home /var/lib/postgresql
-    COPY --from=postgres_base /var/lib/postgresql /var/lib/postgresql
-    COPY --from=postgres_base /usr/lib/postgresql /usr/lib/postgresql
-    COPY --from=postgres_base /usr/share/postgresql /usr/share/postgresql
-    RUN mkdir /var/run/postgresql && chown postgres /var/run/postgresql
-    ENV PATH="${PATH}:/usr/lib/postgresql/13/bin"
-    ENV PGDATA=/var/lib/postgresql/data
 
-    # Extend the shared homeserver config to disable rate-limiting,
-    # set Complement's static shared secret, enable registration, amongst other
-    # tweaks to get Synapse ready for testing.
-    # To do this, we copy the old template out of the way and then include it
-    # with Jinja2.
-    RUN mv /conf/shared.yaml.j2 /conf/shared-orig.yaml.j2
-    COPY conf/workers-shared-extra.yaml.j2 /conf/shared.yaml.j2
+# Install postgresql
+RUN apt-get update && \
+  DEBIAN_FRONTEND=noninteractive apt-get install --no-install-recommends -yqq postgresql-13
 
-    WORKDIR /data
+# Configure a user and create a database for Synapse
+RUN pg_ctlcluster 13 main start &&  su postgres -c "echo \
+ \"ALTER USER postgres PASSWORD 'somesecret'; \
+ CREATE DATABASE synapse \
+  ENCODING 'UTF8' \
+  LC_COLLATE='C' \
+  LC_CTYPE='C' \
+  template=template0;\" | psql" && pg_ctlcluster 13 main stop
 
-    COPY conf/postgres.supervisord.conf /etc/supervisor/conf.d/postgres.conf
+# Extend the shared homeserver config to disable rate-limiting,
+# set Complement's static shared secret, enable registration, amongst other
+# tweaks to get Synapse ready for testing.
+# To do this, we copy the old template out of the way and then include it
+# with Jinja2.
+RUN mv /conf/shared.yaml.j2 /conf/shared-orig.yaml.j2
+COPY conf/workers-shared-extra.yaml.j2 /conf/shared.yaml.j2
 
-    # Copy the entrypoint
-    COPY conf/start_for_complement.sh /
+WORKDIR /data
 
-    # Expose nginx's listener ports
-    EXPOSE 8008 8448
+COPY conf/postgres.supervisord.conf /etc/supervisor/conf.d/postgres.conf
 
-    ENTRYPOINT ["/start_for_complement.sh"]
+# Copy the entrypoint
+COPY conf/start_for_complement.sh /
 
-    # Update the healthcheck to have a shorter check interval
-    HEALTHCHECK --start-period=5s --interval=1s --timeout=1s \
-        CMD /bin/sh /healthcheck.sh
+# Expose nginx's listener ports
+EXPOSE 8008 8448
+
+ENTRYPOINT ["/start_for_complement.sh"]
+
+# Update the healthcheck to have a shorter check interval
+HEALTHCHECK --start-period=5s --interval=1s --timeout=1s \
+    CMD /bin/sh /healthcheck.sh

docker/complement/conf/postgres.supervisord.conf

@@ -1,5 +1,5 @@
 [program:postgres]
-command=/usr/local/bin/prefix-log gosu postgres postgres
+command=/usr/local/bin/prefix-log /usr/bin/pg_ctlcluster 13 main start --foreground
 
 # Only start if START_POSTGRES=1
 autostart=%(ENV_START_POSTGRES)s

docker/complement/conf/workers-shared-extra.yaml.j2

@@ -67,10 +67,6 @@ rc_joins:
     per_second: 9999
     burst_count: 9999
 
-rc_joins_per_room:
-    per_second: 9999
-    burst_count: 9999
-
 rc_3pid_validation:
   per_second: 1000
   burst_count: 1000

docker/conf-workers/supervisord.conf.j2

@@ -19,7 +19,7 @@ username=www-data
 autorestart=true
 
 [program:redis]
-command=/usr/local/bin/prefix-log /usr/local/bin/redis-server
+command=/usr/local/bin/prefix-log /usr/bin/redis-server /etc/redis/redis.conf --daemonize no
 priority=1
 stdout_logfile=/dev/stdout
 stdout_logfile_maxbytes=0

docs/SUMMARY.md

@@ -35,6 +35,7 @@
     - [Application Services](application_services.md)
     - [Server Notices](server_notices.md)
     - [Consent Tracking](consent_tracking.md)
+    - [URL Previews](development/url_previews.md)
     - [User Directory](user_directory.md)
     - [Message Retention Policies](message_retention_policies.md)
     - [Pluggable Modules](modules/index.md)
@@ -68,7 +69,7 @@
       - [Federation](usage/administration/admin_api/federation.md)
     - [Manhole](manhole.md)
     - [Monitoring](metrics-howto.md)
-      - [Reporting Homeserver Usage Statistics](usage/administration/monitoring/reporting_homeserver_usage_statistics.md)
+      - [Reporting Anonymised Statistics](usage/administration/monitoring/reporting_anonymised_statistics.md)
     - [Understanding Synapse Through Grafana Graphs](usage/administration/understanding_synapse_through_grafana_graphs.md)
     - [Useful SQL for Admins](usage/administration/useful_sql_for_admins.md)
     - [Database Maintenance Tools](usage/administration/database_maintenance_tools.md)

docs/admin_api/rooms.md

@@ -59,7 +59,6 @@ The following fields are possible in the JSON response body:
     - `guest_access` - Whether guests can join the room. One of: ["can_join", "forbidden"].
     - `history_visibility` - Who can see the room history. One of: ["invited", "joined", "shared", "world_readable"].
     - `state_events` - Total number of state_events of a room. Complexity of the room.
-    - `room_type` - The type of the room taken from the room's creation event; for example "m.space" if the room is a space. If the room does not define a type, the value will be `null`.
 * `offset` - The current pagination offset in rooms. This parameter should be
              used instead of `next_token` for room offset as `next_token` is
              not intended to be parsed.
@@ -102,8 +101,7 @@ A response body like the following is returned:
       "join_rules": "invite",
       "guest_access": null,
       "history_visibility": "shared",
-      "state_events": 93534,
-      "room_type": "m.space"
+      "state_events": 93534
     },
     ... (8 hidden items) ...
     {
@@ -120,8 +118,7 @@ A response body like the following is returned:
       "join_rules": "invite",
       "guest_access": null,
       "history_visibility": "shared",
-      "state_events": 8345,
-      "room_type": null
+      "state_events": 8345
     }
   ],
   "offset": 0,
@@ -154,8 +151,7 @@ A response body like the following is returned:
       "join_rules": "invite",
       "guest_access": null,
       "history_visibility": "shared",
-      "state_events": 8,
-      "room_type": null
+      "state_events": 8
     }
   ],
   "offset": 0,
@@ -188,8 +184,7 @@ A response body like the following is returned:
       "join_rules": "invite",
       "guest_access": null,
       "history_visibility": "shared",
-      "state_events": 93534,
-      "room_type": null
+      "state_events": 93534
     },
     ... (98 hidden items) ...
     {
@@ -206,8 +201,7 @@ A response body like the following is returned:
       "join_rules": "invite",
       "guest_access": null,
       "history_visibility": "shared",
-      "state_events": 8345,
-      "room_type": "m.space"
+      "state_events": 8345
     }
   ],
   "offset": 0,
@@ -244,9 +238,7 @@ A response body like the following is returned:
       "join_rules": "invite",
       "guest_access": null,
       "history_visibility": "shared",
-      "state_events": 93534,
-      "room_type": "m.space"
-
+      "state_events": 93534
     },
     ... (48 hidden items) ...
     {
@@ -263,9 +255,7 @@ A response body like the following is returned:
       "join_rules": "invite",
       "guest_access": null,
       "history_visibility": "shared",
-      "state_events": 8345,
-      "room_type": null
-
+      "state_events": 8345
     }
   ],
   "offset": 100,
@@ -300,8 +290,6 @@ The following fields are possible in the JSON response body:
 * `guest_access` - Whether guests can join the room. One of: ["can_join", "forbidden"].
 * `history_visibility` - Who can see the room history. One of: ["invited", "joined", "shared", "world_readable"].
 * `state_events` - Total number of state_events of a room. Complexity of the room.
-* `room_type` - The type of the room taken from the room's creation event; for example "m.space" if the room is a space.
-  If the room does not define a type, the value will be `null`.
 
 The API is:
 
@@ -329,8 +317,7 @@ A response body like the following is returned:
   "join_rules": "invite",
   "guest_access": null,
   "history_visibility": "shared",
-  "state_events": 93534,
-  "room_type": "m.space"
+  "state_events": 93534
 }
 ```
 

docs/admin_api/user_admin_api.md

@@ -544,7 +544,7 @@ Gets a list of all local media that a specific `user_id` has created.
 These are media that the user has uploaded themselves
 ([local media](../media_repository.md#local-media)), as well as
 [URL preview images](../media_repository.md#url-previews) requested by the user if the
-[feature is enabled](../usage/configuration/config_documentation.md#url_preview_enabled).
+[feature is enabled](../development/url_previews.md).
 
 By default, the response is ordered by descending creation date and ascending media ID.
 The newest media is on top. You can change the order with parameters

docs/development/dependencies.md

@@ -237,28 +237,3 @@ poetry run pip install build && poetry run python -m build
 because [`build`](https://github.com/pypa/build) is a standardish tool which
 doesn't require poetry. (It's what we use in CI too). However, you could try
 `poetry build` too.
-
-
-# Troubleshooting
-
-## Check the version of poetry with `poetry --version`.
-
-At the time of writing, the 1.2 series is beta only. We have seen some examples
-where the lockfiles generated by 1.2 prereleasese aren't interpreted correctly
-by poetry 1.1.x. For now, use poetry 1.1.14, which includes a critical
-[change](https://github.com/python-poetry/poetry/pull/5973) needed to remain
-[compatible with PyPI](https://github.com/pypi/warehouse/pull/11775).
-
-It can also be useful to check the version of `poetry-core` in use. If you've
-installed `poetry` with `pipx`, try `pipx runpip poetry list | grep poetry-core`.
-
-## Clear caches: `poetry cache clear --all pypi`.
-
-Poetry caches a bunch of information about packages that isn't readily available
-from PyPI. (This is what makes poetry seem slow when doing the first
-`poetry install`.) Try `poetry cache list` and `poetry cache clear --all
-<name of cache>` to see if that fixes things.
-
-## Try `--verbose` or `--dry-run` arguments.
-
-Sometimes useful to see what poetry's internal logic is.

docs/development/url_previews.md

@@ -0,0 +1,61 @@
+URL Previews
+============
+
+The `GET /_matrix/media/r0/preview_url` endpoint provides a generic preview API
+for URLs which outputs [Open Graph](https://ogp.me/) responses (with some Matrix
+specific additions).
+
+This does have trade-offs compared to other designs:
+
+* Pros:
+  * Simple and flexible; can be used by any clients at any point
+* Cons:
+  * If each homeserver provides one of these independently, all the HSes in a
+    room may needlessly DoS the target URI
+  * The URL metadata must be stored somewhere, rather than just using Matrix
+    itself to store the media.
+  * Matrix cannot be used to distribute the metadata between homeservers.
+
+When Synapse is asked to preview a URL it does the following:
+
+1. Checks against a URL blacklist (defined as `url_preview_url_blacklist` in the
+   config).
+2. Checks the in-memory cache by URLs and returns the result if it exists. (This
+   is also used to de-duplicate processing of multiple in-flight requests at once.)
+3. Kicks off a background process to generate a preview:
+   1. Checks the database cache by URL and timestamp and returns the result if it
+      has not expired and was successful (a 2xx return code).
+   2. Checks if the URL matches an [oEmbed](https://oembed.com/) pattern. If it
+      does, update the URL to download.
+   3. Downloads the URL and stores it into a file via the media storage provider
+      and saves the local media metadata.
+   4. If the media is an image:
+      1. Generates thumbnails.
+      2. Generates an Open Graph response based on image properties.
+   5. If the media is HTML:
+      1. Decodes the HTML via the stored file.
+      2. Generates an Open Graph response from the HTML.
+      3. If a JSON oEmbed URL was found in the HTML via autodiscovery:
+         1. Downloads the URL and stores it into a file via the media storage provider
+            and saves the local media metadata.
+         2. Convert the oEmbed response to an Open Graph response.
+         3. Override any Open Graph data from the HTML with data from oEmbed.
+      4. If an image exists in the Open Graph response:
+         1. Downloads the URL and stores it into a file via the media storage
+            provider and saves the local media metadata.
+         2. Generates thumbnails.
+         3. Updates the Open Graph response based on image properties.
+   6. If the media is JSON and an oEmbed URL was found:
+      1. Convert the oEmbed response to an Open Graph response.
+      2. If a thumbnail or image is in the oEmbed response:
+         1. Downloads the URL and stores it into a file via the media storage
+            provider and saves the local media metadata.
+         2. Generates thumbnails.
+         3. Updates the Open Graph response based on image properties.
+   7. Stores the result in the database cache.
+4. Returns the result.
+
+The in-memory cache expires after 1 hour.
+
+Expired entries in the database cache (and their associated media files) are
+deleted every 10 seconds. The default expiration time is 1 hour from download.

docs/media_repository.md

@@ -7,7 +7,8 @@ The media repository
    users.
  * caches avatars, attachments and their thumbnails for media uploaded by remote
    users.
- * caches resources and thumbnails used for URL previews.
+ * caches resources and thumbnails used for
+   [URL previews](development/url_previews.md).
 
 All media in Matrix can be identified by a unique
 [MXC URI](https://spec.matrix.org/latest/client-server-api/#matrix-content-mxc-uris),
@@ -58,6 +59,8 @@ remote_thumbnail/matrix.org/aa/bb/cccccccccccccccccccc/128-96-image-jpeg
 Note that `remote_thumbnail/` does not have an `s`.
 
 ## URL Previews
+See [URL Previews](development/url_previews.md) for documentation on the URL preview
+process.
 
 When generating previews for URLs, Synapse may download and cache various
 resources, including images. These resources are assigned temporary media IDs

docs/modules/password_auth_provider_callbacks.md

@@ -263,7 +263,7 @@ class MyAuthProvider:
             return None
 
         if self.credentials.get(username) == login_dict.get("my_field"):
-            return (self.api.get_qualified_user_id(username), None)
+            return self.api.get_qualified_user_id(username)
 
     async def check_pass(
         self,
@@ -280,5 +280,5 @@ class MyAuthProvider:
             return None
 
         if self.credentials.get(username) == login_dict.get("password"):
-            return (self.api.get_qualified_user_id(username), None)
+            return self.api.get_qualified_user_id(username)
 ```

docs/reverse_proxy.md

@@ -79,32 +79,63 @@ server {
 }
 ```
 
+### Caddy v1
+
+```
+matrix.example.com {
+  proxy /_matrix http://localhost:8008 {
+    transparent
+  }
+
+  proxy /_synapse/client http://localhost:8008 {
+    transparent
+  }
+}
+
+example.com:8448 {
+  proxy / http://localhost:8008 {
+    transparent
+  }
+}
+```
+
 ### Caddy v2
 
 ```
 matrix.example.com {
-  reverse_proxy /_matrix/* localhost:8008
-  reverse_proxy /_synapse/client/* localhost:8008
+  reverse_proxy /_matrix/* http://localhost:8008
+  reverse_proxy /_synapse/client/* http://localhost:8008
 }
 
 example.com:8448 {
-  reverse_proxy localhost:8008
+  reverse_proxy http://localhost:8008
 }
 ```
-
 [Delegation](delegate.md) example:
-
 ```
+(matrix-well-known-header) {
+    # Headers
+    header Access-Control-Allow-Origin "*"
+    header Access-Control-Allow-Methods "GET, POST, PUT, DELETE, OPTIONS"
+    header Access-Control-Allow-Headers "Origin, X-Requested-With, Content-Type, Accept, Authorization"
+    header Content-Type "application/json"
+}
+
 example.com {
-	header /.well-known/matrix/* Content-Type application/json
-	header /.well-known/matrix/* Access-Control-Allow-Origin *
-	respond /.well-known/matrix/server `{"m.server": "matrix.example.com:443"}`
-	respond /.well-known/matrix/client `{"m.homeserver":{"base_url":"https://matrix.example.com"},"m.identity_server":{"base_url":"https://identity.example.com"}}`
+    handle /.well-known/matrix/server {
+        import matrix-well-known-header
+        respond `{"m.server":"matrix.example.com:443"}`
+    }
+
+    handle /.well-known/matrix/client {
+        import matrix-well-known-header
+        respond `{"m.homeserver":{"base_url":"https://matrix.example.com"},"m.identity_server":{"base_url":"https://identity.example.com"}}`
+    }
 }
 
 matrix.example.com {
-    reverse_proxy /_matrix/* localhost:8008
-    reverse_proxy /_synapse/client/* localhost:8008
+    reverse_proxy /_matrix/* http://localhost:8008
+    reverse_proxy /_synapse/client/* http://localhost:8008
 }
 ```
 

docs/sso_mapping_providers.md

@@ -22,7 +22,7 @@ choose their own username.
 In the first case - where users are automatically allocated a Matrix ID - it is
 the responsibility of the mapping provider to normalise the SSO attributes and
 map them to a valid Matrix ID. The [specification for Matrix
-IDs](https://spec.matrix.org/latest/appendices/#user-identifiers) has some
+IDs](https://matrix.org/docs/spec/appendices#user-identifiers) has some
 information about what is considered valid.
 
 If the mapping provider does not assign a Matrix ID, then Synapse will
@@ -37,10 +37,9 @@ as Synapse). The Synapse config is then modified to point to the mapping provide
 ## OpenID Mapping Providers
 
 The OpenID mapping provider can be customized by editing the
-[`oidc_providers.user_mapping_provider.module`](usage/configuration/config_documentation.md#oidc_providers)
-config option.
+`oidc_config.user_mapping_provider.module` config option.
 
-`oidc_providers.user_mapping_provider.config` allows you to provide custom
+`oidc_config.user_mapping_provider.config` allows you to provide custom
 configuration options to the module. Check with the module's documentation for
 what options it provides (if any). The options listed by default are for the
 user mapping provider built in to Synapse. If using a custom module, you should
@@ -59,7 +58,7 @@ A custom mapping provider must specify the following methods:
     - This method should have the `@staticmethod` decoration.
     - Arguments:
         - `config` - A `dict` representing the parsed content of the
-          `oidc_providers.user_mapping_provider.config` homeserver config option.
+          `oidc_config.user_mapping_provider.config` homeserver config option.
            Runs on homeserver startup. Providers should extract and validate
            any option values they need here.
     - Whatever is returned will be passed back to the user mapping provider module's
@@ -103,7 +102,7 @@ A custom mapping provider must specify the following methods:
       will be returned as part of the response during a successful login.
 
       Note that care should be taken to not overwrite any of the parameters
-      usually returned as part of the [login response](https://spec.matrix.org/latest/client-server-api/#post_matrixclientv3login).
+      usually returned as part of the [login response](https://matrix.org/docs/spec/client_server/latest#post-matrix-client-r0-login).
 
 ### Default OpenID Mapping Provider
 
@@ -114,8 +113,7 @@ specified in the config. It is located at
 ## SAML Mapping Providers
 
 The SAML mapping provider can be customized by editing the
-[`saml2_config.user_mapping_provider.module`](docs/usage/configuration/config_documentation.md#saml2_config)
-config option.
+`saml2_config.user_mapping_provider.module` config option.
 
 `saml2_config.user_mapping_provider.config` allows you to provide custom
 configuration options to the module. Check with the module's documentation for

docs/upgrade.md

@@ -89,37 +89,6 @@ process, for example:
     dpkg -i matrix-synapse-py3_1.3.0+stretch1_amd64.deb
     ```
 
-# Upgrading to v1.64.0
-
-## Deprecation of the ability to delegate e-mail verification to identity servers
-
-Synapse v1.66.0 will remove the ability to delegate the tasks of verifying email address ownership, and password reset confirmation, to an identity server.
-
-If you require your homeserver to verify e-mail addresses or to support password resets via e-mail, please configure your homeserver with SMTP access so that it can send e-mails on its own behalf.
-[Consult the configuration documentation for more information.](https://matrix-org.github.io/synapse/latest/usage/configuration/config_documentation.html#email)
-
-The option that will be removed is `account_threepid_delegates.email`.
-
-
-## Changes to the event replication streams
-
-Synapse now includes a flag indicating if an event is an outlier when
-replicating it to other workers. This is a forwards- and backwards-incompatible
-change: v1.63 and workers cannot process events replicated by v1.64 workers, and
-vice versa.
-
-Once all workers are upgraded to v1.64 (or downgraded to v1.63), event
-replication will resume as normal.
-
-## frozendict release
-
-[frozendict 2.3.3](https://github.com/Marco-Sulla/python-frozendict/releases/tag/v2.3.3)
-has recently been released, which fixes a memory leak that occurs during `/sync`
-requests. We advise server administrators who installed Synapse via pip to upgrade
-frozendict with `pip install --upgrade frozendict`. The Docker image
-`matrixdotorg/synapse` and the Debian packages from `packages.matrix.org` already
-include the updated library.
-
 # Upgrading to v1.62.0
 
 ## New signatures for spam checker callbacks

docs/usage/administration/admin_api/README.md

@@ -5,9 +5,8 @@
 Many of the API calls in the admin api will require an `access_token` for a
 server admin. (Note that a server admin is distinct from a room admin.)
 
-An existing user can be marked as a server admin by updating the database directly.
+A user can be marked as a server admin by updating the database directly, e.g.:
 
-Check your [database settings](config_documentation.md#database) in the configuration file, connect to the correct database using either `psql [database name]` (if using PostgreSQL) or `sqlite3 path/to/your/database.db` (if using SQLite) and elevate the user `@foo:bar.com` to administrator.
 ```sql
 UPDATE users SET admin = 1 WHERE name = '@foo:bar.com';
 ```
@@ -19,11 +18,6 @@ already on your `$PATH` depending on how Synapse was installed.
 Finding your user's `access_token` is client-dependent, but will usually be shown in the client's settings.
 
 ## Making an Admin API request
-For security reasons, we [recommend](reverse_proxy.md#synapse-administration-endpoints)
-that the Admin API (`/_synapse/admin/...`) should be hidden from public view using a
-reverse proxy. This means you should typically query the Admin API from a terminal on
-the machine which runs Synapse.
-
 Once you have your `access_token`, you will need to authenticate each request to an Admin API endpoint by
 providing the token as either a query parameter or a request header. To add it as a request header in cURL:
 
@@ -31,17 +25,5 @@ providing the token as either a query parameter or a request header. To add it a
 curl --header "Authorization: Bearer <access_token>" <the_rest_of_your_API_request>
 ```
 
-For example, suppose we want to
-[query the account](user_admin_api.md#query-user-account) of the user
-`@foo:bar.com`. We need an admin access token (e.g.
-`syt_AjfVef2_L33JNpafeif_0feKJfeaf0CQpoZk`), and we need to know which port
-Synapse's [`client` listener](config_documentation.md#listeners) is listening
-on (e.g. `8008`). Then we can use the following command to request the account
-information from the Admin API.
-
-```sh
-curl --header "Authorization: Bearer syt_AjfVef2_L33JNpafeif_0feKJfeaf0CQpoZk" -X GET http://127.0.0.1:8008/_synapse/admin/v2/users/@foo:bar.com
-```
-
 For more details on access tokens in Matrix, please refer to the complete
 [matrix spec documentation](https://matrix.org/docs/spec/client_server/r0.6.1#using-access-tokens).

docs/usage/administration/admin_api/registration_tokens.md

@@ -2,11 +2,11 @@
 
 This API allows you to manage tokens which can be used to authenticate
 registration requests, as proposed in
-[MSC3231](https://github.com/matrix-org/matrix-doc/blob/main/proposals/3231-token-authenticated-registration.md)
-and stabilised in version 1.2 of the Matrix specification.
+[MSC3231](https://github.com/matrix-org/matrix-doc/blob/main/proposals/3231-token-authenticated-registration.md).
 To use it, you will need to enable the `registration_requires_token` config
 option, and authenticate by providing an `access_token` for a server admin:
-see [Admin API](../admin_api).
+see [Admin API](../../usage/administration/admin_api).
+Note that this API is still experimental; not all clients may support it yet.
 
 
 ## Registration token objects

docs/usage/administration/monitoring/reporting_anonymised_statistics.md

@@ -1,11 +1,11 @@
-# Reporting Homeserver Usage Statistics
+# Reporting Anonymised Statistics
 
 When generating your Synapse configuration file, you are asked whether you
-would like to report usage statistics to Matrix.org. These statistics
+would like to report anonymised statistics to Matrix.org. These statistics
 provide the foundation a glimpse into the number of Synapse homeservers
 participating in the network, as well as statistics such as the number of
 rooms being created and messages being sent. This feature is sometimes
-affectionately called "phone home" stats. Reporting
+affectionately called "phone-home" stats. Reporting
 [is optional](../../configuration/config_documentation.md#report_stats)
 and the reporting endpoint
 [can be configured](../../configuration/config_documentation.md#report_stats_endpoint),
@@ -21,9 +21,9 @@ The following statistics are sent to the configured reporting endpoint:
 
 | Statistic Name             | Type   | Description                                                                                                                                                                                                                                                                                     |
 |----------------------------|--------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
-| `homeserver`               | string | The homeserver's server name.                                                                                                                                                                                                                                                                   |
 | `memory_rss`               | int    | The memory usage of the process (in kilobytes on Unix-based systems, bytes on MacOS).                                                                                                                                                                                                           |
 | `cpu_average`              | int    | CPU time in % of a single core (not % of all cores).                                                                                                                                                                                                                                            |              
+| `homeserver`               | string | The homeserver's server name.                                                                                                                                                                                                                                                                   |
 | `server_context`           | string | An arbitrary string used to group statistics from a set of homeservers.                                                                                                                                                                                                                         |
 | `timestamp`                | int    | The current time, represented as the number of seconds since the epoch.                                                                                                                                                                                                                         |                 
 | `uptime_seconds`           | int    | The number of seconds since the homeserver was last started.                                                                                                                                                                                                                                    |

mypy.ini

@@ -84,6 +84,9 @@ disallow_untyped_defs = False
 [mypy-synapse.http.matrixfederationclient]
 disallow_untyped_defs = False
 
+[mypy-synapse.logging.opentracing]
+disallow_untyped_defs = False
+
 [mypy-synapse.metrics._reactor_metrics]
 disallow_untyped_defs = False
 # This module imports select.epoll. That exists on Linux, but doesn't on macOS.

poetry.lock

@@ -177,7 +177,7 @@ optional = false
 python-versions = "*"
 
 [package.extras]
-test = ["hypothesis (==3.55.3)", "flake8 (==3.7.8)"]
+test = ["flake8 (==3.7.8)", "hypothesis (==3.55.3)"]
 
 [[package]]
 name = "constantly"
@@ -290,7 +290,7 @@ importlib-metadata = {version = "*", markers = "python_version < \"3.8\""}
 
 [[package]]
 name = "frozendict"
-version = "2.3.3"
+version = "2.3.0"
 description = "A simple immutable dictionary"
 category = "main"
 optional = false
@@ -435,8 +435,8 @@ optional = false
 python-versions = ">=3.6"
 
 [package.extras]
-trio = ["async-generator", "trio"]
-test = ["async-timeout", "trio", "testpath", "pytest-asyncio", "pytest-trio", "pytest"]
+test = ["pytest", "pytest-trio", "pytest-asyncio", "testpath", "trio", "async-timeout"]
+trio = ["trio", "async-generator"]
 
 [[package]]
 name = "jinja2"
@@ -535,12 +535,12 @@ attrs = "*"
 importlib-metadata = {version = ">=1.4", markers = "python_version < \"3.8\""}
 
 [package.extras]
-test = ["aiounittest", "twisted", "tox"]
-dev = ["twine (==4.0.1)", "build (==0.8.0)", "isort (==5.9.3)", "flake8 (==4.0.1)", "black (==22.3.0)", "mypy (==0.910)", "aiounittest", "twisted", "tox"]
+dev = ["tox", "twisted", "aiounittest", "mypy (==0.910)", "black (==22.3.0)", "flake8 (==4.0.1)", "isort (==5.9.3)", "build (==0.8.0)", "twine (==4.0.1)"]
+test = ["tox", "twisted", "aiounittest"]
 
 [[package]]
 name = "matrix-synapse-ldap3"
-version = "0.2.2"
+version = "0.2.1"
 description = "An LDAP3 auth provider for Synapse"
 category = "main"
 optional = true
@@ -552,7 +552,7 @@ service-identity = "*"
 Twisted = ">=15.1.0"
 
 [package.extras]
-dev = ["isort (==5.9.3)", "flake8 (==4.0.1)", "black (==22.3.0)", "types-setuptools", "mypy (==0.910)", "ldaptor", "tox", "matrix-synapse"]
+dev = ["matrix-synapse", "tox", "ldaptor", "mypy (==0.910)", "types-setuptools", "black (==22.3.0)", "flake8 (==4.0.1)", "isort (==5.9.3)"]
 
 [[package]]
 name = "mccabe"
@@ -820,10 +820,10 @@ optional = false
 python-versions = ">=3.6"
 
 [package.extras]
-tests = ["coverage[toml] (==5.0.4)", "pytest (>=6.0.0,<7.0.0)"]
-docs = ["zope.interface", "sphinx-rtd-theme", "sphinx"]
-dev = ["pre-commit", "mypy", "coverage[toml] (==5.0.4)", "pytest (>=6.0.0,<7.0.0)", "cryptography (>=3.3.1)", "zope.interface", "sphinx-rtd-theme", "sphinx"]
 crypto = ["cryptography (>=3.3.1)"]
+dev = ["sphinx", "sphinx-rtd-theme", "zope.interface", "cryptography (>=3.3.1)", "pytest (>=6.0.0,<7.0.0)", "coverage[toml] (==5.0.4)", "mypy", "pre-commit"]
+docs = ["sphinx", "sphinx-rtd-theme", "zope.interface"]
+tests = ["pytest (>=6.0.0,<7.0.0)", "coverage[toml] (==5.0.4)"]
 
 [[package]]
 name = "pymacaroons"
@@ -1563,7 +1563,7 @@ url_preview = ["lxml"]
 [metadata]
 lock-version = "1.1"
 python-versions = "^3.7.1"
-content-hash = "c24bbcee7e86dbbe7cdbf49f91a25b310bf21095452641e7440129f59b077f78"
+content-hash = "e96625923122e29b6ea5964379828e321b6cede2b020fc32c6f86c09d86d1ae8"
 
 [metadata.files]
 attrs = [
@@ -1753,23 +1753,23 @@ flake8-comprehensions = [
     {file = "flake8_comprehensions-3.8.0-py3-none-any.whl", hash = "sha256:9406314803abe1193c064544ab14fdc43c58424c0882f6ff8a581eb73fc9bb58"},
 ]
 frozendict = [
-    {file = "frozendict-2.3.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:39942914c1217a5a49c7551495a103b3dbd216e19413687e003b859c6b0ebc12"},
-    {file = "frozendict-2.3.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5589256058b31f2b91419fa30b8dc62dbdefe7710e688a3fd5b43849161eecc9"},
-    {file = "frozendict-2.3.3-cp310-cp310-win_amd64.whl", hash = "sha256:35eb7e59e287c41f4f712d4d3d2333354175b155d217b97c99c201d2d8920790"},
-    {file = "frozendict-2.3.3-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:310aaf81793abf4f471895e6fe65e0e74a28a2aaf7b25c2ba6ccd4e35af06842"},
-    {file = "frozendict-2.3.3-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c353c11010a986566a0cb37f9a783c560ffff7d67d5e7fd52221fb03757cdc43"},
-    {file = "frozendict-2.3.3-cp36-cp36m-win_amd64.whl", hash = "sha256:15b5f82aad108125336593cec1b6420c638bf45f449c57e50949fc7654ea5a41"},
-    {file = "frozendict-2.3.3-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:a4737e5257756bd6b877504ff50185b705db577b5330d53040a6cf6417bb3cdb"},
-    {file = "frozendict-2.3.3-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:80a14c11e33e8b0bc09e07bba3732c77a502c39edb8c3959fd9a0e490e031158"},
-    {file = "frozendict-2.3.3-cp37-cp37m-win_amd64.whl", hash = "sha256:027952d1698ac9c766ef43711226b178cdd49d2acbdff396936639ad1d2a5615"},
-    {file = "frozendict-2.3.3-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:ef818d66c85098a37cf42509545a4ba7dd0c4c679d6262123a8dc14cc474bab7"},
-    {file = "frozendict-2.3.3-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:812279f2b270c980112dc4e367b168054f937108f8044eced4199e0ab2945a37"},
-    {file = "frozendict-2.3.3-cp38-cp38-win_amd64.whl", hash = "sha256:c1fb7efbfebc2075f781be3d9774e4ba6ce4fc399148b02097f68d4b3c4bc00a"},
-    {file = "frozendict-2.3.3-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:a0b46d4bf95bce843c0151959d54c3e5b8d0ce29cb44794e820b3ec980d63eee"},
-    {file = "frozendict-2.3.3-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:38c4660f37fcc70a32ff997fe58e40b3fcc60b2017b286e33828efaa16b01308"},
-    {file = "frozendict-2.3.3-cp39-cp39-win_amd64.whl", hash = "sha256:919e3609844fece11ab18bcbf28a3ed20f8108ad4149d7927d413687f281c6c9"},
-    {file = "frozendict-2.3.3-py3-none-any.whl", hash = "sha256:f988b482d08972a196664718167a993a61c9e9f6fe7b0ca2443570b5f20ca44a"},
-    {file = "frozendict-2.3.3.tar.gz", hash = "sha256:398539c52af3c647d103185bbaa1291679f0507ad035fe3bab2a8b0366d52cf1"},
+    {file = "frozendict-2.3.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:e18e2abd144a9433b0a8334582843b2aa0d3b9ac8b209aaa912ad365115fe2e1"},
+    {file = "frozendict-2.3.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:96dc7a02e78da5725e5e642269bb7ae792e0c9f13f10f2e02689175ebbfedb35"},
+    {file = "frozendict-2.3.0-cp310-cp310-win_amd64.whl", hash = "sha256:752a6dcfaf9bb20a7ecab24980e4dbe041f154509c989207caf185522ef85461"},
+    {file = "frozendict-2.3.0-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:5346d9fc1c936c76d33975a9a9f1a067342963105d9a403a99e787c939cc2bb2"},
+    {file = "frozendict-2.3.0-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:60dd2253f1bacb63a7c486ec541a968af4f985ffb06602ee8954a3d39ec6bd2e"},
+    {file = "frozendict-2.3.0-cp36-cp36m-win_amd64.whl", hash = "sha256:b2e044602ce17e5cd86724add46660fb9d80169545164e763300a3b839cb1b79"},
+    {file = "frozendict-2.3.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:a27a69b1ac3591e4258325108aee62b53c0eeb6ad0a993ae68d3c7eaea980420"},
+    {file = "frozendict-2.3.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4f45ef5f6b184d84744fff97b61f6b9a855e24d36b713ea2352fc723a047afa5"},
+    {file = "frozendict-2.3.0-cp37-cp37m-win_amd64.whl", hash = "sha256:2d3f5016650c0e9a192f5024e68fb4d63f670d0ee58b099ed3f5b4c62ea30ecb"},
+    {file = "frozendict-2.3.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:6cf605916f50aabaaba5624c81eb270200f6c2c466c46960237a125ec8fe3ae0"},
+    {file = "frozendict-2.3.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f6da06e44904beae4412199d7e49be4f85c6cc168ab06b77c735ea7da5ce3454"},
+    {file = "frozendict-2.3.0-cp38-cp38-win_amd64.whl", hash = "sha256:1f34793fb409c4fa70ffd25bea87b01f3bd305fb1c6b09e7dff085b126302206"},
+    {file = "frozendict-2.3.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:fd72494a559bdcd28aa71f4aa81860269cd0b7c45fff3e2614a0a053ecfd2a13"},
+    {file = "frozendict-2.3.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:00ea9166aa68cc5feed05986206fdbf35e838a09cb3feef998cf35978ff8a803"},
+    {file = "frozendict-2.3.0-cp39-cp39-win_amd64.whl", hash = "sha256:9ffaf440648b44e0bc694c1a4701801941378ba3ba6541e17750ae4b4aeeb116"},
+    {file = "frozendict-2.3.0-py3-none-any.whl", hash = "sha256:8578fe06815fcdcc672bd5603eebc98361a5317c1c3a13b28c6c810f6ea3b323"},
+    {file = "frozendict-2.3.0.tar.gz", hash = "sha256:da4231adefc5928e7810da2732269d3ad7b5616295b3e693746392a8205ea0b5"},
 ]
 gitdb = [
     {file = "gitdb-4.0.9-py3-none-any.whl", hash = "sha256:8033ad4e853066ba6ca92050b9df2f89301b8fc8bf7e9324d412a63f8bf1a8fd"},
@@ -2055,8 +2055,8 @@ matrix-common = [
     {file = "matrix_common-1.2.1.tar.gz", hash = "sha256:a99dcf02a6bd95b24a5a61b354888a2ac92bf2b4b839c727b8dd9da2cdfa3853"},
 ]
 matrix-synapse-ldap3 = [
-    {file = "matrix-synapse-ldap3-0.2.2.tar.gz", hash = "sha256:b388d95693486eef69adaefd0fd9e84463d52fe17b0214a00efcaa669b73cb74"},
-    {file = "matrix_synapse_ldap3-0.2.2-py3-none-any.whl", hash = "sha256:66ee4c85d7952c6c27fd04c09cdfdf4847b8e8b7d6a7ada6ba1100013bda060f"},
+    {file = "matrix-synapse-ldap3-0.2.1.tar.gz", hash = "sha256:bfb4390f4a262ffb0d6f057ff3aeb1e46d4e52ff420a064d795fb4f555f00285"},
+    {file = "matrix_synapse_ldap3-0.2.1-py3-none-any.whl", hash = "sha256:1b3310a60f1d06466f35905a269b6df95747fd1305f2b7fe638f373963b2aa2c"},
 ]
 mccabe = [
     {file = "mccabe-0.6.1-py2.py3-none-any.whl", hash = "sha256:ab8a6258860da4b6677da4bd2fe5dc2c659cff31b3ee4f7f5d64e79735b80d42"},

pyproject.toml

@@ -54,7 +54,7 @@ skip_gitignore = true
 
 [tool.poetry]
 name = "matrix-synapse"
-version = "1.65.0rc1"
+version = "1.62.0"
 description = "Homeserver for the Matrix decentralised comms protocol"
 authors = ["Matrix.org Team and Contributors <packages@matrix.org>"]
 license = "Apache-2.0"
@@ -110,9 +110,7 @@ jsonschema = ">=3.0.0"
 frozendict = ">=1,!=2.1.2"
 # We require 2.1.0 or higher for type hints. Previous guard was >= 1.1.0
 unpaddedbase64 = ">=2.1.0"
-# We require 1.5.0 to work around an issue when running against the C implementation of
-# frozendict: https://github.com/matrix-org/python-canonicaljson/issues/36
-canonicaljson = "^1.5.0"
+canonicaljson = "^1.4.0"
 # we use the type definitions added in signedjson 1.1.
 signedjson = "^1.1.0"
 # validating SSL certs for IP addresses requires service_identity 18.1.

scripts-dev/build_debian_packages.py

@@ -26,6 +26,7 @@ DISTS = (
     "debian:bookworm",
     "debian:sid",
     "ubuntu:focal",  # 20.04 LTS (our EOL forced by Py38 on 2024-10-14)
+    "ubuntu:impish",  # 21.10  (EOL 2022-07)
     "ubuntu:jammy",  # 22.04 LTS (EOL 2027-04)
 )
 

scripts-dev/complement.sh

@@ -101,7 +101,6 @@ if [ -z "$skip_docker_build" ]; then
     echo_if_github "::group::Build Docker image: matrixdotorg/synapse"
     docker build -t matrixdotorg/synapse \
       --build-arg TEST_ONLY_SKIP_DEP_HASH_VERIFICATION \
-      --build-arg TEST_ONLY_IGNORE_POETRY_LOCKFILE \
       -f "docker/Dockerfile" .
     echo_if_github "::endgroup::"
 

scripts-dev/mypy_synapse_plugin.py

@@ -29,7 +29,7 @@ class SynapsePlugin(Plugin):
         self, fullname: str
     ) -> Optional[Callable[[MethodSigContext], CallableType]]:
         if fullname.startswith(
-            "synapse.util.caches.descriptors.CachedFunction.__call__"
+            "synapse.util.caches.descriptors._CachedFunction.__call__"
         ) or fullname.startswith(
             "synapse.util.caches.descriptors._LruCachedFunction.__call__"
         ):
@@ -38,7 +38,7 @@ class SynapsePlugin(Plugin):
 
 
 def cached_function_method_signature(ctx: MethodSigContext) -> CallableType:
-    """Fixes the `CachedFunction.__call__` signature to be correct.
+    """Fixes the `_CachedFunction.__call__` signature to be correct.
 
     It already has *almost* the correct signature, except:
 

scripts-dev/release.py

@@ -32,7 +32,6 @@ import click
 import commonmark
 import git
 from click.exceptions import ClickException
-from git import GitCommandError, Repo
 from github import Github
 from packaging import version
 
@@ -56,12 +55,9 @@ def run_until_successful(
 def cli() -> None:
     """An interactive script to walk through the parts of creating a release.
 
-    Requirements:
-      - The dev dependencies be installed, which can be done via:
+    Requires the dev dependencies be installed, which can be done via:
 
-            pip install -e .[dev]
-
-      - A checkout of the sytest repository at ../sytest
+        pip install -e .[dev]
 
     Then to use:
 
@@ -79,8 +75,6 @@ def cli() -> None:
 
         # Optional: generate some nice links for the announcement
 
-        ./scripts-dev/release.py merge-back
-
         ./scripts-dev/release.py announce
 
     If the env var GH_TOKEN (or GITHUB_TOKEN) is set, or passed into the
@@ -95,12 +89,10 @@ def prepare() -> None:
     """
 
     # Make sure we're in a git repo.
-    synapse_repo = get_repo_and_check_clean_checkout()
-    sytest_repo = get_repo_and_check_clean_checkout("../sytest", "sytest")
+    repo = get_repo_and_check_clean_checkout()
 
-    click.secho("Updating Synapse and Sytest git repos...")
-    synapse_repo.remote().fetch()
-    sytest_repo.remote().fetch()
+    click.secho("Updating git repo...")
+    repo.remote().fetch()
 
     # Get the current version and AST from root Synapse module.
     current_version = get_package_version()
@@ -174,12 +166,12 @@ def prepare() -> None:
     assert not parsed_new_version.is_postrelease
 
     release_branch_name = get_release_branch_name(parsed_new_version)
-    release_branch = find_ref(synapse_repo, release_branch_name)
+    release_branch = find_ref(repo, release_branch_name)
     if release_branch:
         if release_branch.is_remote():
             # If the release branch only exists on the remote we check it out
             # locally.
-            synapse_repo.git.checkout(release_branch_name)
+            repo.git.checkout(release_branch_name)
     else:
         # If a branch doesn't exist we create one. We ask which one branch it
         # should be based off, defaulting to sensible values depending on the
@@ -195,34 +187,25 @@ def prepare() -> None:
             "Which branch should the release be based on?", default=default
         )
 
-        for repo_name, repo in {"synapse": synapse_repo, "sytest": sytest_repo}.items():
-            base_branch = find_ref(repo, branch_name)
-            if not base_branch:
-                print(f"Could not find base branch {branch_name} for {repo_name}!")
-                click.get_current_context().abort()
+        base_branch = find_ref(repo, branch_name)
+        if not base_branch:
+            print(f"Could not find base branch {branch_name}!")
+            click.get_current_context().abort()
 
-            # Check out the base branch and ensure it's up to date
-            repo.head.set_reference(
-                base_branch, f"check out the base branch for {repo_name}"
-            )
-            repo.head.reset(index=True, working_tree=True)
-            if not base_branch.is_remote():
-                update_branch(repo)
+        # Check out the base branch and ensure it's up to date
+        repo.head.set_reference(base_branch, "check out the base branch")
+        repo.head.reset(index=True, working_tree=True)
+        if not base_branch.is_remote():
+            update_branch(repo)
 
-            # Create the new release branch
-            # Type ignore will no longer be needed after GitPython 3.1.28.
-            # See https://github.com/gitpython-developers/GitPython/pull/1419
-            repo.create_head(release_branch_name, commit=base_branch)  # type: ignore[arg-type]
-
-        # Special-case SyTest: we don't actually prepare any files so we may
-        # as well push it now (and only when we create a release branch;
-        # not on subsequent RCs or full releases).
-        if click.confirm("Push new SyTest branch?", default=True):
-            sytest_repo.git.push("-u", sytest_repo.remote().name, release_branch_name)
+        # Create the new release branch
+        # Type ignore will no longer be needed after GitPython 3.1.28.
+        # See https://github.com/gitpython-developers/GitPython/pull/1419
+        repo.create_head(release_branch_name, commit=base_branch)  # type: ignore[arg-type]
 
     # Switch to the release branch and ensure it's up to date.
-    synapse_repo.git.checkout(release_branch_name)
-    update_branch(synapse_repo)
+    repo.git.checkout(release_branch_name)
+    update_branch(repo)
 
     # Update the version specified in pyproject.toml.
     subprocess.check_output(["poetry", "version", new_version])
@@ -247,15 +230,15 @@ def prepare() -> None:
     run_until_successful('dch -M -r -D stable ""', shell=True)
 
     # Show the user the changes and ask if they want to edit the change log.
-    synapse_repo.git.add("-u")
+    repo.git.add("-u")
     subprocess.run("git diff --cached", shell=True)
 
     if click.confirm("Edit changelog?", default=False):
         click.edit(filename="CHANGES.md")
 
     # Commit the changes.
-    synapse_repo.git.add("-u")
-    synapse_repo.git.commit("-m", new_version)
+    repo.git.add("-u")
+    repo.git.commit("-m", new_version)
 
     # We give the option to bail here in case the user wants to make sure things
     # are OK before pushing.
@@ -263,21 +246,17 @@ def prepare() -> None:
         print("")
         print("Run when ready to push:")
         print("")
-        print(
-            f"\tgit push -u {synapse_repo.remote().name} {synapse_repo.active_branch.name}"
-        )
+        print(f"\tgit push -u {repo.remote().name} {repo.active_branch.name}")
         print("")
         sys.exit(0)
 
     # Otherwise, push and open the changelog in the browser.
-    synapse_repo.git.push(
-        "-u", synapse_repo.remote().name, synapse_repo.active_branch.name
-    )
+    repo.git.push("-u", repo.remote().name, repo.active_branch.name)
 
     print("Opening the changelog in your browser...")
     print("Please ask others to give it a check.")
     click.launch(
-        f"https://github.com/matrix-org/synapse/blob/{synapse_repo.active_branch.name}/CHANGES.md"
+        f"https://github.com/matrix-org/synapse/blob/{repo.active_branch.name}/CHANGES.md"
     )
 
 
@@ -444,79 +423,6 @@ def upload() -> None:
     )
 
 
-def _merge_into(repo: Repo, source: str, target: str) -> None:
-    """
-    Merges branch `source` into branch `target`.
-    Pulls both before merging and pushes the result.
-    """
-
-    # Update our branches and switch to the target branch
-    for branch in [source, target]:
-        click.echo(f"Switching to {branch} and pulling...")
-        repo.heads[branch].checkout()
-        # Pull so we're up to date
-        repo.remote().pull()
-
-    assert repo.active_branch.name == target
-
-    try:
-        # TODO This seemed easier than using GitPython directly
-        click.echo(f"Merging {source}...")
-        repo.git.merge(source)
-    except GitCommandError as exc:
-        # If a merge conflict occurs, give some context and try to
-        # make it easy to abort if necessary.
-        click.echo(exc)
-        if not click.confirm(
-            f"Likely merge conflict whilst merging ({source} → {target}). "
-            f"Have you resolved it?"
-        ):
-            repo.git.merge("--abort")
-            return
-
-    # Push result.
-    click.echo("Pushing...")
-    repo.remote().push()
-
-
-@cli.command()
-def merge_back() -> None:
-    """Merge the release branch back into the appropriate branches.
-    All branches will be automatically pulled from the remote and the results
-    will be pushed to the remote."""
-
-    synapse_repo = get_repo_and_check_clean_checkout()
-    branch_name = synapse_repo.active_branch.name
-
-    if not branch_name.startswith("release-v"):
-        raise RuntimeError("Not on a release branch. This does not seem sensible.")
-
-    # Pull so we're up to date
-    synapse_repo.remote().pull()
-
-    current_version = get_package_version()
-
-    if current_version.is_prerelease:
-        # Release candidate
-        if click.confirm(f"Merge {branch_name} → develop?", default=True):
-            _merge_into(synapse_repo, branch_name, "develop")
-    else:
-        # Full release
-        sytest_repo = get_repo_and_check_clean_checkout("../sytest", "sytest")
-
-        if click.confirm(f"Merge {branch_name} → master?", default=True):
-            _merge_into(synapse_repo, branch_name, "master")
-
-        if click.confirm("Merge master → develop?", default=True):
-            _merge_into(synapse_repo, "master", "develop")
-
-        if click.confirm(f"On SyTest, merge {branch_name} → master?", default=True):
-            _merge_into(sytest_repo, branch_name, "master")
-
-        if click.confirm("On SyTest, merge master → develop?", default=True):
-            _merge_into(sytest_repo, "master", "develop")
-
-
 @cli.command()
 def announce() -> None:
     """Generate markdown to announce the release."""
@@ -563,18 +469,14 @@ def get_release_branch_name(version_number: version.Version) -> str:
     return f"release-v{version_number.major}.{version_number.minor}"
 
 
-def get_repo_and_check_clean_checkout(
-    path: str = ".", name: str = "synapse"
-) -> git.Repo:
+def get_repo_and_check_clean_checkout() -> git.Repo:
     """Get the project repo and check it's not got any uncommitted changes."""
     try:
-        repo = git.Repo(path=path)
+        repo = git.Repo()
     except git.InvalidGitRepositoryError:
-        raise click.ClickException(
-            f"{path} is not a git repository (expecting a {name} repository)."
-        )
+        raise click.ClickException("Not in Synapse repo.")
     if repo.is_dirty():
-        raise click.ClickException(f"Uncommitted changes exist in {path}.")
+        raise click.ClickException("Uncommitted changes exist.")
     return repo
 
 

synapse/_scripts/generate_config.py

@@ -33,7 +33,7 @@ def main() -> None:
     parser.add_argument(
         "--report-stats",
         action="store",
-        help="Whether the generated config reports homeserver usage statistics",
+        help="Whether the generated config reports anonymized usage statistics",
         choices=["yes", "no"],
     )
 

synapse/_scripts/synapse_port_db.py

@@ -166,6 +166,22 @@ IGNORED_TABLES = {
     "ui_auth_sessions",
     "ui_auth_sessions_credentials",
     "ui_auth_sessions_ips",
+    # Groups/communities is no longer supported.
+    "group_attestations_remote",
+    "group_attestations_renewals",
+    "group_invites",
+    "group_roles",
+    "group_room_categories",
+    "group_rooms",
+    "group_summary_roles",
+    "group_summary_room_categories",
+    "group_summary_rooms",
+    "group_summary_users",
+    "group_users",
+    "groups",
+    "local_group_membership",
+    "local_group_updates",
+    "remote_profile_cache",
 }
 
 
@@ -402,15 +418,12 @@ class Porter:
             self.progress.update(table, table_size)  # Mark table as done
             return
 
-        # We sweep over rowids in two directions: one forwards (rowids 1, 2, 3, ...)
-        # and another backwards (rowids 0, -1, -2, ...).
         forward_select = (
             "SELECT rowid, * FROM %s WHERE rowid >= ? ORDER BY rowid LIMIT ?" % (table,)
         )
 
         backward_select = (
-            "SELECT rowid, * FROM %s WHERE rowid <= ? ORDER BY rowid DESC LIMIT ?"
-            % (table,)
+            "SELECT rowid, * FROM %s WHERE rowid <= ? ORDER BY rowid LIMIT ?" % (table,)
         )
 
         do_forward = [True]

synapse/api/auth.py

@@ -26,17 +26,11 @@ from synapse.api.errors import (
     Codes,
     InvalidClientTokenError,
     MissingClientTokenError,
-    UnstableSpecAuthError,
 )
 from synapse.appservice import ApplicationService
 from synapse.http import get_request_user_agent
 from synapse.http.site import SynapseRequest
-from synapse.logging.opentracing import (
-    active_span,
-    force_tracing,
-    start_active_span,
-    trace,
-)
+from synapse.logging.opentracing import active_span, force_tracing, start_active_span
 from synapse.storage.databases.main.registration import TokenLookupResult
 from synapse.types import Requester, UserID, create_requester
 
@@ -112,11 +106,8 @@ class Auth:
                 forgot = await self.store.did_forget(user_id, room_id)
                 if not forgot:
                     return membership, member_event_id
-        raise UnstableSpecAuthError(
-            403,
-            "User %s not in room %s" % (user_id, room_id),
-            errcode=Codes.NOT_JOINED,
-        )
+
+        raise AuthError(403, "User %s not in room %s" % (user_id, room_id))
 
     async def get_user_by_req(
         self,
@@ -572,7 +563,6 @@ class Auth:
 
             return query_params[0].decode("ascii")
 
-    @trace
     async def check_user_in_room_or_world_readable(
         self, room_id: str, user_id: str, allow_departed_users: bool = False
     ) -> Tuple[str, Optional[str]]:
@@ -610,9 +600,8 @@ class Auth:
                 == HistoryVisibility.WORLD_READABLE
             ):
                 return Membership.JOIN, None
-            raise UnstableSpecAuthError(
+            raise AuthError(
                 403,
                 "User %s not in room %s, and room previews are disabled"
                 % (user_id, room_id),
-                errcode=Codes.NOT_JOINED,
             )

synapse/api/constants.py

@@ -257,8 +257,7 @@ class GuestAccess:
 
 class ReceiptTypes:
     READ: Final = "m.read"
-    READ_PRIVATE: Final = "m.read.private"
-    UNSTABLE_READ_PRIVATE: Final = "org.matrix.msc2285.read.private"
+    READ_PRIVATE: Final = "org.matrix.msc2285.read.private"
     FULLY_READ: Final = "m.fully_read"
 
 
@@ -269,4 +268,4 @@ class PublicRoomsFilterFields:
     """
 
     GENERIC_SEARCH_TERM: Final = "generic_search_term"
-    ROOM_TYPES: Final = "room_types"
+    ROOM_TYPES: Final = "org.matrix.msc3827.room_types"

synapse/api/errors.py

@@ -26,7 +26,6 @@ from twisted.web import http
 from synapse.util import json_decoder
 
 if typing.TYPE_CHECKING:
-    from synapse.config.homeserver import HomeServerConfig
     from synapse.types import JsonDict
 
 logger = logging.getLogger(__name__)
@@ -81,12 +80,6 @@ class Codes(str, Enum):
     INVALID_SIGNATURE = "M_INVALID_SIGNATURE"
     USER_DEACTIVATED = "M_USER_DEACTIVATED"
 
-    # Part of MSC3848
-    # https://github.com/matrix-org/matrix-spec-proposals/pull/3848
-    ALREADY_JOINED = "ORG.MATRIX.MSC3848.ALREADY_JOINED"
-    NOT_JOINED = "ORG.MATRIX.MSC3848.NOT_JOINED"
-    INSUFFICIENT_POWER = "ORG.MATRIX.MSC3848.INSUFFICIENT_POWER"
-
     # The account has been suspended on the server.
     # By opposition to `USER_DEACTIVATED`, this is a reversible measure
     # that can possibly be appealed and reverted.
@@ -174,7 +167,7 @@ class SynapseError(CodeMessageException):
         else:
             self._additional_fields = dict(additional_fields)
 
-    def error_dict(self, config: Optional["HomeServerConfig"]) -> "JsonDict":
+    def error_dict(self) -> "JsonDict":
         return cs_error(self.msg, self.errcode, **self._additional_fields)
 
 
@@ -220,7 +213,7 @@ class ConsentNotGivenError(SynapseError):
         )
         self._consent_uri = consent_uri
 
-    def error_dict(self, config: Optional["HomeServerConfig"]) -> "JsonDict":
+    def error_dict(self) -> "JsonDict":
         return cs_error(self.msg, self.errcode, consent_uri=self._consent_uri)
 
 
@@ -304,45 +297,8 @@ class AuthError(SynapseError):
     other poorly-defined times.
     """
 
-    def __init__(
-        self,
-        code: int,
-        msg: str,
-        errcode: str = Codes.FORBIDDEN,
-        additional_fields: Optional[dict] = None,
-    ):
-        super().__init__(code, msg, errcode, additional_fields)
-
-
-class UnstableSpecAuthError(AuthError):
-    """An error raised when a new error code is being proposed to replace a previous one.
-    This error will return a "org.matrix.unstable.errcode" property with the new error code,
-    with the previous error code still being defined in the "errcode" property.
-
-    This error will include `org.matrix.msc3848.unstable.errcode` in the C-S error body.
-    """
-
-    def __init__(
-        self,
-        code: int,
-        msg: str,
-        errcode: str,
-        previous_errcode: str = Codes.FORBIDDEN,
-        additional_fields: Optional[dict] = None,
-    ):
-        self.previous_errcode = previous_errcode
-        super().__init__(code, msg, errcode, additional_fields)
-
-    def error_dict(self, config: Optional["HomeServerConfig"]) -> "JsonDict":
-        fields = {}
-        if config is not None and config.experimental.msc3848_enabled:
-            fields["org.matrix.msc3848.unstable.errcode"] = self.errcode
-        return cs_error(
-            self.msg,
-            self.previous_errcode,
-            **fields,
-            **self._additional_fields,
-        )
+    def __init__(self, code: int, msg: str, errcode: str = Codes.FORBIDDEN):
+        super().__init__(code, msg, errcode)
 
 
 class InvalidClientCredentialsError(SynapseError):
@@ -376,8 +332,8 @@ class InvalidClientTokenError(InvalidClientCredentialsError):
         super().__init__(msg=msg, errcode="M_UNKNOWN_TOKEN")
         self._soft_logout = soft_logout
 
-    def error_dict(self, config: Optional["HomeServerConfig"]) -> "JsonDict":
-        d = super().error_dict(config)
+    def error_dict(self) -> "JsonDict":
+        d = super().error_dict()
         d["soft_logout"] = self._soft_logout
         return d
 
@@ -400,7 +356,7 @@ class ResourceLimitError(SynapseError):
         self.limit_type = limit_type
         super().__init__(code, msg, errcode=errcode)
 
-    def error_dict(self, config: Optional["HomeServerConfig"]) -> "JsonDict":
+    def error_dict(self) -> "JsonDict":
         return cs_error(
             self.msg,
             self.errcode,
@@ -435,7 +391,7 @@ class InvalidCaptchaError(SynapseError):
         super().__init__(code, msg, errcode)
         self.error_url = error_url
 
-    def error_dict(self, config: Optional["HomeServerConfig"]) -> "JsonDict":
+    def error_dict(self) -> "JsonDict":
         return cs_error(self.msg, self.errcode, error_url=self.error_url)
 
 
@@ -452,7 +408,7 @@ class LimitExceededError(SynapseError):
         super().__init__(code, msg, errcode)
         self.retry_after_ms = retry_after_ms
 
-    def error_dict(self, config: Optional["HomeServerConfig"]) -> "JsonDict":
+    def error_dict(self) -> "JsonDict":
         return cs_error(self.msg, self.errcode, retry_after_ms=self.retry_after_ms)
 
 
@@ -467,7 +423,7 @@ class RoomKeysVersionError(SynapseError):
         super().__init__(403, "Wrong room_keys version", Codes.WRONG_ROOM_KEYS_VERSION)
         self.current_version = current_version
 
-    def error_dict(self, config: Optional["HomeServerConfig"]) -> "JsonDict":
+    def error_dict(self) -> "JsonDict":
         return cs_error(self.msg, self.errcode, current_version=self.current_version)
 
 
@@ -507,7 +463,7 @@ class IncompatibleRoomVersionError(SynapseError):
 
         self._room_version = room_version
 
-    def error_dict(self, config: Optional["HomeServerConfig"]) -> "JsonDict":
+    def error_dict(self) -> "JsonDict":
         return cs_error(self.msg, self.errcode, room_version=self._room_version)
 
 
@@ -553,7 +509,7 @@ class UnredactedContentDeletedError(SynapseError):
         )
         self.content_keep_ms = content_keep_ms
 
-    def error_dict(self, config: Optional["HomeServerConfig"]) -> "JsonDict":
+    def error_dict(self) -> "JsonDict":
         extra = {}
         if self.content_keep_ms is not None:
             extra = {"fi.mau.msc2815.content_keep_ms": self.content_keep_ms}

synapse/api/ratelimiting.py

@@ -17,7 +17,7 @@ from collections import OrderedDict
 from typing import Hashable, Optional, Tuple
 
 from synapse.api.errors import LimitExceededError
-from synapse.config.ratelimiting import RatelimitSettings
+from synapse.config.ratelimiting import RateLimitConfig
 from synapse.storage.databases.main import DataStore
 from synapse.types import Requester
 from synapse.util import Clock
@@ -27,33 +27,6 @@ class Ratelimiter:
     """
     Ratelimit actions marked by arbitrary keys.
 
-    (Note that the source code speaks of "actions" and "burst_count" rather than
-    "tokens" and a "bucket_size".)
-
-    This is a "leaky bucket as a meter". For each key to be tracked there is a bucket
-    containing some number 0 <= T <= `burst_count` of tokens corresponding to previously
-    permitted requests for that key. Each bucket starts empty, and gradually leaks
-    tokens at a rate of `rate_hz`.
-
-    Upon an incoming request, we must determine:
-    - the key that this request falls under (which bucket to inspect), and
-    - the cost C of this request in tokens.
-    Then, if there is room in the bucket for C tokens (T + C <= `burst_count`),
-    the request is permitted and `cost` tokens are added to the bucket.
-    Otherwise the request is denied, and the bucket continues to hold T tokens.
-
-    This means that the limiter enforces an average request frequency of `rate_hz`,
-    while accumulating a buffer of up to `burst_count` requests which can be consumed
-    instantaneously.
-
-    The tricky bit is the leaking. We do not want to have a periodic process which
-    leaks every bucket! Instead, we track
-    - the time point when the bucket was last completely empty, and
-    - how many tokens have added to the bucket permitted since then.
-    Then for each incoming request, we can calculate how many tokens have leaked
-    since this time point, and use that to decide if we should accept or reject the
-    request.
-
     Args:
         clock: A homeserver clock, for retrieving the current time
         rate_hz: The long term number of actions that can be performed in a second.
@@ -68,30 +41,14 @@ class Ratelimiter:
         self.burst_count = burst_count
         self.store = store
 
-        # An ordered dictionary representing the token buckets tracked by this rate
-        # limiter. Each entry maps a key of arbitrary type to a tuple representing:
-        #   * The number of tokens currently in the bucket,
-        #   * The time point when the bucket was last completely empty, and
-        #   * The rate_hz (leak rate) of this particular bucket.
+        # A ordered dictionary keeping track of actions, when they were last
+        # performed and how often. Each entry is a mapping from a key of arbitrary type
+        # to a tuple representing:
+        #   * How many times an action has occurred since a point in time
+        #   * The point in time
+        #   * The rate_hz of this particular entry. This can vary per request
         self.actions: OrderedDict[Hashable, Tuple[float, float, float]] = OrderedDict()
 
-    def _get_key(
-        self, requester: Optional[Requester], key: Optional[Hashable]
-    ) -> Hashable:
-        """Use the requester's MXID as a fallback key if no key is provided."""
-        if key is None:
-            if not requester:
-                raise ValueError("Must supply at least one of `requester` or `key`")
-
-            key = requester.user.to_string()
-        return key
-
-    def _get_action_counts(
-        self, key: Hashable, time_now_s: float
-    ) -> Tuple[float, float, float]:
-        """Retrieve the action counts, with a fallback representing an empty bucket."""
-        return self.actions.get(key, (0.0, time_now_s, 0.0))
-
     async def can_do_action(
         self,
         requester: Optional[Requester],
@@ -131,7 +88,11 @@ class Ratelimiter:
                 * The reactor timestamp for when the action can be performed next.
                   -1 if rate_hz is less than or equal to zero
         """
-        key = self._get_key(requester, key)
+        if key is None:
+            if not requester:
+                raise ValueError("Must supply at least one of `requester` or `key`")
+
+            key = requester.user.to_string()
 
         if requester:
             # Disable rate limiting of users belonging to any AS that is configured
@@ -160,7 +121,7 @@ class Ratelimiter:
         self._prune_message_counts(time_now_s)
 
         # Check if there is an existing count entry for this key
-        action_count, time_start, _ = self._get_action_counts(key, time_now_s)
+        action_count, time_start, _ = self.actions.get(key, (0.0, time_now_s, 0.0))
 
         # Check whether performing another action is allowed
         time_delta = time_now_s - time_start
@@ -203,37 +164,6 @@ class Ratelimiter:
 
         return allowed, time_allowed
 
-    def record_action(
-        self,
-        requester: Optional[Requester],
-        key: Optional[Hashable] = None,
-        n_actions: int = 1,
-        _time_now_s: Optional[float] = None,
-    ) -> None:
-        """Record that an action(s) took place, even if they violate the rate limit.
-
-        This is useful for tracking the frequency of events that happen across
-        federation which we still want to impose local rate limits on. For instance, if
-        we are alice.com monitoring a particular room, we cannot prevent bob.com
-        from joining users to that room. However, we can track the number of recent
-        joins in the room and refuse to serve new joins ourselves if there have been too
-        many in the room across both homeservers.
-
-        Args:
-            requester: The requester that is doing the action, if any.
-            key: An arbitrary key used to classify an action. Defaults to the
-                requester's user ID.
-            n_actions: The number of times the user wants to do this action. If the user
-                cannot do all of the actions, the user's action count is not incremented
-                at all.
-            _time_now_s: The current time. Optional, defaults to the current time according
-                to self.clock. Only used by tests.
-        """
-        key = self._get_key(requester, key)
-        time_now_s = _time_now_s if _time_now_s is not None else self.clock.time()
-        action_count, time_start, rate_hz = self._get_action_counts(key, time_now_s)
-        self.actions[key] = (action_count + n_actions, time_start, rate_hz)
-
     def _prune_message_counts(self, time_now_s: float) -> None:
         """Remove message count entries that have not exceeded their defined
         rate_hz limit
@@ -314,8 +244,8 @@ class RequestRatelimiter:
         self,
         store: DataStore,
         clock: Clock,
-        rc_message: RatelimitSettings,
-        rc_admin_redaction: Optional[RatelimitSettings],
+        rc_message: RateLimitConfig,
+        rc_admin_redaction: Optional[RateLimitConfig],
     ):
         self.store = store
         self.clock = clock

synapse/api/room_versions.py

@@ -84,8 +84,6 @@ class RoomVersion:
     # MSC3787: Adds support for a `knock_restricted` join rule, mixing concepts of
     # knocks and restricted join rules into the same join condition.
     msc3787_knock_restricted_join_rule: bool
-    # MSC3667: Enforce integer power levels
-    msc3667_int_only_power_levels: bool
 
 
 class RoomVersions:
@@ -105,7 +103,6 @@ class RoomVersions:
         msc2716_historical=False,
         msc2716_redactions=False,
         msc3787_knock_restricted_join_rule=False,
-        msc3667_int_only_power_levels=False,
     )
     V2 = RoomVersion(
         "2",
@@ -123,7 +120,6 @@ class RoomVersions:
         msc2716_historical=False,
         msc2716_redactions=False,
         msc3787_knock_restricted_join_rule=False,
-        msc3667_int_only_power_levels=False,
     )
     V3 = RoomVersion(
         "3",
@@ -141,7 +137,6 @@ class RoomVersions:
         msc2716_historical=False,
         msc2716_redactions=False,
         msc3787_knock_restricted_join_rule=False,
-        msc3667_int_only_power_levels=False,
     )
     V4 = RoomVersion(
         "4",
@@ -159,7 +154,6 @@ class RoomVersions:
         msc2716_historical=False,
         msc2716_redactions=False,
         msc3787_knock_restricted_join_rule=False,
-        msc3667_int_only_power_levels=False,
     )
     V5 = RoomVersion(
         "5",
@@ -177,7 +171,6 @@ class RoomVersions:
         msc2716_historical=False,
         msc2716_redactions=False,
         msc3787_knock_restricted_join_rule=False,
-        msc3667_int_only_power_levels=False,
     )
     V6 = RoomVersion(
         "6",
@@ -195,7 +188,6 @@ class RoomVersions:
         msc2716_historical=False,
         msc2716_redactions=False,
         msc3787_knock_restricted_join_rule=False,
-        msc3667_int_only_power_levels=False,
     )
     MSC2176 = RoomVersion(
         "org.matrix.msc2176",
@@ -213,7 +205,6 @@ class RoomVersions:
         msc2716_historical=False,
         msc2716_redactions=False,
         msc3787_knock_restricted_join_rule=False,
-        msc3667_int_only_power_levels=False,
     )
     V7 = RoomVersion(
         "7",
@@ -231,7 +222,6 @@ class RoomVersions:
         msc2716_historical=False,
         msc2716_redactions=False,
         msc3787_knock_restricted_join_rule=False,
-        msc3667_int_only_power_levels=False,
     )
     V8 = RoomVersion(
         "8",
@@ -249,7 +239,6 @@ class RoomVersions:
         msc2716_historical=False,
         msc2716_redactions=False,
         msc3787_knock_restricted_join_rule=False,
-        msc3667_int_only_power_levels=False,
     )
     V9 = RoomVersion(
         "9",
@@ -267,7 +256,6 @@ class RoomVersions:
         msc2716_historical=False,
         msc2716_redactions=False,
         msc3787_knock_restricted_join_rule=False,
-        msc3667_int_only_power_levels=False,
     )
     MSC2716v3 = RoomVersion(
         "org.matrix.msc2716v3",
@@ -285,7 +273,6 @@ class RoomVersions:
         msc2716_historical=True,
         msc2716_redactions=True,
         msc3787_knock_restricted_join_rule=False,
-        msc3667_int_only_power_levels=False,
     )
     MSC3787 = RoomVersion(
         "org.matrix.msc3787",
@@ -303,25 +290,6 @@ class RoomVersions:
         msc2716_historical=False,
         msc2716_redactions=False,
         msc3787_knock_restricted_join_rule=True,
-        msc3667_int_only_power_levels=False,
-    )
-    V10 = RoomVersion(
-        "10",
-        RoomDisposition.STABLE,
-        EventFormatVersions.V3,
-        StateResolutionVersions.V2,
-        enforce_key_validity=True,
-        special_case_aliases_auth=False,
-        strict_canonicaljson=True,
-        limit_notifications_power_levels=True,
-        msc2176_redaction_rules=False,
-        msc3083_join_rules=True,
-        msc3375_redaction_rules=True,
-        msc2403_knocking=True,
-        msc2716_historical=False,
-        msc2716_redactions=False,
-        msc3787_knock_restricted_join_rule=True,
-        msc3667_int_only_power_levels=True,
     )
 
 
@@ -340,7 +308,6 @@ KNOWN_ROOM_VERSIONS: Dict[str, RoomVersion] = {
         RoomVersions.V9,
         RoomVersions.MSC2716v3,
         RoomVersions.MSC3787,
-        RoomVersions.V10,
     )
 }
 

synapse/app/admin_cmd.py

@@ -28,22 +28,18 @@ from synapse.config.homeserver import HomeServerConfig
 from synapse.config.logger import setup_logging
 from synapse.events import EventBase
 from synapse.handlers.admin import ExfiltrationWriter
+from synapse.replication.slave.storage._base import BaseSlavedStore
+from synapse.replication.slave.storage.account_data import SlavedAccountDataStore
+from synapse.replication.slave.storage.appservice import SlavedApplicationServiceStore
+from synapse.replication.slave.storage.deviceinbox import SlavedDeviceInboxStore
 from synapse.replication.slave.storage.devices import SlavedDeviceStore
 from synapse.replication.slave.storage.events import SlavedEventStore
 from synapse.replication.slave.storage.filtering import SlavedFilteringStore
 from synapse.replication.slave.storage.push_rule import SlavedPushRuleStore
+from synapse.replication.slave.storage.receipts import SlavedReceiptsStore
+from synapse.replication.slave.storage.registration import SlavedRegistrationStore
 from synapse.server import HomeServer
-from synapse.storage.database import DatabasePool, LoggingDatabaseConnection
-from synapse.storage.databases.main.account_data import AccountDataWorkerStore
-from synapse.storage.databases.main.appservice import (
-    ApplicationServiceTransactionWorkerStore,
-    ApplicationServiceWorkerStore,
-)
-from synapse.storage.databases.main.deviceinbox import DeviceInboxWorkerStore
-from synapse.storage.databases.main.receipts import ReceiptsWorkerStore
-from synapse.storage.databases.main.registration import RegistrationWorkerStore
 from synapse.storage.databases.main.room import RoomWorkerStore
-from synapse.storage.databases.main.tags import TagsWorkerStore
 from synapse.types import StateMap
 from synapse.util import SYNAPSE_VERSION
 from synapse.util.logcontext import LoggingContext
@@ -52,30 +48,19 @@ logger = logging.getLogger("synapse.app.admin_cmd")
 
 
 class AdminCmdSlavedStore(
+    SlavedReceiptsStore,
+    SlavedAccountDataStore,
+    SlavedApplicationServiceStore,
+    SlavedRegistrationStore,
     SlavedFilteringStore,
+    SlavedDeviceInboxStore,
     SlavedDeviceStore,
     SlavedPushRuleStore,
     SlavedEventStore,
-    TagsWorkerStore,
-    DeviceInboxWorkerStore,
-    AccountDataWorkerStore,
-    ApplicationServiceTransactionWorkerStore,
-    ApplicationServiceWorkerStore,
-    RegistrationWorkerStore,
-    ReceiptsWorkerStore,
+    BaseSlavedStore,
     RoomWorkerStore,
 ):
-    def __init__(
-        self,
-        database: DatabasePool,
-        db_conn: LoggingDatabaseConnection,
-        hs: "HomeServer",
-    ):
-        super().__init__(database, db_conn, hs)
-
-        # Annoyingly `filter_events_for_client` assumes that this exists. We
-        # should refactor it to take a `Clock` directly.
-        self.clock = hs.get_clock()
+    pass
 
 
 class AdminCmdServer(HomeServer):

synapse/app/generic_worker.py

@@ -48,12 +48,20 @@ from synapse.http.site import SynapseRequest, SynapseSite
 from synapse.logging.context import LoggingContext
 from synapse.metrics import METRICS_PREFIX, MetricsResource, RegistryProxy
 from synapse.replication.http import REPLICATION_PREFIX, ReplicationRestResource
+from synapse.replication.slave.storage._base import BaseSlavedStore
+from synapse.replication.slave.storage.account_data import SlavedAccountDataStore
+from synapse.replication.slave.storage.appservice import SlavedApplicationServiceStore
+from synapse.replication.slave.storage.deviceinbox import SlavedDeviceInboxStore
 from synapse.replication.slave.storage.devices import SlavedDeviceStore
+from synapse.replication.slave.storage.directory import DirectoryStore
 from synapse.replication.slave.storage.events import SlavedEventStore
 from synapse.replication.slave.storage.filtering import SlavedFilteringStore
 from synapse.replication.slave.storage.keys import SlavedKeyStore
+from synapse.replication.slave.storage.profile import SlavedProfileStore
 from synapse.replication.slave.storage.push_rule import SlavedPushRuleStore
 from synapse.replication.slave.storage.pushers import SlavedPusherStore
+from synapse.replication.slave.storage.receipts import SlavedReceiptsStore
+from synapse.replication.slave.storage.registration import SlavedRegistrationStore
 from synapse.rest.admin import register_servlets_for_media_repo
 from synapse.rest.client import (
     account_data,
@@ -92,15 +100,8 @@ from synapse.rest.key.v2 import KeyApiV2Resource
 from synapse.rest.synapse.client import build_synapse_client_resource_tree
 from synapse.rest.well_known import well_known_resource
 from synapse.server import HomeServer
-from synapse.storage.databases.main.account_data import AccountDataWorkerStore
-from synapse.storage.databases.main.appservice import (
-    ApplicationServiceTransactionWorkerStore,
-    ApplicationServiceWorkerStore,
-)
 from synapse.storage.databases.main.censor_events import CensorEventsStore
 from synapse.storage.databases.main.client_ips import ClientIpWorkerStore
-from synapse.storage.databases.main.deviceinbox import DeviceInboxWorkerStore
-from synapse.storage.databases.main.directory import DirectoryWorkerStore
 from synapse.storage.databases.main.e2e_room_keys import EndToEndRoomKeyStore
 from synapse.storage.databases.main.lock import LockStore
 from synapse.storage.databases.main.media_repository import MediaRepositoryStore
@@ -109,15 +110,11 @@ from synapse.storage.databases.main.monthly_active_users import (
     MonthlyActiveUsersWorkerStore,
 )
 from synapse.storage.databases.main.presence import PresenceStore
-from synapse.storage.databases.main.profile import ProfileWorkerStore
-from synapse.storage.databases.main.receipts import ReceiptsWorkerStore
-from synapse.storage.databases.main.registration import RegistrationWorkerStore
 from synapse.storage.databases.main.room import RoomWorkerStore
 from synapse.storage.databases.main.room_batch import RoomBatchStore
 from synapse.storage.databases.main.search import SearchStore
 from synapse.storage.databases.main.session import SessionStore
 from synapse.storage.databases.main.stats import StatsStore
-from synapse.storage.databases.main.tags import TagsWorkerStore
 from synapse.storage.databases.main.transactions import TransactionWorkerStore
 from synapse.storage.databases.main.ui_auth import UIAuthWorkerStore
 from synapse.storage.databases.main.user_directory import UserDirectoryStore
@@ -230,11 +227,11 @@ class GenericWorkerSlavedStore(
     UIAuthWorkerStore,
     EndToEndRoomKeyStore,
     PresenceStore,
-    DeviceInboxWorkerStore,
+    SlavedDeviceInboxStore,
     SlavedDeviceStore,
+    SlavedReceiptsStore,
     SlavedPushRuleStore,
-    TagsWorkerStore,
-    AccountDataWorkerStore,
+    SlavedAccountDataStore,
     SlavedPusherStore,
     CensorEventsStore,
     ClientIpWorkerStore,
@@ -242,20 +239,19 @@ class GenericWorkerSlavedStore(
     SlavedKeyStore,
     RoomWorkerStore,
     RoomBatchStore,
-    DirectoryWorkerStore,
-    ApplicationServiceTransactionWorkerStore,
-    ApplicationServiceWorkerStore,
-    ProfileWorkerStore,
+    DirectoryStore,
+    SlavedApplicationServiceStore,
+    SlavedRegistrationStore,
+    SlavedProfileStore,
     SlavedFilteringStore,
     MonthlyActiveUsersWorkerStore,
     MediaRepositoryStore,
     ServerMetricsStore,
-    ReceiptsWorkerStore,
-    RegistrationWorkerStore,
     SearchStore,
     TransactionWorkerStore,
     LockStore,
     SessionStore,
+    BaseSlavedStore,
 ):
     # Properties that multiple storage classes define. Tell mypy what the
     # expected type is.

synapse/appservice/api.py

@@ -53,18 +53,6 @@ sent_events_counter = Counter(
     "synapse_appservice_api_sent_events", "Number of events sent to the AS", ["service"]
 )
 
-sent_ephemeral_counter = Counter(
-    "synapse_appservice_api_sent_ephemeral",
-    "Number of ephemeral events sent to the AS",
-    ["service"],
-)
-
-sent_todevice_counter = Counter(
-    "synapse_appservice_api_sent_todevice",
-    "Number of todevice messages sent to the AS",
-    ["service"],
-)
-
 HOUR_IN_MS = 60 * 60 * 1000
 
 
@@ -322,8 +310,6 @@ class ApplicationServiceApi(SimpleHttpClient):
                 )
             sent_transactions_counter.labels(service.id).inc()
             sent_events_counter.labels(service.id).inc(len(serialized_events))
-            sent_ephemeral_counter.labels(service.id).inc(len(ephemeral))
-            sent_todevice_counter.labels(service.id).inc(len(to_device_messages))
             return True
         except CodeMessageException as e:
             logger.warning(

synapse/appservice/scheduler.py

@@ -319,9 +319,7 @@ class _ServiceQueuer:
         rooms_of_interesting_users.update(event.room_id for event in events)
         # EDUs
         rooms_of_interesting_users.update(
-            ephemeral["room_id"]
-            for ephemeral in ephemerals
-            if ephemeral.get("room_id") is not None
+            ephemeral["room_id"] for ephemeral in ephemerals
         )
 
         # Look up the AS users in those rooms
@@ -331,9 +329,8 @@ class _ServiceQueuer:
             )
 
         # Add recipients of to-device messages.
-        users.update(
-            device_message["to_user_id"] for device_message in to_device_messages
-        )
+        # device_message["user_id"] is the ID of the recipient.
+        users.update(device_message["user_id"] for device_message in to_device_messages)
 
         # Compute and return the counts / fallback key usage states
         otk_counts = await self._store.count_bulk_e2e_one_time_keys_for_as(users)

synapse/config/_base.py

@@ -97,16 +97,16 @@ def format_config_error(e: ConfigError) -> Iterator[str]:
 # We split these messages out to allow packages to override with package
 # specific instructions.
 MISSING_REPORT_STATS_CONFIG_INSTRUCTIONS = """\
-Please opt in or out of reporting homeserver usage statistics, by setting
-the `report_stats` key in your config file to either True or False.
+Please opt in or out of reporting anonymized homeserver usage statistics, by
+setting the `report_stats` key in your config file to either True or False.
 """
 
 MISSING_REPORT_STATS_SPIEL = """\
 We would really appreciate it if you could help our project out by reporting
-homeserver usage statistics from your homeserver. Your homeserver's server name,
-along with very basic aggregate data (e.g. number of users) will be reported. But
-it helps us to track the growth of the Matrix community, and helps us to make Matrix
-a success, as well as to convince other networks that they should peer with us.
+anonymized usage statistics from your homeserver. Only very basic aggregate
+data (e.g. number of users) will be reported, but it helps us to track the
+growth of the Matrix community, and helps us to make Matrix a success, as well
+as to convince other networks that they should peer with us.
 
 Thank you.
 """
@@ -621,7 +621,7 @@ class RootConfig:
         generate_group.add_argument(
             "--report-stats",
             action="store",
-            help="Whether the generated config reports homeserver usage statistics.",
+            help="Whether the generated config reports anonymized usage statistics.",
             choices=["yes", "no"],
         )
         generate_group.add_argument(

synapse/config/emailconfig.py

@@ -86,19 +86,14 @@ class EmailConfig(Config):
         if email_config is None:
             email_config = {}
 
-        self.force_tls = email_config.get("force_tls", False)
         self.email_smtp_host = email_config.get("smtp_host", "localhost")
-        self.email_smtp_port = email_config.get(
-            "smtp_port", 465 if self.force_tls else 25
-        )
+        self.email_smtp_port = email_config.get("smtp_port", 25)
         self.email_smtp_user = email_config.get("smtp_user", None)
         self.email_smtp_pass = email_config.get("smtp_pass", None)
         self.require_transport_security = email_config.get(
             "require_transport_security", False
         )
         self.enable_smtp_tls = email_config.get("enable_tls", True)
-        if self.force_tls and not self.enable_smtp_tls:
-            raise ConfigError("email.force_tls requires email.enable_tls to be true")
         if self.require_transport_security and not self.enable_smtp_tls:
             raise ConfigError(
                 "email.require_transport_security requires email.enable_tls to be true"
@@ -148,7 +143,8 @@ class EmailConfig(Config):
 
         if config.get("trust_identity_server_for_password_resets"):
             raise ConfigError(
-                'The config option "trust_identity_server_for_password_resets" has been removed.'
+                'The config option "trust_identity_server_for_password_resets" '
+                'has been replaced by "account_threepid_delegate". '
                 "Please consult the configuration manual at docs/usage/configuration/config_documentation.md for "
                 "details and update your config file."
             )

synapse/config/experimental.py

@@ -32,7 +32,7 @@ class ExperimentalConfig(Config):
         # MSC2716 (importing historical messages)
         self.msc2716_enabled: bool = experimental.get("msc2716_enabled", False)
 
-        # MSC2285 (unstable private read receipts)
+        # MSC2285 (private read receipts)
         self.msc2285_enabled: bool = experimental.get("msc2285_enabled", False)
 
         # MSC3244 (room version capabilities)
@@ -88,5 +88,5 @@ class ExperimentalConfig(Config):
         # MSC3715: dir param on /relations.
         self.msc3715_enabled: bool = experimental.get("msc3715_enabled", False)
 
-        # MSC3848: Introduce errcodes for specific event sending failures
-        self.msc3848_enabled: bool = experimental.get("msc3848_enabled", False)
+        # MSC3827: Filtering of /publicRooms by room type
+        self.msc3827_enabled: bool = experimental.get("msc3827_enabled", False)

synapse/config/ratelimiting.py

@@ -21,7 +21,7 @@ from synapse.types import JsonDict
 from ._base import Config
 
 
-class RatelimitSettings:
+class RateLimitConfig:
     def __init__(
         self,
         config: Dict[str, float],
@@ -34,7 +34,7 @@ class RatelimitSettings:
 
 
 @attr.s(auto_attribs=True)
-class FederationRatelimitSettings:
+class FederationRateLimitConfig:
     window_size: int = 1000
     sleep_limit: int = 10
     sleep_delay: int = 500
@@ -50,11 +50,11 @@ class RatelimitConfig(Config):
         # Load the new-style messages config if it exists. Otherwise fall back
         # to the old method.
         if "rc_message" in config:
-            self.rc_message = RatelimitSettings(
+            self.rc_message = RateLimitConfig(
                 config["rc_message"], defaults={"per_second": 0.2, "burst_count": 10.0}
             )
         else:
-            self.rc_message = RatelimitSettings(
+            self.rc_message = RateLimitConfig(
                 {
                     "per_second": config.get("rc_messages_per_second", 0.2),
                     "burst_count": config.get("rc_message_burst_count", 10.0),
@@ -64,9 +64,9 @@ class RatelimitConfig(Config):
         # Load the new-style federation config, if it exists. Otherwise, fall
         # back to the old method.
         if "rc_federation" in config:
-            self.rc_federation = FederationRatelimitSettings(**config["rc_federation"])
+            self.rc_federation = FederationRateLimitConfig(**config["rc_federation"])
         else:
-            self.rc_federation = FederationRatelimitSettings(
+            self.rc_federation = FederationRateLimitConfig(
                 **{
                     k: v
                     for k, v in {
@@ -80,17 +80,17 @@ class RatelimitConfig(Config):
                 }
             )
 
-        self.rc_registration = RatelimitSettings(config.get("rc_registration", {}))
+        self.rc_registration = RateLimitConfig(config.get("rc_registration", {}))
 
-        self.rc_registration_token_validity = RatelimitSettings(
+        self.rc_registration_token_validity = RateLimitConfig(
             config.get("rc_registration_token_validity", {}),
             defaults={"per_second": 0.1, "burst_count": 5},
         )
 
         rc_login_config = config.get("rc_login", {})
-        self.rc_login_address = RatelimitSettings(rc_login_config.get("address", {}))
-        self.rc_login_account = RatelimitSettings(rc_login_config.get("account", {}))
-        self.rc_login_failed_attempts = RatelimitSettings(
+        self.rc_login_address = RateLimitConfig(rc_login_config.get("address", {}))
+        self.rc_login_account = RateLimitConfig(rc_login_config.get("account", {}))
+        self.rc_login_failed_attempts = RateLimitConfig(
             rc_login_config.get("failed_attempts", {})
         )
 
@@ -101,54 +101,47 @@ class RatelimitConfig(Config):
         rc_admin_redaction = config.get("rc_admin_redaction")
         self.rc_admin_redaction = None
         if rc_admin_redaction:
-            self.rc_admin_redaction = RatelimitSettings(rc_admin_redaction)
+            self.rc_admin_redaction = RateLimitConfig(rc_admin_redaction)
 
-        self.rc_joins_local = RatelimitSettings(
+        self.rc_joins_local = RateLimitConfig(
             config.get("rc_joins", {}).get("local", {}),
             defaults={"per_second": 0.1, "burst_count": 10},
         )
-        self.rc_joins_remote = RatelimitSettings(
+        self.rc_joins_remote = RateLimitConfig(
             config.get("rc_joins", {}).get("remote", {}),
             defaults={"per_second": 0.01, "burst_count": 10},
         )
 
-        # Track the rate of joins to a given room. If there are too many, temporarily
-        # prevent local joins and remote joins via this server.
-        self.rc_joins_per_room = RatelimitSettings(
-            config.get("rc_joins_per_room", {}),
-            defaults={"per_second": 1, "burst_count": 10},
-        )
-
         # Ratelimit cross-user key requests:
         # * For local requests this is keyed by the sending device.
         # * For requests received over federation this is keyed by the origin.
         #
         # Note that this isn't exposed in the configuration as it is obscure.
-        self.rc_key_requests = RatelimitSettings(
+        self.rc_key_requests = RateLimitConfig(
             config.get("rc_key_requests", {}),
             defaults={"per_second": 20, "burst_count": 100},
         )
 
-        self.rc_3pid_validation = RatelimitSettings(
+        self.rc_3pid_validation = RateLimitConfig(
             config.get("rc_3pid_validation") or {},
             defaults={"per_second": 0.003, "burst_count": 5},
         )
 
-        self.rc_invites_per_room = RatelimitSettings(
+        self.rc_invites_per_room = RateLimitConfig(
             config.get("rc_invites", {}).get("per_room", {}),
             defaults={"per_second": 0.3, "burst_count": 10},
         )
-        self.rc_invites_per_user = RatelimitSettings(
+        self.rc_invites_per_user = RateLimitConfig(
             config.get("rc_invites", {}).get("per_user", {}),
             defaults={"per_second": 0.003, "burst_count": 5},
         )
 
-        self.rc_invites_per_issuer = RatelimitSettings(
+        self.rc_invites_per_issuer = RateLimitConfig(
             config.get("rc_invites", {}).get("per_issuer", {}),
             defaults={"per_second": 0.3, "burst_count": 10},
         )
 
-        self.rc_third_party_invite = RatelimitSettings(
+        self.rc_third_party_invite = RateLimitConfig(
             config.get("rc_third_party_invite", {}),
             defaults={
                 "per_second": self.rc_message.per_second,

synapse/config/registration.py

@@ -13,7 +13,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 import argparse
-import logging
 from typing import Any, Optional
 
 from synapse.api.constants import RoomCreationPreset
@@ -21,17 +20,6 @@ from synapse.config._base import Config, ConfigError
 from synapse.types import JsonDict, RoomAlias, UserID
 from synapse.util.stringutils import random_string_with_symbols, strtobool
 
-logger = logging.getLogger(__name__)
-
-LEGACY_EMAIL_DELEGATE_WARNING = """\
-Delegation of email verification to an identity server is now deprecated. To
-continue to allow users to add email addresses to their accounts, and use them for
-password resets, configure Synapse with an SMTP server via the `email` setting, and
-remove `account_threepid_delegates.email`.
-
-This will be an error in a future version.
-"""
-
 
 class RegistrationConfig(Config):
     section = "registration"
@@ -63,9 +51,6 @@ class RegistrationConfig(Config):
         self.bcrypt_rounds = config.get("bcrypt_rounds", 12)
 
         account_threepid_delegates = config.get("account_threepid_delegates") or {}
-        if "email" in account_threepid_delegates:
-            logger.warning(LEGACY_EMAIL_DELEGATE_WARNING)
-
         self.account_threepid_delegate_email = account_threepid_delegates.get("email")
         self.account_threepid_delegate_msisdn = account_threepid_delegates.get("msisdn")
         self.default_identity_server = config.get("default_identity_server")

synapse/config/repository.py

@@ -42,18 +42,6 @@ THUMBNAIL_SIZE_YAML = """\
         #    method: %(method)s
 """
 
-# A map from the given media type to the type of thumbnail we should generate
-# for it.
-THUMBNAIL_SUPPORTED_MEDIA_FORMAT_MAP = {
-    "image/jpeg": "jpeg",
-    "image/jpg": "jpeg",
-    "image/webp": "jpeg",
-    # Thumbnails can only be jpeg or png. We choose png thumbnails for gif
-    # because it can have transparency.
-    "image/gif": "png",
-    "image/png": "png",
-}
-
 HTTP_PROXY_SET_WARNING = """\
 The Synapse config url_preview_ip_range_blacklist will be ignored as an HTTP(s) proxy is configured."""
 
@@ -91,22 +79,13 @@ def parse_thumbnail_requirements(
         width = size["width"]
         height = size["height"]
         method = size["method"]
-
-        for format, thumbnail_format in THUMBNAIL_SUPPORTED_MEDIA_FORMAT_MAP.items():
-            requirement = requirements.setdefault(format, [])
-            if thumbnail_format == "jpeg":
-                requirement.append(
-                    ThumbnailRequirement(width, height, method, "image/jpeg")
-                )
-            elif thumbnail_format == "png":
-                requirement.append(
-                    ThumbnailRequirement(width, height, method, "image/png")
-                )
-            else:
-                raise Exception(
-                    "Unknown thumbnail mapping from %s to %s. This is a Synapse problem, please report!"
-                    % (format, thumbnail_format)
-                )
+        jpeg_thumbnail = ThumbnailRequirement(width, height, method, "image/jpeg")
+        png_thumbnail = ThumbnailRequirement(width, height, method, "image/png")
+        requirements.setdefault("image/jpeg", []).append(jpeg_thumbnail)
+        requirements.setdefault("image/jpg", []).append(jpeg_thumbnail)
+        requirements.setdefault("image/webp", []).append(jpeg_thumbnail)
+        requirements.setdefault("image/gif", []).append(png_thumbnail)
+        requirements.setdefault("image/png", []).append(png_thumbnail)
     return {
         media_type: tuple(thumbnails) for media_type, thumbnails in requirements.items()
     }