update sample config

changelog
add default_power_level_content_override config option.
2022-04-23 15:39:41 +01:00 · 2022-04-23 15:36:41 +01:00 · 2022-04-23 15:25:25 +01:00 · 2022-04-22 18:20:06 +01:00 · 2022-04-22 16:03:46 +01:00 · 2022-04-22 14:23:40 +01:00
142 changed files with 5607 additions and 1109 deletions
@@ -0,0 +1,4 @@
+---
+title: CI run against latest deps is failing
+---
+See https://github.com/{{env.GITHUB_REPOSITORY}}/actions/runs/{{env.GITHUB_RUN_ID}}
@@ -1,8 +0,0 @@
-#!/bin/sh
-
-# replaces the dependency on Twisted in `python_dependencies` with trunk.
-
-set -e
-cd "$(dirname "$0")"/..
-
-sed -i -e 's#"Twisted.*"#"Twisted @ git+https://github.com/twisted/twisted"#' synapse/python_dependencies.py
@@ -8,8 +8,4 @@
 !pyproject.toml
 !poetry.lock

-# TODO: remove these once we have moved over to using poetry-core in pyproject.toml
-!MANIFEST.in
-!setup.py
-
 **/__pycache__
@@ -0,0 +1,156 @@
+# People who are freshly `pip install`ing from PyPI will pull in the latest versions of
+# dependencies which match the broad requirements. Since most CI runs are against
+# the locked poetry environment, run specifically against the latest dependencies to
+# know if there's an upcoming breaking change.
+#
+# As an overview this workflow:
+# - checks out develop,
+# - installs from source, pulling in the dependencies like a fresh `pip install` would, and 
+# - runs mypy and test suites in that checkout.
+#
+# Based on the twisted trunk CI job.
+
+name: Latest dependencies
+
+on:
+  schedule:
+    - cron: 0 7 * * *
+  workflow_dispatch:
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  mypy:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      # The dev dependencies aren't exposed in the wheel metadata (at least with current
+      # poetry-core versions), so we install with poetry.
+      - uses: matrix-org/setup-python-poetry@v1
+        with:
+          python-version: "3.x"
+          poetry-version: "1.2.0b1"
+      # Dump installed versions for debugging.
+      - run: poetry run pip list > before.txt
+      # Upgrade all runtime dependencies only. This is intended to mimic a fresh
+      # `pip install matrix-synapse[all]` as closely as possible.
+      - run: poetry update --no-dev
+      - run: poetry run pip list > after.txt && (diff -u before.txt after.txt || true)
+      - run: poetry run mypy
+  trial:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        include:
+          - database: "sqlite"
+          - database: "postgres"
+            postgres-version: "14"
+
+    steps:
+      - uses: actions/checkout@v2
+      - run: sudo apt-get -qq install xmlsec1
+      - name: Set up PostgreSQL ${{ matrix.postgres-version }}
+        if: ${{ matrix.postgres-version }}
+        run: |
+          docker run -d -p 5432:5432 \
+            -e POSTGRES_PASSWORD=postgres \
+            -e POSTGRES_INITDB_ARGS="--lc-collate C --lc-ctype C --encoding UTF8" \
+            postgres:${{ matrix.postgres-version }}
+      - uses: actions/setup-python@v2
+        with:
+          python-version: "3.x"
+      - run: pip install .[all,test]
+      - name: Await PostgreSQL
+        if: ${{ matrix.postgres-version }}
+        timeout-minutes: 2
+        run: until pg_isready -h localhost; do sleep 1; done
+      - run: python -m twisted.trial --jobs=2 tests
+        env:
+          SYNAPSE_POSTGRES: ${{ matrix.database == 'postgres' || '' }}
+          SYNAPSE_POSTGRES_HOST: localhost
+          SYNAPSE_POSTGRES_USER: postgres
+          SYNAPSE_POSTGRES_PASSWORD: postgres
+      - name: Dump logs
+        # Logs are most useful when the command fails, always include them.
+        if: ${{ always() }}
+        # Note: Dumps to workflow logs instead of using actions/upload-artifact
+        #       This keeps logs colocated with failing jobs
+        #       It also ignores find's exit code; this is a best effort affair
+        run: >-
+          find _trial_temp -name '*.log'
+          -exec echo "::group::{}" \;
+          -exec cat {} \;
+          -exec echo "::endgroup::" \;
+          || true
+
+
+  sytest:
+    runs-on: ubuntu-latest
+    container:
+      image: matrixdotorg/sytest-synapse:testing
+      volumes:
+        - ${{ github.workspace }}:/src
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - sytest-tag: focal
+
+          - sytest-tag: focal
+            postgres: postgres
+            workers: workers
+            redis: redis
+    env:
+      POSTGRES: ${{ matrix.postgres && 1}}
+      WORKERS: ${{ matrix.workers && 1 }}
+      REDIS: ${{ matrix.redis && 1 }}
+      BLACKLIST: ${{ matrix.workers && 'synapse-blacklist-with-workers' }}
+
+    steps:
+      - uses: actions/checkout@v2
+      - name: Ensure sytest runs `pip install`
+        # Delete the lockfile so sytest will `pip install` rather than `poetry install`
+        run: rm /src/poetry.lock
+        working-directory: /src
+      - name: Prepare test blacklist
+        run: cat sytest-blacklist .ci/worker-blacklist > synapse-blacklist-with-workers
+      - name: Run SyTest
+        run: /bootstrap.sh synapse
+        working-directory: /src
+      - name: Summarise results.tap
+        if: ${{ always() }}
+        run: /sytest/scripts/tap_to_gha.pl /logs/results.tap
+      - name: Upload SyTest logs
+        uses: actions/upload-artifact@v2
+        if: ${{ always() }}
+        with:
+          name: Sytest Logs - ${{ job.status }} - (${{ join(matrix.*, ', ') }})
+          path: |
+            /logs/results.tap
+            /logs/**/*.log*
+
+
+  # TODO: run complement (as with twisted trunk, see #12473).
+
+  # open an issue if the build fails, so we know about it.
+  open-issue:
+    if: failure()
+    needs:
+      # TODO: should mypy be included here? It feels more brittle than the other two.
+      - mypy
+      - trial
+      - sytest
+
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v2
+      - uses: JasonEtco/create-an-issue@5d9504915f79f9cc6d791934b8ef34f2353dd74d # v2.5.0, 2020-12-06
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        with:
+          update_existing: true
+          filename: .ci/latest_deps_build_failed_issue_template.md
+
@@ -15,24 +15,18 @@ jobs:
    steps:
      - uses: actions/checkout@v2
      - uses: actions/setup-python@v2
-      - run: pip install -e .
+      - run: pip install .
      - run: scripts-dev/generate_sample_config.sh --check
      - run: scripts-dev/config-lint.sh

  lint:
-    runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        toxenv:
-          - "check_codestyle"
-          - "check_isort"
-          - "mypy"
-
-    steps:
-      - uses: actions/checkout@v2
-      - uses: actions/setup-python@v2
-      - run: pip install tox
-      - run: tox -e ${{ matrix.toxenv }}
+    # This does a vanilla `poetry install` - no extras. I'm slightly anxious
+    # that we might skip some typechecks on code that uses extras. However,
+    # I think the right way to fix this is to mark any extras needed for
+    # typechecking as development dependencies. To detect this, we ought to
+    # turn up mypy's strictness: disallow unknown imports and be accept fewer
+    # uses of `Any`.
+    uses: "matrix-org/backend-meta/.github/workflows/python-poetry-ci.yml@v1"

  lint-crlf:
    runs-on: ubuntu-latest
@@ -71,23 +65,23 @@ jobs:
      matrix:
        python-version: ["3.7", "3.8", "3.9", "3.10"]
        database: ["sqlite"]
-        toxenv: ["py"]
+        extras: ["all"]
        include:
          # Newest Python without optional deps
          - python-version: "3.10"
-            toxenv: "py-noextras"
+            extras: ""

          # Oldest Python with PostgreSQL
          - python-version: "3.7"
            database: "postgres"
            postgres-version: "10"
-            toxenv: "py"
+            extras: "all"

          # Newest Python with newest PostgreSQL
          - python-version: "3.10"
            database: "postgres"
            postgres-version: "14"
-            toxenv: "py"
+            extras: "all"

    steps:
      - uses: actions/checkout@v2
@@ -99,17 +93,16 @@ jobs:
            -e POSTGRES_PASSWORD=postgres \
            -e POSTGRES_INITDB_ARGS="--lc-collate C --lc-ctype C --encoding UTF8" \
            postgres:${{ matrix.postgres-version }}
-      - uses: actions/setup-python@v2
+      - uses: matrix-org/setup-python-poetry@v1
        with:
          python-version: ${{ matrix.python-version }}
-      - run: pip install tox
+          extras: ${{ matrix.extras }}
      - name: Await PostgreSQL
        if: ${{ matrix.postgres-version }}
        timeout-minutes: 2
        run: until pg_isready -h localhost; do sleep 1; done
-      - run: tox -e ${{ matrix.toxenv }}
+      - run: poetry run trial --jobs=2 tests
        env:
-          TRIAL_FLAGS: "--jobs=2"
          SYNAPSE_POSTGRES: ${{ matrix.database == 'postgres' || '' }}
          SYNAPSE_POSTGRES_HOST: localhost
          SYNAPSE_POSTGRES_USER: postgres
@@ -156,23 +149,24 @@ jobs:

  trial-pypy:
    # Very slow; only run if the branch name includes 'pypy'
+    # Note: sqlite only; no postgres. Completely untested since poetry move.
    if: ${{ contains(github.ref, 'pypy') && !failure() && !cancelled() }}
    needs: linting-done
    runs-on: ubuntu-latest
    strategy:
      matrix:
        python-version: ["pypy-3.7"]
+        extras: ["all"]

    steps:
      - uses: actions/checkout@v2
+      # Install libs necessary for PyPy to build binary wheels for dependencies
      - run: sudo apt-get -qq install xmlsec1 libxml2-dev libxslt-dev
-      - uses: actions/setup-python@v2
+      - uses: matrix-org/setup-python-poetry@v1
        with:
          python-version: ${{ matrix.python-version }}
-      - run: pip install tox
-      - run: tox -e py
-        env:
-          TRIAL_FLAGS: "--jobs=2"
+          extras: ${{ matrix.extras }}
+      - run: poetry run trial --jobs=2 tests
      - name: Dump logs
        # Logs are most useful when the command fails, always include them.
        if: ${{ always() }}
@@ -6,16 +6,25 @@ on:

  workflow_dispatch:

+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
 jobs:
  mypy:
    runs-on: ubuntu-latest

    steps:
      - uses: actions/checkout@v2
-      - uses: actions/setup-python@v2
-      - run: .ci/patch_for_twisted_trunk.sh
-      - run: pip install tox
-      - run: tox -e mypy
+      - uses: matrix-org/setup-python-poetry@v1
+        with:
+          python-version: "3.x"
+          extras: "all"
+      - run: |
+          poetry remove twisted
+          poetry add --extras tls git+https://github.com/twisted/twisted.git#trunk
+          poetry install --no-interaction --extras "all test"
+      - run: poetry run mypy

  trial:
    runs-on: ubuntu-latest
@@ -23,14 +32,15 @@ jobs:
    steps:
      - uses: actions/checkout@v2
      - run: sudo apt-get -qq install xmlsec1
-      - uses: actions/setup-python@v2
+      - uses: matrix-org/setup-python-poetry@v1
        with:
-          python-version: 3.7
-      - run: .ci/patch_for_twisted_trunk.sh
-      - run: pip install tox
-      - run: tox -e py
-        env:
-          TRIAL_FLAGS: "--jobs=2"
+          python-version: "3.x"
+          extras: "all test"
+      - run: |
+          poetry remove twisted
+          poetry add --extras tls git+https://github.com/twisted/twisted.git#trunk
+          poetry install --no-interaction --extras "all test"
+      - run: poetry run trial --jobs 2 tests

      - name: Dump logs
        # Logs are most useful when the command fails, always include them.
@@ -55,11 +65,23 @@ jobs:
    steps:
      - uses: actions/checkout@v2
      - name: Patch dependencies
-        run: .ci/patch_for_twisted_trunk.sh
+        # Note: The poetry commands want to create a virtualenv in /src/.venv/,
+        #       but the sytest-synapse container expects it to be in /venv/.
+        #       We symlink it before running poetry so that poetry actually
+        #       ends up installing to `/venv`.
+        run: |
+          ln -s -T /venv /src/.venv
+          poetry remove twisted
+          poetry add --extras tls git+https://github.com/twisted/twisted.git#trunk
+          poetry install --no-interaction --extras "all test"
        working-directory: /src
      - name: Run SyTest
        run: /bootstrap.sh synapse
        working-directory: /src
+        env:
+          # Use offline mode to avoid reinstalling the pinned version of
+          # twisted.
+          OFFLINE: 1
      - name: Summarise results.tap
        if: ${{ always() }}
        run: /sytest/scripts/tap_to_gha.pl /logs/results.tap
@@ -15,8 +15,7 @@ _trial_temp*/
 .DS_Store
 __pycache__/

-# We do want the poetry lockfile. TODO: is there a good reason for ignoring
-# '*.lock' above? If not, let's nuke it.
+# We do want the poetry lockfile.
 !poetry.lock

 # stuff that is likely to exist when you run a server locally
@@ -1,54 +0,0 @@
-include LICENSE
-include VERSION
-include *.rst
-include *.md
-include demo/README
-include demo/demo.tls.dh
-include demo/*.py
-include demo/*.sh
-
-include synapse/py.typed
-recursive-include synapse/storage *.sql
-recursive-include synapse/storage *.sql.postgres
-recursive-include synapse/storage *.sql.sqlite
-recursive-include synapse/storage *.py
-recursive-include synapse/storage *.txt
-recursive-include synapse/storage *.md
-
-recursive-include docs *
-recursive-include scripts-dev *
-recursive-include synapse *.pyi
-recursive-include tests *.py
-recursive-include tests *.pem
-recursive-include tests *.p8
-recursive-include tests *.crt
-recursive-include tests *.key
-
-recursive-include synapse/res *
-recursive-include synapse/static *.css
-recursive-include synapse/static *.gif
-recursive-include synapse/static *.html
-recursive-include synapse/static *.js
-
-exclude .codecov.yml
-exclude .coveragerc
-exclude .dockerignore
-exclude .editorconfig
-exclude Dockerfile
-exclude mypy.ini
-exclude sytest-blacklist
-exclude test_postgresql.sh
-
-include book.toml
-include pyproject.toml
-recursive-include changelog.d *
-
-include .flake8
-prune .circleci
-prune .github
-prune .ci
-prune contrib
-prune debian
-prune demo/etc
-prune docker
-prune stubs
@@ -293,39 +293,42 @@ directory of your choice::
    git clone https://github.com/matrix-org/synapse.git
    cd synapse

-Synapse has a number of external dependencies, that are easiest
-to install using pip and a virtualenv::
+Synapse has a number of external dependencies. We maintain a fixed development
+environment using [poetry](https://python-poetry.org/). First, install poetry. We recommend

-    python3 -m venv ./env
-    source ./env/bin/activate
-    pip install -e ".[all,dev]"
+    pip install --user pipx
+    pipx install poetry
+
+as described `here <https://python-poetry.org/docs/#installing-with-pipx>`_.
+(See `poetry's installation docs <https://python-poetry.org/docs/#installation>`
+for other installation methods.) Then ask poetry to create a virtual environment
+from the project and install Synapse's dependencies::
+
+    poetry install --extras "all test"

 This will run a process of downloading and installing all the needed
-dependencies into a virtual env. If any dependencies fail to install,
-try installing the failing modules individually::
-
-    pip install -e "module-name"
+dependencies into a virtual env.

 We recommend using the demo which starts 3 federated instances running on ports `8080` - `8082`

-    ./demo/start.sh
+    poetry run ./demo/start.sh

-(to stop, you can use `./demo/stop.sh`)
+(to stop, you can use `poetry run ./demo/stop.sh`)

-See the [demo documentation](https://matrix-org.github.io/synapse/develop/development/demo.html)
+See the `demo documentation <https://matrix-org.github.io/synapse/develop/development/demo.html>`_
 for more information.

 If you just want to start a single instance of the app and run it directly::

    # Create the homeserver.yaml config once
-    python -m synapse.app.homeserver \
+    poetry run synapse_homeserver \
      --server-name my.domain.name \
      --config-path homeserver.yaml \
      --generate-config \
      --report-stats=[yes|no]

    # Start the app
-    python -m synapse.app.homeserver --config-path homeserver.yaml
+    poetry run synapse_homeserver --config-path homeserver.yaml


 Running the unit tests
@@ -334,7 +337,7 @@ Running the unit tests
 After getting up and running, you may wish to run Synapse's unit tests to
 check that everything is installed correctly::

-    trial tests
+    poetry run trial tests

 This should end with a 'PASSED' result (note that exact numbers will
 differ)::
@@ -0,0 +1 @@
+Implement [MSC3383](https://github.com/matrix-org/matrix-spec-proposals/pull/3383) for including the destination in server-to-server authentication headers. Contributed by @Bubu and @jcgruenhage for Famedly GmbH.
@@ -0,0 +1 @@
+Prevent a sync request from removing a user's busy presence status.
@@ -0,0 +1 @@
+Fix bug with incremental sync missing events when rejoining/backfilling. Contributed by Nick @ Beeper.
@@ -0,0 +1 @@
+Use poetry to manage Synapse's dependencies.
@@ -0,0 +1 @@
+Fix rendering of the documentation site when using the 'print' feature.
@@ -0,0 +1 @@
+The groups/communities feature in Synapse has been disabled by default.
@@ -0,0 +1 @@
+Enable processing of device list updates asynchronously.
@@ -0,0 +1 @@
+Add a manual documenting config file options.
@@ -0,0 +1 @@
+Remove unstable identifiers from [MSC3440](https://github.com/matrix-org/matrix-doc/pull/3440).
@@ -0,0 +1 @@
+Preparation for faster-room-join work: start a background process to resynchronise the room state after a room join.
@@ -0,0 +1 @@
+Remove an unstable identifier from [MSC3083](https://github.com/matrix-org/matrix-doc/pull/3083).
@@ -0,0 +1 @@
+Preparation for faster-room-join work: Implement a tracking mechanism to allow functions to wait for full room state to arrive.
@@ -0,0 +1 @@
+Run twisted trunk CI job in the locked poetry environment.
@@ -0,0 +1 @@
+Implement [MSC2815](https://github.com/matrix-org/matrix-spec-proposals/pull/2815) to allow room moderators to view redacted event content. Contributed by @tulir.
@@ -0,0 +1 @@
+Run lints under poetry in CI, and remove corresponding tox lint jobs.
@@ -0,0 +1 @@
+Run "main" trial tests under `poetry`.
@@ -0,0 +1 @@
+Bump twisted version in `poetry.lock` to work around [pip bug #9644](https://github.com/pypa/pip/issues/9644).
@@ -0,0 +1 @@
+Change Mutual Rooms' `unstable_features` flag to `uk.half-shot.msc2666.mutual_rooms` which matches the current MSC iteration.
@@ -0,0 +1 @@
+Use `poetry` to manage the virtualenv in debian packages.
@@ -0,0 +1 @@
+Fix typo in the release script help string.
@@ -0,0 +1 @@
+Update documentation to reflect that both the `run_background_tasks_on` option and the options for moving stream writers off of the main process are no longer experimental.
@@ -0,0 +1 @@
+Limit length of device_id to less than 512 characters.
@@ -0,0 +1 @@
+Reintroduce the list of targets to the linter script, to avoid linting unwanted local-only directories during development.
@@ -0,0 +1 @@
+Update worker documentation and replace old `federation_reader` with `generic_worker`.
@@ -0,0 +1 @@
+Dockerfile-workers: reduce the amount we install in the image.
@@ -0,0 +1 @@
+Enable processing of device list updates asynchronously.
@@ -0,0 +1 @@
+Dockerfile-workers: give the master its own log config.
@@ -0,0 +1 @@
+complement-synapse-workers: factor out separate entry point script.
@@ -0,0 +1 @@
+Update `delay_cancellation` to accept any awaitable, rather than just `Deferred`s.
@@ -0,0 +1 @@
+Add a CI job which tests Synapse against the latest version of all dependencies.
@@ -0,0 +1 @@
+Back out experimental implementation of [MSC2314](https://github.com/matrix-org/matrix-spec-proposals/pull/2314).
@@ -0,0 +1 @@
+Strongly recommend `poetry` for development.
@@ -0,0 +1 @@
+Fix a long-standing bug which incorrectly caused `GET /_matrix/client/r3/rooms/{roomId}/event/{eventId}` to return edited events rather than the original.
@@ -0,0 +1 @@
+Use poetry-core instead of setuptools to build wheels.
@@ -0,0 +1 @@
+Fix grammatical error in federation error response when the room version of a room is unknown.
@@ -0,0 +1 @@
+Fix a broken link in `README.rst`.
@@ -0,0 +1 @@
+Fix bug where the admin API for [deleting forward extremities](https://github.com/matrix-org/synapse/blob/erikj/fix_delete_event_response_count/docs/admin_api/rooms.md#deleting-forward-extremities) would always return a count of 1 no matter how many extremities were deleted. Broke in v1.27.0.
@@ -0,0 +1 @@
+Fix a minor typo in the Debian changelogs generated by the release script.
@@ -0,0 +1 @@
+Fix a long-standing bug where the image thumbanils embedded into email notifications were broken.
@@ -0,0 +1 @@
+Remove unnecessary configuration overrides in tests.
@@ -0,0 +1 @@
+Use poetry-core instead of setuptools to build wheels.
@@ -0,0 +1 @@
+Fix a bug in the implementation of MSC3202 where Synapse would use the field name `device_unused_fallback_keys`, rather than `device_unused_fallback_key_types`.
@@ -0,0 +1 @@
+Add default_power_level_content_override config option.
@@ -30,9 +30,19 @@ case $(dpkg-architecture -q DEB_HOST_ARCH) in
        ;;
 esac

-# Use --builtin-venv to use the better `venv` module from CPython 3.4+ rather
-# than the 2/3 compatible `virtualenv`.
+# Manually install Poetry and export a pip-compatible `requirements.txt`
+# We need a Poetry pre-release as the export command is buggy in < 1.2
+TEMP_VENV="$(mktemp -d)"
+python3 -m venv "$TEMP_VENV"
+source "$TEMP_VENV/bin/activate"
+pip install -U pip
+pip install poetry==1.2.0b1
+poetry export --extras all --extras test -o exported_requirements.txt
+deactivate
+rm -rf "$TEMP_VENV"

+# Use --no-deps to only install pinned versions in exported_requirements.txt,
+# and to avoid https://github.com/pypa/pip/issues/9644
 dh_virtualenv \
    --install-suffix "matrix-synapse" \
    --builtin-venv \
@@ -41,9 +51,11 @@ dh_virtualenv \
    --preinstall="lxml" \
    --preinstall="mock" \
    --preinstall="wheel" \
+    --extra-pip-arg="--no-deps" \
    --extra-pip-arg="--no-cache-dir" \
    --extra-pip-arg="--compile" \
-    --extras="all,systemd,test"
+    --extras="all,systemd,test" \
+    --requirements="exported_requirements.txt"

 PACKAGE_BUILD_DIR="debian/matrix-synapse-py3"
 VIRTUALENV_DIR="${PACKAGE_BUILD_DIR}${DH_VIRTUALENV_INSTALL_ROOT}/matrix-synapse"
@@ -1,3 +1,9 @@
+matrix-synapse-py3 (1.58.0+nmu1) UNRELEASED; urgency=medium
+
+  * Use poetry to manage the bundled virtualenv included with this package.
+
+ -- Synapse Packaging team <packages@matrix.org>  Wed, 30 Mar 2022 12:21:43 +0100
+
 matrix-synapse-py3 (1.57.1) stable; urgency=medium

  * New synapse release 1.57.1.
@@ -0,0 +1 @@
+exported_requirements.txt
@@ -59,7 +59,7 @@ RUN --mount=type=cache,target=/root/.cache/pip \
 WORKDIR /synapse

 # Copy just what we need to run `poetry export`...
-COPY pyproject.toml poetry.lock README.rst /synapse/
+COPY pyproject.toml poetry.lock /synapse/

 RUN /root/.local/bin/poetry export --extras all -o /synapse/requirements.txt

@@ -98,9 +98,7 @@ RUN --mount=type=cache,target=/root/.cache/pip \
 # Copy over the rest of the synapse source code.
 COPY synapse /synapse/synapse/
 # ... and what we need to `pip install`.
-# TODO: once pyproject.toml declares poetry-core as its build system, we'll need to copy
-# pyproject.toml here, ditching setup.py and MANIFEST.in.
-COPY setup.py MANIFEST.in README.rst /synapse/
+COPY pyproject.toml README.rst /synapse/

 # Install the synapse package itself.
 RUN pip install --prefix="/install" --no-deps --no-warn-script-location /synapse
@@ -2,10 +2,19 @@
 FROM matrixdotorg/synapse

 # Install deps
-RUN apt-get update
-RUN apt-get install -y supervisor redis nginx
+RUN \
+   --mount=type=cache,target=/var/cache/apt,sharing=locked \
+   --mount=type=cache,target=/var/lib/apt,sharing=locked \
+  apt-get update && \
+  DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
+     redis-server nginx-light

-# Remove the default nginx sites
+# Install supervisord with pip instead of apt, to avoid installing a second
+# copy of python.
+RUN --mount=type=cache,target=/root/.cache/pip \
+    pip install supervisor~=4.2
+
+# Disable the default nginx sites
 RUN rm /etc/nginx/sites-enabled/default

 # Copy Synapse worker, nginx and supervisord configuration template files
@@ -19,5 +28,7 @@ EXPOSE 8080/tcp
 COPY ./docker/configure_workers_and_start.py /configure_workers_and_start.py
 ENTRYPOINT ["/configure_workers_and_start.py"]

+# Replace the healthcheck with one which checks *all* the workers. The script
+# is generated by configure_workers_and_start.py.
 HEALTHCHECK --start-period=5s --interval=15s --timeout=5s \
    CMD /bin/sh /healthcheck.sh
@@ -13,8 +13,8 @@ RUN curl -OL "https://github.com/caddyserver/caddy/releases/download/v2.3.0/cadd
  tar xzf caddy_2.3.0_linux_amd64.tar.gz && rm caddy_2.3.0_linux_amd64.tar.gz && mv caddy /root

 # Install postgresql
-RUN apt-get update
-RUN apt-get install -y postgresql
+RUN apt-get update && \
+  DEBIAN_FRONTEND=noninteractive apt-get install --no-install-recommends -y postgresql-13

 # Configure a user and create a database for Synapse
 RUN pg_ctlcluster 13 main start &&  su postgres -c "echo \
@@ -34,40 +34,14 @@ WORKDIR /data
 # Copy the caddy config
 COPY conf-workers/caddy.complement.json /root/caddy.json

+# Copy the entrypoint
+COPY conf-workers/start-complement-synapse-workers.sh /
+
 # Expose caddy's listener ports
 EXPOSE 8008 8448

-ENTRYPOINT \
-  # Replace the server name in the caddy config
-  sed -i "s/{{ server_name }}/${SERVER_NAME}/g" /root/caddy.json && \
-  # Start postgres
-  pg_ctlcluster 13 main start 2>&1 && \
-  # Start caddy
-  /root/caddy start --config /root/caddy.json 2>&1 && \
-  # Set the server name of the homeserver
-  SYNAPSE_SERVER_NAME=${SERVER_NAME} \
-  # No need to report stats here
-  SYNAPSE_REPORT_STATS=no \
-  # Set postgres authentication details which will be placed in the homeserver config file
-  POSTGRES_PASSWORD=somesecret POSTGRES_USER=postgres POSTGRES_HOST=localhost \
-  # Specify the workers to test with
-  SYNAPSE_WORKER_TYPES="\
-    event_persister, \
-    event_persister, \
-    background_worker, \
-    frontend_proxy, \
-    event_creator, \
-    user_dir, \
-    media_repository, \
-    federation_inbound, \
-    federation_reader, \
-    federation_sender, \
-    synchrotron, \
-    appservice, \
-    pusher" \
-  # Run the script that writes the necessary config files and starts supervisord, which in turn
-  # starts everything else
-  /configure_workers_and_start.py
+ENTRYPOINT /start-complement-synapse-workers.sh

+# Update the healthcheck to have a shorter check interval
 HEALTHCHECK --start-period=5s --interval=1s --timeout=1s \
    CMD /bin/sh /healthcheck.sh
@@ -0,0 +1,50 @@
+#!/bin/bash
+#
+# Default ENTRYPOINT for the docker image used for testing synapse with workers under complement
+
+set -e
+
+function log {
+    d=$(date +"%Y-%m-%d %H:%M:%S,%3N")
+    echo "$d $@"
+}
+
+# Replace the server name in the caddy config
+sed -i "s/{{ server_name }}/${SERVER_NAME}/g" /root/caddy.json
+
+log "starting postgres"
+pg_ctlcluster 13 main start
+
+log "starting caddy"
+/root/caddy start --config /root/caddy.json
+
+# Set the server name of the homeserver
+export SYNAPSE_SERVER_NAME=${SERVER_NAME}
+
+# No need to report stats here
+export SYNAPSE_REPORT_STATS=no
+
+# Set postgres authentication details which will be placed in the homeserver config file
+export POSTGRES_PASSWORD=somesecret
+export POSTGRES_USER=postgres
+export POSTGRES_HOST=localhost
+
+# Specify the workers to test with
+export SYNAPSE_WORKER_TYPES="\
+    event_persister, \
+    event_persister, \
+    background_worker, \
+    frontend_proxy, \
+    event_creator, \
+    user_dir, \
+    media_repository, \
+    federation_inbound, \
+    federation_reader, \
+    federation_sender, \
+    synchrotron, \
+    appservice, \
+    pusher"
+
+# Run the script that writes the necessary config files and starts supervisord, which in turn
+# starts everything else
+exec /configure_workers_and_start.py
@@ -103,8 +103,10 @@ experimental_features:
  spaces_enabled: true
  # Enable history backfilling support
  msc2716_enabled: true
-  # server-side support for partial state in /send_join
+  # server-side support for partial state in /send_join responses
  msc3706_enabled: true
+  # client-side support for partial state in /send_join responses
+  faster_joins: true
  # Enable jump to date endpoint
  msc3030_enabled: true

@@ -5,6 +5,9 @@
 nodaemon=true
 user=root

+[include]
+files = /etc/supervisor/conf.d/*.conf
+
 [program:nginx]
 command=/usr/sbin/nginx -g "daemon off;"
 priority=500
@@ -29,7 +29,7 @@
 import os
 import subprocess
 import sys
-from typing import Any, Dict, Set
+from typing import Any, Dict, Mapping, Set

 import jinja2
 import yaml
@@ -341,7 +341,7 @@ def generate_worker_files(environ, config_path: str, data_dir: str):
    # base shared worker jinja2 template.
    #
    # This config file will be passed to all workers, included Synapse's main process.
-    shared_config = {"listeners": listeners}
+    shared_config: Dict[str, Any] = {"listeners": listeners}

    # The supervisord config. The contents of which will be inserted into the
    # base supervisord jinja2 template.
@@ -446,21 +446,7 @@ def generate_worker_files(environ, config_path: str, data_dir: str):

        # Write out the worker's logging config file

-        # Check whether we should write worker logs to disk, in addition to the console
-        extra_log_template_args = {}
-        if environ.get("SYNAPSE_WORKERS_WRITE_LOGS_TO_DISK"):
-            extra_log_template_args["LOG_FILE_PATH"] = "{dir}/logs/{name}.log".format(
-                dir=data_dir, name=worker_name
-            )
-
-        # Render and write the file
-        log_config_filepath = "/conf/workers/{name}.log.config".format(name=worker_name)
-        convert(
-            "/conf/log.config",
-            log_config_filepath,
-            worker_name=worker_name,
-            **extra_log_template_args,
-        )
+        log_config_filepath = generate_worker_log_config(environ, worker_name, data_dir)

        # Then a worker config file
        convert(
@@ -496,6 +482,10 @@ def generate_worker_files(environ, config_path: str, data_dir: str):

    # Finally, we'll write out the config files.

+    # log config for the master process
+    master_log_config = generate_worker_log_config(environ, "master", data_dir)
+    shared_config["log_config"] = master_log_config
+
    # Shared homeserver config
    convert(
        "/conf/shared.yaml.j2",
@@ -512,9 +502,10 @@ def generate_worker_files(environ, config_path: str, data_dir: str):
    )

    # Supervisord config
+    os.makedirs("/etc/supervisor", exist_ok=True)
    convert(
        "/conf/supervisord.conf.j2",
-        "/etc/supervisor/conf.d/supervisord.conf",
+        "/etc/supervisor/supervisord.conf",
        main_config_path=config_path,
        worker_config=supervisord_config,
    )
@@ -532,12 +523,28 @@ def generate_worker_files(environ, config_path: str, data_dir: str):
        os.mkdir(log_dir)


-def start_supervisord():
-    """Starts up supervisord which then starts and monitors all other necessary processes
+def generate_worker_log_config(
+    environ: Mapping[str, str], worker_name: str, data_dir: str
+) -> str:
+    """Generate a log.config file for the given worker.

-    Raises: CalledProcessError if calling start.py return a non-zero exit code.
+    Returns: the path to the generated file
    """
-    subprocess.run(["/usr/bin/supervisord"], stdin=subprocess.PIPE)
+    # Check whether we should write worker logs to disk, in addition to the console
+    extra_log_template_args = {}
+    if environ.get("SYNAPSE_WORKERS_WRITE_LOGS_TO_DISK"):
+        extra_log_template_args["LOG_FILE_PATH"] = "{dir}/logs/{name}.log".format(
+            dir=data_dir, name=worker_name
+        )
+    # Render and write the file
+    log_config_filepath = "/conf/workers/{name}.log.config".format(name=worker_name)
+    convert(
+        "/conf/log.config",
+        log_config_filepath,
+        worker_name=worker_name,
+        **extra_log_template_args,
+    )
+    return log_config_filepath


 def main(args, environ):
@@ -567,7 +574,13 @@ def main(args, environ):

    # Start supervisord, which will start Synapse, all of the configured worker
    # processes, redis, nginx etc. according to the config we created above.
-    start_supervisord()
+    log("Starting supervisord")
+    os.execl(
+        "/usr/local/bin/supervisord",
+        "supervisord",
+        "-c",
+        "/etc/supervisor/supervisord.conf",
+    )


 if __name__ == "__main__":
@@ -17,6 +17,7 @@
 # Usage
  - [Federation](federate.md)
  - [Configuration](usage/configuration/README.md)
+    - [Configuration Manual](usage/configuration/config_documentation.md) 
    - [Homeserver Sample Config File](usage/configuration/homeserver_sample_config.md)
    - [Logging Sample Config File](usage/configuration/logging_sample_config.md)
    - [Structured Logging](structured_logging.md)
@@ -6,60 +6,36 @@ The Synapse codebase uses a number of code formatting tools in order to
 quickly and automatically check for formatting (and sometimes logical)
 errors in code.

-The necessary tools are detailed below.
+The necessary tools are:

-First install them with:
+- [black](https://black.readthedocs.io/en/stable/), a source code formatter;
+- [isort](https://pycqa.github.io/isort/), which organises each file's imports;
+- [flake8](https://flake8.pycqa.org/en/latest/), which can spot common errors; and
+- [mypy](https://mypy.readthedocs.io/en/stable/), a type checker.
+
+Install them with:

 ```sh
 pip install -e ".[lint,mypy]"
 ```

-   **black**
+The easiest way to run the lints is to invoke the linter script as follows.

-    The Synapse codebase uses [black](https://pypi.org/project/black/)
-    as an opinionated code formatter, ensuring all comitted code is
-    properly formatted.
-
-    Have `black` auto-format your code (it shouldn't change any
-    functionality) with:
-
-    ```sh
-    black .
-    ```
-
-   **flake8**
-
-    `flake8` is a code checking tool. We require code to pass `flake8`
-    before being merged into the codebase.
-
-    Check all application and test code with:
-
-    ```sh
-    flake8 .
-    ```
-
-   **isort**
-
-    `isort` ensures imports are nicely formatted, and can suggest and
-    auto-fix issues such as double-importing.
-
-    Auto-fix imports with:
-
-    ```sh
-    isort .
-    ```
+```sh
+scripts-dev/lint.sh
+```

 It's worth noting that modern IDEs and text editors can run these tools
 automatically on save. It may be worth looking into whether this
 functionality is supported in your editor for a more convenient
-development workflow. It is not, however, recommended to run `flake8` on
-save as it takes a while and is very resource intensive.
+development workflow. It is not, however, recommended to run `flake8` or `mypy`
+on save as they take a while and can be very resource intensive.

 ## General rules

 -   **Naming**:
-    -   Use camel case for class and type names
-    -   Use underscores for functions and variables.
+    -   Use `CamelCase` for class and type names
+    -   Use underscores for `function_names` and `variable_names`.
 -   **Docstrings**: should follow the [google code
    style](https://google.github.io/styleguide/pyguide.html#38-comments-and-docstrings).
    See the
@@ -48,19 +48,28 @@ can find many good git tutorials on the web.

 # 4. Install the dependencies

-Once you have installed Python 3 and added the source, please open a terminal and
-setup a *virtualenv*, as follows:
+Synapse uses the [poetry](https://python-poetry.org/) project to manage its dependencies
+and development environment. Once you have installed Python 3 and added the
+source, you should install `poetry`.
+Of their installation methods, we recommend
+[installing `poetry` using `pipx`](https://python-poetry.org/docs/#installing-with-pipx),
+
+```shell
+pip install --user pipx
+pipx install poetry
+```
+
+but see poetry's [installation instructions](https://python-poetry.org/docs/#installation)
+for other installation methods.
+
+Next, open a terminal and install dependencies as follows:

 ```sh
 cd path/where/you/have/cloned/the/repository
-python3 -m venv ./env
-source ./env/bin/activate
-pip install wheel
-pip install -e ".[all,dev]"
-pip install tox
+poetry install --extras all
 ```

-This will install the developer dependencies for the project.
+This will install the runtime and developer dependencies for the project.


 # 5. Get in touch.
@@ -117,11 +126,10 @@ The linters look at your code and do two things:
 - ensure that your code follows the coding style adopted by the project;
 - catch a number of errors in your code.

-The linter takes no time at all to run as soon as you've [downloaded the dependencies into your python virtual environment](#4-install-the-dependencies).
+The linter takes no time at all to run as soon as you've [downloaded the dependencies](#4-install-the-dependencies).

 ```sh
-source ./env/bin/activate
-./scripts-dev/lint.sh
+poetry run ./scripts-dev/lint.sh
 ```

 Note that this script *will modify your files* to fix styling errors.
@@ -131,15 +139,13 @@ If you wish to restrict the linters to only the files changed since the last com
 (much faster!), you can instead run:

 ```sh
-source ./env/bin/activate
-./scripts-dev/lint.sh -d
+poetry run ./scripts-dev/lint.sh -d
 ```

 Or if you know exactly which files you wish to lint, you can instead run:

 ```sh
-source ./env/bin/activate
-./scripts-dev/lint.sh path/to/file1.py path/to/file2.py path/to/folder
+poetry run ./scripts-dev/lint.sh path/to/file1.py path/to/file2.py path/to/folder
 ```

 ## Run the unit tests (Twisted trial).
@@ -148,16 +154,14 @@ The unit tests run parts of Synapse, including your changes, to see if anything
 was broken. They are slower than the linters but will typically catch more errors.

 ```sh
-source ./env/bin/activate
-trial tests
+poetry run trial tests
 ```

 If you wish to only run *some* unit tests, you may specify
 another module instead of `tests` - or a test class or a method:

 ```sh
-source ./env/bin/activate
-trial tests.rest.admin.test_room tests.handlers.test_admin.ExfiltrateData.test_invite
+poetry run trial tests.rest.admin.test_room tests.handlers.test_admin.ExfiltrateData.test_invite
 ```

 If your tests fail, you may wish to look at the logs (the default log level is `ERROR`):
@@ -169,7 +173,7 @@ less _trial_temp/test.log
 To increase the log level for the tests, set `SYNAPSE_TEST_LOG_LEVEL`:

 ```sh
-SYNAPSE_TEST_LOG_LEVEL=DEBUG trial tests
+SYNAPSE_TEST_LOG_LEVEL=DEBUG poetry run trial tests
 ```

 By default, tests will use an in-memory SQLite database for test data. For additional
@@ -180,7 +184,7 @@ database state to be stored in a file named `test.db` under the trial process'
 working directory. Typically, this ends up being `_trial_temp/test.db`. For example:

 ```sh
-SYNAPSE_TEST_PERSIST_SQLITE_DB=1 trial tests
+SYNAPSE_TEST_PERSIST_SQLITE_DB=1 poetry run trial tests
 ```

 The database file can then be inspected with:
@@ -0,0 +1,239 @@
+# Managing dependencies with Poetry
+
+This is a quick cheat sheet for developers on how to use [`poetry`](https://python-poetry.org/).
+
+# Background
+
+Synapse uses a variety of third-party Python packages to function as a homeserver.
+Some of these are direct dependencies, listed in `pyproject.toml` under the
+`[tool.poetry.dependencies]` section. The rest are transitive dependencies (the
+things that our direct dependencies themselves depend on, and so on recursively.)
+
+We maintain a locked list of all our dependencies (transitive included) so that
+we can track exactly which version of each dependency appears in a given release.
+See [here](https://github.com/matrix-org/synapse/issues/11537#issue-1074469665)
+for discussion of why we wanted this for Synapse. We chose to use
+[`poetry`](https://python-poetry.org/) to manage this locked list; see
+[this comment](https://github.com/matrix-org/synapse/issues/11537#issuecomment-1015975819)
+for the reasoning.
+
+The locked dependencies get included in our "self-contained" releases: namely,
+our docker images and our debian packages. We also use the locked dependencies
+in development and our continuous integration.
+
+Separately, our "broad" dependencies—the version ranges specified in
+`pyproject.toml`—are included as metadata in our "sdists" and "wheels" [uploaded
+to PyPI](https://pypi.org/project/matrix-synapse). Installing from PyPI or from
+the Synapse source tree directly will _not_ use the locked dependencies; instead,
+they'll pull in the latest version of each package available at install time.
+
+## Example dependency
+
+An example may help. We have a broad dependency on
+[`phonenumbers`](https://pypi.org/project/phonenumbers/), as declared in
+this snippet from pyproject.toml [as of Synapse 1.57](
+https://github.com/matrix-org/synapse/blob/release-v1.57/pyproject.toml#L133
+):
+
+```toml
+[tool.poetry.dependencies]
+# ...
+phonenumbers = ">=8.2.0"
+```
+
+In our lockfile this is
+[pinned]( https://github.com/matrix-org/synapse/blob/dfc7646504cef3e4ff396c36089e1c6f1b1634de/poetry.lock#L679-L685)
+to version 8.12.44, even though
+[newer versions are available](https://pypi.org/project/phonenumbers/#history).
+
+```toml
+[[package]]
+name = "phonenumbers"
+version = "8.12.44"
+description = "Python version of Google's common library for parsing, formatting, storing and validating international phone numbers."
+category = "main"
+optional = false
+python-versions = "*"
+```
+
+The lockfile also includes a
+[cryptographic checksum](https://github.com/matrix-org/synapse/blob/release-v1.57/poetry.lock#L2178-L2181)
+of the sdists and wheels provided for this version of `phonenumbers`.
+
+```toml
+[metadata.files]
+# ...
+phonenumbers = [
+    {file = "phonenumbers-8.12.44-py2.py3-none-any.whl", hash = "sha256:cc1299cf37b309ecab6214297663ab86cb3d64ae37fd5b88e904fe7983a874a6"},
+    {file = "phonenumbers-8.12.44.tar.gz", hash = "sha256:26cfd0257d1704fe2f88caff2caabb70d16a877b1e65b6aae51f9fbbe10aa8ce"},
+]
+```
+
+We can see this pinned version inside the docker image for that release:
+
+```
+$ docker pull matrixdotorg/synapse:v1.57.0
+...
+$ docker run --entrypoint pip matrixdotorg/synapse:v1.57.0 show phonenumbers
+Name: phonenumbers
+Version: 8.12.44
+Summary: Python version of Google's common library for parsing, formatting, storing and validating international phone numbers.
+Home-page: https://github.com/daviddrysdale/python-phonenumbers
+Author: David Drysdale
+Author-email: dmd@lurklurk.org
+License: Apache License 2.0
+Location: /usr/local/lib/python3.9/site-packages
+Requires:
+Required-by: matrix-synapse
+```
+
+Whereas the wheel metadata just contains the broad dependencies:
+
+```
+$ cd /tmp
+$ wget https://files.pythonhosted.org/packages/ca/5e/d722d572cc5b3092402b783d6b7185901b444427633bd8a6b00ea0dd41b7/matrix_synapse-1.57.0rc1-py3-none-any.whl
+...
+$ unzip -c matrix_synapse-1.57.0rc1-py3-none-any.whl matrix_synapse-1.57.0rc1.dist-info/METADATA | grep phonenumbers
+Requires-Dist: phonenumbers (>=8.2.0)
+```
+
+# Tooling recommendation: direnv
+
+[`direnv`](https://direnv.net/) is a tool for activating environments in your
+shell inside a given directory. Its support for poetry is unofficial (a
+community wiki recipe only), but works solidly in our experience. We thoroughly
+recommend it for daily use. To use it:
+
+1. [Install `direnv`](https://direnv.net/docs/installation.html) - it's likely
+   packaged for your system already.
+2. Teach direnv about poetry. The [shell config here](https://github.com/direnv/direnv/wiki/Python#poetry)
+   needs to be added to `~/.config/direnv/direnvrc` (or more generally `$XDG_CONFIG_HOME/direnv/direnvrc`).
+3. Mark the synapse checkout as a poetry project: `echo layout poetry > .envrc`.
+4. Convince yourself that you trust this `.envrc` configuration and project.
+   Then formally confirm this to `direnv` by running `direnv allow`.
+
+Then whenever you navigate to the synapse checkout, you should be able to run
+e.g. `mypy` instead of `poetry run mypy`; `python` instead of
+`poetry run python`; and your shell commands will automatically run in the
+context of poetry's venv, without having to run `poetry shell` beforehand.
+
+
+# How do I...
+
+## ...reset my venv to the locked environment?
+
+```shell
+poetry install --extras all --remove-untracked
+```
+
+## ...run a command in the `poetry` virtualenv?
+
+Use `poetry run cmd args` when you need the python virtualenv context.
+To avoid typing `poetry run` all the time, you can run  `poetry shell`
+to start a new shell in the poetry virtualenv context. Within `poetry shell`,
+`python`, `pip`, `mypy`, `trial`, etc. are all run inside the project virtualenv
+and isolated from the rest o the system.
+
+Roughly speaking, the translation from a traditional virtualenv is:
+- `env/bin/activate` -> `poetry shell`, and
+- `deactivate` -> close the terminal (Ctrl-D, `exit`, etc.)
+
+See also the direnv recommendation above, which makes `poetry run` and
+`poetry shell` unnecessary.
+
+
+## ...inspect the `poetry` virtualenv?
+
+Some suggestions:
+
+```shell
+# Current env only
+poetry env info
+# All envs: this allows you to have e.g. a poetry managed venv for Python 3.7,
+# and another for Python 3.10.
+poetry env list --full-path
+poetry run pip list
+```
+
+Note that `poetry show` describes the abstract *lock file* rather than your
+on-disk environment. With that said, `poetry show --tree` can sometimes be
+useful.
+
+
+## ...add a new dependency?
+
+Either:
+- manually update `pyproject.toml`; then `poetry lock --no-update`; or else
+- `poetry add packagename`. See `poetry add --help`; note the `--dev`,
+  `--extras` and `--optional` flags in particular.
+  - **NB**: this specifies the new package with a version given by a "caret bound". This won't get forced to its lowest version in the old deps CI job: see [this TODO](https://github.com/matrix-org/synapse/blob/4e1374373857f2f7a911a31c50476342d9070681/.ci/scripts/test_old_deps.sh#L35-L39).
+
+Include the updated `pyproject.toml` and `poetry.lock` files in your commit.
+
+## ...remove a dependency?
+
+This is not done often and is untested, but
+
+```shell
+poetry remove packagename
+```
+
+ought to do the trick. Alternatively, manually update `pyproject.toml` and
+`poetry lock --no-update`. Include the updated `pyproject.toml` and poetry.lock`
+files in your commit.
+
+## ...update the version range for an existing dependency?
+
+Best done by manually editing `pyproject.toml`, then `poetry lock --no-update`.
+Include the updated `pyproject.toml` and `poetry.lock` in your commit.
+
+## ...update a dependency in the locked environment?
+
+Use
+
+```shell
+poetry update packagename
+```
+
+to use the latest version of `packagename` in the locked environment, without
+affecting the broad dependencies listed in the wheel.
+
+There doesn't seem to be a way to do this whilst locking a _specific_ version of
+`packagename`. We can workaround this (crudely) as follows:
+
+```shell
+poetry add packagename==1.2.3
+# This should update pyproject.lock.
+
+# Now undo the changes to pyproject.toml. For example
+# git restore pyproject.toml
+
+# Get poetry to recompute the content-hash of pyproject.toml without changing
+# the locked package versions.
+poetry lock --no-update
+```
+
+Either way, include the updated `poetry.lock` file in your commit.
+
+## ...export a `requirements.txt` file?
+
+```shell
+poetry export --extras all
+```
+
+Be wary of bugs in `poetry export` and `pip install -r requirements.txt`.
+
+Note: `poetry export` will be made a plugin in Poetry 1.2. Additional config may
+be required.
+
+## ...build a test wheel?
+
+I usually use
+
+```shell
+poetry run pip install build && poetry run python -m build
+```
+
+because [`build`](https://github.com/pypa/build) is a standardish tool which
+doesn't require poetry. (It's what we use in CI too). However, you could try
+`poetry build` too.
@@ -2449,6 +2449,22 @@ push:
 #
 #encryption_enabled_by_default_for_room_type: invite

+# Override the default power levels for rooms created on this server, per
+# room creation preset.
+#
+# Useful if you know that your users need special permissions in rooms
+# that they create (e.g. to send particular types of state events without
+# needing an elevated power level).  This takes the same shape as the
+# `power_level_content_override` parameter in the /createRoom API, but
+# is applied before that parameter.
+#
+# This is something of a workaround in the absence of MSC3779 or MSC3761.
+#
+#default_power_level_content_override:
+#   private_chat: null
+#   trusted_private_chat: null
+#   public_chat: null
+

 # Uncomment to allow non-server-admin users to create groups on this server
 #
@@ -10,15 +10,15 @@ See the folder [system](https://github.com/matrix-org/synapse/tree/develop/docs/
 for the systemd unit files.

 The folder [workers](https://github.com/matrix-org/synapse/tree/develop/docs/systemd-with-workers/workers/)
-contains an example configuration for the `federation_reader` worker.
+contains an example configuration for the `generic_worker` worker.

 ## Synapse configuration files

 See [the worker documentation](../workers.md) for information on how to set up the
 configuration files and reverse-proxy correctly.
-Below is a sample `federation_reader` worker configuration file.
+Below is a sample `generic_worker` worker configuration file.
 ```yaml
-{{#include workers/federation_reader.yaml}}
+{{#include workers/generic_worker.yaml}}
 ```

 Systemd manages daemonization itself, so ensure that none of the configuration
@@ -61,9 +61,9 @@ systemctl stop matrix-synapse.target
 # Restart the master alone
 systemctl start matrix-synapse.service

-# Restart a specific worker (eg. federation_reader); the master is
+# Restart a specific worker (eg. generic_worker); the master is
 # unaffected by this.
-systemctl restart matrix-synapse-worker@federation_reader.service
+systemctl restart matrix-synapse-worker@generic_worker.service

 # Add a new worker (assuming all configs are set up already)
 systemctl enable matrix-synapse-worker@federation_writer.service
@@ -1,13 +0,0 @@
-worker_app: synapse.app.federation_reader
-worker_name: federation_reader1
-
-worker_replication_host: 127.0.0.1
-worker_replication_http_port: 9093
-
-worker_listeners:
-    - type: http
-      port: 8011
-      resources:
-          - names: [federation]
-
-worker_log_config: /etc/matrix-synapse/federation-reader-log.yaml
@@ -0,0 +1,13 @@
+worker_app: synapse.app.generic_worker
+worker_name: generic_worker1
+
+worker_replication_host: 127.0.0.1
+worker_replication_http_port: 9093
+
+worker_listeners:
+  - type: http
+    port: 8011
+    resources:
+      - names: [client, federation]
+
+worker_log_config: /etc/matrix-synapse/generic-worker-log.yaml
@@ -302,14 +302,14 @@ Here are a few things to try:

   (Understanding the output is beyond the scope of this document!)

- * You can test your Matrix homeserver TURN setup with https://test.voip.librepush.net/.
+ * You can test your Matrix homeserver TURN setup with <https://test.voip.librepush.net/>.
   Note that this test is not fully reliable yet, so don't be discouraged if
   the test fails.
   [Here](https://github.com/matrix-org/voip-tester) is the github repo of the
   source of the tester, where you can file bug reports.

 * There is a WebRTC test tool at
-   https://webrtc.github.io/samples/src/content/peerconnection/trickle-ice/. To
+   <https://webrtc.github.io/samples/src/content/peerconnection/trickle-ice/>. To
   use it, you will need a username/password for your TURN server. You can
   either:

@@ -19,32 +19,36 @@ this document.
    packages](setup/installation.md#prebuilt-packages), you will need to follow the
    normal process for upgrading those packages.

+-   If Synapse was installed using pip then upgrade to the latest
+    version by running:
+
+    ```bash
+    pip install --upgrade matrix-synapse
+    ```
+
 -   If Synapse was installed from source, then:

-    1.  Activate the virtualenv before upgrading. For example, if
-        Synapse is installed in a virtualenv in `~/synapse/env` then
+    1.  Obtain the latest version of the source code. Git users can run
+        `git pull` to do this.
+
+    2.  If you're running Synapse in a virtualenv, make sure to activate it before
+        upgrading. For example, if Synapse is installed in a virtualenv in `~/synapse/env` then
        run:

        ```bash
        source ~/synapse/env/bin/activate
-        ```
-
-    2.  If Synapse was installed using pip then upgrade to the latest
-        version by running:
-
-        ```bash
-        pip install --upgrade matrix-synapse
-        ```
-
-        If Synapse was installed using git then upgrade to the latest
-        version by running:
-
-        ```bash
-        git pull
        pip install --upgrade .
        ```
+        Include any relevant extras between square brackets, e.g. `pip install --upgrade ".[postgres,oidc]"`.

-    3.  Restart Synapse:
+    3.  If you're using `poetry` to manage a Synapse installation, run:
+        ```bash
+        poetry install
+        ```
+        Include any relevant extras with `--extras`, e.g. `poetry install --extras postgres --extras oidc`.
+        It's probably easiest to run `poetry install --extras all`.
+
+    4.  Restart Synapse:

        ```bash
        synctl restart
@@ -85,6 +89,13 @@ process, for example:
    dpkg -i matrix-synapse-py3_1.3.0+stretch1_amd64.deb
    ```

+# Upgrading to v1.58.0
+
+## Groups/communities feature has been disabled by default
+
+The non-standard groups/communities feature in Synapse has been disabled by default
+and will be removed in Synapse v1.61.0.
+
 # Upgrading to v1.57.0

 ## Changes to database schema for application services
@@ -75,6 +75,20 @@ function setTocEntry() {
 * Populate sidebar on load
 */
 window.addEventListener('load', () => {
+    // Prevent rendering the table of contents of the "print book" page, as it
+    // will end up being rendered into the output (in a broken-looking way)
+
+    // Get the name of the current page (i.e. 'print.html')
+    const pageNameExtension = window.location.pathname.split('/').pop();
+
+    // Split off the extension (as '.../print' is also a valid page name), which
+    // should result in 'print'
+    const pageName = pageNameExtension.split('.')[0];
+    if (pageName === "print") {
+        // Don't render the table of contents on this page
+        return;
+    }
+
    // Only create table of contents if there is more than one header on the page
    if (headers.length <= 1) {
        return;
@@ -146,12 +146,10 @@ worker_replication_host: 127.0.0.1
 worker_replication_http_port: 9093

 worker_listeners:
- - type: http
-   port: 8083
-   resources:
-     - names:
-       - client
-       - federation
+  - type: http
+    port: 8083
+    resources:
+      - names: [client, federation]

 worker_log_config: /home/matrix/synapse/config/worker1_log_config.yaml
 ```
@@ -343,9 +341,9 @@ effects of bursts of events from that bridge on events sent by normal users.

 #### Stream writers

-Additionally, there is *experimental* support for moving writing of specific
-streams (such as events) off of the main process to a particular worker. (This
-is only supported with Redis-based replication.)
+Additionally, the writing of specific streams (such as events) can be moved off
+of the main process to a particular worker.
+(This is only supported with Redis-based replication.)

 To enable this, the worker must have a HTTP replication listener configured,
 have a `worker_name` and be listed in the `instance_map` config. The same worker
@@ -422,7 +420,7 @@ the stream writer for the `presence` stream:

 #### Background tasks

-There is also *experimental* support for moving background tasks to a separate
+There is also support for moving background tasks to a separate
 worker. Background tasks are run periodically or started via replication. Exactly
 which tasks are configured to run depends on your Synapse configuration (e.g. if
 stats is enabled).
@@ -13,7 +13,6 @@ no_implicit_optional = True
 files =
  docker/,
  scripts-dev/,
-  setup.py,
  synapse/,
  tests/

@@ -234,8 +233,8 @@ disallow_untyped_defs = True
 ;; The `typeshed` project maintains stubs here:
 ;;     https://github.com/python/typeshed/tree/master/stubs
 ;; and for each package `foo` there's a corresponding `types-foo` package on PyPI,
-;; which we can pull in as a dev dependency by adding to `setup.py`'s
-;; `CONDITIONAL_REQUIREMENTS["mypy"]` list.
+;; which we can pull in as a dev dependency by adding to `pyproject.toml`'s
+;; `[tool.poetry.dev-dependencies]` list.

 [mypy-authlib.*]
 ignore_missing_imports = True
@@ -720,7 +720,7 @@ test = ["appdirs (==1.4.4)", "pytest (>=6)", "pytest-cov (>=2.7)", "pytest-mock

 [[package]]
 name = "prometheus-client"
-version = "0.13.1"
+version = "0.14.0"
 description = "Python client for the Prometheus monitoring system."
 category = "main"
 optional = false
@@ -1288,7 +1288,7 @@ urllib3 = ">=1.26.0"

 [[package]]
 name = "twisted"
-version = "22.2.0"
+version = "22.4.0"
 description = "An asynchronous networking framework written in Python"
 category = "main"
 optional = false
@@ -1308,19 +1308,20 @@ typing-extensions = ">=3.6.5"
 "zope.interface" = ">=4.4.2"

 [package.extras]
-all_non_platform = ["cython-test-exception-raiser (>=1.0.2,<2)", "PyHamcrest (>=1.9.0)", "pyopenssl (>=16.0.0)", "service-identity (>=18.1.0)", "idna (>=2.4)", "pyasn1", "cryptography (>=2.6)", "appdirs (>=1.4.0)", "bcrypt (>=3.0.0)", "pyserial (>=3.0)", "h2 (>=3.0,<4.0)", "priority (>=1.1.0,<2.0)", "pywin32 (!=226)", "contextvars (>=2.4,<3)"]
+all_non_platform = ["cython-test-exception-raiser (>=1.0.2,<2)", "PyHamcrest (>=1.9.0)", "pyopenssl (>=16.0.0)", "service-identity (>=18.1.0)", "idna (>=2.4)", "pyasn1", "cryptography (>=2.6)", "appdirs (>=1.4.0)", "bcrypt (>=3.0.0)", "pyserial (>=3.0)", "h2 (>=3.0,<5.0)", "priority (>=1.1.0,<2.0)", "pywin32 (!=226)", "contextvars (>=2.4,<3)"]
 conch = ["pyasn1", "cryptography (>=2.6)", "appdirs (>=1.4.0)", "bcrypt (>=3.0.0)"]
+conch_nacl = ["pyasn1", "cryptography (>=2.6)", "appdirs (>=1.4.0)", "bcrypt (>=3.0.0)", "pynacl"]
 contextvars = ["contextvars (>=2.4,<3)"]
 dev = ["towncrier (>=19.2,<20.0)", "sphinx-rtd-theme (>=0.5,<1.0)", "readthedocs-sphinx-ext (>=2.1,<3.0)", "sphinx (>=4.1.2,<6)", "pyflakes (>=2.2,<3.0)", "twistedchecker (>=0.7,<1.0)", "coverage (>=6b1,<7)", "python-subunit (>=1.4,<2.0)", "pydoctor (>=21.9.0,<21.10.0)"]
 dev_release = ["towncrier (>=19.2,<20.0)", "sphinx-rtd-theme (>=0.5,<1.0)", "readthedocs-sphinx-ext (>=2.1,<3.0)", "sphinx (>=4.1.2,<6)", "pydoctor (>=21.9.0,<21.10.0)"]
-http2 = ["h2 (>=3.0,<4.0)", "priority (>=1.1.0,<2.0)"]
-macos_platform = ["pyobjc-core", "pyobjc-framework-cfnetwork", "pyobjc-framework-cocoa", "cython-test-exception-raiser (>=1.0.2,<2)", "PyHamcrest (>=1.9.0)", "pyopenssl (>=16.0.0)", "service-identity (>=18.1.0)", "idna (>=2.4)", "pyasn1", "cryptography (>=2.6)", "appdirs (>=1.4.0)", "bcrypt (>=3.0.0)", "pyserial (>=3.0)", "h2 (>=3.0,<4.0)", "priority (>=1.1.0,<2.0)", "pywin32 (!=226)", "contextvars (>=2.4,<3)"]
-mypy = ["mypy (==0.930)", "mypy-zope (==0.3.4)", "types-setuptools", "types-pyopenssl", "towncrier (>=19.2,<20.0)", "sphinx-rtd-theme (>=0.5,<1.0)", "readthedocs-sphinx-ext (>=2.1,<3.0)", "sphinx (>=4.1.2,<6)", "pyflakes (>=2.2,<3.0)", "twistedchecker (>=0.7,<1.0)", "coverage (>=6b1,<7)", "cython-test-exception-raiser (>=1.0.2,<2)", "PyHamcrest (>=1.9.0)", "pyopenssl (>=16.0.0)", "service-identity (>=18.1.0)", "idna (>=2.4)", "pyasn1", "cryptography (>=2.6)", "appdirs (>=1.4.0)", "bcrypt (>=3.0.0)", "pyserial (>=3.0)", "h2 (>=3.0,<4.0)", "priority (>=1.1.0,<2.0)", "pywin32 (!=226)", "python-subunit (>=1.4,<2.0)", "contextvars (>=2.4,<3)", "pydoctor (>=21.9.0,<21.10.0)"]
-osx_platform = ["pyobjc-core", "pyobjc-framework-cfnetwork", "pyobjc-framework-cocoa", "cython-test-exception-raiser (>=1.0.2,<2)", "PyHamcrest (>=1.9.0)", "pyopenssl (>=16.0.0)", "service-identity (>=18.1.0)", "idna (>=2.4)", "pyasn1", "cryptography (>=2.6)", "appdirs (>=1.4.0)", "bcrypt (>=3.0.0)", "pyserial (>=3.0)", "h2 (>=3.0,<4.0)", "priority (>=1.1.0,<2.0)", "pywin32 (!=226)", "contextvars (>=2.4,<3)"]
+http2 = ["h2 (>=3.0,<5.0)", "priority (>=1.1.0,<2.0)"]
+macos_platform = ["pyobjc-core", "pyobjc-framework-cfnetwork", "pyobjc-framework-cocoa", "cython-test-exception-raiser (>=1.0.2,<2)", "PyHamcrest (>=1.9.0)", "pyopenssl (>=16.0.0)", "service-identity (>=18.1.0)", "idna (>=2.4)", "pyasn1", "cryptography (>=2.6)", "appdirs (>=1.4.0)", "bcrypt (>=3.0.0)", "pyserial (>=3.0)", "h2 (>=3.0,<5.0)", "priority (>=1.1.0,<2.0)", "pywin32 (!=226)", "contextvars (>=2.4,<3)"]
+mypy = ["mypy (==0.930)", "mypy-zope (==0.3.4)", "types-setuptools", "types-pyopenssl", "towncrier (>=19.2,<20.0)", "sphinx-rtd-theme (>=0.5,<1.0)", "readthedocs-sphinx-ext (>=2.1,<3.0)", "sphinx (>=4.1.2,<6)", "pyflakes (>=2.2,<3.0)", "twistedchecker (>=0.7,<1.0)", "coverage (>=6b1,<7)", "cython-test-exception-raiser (>=1.0.2,<2)", "PyHamcrest (>=1.9.0)", "pyopenssl (>=16.0.0)", "service-identity (>=18.1.0)", "idna (>=2.4)", "pyasn1", "cryptography (>=2.6)", "appdirs (>=1.4.0)", "bcrypt (>=3.0.0)", "pyserial (>=3.0)", "h2 (>=3.0,<5.0)", "priority (>=1.1.0,<2.0)", "pynacl", "pywin32 (!=226)", "python-subunit (>=1.4,<2.0)", "contextvars (>=2.4,<3)", "pydoctor (>=21.9.0,<21.10.0)"]
+osx_platform = ["pyobjc-core", "pyobjc-framework-cfnetwork", "pyobjc-framework-cocoa", "cython-test-exception-raiser (>=1.0.2,<2)", "PyHamcrest (>=1.9.0)", "pyopenssl (>=16.0.0)", "service-identity (>=18.1.0)", "idna (>=2.4)", "pyasn1", "cryptography (>=2.6)", "appdirs (>=1.4.0)", "bcrypt (>=3.0.0)", "pyserial (>=3.0)", "h2 (>=3.0,<5.0)", "priority (>=1.1.0,<2.0)", "pywin32 (!=226)", "contextvars (>=2.4,<3)"]
 serial = ["pyserial (>=3.0)", "pywin32 (!=226)"]
 test = ["cython-test-exception-raiser (>=1.0.2,<2)", "PyHamcrest (>=1.9.0)"]
 tls = ["pyopenssl (>=16.0.0)", "service-identity (>=18.1.0)", "idna (>=2.4)"]
-windows_platform = ["pywin32 (!=226)", "cython-test-exception-raiser (>=1.0.2,<2)", "PyHamcrest (>=1.9.0)", "pyopenssl (>=16.0.0)", "service-identity (>=18.1.0)", "idna (>=2.4)", "pyasn1", "cryptography (>=2.6)", "appdirs (>=1.4.0)", "bcrypt (>=3.0.0)", "pyserial (>=3.0)", "h2 (>=3.0,<4.0)", "priority (>=1.1.0,<2.0)", "pywin32 (!=226)", "contextvars (>=2.4,<3)"]
+windows_platform = ["pywin32 (!=226)", "cython-test-exception-raiser (>=1.0.2,<2)", "PyHamcrest (>=1.9.0)", "pyopenssl (>=16.0.0)", "service-identity (>=18.1.0)", "idna (>=2.4)", "pyasn1", "cryptography (>=2.6)", "appdirs (>=1.4.0)", "bcrypt (>=3.0.0)", "pyserial (>=3.0)", "h2 (>=3.0,<5.0)", "priority (>=1.1.0,<2.0)", "pywin32 (!=226)", "contextvars (>=2.4,<3)"]

 [[package]]
 name = "twisted-iocpsupport"
@@ -2229,8 +2230,8 @@ platformdirs = [
    {file = "platformdirs-2.5.1.tar.gz", hash = "sha256:7535e70dfa32e84d4b34996ea99c5e432fa29a708d0f4e394bbcb2a8faa4f16d"},
 ]
 prometheus-client = [
-    {file = "prometheus_client-0.13.1-py3-none-any.whl", hash = "sha256:357a447fd2359b0a1d2e9b311a0c5778c330cfbe186d880ad5a6b39884652316"},
-    {file = "prometheus_client-0.13.1.tar.gz", hash = "sha256:ada41b891b79fca5638bd5cfe149efa86512eaa55987893becd2c6d8d0a5dfc5"},
+    {file = "prometheus_client-0.14.0-py3-none-any.whl", hash = "sha256:f4aba3fdd1735852049f537c1f0ab177159b7ab76f271ecc4d2f45aa2a1d01f2"},
+    {file = "prometheus_client-0.14.0.tar.gz", hash = "sha256:8f7a922dd5455ad524b6ba212ce8eb2b4b05e073f4ec7218287f88b1cac34750"},
 ]
 psycopg2 = [
    {file = "psycopg2-2.9.3-cp310-cp310-win32.whl", hash = "sha256:083707a696e5e1c330af2508d8fab36f9700b26621ccbcb538abe22e15485362"},
@@ -2596,8 +2597,8 @@ twine = [
    {file = "twine-3.8.0.tar.gz", hash = "sha256:8efa52658e0ae770686a13b675569328f1fba9837e5de1867bfe5f46a9aefe19"},
 ]
 twisted = [
-    {file = "Twisted-22.2.0-py3-none-any.whl", hash = "sha256:5c63c149eb6b8fe1e32a0215b1cef96fabdba04f705d8efb9174b1ccf5b49d49"},
-    {file = "Twisted-22.2.0.tar.gz", hash = "sha256:57f32b1f6838facb8c004c89467840367ad38e9e535f8252091345dba500b4f2"},
+    {file = "Twisted-22.4.0-py3-none-any.whl", hash = "sha256:f9f7a91f94932477a9fc3b169d57f54f96c6e74a23d78d9ce54039a7f48928a2"},
+    {file = "Twisted-22.4.0.tar.gz", hash = "sha256:a047990f57dfae1e0bd2b7df2526d4f16dcdc843774dc108b78c52f2a5f13680"},
 ]
 twisted-iocpsupport = [
    {file = "twisted-iocpsupport-1.0.2.tar.gz", hash = "sha256:72068b206ee809c9c596b57b5287259ea41ddb4774d86725b19f35bf56aa32a9"},
@@ -280,5 +280,5 @@ twine = "*"
 towncrier = ">=18.6.0rc1"

 [build-system]
-requires = ["setuptools"]
-build-backend = "setuptools.build_meta"
+requires = ["poetry-core>=1.0.0"]
+build-backend = "poetry.core.masonry.api"
@@ -64,4 +64,4 @@ docker build -t $COMPLEMENT_BASE_IMAGE -f "docker/complement/$COMPLEMENT_DOCKERF
 # Run the tests!
 echo "Images built; running complement"
 cd "$COMPLEMENT_DIR"
-go test -v -tags synapse_blacklist,msc2716,msc3030 -count=1 "$@" ./tests/...
+go test -v -tags synapse_blacklist,msc2716,msc3030,faster_joins -count=1 "$@" ./tests/...
@@ -124,7 +124,12 @@ def request(
    authorization_headers = []

    for key, sig in signed_json["signatures"][origin_name].items():
-        header = 'X-Matrix origin=%s,key="%s",sig="%s"' % (origin_name, key, sig)
+        header = 'X-Matrix origin=%s,key="%s",sig="%s",destination="%s"' % (
+            origin_name,
+            key,
+            sig,
+            destination,
+        )
        authorization_headers.append(header.encode("ascii"))
        print("Authorization: %s" % header, file=sys.stderr)

@@ -79,8 +79,20 @@ else
  # If we were not asked to lint changed files, and no paths were found as a result,
  # then lint everything!
  if [[ -z ${files+x} ]]; then
-    # Lint all source code files and directories
-      files=( "." )
+      # CI runs each linter on the entire checkout, e.g. `black .`. So don't
+      # rely on this list to *find* lint targets if that misses a file; instead;
+      # use it to exclude files from linters when this can't be done by config.
+      #
+      # To check which files the linters examine, use:
+      #     black --verbose . 2>&1 | \grep -v ignored
+      #     isort --show-files .
+      #     flake8 --verbose .  # This isn't a great option
+      #     mypy has explicit config in mypy.ini; there is also mypy --verbose
+      files=(
+          "synapse" "docker" "tests"
+          "scripts-dev"
+          "contrib" "synmark" "stubs" ".ci"
+      )
  fi
 fi

@@ -69,11 +69,12 @@ def cli():
        # ... wait for assets to build ...

        ./scripts-dev/release.py publish
+
        ./scripts-dev/release.py upload

        # Optional: generate some nice links for the announcement

-        ./scripts-dev/release.py upload
+        ./scripts-dev/release.py announce

    If the env var GH_TOKEN (or GITHUB_TOKEN) is set, or passed into the
    `tag`/`publish` command, then a new draft release will be created/published.
@@ -229,7 +230,7 @@ def prepare():
        debian_version = new_version

    run_until_successful(
-        f'dch -M -v {debian_version} "New synapse release {debian_version}."',
+        f'dch -M -v {debian_version} "New Synapse release {new_version}."',
        shell=True,
    )
    run_until_successful('dch -M -r -D stable ""', shell=True)
@@ -1,9 +0,0 @@
-[check-manifest]
-ignore =
-    .git-blame-ignore-revs
-    contrib
-    contrib/*
-    docs/*
-    pylint.cfg
-    tox.ini
-
@@ -1,183 +0,0 @@
-#!/usr/bin/env python
-
-# Copyright 2014-2017 OpenMarket Ltd
-# Copyright 2017 Vector Creations Ltd
-# Copyright 2017-2018 New Vector Ltd
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-import os
-from typing import Any, Dict
-
-from setuptools import Command, find_packages, setup
-
-here = os.path.abspath(os.path.dirname(__file__))
-
-
-# Some notes on `setup.py test`:
-#
-# Once upon a time we used to try to make `setup.py test` run `tox` to run the
-# tests. That's a bad idea for three reasons:
-#
-# 1: `setup.py test` is supposed to find out whether the tests work in the
-#    *current* environmentt, not whatever tox sets up.
-# 2: Empirically, trying to install tox during the test run wasn't working ("No
-#    module named virtualenv").
-# 3: The tox documentation advises against it[1].
-#
-# Even further back in time, we used to use setuptools_trial [2]. That has its
-# own set of issues: for instance, it requires installation of Twisted to build
-# an sdist (because the recommended mode of usage is to add it to
-# `setup_requires`). That in turn means that in order to successfully run tox
-# you have to have the python header files installed for whichever version of
-# python tox uses (which is python3 on recent ubuntus, for example).
-#
-# So, for now at least, we stick with what appears to be the convention among
-# Twisted projects, and don't attempt to do anything when someone runs
-# `setup.py test`; instead we direct people to run `trial` directly if they
-# care.
-#
-# [1]: http://tox.readthedocs.io/en/2.5.0/example/basic.html#integration-with-setup-py-test-command
-# [2]: https://pypi.python.org/pypi/setuptools_trial
-class TestCommand(Command):
-    def initialize_options(self):
-        pass
-
-    def finalize_options(self):
-        pass
-
-    def run(self):
-        print(
-            """Synapse's tests cannot be run via setup.py. To run them, try:
-     PYTHONPATH="." trial tests
-"""
-        )
-
-
-def read_file(path_segments):
-    """Read a file from the package. Takes a list of strings to join to
-    make the path"""
-    file_path = os.path.join(here, *path_segments)
-    with open(file_path) as f:
-        return f.read()
-
-
-def exec_file(path_segments):
-    """Execute a single python file to get the variables defined in it"""
-    result: Dict[str, Any] = {}
-    code = read_file(path_segments)
-    exec(code, result)
-    return result
-
-
-version = exec_file(("synapse", "__init__.py"))["__version__"]
-dependencies = exec_file(("synapse", "python_dependencies.py"))
-long_description = read_file(("README.rst",))
-
-REQUIREMENTS = dependencies["REQUIREMENTS"]
-CONDITIONAL_REQUIREMENTS = dependencies["CONDITIONAL_REQUIREMENTS"]
-ALL_OPTIONAL_REQUIREMENTS = dependencies["ALL_OPTIONAL_REQUIREMENTS"]
-
-# Make `pip install matrix-synapse[all]` install all the optional dependencies.
-CONDITIONAL_REQUIREMENTS["all"] = list(ALL_OPTIONAL_REQUIREMENTS)
-
-# Developer dependencies should not get included in "all".
-#
-# We pin black so that our tests don't start failing on new releases.
-CONDITIONAL_REQUIREMENTS["lint"] = [
-    "isort==5.7.0",
-    "black==22.3.0",
-    "flake8-comprehensions",
-    "flake8-bugbear==21.3.2",
-    "flake8",
-]
-
-CONDITIONAL_REQUIREMENTS["mypy"] = [
-    "mypy==0.931",
-    "mypy-zope==0.3.5",
-    "types-bleach>=4.1.0",
-    "types-jsonschema>=3.2.0",
-    "types-opentracing>=2.4.2",
-    "types-Pillow>=8.3.4",
-    "types-psycopg2>=2.9.9",
-    "types-pyOpenSSL>=20.0.7",
-    "types-PyYAML>=5.4.10",
-    "types-requests>=2.26.0",
-    "types-setuptools>=57.4.0",
-]
-
-# Dependencies which are exclusively required by unit test code. This is
-# NOT a list of all modules that are necessary to run the unit tests.
-# Tests assume that all optional dependencies are installed.
-#
-# parameterized_class decorator was introduced in parameterized 0.7.0
-CONDITIONAL_REQUIREMENTS["test"] = ["parameterized>=0.7.0", "idna>=2.5"]
-
-CONDITIONAL_REQUIREMENTS["dev"] = (
-    CONDITIONAL_REQUIREMENTS["lint"]
-    + CONDITIONAL_REQUIREMENTS["mypy"]
-    + CONDITIONAL_REQUIREMENTS["test"]
-    + [
-        # The following are used by the release script
-        "click==8.1.0",
-        "redbaron==0.9.2",
-        "GitPython==3.1.14",
-        "commonmark==0.9.1",
-        "pygithub==1.55",
-        # The following are executed as commands by the release script.
-        "twine",
-        "towncrier",
-    ]
-)
-
-setup(
-    name="matrix-synapse",
-    version=version,
-    packages=find_packages(exclude=["tests", "tests.*"]),
-    description="Reference homeserver for the Matrix decentralised comms protocol",
-    install_requires=REQUIREMENTS,
-    extras_require=CONDITIONAL_REQUIREMENTS,
-    include_package_data=True,
-    zip_safe=False,
-    long_description=long_description,
-    long_description_content_type="text/x-rst",
-    python_requires="~=3.7",
-    entry_points={
-        "console_scripts": [
-            # Application
-            "synapse_homeserver = synapse.app.homeserver:main",
-            "synapse_worker = synapse.app.generic_worker:main",
-            "synctl = synapse._scripts.synctl:main",
-            # Scripts
-            "export_signing_key = synapse._scripts.export_signing_key:main",
-            "generate_config = synapse._scripts.generate_config:main",
-            "generate_log_config = synapse._scripts.generate_log_config:main",
-            "generate_signing_key = synapse._scripts.generate_signing_key:main",
-            "hash_password = synapse._scripts.hash_password:main",
-            "register_new_matrix_user = synapse._scripts.register_new_matrix_user:main",
-            "synapse_port_db = synapse._scripts.synapse_port_db:main",
-            "synapse_review_recent_signups = synapse._scripts.review_recent_signups:main",
-            "update_synapse_database = synapse._scripts.update_synapse_database:main",
-        ]
-    },
-    classifiers=[
-        "Development Status :: 5 - Production/Stable",
-        "Topic :: Communications :: Chat",
-        "License :: OSI Approved :: Apache Software License",
-        "Programming Language :: Python :: 3 :: Only",
-        "Programming Language :: Python :: 3.7",
-        "Programming Language :: Python :: 3.8",
-        "Programming Language :: Python :: 3.9",
-        "Programming Language :: Python :: 3.10",
-    ],
-    cmdclass={"test": TestCommand},
-)
@@ -20,6 +20,8 @@ import json
 import os
 import sys

+from matrix_common.versionstring import get_distribution_version_string
+
 # Check that we're not running on an unsupported Python version.
 if sys.version_info < (3, 7):
    print("Synapse requires Python 3.7 or above.")
@@ -68,7 +70,7 @@ try:
 except ImportError:
    pass

-__version__ = "1.57.1"
+__version__ = get_distribution_version_string("matrix-synapse")

 if bool(os.environ.get("SYNAPSE_TEST_PATCH_LOG_CONTEXTS", False)):
    # We import here so that we don't have to install a bunch of deps when
@@ -179,8 +179,6 @@ class RelationTypes:
    REPLACE: Final = "m.replace"
    REFERENCE: Final = "m.reference"
    THREAD: Final = "m.thread"
-    # TODO Remove this in Synapse >= v1.57.0.
-    UNSTABLE_THREAD: Final = "io.element.thread"


 class LimitBlockingTypes:
@@ -79,6 +79,8 @@ class Codes:
    UNABLE_AUTHORISE_JOIN = "M_UNABLE_TO_AUTHORISE_JOIN"
    UNABLE_TO_GRANT_JOIN = "M_UNABLE_TO_GRANT_JOIN"

+    UNREDACTED_CONTENT_DELETED = "FI.MAU.MSC2815_UNREDACTED_CONTENT_DELETED"
+

 class CodeMessageException(RuntimeError):
    """An exception with integer code and message string attributes.
@@ -483,6 +485,22 @@ class RequestSendFailed(RuntimeError):
        self.can_retry = can_retry


+class UnredactedContentDeletedError(SynapseError):
+    def __init__(self, content_keep_ms: Optional[int] = None):
+        super().__init__(
+            404,
+            "The content for that event has already been erased from the database",
+            errcode=Codes.UNREDACTED_CONTENT_DELETED,
+        )
+        self.content_keep_ms = content_keep_ms
+
+    def error_dict(self) -> "JsonDict":
+        extra = {}
+        if self.content_keep_ms is not None:
+            extra = {"fi.mau.msc2815.content_keep_ms": self.content_keep_ms}
+        return cs_error(self.msg, self.errcode, **extra)
+
+
 def cs_error(msg: str, code: str = Codes.UNKNOWN, **kwargs: Any) -> "JsonDict":
    """Utility method for constructing an error response for client-server
    interactions.
@@ -89,9 +89,7 @@ ROOM_EVENT_FILTER_SCHEMA = {
        "org.matrix.not_labels": {"type": "array", "items": {"type": "string"}},
        # MSC3440, filtering by event relations.
        "related_by_senders": {"type": "array", "items": {"type": "string"}},
-        "io.element.relation_senders": {"type": "array", "items": {"type": "string"}},
        "related_by_rel_types": {"type": "array", "items": {"type": "string"}},
-        "io.element.relation_types": {"type": "array", "items": {"type": "string"}},
    },
 }

@@ -323,16 +321,6 @@ class Filter:
        self.related_by_senders = self.filter_json.get("related_by_senders", None)
        self.related_by_rel_types = self.filter_json.get("related_by_rel_types", None)

-        # Fallback to the unstable prefix if the stable version is not given.
-        if hs.config.experimental.msc3440_enabled:
-            self.related_by_senders = self.related_by_senders or self.filter_json.get(
-                "io.element.relation_senders", None
-            )
-            self.related_by_rel_types = (
-                self.related_by_rel_types
-                or self.filter_json.get("io.element.relation_types", None)
-            )
-
    def filters_all_types(self) -> bool:
        return "*" in self.not_types

@@ -42,7 +42,7 @@ logger = logging.getLogger(__name__)
 #   user ID -> {device ID -> {algorithm -> count}}
 TransactionOneTimeKeyCounts = Dict[str, Dict[str, Dict[str, int]]]

-# Type for the `device_unused_fallback_keys` field in an appservice transaction
+# Type for the `device_unused_fallback_key_types` field in an appservice transaction
 #   user ID -> {device ID -> [algorithm]}
 TransactionUnusedFallbackKeys = Dict[str, Dict[str, List[str]]]

@@ -278,7 +278,7 @@ class ApplicationServiceApi(SimpleHttpClient):
                ] = one_time_key_counts
            if unused_fallback_keys:
                body[
-                    "org.matrix.msc3202.device_unused_fallback_keys"
+                    "org.matrix.msc3202.device_unused_fallback_key_types"
                ] = unused_fallback_keys
            if device_list_summary:
                body["org.matrix.msc3202.device_lists"] = {
@@ -26,9 +26,6 @@ class ExperimentalConfig(Config):
    def read_config(self, config: JsonDict, **kwargs: Any) -> None:
        experimental = config.get("experimental_features") or {}

-        # MSC3440 (thread relation)
-        self.msc3440_enabled: bool = experimental.get("msc3440_enabled", False)
-
        # MSC3026 (busy presence state)
        self.msc3026_enabled: bool = experimental.get("msc3026_enabled", False)

@@ -77,7 +74,10 @@ class ExperimentalConfig(Config):
        self.msc3720_enabled: bool = experimental.get("msc3720_enabled", False)

        # The deprecated groups feature.
-        self.groups_enabled: bool = experimental.get("groups_enabled", True)
+        self.groups_enabled: bool = experimental.get("groups_enabled", False)

        # MSC2654: Unread counts
        self.msc2654_enabled: bool = experimental.get("msc2654_enabled", False)
+
+        # MSC2815 (allow room moderators to view redacted event content)
+        self.msc2815_enabled: bool = experimental.get("msc2815_enabled", False)
@@ -63,6 +63,19 @@ class RoomConfig(Config):
                "Invalid value for encryption_enabled_by_default_for_room_type"
            )

+        self.default_power_level_content_override = config.get(
+            "default_power_level_content_override",
+            None,
+        )
+        if self.default_power_level_content_override is not None:
+            for preset in self.default_power_level_content_override:
+                if preset not in vars(RoomCreationPreset).values():
+                    raise ConfigError(
+                        "Unrecognised room preset %s in default_power_level_content_override"
+                        % preset
+                    )
+                # We validate the actual overrides when we try to apply them.
+
    def generate_config_section(self, **kwargs: Any) -> str:
        return """\
        ## Rooms ##
@@ -83,4 +96,20 @@ class RoomConfig(Config):
        # will also not affect rooms created by other servers.
        #
        #encryption_enabled_by_default_for_room_type: invite
+
+        # Override the default power levels for rooms created on this server, per
+        # room creation preset.
+        #
+        # Useful if you know that your users need special permissions in rooms
+        # that they create (e.g. to send particular types of state events without
+        # needing an elevated power level).  This takes the same shape as the
+        # `power_level_content_override` parameter in the /createRoom API, but
+        # is applied before that parameter.
+        #
+        # This is something of a workaround in the absence of MSC3779 or MSC3761.
+        #
+        #default_power_level_content_override:
+        #   private_chat: null
+        #   trusted_private_chat: null
+        #   public_chat: null
        """
@@ -680,14 +680,6 @@ class ServerConfig(Config):
            config.get("use_account_validity_in_account_status") or False
        )

-        # This is a temporary option that enables fully using the new
-        # `device_lists_changes_in_room` without the backwards compat code. This
-        # is primarily for testing. If enabled the server should *not* be
-        # downgraded, as it may lead to missing device list updates.
-        self.use_new_device_lists_changes_in_room = (
-            config.get("use_new_device_lists_changes_in_room") or False
-        )
-
        self.rooms_to_exclude_from_sync: List[str] = (
            config.get("exclude_rooms_from_sync") or []
        )
@@ -39,7 +39,6 @@ from . import EventBase

 if TYPE_CHECKING:
    from synapse.handlers.relations import BundledAggregations
-    from synapse.server import HomeServer


 # Split strings on "." but not "\." This uses a negative lookbehind assertion for '\'
@@ -396,9 +395,6 @@ class EventClientSerializer:
    clients.
    """

-    def __init__(self, hs: "HomeServer"):
-        self._msc3440_enabled = hs.config.experimental.msc3440_enabled
-
    def serialize_event(
        self,
        event: Union[JsonDict, EventBase],
@@ -406,6 +402,7 @@ class EventClientSerializer:
        *,
        config: SerializeEventConfig = _DEFAULT_SERIALIZE_EVENT_CONFIG,
        bundle_aggregations: Optional[Dict[str, "BundledAggregations"]] = None,
+        apply_edits: bool = True,
    ) -> JsonDict:
        """Serializes a single event.

@@ -413,10 +410,10 @@ class EventClientSerializer:
            event: The event being serialized.
            time_now: The current time in milliseconds
            config: Event serialization config
-            bundle_aggregations: Whether to include the bundled aggregations for this
-                event. Only applies to non-state events. (State events never include
-                bundled aggregations.)
-
+            bundle_aggregations: A map from event_id to the aggregations to be bundled
+               into the event.
+            apply_edits: Whether the content of the event should be modified to reflect
+               any replacement in `bundle_aggregations[<event_id>].replace`.
        Returns:
            The serialized event
        """
@@ -434,8 +431,9 @@ class EventClientSerializer:
                    event,
                    time_now,
                    config,
-                    bundle_aggregations[event.event_id],
+                    event_aggregations,
                    serialized_event,
+                    apply_edits=apply_edits,
                )

        return serialized_event
@@ -474,6 +472,7 @@ class EventClientSerializer:
        config: SerializeEventConfig,
        aggregations: "BundledAggregations",
        serialized_event: JsonDict,
+        apply_edits: bool,
    ) -> None:
        """Potentially injects bundled aggregations into the unsigned portion of the serialized event.

@@ -483,7 +482,8 @@ class EventClientSerializer:
            aggregations: The bundled aggregation to serialize.
            serialized_event: The serialized event which may be modified.
            config: Event serialization config
-
+            apply_edits: Whether the content of the event should be modified to reflect
+               any replacement in `aggregations.replace`.
        """
        serialized_aggregations = {}

@@ -494,9 +494,10 @@ class EventClientSerializer:
            serialized_aggregations[RelationTypes.REFERENCE] = aggregations.references

        if aggregations.replace:
-            # If there is an edit, apply it to the event.
+            # If there is an edit, optionally apply it to the event.
            edit = aggregations.replace
-            self._apply_edit(event, serialized_event, edit)
+            if apply_edits:
+                self._apply_edit(event, serialized_event, edit)

            # Include information about it in the relations dict.
            serialized_aggregations[RelationTypes.REPLACE] = {
@@ -525,8 +526,6 @@ class EventClientSerializer:
                "current_user_participated": thread.current_user_participated,
            }
            serialized_aggregations[RelationTypes.THREAD] = thread_summary
-            if self._msc3440_enabled:
-                serialized_aggregations[RelationTypes.UNSTABLE_THREAD] = thread_summary

        # Include the bundled aggregations in the event.
        if serialized_aggregations:
@@ -515,7 +515,7 @@ class FederationServer(FederationBase):
        )

    async def on_room_state_request(
-        self, origin: str, room_id: str, event_id: Optional[str]
+        self, origin: str, room_id: str, event_id: str
    ) -> Tuple[int, JsonDict]:
        origin_host, _ = parse_server_name(origin)
        await self.check_server_matches_acl(origin_host, room_id)
@@ -530,18 +530,13 @@ class FederationServer(FederationBase):
        # - but that's non-trivial to get right, and anyway somewhat defeats
        # the point of the linearizer.
        async with self._server_linearizer.queue((origin, room_id)):
-            resp: JsonDict = dict(
-                await self._state_resp_cache.wrap(
-                    (room_id, event_id),
-                    self._on_context_state_request_compute,
-                    room_id,
-                    event_id,
-                )
+            resp = await self._state_resp_cache.wrap(
+                (room_id, event_id),
+                self._on_context_state_request_compute,
+                room_id,
+                event_id,
            )

-        room_version = await self.store.get_room_version_id(room_id)
-        resp["room_version"] = room_version
-
        return 200, resp

    async def on_state_ids_request(
@@ -574,14 +569,11 @@ class FederationServer(FederationBase):
        return {"pdu_ids": state_ids, "auth_chain_ids": list(auth_chain_ids)}

    async def _on_context_state_request_compute(
-        self, room_id: str, event_id: Optional[str]
+        self, room_id: str, event_id: str
    ) -> Dict[str, list]:
        pdus: Collection[EventBase]
-        if event_id:
-            event_ids = await self.handler.get_state_ids_for_pdu(room_id, event_id)
-            pdus = await self.store.get_events_as_list(event_ids)
-        else:
-            pdus = (await self.state.get_current_state(room_id)).values()
+        event_ids = await self.handler.get_state_ids_for_pdu(room_id, event_id)
+        pdus = await self.store.get_events_as_list(event_ids)

        auth_chain = await self.store.get_auth_chain(
            room_id, [pdu.event_id for pdu in pdus]
@@ -687,8 +679,6 @@ class FederationServer(FederationBase):
        time_now = self._clock.time_msec()
        event_json = event.get_pdu_json(time_now)
        resp = {
-            # TODO Remove the unstable prefix when servers have updated.
-            "org.matrix.msc3083.v2.event": event_json,
            "event": event_json,
            "state": [p.get_pdu_json(time_now) for p in state_events],
            "auth_chain": [p.get_pdu_json(time_now) for p in auth_chain_events],
@@ -1380,16 +1380,6 @@ class SendJoinParser(ByteParser[SendJoinResponse]):
                prefix + "auth_chain.item",
                use_float=True,
            ),
-            # TODO Remove the unstable prefix when servers have updated.
-            #
-            # By re-using the same event dictionary this will cause the parsing of
-            # org.matrix.msc3083.v2.event and event to stomp over each other.
-            # Generally this should be fine.
-            ijson.kvitems_coro(
-                _event_parser(self._response.event_dict),
-                prefix + "org.matrix.msc3083.v2.event",
-                use_float=True,
-            ),
            ijson.kvitems_coro(
                _event_parser(self._response.event_dict),
                prefix + "event",
@@ -16,7 +16,8 @@ import functools
 import logging
 import re
 import time
-from typing import TYPE_CHECKING, Any, Awaitable, Callable, Optional, Tuple, cast
+from http import HTTPStatus
+from typing import TYPE_CHECKING, Any, Awaitable, Callable, Dict, Optional, Tuple, cast

 from synapse.api.errors import Codes, FederationDeniedError, SynapseError
 from synapse.api.urls import FEDERATION_V1_PREFIX
@@ -86,15 +87,24 @@ class Authenticator:

        if not auth_headers:
            raise NoAuthenticationError(
-                401, "Missing Authorization headers", Codes.UNAUTHORIZED
+                HTTPStatus.UNAUTHORIZED,
+                "Missing Authorization headers",
+                Codes.UNAUTHORIZED,
            )

        for auth in auth_headers:
            if auth.startswith(b"X-Matrix"):
-                (origin, key, sig) = _parse_auth_header(auth)
+                (origin, key, sig, destination) = _parse_auth_header(auth)
                json_request["origin"] = origin
                json_request["signatures"].setdefault(origin, {})[key] = sig

+                # if the origin_server sent a destination along it needs to match our own server_name
+                if destination is not None and destination != self.server_name:
+                    raise AuthenticationError(
+                        HTTPStatus.UNAUTHORIZED,
+                        "Destination mismatch in auth header",
+                        Codes.UNAUTHORIZED,
+                    )
        if (
            self.federation_domain_whitelist is not None
            and origin not in self.federation_domain_whitelist
@@ -103,7 +113,9 @@ class Authenticator:

        if origin is None or not json_request["signatures"]:
            raise NoAuthenticationError(
-                401, "Missing Authorization headers", Codes.UNAUTHORIZED
+                HTTPStatus.UNAUTHORIZED,
+                "Missing Authorization headers",
+                Codes.UNAUTHORIZED,
            )

        await self.keyring.verify_json_for_server(
@@ -142,13 +154,14 @@ class Authenticator:
            logger.exception("Error resetting retry timings on %s", origin)


-def _parse_auth_header(header_bytes: bytes) -> Tuple[str, str, str]:
+def _parse_auth_header(header_bytes: bytes) -> Tuple[str, str, str, Optional[str]]:
    """Parse an X-Matrix auth header

    Args:
        header_bytes: header value

    Returns:
+        origin, key id, signature, destination.
        origin, key id, signature.

    Raises:
@@ -157,7 +170,9 @@ def _parse_auth_header(header_bytes: bytes) -> Tuple[str, str, str]:
    try:
        header_str = header_bytes.decode("utf-8")
        params = header_str.split(" ")[1].split(",")
-        param_dict = {k: v for k, v in (kv.split("=", maxsplit=1) for kv in params)}
+        param_dict: Dict[str, str] = {
+            k: v for k, v in [param.split("=", maxsplit=1) for param in params]
+        }

        def strip_quotes(value: str) -> str:
            if value.startswith('"'):
@@ -172,7 +187,15 @@ def _parse_auth_header(header_bytes: bytes) -> Tuple[str, str, str]:

        key = strip_quotes(param_dict["key"])
        sig = strip_quotes(param_dict["sig"])
-        return origin, key, sig
+
+        # get the destination server_name from the auth header if it exists
+        destination = param_dict.get("destination")
+        if destination is not None:
+            destination = strip_quotes(destination)
+        else:
+            destination = None
+
+        return origin, key, sig, destination
    except Exception as e:
        logger.warning(
            "Error parsing auth header '%s': %s",
@@ -180,7 +203,7 @@ def _parse_auth_header(header_bytes: bytes) -> Tuple[str, str, str]:
            e,
        )
        raise AuthenticationError(
-            400, "Malformed Authorization header", Codes.UNAUTHORIZED
+            HTTPStatus.BAD_REQUEST, "Malformed Authorization header", Codes.UNAUTHORIZED
        )


@@ -160,7 +160,7 @@ class FederationStateV1Servlet(BaseFederationServerServlet):
        return await self.handler.on_room_state_request(
            origin,
            room_id,
-            parse_string_from_args(query, "event_id", None, required=False),
+            parse_string_from_args(query, "event_id", None, required=True),
        )


@@ -291,12 +291,6 @@ class DeviceHandler(DeviceWorkerHandler):
        # On start up check if there are any updates pending.
        hs.get_reactor().callWhenRunning(self._handle_new_device_update_async)

-        # Used to decide if we calculate outbound pokes up front or not. By
-        # default we do to allow safely downgrading Synapse.
-        self.use_new_device_lists_changes_in_room = (
-            hs.config.server.use_new_device_lists_changes_in_room
-        )
-
    def _check_device_name_length(self, name: Optional[str]) -> None:
        """
        Checks whether a device name is longer than the maximum allowed length.
@@ -490,23 +484,9 @@ class DeviceHandler(DeviceWorkerHandler):

        room_ids = await self.store.get_rooms_for_user(user_id)

-        hosts: Optional[Set[str]] = None
-        if not self.use_new_device_lists_changes_in_room:
-            hosts = set()
-
-            if self.hs.is_mine_id(user_id):
-                for room_id in room_ids:
-                    joined_users = await self.store.get_users_in_room(room_id)
-                    hosts.update(get_domain_from_id(u) for u in joined_users)
-
-                set_tag("target_hosts", hosts)
-
-                hosts.discard(self.server_name)
-
        position = await self.store.add_device_change_to_streams(
            user_id,
            device_ids,
-            hosts=hosts,
            room_ids=room_ids,
        )

@@ -528,14 +508,6 @@ class DeviceHandler(DeviceWorkerHandler):
        # We may need to do some processing asynchronously.
        self._handle_new_device_update_async()

-        if hosts:
-            logger.info(
-                "Sending device list update notif for %r to: %r", user_id, hosts
-            )
-            for host in hosts:
-                self.federation_sender.send_device_messages(host, immediate=False)
-                log_kv({"message": "sent device update to host", "host": host})
-
    async def notify_user_signature_update(
        self, from_user_id: str, user_ids: List[str]
    ) -> None:
@@ -677,9 +649,13 @@ class DeviceHandler(DeviceWorkerHandler):
                        return

                for user_id, device_id, room_id, stream_id, opentracing_context in rows:
-                    joined_user_ids = await self.store.get_users_in_room(room_id)
-                    hosts = {get_domain_from_id(u) for u in joined_user_ids}
-                    hosts.discard(self.server_name)
+                    hosts = set()
+
+                    # Ignore any users that aren't ours
+                    if self.hs.is_mine_id(user_id):
+                        joined_user_ids = await self.store.get_users_in_room(room_id)
+                        hosts = {get_domain_from_id(u) for u in joined_user_ids}
+                        hosts.discard(self.server_name)

                    # Check if we've already sent this update to some hosts
                    if current_stream_id == stream_id:
@@ -16,11 +16,12 @@ import logging
 import random
 from typing import TYPE_CHECKING, Iterable, List, Optional

-from synapse.api.constants import EduTypes, EventTypes, Membership
+from synapse.api.constants import EduTypes, EventTypes, Membership, PresenceState
 from synapse.api.errors import AuthError, SynapseError
 from synapse.events import EventBase
 from synapse.events.utils import SerializeEventConfig
 from synapse.handlers.presence import format_user_presence_state
+from synapse.storage.databases.main.events_worker import EventRedactBehaviour
 from synapse.streams.config import PaginationConfig
 from synapse.types import JsonDict, UserID
 from synapse.visibility import filter_events_for_client
@@ -67,7 +68,9 @@ class EventStreamHandler:
        presence_handler = self.hs.get_presence_handler()

        context = await presence_handler.user_syncing(
-            auth_user_id, affect_presence=affect_presence
+            auth_user_id,
+            affect_presence=affect_presence,
+            presence_state=PresenceState.ONLINE,
        )
        with context:
            if timeout:
@@ -139,7 +142,11 @@ class EventHandler:
        self.storage = hs.get_storage()

    async def get_event(
-        self, user: UserID, room_id: Optional[str], event_id: str
+        self,
+        user: UserID,
+        room_id: Optional[str],
+        event_id: str,
+        show_redacted: bool = False,
    ) -> Optional[EventBase]:
        """Retrieve a single specified event.

@@ -148,6 +155,7 @@ class EventHandler:
            room_id: The expected room id. We'll return None if the
                event's room does not match.
            event_id: The event ID to obtain.
+            show_redacted: Should the full content of redacted events be returned?
        Returns:
            An event, or None if there is no event matching this ID.
        Raises:
@@ -155,7 +163,12 @@ class EventHandler:
            AuthError if the user does not have the rights to inspect this
            event.
        """
-        event = await self.store.get_event(event_id, check_room_id=room_id)
+        redact_behaviour = (
+            EventRedactBehaviour.AS_IS if show_redacted else EventRedactBehaviour.REDACT
+        )
+        event = await self.store.get_event(
+            event_id, check_room_id=room_id, redact_behaviour=redact_behaviour
+        )

        if not event:
            return None
--- a/Show More
+++ b/Show More
Author	SHA1	Message	Date
Matthew Hodgson	ba33466939	update sample config	2022-04-23 15:39:41 +01:00
Matthew Hodgson	8cb9abe8e7	changelog	2022-04-23 15:36:41 +01:00
Matthew Hodgson	0a6f1d1ac2	add default_power_level_content_override config option. Lets you override the default power levels for rooms created on this server, per room creation preset. Useful if you know that your users need special permissions in rooms that they create (e.g. to send particular types of state events without needing an elevated power level). This takes the same shape as the power_level_content_override parameter in the /createRoom API, but is applied before that parameter. This is something of a workaround in the absence of MSC3779 or MSC3761.	2022-04-23 15:25:25 +01:00
Sean Quah	a50fb411b3	Update `delay_cancellation` to accept any awaitable (#12468 ) This will mainly be useful when dealing with module callbacks, which are all typed as returning `Awaitable`s instead of coroutines or `Deferred`s. Signed-off-by: Sean Quah <seanq@element.io>	2022-04-22 18:20:06 +01:00
Will Hunt	b82fff66df	MSC3202: Fix device_unused_fallback_keys -> device_unused_fallback_key_types (#12520 ) * Fix device_unused_fallback_keys -> device_unused_fallback_key_types * changelog	2022-04-22 16:03:46 +01:00
Richard van der Hoff	f46b223354	turn-howto: fix some links	2022-04-22 14:23:40 +01:00
Richard van der Hoff	f5668f0b4a	Await un-partial-stating after a partial-state join (#12399 ) When we join a room via the faster-joins mechanism, we end up with "partial state" at some points on the event DAG. Many parts of the codebase need to wait for the full state to load. So, we implement a mechanism to keep track of which events have partial state, and wait for them to be fully-populated.	2022-04-21 07:42:03 +01:00
David Robertson	09b4f6e46d	Remove leftover references to setup.py (#12514 ) * Remove leftover references to setup.py Missed in #12478. * Changelog	2022-04-20 18:16:49 +00:00
Olivier Wilkinson (reivilibre)	01c8f9ca69	Merge branch 'master' into develop	2022-04-20 17:44:19 +01:00
David Robertson	e5a76ec00b	Dump setuptools; correct pyproject version number (#12478 )	2022-04-20 17:33:20 +01:00
Patrick Cloke	103f51d867	Fix Jinja templating error when generating thumbnail URLs. (#12510 ) scale is meant to be a constant string, not refer to a variable.	2022-04-20 12:03:03 -04:00
David Robertson	ecef741add	Recommend poetry in docs (#12475 ) * Recommend poetry in docs - readme - contributor guide - upgrade notes - new dev cheat sheet for poetry Co-authored-by: Shay <hillerys@element.io> Co-authored-by: Patrick Cloke <clokep@users.noreply.github.com>	2022-04-20 15:18:21 +01:00
Patrick Cloke	d0c1f4ca4c	Remove unnecessary config overrides for MSC3666. (#12511 )	2022-04-20 09:56:59 -04:00
Tulir Asokan	4bc8cb4669	Implement MSC2815: allow room moderators to view redacted event content (#12427 ) Implements matrix-org/matrix-spec-proposals#2815 Signed-off-by: Tulir Asokan <tulir@maunium.net>	2022-04-20 12:57:39 +01:00
David Robertson	eed38c5027	Add CI job to act as a canary for testing against latest dependencies (#12472 ) Co-authored-by: Patrick Cloke <clokep@users.noreply.github.com>	2022-04-20 12:48:44 +01:00
Erik Johnston	c1482a352a	Fix returned count of delete extremities admin API (#12496 )	2022-04-19 16:49:45 +01:00
Richard van der Hoff	b80bb7e452	Fix `/room/.../event/...` to return the original event after any edits (#12476 ) This is what the MSC (now) requires. Fixes https://github.com/matrix-org/synapse/issues/10310.	2022-04-19 16:42:19 +01:00
Richard van der Hoff	798deb3a10	Fix typo in deb changelogs from release script (#12497 ) The release script used to incorrectly write `New synapse release 1.57.0~rc1.` instead of `New synapse release 1.57.0rc1.`	2022-04-19 16:41:52 +01:00
Jan Christian Grünhage	a1f87f57ff	Implement MSC3383: include destination in X-Matrix auth header (#11398 ) Co-authored-by: Jan Christian Grünhage <jan.christian@gruenhage.xyz> Co-authored-by: Marcus Hoffmann <bubu@bubu1.eu>	2022-04-19 16:23:53 +01:00
Dirk Klimpel	fbdee86004	Fix a link in `README.rst` (#12495 ) * Fix a link in `README.rst` * newsfile	2022-04-19 13:00:41 +00:00
Richard van der Hoff	7dec4ce7e4	fix typo in debian changelog	2022-04-19 13:45:27 +01:00
Olivier Wilkinson (reivilibre)	dbe016e258	Remove 'Non-maintainer upload' line from Debian changelog	2022-04-19 13:33:36 +01:00
Olivier Wilkinson (reivilibre)	0921d93dcd	Merge branch 'master' into develop	2022-04-19 13:28:28 +01:00
Richard van der Hoff	b121a3ad2b	Back out implementation of MSC2314 (#12474 ) MSC2314 has now been closed, so we're backing out its implementation, which originally happened in #6176. Unfortunately it's not a direct revert, as that PR mixed in a bunch of unrelated changes to tests etc.	2022-04-19 11:17:29 +00:00
Travis Ralston	f8d3ee9570	Fix grammatical error in error message (#12483 ) * Fix grammatical error in error message * changelog	2022-04-18 12:41:55 -06:00
Shay	3c758d9808	Add a manual documenting config file options (#12368 )	2022-04-18 11:32:30 -07:00
Richard van der Hoff	aaaff98202	Dockerfile-workers: reduce the amount we install (#12464 ) This is an attempt to reduce the rebuild time. In short, we reduce the amount of stuff that the dockerfile installs, so as to give a faster startup.	2022-04-14 15:36:49 +01:00
reivilibre	7efddbebef	Update documentation to reflect that `run_background_tasks_on` is no longer experimental. (#12451 ) * Background workers aren't experimental anymore * Newsfile Signed-off-by: Olivier Wilkinson (reivilibre) <oliverw@matrix.org> * Stream writers aren't experimental either	2022-04-14 15:25:22 +01:00
Richard van der Hoff	960b4fb409	complement-synapse-workers: factor out separate entry point script (#12467 ) ... with a bit more verbosity.	2022-04-14 14:56:10 +01:00
Dirk Klimpel	a743f7d33e	Replace `federation_reader` with `generic_worker` in docs (#12457 )	2022-04-14 13:09:07 +01:00
Erik Johnston	0b014eb25e	Only send out device list updates for our own users (#12465 ) Broke in #12365	2022-04-14 13:05:31 +01:00
David Robertson	535a689cfc	Reintroduce the lint targets in the linter script (#12455 )	2022-04-14 11:33:06 +01:00
David Robertson	6b3e0ea6bd	Use `poetry` to manage the virtualenv in debian packages (#12449 ) * Use `poetry` to build venv in debian packages Co-authored-by: Dan Callahan <danc@element.io> Co-authored-by: Shay <hillerys@element.io> * Changelog * Only pull in from requirements.txt Addresses the same problem as #12439. * Include `test` and `all` extras `poetry export` helpfully silently ignores an unknown extra Haven't seen this before because it's the only place we export `all` and `test`. I could have __sworm__ that the syntax `--extra "all test"` worked for `poetry install`... * Clean up requirements file on subsequence builds * Fix shell syntax Co-authored-by: Dan Callahan <danc@element.io> Co-authored-by: Shay <hillerys@element.io>	2022-04-14 11:03:24 +01:00
Richard van der Hoff	8af8a9bce5	Dockerfile-workers: give the master its own log config (#12466 ) When we run a worker-mode synapse under docker, everything gets logged to stdout. Currently, output from the workers is tacked with a worker name, for example: ``` 2022-04-13 15:27:56,810 - worker:frontend_proxy1 - synapse.util.caches.lrucache - 154 - INFO - LruCache._expire_old_entries-0 - Dropped 0 items from caches ``` - note `worker:frontend_proxy1`. No such tag is applied to log lines from the master, which makes somewhat confusing reading. To fix this, we generate a dedicated log config file for the master in the same way that we do for the workers, and use that.	2022-04-13 20:50:08 +01:00
Shay	8e2759f2d8	Limit `device_id` size to 512B (#12454 ) *	2022-04-13 10:04:01 -07:00
Andrew Morgan	0922462fc7	docs: Don't render the table of contents on the print page (#12340 )	2022-04-13 08:27:51 -07:00
David Baker	73d8ded0b0	Prevent a sync request from removing a user's busy presence status (#12213 ) In trying to use the MSC3026 busy presence status, the user's status would be set back to 'online' next time they synced. This change makes it so that syncing does not affect a user's presence status if it is currently set to 'busy': it must be removed through the presence API. The MSC defers to implementations on the behaviour of busy presence, so this ought to remain compatible with the MSC.	2022-04-13 16:21:07 +01:00
Nick Mills-Barrett	e3a49f4784	Fix missing sync events during historical batch imports (#12319 ) Discovered after much in-depth investigation in #12281. Closes: #12281 Closes: #3305 Signed off by: Nick Mills-Barrett nick@beeper.com	2022-04-13 11:38:35 +01:00
David Robertson	d24cd17820	Use poetry lockfile in twisted trunk CI job (#12425 ) Fixes #12458 Co-authored-by: Sean Quah <seanq@element.io> Co-authored-by: Dan Callahan <danc@element.io>	2022-04-13 11:26:53 +01:00
Jonathan de Jong	36d8b83888	Rename Mutual Rooms `unstable_features` flag to match MSC (#12445 ) Signed-off-by: Jonathan de Jong <jonathan@automatia.nl>	2022-04-13 10:32:44 +01:00
Shay	32545d2e26	Bump twisted version to the latest in lockfile (#12441 )	2022-04-12 13:46:55 -07:00
David Robertson	5a275a2377	Run "main" trial tests under poetry (#12438 ) * Run "main" trial tests under poetry Olddeps and twisted trunk tests are handled in separate PRs. The PyPy config is a best-effort only; it's completely untested. Pulled out from #12337. * Changelog	2022-04-12 17:41:21 +01:00
David Robertson	58c657322a	Run lints under poetry in CI; remove lint tox jobs (#12434 ) Co-authored-by: Dan Callahan <danc@element.io>	2022-04-12 17:35:48 +01:00
Erik Johnston	aa28110264	Process device list updates asynchronously (#12365 )	2022-04-12 16:50:40 +01:00
Patrick Cloke	4bdbebccb9	Remove the unstable event field for `/send_join` per MSC3083. (#12395 ) This was missed when initially stabilising room version 8 and was left in as a compatibility shim. Most homeservers have upgraded to a version which expects the proper field name, and the failure mode is reasonable (a user on an older server may have to attempt joining the room twice with an obscure error message the first time).	2022-04-12 11:27:45 -04:00
Erik Johnston	ba1588461b	Fix typos in release script docs (#12450 )	2022-04-12 15:12:57 +00:00
Erik Johnston	a468768104	Merge branch 'release-v1.57' into develop	2022-04-12 15:28:41 +01:00
Patrick Cloke	9535fd0f9c	Disable groups/communities by default. (#12344 ) This disables the endpoints (and sync response fields) for groups/communities by default.	2022-04-12 10:20:46 -04:00
Richard van der Hoff	320186319a	Resync state after partial-state join (#12394 ) We work through all the events with partial state, updating the state at each of them. Once it's done, we recalculate the state for the whole room, and then mark the room as having complete state.	2022-04-12 13:23:43 +00:00
Patrick Cloke	86cf6a3a17	Remove references to unstable identifiers from MSC3440. (#12382 ) Removes references to unstable thread relation, unstable identifiers for filtering parameters, and the experimental config flag.	2022-04-12 08:42:03 -04:00
				`@@ -0,0 +1 @@`
				`Implement [MSC3383](https://github.com/matrix-org/matrix-spec-proposals/pull/3383) for including the destination in server-to-server authentication headers. Contributed by @Bubu and @jcgruenhage for Famedly GmbH.`
				`@@ -0,0 +1 @@`
				`Prevent a sync request from removing a user's busy presence status.`
				`@@ -0,0 +1 @@`
				`Fix bug with incremental sync missing events when rejoining/backfilling. Contributed by Nick @ Beeper.`
				`@@ -0,0 +1 @@`
				`Use poetry to manage Synapse's dependencies.`
				`@@ -0,0 +1 @@`
				`Fix rendering of the documentation site when using the 'print' feature.`
				`@@ -0,0 +1 @@`
				`The groups/communities feature in Synapse has been disabled by default.`
				`@@ -0,0 +1 @@`
				`Enable processing of device list updates asynchronously.`
				`@@ -0,0 +1 @@`
				`Add a manual documenting config file options.`
				`@@ -0,0 +1 @@`
				`Remove unstable identifiers from [MSC3440](https://github.com/matrix-org/matrix-doc/pull/3440).`