changelog

.ci/scripts/test_old_deps.sh

@@ -1,14 +1,12 @@
 #!/usr/bin/env bash
-# this script is run by GitHub Actions in a plain `focal` container; it installs the
-# minimal requirements for tox and hands over to the py3-old tox environment.
 
-# Prevent tzdata from asking for user input
-export DEBIAN_FRONTEND=noninteractive
+# this script is run by GitHub Actions in a plain `bionic` container; it installs the
+# minimal requirements for tox and hands over to the py3-old tox environment.
 
 set -ex
 
 apt-get update
-apt-get install -y python3 python3-dev python3-pip libxml2-dev libxslt-dev xmlsec1 zlib1g-dev tox libjpeg-dev libwebp-dev
+apt-get install -y python3 python3-dev python3-pip libxml2-dev libxslt-dev xmlsec1 zlib1g-dev tox
 
 export LANG="C.UTF-8"
 

.github/PULL_REQUEST_TEMPLATE.md

@@ -8,7 +8,6 @@
   - Use markdown where necessary, mostly for `code blocks`.
   - End with either a period (.) or an exclamation mark (!).
   - Start with a capital letter.
-  - Feel free to credit yourself, by adding a sentence "Contributed by @github_username." or "Contributed by [Your Name]." to the end of the entry.
 * [ ] Pull request includes a [sign off](https://matrix-org.github.io/synapse/latest/development/contributing_guide.html#sign-off)
 * [ ] [Code style](https://matrix-org.github.io/synapse/latest/code_style.html) is correct
   (run the [linters](https://matrix-org.github.io/synapse/latest/development/contributing_guide.html#run-the-linters))

.github/workflows/docker.yml

@@ -34,8 +34,6 @@ jobs:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_TOKEN }}
 
-      # TODO: consider using https://github.com/docker/metadata-action instead of this
-      # custom magic
       - name: Calculate docker image tag
         id: set-tag
         run: |
@@ -55,6 +53,18 @@ jobs:
           esac
           echo "::set-output name=tag::$tag"
 
+        # for release builds, we want to get the amd64 image out asap, so first
+        # we do an amd64-only build, before following up with a multiarch build.
+      - name: Build and push amd64
+        uses: docker/build-push-action@v2
+        if: "${{ startsWith(github.ref, 'refs/tags/v') }}"
+        with:
+          push: true
+          labels: "gitsha1=${{ github.sha }}"
+          tags: "matrixdotorg/synapse:${{ steps.set-tag.outputs.tag }}"
+          file: "docker/Dockerfile"
+          platforms: linux/amd64
+
       - name: Build and push all platforms
         uses: docker/build-push-action@v2
         with:
@@ -63,69 +73,3 @@ jobs:
           tags: "matrixdotorg/synapse:${{ steps.set-tag.outputs.tag }}"
           file: "docker/Dockerfile"
           platforms: linux/amd64,linux/arm64
-
-  build_workers_test:
-    runs-on: ubuntu-latest
-
-    # The worker test image depends on the base image, so we must build the base
-    # first.
-    needs: build
-
-    permissions:
-      packages: write
-      contents: read
-
-    steps:
-      - name: Set up QEMU
-        id: qemu
-        uses: docker/setup-qemu-action@v1
-        with:
-          platforms: arm64
-
-      - name: Set up Docker Buildx
-        id: buildx
-        uses: docker/setup-buildx-action@v1
-
-      - name: Inspect builder
-        run: docker buildx inspect
-
-      - name: Login to GitHub Container Registry (for worker-testing-only image)
-        if: github.event_name != 'pull_request'
-        uses: docker/login-action@v1
-        with:
-          registry: ghcr.io
-          username: ${{ github.repository_owner }}
-          password: ${{ secrets.GITHUB_TOKEN }}
-
-      # This must match EXACTLY the one in the `build` pipeline.
-      - name: Calculate docker image tag
-        id: set-tag
-        run: |
-          case "${GITHUB_REF}" in
-              refs/heads/develop)
-                  tag=develop
-                  ;;
-              refs/heads/master|refs/heads/main)
-                  tag=latest
-                  ;;
-              refs/tags/*)
-                  tag=${GITHUB_REF#refs/tags/}
-                  ;;
-              *)
-                  tag=${GITHUB_SHA}
-                  ;;
-          esac
-          echo "::set-output name=tag::$tag"
-
-      # This image is solely intended to be used for automated test tools,
-      # such as mx-tester.
-      - name: Build and push worker-testing-only image for all platforms
-        uses: docker/build-push-action@v2
-        with:
-          push: true
-          build-args: |
-            "base_version=${{ steps.set-tag.outputs.tag }}"
-          labels: "gitsha1=${{ github.sha }}"
-          tags: "ghcr.io/matrix-org/synapse-workers-testing-only:${{ steps.set-tag.outputs.tag }}"
-          file: "docker/Dockerfile-workers"
-          platforms: linux/amd64,linux/arm64

.github/workflows/tests.yml

@@ -141,7 +141,7 @@ jobs:
     steps:
       - uses: actions/checkout@v2
       - name: Test with old deps
-        uses: docker://ubuntu:focal # For old python and sqlite
+        uses: docker://ubuntu:bionic # For old python and sqlite
         with:
           workdir: /github/workspace
           entrypoint: .ci/scripts/test_old_deps.sh
@@ -213,15 +213,15 @@ jobs:
       fail-fast: false
       matrix:
         include:
-          - sytest-tag: focal
+          - sytest-tag: bionic
 
-          - sytest-tag: focal
+          - sytest-tag: bionic
             postgres: postgres
 
           - sytest-tag: testing
             postgres: postgres
 
-          - sytest-tag: focal
+          - sytest-tag: bionic
             postgres: multi-postgres
             workers: workers
 
@@ -323,22 +323,17 @@ jobs:
     if: ${{ !failure() && !cancelled() }}
     needs: linting-done
     runs-on: ubuntu-latest
+    container:
+      # https://github.com/matrix-org/complement/blob/master/dockerfiles/ComplementCIBuildkite.Dockerfile
+      image: matrixdotorg/complement:latest
+      env:
+        CI: true
+      ports:
+        - 8448:8448
+      volumes:
+        - /var/run/docker.sock:/var/run/docker.sock
 
     steps:
-      # The path is set via a file given by $GITHUB_PATH. We need both Go 1.17 and GOPATH on the path to run Complement.
-      # See https://docs.github.com/en/actions/using-workflows/workflow-commands-for-github-actions#adding-a-system-path
-      - name: "Set Go Version"
-        run: |
-          # Add Go 1.17 to the PATH: see https://github.com/actions/virtual-environments/blob/main/images/linux/Ubuntu2004-Readme.md#environment-variables-2
-          echo "$GOROOT_1_17_X64/bin" >> $GITHUB_PATH
-          # Add the Go path to the PATH: We need this so we can call gotestfmt
-          echo "~/go/bin" >> $GITHUB_PATH
-
-      - name: "Install Complement Dependencies"
-        run: |
-          sudo apt-get update && sudo apt-get install -y libolm3 libolm-dev
-          go get -v github.com/haveyoudebuggedit/gotestfmt/v2/cmd/gotestfmt@latest
-
       - name: Run actions/checkout@v2 for synapse
         uses: actions/checkout@v2
         with:
@@ -371,8 +366,6 @@ jobs:
       # Build initial Synapse image
       - run: docker build -t matrixdotorg/synapse:latest -f docker/Dockerfile .
         working-directory: synapse
-        env:
-          DOCKER_BUILDKIT: 1
 
       # Build a ready-to-run Synapse image based on the initial image above.
       # This new image includes a config file, keys for signing and TLS, and
@@ -381,11 +374,7 @@ jobs:
         working-directory: complement/dockerfiles
 
       # Run Complement
-      - run: |
-          set -o pipefail
-          go test -v -json -tags synapse_blacklist,msc2403 ./tests/... 2>&1 | gotestfmt
-        shell: bash
-        name: Run Complement Tests
+      - run: go test -v -tags synapse_blacklist,msc2403 ./tests/...
         env:
           COMPLEMENT_BASE_IMAGE: complement-synapse:latest
         working-directory: complement

.github/workflows/twisted_trunk.yml

@@ -25,7 +25,7 @@ jobs:
       - run: sudo apt-get -qq install xmlsec1
       - uses: actions/setup-python@v2
         with:
-          python-version: 3.7
+          python-version: 3.6
       - run: .ci/patch_for_twisted_trunk.sh
       - run: pip install tox
       - run: tox -e py

.gitignore

@@ -50,7 +50,3 @@ __pycache__/
 
 # docs
 book/
-
-# complement
-/complement-*
-/master.tar.gz

CHANGES.md

@@ -1,123 +1,6 @@
-Synapse 1.51.0 (2022-01-25)
-===========================
-
-No significant changes since 1.51.0rc2.
-
-Synapse 1.51.0 deprecates `webclient` listeners and non-HTTP(S) `web_client_location`s. Support for these will be removed in Synapse 1.53.0, at which point Synapse will not be capable of directly serving a web client for Matrix. See the [upgrade notes](https://matrix-org.github.io/synapse/develop/upgrade#upgrading-to-v1510).
-
-Synapse 1.51.0rc2 (2022-01-24)
-==============================
-
-Bugfixes
---------
-
-- Fix a bug introduced in Synapse 1.40.0 that caused Synapse to fail to process incoming federation traffic after handling a large amount of events in a v1 room. ([\#11806](https://github.com/matrix-org/synapse/issues/11806))
-
-
-Synapse 1.50.2 (2022-01-24)
-===========================
-
-This release includes the same bugfix as Synapse 1.51.0rc2.
-
-Bugfixes
---------
-
-- Fix a bug introduced in Synapse 1.40.0 that caused Synapse to fail to process incoming federation traffic after handling a large amount of events in a v1 room. ([\#11806](https://github.com/matrix-org/synapse/issues/11806))
-
-
-Synapse 1.51.0rc1 (2022-01-21)
-==============================
-
-Features
---------
-
-- Add `track_puppeted_user_ips` config flag to record client IP addresses against puppeted users, and include the puppeted users in monthly active user counts. ([\#11561](https://github.com/matrix-org/synapse/issues/11561), [\#11749](https://github.com/matrix-org/synapse/issues/11749), [\#11757](https://github.com/matrix-org/synapse/issues/11757))
-- Include whether the requesting user has participated in a thread when generating a summary for [MSC3440](https://github.com/matrix-org/matrix-doc/pull/3440). ([\#11577](https://github.com/matrix-org/synapse/issues/11577))
-- Return an `M_FORBIDDEN` error code instead of `M_UNKNOWN` when a spam checker module prevents a user from creating a room. ([\#11672](https://github.com/matrix-org/synapse/issues/11672))
-- Add a flag to the `synapse_review_recent_signups` script to ignore and filter appservice users. ([\#11675](https://github.com/matrix-org/synapse/issues/11675), [\#11770](https://github.com/matrix-org/synapse/issues/11770))
-
-
-Bugfixes
---------
-
-- Fix a long-standing issue which could cause Synapse to incorrectly accept data in the unsigned field of events
-  received over federation. ([\#11530](https://github.com/matrix-org/synapse/issues/11530))
-- Fix a long-standing bug where Synapse wouldn't cache a response indicating that a remote user has no devices. ([\#11587](https://github.com/matrix-org/synapse/issues/11587))
-- Fix an error that occurs whilst trying to get the federation status of a destination server that was working normally. This admin API was newly introduced in Synapse v1.49.0. ([\#11593](https://github.com/matrix-org/synapse/issues/11593))
-- Fix bundled aggregations not being included in the `/sync` response, per [MSC2675](https://github.com/matrix-org/matrix-doc/pull/2675). ([\#11612](https://github.com/matrix-org/synapse/issues/11612), [\#11659](https://github.com/matrix-org/synapse/issues/11659), [\#11791](https://github.com/matrix-org/synapse/issues/11791))
-- Fix the `/_matrix/client/v1/room/{roomId}/hierarchy` endpoint returning incorrect fields which have been present since Synapse 1.49.0. ([\#11667](https://github.com/matrix-org/synapse/issues/11667))
-- Fix preview of some GIF URLs (like tenor.com). Contributed by Philippe Daouadi. ([\#11669](https://github.com/matrix-org/synapse/issues/11669))
-- Fix a bug where only the first 50 rooms from a space were returned from the `/hierarchy` API. This has existed since the introduction of the API in Synapse v1.41.0. ([\#11695](https://github.com/matrix-org/synapse/issues/11695))
-- Fix a bug introduced in Synapse v1.18.0 where password reset and address validation emails would not be sent if their subject was configured to use the 'app' template variable. Contributed by @br4nnigan. ([\#11710](https://github.com/matrix-org/synapse/issues/11710), [\#11745](https://github.com/matrix-org/synapse/issues/11745))
-- Make the 'List Rooms' Admin API sort stable. Contributed by Daniël Sonck. ([\#11737](https://github.com/matrix-org/synapse/issues/11737))
-- Fix a long-standing bug where space hierarchy over federation would only work correctly some of the time. ([\#11775](https://github.com/matrix-org/synapse/issues/11775))
-- Fix a bug introduced in Synapse v1.46.0 that prevented `on_logged_out` module callbacks from being correctly awaited by Synapse. ([\#11786](https://github.com/matrix-org/synapse/issues/11786))
-
-
-Improved Documentation
-----------------------
-
-- Warn against using a Let's Encrypt certificate for TLS/DTLS TURN server client connections, and suggest using ZeroSSL certificate instead. This works around client-side connectivity errors caused by WebRTC libraries that reject Let's Encrypt certificates. Contibuted by @AndrewFerr. ([\#11686](https://github.com/matrix-org/synapse/issues/11686))
-- Document the new `SYNAPSE_TEST_PERSIST_SQLITE_DB` environment variable in the contributing guide. ([\#11715](https://github.com/matrix-org/synapse/issues/11715))
-- Document that the minimum supported PostgreSQL version is now 10. ([\#11725](https://github.com/matrix-org/synapse/issues/11725))
-- Fix typo in demo docs: differnt. ([\#11735](https://github.com/matrix-org/synapse/issues/11735))
-- Update room spec URL in config files. ([\#11739](https://github.com/matrix-org/synapse/issues/11739))
-- Mention `python3-venv` and `libpq-dev` dependencies in the contribution guide. ([\#11740](https://github.com/matrix-org/synapse/issues/11740))
-- Update documentation for configuring login with Facebook. ([\#11755](https://github.com/matrix-org/synapse/issues/11755))
-- Update installation instructions to note that Python 3.6 is no longer supported. ([\#11781](https://github.com/matrix-org/synapse/issues/11781))
-
-
-Deprecations and Removals
--------------------------
-
-- Remove the unstable `/send_relation` endpoint. ([\#11682](https://github.com/matrix-org/synapse/issues/11682))
-- Remove `python_twisted_reactor_pending_calls` Prometheus metric. ([\#11724](https://github.com/matrix-org/synapse/issues/11724))
-- Remove the `password_hash` field from the response dictionaries of the [Users Admin API](https://matrix-org.github.io/synapse/latest/admin_api/user_admin_api.html). ([\#11576](https://github.com/matrix-org/synapse/issues/11576))
-- **Deprecate support for `webclient` listeners and non-HTTP(S) `web_client_location` configuration. ([\#11774](https://github.com/matrix-org/synapse/issues/11774), [\#11783](https://github.com/matrix-org/synapse/issues/11783))**
-
-
-Internal Changes
-----------------
-
-- Run `pyupgrade --py37-plus --keep-percent-format` on Synapse. ([\#11685](https://github.com/matrix-org/synapse/issues/11685))
-- Use buildkit's cache feature to speed up docker builds. ([\#11691](https://github.com/matrix-org/synapse/issues/11691))
-- Use `auto_attribs` and native type hints for attrs classes. ([\#11692](https://github.com/matrix-org/synapse/issues/11692), [\#11768](https://github.com/matrix-org/synapse/issues/11768))
-- Remove debug logging for #4422, which has been closed since Synapse 0.99. ([\#11693](https://github.com/matrix-org/synapse/issues/11693))
-- Remove fallback code for Python 2. ([\#11699](https://github.com/matrix-org/synapse/issues/11699))
-- Add a test for [an edge case](https://github.com/matrix-org/synapse/pull/11532#discussion_r769104461) in the `/sync` logic. ([\#11701](https://github.com/matrix-org/synapse/issues/11701))
-- Add the option to write SQLite test dbs to disk when running tests. ([\#11702](https://github.com/matrix-org/synapse/issues/11702))
-- Improve Complement test output for Gitub Actions. ([\#11707](https://github.com/matrix-org/synapse/issues/11707))
-- Fix docstring on `add_account_data_for_user`. ([\#11716](https://github.com/matrix-org/synapse/issues/11716))
-- Complement environment variable name change and update `.gitignore`. ([\#11718](https://github.com/matrix-org/synapse/issues/11718))
-- Simplify calculation of Prometheus metrics for garbage collection. ([\#11723](https://github.com/matrix-org/synapse/issues/11723))
-- Improve accuracy of `python_twisted_reactor_tick_time` Prometheus metric. ([\#11724](https://github.com/matrix-org/synapse/issues/11724), [\#11771](https://github.com/matrix-org/synapse/issues/11771))
-- Minor efficiency improvements when inserting many values into the database. ([\#11742](https://github.com/matrix-org/synapse/issues/11742))
-- Invite PR authors to give themselves credit in the changelog. ([\#11744](https://github.com/matrix-org/synapse/issues/11744))
-- Add optional debugging to investigate [issue 8631](https://github.com/matrix-org/synapse/issues/8631). ([\#11760](https://github.com/matrix-org/synapse/issues/11760))
-- Remove `log_function` utility function and its uses. ([\#11761](https://github.com/matrix-org/synapse/issues/11761))
-- Add a unit test that checks both `client` and `webclient` resources will function when simultaneously enabled. ([\#11765](https://github.com/matrix-org/synapse/issues/11765))
-- Allow overriding complement commit using `COMPLEMENT_REF`. ([\#11766](https://github.com/matrix-org/synapse/issues/11766))
-- Add some comments and type annotations for `_update_outliers_txn`. ([\#11776](https://github.com/matrix-org/synapse/issues/11776))
-
-
-Synapse 1.50.1 (2022-01-18)
-===========================
-
-This release fixes a bug in Synapse 1.50.0 that could prevent clients from being able to connect to Synapse if the `webclient` resource was enabled. Further details are available in [this issue](https://github.com/matrix-org/synapse/issues/11763).
-
-Bugfixes
---------
-
-- Fix a bug introduced in Synapse 1.50.0rc1 that could cause Matrix clients to be unable to connect to Synapse instances with the `webclient` resource enabled. ([\#11764](https://github.com/matrix-org/synapse/issues/11764))
-
-
 Synapse 1.50.0 (2022-01-18)
 ===========================
 
-**This release contains a critical bug that may prevent clients from being able to connect. 
-As such, it is not recommended to upgrade to 1.50.0. Instead, please upgrade straight to
-to 1.50.1. Further details are available in [this issue](https://github.com/matrix-org/synapse/issues/11763).**
-
 Please note that we now only support Python 3.7+ and PostgreSQL 10+ (if applicable), because Python 3.6 and PostgreSQL 9.6 have reached end-of-life.
 
 No significant changes since 1.50.0rc2.

changelog.d/11612.bugfix

@@ -1 +0,0 @@
-Include the bundled aggregations in the `/sync` response, per [MSC2675](https://github.com/matrix-org/matrix-doc/pull/2675).

changelog.d/11621.feature

@@ -1 +0,0 @@
-Remove account data (including client config, push rules and ignored users) upon user deactivation.

changelog.d/11639.feature

@@ -1 +0,0 @@
-Add admin API to reset connection timeouts for remote server.

changelog.d/11658.feature

@@ -1 +0,0 @@
-Add an admin API to get a list of rooms that federate with a given remote homeserver.

changelog.d/11683.removal

@@ -1 +0,0 @@
-Drop support for Python 3.6, which is EOL.

changelog.d/11743.feature

@@ -1 +0,0 @@
-Add a config flag to inhibit M_USER_IN_USE during registration.

changelog.d/11765.misc

@@ -0,0 +1 @@
+Add a unit test that checks both `client` and `webclient` resources will function when simultaneously enabled.

changelog.d/11767.bugfix

@@ -1 +0,0 @@
-Fix a long-standing bug when previewing Reddit URLs which do not contain an image.

changelog.d/11784.bugfix

@@ -1 +0,0 @@
-Fix a long-standing bug that media streams could cause long-lived connections when generating URL previews.

changelog.d/11788.feature

@@ -1 +0,0 @@
-Remove account data (including client config, push rules and ignored users) upon user deactivation.

changelog.d/11789.feature

@@ -1 +0,0 @@
-Remove account data (including client config, push rules and ignored users) upon user deactivation.

changelog.d/11790.feature

@@ -1 +0,0 @@
-Add a module callback to set username at registration.

changelog.d/11792.misc

@@ -1 +0,0 @@
-Preparation for database schema simplifications: add `state_key` and `rejection_reason` columns to `events` table.

changelog.d/11793.misc

@@ -1 +0,0 @@
-Add `FrozenEvent.get_state_key` and use it in a couple of places.

changelog.d/11794.misc

@@ -1 +0,0 @@
-Preparation for database schema simplifications: stop reading from `event_reference_hashes`.

changelog.d/11795.misc

@@ -1 +0,0 @@
-Drop unused table `public_room_list_stream`.

changelog.d/11798.bugfix

@@ -1 +0,0 @@
-Include a `prev_content` field in state events sent to Application Services. Contributed by @totallynotvaishnav.

changelog.d/11799.misc

@@ -1 +0,0 @@
-Preparation for reducing Postgres serialization errors: allow setting transaction isolation level. Contributed by Nick @ Beeper.

changelog.d/11810.misc

@@ -1 +0,0 @@
-Docker: skip the initial amd64-only build and go straight to multiarch.

changelog.d/11811.misc

@@ -1 +0,0 @@
-Run Complement on the Github Actions VM and not inside a Docker container.

changelog.d/11813.misc

@@ -1 +0,0 @@
-Log module names at startup.

changelog.d/11815.misc

@@ -1 +0,0 @@
-Improve type safety of bundled aggregations code.

changelog.d/11816.misc

@@ -1 +0,0 @@
-Drop support for Python 3.6, which is EOL.

changelog.d/11817.misc

@@ -1 +0,0 @@
-Correct a type annotation in the event validation logic.

changelog.d/11820.doc

@@ -1 +0,0 @@
-Update pypi installation docs to indicate that we now support Python 3.10.

changelog.d/11821.doc

@@ -1 +0,0 @@
-Add missing steps to the contribution submission process in the documentation.  Contributed by @sequentialread.

changelog.d/11823.misc

@@ -1 +0,0 @@
-Minor updates and documentation for database schema delta files.

changelog.d/11827.bugfix

@@ -1 +0,0 @@
-Fix a bug introduced in Synapse 0.33.3 causing requests to sometimes log strings such as `HTTPStatus.OK` instead of integer status codes.

changelog.d/11830.misc

@@ -1 +0,0 @@
-Correct a type annotation in the event validation logic.

changelog.d/11834.misc

@@ -1 +0,0 @@
-Workaround a type annotation problem in `prometheus_client` 0.13.0.

changelog.d/11836.misc

@@ -1 +0,0 @@
-Minor performance improvement in room state lookup.

changelog.d/11838.misc

@@ -1 +0,0 @@
-Fix some indentation inconsistencies in the sample config.

changelog.d/11847.misc

@@ -1 +0,0 @@
-Preparation for reducing Postgres serialization errors: allow setting transaction isolation level. Contributed by Nick @ Beeper.

changelog.d/11852.misc

@@ -1 +0,0 @@
-Build Docker images for using worker-mode Synapse in automated test tools.

contrib/prometheus/consoles/synapse.html

@@ -92,6 +92,22 @@ new PromConsole.Graph({
 })
 </script>
 
+<h3>Pending calls per tick</h3>
+<div id="reactor_pending_calls"></div>
+<script>
+new PromConsole.Graph({
+  node: document.querySelector("#reactor_pending_calls"),
+  expr: "rate(python_twisted_reactor_pending_calls_sum[30s]) / rate(python_twisted_reactor_pending_calls_count[30s])",
+  name: "[[job]]-[[index]]",
+  min: 0,
+  renderer: "line",
+  height: 150,
+  yAxisFormatter: PromConsole.NumberFormatter.humanize,
+  yHoverFormatter: PromConsole.NumberFormatter.humanize,
+  yTitle: "Pending Calls"
+})
+</script>
+
 <h1>Storage</h1>
 
 <h3>Queries</h3>

debian/changelog

@@ -1,33 +1,3 @@
-matrix-synapse-py3 (1.51.0) stable; urgency=medium
-
-  * New synapse release 1.51.0.
-
- -- Synapse Packaging team <packages@matrix.org>  Tue, 25 Jan 2022 11:28:51 +0000
-
-matrix-synapse-py3 (1.51.0~rc2) stable; urgency=medium
-
-  * New synapse release 1.51.0~rc2.
-
- -- Synapse Packaging team <packages@matrix.org>  Mon, 24 Jan 2022 12:25:00 +0000
-
-matrix-synapse-py3 (1.51.0~rc1) stable; urgency=medium
-
-  * New synapse release 1.51.0~rc1.
-
- -- Synapse Packaging team <packages@matrix.org>  Fri, 21 Jan 2022 10:46:02 +0000
-
-matrix-synapse-py3 (1.50.2) stable; urgency=medium
-
-  * New synapse release 1.50.2.
-
- -- Synapse Packaging team <packages@matrix.org>  Mon, 24 Jan 2022 13:37:11 +0000
-
-matrix-synapse-py3 (1.50.1) stable; urgency=medium
-
-  * New synapse release 1.50.1.
-
- -- Synapse Packaging team <packages@matrix.org>  Tue, 18 Jan 2022 16:06:26 +0000
-
 matrix-synapse-py3 (1.50.0) stable; urgency=medium
 
   * New synapse release 1.50.0.

demo/README

@@ -22,5 +22,5 @@ Logs and sqlitedb will be stored in demo/808{0,1,2}.{log,db}
 
 
 
-Also note that when joining a public room on a different HS via "#foo:bar.net", then you are (in the current impl) joining a room with room_id "foo". This means that it won't work if your HS already has a room with that name.
+Also note that when joining a public room on a differnt HS via "#foo:bar.net", then you are (in the current impl) joining a room with room_id "foo". This means that it won't work if your HS already has a room with that name.
 

docker/Dockerfile

@@ -1,17 +1,14 @@
 # Dockerfile to build the matrixdotorg/synapse docker images.
 #
-# Note that it uses features which are only available in BuildKit - see
-# https://docs.docker.com/go/buildkit/ for more information.
-#
 # To build the image, run `docker build` command from the root of the
 # synapse repository:
 #
-#    DOCKER_BUILDKIT=1 docker build -f docker/Dockerfile .
+#    docker build -f docker/Dockerfile .
 #
 # There is an optional PYTHON_VERSION build argument which sets the
 # version of python to build against: for example:
 #
-#    DOCKER_BUILDKIT=1 docker build -f docker/Dockerfile --build-arg PYTHON_VERSION=3.9 .
+#    docker build -f docker/Dockerfile --build-arg PYTHON_VERSION=3.6 .
 #
 
 ARG PYTHON_VERSION=3.8
@@ -22,16 +19,7 @@ ARG PYTHON_VERSION=3.8
 FROM docker.io/python:${PYTHON_VERSION}-slim as builder
 
 # install the OS build deps
-#
-# RUN --mount is specific to buildkit and is documented at
-# https://github.com/moby/buildkit/blob/master/frontend/dockerfile/docs/syntax.md#build-mounts-run---mount.
-# Here we use it to set up a cache for apt, to improve rebuild speeds on
-# slow connections.
-#
-RUN \
-   --mount=type=cache,target=/var/cache/apt,sharing=locked \
-   --mount=type=cache,target=/var/lib/apt,sharing=locked \
- apt-get update && apt-get install -y \
+RUN apt-get update && apt-get install -y \
     build-essential \
     libffi-dev \
     libjpeg-dev \
@@ -56,8 +44,7 @@ COPY synapse/python_dependencies.py /synapse/synapse/python_dependencies.py
 # used while you develop on the source
 #
 # This is aiming at installing the `install_requires` and `extras_require` from `setup.py`
-RUN --mount=type=cache,target=/root/.cache/pip \
-  pip install --prefix="/install" --no-warn-script-location \
+RUN pip install --prefix="/install" --no-warn-script-location \
     /synapse[all]
 
 # Copy over the rest of the project
@@ -79,10 +66,7 @@ LABEL org.opencontainers.image.documentation='https://github.com/matrix-org/syna
 LABEL org.opencontainers.image.source='https://github.com/matrix-org/synapse.git'
 LABEL org.opencontainers.image.licenses='Apache-2.0'
 
-RUN \
-   --mount=type=cache,target=/var/cache/apt,sharing=locked \
-   --mount=type=cache,target=/var/lib/apt,sharing=locked \
-  apt-get update && apt-get install -y \
+RUN apt-get update && apt-get install -y \
     curl \
     gosu \
     libjpeg62-turbo \

docker/Dockerfile-pgtests

@@ -1,6 +1,6 @@
 # Use the Sytest image that comes with a lot of the build dependencies
 # pre-installed
-FROM matrixdotorg/sytest:focal
+FROM matrixdotorg/sytest:bionic
 
 # The Sytest image doesn't come with python, so install that
 RUN apt-get update && apt-get -qq install -y python3 python3-dev python3-pip

docker/Dockerfile-workers

@@ -1,8 +1,5 @@
-ARG base_version=latest
-
 # Inherit from the official Synapse docker image
-
-FROM matrixdotorg/synapse:$base_version
+FROM matrixdotorg/synapse
 
 # Install deps
 RUN apt-get update

docker/run_pg_tests.sh

@@ -16,4 +16,4 @@ sudo -u postgres /usr/lib/postgresql/10/bin/pg_ctl -w -D /var/lib/postgresql/dat
 # Run the tests
 cd /src
 export TRIAL_FLAGS="-j 4"
-tox --workdir=./.tox-pg-container -e py37-postgres "$@"
+tox --workdir=./.tox-pg-container -e py36-postgres "$@"

docs/admin_api/user_admin_api.md

@@ -15,10 +15,9 @@ server admin: [Admin API](../usage/administration/admin_api)
 
 It returns a JSON body like the following:
 
-```jsonc
+```json
 {
-    "name": "@user:example.com",
-    "displayname": "User", // can be null if not set
+    "displayname": "User",
     "threepids": [
         {
             "medium": "email",
@@ -33,11 +32,11 @@ It returns a JSON body like the following:
             "validated_at": 1586458409743
         }
     ],
-    "avatar_url": "<avatar_url>",  // can be null if not set
-    "is_guest": 0,
+    "avatar_url": "<avatar_url>",
     "admin": 0,
     "deactivated": 0,
     "shadow_banned": 0,
+    "password_hash": "$2b$12$p9B4GkqYdRTPGD",
     "creation_ts": 1560432506,
     "appservice_id": null,
     "consent_server_notice_sent": null,
@@ -353,11 +352,6 @@ The following actions are performed when deactivating an user:
 - Remove the user from the user directory
 - Reject all pending invites
 - Remove all account validity information related to the user
-- Remove the arbitrary data store known as *account data*. For example, this includes:
-    - list of ignored users;
-    - push rules;
-    - secret storage keys; and
-    - cross-signing keys.
 
 The following additional actions are performed during deactivation if `erase`
 is set to `true`:
@@ -371,6 +365,7 @@ The following actions are **NOT** performed. The list may be incomplete.
 - Remove mappings of SSO IDs
 - [Delete media uploaded](#delete-media-uploaded-by-a-user) by user (included avatar images)
 - Delete sent and received messages
+- Delete E2E cross-signing keys
 - Remove the user's creation (registration) timestamp
 - [Remove rate limit overrides](#override-ratelimiting-for-users)
 - Remove from monthly active users

docs/admin_api/version_api.md

@@ -16,6 +16,6 @@ It returns a JSON body like the following:
 ```json
 {
     "server_version": "0.99.2rc1 (b=develop, abcdef123)",
-    "python_version": "3.7.8"
+    "python_version": "3.6.8"
 }
 ```

docs/development/contributing_guide.md

@@ -20,9 +20,7 @@ recommended for development. More information about WSL can be found at
 <https://docs.microsoft.com/en-us/windows/wsl/install>. Running Synapse natively
 on Windows is not officially supported.
 
-The code of Synapse is written in Python 3. To do pretty much anything, you'll need [a recent version of Python 3](https://www.python.org/downloads/). Your Python also needs support for [virtual environments](https://docs.python.org/3/library/venv.html). This is usually built-in, but some Linux distributions like Debian and Ubuntu split it out into its own package. Running `sudo apt install python3-venv` should be enough.
-
-Synapse can connect to PostgreSQL via the [psycopg2](https://pypi.org/project/psycopg2/) Python library. Building this library from source requires access to PostgreSQL's C header files. On Debian or Ubuntu Linux, these can be installed with `sudo apt install libpq-dev`.
+The code of Synapse is written in Python 3. To do pretty much anything, you'll need [a recent version of Python 3](https://wiki.python.org/moin/BeginnersGuide/Download).
 
 The source code of Synapse is hosted on GitHub. You will also need [a recent version of git](https://github.com/git-guides/install-git).
 
@@ -55,7 +53,6 @@ setup a *virtualenv*, as follows:
 cd path/where/you/have/cloned/the/repository
 python3 -m venv ./env
 source ./env/bin/activate
-pip install wheel
 pip install -e ".[all,dev]"
 pip install tox
 ```
@@ -117,7 +114,7 @@ The linters look at your code and do two things:
 - ensure that your code follows the coding style adopted by the project;
 - catch a number of errors in your code.
 
-The linter takes no time at all to run as soon as you've [downloaded the dependencies into your python virtual environment](#4-install-the-dependencies).
+They're pretty fast, don't hesitate!
 
 ```sh
 source ./env/bin/activate
@@ -172,27 +169,6 @@ To increase the log level for the tests, set `SYNAPSE_TEST_LOG_LEVEL`:
 SYNAPSE_TEST_LOG_LEVEL=DEBUG trial tests
 ```
 
-By default, tests will use an in-memory SQLite database for test data. For additional
-help with debugging, one can use an on-disk SQLite database file instead, in order to
-review database state during and after running tests. This can be done by setting
-the `SYNAPSE_TEST_PERSIST_SQLITE_DB` environment variable. Doing so will cause the
-database state to be stored in a file named `test.db` under the trial process'
-working directory. Typically, this ends up being `_trial_temp/test.db`. For example:
-
-```sh
-SYNAPSE_TEST_PERSIST_SQLITE_DB=1 trial tests
-```
-
-The database file can then be inspected with:
-
-```sh
-sqlite3 _trial_temp/test.db
-```
-
-Note that the database file is cleared at the beginning of each test run. Thus it 
-will always only contain the data generated by the *last run test*. Though generally
-when debugging, one is only running a single test anyway.
-
 ### Running tests under PostgreSQL
 
 Invoking `trial` as above will use an in-memory SQLite database. This is great for

docs/development/database_schema.md

@@ -96,60 +96,6 @@ Ensure postgres is installed, then run:
 NB at the time of writing, this script predates the split into separate `state`/`main`
 databases so will require updates to handle that correctly.
 
-## Delta files
-
-Delta files define the steps required to upgrade the database from an earlier version.
-They can be written as either a file containing a series of SQL statements, or a Python
-module.
-
-Synapse remembers which delta files it has applied to a database (they are stored in the
-`applied_schema_deltas` table) and will not re-apply them (even if a given file is
-subsequently updated).
-
-Delta files should be placed in a directory named `synapse/storage/schema/<database>/delta/<version>/`.
-They are applied in alphanumeric order, so  by convention the first two characters
-of the filename should be an integer such as `01`, to put the file in the right order.
-
-### SQL delta files
-
-These should be named `*.sql`, or —  for changes which should only be applied for a
-given database engine — `*.sql.posgres` or `*.sql.sqlite`. For example, a delta which
-adds a new column to the `foo` table might be called `01add_bar_to_foo.sql`.
-
-Note that our SQL parser is a bit simple - it understands comments (`--` and `/*...*/`),
-but complex statements which require a `;` in the middle of them (such as `CREATE
-TRIGGER`) are beyond it and you'll have to use a Python delta file.
-
-### Python delta files
-
-For more flexibility, a delta file can take the form of a python module. These should
-be named `*.py`. Note that database-engine-specific modules are not supported here –
-instead you can write `if isinstance(database_engine, PostgresEngine)` or similar.
-
-A Python delta module should define either or both of the following functions:
-
-```python
-import synapse.config.homeserver
-import synapse.storage.engines
-import synapse.storage.types
-
-
-def run_create(
-    cur: synapse.storage.types.Cursor,
-    database_engine: synapse.storage.engines.BaseDatabaseEngine,
-) -> None:
-    """Called whenever an existing or new database is to be upgraded"""
-    ...
-
-def run_upgrade(
-    cur: synapse.storage.types.Cursor,
-    database_engine: synapse.storage.engines.BaseDatabaseEngine,
-    config: synapse.config.homeserver.HomeServerConfig,
-) -> None:
-    """Called whenever an existing database is to be upgraded."""
-    ...
-```
-
 ## Boolean columns
 
 Boolean columns require special treatment, since SQLite treats booleans the

docs/development/url_previews.md

@@ -35,12 +35,7 @@ When Synapse is asked to preview a URL it does the following:
    5. If the media is HTML:
       1. Decodes the HTML via the stored file.
       2. Generates an Open Graph response from the HTML.
-      3. If a JSON oEmbed URL was found in the HTML via autodiscovery:
-         1. Downloads the URL and stores it into a file via the media storage provider
-            and saves the local media metadata.
-         2. Convert the oEmbed response to an Open Graph response.
-         3. Override any Open Graph data from the HTML with data from oEmbed.
-      4. If an image exists in the Open Graph response:
+      3. If an image exists in the Open Graph response:
          1. Downloads the URL and stores it into a file via the media storage
             provider and saves the local media metadata.
          2. Generates thumbnails.

docs/modules/password_auth_provider_callbacks.md

@@ -105,68 +105,6 @@ device ID), and the (now deactivated) access token.
 
 If multiple modules implement this callback, Synapse runs them all in order.
 
-### `get_username_for_registration`
-
-_First introduced in Synapse v1.52.0_
-
-```python
-async def get_username_for_registration(
-    uia_results: Dict[str, Any],
-    params: Dict[str, Any],
-) -> Optional[str]
-```
-
-Called when registering a new user. The module can return a username to set for the user
-being registered by returning it as a string, or `None` if it doesn't wish to force a
-username for this user. If a username is returned, it will be used as the local part of a
-user's full Matrix ID (e.g. it's `alice` in `@alice:example.com`).
-
-This callback is called once [User-Interactive Authentication](https://spec.matrix.org/latest/client-server-api/#user-interactive-authentication-api)
-has been completed by the user. It is not called when registering a user via SSO. It is
-passed two dictionaries, which include the information that the user has provided during
-the registration process.
-
-The first dictionary contains the results of the [User-Interactive Authentication](https://spec.matrix.org/latest/client-server-api/#user-interactive-authentication-api)
-flow followed by the user. Its keys are the identifiers of every step involved in the flow,
-associated with either a boolean value indicating whether the step was correctly completed,
-or additional information (e.g. email address, phone number...). A list of most existing
-identifiers can be found in the [Matrix specification](https://spec.matrix.org/v1.1/client-server-api/#authentication-types).
-Here's an example featuring all currently supported keys:
-
-```python
-{
-    "m.login.dummy": True,  # Dummy authentication
-    "m.login.terms": True,  # User has accepted the terms of service for the homeserver
-    "m.login.recaptcha": True,  # User has completed the recaptcha challenge
-    "m.login.email.identity": {  # User has provided and verified an email address
-        "medium": "email",
-        "address": "alice@example.com",
-        "validated_at": 1642701357084,
-    },
-    "m.login.msisdn": {  # User has provided and verified a phone number
-        "medium": "msisdn",
-        "address": "33123456789",
-        "validated_at": 1642701357084,
-    },
-    "org.matrix.msc3231.login.registration_token": "sometoken",  # User has registered through the flow described in MSC3231
-}
-```
-
-The second dictionary contains the parameters provided by the user's client in the request
-to `/_matrix/client/v3/register`. See the [Matrix specification](https://spec.matrix.org/latest/client-server-api/#post_matrixclientv3register)
-for a complete list of these parameters.
-
-If the module cannot, or does not wish to, generate a username for this user, it must
-return `None`.
-
-If multiple modules implement this callback, they will be considered in order. If a
-callback returns `None`, Synapse falls through to the next one. The value of the first
-callback that does not return `None` will be used. If this happens, Synapse will not call
-any of the subsequent implementations of this callback. If every callback return `None`,
-the username provided by the user is used, if any (otherwise one is automatically
-generated).
-
-
 ## Example
 
 The example module below implements authentication checkers for two different login types: 

docs/openid.md

@@ -390,6 +390,9 @@ oidc_providers:
 
 ### Facebook
 
+Like Github, Facebook provide a custom OAuth2 API rather than an OIDC-compliant
+one so requires a little more configuration.
+
 0. You will need a Facebook developer account. You can register for one
    [here](https://developers.facebook.com/async/registration/).
 1. On the [apps](https://developers.facebook.com/apps/) page of the developer
@@ -409,28 +412,24 @@ Synapse config:
     idp_name: Facebook
     idp_brand: "facebook"  # optional: styling hint for clients
     discover: false
-    issuer: "https://www.facebook.com"
+    issuer: "https://facebook.com"
     client_id: "your-client-id" # TO BE FILLED
     client_secret: "your-client-secret" # TO BE FILLED
     scopes: ["openid", "email"]
-    authorization_endpoint: "https://facebook.com/dialog/oauth"
-    token_endpoint: "https://graph.facebook.com/v9.0/oauth/access_token"
-    jwks_uri: "https://www.facebook.com/.well-known/oauth/openid/jwks/"
+    authorization_endpoint: https://facebook.com/dialog/oauth
+    token_endpoint: https://graph.facebook.com/v9.0/oauth/access_token
+    user_profile_method: "userinfo_endpoint"
+    userinfo_endpoint: "https://graph.facebook.com/v9.0/me?fields=id,name,email,picture"
     user_mapping_provider:
       config:
+        subject_claim: "id"
         display_name_template: "{{ user.name }}"
-        email_template: "{{ '{{ user.email }}' }}"
 ```
 
 Relevant documents:
- * [Manually Build a Login Flow](https://developers.facebook.com/docs/facebook-login/manually-build-a-login-flow)
- * [Using Facebook's Graph API](https://developers.facebook.com/docs/graph-api/using-graph-api/)
- * [Reference to the User endpoint](https://developers.facebook.com/docs/graph-api/reference/user)
-
-Facebook do have an [OIDC discovery endpoint](https://www.facebook.com/.well-known/openid-configuration),
-but it has a `response_types_supported` which excludes "code" (which we rely on, and
-is even mentioned in their [documentation](https://developers.facebook.com/docs/facebook-login/manually-build-a-login-flow#login)),
-so we have to disable discovery and configure the URIs manually.
+ * https://developers.facebook.com/docs/facebook-login/manually-build-a-login-flow
+ * Using Facebook's Graph API: https://developers.facebook.com/docs/graph-api/using-graph-api/
+ * Reference to the User endpoint: https://developers.facebook.com/docs/graph-api/reference/user
 
 ### Gitea
 

docs/sample_config.yaml

@@ -41,11 +41,11 @@
 # documentation on how to configure or create custom modules for Synapse.
 #
 modules:
-  #- module: my_super_module.MySuperClass
-  #  config:
-  #    do_thing: true
-  #- module: my_other_super_module.SomeClass
-  #  config: {}
+    # - module: my_super_module.MySuperClass
+    #   config:
+    #       do_thing: true
+    # - module: my_other_super_module.SomeClass
+    #   config: {}
 
 
 ## Server ##
@@ -74,7 +74,13 @@ server_name: "SERVERNAME"
 #
 pid_file: DATADIR/homeserver.pid
 
-# The absolute URL to the web client which / will redirect to.
+# The absolute URL to the web client which /_matrix/client will redirect
+# to if 'webclient' is configured under the 'listeners' configuration.
+#
+# This option can be also set to the filesystem path to the web client
+# which will be served at /_matrix/client/ if 'webclient' is configured
+# under the 'listeners' configuration, however this is a security risk:
+# https://github.com/matrix-org/synapse#security-note
 #
 #web_client_location: https://riot.example.com/
 
@@ -158,7 +164,7 @@ presence:
 # The default room version for newly created rooms.
 #
 # Known room versions are listed here:
-# https://spec.matrix.org/latest/rooms/#complete-list-of-room-versions
+# https://matrix.org/docs/spec/#complete-list-of-room-versions
 #
 # For example, for room version 1, default_room_version should be set
 # to "1".
@@ -304,6 +310,8 @@ presence:
 #   static: static resources under synapse/static (/_matrix/static). (Mostly
 #       useful for 'fallback authentication'.)
 #
+#   webclient: A web client. Requires web_client_location to be set.
+#
 listeners:
   # TLS-enabled listener: for when matrix traffic is sent directly to synapse.
   #
@@ -1428,16 +1436,6 @@ account_threepid_delegates:
 #
 #auto_join_rooms_for_guests: false
 
-# Whether to inhibit errors raised when registering a new account if the user ID
-# already exists. If turned on, that requests to /register/available will always
-# show a user ID as available, and Synapse won't raise an error when starting
-# a registration with a user ID that already exists. However, Synapse will still
-# raise an error if the registration completes and the username conflicts.
-#
-# Defaults to false.
-#
-#inhibit_user_in_use_error: true
-
 
 ## Metrics ###
 
@@ -1505,21 +1503,6 @@ room_prejoin_state:
    #additional_event_types:
    #  - org.example.custom.event.type
 
-# We record the IP address of clients used to access the API for various
-# reasons, including displaying it to the user in the "Where you're signed in"
-# dialog.
-#
-# By default, when puppeting another user via the admin API, the client IP
-# address is recorded against the user who created the access token (ie, the
-# admin user), and *not* the puppeted user.
-#
-# Uncomment the following to also record the IP address against the puppeted
-# user. (This also means that the puppeted user will count as an "active" user
-# for the purpose of monthly active user tracking - see 'limit_usage_by_mau' etc
-# above.)
-#
-#track_puppeted_user_ips: true
-
 
 # A list of application service config files to use
 #
@@ -1887,13 +1870,10 @@ saml2_config:
 #       Defaults to false. Avoid this in production.
 #
 #   user_profile_method: Whether to fetch the user profile from the userinfo
-#       endpoint, or to rely on the data returned in the id_token from the
-#       token_endpoint.
+#       endpoint. Valid values are: 'auto' or 'userinfo_endpoint'.
 #
-#       Valid values are: 'auto' or 'userinfo_endpoint'.
-#
-#       Defaults to 'auto', which uses the userinfo endpoint if 'openid' is
-#       not included in 'scopes'. Set to 'userinfo_endpoint' to always use the
+#       Defaults to 'auto', which fetches the userinfo endpoint if 'openid' is
+#       included in 'scopes'. Set to 'userinfo_endpoint' to always fetch the
 #       userinfo endpoint.
 #
 #   allow_existing_users: set to 'true' to allow a user logging in via OIDC to

docs/setup/installation.md

@@ -194,7 +194,7 @@ When following this route please make sure that the [Platform-specific prerequis
 System requirements:
 
 - POSIX-compliant system (tested on Linux & OS X)
-- Python 3.7 or later, up to Python 3.10.
+- Python 3.6 or later, up to Python 3.9.
 - At least 1GB of free RAM if you want to join large public rooms like #matrix:matrix.org
 
 To install the Synapse homeserver run:

docs/turn-howto.md

@@ -137,10 +137,6 @@ This will install and start a systemd service called `coturn`.
 
     # TLS private key file
     pkey=/path/to/privkey.pem
-
-    # Ensure the configuration lines that disable TLS/DTLS are commented-out or removed
-    #no-tls
-    #no-dtls
     ```
 
     In this case, replace the `turn:` schemes in the `turn_uris` settings below
@@ -149,14 +145,6 @@ This will install and start a systemd service called `coturn`.
     We recommend that you only try to set up TLS/DTLS once you have set up a
     basic installation and got it working.
 
-    NB: If your TLS certificate was provided by Let's Encrypt, TLS/DTLS will
-    not work with any Matrix client that uses Chromium's WebRTC library. This
-    currently includes Element Android & iOS; for more details, see their
-    [respective](https://github.com/vector-im/element-android/issues/1533)
-    [issues](https://github.com/vector-im/element-ios/issues/2712) as well as the underlying
-    [WebRTC issue](https://bugs.chromium.org/p/webrtc/issues/detail?id=11710).
-    Consider using a ZeroSSL certificate for your TURN server as a working alternative.
-
 1.  Ensure your firewall allows traffic into the TURN server on the ports
     you've configured it to listen on (By default: 3478 and 5349 for TURN
     traffic (remember to allow both TCP and UDP traffic), and ports 49152-65535
@@ -262,10 +250,6 @@ Here are a few things to try:
  * Check that you have opened your firewall to allow UDP traffic to the UDP
    relay ports (49152-65535 by default).
 
- * Try disabling `coturn`'s TLS/DTLS listeners and enable only its (unencrypted)
-   TCP/UDP listeners. (This will only leave signaling traffic unencrypted;
-   voice & video WebRTC traffic is always encrypted.)
-
  * Some WebRTC implementations (notably, that of Google Chrome) appear to get
    confused by TURN servers which are reachable over IPv6 (this appears to be
    an unexpected side-effect of its handling of multiple IP addresses as

docs/upgrade.md

@@ -85,17 +85,6 @@ process, for example:
     dpkg -i matrix-synapse-py3_1.3.0+stretch1_amd64.deb
     ```
 
-# Upgrading to v1.51.0
-
-## Deprecation of `webclient` listeners and non-HTTP(S) `web_client_location`
-
-Listeners of type  `webclient` are deprecated and scheduled to be removed in
-Synapse v1.53.0.
-
-Similarly, a non-HTTP(S) `web_client_location` configuration is deprecated and
-will become a configuration error in Synapse v1.53.0.
-
-
 # Upgrading to v1.50.0
 
 ## Dropping support for old Python and Postgres versions

docs/usage/administration/admin_api/federation.md

@@ -86,7 +86,7 @@ The following fields are returned in the JSON response body:
 - `next_token`: string representing a positive integer - Indication for pagination. See above.
 - `total` - integer - Total number of destinations.
 
-## Destination Details API
+# Destination Details API
 
 This API gets the retry timing info for a specific remote server.
 
@@ -108,105 +108,7 @@ A response body like the following is returned:
 }
 ```
 
-**Parameters**
-
-The following parameters should be set in the URL:
-
-- `destination` - Name of the remote server.
-
 **Response**
 
 The response fields are the same like in the `destinations` array in
 [List of destinations](#list-of-destinations) response.
-
-## Destination rooms
-
-This API gets the rooms that federate with a specific remote server.
-
-The API is:
-
-```
-GET /_synapse/admin/v1/federation/destinations/<destination>/rooms
-```
-
-A response body like the following is returned:
-
-```json
-{
-   "rooms":[
-      {
-         "room_id": "!OGEhHVWSdvArJzumhm:matrix.org",
-         "stream_ordering": 8326
-      },
-      {
-         "room_id": "!xYvNcQPhnkrdUmYczI:matrix.org",
-         "stream_ordering": 93534
-      }
-   ],
-   "total": 2
-}
-```
-
-To paginate, check for `next_token` and if present, call the endpoint again
-with `from` set to the value of `next_token`. This will return a new page.
-
-If the endpoint does not return a `next_token` then there are no more destinations
-to paginate through.
-
-**Parameters**
-
-The following parameters should be set in the URL:
-
-- `destination` - Name of the remote server.
-
-The following query parameters are available:
-
-- `from` - Offset in the returned list. Defaults to `0`.
-- `limit` - Maximum amount of destinations to return. Defaults to `100`.
-- `dir` - Direction of room order by `room_id`. Either `f` for forwards or `b` for
-  backwards. Defaults to `f`.
-
-**Response**
-
-The following fields are returned in the JSON response body:
-
-- `rooms` - An array of objects, each containing information about a room.
-  Room objects contain the following fields:
-  - `room_id` - string - The ID of the room.
-  - `stream_ordering` - integer -  The stream ordering of the most recent
-    successfully-sent [PDU](understanding_synapse_through_grafana_graphs.md#federation)
-    to this destination in this room.
-- `next_token`: string representing a positive integer - Indication for pagination. See above.
-- `total` - integer - Total number of destinations.
-
-## Reset connection timeout
-
-Synapse makes federation requests to other homeservers. If a federation request fails,
-Synapse will mark the destination homeserver as offline, preventing any future requests
-to that server for a "cooldown" period. This period grows over time if the server
-continues to fail its responses
-([exponential backoff](https://en.wikipedia.org/wiki/Exponential_backoff)).
-
-Admins can cancel the cooldown period with this API.
-
-This API resets the retry timing for a specific remote server and tries to connect to
-the remote server again. It does not wait for the next `retry_interval`.
-The connection must have previously run into an error and `retry_last_ts`
-([Destination Details API](#destination-details-api)) must not be equal to `0`.
-
-The connection attempt is carried out in the background and can take a while
-even if the API already returns the http status 200.
-
-The API is:
-
-```
-POST /_synapse/admin/v1/federation/destinations/<destination>/reset_connection
-
-{}
-```
-
-**Parameters**
-
-The following parameters should be set in the URL:
-
-- `destination` - Name of the remote server.

scripts-dev/complement.sh

@@ -8,8 +8,7 @@
 # By default the script will fetch the latest Complement master branch and
 # run tests with that. This can be overridden to use a custom Complement
 # checkout by setting the COMPLEMENT_DIR environment variable to the
-# filepath of a local Complement checkout or by setting the COMPLEMENT_REF
-# environment variable to pull a different branch or commit.
+# filepath of a local Complement checkout.
 #
 # By default Synapse is run in monolith mode. This can be overridden by
 # setting the WORKERS environment variable.
@@ -24,20 +23,16 @@
 # Exit if a line returns a non-zero exit code
 set -e
 
-# enable buildkit for the docker builds
-export DOCKER_BUILDKIT=1
-
 # Change to the repository root
 cd "$(dirname $0)/.."
 
 # Check for a user-specified Complement checkout
 if [[ -z "$COMPLEMENT_DIR" ]]; then
-  COMPLEMENT_REF=${COMPLEMENT_REF:-master}
-  echo "COMPLEMENT_DIR not set. Fetching Complement checkout from ${COMPLEMENT_REF}..."
-  wget -Nq https://github.com/matrix-org/complement/archive/${COMPLEMENT_REF}.tar.gz
-  tar -xzf ${COMPLEMENT_REF}.tar.gz
-  COMPLEMENT_DIR=complement-${COMPLEMENT_REF}
-  echo "Checkout available at 'complement-${COMPLEMENT_REF}'"
+  echo "COMPLEMENT_DIR not set. Fetching the latest Complement checkout..."
+  wget -Nq https://github.com/matrix-org/complement/archive/master.tar.gz
+  tar -xzf master.tar.gz
+  COMPLEMENT_DIR=complement-master
+  echo "Checkout available at 'complement-master'"
 fi
 
 # Build the base Synapse image from the local checkout
@@ -52,7 +47,7 @@ if [[ -n "$WORKERS" ]]; then
   COMPLEMENT_DOCKERFILE=SynapseWorkers.Dockerfile
   # And provide some more configuration to complement.
   export COMPLEMENT_CA=true
-  export COMPLEMENT_SPAWN_HS_TIMEOUT_SECS=25
+  export COMPLEMENT_VERSION_CHECK_ITERATIONS=500
 else
   export COMPLEMENT_BASE_IMAGE=complement-synapse
   COMPLEMENT_DOCKERFILE=Synapse.Dockerfile
@@ -70,5 +65,4 @@ if [[ -n "$1" ]]; then
 fi
 
 # Run the tests!
-echo "Images built; running complement"
 go test -v -tags synapse_blacklist,msc2403 -count=1 $EXTRA_COMPLEMENT_ARGS ./tests/...

setup.py

@@ -150,7 +150,7 @@ setup(
     zip_safe=False,
     long_description=long_description,
     long_description_content_type="text/x-rst",
-    python_requires="~=3.7",
+    python_requires="~=3.6",
     entry_points={
         "console_scripts": [
             "synapse_homeserver = synapse.app.homeserver:main",

synapse/__init__.py

@@ -21,8 +21,8 @@ import os
 import sys
 
 # Check that we're not running on an unsupported Python version.
-if sys.version_info < (3, 7):
-    print("Synapse requires Python 3.7 or above.")
+if sys.version_info < (3, 6):
+    print("Synapse requires Python 3.6 or above.")
     sys.exit(1)
 
 # Twisted and canonicaljson will fail to import when this file is executed to
@@ -47,7 +47,7 @@ try:
 except ImportError:
     pass
 
-__version__ = "1.51.0"
+__version__ = "1.50.0"
 
 if bool(os.environ.get("SYNAPSE_TEST_PATCH_LOG_CONTEXTS", False)):
     # We import here so that we don't have to install a bunch of deps when

synapse/_scripts/review_recent_signups.py

@@ -46,9 +46,7 @@ class UserInfo:
     ips: List[str] = attr.Factory(list)
 
 
-def get_recent_users(
-    txn: LoggingTransaction, since_ms: int, exclude_app_service: bool
-) -> List[UserInfo]:
+def get_recent_users(txn: LoggingTransaction, since_ms: int) -> List[UserInfo]:
     """Fetches recently registered users and some info on them."""
 
     sql = """
@@ -58,9 +56,6 @@ def get_recent_users(
             AND deactivated = 0
     """
 
-    if exclude_app_service:
-        sql += " AND appservice_id IS NULL"
-
     txn.execute(sql, (since_ms / 1000,))
 
     user_infos = [UserInfo(user_id, creation_ts) for user_id, creation_ts in txn]
@@ -118,7 +113,7 @@ def main() -> None:
         "-e",
         "--exclude-emails",
         action="store_true",
-        help="Exclude users that have validated email addresses.",
+        help="Exclude users that have validated email addresses",
     )
     parser.add_argument(
         "-u",
@@ -126,12 +121,6 @@ def main() -> None:
         action="store_true",
         help="Only print user IDs that match.",
     )
-    parser.add_argument(
-        "-a",
-        "--exclude-app-service",
-        help="Exclude appservice users.",
-        action="store_true",
-    )
 
     config = ReviewConfig()
 
@@ -144,7 +133,6 @@ def main() -> None:
 
     since_ms = time.time() * 1000 - Config.parse_duration(config_args.since)
     exclude_users_with_email = config_args.exclude_emails
-    exclude_users_with_appservice = config_args.exclude_app_service
     include_context = not config_args.only_users
 
     for database_config in config.database.databases:
@@ -155,7 +143,7 @@ def main() -> None:
 
     with make_conn(database_config, engine, "review_recent_signups") as db_conn:
         # This generates a type of Cursor, not LoggingTransaction.
-        user_infos = get_recent_users(db_conn.cursor(), since_ms, exclude_users_with_appservice)  # type: ignore[arg-type]
+        user_infos = get_recent_users(db_conn.cursor(), since_ms)  # type: ignore[arg-type]
 
     for user_info in user_infos:
         if exclude_users_with_email and user_info.emails:

synapse/api/auth.py

@@ -71,7 +71,6 @@ class Auth:
         self._auth_blocking = AuthBlocking(self.hs)
 
         self._track_appservice_user_ips = hs.config.appservice.track_appservice_user_ips
-        self._track_puppeted_user_ips = hs.config.api.track_puppeted_user_ips
         self._macaroon_secret_key = hs.config.key.macaroon_secret_key
         self._force_tracing_for_users = hs.config.tracing.force_tracing_for_users
 
@@ -247,18 +246,6 @@ class Auth:
                     user_agent=user_agent,
                     device_id=device_id,
                 )
-                # Track also the puppeted user client IP if enabled and the user is puppeting
-                if (
-                    user_info.user_id != user_info.token_owner
-                    and self._track_puppeted_user_ips
-                ):
-                    await self.store.insert_client_ip(
-                        user_id=user_info.user_id,
-                        access_token=access_token,
-                        ip=ip_addr,
-                        user_agent=user_agent,
-                        device_id=device_id,
-                    )
 
             if is_guest and not allow_guest:
                 raise AuthError(

synapse/api/room_versions.py

@@ -46,41 +46,41 @@ class RoomDisposition:
     UNSTABLE = "unstable"
 
 
-@attr.s(slots=True, frozen=True, auto_attribs=True)
+@attr.s(slots=True, frozen=True)
 class RoomVersion:
     """An object which describes the unique attributes of a room version."""
 
-    identifier: str  # the identifier for this version
-    disposition: str  # one of the RoomDispositions
-    event_format: int  # one of the EventFormatVersions
-    state_res: int  # one of the StateResolutionVersions
-    enforce_key_validity: bool
+    identifier = attr.ib(type=str)  # the identifier for this version
+    disposition = attr.ib(type=str)  # one of the RoomDispositions
+    event_format = attr.ib(type=int)  # one of the EventFormatVersions
+    state_res = attr.ib(type=int)  # one of the StateResolutionVersions
+    enforce_key_validity = attr.ib(type=bool)
 
     # Before MSC2432, m.room.aliases had special auth rules and redaction rules
-    special_case_aliases_auth: bool
+    special_case_aliases_auth = attr.ib(type=bool)
     # Strictly enforce canonicaljson, do not allow:
     # * Integers outside the range of [-2 ^ 53 + 1, 2 ^ 53 - 1]
     # * Floats
     # * NaN, Infinity, -Infinity
-    strict_canonicaljson: bool
+    strict_canonicaljson = attr.ib(type=bool)
     # MSC2209: Check 'notifications' key while verifying
     # m.room.power_levels auth rules.
-    limit_notifications_power_levels: bool
+    limit_notifications_power_levels = attr.ib(type=bool)
     # MSC2174/MSC2176: Apply updated redaction rules algorithm.
-    msc2176_redaction_rules: bool
+    msc2176_redaction_rules = attr.ib(type=bool)
     # MSC3083: Support the 'restricted' join_rule.
-    msc3083_join_rules: bool
+    msc3083_join_rules = attr.ib(type=bool)
     # MSC3375: Support for the proper redaction rules for MSC3083. This mustn't
     #          be enabled if MSC3083 is not.
-    msc3375_redaction_rules: bool
+    msc3375_redaction_rules = attr.ib(type=bool)
     # MSC2403: Allows join_rules to be set to 'knock', changes auth rules to allow sending
     # m.room.membership event with membership 'knock'.
-    msc2403_knocking: bool
+    msc2403_knocking = attr.ib(type=bool)
     # MSC2716: Adds m.room.power_levels -> content.historical field to control
     # whether "insertion", "chunk", "marker" events can be sent
-    msc2716_historical: bool
+    msc2716_historical = attr.ib(type=bool)
     # MSC2716: Adds support for redacting "insertion", "chunk", and "marker" events
-    msc2716_redactions: bool
+    msc2716_redactions = attr.ib(type=bool)
 
 
 class RoomVersions:

synapse/app/_base.py

@@ -16,6 +16,7 @@ import atexit
 import gc
 import logging
 import os
+import platform
 import signal
 import socket
 import sys
@@ -59,7 +60,7 @@ from synapse.events.spamcheck import load_legacy_spam_checkers
 from synapse.events.third_party_rules import load_legacy_third_party_event_rules
 from synapse.handlers.auth import load_legacy_password_auth_providers
 from synapse.logging.context import PreserveLoggingContext
-from synapse.metrics import install_gc_manager, register_threadpool
+from synapse.metrics import register_threadpool
 from synapse.metrics.background_process_metrics import wrap_as_background_process
 from synapse.metrics.jemalloc import setup_jemalloc_stats
 from synapse.types import ISynapseReactor
@@ -158,7 +159,6 @@ def start_reactor(
         change_resource_limit(soft_file_limit)
         if gc_thresholds:
             gc.set_threshold(*gc_thresholds)
-        install_gc_manager()
         run_command()
 
     # make sure that we run the reactor with the sentinel log context,
@@ -435,8 +435,7 @@ async def start(hs: "HomeServer") -> None:
     # before we start the listeners.
     module_api = hs.get_module_api()
     for module, config in hs.config.modules.loaded_modules:
-        m = module(config=config, api=module_api)
-        logger.info("Loaded module %s", m)
+        module(config=config, api=module_api)
 
     load_legacy_spam_checkers(hs)
     load_legacy_third_party_event_rules(hs)
@@ -468,13 +467,15 @@ async def start(hs: "HomeServer") -> None:
     # everything currently allocated are things that will be used for the
     # rest of time. Doing so means less work each GC (hopefully).
     #
-    # PyPy does not (yet?) implement gc.freeze()
-    if hasattr(gc, "freeze"):
+    # This only works on Python 3.7
+    if platform.python_implementation() == "CPython" and sys.version_info >= (3, 7):
         gc.collect()
         gc.freeze()
 
-        # Speed up shutdowns by freezing all allocated objects. This moves everything
-        # into the permanent generation and excludes them from the final GC.
+    # Speed up shutdowns by freezing all allocated objects. This moves everything
+    # into the permanent generation and excludes them from the final GC.
+    # Unfortunately only works on Python 3.7
+    if platform.python_implementation() == "CPython" and sys.version_info >= (3, 7):
         atexit.register(gc.freeze)
 
 

synapse/app/homeserver.py

@@ -27,6 +27,7 @@ import synapse
 import synapse.config.logger
 from synapse import events
 from synapse.api.urls import (
+    CLIENT_API_PREFIX,
     FEDERATION_PREFIX,
     LEGACY_MEDIA_PREFIX,
     MEDIA_R0_PREFIX,
@@ -131,18 +132,9 @@ class SynapseHomeServer(HomeServer):
         resources.update(self._module_web_resources)
         self._module_web_resources_consumed = True
 
-        # Try to find something useful to serve at '/':
-        #
-        # 1. Redirect to the web client if it is an HTTP(S) URL.
-        # 2. Redirect to the web client served via Synapse.
-        # 3. Redirect to the static "Synapse is running" page.
-        # 4. Do not redirect and use a blank resource.
-        if self.config.server.web_client_location_is_redirect:
-            root_resource: Resource = RootOptionsRedirectResource(
-                self.config.server.web_client_location
-            )
-        elif WEB_CLIENT_PREFIX in resources:
-            root_resource = RootOptionsRedirectResource(WEB_CLIENT_PREFIX)
+        # try to find something useful to redirect '/' to
+        if WEB_CLIENT_PREFIX in resources:
+            root_resource: Resource = RootOptionsRedirectResource(WEB_CLIENT_PREFIX)
         elif STATIC_PREFIX in resources:
             root_resource = RootOptionsRedirectResource(STATIC_PREFIX)
         else:
@@ -201,13 +193,7 @@ class SynapseHomeServer(HomeServer):
 
             resources.update(
                 {
-                    "/_matrix/client/api/v1": client_resource,
-                    "/_matrix/client/r0": client_resource,
-                    "/_matrix/client/v1": client_resource,
-                    "/_matrix/client/v3": client_resource,
-                    "/_matrix/client/unstable": client_resource,
-                    "/_matrix/client/v2_alpha": client_resource,
-                    "/_matrix/client/versions": client_resource,
+                    CLIENT_API_PREFIX: client_resource,
                     "/.well-known": well_known_resource(self),
                     "/_synapse/admin": AdminRestResource(self),
                     **build_synapse_client_resource_tree(self),
@@ -271,15 +257,15 @@ class SynapseHomeServer(HomeServer):
             resources[SERVER_KEY_V2_PREFIX] = KeyApiV2Resource(self)
 
         if name == "webclient":
-            # webclient listeners are deprecated as of Synapse v1.51.0, remove it
-            # in > v1.53.0.
             webclient_loc = self.config.server.web_client_location
 
             if webclient_loc is None:
                 logger.warning(
                     "Not enabling webclient resource, as web_client_location is unset."
                 )
-            elif self.config.server.web_client_location_is_redirect:
+            elif webclient_loc.startswith("http://") or webclient_loc.startswith(
+                "https://"
+            ):
                 resources[WEB_CLIENT_PREFIX] = RootRedirect(webclient_loc)
             else:
                 logger.warning(

synapse/config/api.py

@@ -29,7 +29,6 @@ class ApiConfig(Config):
     def read_config(self, config: JsonDict, **kwargs):
         validate_config(_MAIN_SCHEMA, config, ())
         self.room_prejoin_state = list(self._get_prejoin_state_types(config))
-        self.track_puppeted_user_ips = config.get("track_puppeted_user_ips", False)
 
     def generate_config_section(cls, **kwargs) -> str:
         formatted_default_state_types = "\n".join(
@@ -60,21 +59,6 @@ class ApiConfig(Config):
            #
            #additional_event_types:
            #  - org.example.custom.event.type
-
-        # We record the IP address of clients used to access the API for various
-        # reasons, including displaying it to the user in the "Where you're signed in"
-        # dialog.
-        #
-        # By default, when puppeting another user via the admin API, the client IP
-        # address is recorded against the user who created the access token (ie, the
-        # admin user), and *not* the puppeted user.
-        #
-        # Uncomment the following to also record the IP address against the puppeted
-        # user. (This also means that the puppeted user will count as an "active" user
-        # for the purpose of monthly active user tracking - see 'limit_usage_by_mau' etc
-        # above.)
-        #
-        #track_puppeted_user_ips: true
         """ % {
             "formatted_default_state_types": formatted_default_state_types
         }
@@ -154,8 +138,5 @@ _MAIN_SCHEMA = {
     "properties": {
         "room_prejoin_state": _ROOM_PREJOIN_STATE_CONFIG_SCHEMA,
         "room_invite_state_types": _ROOM_INVITE_STATE_TYPES_SCHEMA,
-        "track_puppeted_user_ips": {
-            "type": "boolean",
-        },
     },
 }

synapse/config/emailconfig.py

@@ -55,19 +55,19 @@ https://matrix-org.github.io/synapse/latest/templates.html
 ---------------------------------------------------------------------------------------"""
 
 
-@attr.s(slots=True, frozen=True, auto_attribs=True)
+@attr.s(slots=True, frozen=True)
 class EmailSubjectConfig:
-    message_from_person_in_room: str
-    message_from_person: str
-    messages_from_person: str
-    messages_in_room: str
-    messages_in_room_and_others: str
-    messages_from_person_and_others: str
-    invite_from_person: str
-    invite_from_person_to_room: str
-    invite_from_person_to_space: str
-    password_reset: str
-    email_validation: str
+    message_from_person_in_room = attr.ib(type=str)
+    message_from_person = attr.ib(type=str)
+    messages_from_person = attr.ib(type=str)
+    messages_in_room = attr.ib(type=str)
+    messages_in_room_and_others = attr.ib(type=str)
+    messages_from_person_and_others = attr.ib(type=str)
+    invite_from_person = attr.ib(type=str)
+    invite_from_person_to_room = attr.ib(type=str)
+    invite_from_person_to_space = attr.ib(type=str)
+    password_reset = attr.ib(type=str)
+    email_validation = attr.ib(type=str)
 
 
 class EmailConfig(Config):

synapse/config/modules.py

@@ -41,9 +41,9 @@ class ModulesConfig(Config):
             # documentation on how to configure or create custom modules for Synapse.
             #
             modules:
-              #- module: my_super_module.MySuperClass
-              #  config:
-              #    do_thing: true
-              #- module: my_other_super_module.SomeClass
-              #  config: {}
+                # - module: my_super_module.MySuperClass
+                #   config:
+                #       do_thing: true
+                # - module: my_other_super_module.SomeClass
+                #   config: {}
             """

synapse/config/oidc.py

@@ -148,13 +148,10 @@ class OIDCConfig(Config):
         #       Defaults to false. Avoid this in production.
         #
         #   user_profile_method: Whether to fetch the user profile from the userinfo
-        #       endpoint, or to rely on the data returned in the id_token from the
-        #       token_endpoint.
+        #       endpoint. Valid values are: 'auto' or 'userinfo_endpoint'.
         #
-        #       Valid values are: 'auto' or 'userinfo_endpoint'.
-        #
-        #       Defaults to 'auto', which uses the userinfo endpoint if 'openid' is
-        #       not included in 'scopes'. Set to 'userinfo_endpoint' to always use the
+        #       Defaults to 'auto', which fetches the userinfo endpoint if 'openid' is
+        #       included in 'scopes'. Set to 'userinfo_endpoint' to always fetch the
         #       userinfo endpoint.
         #
         #   allow_existing_users: set to 'true' to allow a user logging in via OIDC to

synapse/config/registration.py

@@ -190,8 +190,6 @@ class RegistrationConfig(Config):
         # The success template used during fallback auth.
         self.fallback_success_template = self.read_template("auth_success.html")
 
-        self.inhibit_user_in_use_error = config.get("inhibit_user_in_use_error", False)
-
     def generate_config_section(self, generate_secrets=False, **kwargs):
         if generate_secrets:
             registration_shared_secret = 'registration_shared_secret: "%s"' % (
@@ -448,16 +446,6 @@ class RegistrationConfig(Config):
         # Defaults to true.
         #
         #auto_join_rooms_for_guests: false
-
-        # Whether to inhibit errors raised when registering a new account if the user ID
-        # already exists. If turned on, that requests to /register/available will always
-        # show a user ID as available, and Synapse won't raise an error when starting
-        # a registration with a user ID that already exists. However, Synapse will still
-        # raise an error if the registration completes and the username conflicts.
-        #
-        # Defaults to false.
-        #
-        #inhibit_user_in_use_error: true
         """
             % locals()
         )

synapse/config/server.py

@@ -200,8 +200,8 @@ class HttpListenerConfig:
     """Object describing the http-specific parts of the config of a listener"""
 
     x_forwarded: bool = False
-    resources: List[HttpResourceConfig] = attr.Factory(list)
-    additional_resources: Dict[str, dict] = attr.Factory(dict)
+    resources: List[HttpResourceConfig] = attr.ib(factory=list)
+    additional_resources: Dict[str, dict] = attr.ib(factory=dict)
     tag: Optional[str] = None
 
 
@@ -259,6 +259,7 @@ class ServerConfig(Config):
             raise ConfigError(str(e))
 
         self.pid_file = self.abspath(config.get("pid_file"))
+        self.web_client_location = config.get("web_client_location", None)
         self.soft_file_limit = config.get("soft_file_limit", 0)
         self.daemonize = config.get("daemonize")
         self.print_pidfile = config.get("print_pidfile")
@@ -505,17 +506,8 @@ class ServerConfig(Config):
                     l2.append(listener)
             self.listeners = l2
 
-        self.web_client_location = config.get("web_client_location", None)
-        self.web_client_location_is_redirect = self.web_client_location and (
-            self.web_client_location.startswith("http://")
-            or self.web_client_location.startswith("https://")
-        )
-        # A non-HTTP(S) web client location is deprecated.
-        if self.web_client_location and not self.web_client_location_is_redirect:
-            logger.warning(NO_MORE_NONE_HTTP_WEB_CLIENT_LOCATION_WARNING)
-
-        # Warn if webclient is configured for a worker.
-        _warn_if_webclient_configured(self.listeners)
+        if not self.web_client_location:
+            _warn_if_webclient_configured(self.listeners)
 
         self.gc_thresholds = read_gc_thresholds(config.get("gc_thresholds", None))
         self.gc_seconds = self.read_gc_intervals(config.get("gc_min_interval", None))
@@ -801,7 +793,13 @@ class ServerConfig(Config):
         #
         pid_file: %(pid_file)s
 
-        # The absolute URL to the web client which / will redirect to.
+        # The absolute URL to the web client which /_matrix/client will redirect
+        # to if 'webclient' is configured under the 'listeners' configuration.
+        #
+        # This option can be also set to the filesystem path to the web client
+        # which will be served at /_matrix/client/ if 'webclient' is configured
+        # under the 'listeners' configuration, however this is a security risk:
+        # https://github.com/matrix-org/synapse#security-note
         #
         #web_client_location: https://riot.example.com/
 
@@ -885,7 +883,7 @@ class ServerConfig(Config):
         # The default room version for newly created rooms.
         #
         # Known room versions are listed here:
-        # https://spec.matrix.org/latest/rooms/#complete-list-of-room-versions
+        # https://matrix.org/docs/spec/#complete-list-of-room-versions
         #
         # For example, for room version 1, default_room_version should be set
         # to "1".
@@ -1013,6 +1011,8 @@ class ServerConfig(Config):
         #   static: static resources under synapse/static (/_matrix/static). (Mostly
         #       useful for 'fallback authentication'.)
         #
+        #   webclient: A web client. Requires web_client_location to be set.
+        #
         listeners:
           # TLS-enabled listener: for when matrix traffic is sent directly to synapse.
           #
@@ -1349,15 +1349,9 @@ def parse_listener_def(listener: Any) -> ListenerConfig:
     return ListenerConfig(port, bind_addresses, listener_type, tls, http_config)
 
 
-NO_MORE_NONE_HTTP_WEB_CLIENT_LOCATION_WARNING = """
-Synapse no longer supports serving a web client. To remove this warning,
-configure 'web_client_location' with an HTTP(S) URL.
-"""
-
-
 NO_MORE_WEB_CLIENT_WARNING = """
-Synapse no longer includes a web client. To redirect the root resource to a web client, configure
-'web_client_location'. To remove this warning, remove 'webclient' from the 'listeners'
+Synapse no longer includes a web client. To enable a web client, configure
+web_client_location. To remove this warning, remove 'webclient' from the 'listeners'
 configuration.
 """
 

synapse/config/workers.py

@@ -51,12 +51,12 @@ def _instance_to_list_converter(obj: Union[str, List[str]]) -> List[str]:
     return obj
 
 
-@attr.s(auto_attribs=True)
+@attr.s
 class InstanceLocationConfig:
     """The host and port to talk to an instance via HTTP replication."""
 
-    host: str
-    port: int
+    host = attr.ib(type=str)
+    port = attr.ib(type=int)
 
 
 @attr.s
@@ -77,28 +77,34 @@ class WriterLocations:
             can only be a single instance.
     """
 
-    events: List[str] = attr.ib(
+    events = attr.ib(
         default=["master"],
+        type=List[str],
         converter=_instance_to_list_converter,
     )
-    typing: List[str] = attr.ib(
+    typing = attr.ib(
         default=["master"],
+        type=List[str],
         converter=_instance_to_list_converter,
     )
-    to_device: List[str] = attr.ib(
+    to_device = attr.ib(
         default=["master"],
+        type=List[str],
         converter=_instance_to_list_converter,
     )
-    account_data: List[str] = attr.ib(
+    account_data = attr.ib(
         default=["master"],
+        type=List[str],
         converter=_instance_to_list_converter,
     )
-    receipts: List[str] = attr.ib(
+    receipts = attr.ib(
         default=["master"],
+        type=List[str],
         converter=_instance_to_list_converter,
     )
-    presence: List[str] = attr.ib(
+    presence = attr.ib(
         default=["master"],
+        type=List[str],
         converter=_instance_to_list_converter,
     )
 

synapse/crypto/keyring.py

@@ -58,7 +58,7 @@ if TYPE_CHECKING:
 logger = logging.getLogger(__name__)
 
 
-@attr.s(slots=True, frozen=True, cmp=False, auto_attribs=True)
+@attr.s(slots=True, cmp=False)
 class VerifyJsonRequest:
     """
     A request to verify a JSON object.
@@ -78,10 +78,10 @@ class VerifyJsonRequest:
         key_ids: The set of key_ids to that could be used to verify the JSON object
     """
 
-    server_name: str
-    get_json_object: Callable[[], JsonDict]
-    minimum_valid_until_ts: int
-    key_ids: List[str]
+    server_name = attr.ib(type=str)
+    get_json_object = attr.ib(type=Callable[[], JsonDict])
+    minimum_valid_until_ts = attr.ib(type=int)
+    key_ids = attr.ib(type=List[str])
 
     @staticmethod
     def from_json_object(
@@ -124,7 +124,7 @@ class KeyLookupError(ValueError):
     pass
 
 
-@attr.s(slots=True, frozen=True, auto_attribs=True)
+@attr.s(slots=True)
 class _FetchKeyRequest:
     """A request for keys for a given server.
 
@@ -138,9 +138,9 @@ class _FetchKeyRequest:
         key_ids: The IDs of the keys to attempt to fetch
     """
 
-    server_name: str
-    minimum_valid_until_ts: int
-    key_ids: List[str]
+    server_name = attr.ib(type=str)
+    minimum_valid_until_ts = attr.ib(type=int)
+    key_ids = attr.ib(type=List[str])
 
 
 class Keyring:

synapse/events/__init__.py

@@ -315,11 +315,10 @@ class EventBase(metaclass=abc.ABCMeta):
     redacts: DefaultDictProperty[Optional[str]] = DefaultDictProperty("redacts", None)
     room_id: DictProperty[str] = DictProperty("room_id")
     sender: DictProperty[str] = DictProperty("sender")
-    # TODO state_key should be Optional[str]. This is generally asserted in Synapse
-    # by calling is_state() first (which ensures it is not None), but it is hard (not possible?)
+    # TODO state_key should be Optional[str], this is generally asserted in Synapse
+    # by calling is_state() first (which ensures this), but it is hard (not possible?)
     # to properly annotate that calling is_state() asserts that state_key exists
-    # and is non-None. It would be better to replace such direct references with
-    # get_state_key() (and a check for None).
+    # and is non-None.
     state_key: DictProperty[str] = DictProperty("state_key")
     type: DictProperty[str] = DictProperty("type")
     user_id: DictProperty[str] = DictProperty("sender")
@@ -333,11 +332,7 @@ class EventBase(metaclass=abc.ABCMeta):
         return self.content["membership"]
 
     def is_state(self) -> bool:
-        return self.get_state_key() is not None
-
-    def get_state_key(self) -> Optional[str]:
-        """Get the state key of this event, or None if it's not a state event"""
-        return self._dict.get("state_key")
+        return hasattr(self, "state_key") and self.state_key is not None
 
     def get_dict(self) -> JsonDict:
         d = dict(self._dict)

synapse/events/snapshot.py

@@ -28,7 +28,7 @@ if TYPE_CHECKING:
     from synapse.storage.databases.main import DataStore
 
 
-@attr.s(slots=True, auto_attribs=True)
+@attr.s(slots=True)
 class EventContext:
     """
     Holds information relevant to persisting an event
@@ -103,15 +103,15 @@ class EventContext:
             accessed via get_prev_state_ids.
     """
 
-    rejected: Union[bool, str] = False
-    _state_group: Optional[int] = None
-    state_group_before_event: Optional[int] = None
-    prev_group: Optional[int] = None
-    delta_ids: Optional[StateMap[str]] = None
-    app_service: Optional[ApplicationService] = None
+    rejected = attr.ib(default=False, type=Union[bool, str])
+    _state_group = attr.ib(default=None, type=Optional[int])
+    state_group_before_event = attr.ib(default=None, type=Optional[int])
+    prev_group = attr.ib(default=None, type=Optional[int])
+    delta_ids = attr.ib(default=None, type=Optional[StateMap[str]])
+    app_service = attr.ib(default=None, type=Optional[ApplicationService])
 
-    _current_state_ids: Optional[StateMap[str]] = None
-    _prev_state_ids: Optional[StateMap[str]] = None
+    _current_state_ids = attr.ib(default=None, type=Optional[StateMap[str]])
+    _prev_state_ids = attr.ib(default=None, type=Optional[StateMap[str]])
 
     @staticmethod
     def with_state(
@@ -163,7 +163,7 @@ class EventContext:
         return {
             "prev_state_id": prev_state_id,
             "event_type": event.type,
-            "event_state_key": event.get_state_key(),
+            "event_state_key": event.state_key if event.is_state() else None,
             "state_group": self._state_group,
             "state_group_before_event": self.state_group_before_event,
             "rejected": self.rejected,

synapse/events/utils.py

@@ -32,13 +32,13 @@ from synapse.api.constants import EventContentFields, EventTypes, RelationTypes
 from synapse.api.errors import Codes, SynapseError
 from synapse.api.room_versions import RoomVersion
 from synapse.types import JsonDict
+from synapse.util.async_helpers import yieldable_gather_results
 from synapse.util.frozenutils import unfreeze
 
 from . import EventBase
 
 if TYPE_CHECKING:
-    from synapse.storage.databases.main.relations import BundledAggregations
-
+    from synapse.server import HomeServer
 
 # Split strings on "." but not "\." This uses a negative lookbehind assertion for '\'
 # (?<!stuff) matches if the current position in the string is not preceded
@@ -385,12 +385,17 @@ class EventClientSerializer:
     clients.
     """
 
-    def serialize_event(
+    def __init__(self, hs: "HomeServer"):
+        self.store = hs.get_datastore()
+        self._msc1849_enabled = hs.config.experimental.msc1849_enabled
+        self._msc3440_enabled = hs.config.experimental.msc3440_enabled
+
+    async def serialize_event(
         self,
         event: Union[JsonDict, EventBase],
         time_now: int,
         *,
-        bundle_aggregations: Optional[Dict[str, "BundledAggregations"]] = None,
+        bundle_aggregations: bool = False,
         **kwargs: Any,
     ) -> JsonDict:
         """Serializes a single event.
@@ -413,46 +418,66 @@ class EventClientSerializer:
         serialized_event = serialize_event(event, time_now, **kwargs)
 
         # Check if there are any bundled aggregations to include with the event.
-        if bundle_aggregations:
-            event_aggregations = bundle_aggregations.get(event.event_id)
-            if event_aggregations:
-                self._inject_bundled_aggregations(
-                    event,
-                    time_now,
-                    bundle_aggregations[event.event_id],
-                    serialized_event,
-                )
+        #
+        # Do not bundle aggregations if any of the following at true:
+        #
+        # * Support is disabled via the configuration or the caller.
+        # * The event is a state event.
+        # * The event has been redacted.
+        if (
+            self._msc1849_enabled
+            and bundle_aggregations
+            and not event.is_state()
+            and not event.internal_metadata.is_redacted()
+        ):
+            await self._injected_bundled_aggregations(event, time_now, serialized_event)
 
         return serialized_event
 
-    def _inject_bundled_aggregations(
-        self,
-        event: EventBase,
-        time_now: int,
-        aggregations: "BundledAggregations",
-        serialized_event: JsonDict,
+    async def _injected_bundled_aggregations(
+        self, event: EventBase, time_now: int, serialized_event: JsonDict
     ) -> None:
         """Potentially injects bundled aggregations into the unsigned portion of the serialized event.
 
         Args:
             event: The event being serialized.
             time_now: The current time in milliseconds
-            aggregations: The bundled aggregation to serialize.
             serialized_event: The serialized event which may be modified.
 
         """
-        serialized_aggregations = {}
+        # Do not bundle aggregations for an event which represents an edit or an
+        # annotation. It does not make sense for them to have related events.
+        relates_to = event.content.get("m.relates_to")
+        if isinstance(relates_to, (dict, frozendict)):
+            relation_type = relates_to.get("rel_type")
+            if relation_type in (RelationTypes.ANNOTATION, RelationTypes.REPLACE):
+                return
 
-        if aggregations.annotations:
-            serialized_aggregations[RelationTypes.ANNOTATION] = aggregations.annotations
+        event_id = event.event_id
+        room_id = event.room_id
 
-        if aggregations.references:
-            serialized_aggregations[RelationTypes.REFERENCE] = aggregations.references
+        # The bundled aggregations to include.
+        aggregations = {}
 
-        if aggregations.replace:
+        annotations = await self.store.get_aggregation_groups_for_event(
+            event_id, room_id
+        )
+        if annotations.chunk:
+            aggregations[RelationTypes.ANNOTATION] = annotations.to_dict()
+
+        references = await self.store.get_relations_for_event(
+            event_id, room_id, RelationTypes.REFERENCE, direction="f"
+        )
+        if references.chunk:
+            aggregations[RelationTypes.REFERENCE] = references.to_dict()
+
+        edit = None
+        if event.type == EventTypes.Message:
+            edit = await self.store.get_applicable_edit(event_id, room_id)
+
+        if edit:
             # If there is an edit replace the content, preserving existing
             # relations.
-            edit = aggregations.replace
 
             # Ensure we take copies of the edit content, otherwise we risk modifying
             # the original event.
@@ -470,30 +495,34 @@ class EventClientSerializer:
             else:
                 serialized_event["content"].pop("m.relates_to", None)
 
-            serialized_aggregations[RelationTypes.REPLACE] = {
+            aggregations[RelationTypes.REPLACE] = {
                 "event_id": edit.event_id,
                 "origin_server_ts": edit.origin_server_ts,
                 "sender": edit.sender,
             }
 
         # If this event is the start of a thread, include a summary of the replies.
-        if aggregations.thread:
-            serialized_aggregations[RelationTypes.THREAD] = {
-                # Don't bundle aggregations as this could recurse forever.
-                "latest_event": self.serialize_event(
-                    aggregations.thread.latest_event, time_now, bundle_aggregations=None
-                ),
-                "count": aggregations.thread.count,
-                "current_user_participated": aggregations.thread.current_user_participated,
-            }
+        if self._msc3440_enabled:
+            (
+                thread_count,
+                latest_thread_event,
+            ) = await self.store.get_thread_summary(event_id, room_id)
+            if latest_thread_event:
+                aggregations[RelationTypes.THREAD] = {
+                    # Don't bundle aggregations as this could recurse forever.
+                    "latest_event": await self.serialize_event(
+                        latest_thread_event, time_now, bundle_aggregations=False
+                    ),
+                    "count": thread_count,
+                }
 
-        # Include the bundled aggregations in the event.
-        if serialized_aggregations:
+        # If any bundled aggregations were found, include them.
+        if aggregations:
             serialized_event["unsigned"].setdefault("m.relations", {}).update(
-                serialized_aggregations
+                aggregations
             )
 
-    def serialize_events(
+    async def serialize_events(
         self, events: Iterable[Union[JsonDict, EventBase]], time_now: int, **kwargs: Any
     ) -> List[JsonDict]:
         """Serializes multiple events.
@@ -506,9 +535,9 @@ class EventClientSerializer:
         Returns:
             The list of serialized events
         """
-        return [
-            self.serialize_event(event, time_now=time_now, **kwargs) for event in events
-        ]
+        return await yieldable_gather_results(
+            self.serialize_event, events, time_now=time_now, **kwargs
+        )
 
 
 def copy_power_levels_contents(

synapse/events/validator.py

@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 import collections.abc
-from typing import Iterable, Type, Union
+from typing import Iterable, Union
 
 import jsonschema
 
@@ -246,7 +246,7 @@ POWER_LEVELS_SCHEMA = {
 
 # This could return something newer than Draft 7, but that's the current "latest"
 # validator.
-def _create_power_level_validator() -> Type[jsonschema.Draft7Validator]:
+def _create_power_level_validator() -> jsonschema.Draft7Validator:
     validator = jsonschema.validators.validator_for(POWER_LEVELS_SCHEMA)
 
     # by default jsonschema does not consider a frozendict to be an object so

synapse/federation/federation_base.py

@@ -230,10 +230,6 @@ def event_from_pdu_json(pdu_json: JsonDict, room_version: RoomVersion) -> EventB
     # origin, etc etc)
     assert_params_in_dict(pdu_json, ("type", "depth"))
 
-    # Strip any unauthorized values from "unsigned" if they exist
-    if "unsigned" in pdu_json:
-        _strip_unsigned_values(pdu_json)
-
     depth = pdu_json["depth"]
     if not isinstance(depth, int):
         raise SynapseError(400, "Depth %r not an intger" % (depth,), Codes.BAD_JSON)
@@ -249,24 +245,3 @@ def event_from_pdu_json(pdu_json: JsonDict, room_version: RoomVersion) -> EventB
 
     event = make_event_from_dict(pdu_json, room_version)
     return event
-
-
-def _strip_unsigned_values(pdu_dict: JsonDict) -> None:
-    """
-    Strip any unsigned values unless specifically allowed, as defined by the whitelist.
-
-    pdu: the json dict to strip values from. Note that the dict is mutated by this
-    function
-    """
-    unsigned = pdu_dict["unsigned"]
-
-    if not isinstance(unsigned, dict):
-        pdu_dict["unsigned"] = {}
-
-    if pdu_dict["type"] == "m.room.member":
-        whitelist = ["knock_room_state", "invite_room_state", "age"]
-    else:
-        whitelist = ["age"]
-
-    filtered_unsigned = {k: v for k, v in unsigned.items() if k in whitelist}
-    pdu_dict["unsigned"] = filtered_unsigned

synapse/federation/federation_client.py

@@ -56,6 +56,7 @@ from synapse.api.room_versions import (
 from synapse.events import EventBase, builder
 from synapse.federation.federation_base import FederationBase, event_from_pdu_json
 from synapse.federation.transport.client import SendJoinResponse
+from synapse.logging.utils import log_function
 from synapse.types import JsonDict, get_domain_from_id
 from synapse.util.async_helpers import concurrently_execute
 from synapse.util.caches.expiringcache import ExpiringCache
@@ -118,8 +119,7 @@ class FederationClient(FederationBase):
         # It is a map of (room ID, suggested-only) -> the response of
         # get_room_hierarchy.
         self._get_room_hierarchy_cache: ExpiringCache[
-            Tuple[str, bool],
-            Tuple[JsonDict, Sequence[JsonDict], Sequence[JsonDict], Sequence[str]],
+            Tuple[str, bool], Tuple[JsonDict, Sequence[JsonDict], Sequence[str]]
         ] = ExpiringCache(
             cache_name="get_room_hierarchy_cache",
             clock=self._clock,
@@ -144,6 +144,7 @@ class FederationClient(FederationBase):
             if destination_dict:
                 self.pdu_destination_tried[event_id] = destination_dict
 
+    @log_function
     async def make_query(
         self,
         destination: str,
@@ -177,6 +178,7 @@ class FederationClient(FederationBase):
             ignore_backoff=ignore_backoff,
         )
 
+    @log_function
     async def query_client_keys(
         self, destination: str, content: JsonDict, timeout: int
     ) -> JsonDict:
@@ -194,6 +196,7 @@ class FederationClient(FederationBase):
             destination, content, timeout
         )
 
+    @log_function
     async def query_user_devices(
         self, destination: str, user_id: str, timeout: int = 30000
     ) -> JsonDict:
@@ -205,6 +208,7 @@ class FederationClient(FederationBase):
             destination, user_id, timeout
         )
 
+    @log_function
     async def claim_client_keys(
         self, destination: str, content: JsonDict, timeout: int
     ) -> JsonDict:
@@ -1334,7 +1338,7 @@ class FederationClient(FederationBase):
         destinations: Iterable[str],
         room_id: str,
         suggested_only: bool,
-    ) -> Tuple[JsonDict, Sequence[JsonDict], Sequence[JsonDict], Sequence[str]]:
+    ) -> Tuple[JsonDict, Sequence[JsonDict], Sequence[str]]:
         """
         Call other servers to get a hierarchy of the given room.
 
@@ -1349,8 +1353,7 @@ class FederationClient(FederationBase):
 
         Returns:
             A tuple of:
-                The room as a JSON dictionary, without a "children_state" key.
-                A list of `m.space.child` state events.
+                The room as a JSON dictionary.
                 A list of children rooms, as JSON dictionaries.
                 A list of inaccessible children room IDs.
 
@@ -1365,7 +1368,7 @@ class FederationClient(FederationBase):
 
         async def send_request(
             destination: str,
-        ) -> Tuple[JsonDict, Sequence[JsonDict], Sequence[JsonDict], Sequence[str]]:
+        ) -> Tuple[JsonDict, Sequence[JsonDict], Sequence[str]]:
             try:
                 res = await self.transport_layer.get_room_hierarchy(
                     destination=destination,
@@ -1394,7 +1397,7 @@ class FederationClient(FederationBase):
                 raise InvalidResponseError("'room' must be a dict")
 
             # Validate children_state of the room.
-            children_state = room.pop("children_state", [])
+            children_state = room.get("children_state", [])
             if not isinstance(children_state, Sequence):
                 raise InvalidResponseError("'room.children_state' must be a list")
             if any(not isinstance(e, dict) for e in children_state):
@@ -1423,7 +1426,7 @@ class FederationClient(FederationBase):
                     "Invalid room ID in 'inaccessible_children' list"
                 )
 
-            return room, children_state, children, inaccessible_children
+            return room, children, inaccessible_children
 
         try:
             result = await self._try_destination_list(
@@ -1471,6 +1474,8 @@ class FederationClient(FederationBase):
                 if event.room_id == room_id:
                     children_events.append(event.data)
                     children_room_ids.add(event.state_key)
+            # And add them under the requested room.
+            requested_room["children_state"] = children_events
 
             # Find the children rooms.
             children = []
@@ -1480,7 +1485,7 @@ class FederationClient(FederationBase):
 
             # It isn't clear from the response whether some of the rooms are
             # not accessible.
-            result = (requested_room, children_events, children, ())
+            result = (requested_room, children, ())
 
         # Cache the result to avoid fetching data over federation every time.
         self._get_room_hierarchy_cache[(room_id, suggested_only)] = result

synapse/federation/federation_server.py

@@ -58,6 +58,7 @@ from synapse.logging.context import (
     run_in_background,
 )
 from synapse.logging.opentracing import log_kv, start_active_span_from_edu, trace
+from synapse.logging.utils import log_function
 from synapse.metrics.background_process_metrics import wrap_as_background_process
 from synapse.replication.http.federation import (
     ReplicationFederationSendEduRestServlet,
@@ -858,6 +859,7 @@ class FederationServer(FederationBase):
             res = {"auth_chain": [a.get_pdu_json(time_now) for a in auth_pdus]}
         return 200, res
 
+    @log_function
     async def on_query_client_keys(
         self, origin: str, content: Dict[str, str]
     ) -> Tuple[int, Dict[str, Any]]:
@@ -938,6 +940,7 @@ class FederationServer(FederationBase):
 
         return {"events": [ev.get_pdu_json(time_now) for ev in missing_events]}
 
+    @log_function
     async def on_openid_userinfo(self, token: str) -> Optional[str]:
         ts_now_ms = self._clock.time_msec()
         return await self.store.get_user_id_for_open_id_token(token, ts_now_ms)

synapse/federation/persistence.py

@@ -23,6 +23,7 @@ import logging
 from typing import Optional, Tuple
 
 from synapse.federation.units import Transaction
+from synapse.logging.utils import log_function
 from synapse.storage.databases.main import DataStore
 from synapse.types import JsonDict
 
@@ -35,6 +36,7 @@ class TransactionActions:
     def __init__(self, datastore: DataStore):
         self.store = datastore
 
+    @log_function
     async def have_responded(
         self, origin: str, transaction: Transaction
     ) -> Optional[Tuple[int, JsonDict]]:
@@ -51,6 +53,7 @@ class TransactionActions:
 
         return await self.store.get_received_txn_response(transaction_id, origin)
 
+    @log_function
     async def set_response(
         self, origin: str, transaction: Transaction, code: int, response: JsonDict
     ) -> None:

synapse/federation/sender/per_destination_queue.py

@@ -607,18 +607,18 @@ class PerDestinationQueue:
         self._pending_pdus = []
 
 
-@attr.s(slots=True, auto_attribs=True)
+@attr.s(slots=True)
 class _TransactionQueueManager:
     """A helper async context manager for pulling stuff off the queues and
     tracking what was last successfully sent, etc.
     """
 
-    queue: PerDestinationQueue
+    queue = attr.ib(type=PerDestinationQueue)
 
-    _device_stream_id: Optional[int] = None
-    _device_list_id: Optional[int] = None
-    _last_stream_ordering: Optional[int] = None
-    _pdus: List[EventBase] = attr.Factory(list)
+    _device_stream_id = attr.ib(type=Optional[int], default=None)
+    _device_list_id = attr.ib(type=Optional[int], default=None)
+    _last_stream_ordering = attr.ib(type=Optional[int], default=None)
+    _pdus = attr.ib(type=List[EventBase], factory=list)
 
     async def __aenter__(self) -> Tuple[List[EventBase], List[Edu]]:
         # First we calculate the EDUs we want to send, if any.

synapse/federation/sender/transaction_manager.py

@@ -35,7 +35,6 @@ if TYPE_CHECKING:
     import synapse.server
 
 logger = logging.getLogger(__name__)
-issue_8631_logger = logging.getLogger("synapse.8631_debug")
 
 last_pdu_ts_metric = Gauge(
     "synapse_federation_last_sent_pdu_time",
@@ -125,17 +124,6 @@ class TransactionManager:
                 len(pdus),
                 len(edus),
             )
-            if issue_8631_logger.isEnabledFor(logging.DEBUG):
-                DEVICE_UPDATE_EDUS = {"m.device_list_update", "m.signing_key_update"}
-                device_list_updates = [
-                    edu.content for edu in edus if edu.edu_type in DEVICE_UPDATE_EDUS
-                ]
-                if device_list_updates:
-                    issue_8631_logger.debug(
-                        "about to send txn [%s] including device list updates: %s",
-                        transaction.transaction_id,
-                        device_list_updates,
-                    )
 
             # Actually send the transaction
 

synapse/federation/transport/client.py

@@ -44,6 +44,7 @@ from synapse.api.urls import (
 from synapse.events import EventBase, make_event_from_dict
 from synapse.federation.units import Transaction
 from synapse.http.matrixfederationclient import ByteParser
+from synapse.logging.utils import log_function
 from synapse.types import JsonDict
 
 logger = logging.getLogger(__name__)
@@ -61,6 +62,7 @@ class TransportLayerClient:
         self.server_name = hs.hostname
         self.client = hs.get_federation_http_client()
 
+    @log_function
     async def get_room_state_ids(
         self, destination: str, room_id: str, event_id: str
     ) -> JsonDict:
@@ -86,6 +88,7 @@ class TransportLayerClient:
             try_trailing_slash_on_400=True,
         )
 
+    @log_function
     async def get_event(
         self, destination: str, event_id: str, timeout: Optional[int] = None
     ) -> JsonDict:
@@ -108,6 +111,7 @@ class TransportLayerClient:
             destination, path=path, timeout=timeout, try_trailing_slash_on_400=True
         )
 
+    @log_function
     async def backfill(
         self, destination: str, room_id: str, event_tuples: Collection[str], limit: int
     ) -> Optional[JsonDict]:
@@ -145,6 +149,7 @@ class TransportLayerClient:
             destination, path=path, args=args, try_trailing_slash_on_400=True
         )
 
+    @log_function
     async def timestamp_to_event(
         self, destination: str, room_id: str, timestamp: int, direction: str
     ) -> Union[JsonDict, List]:
@@ -180,6 +185,7 @@ class TransportLayerClient:
 
         return remote_response
 
+    @log_function
     async def send_transaction(
         self,
         transaction: Transaction,
@@ -228,6 +234,7 @@ class TransportLayerClient:
             try_trailing_slash_on_400=True,
         )
 
+    @log_function
     async def make_query(
         self,
         destination: str,
@@ -247,6 +254,7 @@ class TransportLayerClient:
             ignore_backoff=ignore_backoff,
         )
 
+    @log_function
     async def make_membership_event(
         self,
         destination: str,
@@ -309,6 +317,7 @@ class TransportLayerClient:
             ignore_backoff=ignore_backoff,
         )
 
+    @log_function
     async def send_join_v1(
         self,
         room_version: RoomVersion,
@@ -327,6 +336,7 @@ class TransportLayerClient:
             max_response_size=MAX_RESPONSE_SIZE_SEND_JOIN,
         )
 
+    @log_function
     async def send_join_v2(
         self,
         room_version: RoomVersion,
@@ -345,6 +355,7 @@ class TransportLayerClient:
             max_response_size=MAX_RESPONSE_SIZE_SEND_JOIN,
         )
 
+    @log_function
     async def send_leave_v1(
         self, destination: str, room_id: str, event_id: str, content: JsonDict
     ) -> Tuple[int, JsonDict]:
@@ -361,6 +372,7 @@ class TransportLayerClient:
             ignore_backoff=True,
         )
 
+    @log_function
     async def send_leave_v2(
         self, destination: str, room_id: str, event_id: str, content: JsonDict
     ) -> JsonDict:
@@ -377,6 +389,7 @@ class TransportLayerClient:
             ignore_backoff=True,
         )
 
+    @log_function
     async def send_knock_v1(
         self,
         destination: str,
@@ -410,6 +423,7 @@ class TransportLayerClient:
             destination=destination, path=path, data=content
         )
 
+    @log_function
     async def send_invite_v1(
         self, destination: str, room_id: str, event_id: str, content: JsonDict
     ) -> Tuple[int, JsonDict]:
@@ -419,6 +433,7 @@ class TransportLayerClient:
             destination=destination, path=path, data=content, ignore_backoff=True
         )
 
+    @log_function
     async def send_invite_v2(
         self, destination: str, room_id: str, event_id: str, content: JsonDict
     ) -> JsonDict:
@@ -428,6 +443,7 @@ class TransportLayerClient:
             destination=destination, path=path, data=content, ignore_backoff=True
         )
 
+    @log_function
     async def get_public_rooms(
         self,
         remote_server: str,
@@ -500,6 +516,7 @@ class TransportLayerClient:
 
         return response
 
+    @log_function
     async def exchange_third_party_invite(
         self, destination: str, room_id: str, event_dict: JsonDict
     ) -> JsonDict:
@@ -509,6 +526,7 @@ class TransportLayerClient:
             destination=destination, path=path, data=event_dict
         )
 
+    @log_function
     async def get_event_auth(
         self, destination: str, room_id: str, event_id: str
     ) -> JsonDict:
@@ -516,6 +534,7 @@ class TransportLayerClient:
 
         return await self.client.get_json(destination=destination, path=path)
 
+    @log_function
     async def query_client_keys(
         self, destination: str, query_content: JsonDict, timeout: int
     ) -> JsonDict:
@@ -557,6 +576,7 @@ class TransportLayerClient:
             destination=destination, path=path, data=query_content, timeout=timeout
         )
 
+    @log_function
     async def query_user_devices(
         self, destination: str, user_id: str, timeout: int
     ) -> JsonDict:
@@ -596,6 +616,7 @@ class TransportLayerClient:
             destination=destination, path=path, timeout=timeout
         )
 
+    @log_function
     async def claim_client_keys(
         self, destination: str, query_content: JsonDict, timeout: int
     ) -> JsonDict:
@@ -634,6 +655,7 @@ class TransportLayerClient:
             destination=destination, path=path, data=query_content, timeout=timeout
         )
 
+    @log_function
     async def get_missing_events(
         self,
         destination: str,
@@ -658,6 +680,7 @@ class TransportLayerClient:
             timeout=timeout,
         )
 
+    @log_function
     async def get_group_profile(
         self, destination: str, group_id: str, requester_user_id: str
     ) -> JsonDict:
@@ -671,6 +694,7 @@ class TransportLayerClient:
             ignore_backoff=True,
         )
 
+    @log_function
     async def update_group_profile(
         self, destination: str, group_id: str, requester_user_id: str, content: JsonDict
     ) -> JsonDict:
@@ -692,6 +716,7 @@ class TransportLayerClient:
             ignore_backoff=True,
         )
 
+    @log_function
     async def get_group_summary(
         self, destination: str, group_id: str, requester_user_id: str
     ) -> JsonDict:
@@ -705,6 +730,7 @@ class TransportLayerClient:
             ignore_backoff=True,
         )
 
+    @log_function
     async def get_rooms_in_group(
         self, destination: str, group_id: str, requester_user_id: str
     ) -> JsonDict:
@@ -772,6 +798,7 @@ class TransportLayerClient:
             ignore_backoff=True,
         )
 
+    @log_function
     async def get_users_in_group(
         self, destination: str, group_id: str, requester_user_id: str
     ) -> JsonDict:
@@ -785,6 +812,7 @@ class TransportLayerClient:
             ignore_backoff=True,
         )
 
+    @log_function
     async def get_invited_users_in_group(
         self, destination: str, group_id: str, requester_user_id: str
     ) -> JsonDict:
@@ -798,6 +826,7 @@ class TransportLayerClient:
             ignore_backoff=True,
         )
 
+    @log_function
     async def accept_group_invite(
         self, destination: str, group_id: str, user_id: str, content: JsonDict
     ) -> JsonDict:
@@ -808,6 +837,7 @@ class TransportLayerClient:
             destination=destination, path=path, data=content, ignore_backoff=True
         )
 
+    @log_function
     def join_group(
         self, destination: str, group_id: str, user_id: str, content: JsonDict
     ) -> Awaitable[JsonDict]:
@@ -818,6 +848,7 @@ class TransportLayerClient:
             destination=destination, path=path, data=content, ignore_backoff=True
         )
 
+    @log_function
     async def invite_to_group(
         self,
         destination: str,
@@ -837,6 +868,7 @@ class TransportLayerClient:
             ignore_backoff=True,
         )
 
+    @log_function
     async def invite_to_group_notification(
         self, destination: str, group_id: str, user_id: str, content: JsonDict
     ) -> JsonDict:
@@ -850,6 +882,7 @@ class TransportLayerClient:
             destination=destination, path=path, data=content, ignore_backoff=True
         )
 
+    @log_function
     async def remove_user_from_group(
         self,
         destination: str,
@@ -869,6 +902,7 @@ class TransportLayerClient:
             ignore_backoff=True,
         )
 
+    @log_function
     async def remove_user_from_group_notification(
         self, destination: str, group_id: str, user_id: str, content: JsonDict
     ) -> JsonDict:
@@ -882,6 +916,7 @@ class TransportLayerClient:
             destination=destination, path=path, data=content, ignore_backoff=True
         )
 
+    @log_function
     async def renew_group_attestation(
         self, destination: str, group_id: str, user_id: str, content: JsonDict
     ) -> JsonDict:
@@ -895,6 +930,7 @@ class TransportLayerClient:
             destination=destination, path=path, data=content, ignore_backoff=True
         )
 
+    @log_function
     async def update_group_summary_room(
         self,
         destination: str,
@@ -923,6 +959,7 @@ class TransportLayerClient:
             ignore_backoff=True,
         )
 
+    @log_function
     async def delete_group_summary_room(
         self,
         destination: str,
@@ -949,6 +986,7 @@ class TransportLayerClient:
             ignore_backoff=True,
         )
 
+    @log_function
     async def get_group_categories(
         self, destination: str, group_id: str, requester_user_id: str
     ) -> JsonDict:
@@ -962,6 +1000,7 @@ class TransportLayerClient:
             ignore_backoff=True,
         )
 
+    @log_function
     async def get_group_category(
         self, destination: str, group_id: str, requester_user_id: str, category_id: str
     ) -> JsonDict:
@@ -975,6 +1014,7 @@ class TransportLayerClient:
             ignore_backoff=True,
         )
 
+    @log_function
     async def update_group_category(
         self,
         destination: str,
@@ -994,6 +1034,7 @@ class TransportLayerClient:
             ignore_backoff=True,
         )
 
+    @log_function
     async def delete_group_category(
         self, destination: str, group_id: str, requester_user_id: str, category_id: str
     ) -> JsonDict:
@@ -1007,6 +1048,7 @@ class TransportLayerClient:
             ignore_backoff=True,
         )
 
+    @log_function
     async def get_group_roles(
         self, destination: str, group_id: str, requester_user_id: str
     ) -> JsonDict:
@@ -1020,6 +1062,7 @@ class TransportLayerClient:
             ignore_backoff=True,
         )
 
+    @log_function
     async def get_group_role(
         self, destination: str, group_id: str, requester_user_id: str, role_id: str
     ) -> JsonDict:
@@ -1033,6 +1076,7 @@ class TransportLayerClient:
             ignore_backoff=True,
         )
 
+    @log_function
     async def update_group_role(
         self,
         destination: str,
@@ -1052,6 +1096,7 @@ class TransportLayerClient:
             ignore_backoff=True,
         )
 
+    @log_function
     async def delete_group_role(
         self, destination: str, group_id: str, requester_user_id: str, role_id: str
     ) -> JsonDict:
@@ -1065,6 +1110,7 @@ class TransportLayerClient:
             ignore_backoff=True,
         )
 
+    @log_function
     async def update_group_summary_user(
         self,
         destination: str,
@@ -1090,6 +1136,7 @@ class TransportLayerClient:
             ignore_backoff=True,
         )
 
+    @log_function
     async def set_group_join_policy(
         self, destination: str, group_id: str, requester_user_id: str, content: JsonDict
     ) -> JsonDict:
@@ -1104,6 +1151,7 @@ class TransportLayerClient:
             ignore_backoff=True,
         )
 
+    @log_function
     async def delete_group_summary_user(
         self,
         destination: str,

synapse/federation/transport/server/__init__.py

@@ -13,7 +13,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 import logging
-from typing import TYPE_CHECKING, Dict, Iterable, List, Optional, Tuple, Type
+from typing import Dict, Iterable, List, Optional, Tuple, Type
 
 from typing_extensions import Literal
 
@@ -36,19 +36,17 @@ from synapse.http.servlet import (
     parse_integer_from_args,
     parse_string_from_args,
 )
+from synapse.server import HomeServer
 from synapse.types import JsonDict, ThirdPartyInstanceID
 from synapse.util.ratelimitutils import FederationRateLimiter
 
-if TYPE_CHECKING:
-    from synapse.server import HomeServer
-
 logger = logging.getLogger(__name__)
 
 
 class TransportLayerServer(JsonResource):
     """Handles incoming federation HTTP requests"""
 
-    def __init__(self, hs: "HomeServer", servlet_groups: Optional[List[str]] = None):
+    def __init__(self, hs: HomeServer, servlet_groups: Optional[List[str]] = None):
         """Initialize the TransportLayerServer
 
         Will by default register all servlets. For custom behaviour, pass in
@@ -115,7 +113,7 @@ class PublicRoomList(BaseFederationServlet):
 
     def __init__(
         self,
-        hs: "HomeServer",
+        hs: HomeServer,
         authenticator: Authenticator,
         ratelimiter: FederationRateLimiter,
         server_name: str,
@@ -205,7 +203,7 @@ class FederationGroupsRenewAttestaionServlet(BaseFederationServlet):
 
     def __init__(
         self,
-        hs: "HomeServer",
+        hs: HomeServer,
         authenticator: Authenticator,
         ratelimiter: FederationRateLimiter,
         server_name: str,
@@ -253,7 +251,7 @@ class OpenIdUserInfo(BaseFederationServlet):
 
     def __init__(
         self,
-        hs: "HomeServer",
+        hs: HomeServer,
         authenticator: Authenticator,
         ratelimiter: FederationRateLimiter,
         server_name: str,
@@ -299,7 +297,7 @@ DEFAULT_SERVLET_GROUPS: Dict[str, Iterable[Type[BaseFederationServlet]]] = {
 
 
 def register_servlets(
-    hs: "HomeServer",
+    hs: HomeServer,
     resource: HttpServer,
     authenticator: Authenticator,
     ratelimiter: FederationRateLimiter,

synapse/federation/transport/server/_base.py

@@ -15,7 +15,7 @@
 import functools
 import logging
 import re
-from typing import TYPE_CHECKING, Any, Awaitable, Callable, Optional, Tuple, cast
+from typing import Any, Awaitable, Callable, Optional, Tuple, cast
 
 from synapse.api.errors import Codes, FederationDeniedError, SynapseError
 from synapse.api.urls import FEDERATION_V1_PREFIX
@@ -29,13 +29,11 @@ from synapse.logging.opentracing import (
     start_active_span_follows_from,
     whitelisted_homeserver,
 )
+from synapse.server import HomeServer
 from synapse.types import JsonDict
 from synapse.util.ratelimitutils import FederationRateLimiter
 from synapse.util.stringutils import parse_and_validate_server_name
 
-if TYPE_CHECKING:
-    from synapse.server import HomeServer
-
 logger = logging.getLogger(__name__)
 
 
@@ -48,7 +46,7 @@ class NoAuthenticationError(AuthenticationError):
 
 
 class Authenticator:
-    def __init__(self, hs: "HomeServer"):
+    def __init__(self, hs: HomeServer):
         self._clock = hs.get_clock()
         self.keyring = hs.get_keyring()
         self.server_name = hs.hostname
@@ -116,11 +114,11 @@ class Authenticator:
         # alive
         retry_timings = await self.store.get_destination_retry_timings(origin)
         if retry_timings and retry_timings.retry_last_ts:
-            run_in_background(self.reset_retry_timings, origin)
+            run_in_background(self._reset_retry_timings, origin)
 
         return origin
 
-    async def reset_retry_timings(self, origin: str) -> None:
+    async def _reset_retry_timings(self, origin: str) -> None:
         try:
             logger.info("Marking origin %r as up", origin)
             await self.store.set_destination_retry_timings(origin, None, 0, 0)
@@ -229,7 +227,7 @@ class BaseFederationServlet:
 
     def __init__(
         self,
-        hs: "HomeServer",
+        hs: HomeServer,
         authenticator: Authenticator,
         ratelimiter: FederationRateLimiter,
         server_name: str,

synapse/federation/transport/server/federation.py

@@ -12,17 +12,7 @@
 #  See the License for the specific language governing permissions and
 #  limitations under the License.
 import logging
-from typing import (
-    TYPE_CHECKING,
-    Dict,
-    List,
-    Mapping,
-    Optional,
-    Sequence,
-    Tuple,
-    Type,
-    Union,
-)
+from typing import Dict, List, Mapping, Optional, Sequence, Tuple, Type, Union
 
 from typing_extensions import Literal
 
@@ -40,15 +30,12 @@ from synapse.http.servlet import (
     parse_string_from_args,
     parse_strings_from_args,
 )
+from synapse.server import HomeServer
 from synapse.types import JsonDict
 from synapse.util.ratelimitutils import FederationRateLimiter
 from synapse.util.versionstring import get_version_string
 
-if TYPE_CHECKING:
-    from synapse.server import HomeServer
-
 logger = logging.getLogger(__name__)
-issue_8631_logger = logging.getLogger("synapse.8631_debug")
 
 
 class BaseFederationServerServlet(BaseFederationServlet):
@@ -59,7 +46,7 @@ class BaseFederationServerServlet(BaseFederationServlet):
 
     def __init__(
         self,
-        hs: "HomeServer",
+        hs: HomeServer,
         authenticator: Authenticator,
         ratelimiter: FederationRateLimiter,
         server_name: str,
@@ -108,20 +95,6 @@ class FederationSendServlet(BaseFederationServerServlet):
                 len(transaction_data.get("edus", [])),
             )
 
-            if issue_8631_logger.isEnabledFor(logging.DEBUG):
-                DEVICE_UPDATE_EDUS = {"m.device_list_update", "m.signing_key_update"}
-                device_list_updates = [
-                    edu.content
-                    for edu in transaction_data.get("edus", [])
-                    if edu.edu_type in DEVICE_UPDATE_EDUS
-                ]
-                if device_list_updates:
-                    issue_8631_logger.debug(
-                        "received transaction [%s] including device list updates: %s",
-                        transaction_id,
-                        device_list_updates,
-                    )
-
         except Exception as e:
             logger.exception(e)
             return 400, {"error": "Invalid transaction"}
@@ -608,7 +581,7 @@ class FederationSpaceSummaryServlet(BaseFederationServlet):
 
     def __init__(
         self,
-        hs: "HomeServer",
+        hs: HomeServer,
         authenticator: Authenticator,
         ratelimiter: FederationRateLimiter,
         server_name: str,
@@ -682,7 +655,7 @@ class FederationRoomHierarchyServlet(BaseFederationServlet):
 
     def __init__(
         self,
-        hs: "HomeServer",
+        hs: HomeServer,
         authenticator: Authenticator,
         ratelimiter: FederationRateLimiter,
         server_name: str,
@@ -718,7 +691,7 @@ class RoomComplexityServlet(BaseFederationServlet):
 
     def __init__(
         self,
-        hs: "HomeServer",
+        hs: HomeServer,
         authenticator: Authenticator,
         ratelimiter: FederationRateLimiter,
         server_name: str,

synapse/federation/transport/server/groups_local.py

@@ -11,7 +11,7 @@
 #  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 #  See the License for the specific language governing permissions and
 #  limitations under the License.
-from typing import TYPE_CHECKING, Dict, List, Tuple, Type
+from typing import Dict, List, Tuple, Type
 
 from synapse.api.errors import SynapseError
 from synapse.federation.transport.server._base import (
@@ -19,12 +19,10 @@ from synapse.federation.transport.server._base import (
     BaseFederationServlet,
 )
 from synapse.handlers.groups_local import GroupsLocalHandler
+from synapse.server import HomeServer
 from synapse.types import JsonDict, get_domain_from_id
 from synapse.util.ratelimitutils import FederationRateLimiter
 
-if TYPE_CHECKING:
-    from synapse.server import HomeServer
-
 
 class BaseGroupsLocalServlet(BaseFederationServlet):
     """Abstract base class for federation servlet classes which provides a groups local handler.
@@ -34,7 +32,7 @@ class BaseGroupsLocalServlet(BaseFederationServlet):
 
     def __init__(
         self,
-        hs: "HomeServer",
+        hs: HomeServer,
         authenticator: Authenticator,
         ratelimiter: FederationRateLimiter,
         server_name: str,

synapse/federation/transport/server/groups_server.py

@@ -11,7 +11,7 @@
 #  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 #  See the License for the specific language governing permissions and
 #  limitations under the License.
-from typing import TYPE_CHECKING, Dict, List, Tuple, Type
+from typing import Dict, List, Tuple, Type
 
 from typing_extensions import Literal
 
@@ -22,12 +22,10 @@ from synapse.federation.transport.server._base import (
     BaseFederationServlet,
 )
 from synapse.http.servlet import parse_string_from_args
+from synapse.server import HomeServer
 from synapse.types import JsonDict, get_domain_from_id
 from synapse.util.ratelimitutils import FederationRateLimiter
 
-if TYPE_CHECKING:
-    from synapse.server import HomeServer
-
 
 class BaseGroupsServerServlet(BaseFederationServlet):
     """Abstract base class for federation servlet classes which provides a groups server handler.
@@ -37,7 +35,7 @@ class BaseGroupsServerServlet(BaseFederationServlet):
 
     def __init__(
         self,
-        hs: "HomeServer",
+        hs: HomeServer,
         authenticator: Authenticator,
         ratelimiter: FederationRateLimiter,
         server_name: str,

synapse/handlers/account_data.py

@@ -77,7 +77,7 @@ class AccountDataHandler:
     async def add_account_data_for_user(
         self, user_id: str, account_data_type: str, content: JsonDict
     ) -> int:
-        """Add some global account_data for a user.
+        """Add some account_data to a room for a user.
 
         Args:
             user_id: The user to add a tag for.

synapse/handlers/admin.py

@@ -55,47 +55,21 @@ class AdminHandler:
 
     async def get_user(self, user: UserID) -> Optional[JsonDict]:
         """Function to get user details"""
-        user_info_dict = await self.store.get_user_by_id(user.to_string())
-        if user_info_dict is None:
-            return None
-
-        # Restrict returned information to a known set of fields. This prevents additional
-        # fields added to get_user_by_id from modifying Synapse's external API surface.
-        user_info_to_return = {
-            "name",
-            "admin",
-            "deactivated",
-            "shadow_banned",
-            "creation_ts",
-            "appservice_id",
-            "consent_server_notice_sent",
-            "consent_version",
-            "user_type",
-            "is_guest",
-        }
-
-        # Restrict returned keys to a known set.
-        user_info_dict = {
-            key: value
-            for key, value in user_info_dict.items()
-            if key in user_info_to_return
-        }
-
-        # Add additional user metadata
-        profile = await self.store.get_profileinfo(user.localpart)
-        threepids = await self.store.user_get_threepids(user.to_string())
-        external_ids = [
-            ({"auth_provider": auth_provider, "external_id": external_id})
-            for auth_provider, external_id in await self.store.get_external_ids_by_user(
-                user.to_string()
-            )
-        ]
-        user_info_dict["displayname"] = profile.display_name
-        user_info_dict["avatar_url"] = profile.avatar_url
-        user_info_dict["threepids"] = threepids
-        user_info_dict["external_ids"] = external_ids
-
-        return user_info_dict
+        ret = await self.store.get_user_by_id(user.to_string())
+        if ret:
+            profile = await self.store.get_profileinfo(user.localpart)
+            threepids = await self.store.user_get_threepids(user.to_string())
+            external_ids = [
+                ({"auth_provider": auth_provider, "external_id": external_id})
+                for auth_provider, external_id in await self.store.get_external_ids_by_user(
+                    user.to_string()
+                )
+            ]
+            ret["displayname"] = profile.display_name
+            ret["avatar_url"] = profile.avatar_url
+            ret["threepids"] = threepids
+            ret["external_ids"] = external_ids
+        return ret
 
     async def export_user_data(self, user_id: str, writer: "ExfiltrationWriter") -> Any:
         """Write all data we have on the user to the given writer.

synapse/handlers/auth.py

@@ -168,25 +168,25 @@ def login_id_phone_to_thirdparty(identifier: JsonDict) -> Dict[str, str]:
     }
 
 
-@attr.s(slots=True, auto_attribs=True)
+@attr.s(slots=True)
 class SsoLoginExtraAttributes:
     """Data we track about SAML2 sessions"""
 
     # time the session was created, in milliseconds
-    creation_time: int
-    extra_attributes: JsonDict
+    creation_time = attr.ib(type=int)
+    extra_attributes = attr.ib(type=JsonDict)
 
 
-@attr.s(slots=True, frozen=True, auto_attribs=True)
+@attr.s(slots=True, frozen=True)
 class LoginTokenAttributes:
     """Data we store in a short-term login token"""
 
-    user_id: str
+    user_id = attr.ib(type=str)
 
-    auth_provider_id: str
+    auth_provider_id = attr.ib(type=str)
     """The SSO Identity Provider that the user authenticated with, to get this token."""
 
-    auth_provider_session_id: Optional[str]
+    auth_provider_session_id = attr.ib(type=Optional[str])
     """The session ID advertised by the SSO Identity Provider."""
 
 
@@ -2060,10 +2060,6 @@ CHECK_AUTH_CALLBACK = Callable[
         Optional[Tuple[str, Optional[Callable[["LoginResponse"], Awaitable[None]]]]]
     ],
 ]
-GET_USERNAME_FOR_REGISTRATION_CALLBACK = Callable[
-    [JsonDict, JsonDict],
-    Awaitable[Optional[str]],
-]
 
 
 class PasswordAuthProvider:
@@ -2076,9 +2072,6 @@ class PasswordAuthProvider:
         # lists of callbacks
         self.check_3pid_auth_callbacks: List[CHECK_3PID_AUTH_CALLBACK] = []
         self.on_logged_out_callbacks: List[ON_LOGGED_OUT_CALLBACK] = []
-        self.get_username_for_registration_callbacks: List[
-            GET_USERNAME_FOR_REGISTRATION_CALLBACK
-        ] = []
 
         # Mapping from login type to login parameters
         self._supported_login_types: Dict[str, Iterable[str]] = {}
@@ -2093,9 +2086,6 @@ class PasswordAuthProvider:
         auth_checkers: Optional[
             Dict[Tuple[str, Tuple[str, ...]], CHECK_AUTH_CALLBACK]
         ] = None,
-        get_username_for_registration: Optional[
-            GET_USERNAME_FOR_REGISTRATION_CALLBACK
-        ] = None,
     ) -> None:
         # Register check_3pid_auth callback
         if check_3pid_auth is not None:
@@ -2140,11 +2130,6 @@ class PasswordAuthProvider:
                 # Add the new method to the list of auth_checker_callbacks for this login type
                 self.auth_checker_callbacks.setdefault(login_type, []).append(callback)
 
-        if get_username_for_registration is not None:
-            self.get_username_for_registration_callbacks.append(
-                get_username_for_registration,
-            )
-
     def get_supported_login_types(self) -> Mapping[str, Iterable[str]]:
         """Get the login types supported by this password provider
 
@@ -2296,50 +2281,7 @@ class PasswordAuthProvider:
         # call all of the on_logged_out callbacks
         for callback in self.on_logged_out_callbacks:
             try:
-                await callback(user_id, device_id, access_token)
+                callback(user_id, device_id, access_token)
             except Exception as e:
                 logger.warning("Failed to run module API callback %s: %s", callback, e)
                 continue
-
-    async def get_username_for_registration(
-        self,
-        uia_results: JsonDict,
-        params: JsonDict,
-    ) -> Optional[str]:
-        """Defines the username to use when registering the user, using the credentials
-        and parameters provided during the UIA flow.
-
-        Stops at the first callback that returns a string.
-
-        Args:
-            uia_results: The credentials provided during the UIA flow.
-            params: The parameters provided by the registration request.
-
-        Returns:
-            The localpart to use when registering this user, or None if no module
-            returned a localpart.
-        """
-        for callback in self.get_username_for_registration_callbacks:
-            try:
-                res = await callback(uia_results, params)
-
-                if isinstance(res, str):
-                    return res
-                elif res is not None:
-                    # mypy complains that this line is unreachable because it assumes the
-                    # data returned by the module fits the expected type. We just want
-                    # to make sure this is the case.
-                    logger.warning(  # type: ignore[unreachable]
-                        "Ignoring non-string value returned by"
-                        " get_username_for_registration callback %s: %s",
-                        callback,
-                        res,
-                    )
-            except Exception as e:
-                logger.error(
-                    "Module raised an exception in get_username_for_registration: %s",
-                    e,
-                )
-                raise SynapseError(code=500, msg="Internal Server Error")
-
-        return None

synapse/handlers/deactivate_account.py

@@ -157,9 +157,6 @@ class DeactivateAccountHandler:
         # Mark the user as deactivated.
         await self.store.set_user_deactivated_status(user_id, True)
 
-        # Remove account data (including ignored users and push rules).
-        await self.store.purge_account_data_for_user(user_id)
-
         return identity_server_supports_unbinding
 
     async def _reject_pending_invites_for_user(self, user_id: str) -> None:

synapse/handlers/device.py

@@ -948,16 +948,8 @@ class DeviceListUpdater:
             devices = []
             ignore_devices = True
         else:
-            prev_stream_id = await self.store.get_device_list_last_stream_id_for_remote(
-                user_id
-            )
             cached_devices = await self.store.get_cached_devices_for_user(user_id)
-
-            # To ensure that a user with no devices is cached, we skip the resync only
-            # if we have a stream_id from previously writing a cache entry.
-            if prev_stream_id is not None and cached_devices == {
-                d["device_id"]: d for d in devices
-            }:
+            if cached_devices == {d["device_id"]: d for d in devices}:
                 logging.info(
                     "Skipping device list resync for %s, as our cache matches already",
                     user_id,

synapse/handlers/e2e_keys.py

@@ -1321,14 +1321,14 @@ def _one_time_keys_match(old_key_json: str, new_key: JsonDict) -> bool:
     return old_key == new_key_copy
 
 
-@attr.s(slots=True, auto_attribs=True)
+@attr.s(slots=True)
 class SignatureListItem:
     """An item in the signature list as used by upload_signatures_for_device_keys."""
 
-    signing_key_id: str
-    target_user_id: str
-    target_device_id: str
-    signature: JsonDict
+    signing_key_id = attr.ib(type=str)
+    target_user_id = attr.ib(type=str)
+    target_device_id = attr.ib(type=str)
+    signature = attr.ib(type=JsonDict)
 
 
 class SigningKeyEduUpdater:

synapse/handlers/events.py

@@ -20,6 +20,7 @@ from synapse.api.constants import EduTypes, EventTypes, Membership
 from synapse.api.errors import AuthError, SynapseError
 from synapse.events import EventBase
 from synapse.handlers.presence import format_user_presence_state
+from synapse.logging.utils import log_function
 from synapse.streams.config import PaginationConfig
 from synapse.types import JsonDict, UserID
 from synapse.visibility import filter_events_for_client
@@ -42,6 +43,7 @@ class EventStreamHandler:
         self._server_notices_sender = hs.get_server_notices_sender()
         self._event_serializer = hs.get_event_client_serializer()
 
+    @log_function
     async def get_stream(
         self,
         auth_user_id: str,
@@ -117,7 +119,7 @@ class EventStreamHandler:
 
             events.extend(to_add)
 
-            chunks = self._event_serializer.serialize_events(
+            chunks = await self._event_serializer.serialize_events(
                 events,
                 time_now,
                 as_client_event=as_client_event,

synapse/handlers/federation.py

@@ -51,6 +51,7 @@ from synapse.logging.context import (
     preserve_fn,
     run_in_background,
 )
+from synapse.logging.utils import log_function
 from synapse.replication.http.federation import (
     ReplicationCleanRoomRestServlet,
     ReplicationStoreRoomOnOutlierMembershipRestServlet,
@@ -555,6 +556,7 @@ class FederationHandler:
 
             run_in_background(self._handle_queued_pdus, room_queue)
 
+    @log_function
     async def do_knock(
         self,
         target_hosts: List[str],
@@ -926,6 +928,7 @@ class FederationHandler:
 
         return event
 
+    @log_function
     async def on_make_knock_request(
         self, origin: str, room_id: str, user_id: str
     ) -> EventBase:
@@ -1036,6 +1039,7 @@ class FederationHandler:
         else:
             return []
 
+    @log_function
     async def on_backfill_request(
         self, origin: str, room_id: str, pdu_list: List[str], limit: int
     ) -> List[EventBase]:
@@ -1052,6 +1056,7 @@ class FederationHandler:
 
         return events
 
+    @log_function
     async def get_persisted_pdu(
         self, origin: str, event_id: str
     ) -> Optional[EventBase]:
@@ -1113,6 +1118,7 @@ class FederationHandler:
 
         return missing_events
 
+    @log_function
     async def exchange_third_party_invite(
         self, sender_user_id: str, target_user_id: str, room_id: str, signed: JsonDict
     ) -> None: