Use getattr() rather than __getattr__()

A partial revert of #11478. This turns out to have had a significant CPU impact
on initial-sync handling. For now, let's disable it, until we find a more
efficient way of achieving this.

.ci/scripts/test_old_deps.sh

@@ -1,14 +1,12 @@
 #!/usr/bin/env bash
-# this script is run by GitHub Actions in a plain `focal` container; it installs the
-# minimal requirements for tox and hands over to the py3-old tox environment.
 
-# Prevent tzdata from asking for user input
-export DEBIAN_FRONTEND=noninteractive
+# this script is run by GitHub Actions in a plain `bionic` container; it installs the
+# minimal requirements for tox and hands over to the py3-old tox environment.
 
 set -ex
 
 apt-get update
-apt-get install -y python3 python3-dev python3-pip libxml2-dev libxslt-dev xmlsec1 zlib1g-dev tox libjpeg-dev libwebp-dev
+apt-get install -y python3 python3-dev python3-pip libxml2-dev libxslt-dev xmlsec1 zlib1g-dev tox
 
 export LANG="C.UTF-8"
 

.github/PULL_REQUEST_TEMPLATE.md

@@ -8,7 +8,6 @@
   - Use markdown where necessary, mostly for `code blocks`.
   - End with either a period (.) or an exclamation mark (!).
   - Start with a capital letter.
-  - Feel free to credit yourself, by adding a sentence "Contributed by @github_username." or "Contributed by [Your Name]." to the end of the entry.
 * [ ] Pull request includes a [sign off](https://matrix-org.github.io/synapse/latest/development/contributing_guide.html#sign-off)
 * [ ] [Code style](https://matrix-org.github.io/synapse/latest/code_style.html) is correct
   (run the [linters](https://matrix-org.github.io/synapse/latest/development/contributing_guide.html#run-the-linters))

.github/workflows/docker.yml

@@ -34,8 +34,6 @@ jobs:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_TOKEN }}
 
-      # TODO: consider using https://github.com/docker/metadata-action instead of this
-      # custom magic
       - name: Calculate docker image tag
         id: set-tag
         run: |
@@ -55,6 +53,18 @@ jobs:
           esac
           echo "::set-output name=tag::$tag"
 
+        # for release builds, we want to get the amd64 image out asap, so first
+        # we do an amd64-only build, before following up with a multiarch build.
+      - name: Build and push amd64
+        uses: docker/build-push-action@v2
+        if: "${{ startsWith(github.ref, 'refs/tags/v') }}"
+        with:
+          push: true
+          labels: "gitsha1=${{ github.sha }}"
+          tags: "matrixdotorg/synapse:${{ steps.set-tag.outputs.tag }}"
+          file: "docker/Dockerfile"
+          platforms: linux/amd64
+
       - name: Build and push all platforms
         uses: docker/build-push-action@v2
         with:

.github/workflows/tests.yml

@@ -141,7 +141,7 @@ jobs:
     steps:
       - uses: actions/checkout@v2
       - name: Test with old deps
-        uses: docker://ubuntu:focal # For old python and sqlite
+        uses: docker://ubuntu:bionic # For old python and sqlite
         with:
           workdir: /github/workspace
           entrypoint: .ci/scripts/test_old_deps.sh
@@ -213,15 +213,15 @@ jobs:
       fail-fast: false
       matrix:
         include:
-          - sytest-tag: focal
+          - sytest-tag: bionic
 
-          - sytest-tag: focal
+          - sytest-tag: bionic
             postgres: postgres
 
           - sytest-tag: testing
             postgres: postgres
 
-          - sytest-tag: focal
+          - sytest-tag: bionic
             postgres: multi-postgres
             workers: workers
 
@@ -323,22 +323,17 @@ jobs:
     if: ${{ !failure() && !cancelled() }}
     needs: linting-done
     runs-on: ubuntu-latest
+    container:
+      # https://github.com/matrix-org/complement/blob/master/dockerfiles/ComplementCIBuildkite.Dockerfile
+      image: matrixdotorg/complement:latest
+      env:
+        CI: true
+      ports:
+        - 8448:8448
+      volumes:
+        - /var/run/docker.sock:/var/run/docker.sock
 
     steps:
-      # The path is set via a file given by $GITHUB_PATH. We need both Go 1.17 and GOPATH on the path to run Complement.
-      # See https://docs.github.com/en/actions/using-workflows/workflow-commands-for-github-actions#adding-a-system-path
-      - name: "Set Go Version"
-        run: |
-          # Add Go 1.17 to the PATH: see https://github.com/actions/virtual-environments/blob/main/images/linux/Ubuntu2004-Readme.md#environment-variables-2
-          echo "$GOROOT_1_17_X64/bin" >> $GITHUB_PATH
-          # Add the Go path to the PATH: We need this so we can call gotestfmt
-          echo "~/go/bin" >> $GITHUB_PATH
-
-      - name: "Install Complement Dependencies"
-        run: |
-          sudo apt-get update && sudo apt-get install -y libolm3 libolm-dev
-          go get -v github.com/haveyoudebuggedit/gotestfmt/v2/cmd/gotestfmt@latest
-
       - name: Run actions/checkout@v2 for synapse
         uses: actions/checkout@v2
         with:
@@ -371,8 +366,6 @@ jobs:
       # Build initial Synapse image
       - run: docker build -t matrixdotorg/synapse:latest -f docker/Dockerfile .
         working-directory: synapse
-        env:
-          DOCKER_BUILDKIT: 1
 
       # Build a ready-to-run Synapse image based on the initial image above.
       # This new image includes a config file, keys for signing and TLS, and
@@ -381,11 +374,7 @@ jobs:
         working-directory: complement/dockerfiles
 
       # Run Complement
-      - run: |
-          set -o pipefail
-          go test -v -json -tags synapse_blacklist,msc2403 ./tests/... 2>&1 | gotestfmt
-        shell: bash
-        name: Run Complement Tests
+      - run: go test -v -tags synapse_blacklist,msc2403 ./tests/...
         env:
           COMPLEMENT_BASE_IMAGE: complement-synapse:latest
         working-directory: complement

.github/workflows/twisted_trunk.yml

@@ -25,7 +25,7 @@ jobs:
       - run: sudo apt-get -qq install xmlsec1
       - uses: actions/setup-python@v2
         with:
-          python-version: 3.7
+          python-version: 3.6
       - run: .ci/patch_for_twisted_trunk.sh
       - run: pip install tox
       - run: tox -e py

.gitignore

@@ -50,7 +50,3 @@ __pycache__/
 
 # docs
 book/
-
-# complement
-/complement-*
-/master.tar.gz

CHANGES.md

@@ -1,229 +1,8 @@
-Synapse 1.52.0 (2022-02-08)
-===========================
-
-No significant changes since 1.52.0rc1.
-
-Note that [Twisted 22.1.0](https://github.com/twisted/twisted/releases/tag/twisted-22.1.0)
-has recently been released, which fixes a [security issue](https://github.com/twisted/twisted/security/advisories/GHSA-92x2-jw7w-xvvx)
-within the Twisted library. We do not believe Synapse is affected by this vulnerability,
-though we advise server administrators who installed Synapse via pip to upgrade Twisted
-with `pip install --upgrade Twisted` as a matter of good practice. The Docker image
-`matrixdotorg/synapse` and the Debian packages from `packages.matrix.org` are using the
-updated library.
-
-
-Synapse 1.52.0rc1 (2022-02-01)
-==============================
-
-Features
---------
-
-- Remove account data (including client config, push rules and ignored users) upon user deactivation. ([\#11621](https://github.com/matrix-org/synapse/issues/11621), [\#11788](https://github.com/matrix-org/synapse/issues/11788), [\#11789](https://github.com/matrix-org/synapse/issues/11789))
-- Add an admin API to reset connection timeouts for remote server. ([\#11639](https://github.com/matrix-org/synapse/issues/11639))
-- Add an admin API to get a list of rooms that federate with a given remote homeserver. ([\#11658](https://github.com/matrix-org/synapse/issues/11658))
-- Add a config flag to inhibit `M_USER_IN_USE` during registration. ([\#11743](https://github.com/matrix-org/synapse/issues/11743))
-- Add a module callback to set username at registration. ([\#11790](https://github.com/matrix-org/synapse/issues/11790))
-- Allow configuring a maximum file size as well as a list of allowed content types for avatars. ([\#11846](https://github.com/matrix-org/synapse/issues/11846))
-
-
-Bugfixes
---------
-
-- Include the bundled aggregations in the `/sync` response, per [MSC2675](https://github.com/matrix-org/matrix-doc/pull/2675). ([\#11612](https://github.com/matrix-org/synapse/issues/11612))
-- Fix a long-standing bug when previewing Reddit URLs which do not contain an image. ([\#11767](https://github.com/matrix-org/synapse/issues/11767))
-- Fix a long-standing bug that media streams could cause long-lived connections when generating URL previews. ([\#11784](https://github.com/matrix-org/synapse/issues/11784))
-- Include a `prev_content` field in state events sent to Application Services. Contributed by @totallynotvaishnav. ([\#11798](https://github.com/matrix-org/synapse/issues/11798))
-- Fix a bug introduced in Synapse 0.33.3 causing requests to sometimes log strings such as `HTTPStatus.OK` instead of integer status codes. ([\#11827](https://github.com/matrix-org/synapse/issues/11827))
-
-
-Improved Documentation
-----------------------
-
-- Update pypi installation docs to indicate that we now support Python 3.10. ([\#11820](https://github.com/matrix-org/synapse/issues/11820))
-- Add missing steps to the contribution submission process in the documentation.  Contributed by @sequentialread. ([\#11821](https://github.com/matrix-org/synapse/issues/11821))
-- Remove not needed old table of contents in documentation. ([\#11860](https://github.com/matrix-org/synapse/issues/11860))
-- Consolidate the `access_token` information at the top of each relevant page in the Admin API documentation. ([\#11861](https://github.com/matrix-org/synapse/issues/11861))
-
-
-Deprecations and Removals
--------------------------
-
-- Drop support for Python 3.6, which is EOL. ([\#11683](https://github.com/matrix-org/synapse/issues/11683))
-- Remove the `experimental_msc1849_support_enabled` flag as the features are now stable. ([\#11843](https://github.com/matrix-org/synapse/issues/11843))
-
-
-Internal Changes
-----------------
-
-- Preparation for database schema simplifications: add `state_key` and `rejection_reason` columns to `events` table. ([\#11792](https://github.com/matrix-org/synapse/issues/11792))
-- Add `FrozenEvent.get_state_key` and use it in a couple of places. ([\#11793](https://github.com/matrix-org/synapse/issues/11793))
-- Preparation for database schema simplifications: stop reading from `event_reference_hashes`. ([\#11794](https://github.com/matrix-org/synapse/issues/11794))
-- Drop unused table `public_room_list_stream`. ([\#11795](https://github.com/matrix-org/synapse/issues/11795))
-- Preparation for reducing Postgres serialization errors: allow setting transaction isolation level. Contributed by Nick @ Beeper. ([\#11799](https://github.com/matrix-org/synapse/issues/11799), [\#11847](https://github.com/matrix-org/synapse/issues/11847))
-- Docker: skip the initial amd64-only build and go straight to multiarch. ([\#11810](https://github.com/matrix-org/synapse/issues/11810))
-- Run Complement on the Github Actions VM and not inside a Docker container. ([\#11811](https://github.com/matrix-org/synapse/issues/11811))
-- Log module names at startup. ([\#11813](https://github.com/matrix-org/synapse/issues/11813))
-- Improve type safety of bundled aggregations code. ([\#11815](https://github.com/matrix-org/synapse/issues/11815))
-- Correct a type annotation in the event validation logic. ([\#11817](https://github.com/matrix-org/synapse/issues/11817), [\#11830](https://github.com/matrix-org/synapse/issues/11830))
-- Minor updates and documentation for database schema delta files. ([\#11823](https://github.com/matrix-org/synapse/issues/11823))
-- Workaround a type annotation problem in `prometheus_client` 0.13.0. ([\#11834](https://github.com/matrix-org/synapse/issues/11834))
-- Minor performance improvement in room state lookup. ([\#11836](https://github.com/matrix-org/synapse/issues/11836))
-- Fix some indentation inconsistencies in the sample config. ([\#11838](https://github.com/matrix-org/synapse/issues/11838))
-- Add type hints to `tests/rest/admin`. ([\#11851](https://github.com/matrix-org/synapse/issues/11851))
-
-
-Synapse 1.51.0 (2022-01-25)
-===========================
-
-No significant changes since 1.51.0rc2.
-
-Synapse 1.51.0 deprecates `webclient` listeners and non-HTTP(S) `web_client_location`s. Support for these will be removed in Synapse 1.53.0, at which point Synapse will not be capable of directly serving a web client for Matrix. See the [upgrade notes](https://matrix-org.github.io/synapse/develop/upgrade#upgrading-to-v1510).
-
-Synapse 1.51.0rc2 (2022-01-24)
-==============================
-
-Bugfixes
---------
-
-- Fix a bug introduced in Synapse 1.40.0 that caused Synapse to fail to process incoming federation traffic after handling a large amount of events in a v1 room. ([\#11806](https://github.com/matrix-org/synapse/issues/11806))
-
-
-Synapse 1.50.2 (2022-01-24)
-===========================
-
-This release includes the same bugfix as Synapse 1.51.0rc2.
-
-Bugfixes
---------
-
-- Fix a bug introduced in Synapse 1.40.0 that caused Synapse to fail to process incoming federation traffic after handling a large amount of events in a v1 room. ([\#11806](https://github.com/matrix-org/synapse/issues/11806))
-
-
-Synapse 1.51.0rc1 (2022-01-21)
-==============================
-
-Features
---------
-
-- Add `track_puppeted_user_ips` config flag to record client IP addresses against puppeted users, and include the puppeted users in monthly active user counts. ([\#11561](https://github.com/matrix-org/synapse/issues/11561), [\#11749](https://github.com/matrix-org/synapse/issues/11749), [\#11757](https://github.com/matrix-org/synapse/issues/11757))
-- Include whether the requesting user has participated in a thread when generating a summary for [MSC3440](https://github.com/matrix-org/matrix-doc/pull/3440). ([\#11577](https://github.com/matrix-org/synapse/issues/11577))
-- Return an `M_FORBIDDEN` error code instead of `M_UNKNOWN` when a spam checker module prevents a user from creating a room. ([\#11672](https://github.com/matrix-org/synapse/issues/11672))
-- Add a flag to the `synapse_review_recent_signups` script to ignore and filter appservice users. ([\#11675](https://github.com/matrix-org/synapse/issues/11675), [\#11770](https://github.com/matrix-org/synapse/issues/11770))
-
-
-Bugfixes
---------
-
-- Fix a long-standing issue which could cause Synapse to incorrectly accept data in the unsigned field of events
-  received over federation. ([\#11530](https://github.com/matrix-org/synapse/issues/11530))
-- Fix a long-standing bug where Synapse wouldn't cache a response indicating that a remote user has no devices. ([\#11587](https://github.com/matrix-org/synapse/issues/11587))
-- Fix an error that occurs whilst trying to get the federation status of a destination server that was working normally. This admin API was newly introduced in Synapse v1.49.0. ([\#11593](https://github.com/matrix-org/synapse/issues/11593))
-- Fix bundled aggregations not being included in the `/sync` response, per [MSC2675](https://github.com/matrix-org/matrix-doc/pull/2675). ([\#11612](https://github.com/matrix-org/synapse/issues/11612), [\#11659](https://github.com/matrix-org/synapse/issues/11659), [\#11791](https://github.com/matrix-org/synapse/issues/11791))
-- Fix the `/_matrix/client/v1/room/{roomId}/hierarchy` endpoint returning incorrect fields which have been present since Synapse 1.49.0. ([\#11667](https://github.com/matrix-org/synapse/issues/11667))
-- Fix preview of some GIF URLs (like tenor.com). Contributed by Philippe Daouadi. ([\#11669](https://github.com/matrix-org/synapse/issues/11669))
-- Fix a bug where only the first 50 rooms from a space were returned from the `/hierarchy` API. This has existed since the introduction of the API in Synapse v1.41.0. ([\#11695](https://github.com/matrix-org/synapse/issues/11695))
-- Fix a bug introduced in Synapse v1.18.0 where password reset and address validation emails would not be sent if their subject was configured to use the 'app' template variable. Contributed by @br4nnigan. ([\#11710](https://github.com/matrix-org/synapse/issues/11710), [\#11745](https://github.com/matrix-org/synapse/issues/11745))
-- Make the 'List Rooms' Admin API sort stable. Contributed by Daniël Sonck. ([\#11737](https://github.com/matrix-org/synapse/issues/11737))
-- Fix a long-standing bug where space hierarchy over federation would only work correctly some of the time. ([\#11775](https://github.com/matrix-org/synapse/issues/11775))
-- Fix a bug introduced in Synapse v1.46.0 that prevented `on_logged_out` module callbacks from being correctly awaited by Synapse. ([\#11786](https://github.com/matrix-org/synapse/issues/11786))
-
-
-Improved Documentation
-----------------------
-
-- Warn against using a Let's Encrypt certificate for TLS/DTLS TURN server client connections, and suggest using ZeroSSL certificate instead. This works around client-side connectivity errors caused by WebRTC libraries that reject Let's Encrypt certificates. Contibuted by @AndrewFerr. ([\#11686](https://github.com/matrix-org/synapse/issues/11686))
-- Document the new `SYNAPSE_TEST_PERSIST_SQLITE_DB` environment variable in the contributing guide. ([\#11715](https://github.com/matrix-org/synapse/issues/11715))
-- Document that the minimum supported PostgreSQL version is now 10. ([\#11725](https://github.com/matrix-org/synapse/issues/11725))
-- Fix typo in demo docs: differnt. ([\#11735](https://github.com/matrix-org/synapse/issues/11735))
-- Update room spec URL in config files. ([\#11739](https://github.com/matrix-org/synapse/issues/11739))
-- Mention `python3-venv` and `libpq-dev` dependencies in the contribution guide. ([\#11740](https://github.com/matrix-org/synapse/issues/11740))
-- Update documentation for configuring login with Facebook. ([\#11755](https://github.com/matrix-org/synapse/issues/11755))
-- Update installation instructions to note that Python 3.6 is no longer supported. ([\#11781](https://github.com/matrix-org/synapse/issues/11781))
-
-
-Deprecations and Removals
--------------------------
-
-- Remove the unstable `/send_relation` endpoint. ([\#11682](https://github.com/matrix-org/synapse/issues/11682))
-- Remove `python_twisted_reactor_pending_calls` Prometheus metric. ([\#11724](https://github.com/matrix-org/synapse/issues/11724))
-- Remove the `password_hash` field from the response dictionaries of the [Users Admin API](https://matrix-org.github.io/synapse/latest/admin_api/user_admin_api.html). ([\#11576](https://github.com/matrix-org/synapse/issues/11576))
-- **Deprecate support for `webclient` listeners and non-HTTP(S) `web_client_location` configuration. ([\#11774](https://github.com/matrix-org/synapse/issues/11774), [\#11783](https://github.com/matrix-org/synapse/issues/11783))**
-
-
-Internal Changes
-----------------
-
-- Run `pyupgrade --py37-plus --keep-percent-format` on Synapse. ([\#11685](https://github.com/matrix-org/synapse/issues/11685))
-- Use buildkit's cache feature to speed up docker builds. ([\#11691](https://github.com/matrix-org/synapse/issues/11691))
-- Use `auto_attribs` and native type hints for attrs classes. ([\#11692](https://github.com/matrix-org/synapse/issues/11692), [\#11768](https://github.com/matrix-org/synapse/issues/11768))
-- Remove debug logging for #4422, which has been closed since Synapse 0.99. ([\#11693](https://github.com/matrix-org/synapse/issues/11693))
-- Remove fallback code for Python 2. ([\#11699](https://github.com/matrix-org/synapse/issues/11699))
-- Add a test for [an edge case](https://github.com/matrix-org/synapse/pull/11532#discussion_r769104461) in the `/sync` logic. ([\#11701](https://github.com/matrix-org/synapse/issues/11701))
-- Add the option to write SQLite test dbs to disk when running tests. ([\#11702](https://github.com/matrix-org/synapse/issues/11702))
-- Improve Complement test output for Gitub Actions. ([\#11707](https://github.com/matrix-org/synapse/issues/11707))
-- Fix docstring on `add_account_data_for_user`. ([\#11716](https://github.com/matrix-org/synapse/issues/11716))
-- Complement environment variable name change and update `.gitignore`. ([\#11718](https://github.com/matrix-org/synapse/issues/11718))
-- Simplify calculation of Prometheus metrics for garbage collection. ([\#11723](https://github.com/matrix-org/synapse/issues/11723))
-- Improve accuracy of `python_twisted_reactor_tick_time` Prometheus metric. ([\#11724](https://github.com/matrix-org/synapse/issues/11724), [\#11771](https://github.com/matrix-org/synapse/issues/11771))
-- Minor efficiency improvements when inserting many values into the database. ([\#11742](https://github.com/matrix-org/synapse/issues/11742))
-- Invite PR authors to give themselves credit in the changelog. ([\#11744](https://github.com/matrix-org/synapse/issues/11744))
-- Add optional debugging to investigate [issue 8631](https://github.com/matrix-org/synapse/issues/8631). ([\#11760](https://github.com/matrix-org/synapse/issues/11760))
-- Remove `log_function` utility function and its uses. ([\#11761](https://github.com/matrix-org/synapse/issues/11761))
-- Add a unit test that checks both `client` and `webclient` resources will function when simultaneously enabled. ([\#11765](https://github.com/matrix-org/synapse/issues/11765))
-- Allow overriding complement commit using `COMPLEMENT_REF`. ([\#11766](https://github.com/matrix-org/synapse/issues/11766))
-- Add some comments and type annotations for `_update_outliers_txn`. ([\#11776](https://github.com/matrix-org/synapse/issues/11776))
-
-
-Synapse 1.50.1 (2022-01-18)
-===========================
-
-This release fixes a bug in Synapse 1.50.0 that could prevent clients from being able to connect to Synapse if the `webclient` resource was enabled. Further details are available in [this issue](https://github.com/matrix-org/synapse/issues/11763).
-
-Bugfixes
---------
-
-- Fix a bug introduced in Synapse 1.50.0rc1 that could cause Matrix clients to be unable to connect to Synapse instances with the `webclient` resource enabled. ([\#11764](https://github.com/matrix-org/synapse/issues/11764))
-
-
-Synapse 1.50.0 (2022-01-18)
-===========================
-
-**This release contains a critical bug that may prevent clients from being able to connect. 
-As such, it is not recommended to upgrade to 1.50.0. Instead, please upgrade straight to
-to 1.50.1. Further details are available in [this issue](https://github.com/matrix-org/synapse/issues/11763).**
-
-Please note that we now only support Python 3.7+ and PostgreSQL 10+ (if applicable), because Python 3.6 and PostgreSQL 9.6 have reached end-of-life.
-
-No significant changes since 1.50.0rc2.
-
-
-Synapse 1.50.0rc2 (2022-01-14)
-==============================
-
-This release candidate fixes a federation-breaking regression introduced in Synapse 1.50.0rc1.
-
-Bugfixes
---------
-
-- Fix a bug introduced in Synapse v1.0.0 whereby some device list updates would not be sent to remote homeservers if there were too many to send at once. ([\#11729](https://github.com/matrix-org/synapse/issues/11729))
-- Fix a bug introduced in Synapse v1.50.0rc1 whereby outbound federation could fail because too many EDUs were produced for device updates. ([\#11730](https://github.com/matrix-org/synapse/issues/11730))
-
-
-Improved Documentation
-----------------------
-
-- Document that now the minimum supported PostgreSQL version is 10. ([\#11725](https://github.com/matrix-org/synapse/issues/11725))
-
-
-Internal Changes
-----------------
-
-- Fix a typechecker problem related to our (ab)use of `nacl.signing.SigningKey`s. ([\#11714](https://github.com/matrix-org/synapse/issues/11714))
-
-
 Synapse 1.50.0rc1 (2022-01-05)
 ==============================
 
+Please note that we now only support Python 3.7+ and PostgreSQL 10+ (if applicable), because Python 3.6 and PostgreSQL 9.6 have reached end-of-life.
+
 
 Features
 --------
@@ -263,7 +42,6 @@ Deprecations and Removals
 -------------------------
 
 - Replace `mock` package by its standard library version. ([\#11588](https://github.com/matrix-org/synapse/issues/11588))
-- Drop support for Python 3.6 and Ubuntu 18.04. ([\#11633](https://github.com/matrix-org/synapse/issues/11633))
 
 
 Internal Changes
@@ -299,13 +77,13 @@ Internal Changes
 - Improve OpenTracing support for requests which use a `ResponseCache`. ([\#11607](https://github.com/matrix-org/synapse/issues/11607))
 - Improve OpenTracing support for incoming HTTP requests. ([\#11618](https://github.com/matrix-org/synapse/issues/11618))
 - A number of improvements to opentracing support. ([\#11619](https://github.com/matrix-org/synapse/issues/11619))
+- Drop support for Python 3.6 and Ubuntu 18.04. ([\#11633](https://github.com/matrix-org/synapse/issues/11633))
 - Refactor the way that the `outlier` flag is set on events received over federation. ([\#11634](https://github.com/matrix-org/synapse/issues/11634))
 - Improve the error messages from  `get_create_event_for_room`. ([\#11638](https://github.com/matrix-org/synapse/issues/11638))
 - Remove redundant `get_current_events_token` method. ([\#11643](https://github.com/matrix-org/synapse/issues/11643))
 - Convert `namedtuples` to `attrs`. ([\#11665](https://github.com/matrix-org/synapse/issues/11665), [\#11574](https://github.com/matrix-org/synapse/issues/11574))
 - Update the `/capabilities` response to include whether support for [MSC3440](https://github.com/matrix-org/matrix-doc/pull/3440) is available. ([\#11690](https://github.com/matrix-org/synapse/issues/11690))
 - Send the `Accept` header in HTTP requests made using `SimpleHttpClient.get_json`. ([\#11677](https://github.com/matrix-org/synapse/issues/11677))
-- Work around Mjolnir compatibility issue by adding an import for `glob_to_regex` in `synapse.util`, where it moved from. ([\#11696](https://github.com/matrix-org/synapse/issues/11696))
 
 
 Synapse 1.49.2 (2021-12-21)

book.toml

@@ -34,14 +34,6 @@ additional-css = [
     "docs/website_files/table-of-contents.css",
     "docs/website_files/remove-nav-buttons.css",
     "docs/website_files/indent-section-headers.css",
-    "docs/website_files/version-picker.css",
 ]
-additional-js = [
-    "docs/website_files/table-of-contents.js",
-    "docs/website_files/version-picker.js",
-    "docs/website_files/version.js",
-]
-theme = "docs/website_files/theme"
-
-[preprocessor.schema_versions]
-command = "./scripts-dev/schema_versions.py"
+additional-js = ["docs/website_files/table-of-contents.js"]
+theme = "docs/website_files/theme"

changelog.d/11583.bugfix

@@ -0,0 +1 @@
+Fix a performance regression in `/sync` handling, introduced in 1.49.0.

changelog.d/11696.misc

@@ -0,0 +1 @@
+Work around Mjolnir compatibility issue by adding an import for `glob_to_regex` in `synapse.util`, where it moved from.

contrib/prometheus/consoles/synapse.html

@@ -92,6 +92,22 @@ new PromConsole.Graph({
 })
 </script>
 
+<h3>Pending calls per tick</h3>
+<div id="reactor_pending_calls"></div>
+<script>
+new PromConsole.Graph({
+  node: document.querySelector("#reactor_pending_calls"),
+  expr: "rate(python_twisted_reactor_pending_calls_sum[30s]) / rate(python_twisted_reactor_pending_calls_count[30s])",
+  name: "[[job]]-[[index]]",
+  min: 0,
+  renderer: "line",
+  height: 150,
+  yAxisFormatter: PromConsole.NumberFormatter.humanize,
+  yHoverFormatter: PromConsole.NumberFormatter.humanize,
+  yTitle: "Pending Calls"
+})
+</script>
+
 <h1>Storage</h1>
 
 <h3>Queries</h3>

debian/changelog

@@ -1,57 +1,3 @@
-matrix-synapse-py3 (1.52.0) stable; urgency=medium
-
-  * New synapse release 1.52.0.
-
- -- Synapse Packaging team <packages@matrix.org>  Tue, 08 Feb 2022 11:34:54 +0000
-
-matrix-synapse-py3 (1.52.0~rc1) stable; urgency=medium
-
-  * New synapse release 1.52.0~rc1.
-
- -- Synapse Packaging team <packages@matrix.org>  Tue, 01 Feb 2022 11:04:09 +0000
-
-matrix-synapse-py3 (1.51.0) stable; urgency=medium
-
-  * New synapse release 1.51.0.
-
- -- Synapse Packaging team <packages@matrix.org>  Tue, 25 Jan 2022 11:28:51 +0000
-
-matrix-synapse-py3 (1.51.0~rc2) stable; urgency=medium
-
-  * New synapse release 1.51.0~rc2.
-
- -- Synapse Packaging team <packages@matrix.org>  Mon, 24 Jan 2022 12:25:00 +0000
-
-matrix-synapse-py3 (1.51.0~rc1) stable; urgency=medium
-
-  * New synapse release 1.51.0~rc1.
-
- -- Synapse Packaging team <packages@matrix.org>  Fri, 21 Jan 2022 10:46:02 +0000
-
-matrix-synapse-py3 (1.50.2) stable; urgency=medium
-
-  * New synapse release 1.50.2.
-
- -- Synapse Packaging team <packages@matrix.org>  Mon, 24 Jan 2022 13:37:11 +0000
-
-matrix-synapse-py3 (1.50.1) stable; urgency=medium
-
-  * New synapse release 1.50.1.
-
- -- Synapse Packaging team <packages@matrix.org>  Tue, 18 Jan 2022 16:06:26 +0000
-
-matrix-synapse-py3 (1.50.0) stable; urgency=medium
-
-  * New synapse release 1.50.0.
-
- -- Synapse Packaging team <packages@matrix.org>  Tue, 18 Jan 2022 10:40:38 +0000
-
-matrix-synapse-py3 (1.50.0~rc2) stable; urgency=medium
-
-  * New synapse release 1.50.0~rc2.
-
- -- Synapse Packaging team <packages@matrix.org>  Fri, 14 Jan 2022 11:18:06 +0000
-
 matrix-synapse-py3 (1.50.0~rc1) stable; urgency=medium
 
   * New synapse release 1.50.0~rc1.

demo/README

@@ -22,5 +22,5 @@ Logs and sqlitedb will be stored in demo/808{0,1,2}.{log,db}
 
 
 
-Also note that when joining a public room on a different HS via "#foo:bar.net", then you are (in the current impl) joining a room with room_id "foo". This means that it won't work if your HS already has a room with that name.
+Also note that when joining a public room on a differnt HS via "#foo:bar.net", then you are (in the current impl) joining a room with room_id "foo". This means that it won't work if your HS already has a room with that name.
 

docker/Dockerfile

@@ -1,17 +1,14 @@
 # Dockerfile to build the matrixdotorg/synapse docker images.
 #
-# Note that it uses features which are only available in BuildKit - see
-# https://docs.docker.com/go/buildkit/ for more information.
-#
 # To build the image, run `docker build` command from the root of the
 # synapse repository:
 #
-#    DOCKER_BUILDKIT=1 docker build -f docker/Dockerfile .
+#    docker build -f docker/Dockerfile .
 #
 # There is an optional PYTHON_VERSION build argument which sets the
 # version of python to build against: for example:
 #
-#    DOCKER_BUILDKIT=1 docker build -f docker/Dockerfile --build-arg PYTHON_VERSION=3.9 .
+#    docker build -f docker/Dockerfile --build-arg PYTHON_VERSION=3.6 .
 #
 
 ARG PYTHON_VERSION=3.8
@@ -22,16 +19,7 @@ ARG PYTHON_VERSION=3.8
 FROM docker.io/python:${PYTHON_VERSION}-slim as builder
 
 # install the OS build deps
-#
-# RUN --mount is specific to buildkit and is documented at
-# https://github.com/moby/buildkit/blob/master/frontend/dockerfile/docs/syntax.md#build-mounts-run---mount.
-# Here we use it to set up a cache for apt, to improve rebuild speeds on
-# slow connections.
-#
-RUN \
-   --mount=type=cache,target=/var/cache/apt,sharing=locked \
-   --mount=type=cache,target=/var/lib/apt,sharing=locked \
- apt-get update && apt-get install -y \
+RUN apt-get update && apt-get install -y \
     build-essential \
     libffi-dev \
     libjpeg-dev \
@@ -56,8 +44,7 @@ COPY synapse/python_dependencies.py /synapse/synapse/python_dependencies.py
 # used while you develop on the source
 #
 # This is aiming at installing the `install_requires` and `extras_require` from `setup.py`
-RUN --mount=type=cache,target=/root/.cache/pip \
-  pip install --prefix="/install" --no-warn-script-location \
+RUN pip install --prefix="/install" --no-warn-script-location \
     /synapse[all]
 
 # Copy over the rest of the project
@@ -79,10 +66,7 @@ LABEL org.opencontainers.image.documentation='https://github.com/matrix-org/syna
 LABEL org.opencontainers.image.source='https://github.com/matrix-org/synapse.git'
 LABEL org.opencontainers.image.licenses='Apache-2.0'
 
-RUN \
-   --mount=type=cache,target=/var/cache/apt,sharing=locked \
-   --mount=type=cache,target=/var/lib/apt,sharing=locked \
-  apt-get update && apt-get install -y \
+RUN apt-get update && apt-get install -y \
     curl \
     gosu \
     libjpeg62-turbo \

docker/Dockerfile-pgtests

@@ -1,6 +1,6 @@
 # Use the Sytest image that comes with a lot of the build dependencies
 # pre-installed
-FROM matrixdotorg/sytest:focal
+FROM matrixdotorg/sytest:bionic
 
 # The Sytest image doesn't come with python, so install that
 RUN apt-get update && apt-get -qq install -y python3 python3-dev python3-pip

docker/run_pg_tests.sh

@@ -16,4 +16,4 @@ sudo -u postgres /usr/lib/postgresql/10/bin/pg_ctl -w -D /var/lib/postgresql/dat
 # Run the tests
 cd /src
 export TRIAL_FLAGS="-j 4"
-tox --workdir=./.tox-pg-container -e py37-postgres "$@"
+tox --workdir=./.tox-pg-container -e py36-postgres "$@"

docs/MSC1711_certificates_FAQ.md

@@ -44,6 +44,27 @@ For more details and context on the release of the r0.1 Server/Server API and
 imminent Matrix 1.0 release, you can also see our
 [main talk from FOSDEM 2019](https://matrix.org/blog/2019/02/04/matrix-at-fosdem-2019/).
 
+## Contents
+* Timeline
+* Configuring certificates for compatibility with Synapse 1.0
+* FAQ
+  * Synapse 0.99.0 has just been released, what do I need to do right now?
+  * How do I upgrade?
+  * What will happen if I do not set up a valid federation certificate
+    immediately?
+  * What will happen if I do nothing at all?
+  * When do I need a SRV record or .well-known URI?
+  * Can I still use an SRV record?
+  * I have created a .well-known URI. Do I still need an SRV record?
+  * It used to work just fine, why are you breaking everything?
+  * Can I manage my own certificates rather than having Synapse renew
+    certificates itself?
+  * Do you still recommend against using a reverse proxy on the federation port?
+  * Do I still need to give my TLS certificates to Synapse if I am using a
+    reverse proxy?
+  * Do I need the same certificate for the client and federation port?
+  * How do I tell Synapse to reload my keys/certificates after I replace them?
+
 ## Timeline
 
 **5th Feb 2019  - Synapse 0.99.0 is released.**

docs/admin_api/account_validity.md

@@ -4,9 +4,6 @@ This API allows a server administrator to manage the validity of an account. To
 use it, you must enable the account validity feature (under
 `account_validity`) in Synapse's configuration.
 
-To use it, you will need to authenticate by providing an `access_token`
-for a server admin: see [Admin API](../usage/administration/admin_api).
-
 ## Renew account
 
 This API extends the validity of an account by as much time as configured in the

docs/admin_api/delete_group.md

@@ -4,11 +4,11 @@ This API lets a server admin delete a local group. Doing so will kick all
 users out of the group so that their clients will correctly handle the group
 being deleted.
 
-To use it, you will need to authenticate by providing an `access_token`
-for a server admin: see [Admin API](../usage/administration/admin_api).
-
 The API is:
 
 ```
 POST /_synapse/admin/v1/delete_group/<group_id>
 ```
+
+To use it, you will need to authenticate by providing an `access_token` for a
+server admin: see [Admin API](../usage/administration/admin_api).

docs/admin_api/event_reports.md

@@ -2,13 +2,12 @@
 
 This API returns information about reported events.
 
-To use it, you will need to authenticate by providing an `access_token`
-for a server admin: see [Admin API](../usage/administration/admin_api).
-
 The api is:
 ```
 GET /_synapse/admin/v1/event_reports?from=0&limit=10
 ```
+To use it, you will need to authenticate by providing an `access_token` for a
+server admin: see [Admin API](../usage/administration/admin_api).
 
 It returns a JSON body like the following:
 
@@ -95,6 +94,8 @@ The api is:
 ```
 GET /_synapse/admin/v1/event_reports/<report_id>
 ```
+To use it, you will need to authenticate by providing an `access_token` for a
+server admin: see [Admin API](../usage/administration/admin_api).
 
 It returns a JSON body like the following:
 

docs/admin_api/media_admin_api.md

@@ -1,10 +1,24 @@
+# Contents
+- [Querying media](#querying-media)
+  * [List all media in a room](#list-all-media-in-a-room)
+  * [List all media uploaded by a user](#list-all-media-uploaded-by-a-user)
+- [Quarantine media](#quarantine-media)
+  * [Quarantining media by ID](#quarantining-media-by-id)
+  * [Remove media from quarantine by ID](#remove-media-from-quarantine-by-id)
+  * [Quarantining media in a room](#quarantining-media-in-a-room)
+  * [Quarantining all media of a user](#quarantining-all-media-of-a-user)
+  * [Protecting media from being quarantined](#protecting-media-from-being-quarantined)
+  * [Unprotecting media from being quarantined](#unprotecting-media-from-being-quarantined)
+- [Delete local media](#delete-local-media)
+  * [Delete a specific local media](#delete-a-specific-local-media)
+  * [Delete local media by date or size](#delete-local-media-by-date-or-size)
+  * [Delete media uploaded by a user](#delete-media-uploaded-by-a-user)
+- [Purge Remote Media API](#purge-remote-media-api)
+
 # Querying media
 
 These APIs allow extracting media information from the homeserver.
 
-To use it, you will need to authenticate by providing an `access_token`
-for a server admin: see [Admin API](../usage/administration/admin_api).
-
 ## List all media in a room
 
 This API gets a list of known media in a room.
@@ -14,6 +28,8 @@ The API is:
 ```
 GET /_synapse/admin/v1/room/<room_id>/media
 ```
+To use it, you will need to authenticate by providing an `access_token` for a
+server admin: see [Admin API](../usage/administration/admin_api).
 
 The API returns a JSON body like the following:
 ```json
@@ -301,5 +317,8 @@ The following fields are returned in the JSON response body:
 
 * `deleted`: integer - The number of media items successfully deleted
 
+To use it, you will need to authenticate by providing an `access_token` for a
+server admin: see [Admin API](../usage/administration/admin_api).
+
 If the user re-requests purged remote media, synapse will re-request the media
 from the originating server.

docs/admin_api/purge_history_api.md

@@ -10,15 +10,15 @@ paginate further back in the room from the point being purged from.
 Note that Synapse requires at least one message in each room, so it will never
 delete the last message in a room.
 
-To use it, you will need to authenticate by providing an `access_token`
-for a server admin: see [Admin API](../usage/administration/admin_api).
-
 The API is:
 
 ```
 POST /_synapse/admin/v1/purge_history/<room_id>[/<event_id>]
 ```
 
+To use it, you will need to authenticate by providing an `access_token` for a
+server admin: [Admin API](../usage/administration/admin_api)
+
 By default, events sent by local users are not deleted, as they may represent
 the only copies of this content in existence. (Events sent by remote users are
 deleted.)
@@ -57,6 +57,9 @@ It is possible to poll for updates on recent purges with a second API;
 GET /_synapse/admin/v1/purge_history_status/<purge_id>
 ```
 
+Again, you will need to authenticate by providing an `access_token` for a
+server admin.
+
 This API returns a JSON body like the following:
 
 ```json

docs/admin_api/room_membership.md

@@ -5,9 +5,6 @@ to a room with a given `room_id_or_alias`. You can only modify the membership of
 local users. The server administrator must be in the room and have permission to
 invite users.
 
-To use it, you will need to authenticate by providing an `access_token`
-for a server admin: see [Admin API](../usage/administration/admin_api).
-
 ## Parameters
 
 The following parameters are available:
@@ -26,6 +23,9 @@ POST /_synapse/admin/v1/join/<room_id_or_alias>
 }
 ```
 
+To use it, you will need to authenticate by providing an `access_token` for a
+server admin: see [Admin API](../usage/administration/admin_api).
+
 Response:
 
 ```json

docs/admin_api/rooms.md

@@ -1,12 +1,24 @@
+# Contents
+- [List Room API](#list-room-api)
+- [Room Details API](#room-details-api)
+- [Room Members API](#room-members-api)
+- [Room State API](#room-state-api)
+- [Block Room API](#block-room-api)
+- [Delete Room API](#delete-room-api)
+  * [Version 1 (old version)](#version-1-old-version)
+  * [Version 2 (new version)](#version-2-new-version)
+  * [Status of deleting rooms](#status-of-deleting-rooms)
+  * [Undoing room shutdowns](#undoing-room-shutdowns)
+- [Make Room Admin API](#make-room-admin-api)
+- [Forward Extremities Admin API](#forward-extremities-admin-api)
+- [Event Context API](#event-context-api)
+
 # List Room API
 
 The List Room admin API allows server admins to get a list of rooms on their
 server. There are various parameters available that allow for filtering and
 sorting the returned list. This API supports pagination.
 
-To use it, you will need to authenticate by providing an `access_token`
-for a server admin: see [Admin API](../usage/administration/admin_api).
-
 **Parameters**
 
 The following query parameters are available:
@@ -481,6 +493,9 @@ several minutes or longer.
 The local server will only have the power to move local user and room aliases to
 the new room. Users on other servers will be unaffected.
 
+To use it, you will need to authenticate by providing an ``access_token`` for a
+server admin: see [Admin API](../usage/administration/admin_api).
+
 ## Version 1 (old version)
 
 This version works synchronously. That means you only get the response once the server has

docs/admin_api/statistics.md

@@ -3,15 +3,15 @@
 Returns information about all local media usage of users. Gives the
 possibility to filter them by time and user.
 
-To use it, you will need to authenticate by providing an `access_token`
-for a server admin: see [Admin API](../usage/administration/admin_api).
-
 The API is:
 
 ```
 GET /_synapse/admin/v1/statistics/users/media
 ```
 
+To use it, you will need to authenticate by providing an `access_token`
+for a server admin: see [Admin API](../usage/administration/admin_api).
+
 A response body like the following is returned:
 
 ```json

docs/admin_api/user_admin_api.md

@@ -1,8 +1,5 @@
 # User Admin API
 
-To use it, you will need to authenticate by providing an `access_token`
-for a server admin: see [Admin API](../usage/administration/admin_api).
-
 ## Query User Account
 
 This API returns information about a specific user account.
@@ -13,12 +10,14 @@ The api is:
 GET /_synapse/admin/v2/users/<user_id>
 ```
 
+To use it, you will need to authenticate by providing an `access_token` for a
+server admin: [Admin API](../usage/administration/admin_api)
+
 It returns a JSON body like the following:
 
-```jsonc
+```json
 {
-    "name": "@user:example.com",
-    "displayname": "User", // can be null if not set
+    "displayname": "User",
     "threepids": [
         {
             "medium": "email",
@@ -33,11 +32,11 @@ It returns a JSON body like the following:
             "validated_at": 1586458409743
         }
     ],
-    "avatar_url": "<avatar_url>",  // can be null if not set
-    "is_guest": 0,
+    "avatar_url": "<avatar_url>",
     "admin": 0,
     "deactivated": 0,
     "shadow_banned": 0,
+    "password_hash": "$2b$12$p9B4GkqYdRTPGD",
     "creation_ts": 1560432506,
     "appservice_id": null,
     "consent_server_notice_sent": null,
@@ -104,6 +103,9 @@ with a body of:
 }
 ```
 
+To use it, you will need to authenticate by providing an `access_token` for a
+server admin: [Admin API](../usage/administration/admin_api)
+
 Returns HTTP status code:
 - `201` - When a new user object was created.
 - `200` - When a user was modified.
@@ -153,6 +155,9 @@ By default, the response is ordered by ascending user ID.
 GET /_synapse/admin/v2/users?from=0&limit=10&guests=false
 ```
 
+To use it, you will need to authenticate by providing an `access_token` for a
+server admin: [Admin API](../usage/administration/admin_api)
+
 A response body like the following is returned:
 
 ```json
@@ -272,6 +277,9 @@ GET /_matrix/client/r0/admin/whois/<userId>
 See also: [Client Server
 API Whois](https://matrix.org/docs/spec/client_server/r0.6.1#get-matrix-client-r0-admin-whois-userid).
 
+To use it, you will need to authenticate by providing an `access_token` for a
+server admin: [Admin API](../usage/administration/admin_api)
+
 It returns a JSON body like the following:
 
 ```json
@@ -326,6 +334,9 @@ with a body of:
 }
 ```
 
+To use it, you will need to authenticate by providing an `access_token` for a
+server admin: [Admin API](../usage/administration/admin_api)
+
 The erase parameter is optional and defaults to `false`.
 An empty body may be passed for backwards compatibility.
 
@@ -341,11 +352,6 @@ The following actions are performed when deactivating an user:
 - Remove the user from the user directory
 - Reject all pending invites
 - Remove all account validity information related to the user
-- Remove the arbitrary data store known as *account data*. For example, this includes:
-    - list of ignored users;
-    - push rules;
-    - secret storage keys; and
-    - cross-signing keys.
 
 The following additional actions are performed during deactivation if `erase`
 is set to `true`:
@@ -359,6 +365,7 @@ The following actions are **NOT** performed. The list may be incomplete.
 - Remove mappings of SSO IDs
 - [Delete media uploaded](#delete-media-uploaded-by-a-user) by user (included avatar images)
 - Delete sent and received messages
+- Delete E2E cross-signing keys
 - Remove the user's creation (registration) timestamp
 - [Remove rate limit overrides](#override-ratelimiting-for-users)
 - Remove from monthly active users
@@ -382,6 +389,9 @@ with a body of:
 }
 ```
 
+To use it, you will need to authenticate by providing an `access_token` for a
+server admin: [Admin API](../usage/administration/admin_api)
+
 The parameter `new_password` is required.
 The parameter `logout_devices` is optional and defaults to `true`.
 
@@ -394,6 +404,9 @@ The api is:
 GET /_synapse/admin/v1/users/<user_id>/admin
 ```
 
+To use it, you will need to authenticate by providing an `access_token` for a
+server admin: [Admin API](../usage/administration/admin_api)
+
 A response body like the following is returned:
 
 ```json
@@ -421,6 +434,10 @@ with a body of:
 }
 ```
 
+To use it, you will need to authenticate by providing an `access_token` for a
+server admin: [Admin API](../usage/administration/admin_api)
+
+
 ## List room memberships of a user
 
 Gets a list of all `room_id` that a specific `user_id` is member.
@@ -431,6 +448,9 @@ The API is:
 GET /_synapse/admin/v1/users/<user_id>/joined_rooms
 ```
 
+To use it, you will need to authenticate by providing an `access_token` for a
+server admin: [Admin API](../usage/administration/admin_api)
+
 A response body like the following is returned:
 
 ```json
@@ -549,6 +569,9 @@ The API is:
 GET /_synapse/admin/v1/users/<user_id>/media
 ```
 
+To use it, you will need to authenticate by providing an `access_token` for a
+server admin: [Admin API](../usage/administration/admin_api)
+
 A response body like the following is returned:
 
 ```json
@@ -663,6 +686,9 @@ The API is:
 DELETE /_synapse/admin/v1/users/<user_id>/media
 ```
 
+To use it, you will need to authenticate by providing an `access_token` for a
+server admin: [Admin API](../usage/administration/admin_api)
+
 A response body like the following is returned:
 
 ```json
@@ -735,6 +761,9 @@ The API is:
 GET /_synapse/admin/v2/users/<user_id>/devices
 ```
 
+To use it, you will need to authenticate by providing an `access_token` for a
+server admin: [Admin API](../usage/administration/admin_api)
+
 A response body like the following is returned:
 
 ```json
@@ -800,6 +829,9 @@ POST /_synapse/admin/v2/users/<user_id>/delete_devices
 }
 ```
 
+To use it, you will need to authenticate by providing an `access_token` for a
+server admin: [Admin API](../usage/administration/admin_api)
+
 An empty JSON dict is returned.
 
 **Parameters**
@@ -821,6 +853,9 @@ The API is:
 GET /_synapse/admin/v2/users/<user_id>/devices/<device_id>
 ```
 
+To use it, you will need to authenticate by providing an `access_token` for a
+server admin: [Admin API](../usage/administration/admin_api)
+
 A response body like the following is returned:
 
 ```json
@@ -866,6 +901,9 @@ PUT /_synapse/admin/v2/users/<user_id>/devices/<device_id>
 }
 ```
 
+To use it, you will need to authenticate by providing an `access_token` for a
+server admin: [Admin API](../usage/administration/admin_api)
+
 An empty JSON dict is returned.
 
 **Parameters**
@@ -892,6 +930,9 @@ DELETE /_synapse/admin/v2/users/<user_id>/devices/<device_id>
 {}
 ```
 
+To use it, you will need to authenticate by providing an `access_token` for a
+server admin: [Admin API](../usage/administration/admin_api)
+
 An empty JSON dict is returned.
 
 **Parameters**
@@ -910,6 +951,9 @@ The API is:
 GET /_synapse/admin/v1/users/<user_id>/pushers
 ```
 
+To use it, you will need to authenticate by providing an `access_token` for a
+server admin: [Admin API](../usage/administration/admin_api)
+
 A response body like the following is returned:
 
 ```json
@@ -1004,6 +1048,9 @@ To un-shadow-ban a user the API is:
 DELETE /_synapse/admin/v1/users/<user_id>/shadow_ban
 ```
 
+To use it, you will need to authenticate by providing an `access_token` for a
+server admin: [Admin API](../usage/administration/admin_api)
+
 An empty JSON dict is returned in both cases.
 
 **Parameters**
@@ -1026,6 +1073,9 @@ The API is:
 GET /_synapse/admin/v1/users/<user_id>/override_ratelimit
 ```
 
+To use it, you will need to authenticate by providing an `access_token` for a
+server admin: [Admin API](../usage/administration/admin_api)
+
 A response body like the following is returned:
 
 ```json
@@ -1065,6 +1115,9 @@ The API is:
 POST /_synapse/admin/v1/users/<user_id>/override_ratelimit
 ```
 
+To use it, you will need to authenticate by providing an `access_token` for a
+server admin: [Admin API](../usage/administration/admin_api)
+
 A response body like the following is returned:
 
 ```json
@@ -1107,6 +1160,9 @@ The API is:
 DELETE /_synapse/admin/v1/users/<user_id>/override_ratelimit
 ```
 
+To use it, you will need to authenticate by providing an `access_token` for a
+server admin: [Admin API](../usage/administration/admin_api)
+
 An empty JSON dict is returned.
 
 ```json
@@ -1135,5 +1191,7 @@ The API is:
 GET /_synapse/admin/v1/username_available?username=$localpart
 ```
 
-The request and response format is the same as the
-[/_matrix/client/r0/register/available](https://matrix.org/docs/spec/client_server/r0.6.0#get-matrix-client-r0-register-available) API.
+The request and response format is the same as the [/_matrix/client/r0/register/available](https://matrix.org/docs/spec/client_server/r0.6.0#get-matrix-client-r0-register-available) API.
+
+To use it, you will need to authenticate by providing an `access_token` for a
+server admin: [Admin API](../usage/administration/admin_api)

docs/admin_api/version_api.md

@@ -16,6 +16,6 @@ It returns a JSON body like the following:
 ```json
 {
     "server_version": "0.99.2rc1 (b=develop, abcdef123)",
-    "python_version": "3.7.8"
+    "python_version": "3.6.8"
 }
 ```

docs/development/contributing_guide.md

@@ -20,9 +20,7 @@ recommended for development. More information about WSL can be found at
 <https://docs.microsoft.com/en-us/windows/wsl/install>. Running Synapse natively
 on Windows is not officially supported.
 
-The code of Synapse is written in Python 3. To do pretty much anything, you'll need [a recent version of Python 3](https://www.python.org/downloads/). Your Python also needs support for [virtual environments](https://docs.python.org/3/library/venv.html). This is usually built-in, but some Linux distributions like Debian and Ubuntu split it out into its own package. Running `sudo apt install python3-venv` should be enough.
-
-Synapse can connect to PostgreSQL via the [psycopg2](https://pypi.org/project/psycopg2/) Python library. Building this library from source requires access to PostgreSQL's C header files. On Debian or Ubuntu Linux, these can be installed with `sudo apt install libpq-dev`.
+The code of Synapse is written in Python 3. To do pretty much anything, you'll need [a recent version of Python 3](https://wiki.python.org/moin/BeginnersGuide/Download).
 
 The source code of Synapse is hosted on GitHub. You will also need [a recent version of git](https://github.com/git-guides/install-git).
 
@@ -55,7 +53,6 @@ setup a *virtualenv*, as follows:
 cd path/where/you/have/cloned/the/repository
 python3 -m venv ./env
 source ./env/bin/activate
-pip install wheel
 pip install -e ".[all,dev]"
 pip install tox
 ```
@@ -117,7 +114,7 @@ The linters look at your code and do two things:
 - ensure that your code follows the coding style adopted by the project;
 - catch a number of errors in your code.
 
-The linter takes no time at all to run as soon as you've [downloaded the dependencies into your python virtual environment](#4-install-the-dependencies).
+They're pretty fast, don't hesitate!
 
 ```sh
 source ./env/bin/activate
@@ -172,27 +169,6 @@ To increase the log level for the tests, set `SYNAPSE_TEST_LOG_LEVEL`:
 SYNAPSE_TEST_LOG_LEVEL=DEBUG trial tests
 ```
 
-By default, tests will use an in-memory SQLite database for test data. For additional
-help with debugging, one can use an on-disk SQLite database file instead, in order to
-review database state during and after running tests. This can be done by setting
-the `SYNAPSE_TEST_PERSIST_SQLITE_DB` environment variable. Doing so will cause the
-database state to be stored in a file named `test.db` under the trial process'
-working directory. Typically, this ends up being `_trial_temp/test.db`. For example:
-
-```sh
-SYNAPSE_TEST_PERSIST_SQLITE_DB=1 trial tests
-```
-
-The database file can then be inspected with:
-
-```sh
-sqlite3 _trial_temp/test.db
-```
-
-Note that the database file is cleared at the beginning of each test run. Thus it 
-will always only contain the data generated by the *last run test*. Though generally
-when debugging, one is only running a single test anyway.
-
 ### Running tests under PostgreSQL
 
 Invoking `trial` as above will use an in-memory SQLite database. This is great for

docs/development/database_schema.md

@@ -96,60 +96,6 @@ Ensure postgres is installed, then run:
 NB at the time of writing, this script predates the split into separate `state`/`main`
 databases so will require updates to handle that correctly.
 
-## Delta files
-
-Delta files define the steps required to upgrade the database from an earlier version.
-They can be written as either a file containing a series of SQL statements, or a Python
-module.
-
-Synapse remembers which delta files it has applied to a database (they are stored in the
-`applied_schema_deltas` table) and will not re-apply them (even if a given file is
-subsequently updated).
-
-Delta files should be placed in a directory named `synapse/storage/schema/<database>/delta/<version>/`.
-They are applied in alphanumeric order, so  by convention the first two characters
-of the filename should be an integer such as `01`, to put the file in the right order.
-
-### SQL delta files
-
-These should be named `*.sql`, or —  for changes which should only be applied for a
-given database engine — `*.sql.posgres` or `*.sql.sqlite`. For example, a delta which
-adds a new column to the `foo` table might be called `01add_bar_to_foo.sql`.
-
-Note that our SQL parser is a bit simple - it understands comments (`--` and `/*...*/`),
-but complex statements which require a `;` in the middle of them (such as `CREATE
-TRIGGER`) are beyond it and you'll have to use a Python delta file.
-
-### Python delta files
-
-For more flexibility, a delta file can take the form of a python module. These should
-be named `*.py`. Note that database-engine-specific modules are not supported here –
-instead you can write `if isinstance(database_engine, PostgresEngine)` or similar.
-
-A Python delta module should define either or both of the following functions:
-
-```python
-import synapse.config.homeserver
-import synapse.storage.engines
-import synapse.storage.types
-
-
-def run_create(
-    cur: synapse.storage.types.Cursor,
-    database_engine: synapse.storage.engines.BaseDatabaseEngine,
-) -> None:
-    """Called whenever an existing or new database is to be upgraded"""
-    ...
-
-def run_upgrade(
-    cur: synapse.storage.types.Cursor,
-    database_engine: synapse.storage.engines.BaseDatabaseEngine,
-    config: synapse.config.homeserver.HomeServerConfig,
-) -> None:
-    """Called whenever an existing database is to be upgraded."""
-    ...
-```
-
 ## Boolean columns
 
 Boolean columns require special treatment, since SQLite treats booleans the

docs/development/url_previews.md

@@ -35,12 +35,7 @@ When Synapse is asked to preview a URL it does the following:
    5. If the media is HTML:
       1. Decodes the HTML via the stored file.
       2. Generates an Open Graph response from the HTML.
-      3. If a JSON oEmbed URL was found in the HTML via autodiscovery:
-         1. Downloads the URL and stores it into a file via the media storage provider
-            and saves the local media metadata.
-         2. Convert the oEmbed response to an Open Graph response.
-         3. Override any Open Graph data from the HTML with data from oEmbed.
-      4. If an image exists in the Open Graph response:
+      3. If an image exists in the Open Graph response:
          1. Downloads the URL and stores it into a file via the media storage
             provider and saves the local media metadata.
          2. Generates thumbnails.

docs/modules/password_auth_provider_callbacks.md

@@ -105,68 +105,6 @@ device ID), and the (now deactivated) access token.
 
 If multiple modules implement this callback, Synapse runs them all in order.
 
-### `get_username_for_registration`
-
-_First introduced in Synapse v1.52.0_
-
-```python
-async def get_username_for_registration(
-    uia_results: Dict[str, Any],
-    params: Dict[str, Any],
-) -> Optional[str]
-```
-
-Called when registering a new user. The module can return a username to set for the user
-being registered by returning it as a string, or `None` if it doesn't wish to force a
-username for this user. If a username is returned, it will be used as the local part of a
-user's full Matrix ID (e.g. it's `alice` in `@alice:example.com`).
-
-This callback is called once [User-Interactive Authentication](https://spec.matrix.org/latest/client-server-api/#user-interactive-authentication-api)
-has been completed by the user. It is not called when registering a user via SSO. It is
-passed two dictionaries, which include the information that the user has provided during
-the registration process.
-
-The first dictionary contains the results of the [User-Interactive Authentication](https://spec.matrix.org/latest/client-server-api/#user-interactive-authentication-api)
-flow followed by the user. Its keys are the identifiers of every step involved in the flow,
-associated with either a boolean value indicating whether the step was correctly completed,
-or additional information (e.g. email address, phone number...). A list of most existing
-identifiers can be found in the [Matrix specification](https://spec.matrix.org/v1.1/client-server-api/#authentication-types).
-Here's an example featuring all currently supported keys:
-
-```python
-{
-    "m.login.dummy": True,  # Dummy authentication
-    "m.login.terms": True,  # User has accepted the terms of service for the homeserver
-    "m.login.recaptcha": True,  # User has completed the recaptcha challenge
-    "m.login.email.identity": {  # User has provided and verified an email address
-        "medium": "email",
-        "address": "alice@example.com",
-        "validated_at": 1642701357084,
-    },
-    "m.login.msisdn": {  # User has provided and verified a phone number
-        "medium": "msisdn",
-        "address": "33123456789",
-        "validated_at": 1642701357084,
-    },
-    "org.matrix.msc3231.login.registration_token": "sometoken",  # User has registered through the flow described in MSC3231
-}
-```
-
-The second dictionary contains the parameters provided by the user's client in the request
-to `/_matrix/client/v3/register`. See the [Matrix specification](https://spec.matrix.org/latest/client-server-api/#post_matrixclientv3register)
-for a complete list of these parameters.
-
-If the module cannot, or does not wish to, generate a username for this user, it must
-return `None`.
-
-If multiple modules implement this callback, they will be considered in order. If a
-callback returns `None`, Synapse falls through to the next one. The value of the first
-callback that does not return `None` will be used. If this happens, Synapse will not call
-any of the subsequent implementations of this callback. If every callback return `None`,
-the username provided by the user is used, if any (otherwise one is automatically
-generated).
-
-
 ## Example
 
 The example module below implements authentication checkers for two different login types: 

docs/openid.md

@@ -390,6 +390,9 @@ oidc_providers:
 
 ### Facebook
 
+Like Github, Facebook provide a custom OAuth2 API rather than an OIDC-compliant
+one so requires a little more configuration.
+
 0. You will need a Facebook developer account. You can register for one
    [here](https://developers.facebook.com/async/registration/).
 1. On the [apps](https://developers.facebook.com/apps/) page of the developer
@@ -409,28 +412,24 @@ Synapse config:
     idp_name: Facebook
     idp_brand: "facebook"  # optional: styling hint for clients
     discover: false
-    issuer: "https://www.facebook.com"
+    issuer: "https://facebook.com"
     client_id: "your-client-id" # TO BE FILLED
     client_secret: "your-client-secret" # TO BE FILLED
     scopes: ["openid", "email"]
-    authorization_endpoint: "https://facebook.com/dialog/oauth"
-    token_endpoint: "https://graph.facebook.com/v9.0/oauth/access_token"
-    jwks_uri: "https://www.facebook.com/.well-known/oauth/openid/jwks/"
+    authorization_endpoint: https://facebook.com/dialog/oauth
+    token_endpoint: https://graph.facebook.com/v9.0/oauth/access_token
+    user_profile_method: "userinfo_endpoint"
+    userinfo_endpoint: "https://graph.facebook.com/v9.0/me?fields=id,name,email,picture"
     user_mapping_provider:
       config:
+        subject_claim: "id"
         display_name_template: "{{ user.name }}"
-        email_template: "{{ '{{ user.email }}' }}"
 ```
 
 Relevant documents:
- * [Manually Build a Login Flow](https://developers.facebook.com/docs/facebook-login/manually-build-a-login-flow)
- * [Using Facebook's Graph API](https://developers.facebook.com/docs/graph-api/using-graph-api/)
- * [Reference to the User endpoint](https://developers.facebook.com/docs/graph-api/reference/user)
-
-Facebook do have an [OIDC discovery endpoint](https://www.facebook.com/.well-known/openid-configuration),
-but it has a `response_types_supported` which excludes "code" (which we rely on, and
-is even mentioned in their [documentation](https://developers.facebook.com/docs/facebook-login/manually-build-a-login-flow#login)),
-so we have to disable discovery and configure the URIs manually.
+ * https://developers.facebook.com/docs/facebook-login/manually-build-a-login-flow
+ * Using Facebook's Graph API: https://developers.facebook.com/docs/graph-api/using-graph-api/
+ * Reference to the User endpoint: https://developers.facebook.com/docs/graph-api/reference/user
 
 ### Gitea
 

docs/postgres.md

@@ -1,6 +1,6 @@
 # Using Postgres
 
-Synapse supports PostgreSQL versions 10 or later.
+Synapse supports PostgreSQL versions 9.6 or later.
 
 ## Install postgres client libraries
 

docs/sample_config.yaml

@@ -41,11 +41,11 @@
 # documentation on how to configure or create custom modules for Synapse.
 #
 modules:
-  #- module: my_super_module.MySuperClass
-  #  config:
-  #    do_thing: true
-  #- module: my_other_super_module.SomeClass
-  #  config: {}
+    # - module: my_super_module.MySuperClass
+    #   config:
+    #       do_thing: true
+    # - module: my_other_super_module.SomeClass
+    #   config: {}
 
 
 ## Server ##
@@ -74,7 +74,13 @@ server_name: "SERVERNAME"
 #
 pid_file: DATADIR/homeserver.pid
 
-# The absolute URL to the web client which / will redirect to.
+# The absolute URL to the web client which /_matrix/client will redirect
+# to if 'webclient' is configured under the 'listeners' configuration.
+#
+# This option can be also set to the filesystem path to the web client
+# which will be served at /_matrix/client/ if 'webclient' is configured
+# under the 'listeners' configuration, however this is a security risk:
+# https://github.com/matrix-org/synapse#security-note
 #
 #web_client_location: https://riot.example.com/
 
@@ -158,7 +164,7 @@ presence:
 # The default room version for newly created rooms.
 #
 # Known room versions are listed here:
-# https://spec.matrix.org/latest/rooms/#complete-list-of-room-versions
+# https://matrix.org/docs/spec/#complete-list-of-room-versions
 #
 # For example, for room version 1, default_room_version should be set
 # to "1".
@@ -304,6 +310,8 @@ presence:
 #   static: static resources under synapse/static (/_matrix/static). (Mostly
 #       useful for 'fallback authentication'.)
 #
+#   webclient: A web client. Requires web_client_location to be set.
+#
 listeners:
   # TLS-enabled listener: for when matrix traffic is sent directly to synapse.
   #
@@ -471,20 +479,6 @@ limit_remote_rooms:
 #
 #allow_per_room_profiles: false
 
-# The largest allowed file size for a user avatar. Defaults to no restriction.
-#
-# Note that user avatar changes will not work if this is set without
-# using Synapse's media repository.
-#
-#max_avatar_size: 10M
-
-# The MIME types allowed for user avatars. Defaults to no restriction.
-#
-# Note that user avatar changes will not work if this is set without
-# using Synapse's media repository.
-#
-#allowed_avatar_mimetypes: ["image/png", "image/jpeg", "image/gif"]
-
 # How long to keep redacted events in unredacted form in the database. After
 # this period redacted events get replaced with their redacted form in the DB.
 #
@@ -1442,16 +1436,6 @@ account_threepid_delegates:
 #
 #auto_join_rooms_for_guests: false
 
-# Whether to inhibit errors raised when registering a new account if the user ID
-# already exists. If turned on, that requests to /register/available will always
-# show a user ID as available, and Synapse won't raise an error when starting
-# a registration with a user ID that already exists. However, Synapse will still
-# raise an error if the registration completes and the username conflicts.
-#
-# Defaults to false.
-#
-#inhibit_user_in_use_error: true
-
 
 ## Metrics ###
 
@@ -1519,21 +1503,6 @@ room_prejoin_state:
    #additional_event_types:
    #  - org.example.custom.event.type
 
-# We record the IP address of clients used to access the API for various
-# reasons, including displaying it to the user in the "Where you're signed in"
-# dialog.
-#
-# By default, when puppeting another user via the admin API, the client IP
-# address is recorded against the user who created the access token (ie, the
-# admin user), and *not* the puppeted user.
-#
-# Uncomment the following to also record the IP address against the puppeted
-# user. (This also means that the puppeted user will count as an "active" user
-# for the purpose of monthly active user tracking - see 'limit_usage_by_mau' etc
-# above.)
-#
-#track_puppeted_user_ips: true
-
 
 # A list of application service config files to use
 #
@@ -1901,13 +1870,10 @@ saml2_config:
 #       Defaults to false. Avoid this in production.
 #
 #   user_profile_method: Whether to fetch the user profile from the userinfo
-#       endpoint, or to rely on the data returned in the id_token from the
-#       token_endpoint.
+#       endpoint. Valid values are: 'auto' or 'userinfo_endpoint'.
 #
-#       Valid values are: 'auto' or 'userinfo_endpoint'.
-#
-#       Defaults to 'auto', which uses the userinfo endpoint if 'openid' is
-#       not included in 'scopes'. Set to 'userinfo_endpoint' to always use the
+#       Defaults to 'auto', which fetches the userinfo endpoint if 'openid' is
+#       included in 'scopes'. Set to 'userinfo_endpoint' to always fetch the
 #       userinfo endpoint.
 #
 #   allow_existing_users: set to 'true' to allow a user logging in via OIDC to

docs/setup/installation.md

@@ -194,7 +194,7 @@ When following this route please make sure that the [Platform-specific prerequis
 System requirements:
 
 - POSIX-compliant system (tested on Linux & OS X)
-- Python 3.7 or later, up to Python 3.10.
+- Python 3.6 or later, up to Python 3.9.
 - At least 1GB of free RAM if you want to join large public rooms like #matrix:matrix.org
 
 To install the Synapse homeserver run:

docs/turn-howto.md

@@ -137,10 +137,6 @@ This will install and start a systemd service called `coturn`.
 
     # TLS private key file
     pkey=/path/to/privkey.pem
-
-    # Ensure the configuration lines that disable TLS/DTLS are commented-out or removed
-    #no-tls
-    #no-dtls
     ```
 
     In this case, replace the `turn:` schemes in the `turn_uris` settings below
@@ -149,14 +145,6 @@ This will install and start a systemd service called `coturn`.
     We recommend that you only try to set up TLS/DTLS once you have set up a
     basic installation and got it working.
 
-    NB: If your TLS certificate was provided by Let's Encrypt, TLS/DTLS will
-    not work with any Matrix client that uses Chromium's WebRTC library. This
-    currently includes Element Android & iOS; for more details, see their
-    [respective](https://github.com/vector-im/element-android/issues/1533)
-    [issues](https://github.com/vector-im/element-ios/issues/2712) as well as the underlying
-    [WebRTC issue](https://bugs.chromium.org/p/webrtc/issues/detail?id=11710).
-    Consider using a ZeroSSL certificate for your TURN server as a working alternative.
-
 1.  Ensure your firewall allows traffic into the TURN server on the ports
     you've configured it to listen on (By default: 3478 and 5349 for TURN
     traffic (remember to allow both TCP and UDP traffic), and ports 49152-65535
@@ -262,10 +250,6 @@ Here are a few things to try:
  * Check that you have opened your firewall to allow UDP traffic to the UDP
    relay ports (49152-65535 by default).
 
- * Try disabling `coturn`'s TLS/DTLS listeners and enable only its (unencrypted)
-   TCP/UDP listeners. (This will only leave signaling traffic unencrypted;
-   voice & video WebRTC traffic is always encrypted.)
-
  * Some WebRTC implementations (notably, that of Google Chrome) appear to get
    confused by TURN servers which are reachable over IPv6 (this appears to be
    an unexpected side-effect of its handling of multiple IP addresses as

docs/upgrade.md

@@ -85,29 +85,6 @@ process, for example:
     dpkg -i matrix-synapse-py3_1.3.0+stretch1_amd64.deb
     ```
 
-# Upgrading to v1.52.0
-
-## Twisted security release
-
-Note that [Twisted 22.1.0](https://github.com/twisted/twisted/releases/tag/twisted-22.1.0)
-has recently been released, which fixes a [security issue](https://github.com/twisted/twisted/security/advisories/GHSA-92x2-jw7w-xvvx)
-within the Twisted library. We do not believe Synapse is affected by this vulnerability,
-though we advise server administrators who installed Synapse via pip to upgrade Twisted
-with `pip install --upgrade Twisted` as a matter of good practice. The Docker image
-`matrixdotorg/synapse` and the Debian packages from `packages.matrix.org` are using the
-updated library.
-
-# Upgrading to v1.51.0
-
-## Deprecation of `webclient` listeners and non-HTTP(S) `web_client_location`
-
-Listeners of type  `webclient` are deprecated and scheduled to be removed in
-Synapse v1.53.0.
-
-Similarly, a non-HTTP(S) `web_client_location` configuration is deprecated and
-will become a configuration error in Synapse v1.53.0.
-
-
 # Upgrading to v1.50.0
 
 ## Dropping support for old Python and Postgres versions

docs/usage/administration/admin_api/federation.md

@@ -86,7 +86,7 @@ The following fields are returned in the JSON response body:
 - `next_token`: string representing a positive integer - Indication for pagination. See above.
 - `total` - integer - Total number of destinations.
 
-## Destination Details API
+# Destination Details API
 
 This API gets the retry timing info for a specific remote server.
 
@@ -108,105 +108,7 @@ A response body like the following is returned:
 }
 ```
 
-**Parameters**
-
-The following parameters should be set in the URL:
-
-- `destination` - Name of the remote server.
-
 **Response**
 
 The response fields are the same like in the `destinations` array in
 [List of destinations](#list-of-destinations) response.
-
-## Destination rooms
-
-This API gets the rooms that federate with a specific remote server.
-
-The API is:
-
-```
-GET /_synapse/admin/v1/federation/destinations/<destination>/rooms
-```
-
-A response body like the following is returned:
-
-```json
-{
-   "rooms":[
-      {
-         "room_id": "!OGEhHVWSdvArJzumhm:matrix.org",
-         "stream_ordering": 8326
-      },
-      {
-         "room_id": "!xYvNcQPhnkrdUmYczI:matrix.org",
-         "stream_ordering": 93534
-      }
-   ],
-   "total": 2
-}
-```
-
-To paginate, check for `next_token` and if present, call the endpoint again
-with `from` set to the value of `next_token`. This will return a new page.
-
-If the endpoint does not return a `next_token` then there are no more destinations
-to paginate through.
-
-**Parameters**
-
-The following parameters should be set in the URL:
-
-- `destination` - Name of the remote server.
-
-The following query parameters are available:
-
-- `from` - Offset in the returned list. Defaults to `0`.
-- `limit` - Maximum amount of destinations to return. Defaults to `100`.
-- `dir` - Direction of room order by `room_id`. Either `f` for forwards or `b` for
-  backwards. Defaults to `f`.
-
-**Response**
-
-The following fields are returned in the JSON response body:
-
-- `rooms` - An array of objects, each containing information about a room.
-  Room objects contain the following fields:
-  - `room_id` - string - The ID of the room.
-  - `stream_ordering` - integer -  The stream ordering of the most recent
-    successfully-sent [PDU](understanding_synapse_through_grafana_graphs.md#federation)
-    to this destination in this room.
-- `next_token`: string representing a positive integer - Indication for pagination. See above.
-- `total` - integer - Total number of destinations.
-
-## Reset connection timeout
-
-Synapse makes federation requests to other homeservers. If a federation request fails,
-Synapse will mark the destination homeserver as offline, preventing any future requests
-to that server for a "cooldown" period. This period grows over time if the server
-continues to fail its responses
-([exponential backoff](https://en.wikipedia.org/wiki/Exponential_backoff)).
-
-Admins can cancel the cooldown period with this API.
-
-This API resets the retry timing for a specific remote server and tries to connect to
-the remote server again. It does not wait for the next `retry_interval`.
-The connection must have previously run into an error and `retry_last_ts`
-([Destination Details API](#destination-details-api)) must not be equal to `0`.
-
-The connection attempt is carried out in the background and can take a while
-even if the API already returns the http status 200.
-
-The API is:
-
-```
-POST /_synapse/admin/v1/federation/destinations/<destination>/reset_connection
-
-{}
-```
-
-**Parameters**
-
-The following parameters should be set in the URL:
-
-- `destination` - Name of the remote server.

docs/website_files/README.md

@@ -24,11 +24,6 @@ Finally, we also stylise the chapter titles in the left sidebar by indenting the
 slightly so that they are more visually distinguishable from the section headers
 (the bold titles). This is done through the `indent-section-headers.css` file.
 
-In addition to these modifications, we have added a version picker to the documentation.
-Users can switch between documentations for different versions of Synapse.
-This functionality was implemented through the `version-picker.js` and
-`version-picker.css` files.
-
 More information can be found in mdbook's official documentation for
 [injecting page JS/CSS](https://rust-lang.github.io/mdBook/format/config.html)
 and

docs/website_files/theme/index.hbs

@@ -131,18 +131,6 @@
                             <i class="fa fa-search"></i>
                         </button>
                         {{/if}}
-                        <div class="version-picker">
-                            <div class="dropdown">
-                                <div class="select">
-                                    <span></span>
-                                    <i class="fa fa-chevron-down"></i>
-                                </div>
-                                <input type="hidden" name="version">
-                                <ul class="dropdown-menu">
-                                    <!-- Versions will be added dynamically in version-picker.js -->
-                                </ul>
-                            </div>
-                        </div>      
                     </div>
 
                     <h1 class="menu-title">{{ book_title }}</h1>
@@ -321,4 +309,4 @@
         {{/if}}
 
     </body>
-</html>
+</html>

docs/website_files/version-picker.css

@@ -1,78 +0,0 @@
-.version-picker {
-    display: flex;
-    align-items: center;
-}
-
-.version-picker .dropdown {
-    width: 130px;
-    max-height: 29px;
-    margin-left: 10px;
-    display: inline-block;
-    border-radius: 4px;
-    border: 1px solid var(--theme-popup-border);
-    position: relative;
-    font-size: 13px;
-    color: var(--fg);
-    height: 100%;
-    text-align: left;
-}
-.version-picker .dropdown .select {
-    cursor: pointer;
-    display: block;
-    padding: 5px 2px 5px 15px;
-}
-.version-picker .dropdown .select > i {
-    font-size: 10px;
-    color: var(--fg);
-    cursor: pointer;
-    float: right;
-    line-height: 20px !important;
-}
-.version-picker .dropdown:hover {
-    border: 1px solid var(--theme-popup-border);
-}
-.version-picker .dropdown:active {
-    background-color: var(--theme-popup-bg);
-}
-.version-picker .dropdown.active:hover,
-.version-picker .dropdown.active {
-    border: 1px solid var(--theme-popup-border);
-    border-radius: 2px 2px 0 0;
-    background-color: var(--theme-popup-bg);
-}
-.version-picker .dropdown.active .select > i {
-    transform: rotate(-180deg);
-}
-.version-picker .dropdown .dropdown-menu {
-    position: absolute;
-    background-color: var(--theme-popup-bg);
-    width: 100%;
-    left: -1px;
-    right: 1px;
-    margin-top: 1px;
-    border: 1px solid var(--theme-popup-border);
-    border-radius: 0 0 4px 4px;
-    overflow: hidden;
-    display: none;
-    max-height: 300px;
-    overflow-y: auto;
-    z-index: 9;
-}
-.version-picker .dropdown .dropdown-menu li {
-    font-size: 12px;
-    padding: 6px 20px;
-    cursor: pointer;
-} 
-.version-picker .dropdown .dropdown-menu {
-    padding: 0;
-    list-style: none;
-}
-.version-picker .dropdown .dropdown-menu li:hover {
-    background-color: var(--theme-hover);
-}
-.version-picker .dropdown .dropdown-menu li.active::before {
-    display: inline-block;
-    content: "✓";
-    margin-inline-start: -14px;
-    width: 14px;
-}

docs/website_files/version-picker.js

@@ -1,127 +0,0 @@
-
-const dropdown = document.querySelector('.version-picker .dropdown');
-const dropdownMenu = dropdown.querySelector('.dropdown-menu');
-
-fetchVersions(dropdown, dropdownMenu).then(() => {
-    initializeVersionDropdown(dropdown, dropdownMenu);
-});
-
-/**
- * Initialize the dropdown functionality for version selection.
- * 
- * @param {Element} dropdown - The dropdown element.
- * @param {Element} dropdownMenu - The dropdown menu element.
- */
-function initializeVersionDropdown(dropdown, dropdownMenu) {
-    // Toggle the dropdown menu on click
-    dropdown.addEventListener('click', function () {
-        this.setAttribute('tabindex', 1);
-        this.classList.toggle('active');
-        dropdownMenu.style.display = (dropdownMenu.style.display === 'block') ? 'none' : 'block';
-    });
-  
-    // Remove the 'active' class and hide the dropdown menu on focusout
-    dropdown.addEventListener('focusout', function () {
-        this.classList.remove('active');
-        dropdownMenu.style.display = 'none';
-    });
-  
-    // Handle item selection within the dropdown menu
-    const dropdownMenuItems = dropdownMenu.querySelectorAll('li');    
-    dropdownMenuItems.forEach(function (item) {
-        item.addEventListener('click', function () {
-            dropdownMenuItems.forEach(function (item) {
-                item.classList.remove('active');
-            });
-            this.classList.add('active');
-            dropdown.querySelector('span').textContent = this.textContent;
-            dropdown.querySelector('input').value = this.getAttribute('id');
-
-            window.location.href = changeVersion(window.location.href, this.textContent);
-        });
-    });
-};
-
-/**
- * This function fetches the available versions from a GitHub repository
- * and inserts them into the version picker.
- * 
- * @param {Element} dropdown - The dropdown element.
- * @param {Element} dropdownMenu - The dropdown menu element.
- * @returns {Promise<Array<string>>} A promise that resolves with an array of available versions.
- */
-function fetchVersions(dropdown, dropdownMenu) {
-    return new Promise((resolve, reject) => {
-        window.addEventListener("load", () => {
-
-            fetch("https://api.github.com/repos/matrix-org/synapse/git/trees/gh-pages", {
-                cache: "force-cache",
-            }).then(res => 
-                res.json()
-            ).then(resObject => {
-                const excluded = ['dev-docs', 'v1.91.0', 'v1.80.0', 'v1.69.0'];
-                const tree = resObject.tree.filter(item => item.type === "tree" && !excluded.includes(item.path));
-                const versions = tree.map(item => item.path).sort(sortVersions);
-
-                // Create a list of <li> items for versions
-                versions.forEach((version) => {
-                    const li = document.createElement("li");
-                    li.textContent = version;
-                    li.id = version;
-    
-                    if (window.SYNAPSE_VERSION === version) {
-                        li.classList.add('active');
-                        dropdown.querySelector('span').textContent = version;
-                        dropdown.querySelector('input').value = version;
-                    }
-    
-                    dropdownMenu.appendChild(li);
-                });
-
-                resolve(versions);
-
-            }).catch(ex => {
-                console.error("Failed to fetch version data", ex);
-                reject(ex);
-            })
-        });
-    });
-}
-
-/**
- * Custom sorting function to sort an array of version strings.
- *
- * @param {string} a - The first version string to compare.
- * @param {string} b - The second version string to compare.
- * @returns {number} - A negative number if a should come before b, a positive number if b should come before a, or 0 if they are equal.
- */
-function sortVersions(a, b) {
-    // Put 'develop' and 'latest' at the top
-    if (a === 'develop' || a === 'latest') return -1;
-    if (b === 'develop' || b === 'latest') return 1;
-
-    const versionA = (a.match(/v\d+(\.\d+)+/) || [])[0];
-    const versionB = (b.match(/v\d+(\.\d+)+/) || [])[0];
-
-    return versionB.localeCompare(versionA);
-}
-
-/**
- * Change the version in a URL path.
- *
- * @param {string} url - The original URL to be modified.
- * @param {string} newVersion - The new version to replace the existing version in the URL.
- * @returns {string} The updated URL with the new version.
- */
-function changeVersion(url, newVersion) {
-    const parsedURL = new URL(url);
-    const pathSegments = parsedURL.pathname.split('/');
-  
-    // Modify the version
-    pathSegments[2] = newVersion;
-
-    // Reconstruct the URL
-    parsedURL.pathname = pathSegments.join('/');
-  
-    return parsedURL.href;
-}

docs/website_files/version.js

@@ -1 +0,0 @@
-window.SYNAPSE_VERSION = 'v1.52';

mypy.ini

@@ -77,6 +77,9 @@ exclude = (?x)
    |tests/push/test_http.py
    |tests/push/test_presentable_names.py
    |tests/push/test_push_rule_evaluator.py
+   |tests/rest/admin/test_admin.py
+   |tests/rest/admin/test_user.py
+   |tests/rest/admin/test_username_available.py
    |tests/rest/client/test_account.py
    |tests/rest/client/test_events.py
    |tests/rest/client/test_filter.py

scripts-dev/complement.sh

@@ -8,8 +8,7 @@
 # By default the script will fetch the latest Complement master branch and
 # run tests with that. This can be overridden to use a custom Complement
 # checkout by setting the COMPLEMENT_DIR environment variable to the
-# filepath of a local Complement checkout or by setting the COMPLEMENT_REF
-# environment variable to pull a different branch or commit.
+# filepath of a local Complement checkout.
 #
 # By default Synapse is run in monolith mode. This can be overridden by
 # setting the WORKERS environment variable.
@@ -24,20 +23,16 @@
 # Exit if a line returns a non-zero exit code
 set -e
 
-# enable buildkit for the docker builds
-export DOCKER_BUILDKIT=1
-
 # Change to the repository root
 cd "$(dirname $0)/.."
 
 # Check for a user-specified Complement checkout
 if [[ -z "$COMPLEMENT_DIR" ]]; then
-  COMPLEMENT_REF=${COMPLEMENT_REF:-master}
-  echo "COMPLEMENT_DIR not set. Fetching Complement checkout from ${COMPLEMENT_REF}..."
-  wget -Nq https://github.com/matrix-org/complement/archive/${COMPLEMENT_REF}.tar.gz
-  tar -xzf ${COMPLEMENT_REF}.tar.gz
-  COMPLEMENT_DIR=complement-${COMPLEMENT_REF}
-  echo "Checkout available at 'complement-${COMPLEMENT_REF}'"
+  echo "COMPLEMENT_DIR not set. Fetching the latest Complement checkout..."
+  wget -Nq https://github.com/matrix-org/complement/archive/master.tar.gz
+  tar -xzf master.tar.gz
+  COMPLEMENT_DIR=complement-master
+  echo "Checkout available at 'complement-master'"
 fi
 
 # Build the base Synapse image from the local checkout
@@ -52,7 +47,7 @@ if [[ -n "$WORKERS" ]]; then
   COMPLEMENT_DOCKERFILE=SynapseWorkers.Dockerfile
   # And provide some more configuration to complement.
   export COMPLEMENT_CA=true
-  export COMPLEMENT_SPAWN_HS_TIMEOUT_SECS=25
+  export COMPLEMENT_VERSION_CHECK_ITERATIONS=500
 else
   export COMPLEMENT_BASE_IMAGE=complement-synapse
   COMPLEMENT_DOCKERFILE=Synapse.Dockerfile
@@ -70,5 +65,4 @@ if [[ -n "$1" ]]; then
 fi
 
 # Run the tests!
-echo "Images built; running complement"
 go test -v -tags synapse_blacklist,msc2403 -count=1 $EXTRA_COMPLEMENT_ARGS ./tests/...

setup.py

@@ -150,7 +150,7 @@ setup(
     zip_safe=False,
     long_description=long_description,
     long_description_content_type="text/x-rst",
-    python_requires="~=3.7",
+    python_requires="~=3.6",
     entry_points={
         "console_scripts": [
             "synapse_homeserver = synapse.app.homeserver:main",

synapse/__init__.py

@@ -21,8 +21,8 @@ import os
 import sys
 
 # Check that we're not running on an unsupported Python version.
-if sys.version_info < (3, 7):
-    print("Synapse requires Python 3.7 or above.")
+if sys.version_info < (3, 6):
+    print("Synapse requires Python 3.6 or above.")
     sys.exit(1)
 
 # Twisted and canonicaljson will fail to import when this file is executed to
@@ -47,7 +47,7 @@ try:
 except ImportError:
     pass
 
-__version__ = "1.52.0"
+__version__ = "1.50.0rc1"
 
 if bool(os.environ.get("SYNAPSE_TEST_PATCH_LOG_CONTEXTS", False)):
     # We import here so that we don't have to install a bunch of deps when

synapse/_scripts/review_recent_signups.py

@@ -46,9 +46,7 @@ class UserInfo:
     ips: List[str] = attr.Factory(list)
 
 
-def get_recent_users(
-    txn: LoggingTransaction, since_ms: int, exclude_app_service: bool
-) -> List[UserInfo]:
+def get_recent_users(txn: LoggingTransaction, since_ms: int) -> List[UserInfo]:
     """Fetches recently registered users and some info on them."""
 
     sql = """
@@ -58,9 +56,6 @@ def get_recent_users(
             AND deactivated = 0
     """
 
-    if exclude_app_service:
-        sql += " AND appservice_id IS NULL"
-
     txn.execute(sql, (since_ms / 1000,))
 
     user_infos = [UserInfo(user_id, creation_ts) for user_id, creation_ts in txn]
@@ -118,7 +113,7 @@ def main() -> None:
         "-e",
         "--exclude-emails",
         action="store_true",
-        help="Exclude users that have validated email addresses.",
+        help="Exclude users that have validated email addresses",
     )
     parser.add_argument(
         "-u",
@@ -126,12 +121,6 @@ def main() -> None:
         action="store_true",
         help="Only print user IDs that match.",
     )
-    parser.add_argument(
-        "-a",
-        "--exclude-app-service",
-        help="Exclude appservice users.",
-        action="store_true",
-    )
 
     config = ReviewConfig()
 
@@ -144,7 +133,6 @@ def main() -> None:
 
     since_ms = time.time() * 1000 - Config.parse_duration(config_args.since)
     exclude_users_with_email = config_args.exclude_emails
-    exclude_users_with_appservice = config_args.exclude_app_service
     include_context = not config_args.only_users
 
     for database_config in config.database.databases:
@@ -155,7 +143,7 @@ def main() -> None:
 
     with make_conn(database_config, engine, "review_recent_signups") as db_conn:
         # This generates a type of Cursor, not LoggingTransaction.
-        user_infos = get_recent_users(db_conn.cursor(), since_ms, exclude_users_with_appservice)  # type: ignore[arg-type]
+        user_infos = get_recent_users(db_conn.cursor(), since_ms)  # type: ignore[arg-type]
 
     for user_info in user_infos:
         if exclude_users_with_email and user_info.emails:

synapse/api/auth.py

@@ -71,7 +71,6 @@ class Auth:
         self._auth_blocking = AuthBlocking(self.hs)
 
         self._track_appservice_user_ips = hs.config.appservice.track_appservice_user_ips
-        self._track_puppeted_user_ips = hs.config.api.track_puppeted_user_ips
         self._macaroon_secret_key = hs.config.key.macaroon_secret_key
         self._force_tracing_for_users = hs.config.tracing.force_tracing_for_users
 
@@ -247,18 +246,6 @@ class Auth:
                     user_agent=user_agent,
                     device_id=device_id,
                 )
-                # Track also the puppeted user client IP if enabled and the user is puppeting
-                if (
-                    user_info.user_id != user_info.token_owner
-                    and self._track_puppeted_user_ips
-                ):
-                    await self.store.insert_client_ip(
-                        user_id=user_info.user_id,
-                        access_token=access_token,
-                        ip=ip_addr,
-                        user_agent=user_agent,
-                        device_id=device_id,
-                    )
 
             if is_guest and not allow_guest:
                 raise AuthError(

synapse/api/room_versions.py

@@ -46,41 +46,41 @@ class RoomDisposition:
     UNSTABLE = "unstable"
 
 
-@attr.s(slots=True, frozen=True, auto_attribs=True)
+@attr.s(slots=True, frozen=True)
 class RoomVersion:
     """An object which describes the unique attributes of a room version."""
 
-    identifier: str  # the identifier for this version
-    disposition: str  # one of the RoomDispositions
-    event_format: int  # one of the EventFormatVersions
-    state_res: int  # one of the StateResolutionVersions
-    enforce_key_validity: bool
+    identifier = attr.ib(type=str)  # the identifier for this version
+    disposition = attr.ib(type=str)  # one of the RoomDispositions
+    event_format = attr.ib(type=int)  # one of the EventFormatVersions
+    state_res = attr.ib(type=int)  # one of the StateResolutionVersions
+    enforce_key_validity = attr.ib(type=bool)
 
     # Before MSC2432, m.room.aliases had special auth rules and redaction rules
-    special_case_aliases_auth: bool
+    special_case_aliases_auth = attr.ib(type=bool)
     # Strictly enforce canonicaljson, do not allow:
     # * Integers outside the range of [-2 ^ 53 + 1, 2 ^ 53 - 1]
     # * Floats
     # * NaN, Infinity, -Infinity
-    strict_canonicaljson: bool
+    strict_canonicaljson = attr.ib(type=bool)
     # MSC2209: Check 'notifications' key while verifying
     # m.room.power_levels auth rules.
-    limit_notifications_power_levels: bool
+    limit_notifications_power_levels = attr.ib(type=bool)
     # MSC2174/MSC2176: Apply updated redaction rules algorithm.
-    msc2176_redaction_rules: bool
+    msc2176_redaction_rules = attr.ib(type=bool)
     # MSC3083: Support the 'restricted' join_rule.
-    msc3083_join_rules: bool
+    msc3083_join_rules = attr.ib(type=bool)
     # MSC3375: Support for the proper redaction rules for MSC3083. This mustn't
     #          be enabled if MSC3083 is not.
-    msc3375_redaction_rules: bool
+    msc3375_redaction_rules = attr.ib(type=bool)
     # MSC2403: Allows join_rules to be set to 'knock', changes auth rules to allow sending
     # m.room.membership event with membership 'knock'.
-    msc2403_knocking: bool
+    msc2403_knocking = attr.ib(type=bool)
     # MSC2716: Adds m.room.power_levels -> content.historical field to control
     # whether "insertion", "chunk", "marker" events can be sent
-    msc2716_historical: bool
+    msc2716_historical = attr.ib(type=bool)
     # MSC2716: Adds support for redacting "insertion", "chunk", and "marker" events
-    msc2716_redactions: bool
+    msc2716_redactions = attr.ib(type=bool)
 
 
 class RoomVersions:

synapse/app/_base.py

@@ -16,6 +16,7 @@ import atexit
 import gc
 import logging
 import os
+import platform
 import signal
 import socket
 import sys
@@ -59,7 +60,7 @@ from synapse.events.spamcheck import load_legacy_spam_checkers
 from synapse.events.third_party_rules import load_legacy_third_party_event_rules
 from synapse.handlers.auth import load_legacy_password_auth_providers
 from synapse.logging.context import PreserveLoggingContext
-from synapse.metrics import install_gc_manager, register_threadpool
+from synapse.metrics import register_threadpool
 from synapse.metrics.background_process_metrics import wrap_as_background_process
 from synapse.metrics.jemalloc import setup_jemalloc_stats
 from synapse.types import ISynapseReactor
@@ -158,7 +159,6 @@ def start_reactor(
         change_resource_limit(soft_file_limit)
         if gc_thresholds:
             gc.set_threshold(*gc_thresholds)
-        install_gc_manager()
         run_command()
 
     # make sure that we run the reactor with the sentinel log context,
@@ -435,8 +435,7 @@ async def start(hs: "HomeServer") -> None:
     # before we start the listeners.
     module_api = hs.get_module_api()
     for module, config in hs.config.modules.loaded_modules:
-        m = module(config=config, api=module_api)
-        logger.info("Loaded module %s", m)
+        module(config=config, api=module_api)
 
     load_legacy_spam_checkers(hs)
     load_legacy_third_party_event_rules(hs)
@@ -468,13 +467,15 @@ async def start(hs: "HomeServer") -> None:
     # everything currently allocated are things that will be used for the
     # rest of time. Doing so means less work each GC (hopefully).
     #
-    # PyPy does not (yet?) implement gc.freeze()
-    if hasattr(gc, "freeze"):
+    # This only works on Python 3.7
+    if platform.python_implementation() == "CPython" and sys.version_info >= (3, 7):
         gc.collect()
         gc.freeze()
 
-        # Speed up shutdowns by freezing all allocated objects. This moves everything
-        # into the permanent generation and excludes them from the final GC.
+    # Speed up shutdowns by freezing all allocated objects. This moves everything
+    # into the permanent generation and excludes them from the final GC.
+    # Unfortunately only works on Python 3.7
+    if platform.python_implementation() == "CPython" and sys.version_info >= (3, 7):
         atexit.register(gc.freeze)
 
 

synapse/app/homeserver.py

@@ -27,6 +27,7 @@ import synapse
 import synapse.config.logger
 from synapse import events
 from synapse.api.urls import (
+    CLIENT_API_PREFIX,
     FEDERATION_PREFIX,
     LEGACY_MEDIA_PREFIX,
     MEDIA_R0_PREFIX,
@@ -131,18 +132,9 @@ class SynapseHomeServer(HomeServer):
         resources.update(self._module_web_resources)
         self._module_web_resources_consumed = True
 
-        # Try to find something useful to serve at '/':
-        #
-        # 1. Redirect to the web client if it is an HTTP(S) URL.
-        # 2. Redirect to the web client served via Synapse.
-        # 3. Redirect to the static "Synapse is running" page.
-        # 4. Do not redirect and use a blank resource.
-        if self.config.server.web_client_location_is_redirect:
-            root_resource: Resource = RootOptionsRedirectResource(
-                self.config.server.web_client_location
-            )
-        elif WEB_CLIENT_PREFIX in resources:
-            root_resource = RootOptionsRedirectResource(WEB_CLIENT_PREFIX)
+        # try to find something useful to redirect '/' to
+        if WEB_CLIENT_PREFIX in resources:
+            root_resource: Resource = RootOptionsRedirectResource(WEB_CLIENT_PREFIX)
         elif STATIC_PREFIX in resources:
             root_resource = RootOptionsRedirectResource(STATIC_PREFIX)
         else:
@@ -201,13 +193,7 @@ class SynapseHomeServer(HomeServer):
 
             resources.update(
                 {
-                    "/_matrix/client/api/v1": client_resource,
-                    "/_matrix/client/r0": client_resource,
-                    "/_matrix/client/v1": client_resource,
-                    "/_matrix/client/v3": client_resource,
-                    "/_matrix/client/unstable": client_resource,
-                    "/_matrix/client/v2_alpha": client_resource,
-                    "/_matrix/client/versions": client_resource,
+                    CLIENT_API_PREFIX: client_resource,
                     "/.well-known": well_known_resource(self),
                     "/_synapse/admin": AdminRestResource(self),
                     **build_synapse_client_resource_tree(self),
@@ -271,15 +257,15 @@ class SynapseHomeServer(HomeServer):
             resources[SERVER_KEY_V2_PREFIX] = KeyApiV2Resource(self)
 
         if name == "webclient":
-            # webclient listeners are deprecated as of Synapse v1.51.0, remove it
-            # in > v1.53.0.
             webclient_loc = self.config.server.web_client_location
 
             if webclient_loc is None:
                 logger.warning(
                     "Not enabling webclient resource, as web_client_location is unset."
                 )
-            elif self.config.server.web_client_location_is_redirect:
+            elif webclient_loc.startswith("http://") or webclient_loc.startswith(
+                "https://"
+            ):
                 resources[WEB_CLIENT_PREFIX] = RootRedirect(webclient_loc)
             else:
                 logger.warning(

synapse/config/api.py

@@ -29,7 +29,6 @@ class ApiConfig(Config):
     def read_config(self, config: JsonDict, **kwargs):
         validate_config(_MAIN_SCHEMA, config, ())
         self.room_prejoin_state = list(self._get_prejoin_state_types(config))
-        self.track_puppeted_user_ips = config.get("track_puppeted_user_ips", False)
 
     def generate_config_section(cls, **kwargs) -> str:
         formatted_default_state_types = "\n".join(
@@ -60,21 +59,6 @@ class ApiConfig(Config):
            #
            #additional_event_types:
            #  - org.example.custom.event.type
-
-        # We record the IP address of clients used to access the API for various
-        # reasons, including displaying it to the user in the "Where you're signed in"
-        # dialog.
-        #
-        # By default, when puppeting another user via the admin API, the client IP
-        # address is recorded against the user who created the access token (ie, the
-        # admin user), and *not* the puppeted user.
-        #
-        # Uncomment the following to also record the IP address against the puppeted
-        # user. (This also means that the puppeted user will count as an "active" user
-        # for the purpose of monthly active user tracking - see 'limit_usage_by_mau' etc
-        # above.)
-        #
-        #track_puppeted_user_ips: true
         """ % {
             "formatted_default_state_types": formatted_default_state_types
         }
@@ -154,8 +138,5 @@ _MAIN_SCHEMA = {
     "properties": {
         "room_prejoin_state": _ROOM_PREJOIN_STATE_CONFIG_SCHEMA,
         "room_invite_state_types": _ROOM_INVITE_STATE_TYPES_SCHEMA,
-        "track_puppeted_user_ips": {
-            "type": "boolean",
-        },
     },
 }

synapse/config/emailconfig.py

@@ -55,19 +55,19 @@ https://matrix-org.github.io/synapse/latest/templates.html
 ---------------------------------------------------------------------------------------"""
 
 
-@attr.s(slots=True, frozen=True, auto_attribs=True)
+@attr.s(slots=True, frozen=True)
 class EmailSubjectConfig:
-    message_from_person_in_room: str
-    message_from_person: str
-    messages_from_person: str
-    messages_in_room: str
-    messages_in_room_and_others: str
-    messages_from_person_and_others: str
-    invite_from_person: str
-    invite_from_person_to_room: str
-    invite_from_person_to_space: str
-    password_reset: str
-    email_validation: str
+    message_from_person_in_room = attr.ib(type=str)
+    message_from_person = attr.ib(type=str)
+    messages_from_person = attr.ib(type=str)
+    messages_in_room = attr.ib(type=str)
+    messages_in_room_and_others = attr.ib(type=str)
+    messages_from_person_and_others = attr.ib(type=str)
+    invite_from_person = attr.ib(type=str)
+    invite_from_person_to_room = attr.ib(type=str)
+    invite_from_person_to_space = attr.ib(type=str)
+    password_reset = attr.ib(type=str)
+    email_validation = attr.ib(type=str)
 
 
 class EmailConfig(Config):

synapse/config/experimental.py

@@ -24,6 +24,8 @@ class ExperimentalConfig(Config):
     def read_config(self, config: JsonDict, **kwargs):
         experimental = config.get("experimental_features") or {}
 
+        # Whether to enable experimental MSC1849 (aka relations) support
+        self.msc1849_enabled = config.get("experimental_msc1849_support_enabled", True)
         # MSC3440 (thread relation)
         self.msc3440_enabled: bool = experimental.get("msc3440_enabled", False)
 

synapse/config/modules.py

@@ -41,9 +41,9 @@ class ModulesConfig(Config):
             # documentation on how to configure or create custom modules for Synapse.
             #
             modules:
-              #- module: my_super_module.MySuperClass
-              #  config:
-              #    do_thing: true
-              #- module: my_other_super_module.SomeClass
-              #  config: {}
+                # - module: my_super_module.MySuperClass
+                #   config:
+                #       do_thing: true
+                # - module: my_other_super_module.SomeClass
+                #   config: {}
             """

synapse/config/oidc.py

@@ -148,13 +148,10 @@ class OIDCConfig(Config):
         #       Defaults to false. Avoid this in production.
         #
         #   user_profile_method: Whether to fetch the user profile from the userinfo
-        #       endpoint, or to rely on the data returned in the id_token from the
-        #       token_endpoint.
+        #       endpoint. Valid values are: 'auto' or 'userinfo_endpoint'.
         #
-        #       Valid values are: 'auto' or 'userinfo_endpoint'.
-        #
-        #       Defaults to 'auto', which uses the userinfo endpoint if 'openid' is
-        #       not included in 'scopes'. Set to 'userinfo_endpoint' to always use the
+        #       Defaults to 'auto', which fetches the userinfo endpoint if 'openid' is
+        #       included in 'scopes'. Set to 'userinfo_endpoint' to always fetch the
         #       userinfo endpoint.
         #
         #   allow_existing_users: set to 'true' to allow a user logging in via OIDC to

synapse/config/registration.py

@@ -190,8 +190,6 @@ class RegistrationConfig(Config):
         # The success template used during fallback auth.
         self.fallback_success_template = self.read_template("auth_success.html")
 
-        self.inhibit_user_in_use_error = config.get("inhibit_user_in_use_error", False)
-
     def generate_config_section(self, generate_secrets=False, **kwargs):
         if generate_secrets:
             registration_shared_secret = 'registration_shared_secret: "%s"' % (
@@ -448,16 +446,6 @@ class RegistrationConfig(Config):
         # Defaults to true.
         #
         #auto_join_rooms_for_guests: false
-
-        # Whether to inhibit errors raised when registering a new account if the user ID
-        # already exists. If turned on, that requests to /register/available will always
-        # show a user ID as available, and Synapse won't raise an error when starting
-        # a registration with a user ID that already exists. However, Synapse will still
-        # raise an error if the registration completes and the username conflicts.
-        #
-        # Defaults to false.
-        #
-        #inhibit_user_in_use_error: true
         """
             % locals()
         )

synapse/config/server.py

@@ -200,8 +200,8 @@ class HttpListenerConfig:
     """Object describing the http-specific parts of the config of a listener"""
 
     x_forwarded: bool = False
-    resources: List[HttpResourceConfig] = attr.Factory(list)
-    additional_resources: Dict[str, dict] = attr.Factory(dict)
+    resources: List[HttpResourceConfig] = attr.ib(factory=list)
+    additional_resources: Dict[str, dict] = attr.ib(factory=dict)
     tag: Optional[str] = None
 
 
@@ -259,6 +259,7 @@ class ServerConfig(Config):
             raise ConfigError(str(e))
 
         self.pid_file = self.abspath(config.get("pid_file"))
+        self.web_client_location = config.get("web_client_location", None)
         self.soft_file_limit = config.get("soft_file_limit", 0)
         self.daemonize = config.get("daemonize")
         self.print_pidfile = config.get("print_pidfile")
@@ -489,19 +490,6 @@ class ServerConfig(Config):
         # events with profile information that differ from the target's global profile.
         self.allow_per_room_profiles = config.get("allow_per_room_profiles", True)
 
-        # The maximum size an avatar can have, in bytes.
-        self.max_avatar_size = config.get("max_avatar_size")
-        if self.max_avatar_size is not None:
-            self.max_avatar_size = self.parse_size(self.max_avatar_size)
-
-        # The MIME types allowed for an avatar.
-        self.allowed_avatar_mimetypes = config.get("allowed_avatar_mimetypes")
-        if self.allowed_avatar_mimetypes and not isinstance(
-            self.allowed_avatar_mimetypes,
-            list,
-        ):
-            raise ConfigError("allowed_avatar_mimetypes must be a list")
-
         self.listeners = [parse_listener_def(x) for x in config.get("listeners", [])]
 
         # no_tls is not really supported any more, but let's grandfather it in
@@ -518,17 +506,8 @@ class ServerConfig(Config):
                     l2.append(listener)
             self.listeners = l2
 
-        self.web_client_location = config.get("web_client_location", None)
-        self.web_client_location_is_redirect = self.web_client_location and (
-            self.web_client_location.startswith("http://")
-            or self.web_client_location.startswith("https://")
-        )
-        # A non-HTTP(S) web client location is deprecated.
-        if self.web_client_location and not self.web_client_location_is_redirect:
-            logger.warning(NO_MORE_NONE_HTTP_WEB_CLIENT_LOCATION_WARNING)
-
-        # Warn if webclient is configured for a worker.
-        _warn_if_webclient_configured(self.listeners)
+        if not self.web_client_location:
+            _warn_if_webclient_configured(self.listeners)
 
         self.gc_thresholds = read_gc_thresholds(config.get("gc_thresholds", None))
         self.gc_seconds = self.read_gc_intervals(config.get("gc_min_interval", None))
@@ -814,7 +793,13 @@ class ServerConfig(Config):
         #
         pid_file: %(pid_file)s
 
-        # The absolute URL to the web client which / will redirect to.
+        # The absolute URL to the web client which /_matrix/client will redirect
+        # to if 'webclient' is configured under the 'listeners' configuration.
+        #
+        # This option can be also set to the filesystem path to the web client
+        # which will be served at /_matrix/client/ if 'webclient' is configured
+        # under the 'listeners' configuration, however this is a security risk:
+        # https://github.com/matrix-org/synapse#security-note
         #
         #web_client_location: https://riot.example.com/
 
@@ -898,7 +883,7 @@ class ServerConfig(Config):
         # The default room version for newly created rooms.
         #
         # Known room versions are listed here:
-        # https://spec.matrix.org/latest/rooms/#complete-list-of-room-versions
+        # https://matrix.org/docs/spec/#complete-list-of-room-versions
         #
         # For example, for room version 1, default_room_version should be set
         # to "1".
@@ -1026,6 +1011,8 @@ class ServerConfig(Config):
         #   static: static resources under synapse/static (/_matrix/static). (Mostly
         #       useful for 'fallback authentication'.)
         #
+        #   webclient: A web client. Requires web_client_location to be set.
+        #
         listeners:
           # TLS-enabled listener: for when matrix traffic is sent directly to synapse.
           #
@@ -1181,20 +1168,6 @@ class ServerConfig(Config):
         #
         #allow_per_room_profiles: false
 
-        # The largest allowed file size for a user avatar. Defaults to no restriction.
-        #
-        # Note that user avatar changes will not work if this is set without
-        # using Synapse's media repository.
-        #
-        #max_avatar_size: 10M
-
-        # The MIME types allowed for user avatars. Defaults to no restriction.
-        #
-        # Note that user avatar changes will not work if this is set without
-        # using Synapse's media repository.
-        #
-        #allowed_avatar_mimetypes: ["image/png", "image/jpeg", "image/gif"]
-
         # How long to keep redacted events in unredacted form in the database. After
         # this period redacted events get replaced with their redacted form in the DB.
         #
@@ -1376,15 +1349,9 @@ def parse_listener_def(listener: Any) -> ListenerConfig:
     return ListenerConfig(port, bind_addresses, listener_type, tls, http_config)
 
 
-NO_MORE_NONE_HTTP_WEB_CLIENT_LOCATION_WARNING = """
-Synapse no longer supports serving a web client. To remove this warning,
-configure 'web_client_location' with an HTTP(S) URL.
-"""
-
-
 NO_MORE_WEB_CLIENT_WARNING = """
-Synapse no longer includes a web client. To redirect the root resource to a web client, configure
-'web_client_location'. To remove this warning, remove 'webclient' from the 'listeners'
+Synapse no longer includes a web client. To enable a web client, configure
+web_client_location. To remove this warning, remove 'webclient' from the 'listeners'
 configuration.
 """
 

synapse/config/workers.py

@@ -51,12 +51,12 @@ def _instance_to_list_converter(obj: Union[str, List[str]]) -> List[str]:
     return obj
 
 
-@attr.s(auto_attribs=True)
+@attr.s
 class InstanceLocationConfig:
     """The host and port to talk to an instance via HTTP replication."""
 
-    host: str
-    port: int
+    host = attr.ib(type=str)
+    port = attr.ib(type=int)
 
 
 @attr.s
@@ -77,28 +77,34 @@ class WriterLocations:
             can only be a single instance.
     """
 
-    events: List[str] = attr.ib(
+    events = attr.ib(
         default=["master"],
+        type=List[str],
         converter=_instance_to_list_converter,
     )
-    typing: List[str] = attr.ib(
+    typing = attr.ib(
         default=["master"],
+        type=List[str],
         converter=_instance_to_list_converter,
     )
-    to_device: List[str] = attr.ib(
+    to_device = attr.ib(
         default=["master"],
+        type=List[str],
         converter=_instance_to_list_converter,
     )
-    account_data: List[str] = attr.ib(
+    account_data = attr.ib(
         default=["master"],
+        type=List[str],
         converter=_instance_to_list_converter,
     )
-    receipts: List[str] = attr.ib(
+    receipts = attr.ib(
         default=["master"],
+        type=List[str],
         converter=_instance_to_list_converter,
     )
-    presence: List[str] = attr.ib(
+    presence = attr.ib(
         default=["master"],
+        type=List[str],
         converter=_instance_to_list_converter,
     )
 

synapse/crypto/keyring.py

@@ -58,7 +58,7 @@ if TYPE_CHECKING:
 logger = logging.getLogger(__name__)
 
 
-@attr.s(slots=True, frozen=True, cmp=False, auto_attribs=True)
+@attr.s(slots=True, cmp=False)
 class VerifyJsonRequest:
     """
     A request to verify a JSON object.
@@ -78,10 +78,10 @@ class VerifyJsonRequest:
         key_ids: The set of key_ids to that could be used to verify the JSON object
     """
 
-    server_name: str
-    get_json_object: Callable[[], JsonDict]
-    minimum_valid_until_ts: int
-    key_ids: List[str]
+    server_name = attr.ib(type=str)
+    get_json_object = attr.ib(type=Callable[[], JsonDict])
+    minimum_valid_until_ts = attr.ib(type=int)
+    key_ids = attr.ib(type=List[str])
 
     @staticmethod
     def from_json_object(
@@ -124,7 +124,7 @@ class KeyLookupError(ValueError):
     pass
 
 
-@attr.s(slots=True, frozen=True, auto_attribs=True)
+@attr.s(slots=True)
 class _FetchKeyRequest:
     """A request for keys for a given server.
 
@@ -138,9 +138,9 @@ class _FetchKeyRequest:
         key_ids: The IDs of the keys to attempt to fetch
     """
 
-    server_name: str
-    minimum_valid_until_ts: int
-    key_ids: List[str]
+    server_name = attr.ib(type=str)
+    minimum_valid_until_ts = attr.ib(type=int)
+    key_ids = attr.ib(type=List[str])
 
 
 class Keyring:

synapse/events/__init__.py

@@ -315,11 +315,10 @@ class EventBase(metaclass=abc.ABCMeta):
     redacts: DefaultDictProperty[Optional[str]] = DefaultDictProperty("redacts", None)
     room_id: DictProperty[str] = DictProperty("room_id")
     sender: DictProperty[str] = DictProperty("sender")
-    # TODO state_key should be Optional[str]. This is generally asserted in Synapse
-    # by calling is_state() first (which ensures it is not None), but it is hard (not possible?)
+    # TODO state_key should be Optional[str], this is generally asserted in Synapse
+    # by calling is_state() first (which ensures this), but it is hard (not possible?)
     # to properly annotate that calling is_state() asserts that state_key exists
-    # and is non-None. It would be better to replace such direct references with
-    # get_state_key() (and a check for None).
+    # and is non-None.
     state_key: DictProperty[str] = DictProperty("state_key")
     type: DictProperty[str] = DictProperty("type")
     user_id: DictProperty[str] = DictProperty("sender")
@@ -333,11 +332,7 @@ class EventBase(metaclass=abc.ABCMeta):
         return self.content["membership"]
 
     def is_state(self) -> bool:
-        return self.get_state_key() is not None
-
-    def get_state_key(self) -> Optional[str]:
-        """Get the state key of this event, or None if it's not a state event"""
-        return self._dict.get("state_key")
+        return hasattr(self, "state_key") and self.state_key is not None
 
     def get_dict(self) -> JsonDict:
         d = dict(self._dict)

synapse/events/snapshot.py

@@ -28,7 +28,7 @@ if TYPE_CHECKING:
     from synapse.storage.databases.main import DataStore
 
 
-@attr.s(slots=True, auto_attribs=True)
+@attr.s(slots=True)
 class EventContext:
     """
     Holds information relevant to persisting an event
@@ -103,15 +103,15 @@ class EventContext:
             accessed via get_prev_state_ids.
     """
 
-    rejected: Union[bool, str] = False
-    _state_group: Optional[int] = None
-    state_group_before_event: Optional[int] = None
-    prev_group: Optional[int] = None
-    delta_ids: Optional[StateMap[str]] = None
-    app_service: Optional[ApplicationService] = None
+    rejected = attr.ib(default=False, type=Union[bool, str])
+    _state_group = attr.ib(default=None, type=Optional[int])
+    state_group_before_event = attr.ib(default=None, type=Optional[int])
+    prev_group = attr.ib(default=None, type=Optional[int])
+    delta_ids = attr.ib(default=None, type=Optional[StateMap[str]])
+    app_service = attr.ib(default=None, type=Optional[ApplicationService])
 
-    _current_state_ids: Optional[StateMap[str]] = None
-    _prev_state_ids: Optional[StateMap[str]] = None
+    _current_state_ids = attr.ib(default=None, type=Optional[StateMap[str]])
+    _prev_state_ids = attr.ib(default=None, type=Optional[StateMap[str]])
 
     @staticmethod
     def with_state(
@@ -163,7 +163,7 @@ class EventContext:
         return {
             "prev_state_id": prev_state_id,
             "event_type": event.type,
-            "event_state_key": event.get_state_key(),
+            "event_state_key": event.state_key if event.is_state() else None,
             "state_group": self._state_group,
             "state_group_before_event": self.state_group_before_event,
             "rejected": self.rejected,

synapse/events/utils.py

@@ -32,13 +32,13 @@ from synapse.api.constants import EventContentFields, EventTypes, RelationTypes
 from synapse.api.errors import Codes, SynapseError
 from synapse.api.room_versions import RoomVersion
 from synapse.types import JsonDict
+from synapse.util.async_helpers import yieldable_gather_results
 from synapse.util.frozenutils import unfreeze
 
 from . import EventBase
 
 if TYPE_CHECKING:
-    from synapse.storage.databases.main.relations import BundledAggregations
-
+    from synapse.server import HomeServer
 
 # Split strings on "." but not "\." This uses a negative lookbehind assertion for '\'
 # (?<!stuff) matches if the current position in the string is not preceded
@@ -385,12 +385,17 @@ class EventClientSerializer:
     clients.
     """
 
-    def serialize_event(
+    def __init__(self, hs: "HomeServer"):
+        self.store = hs.get_datastore()
+        self._msc1849_enabled = hs.config.experimental.msc1849_enabled
+        self._msc3440_enabled = hs.config.experimental.msc3440_enabled
+
+    async def serialize_event(
         self,
         event: Union[JsonDict, EventBase],
         time_now: int,
         *,
-        bundle_aggregations: Optional[Dict[str, "BundledAggregations"]] = None,
+        bundle_aggregations: bool = False,
         **kwargs: Any,
     ) -> JsonDict:
         """Serializes a single event.
@@ -413,46 +418,66 @@ class EventClientSerializer:
         serialized_event = serialize_event(event, time_now, **kwargs)
 
         # Check if there are any bundled aggregations to include with the event.
-        if bundle_aggregations:
-            event_aggregations = bundle_aggregations.get(event.event_id)
-            if event_aggregations:
-                self._inject_bundled_aggregations(
-                    event,
-                    time_now,
-                    bundle_aggregations[event.event_id],
-                    serialized_event,
-                )
+        #
+        # Do not bundle aggregations if any of the following at true:
+        #
+        # * Support is disabled via the configuration or the caller.
+        # * The event is a state event.
+        # * The event has been redacted.
+        if (
+            self._msc1849_enabled
+            and bundle_aggregations
+            and not event.is_state()
+            and not event.internal_metadata.is_redacted()
+        ):
+            await self._injected_bundled_aggregations(event, time_now, serialized_event)
 
         return serialized_event
 
-    def _inject_bundled_aggregations(
-        self,
-        event: EventBase,
-        time_now: int,
-        aggregations: "BundledAggregations",
-        serialized_event: JsonDict,
+    async def _injected_bundled_aggregations(
+        self, event: EventBase, time_now: int, serialized_event: JsonDict
     ) -> None:
         """Potentially injects bundled aggregations into the unsigned portion of the serialized event.
 
         Args:
             event: The event being serialized.
             time_now: The current time in milliseconds
-            aggregations: The bundled aggregation to serialize.
             serialized_event: The serialized event which may be modified.
 
         """
-        serialized_aggregations = {}
+        # Do not bundle aggregations for an event which represents an edit or an
+        # annotation. It does not make sense for them to have related events.
+        relates_to = event.content.get("m.relates_to")
+        if isinstance(relates_to, (dict, frozendict)):
+            relation_type = relates_to.get("rel_type")
+            if relation_type in (RelationTypes.ANNOTATION, RelationTypes.REPLACE):
+                return
 
-        if aggregations.annotations:
-            serialized_aggregations[RelationTypes.ANNOTATION] = aggregations.annotations
+        event_id = event.event_id
+        room_id = event.room_id
 
-        if aggregations.references:
-            serialized_aggregations[RelationTypes.REFERENCE] = aggregations.references
+        # The bundled aggregations to include.
+        aggregations = {}
 
-        if aggregations.replace:
+        annotations = await self.store.get_aggregation_groups_for_event(
+            event_id, room_id
+        )
+        if annotations.chunk:
+            aggregations[RelationTypes.ANNOTATION] = annotations.to_dict()
+
+        references = await self.store.get_relations_for_event(
+            event_id, room_id, RelationTypes.REFERENCE, direction="f"
+        )
+        if references.chunk:
+            aggregations[RelationTypes.REFERENCE] = references.to_dict()
+
+        edit = None
+        if event.type == EventTypes.Message:
+            edit = await self.store.get_applicable_edit(event_id, room_id)
+
+        if edit:
             # If there is an edit replace the content, preserving existing
             # relations.
-            edit = aggregations.replace
 
             # Ensure we take copies of the edit content, otherwise we risk modifying
             # the original event.
@@ -470,30 +495,34 @@ class EventClientSerializer:
             else:
                 serialized_event["content"].pop("m.relates_to", None)
 
-            serialized_aggregations[RelationTypes.REPLACE] = {
+            aggregations[RelationTypes.REPLACE] = {
                 "event_id": edit.event_id,
                 "origin_server_ts": edit.origin_server_ts,
                 "sender": edit.sender,
             }
 
         # If this event is the start of a thread, include a summary of the replies.
-        if aggregations.thread:
-            serialized_aggregations[RelationTypes.THREAD] = {
-                # Don't bundle aggregations as this could recurse forever.
-                "latest_event": self.serialize_event(
-                    aggregations.thread.latest_event, time_now, bundle_aggregations=None
-                ),
-                "count": aggregations.thread.count,
-                "current_user_participated": aggregations.thread.current_user_participated,
-            }
+        if self._msc3440_enabled:
+            (
+                thread_count,
+                latest_thread_event,
+            ) = await self.store.get_thread_summary(event_id, room_id)
+            if latest_thread_event:
+                aggregations[RelationTypes.THREAD] = {
+                    # Don't bundle aggregations as this could recurse forever.
+                    "latest_event": await self.serialize_event(
+                        latest_thread_event, time_now, bundle_aggregations=False
+                    ),
+                    "count": thread_count,
+                }
 
-        # Include the bundled aggregations in the event.
-        if serialized_aggregations:
+        # If any bundled aggregations were found, include them.
+        if aggregations:
             serialized_event["unsigned"].setdefault("m.relations", {}).update(
-                serialized_aggregations
+                aggregations
             )
 
-    def serialize_events(
+    async def serialize_events(
         self, events: Iterable[Union[JsonDict, EventBase]], time_now: int, **kwargs: Any
     ) -> List[JsonDict]:
         """Serializes multiple events.
@@ -506,9 +535,9 @@ class EventClientSerializer:
         Returns:
             The list of serialized events
         """
-        return [
-            self.serialize_event(event, time_now=time_now, **kwargs) for event in events
-        ]
+        return await yieldable_gather_results(
+            self.serialize_event, events, time_now=time_now, **kwargs
+        )
 
 
 def copy_power_levels_contents(

synapse/events/validator.py

@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 import collections.abc
-from typing import Iterable, Type, Union
+from typing import Iterable, Union
 
 import jsonschema
 
@@ -246,7 +246,7 @@ POWER_LEVELS_SCHEMA = {
 
 # This could return something newer than Draft 7, but that's the current "latest"
 # validator.
-def _create_power_level_validator() -> Type[jsonschema.Draft7Validator]:
+def _create_power_level_validator() -> jsonschema.Draft7Validator:
     validator = jsonschema.validators.validator_for(POWER_LEVELS_SCHEMA)
 
     # by default jsonschema does not consider a frozendict to be an object so

synapse/federation/federation_base.py

@@ -230,10 +230,6 @@ def event_from_pdu_json(pdu_json: JsonDict, room_version: RoomVersion) -> EventB
     # origin, etc etc)
     assert_params_in_dict(pdu_json, ("type", "depth"))
 
-    # Strip any unauthorized values from "unsigned" if they exist
-    if "unsigned" in pdu_json:
-        _strip_unsigned_values(pdu_json)
-
     depth = pdu_json["depth"]
     if not isinstance(depth, int):
         raise SynapseError(400, "Depth %r not an intger" % (depth,), Codes.BAD_JSON)
@@ -249,24 +245,3 @@ def event_from_pdu_json(pdu_json: JsonDict, room_version: RoomVersion) -> EventB
 
     event = make_event_from_dict(pdu_json, room_version)
     return event
-
-
-def _strip_unsigned_values(pdu_dict: JsonDict) -> None:
-    """
-    Strip any unsigned values unless specifically allowed, as defined by the whitelist.
-
-    pdu: the json dict to strip values from. Note that the dict is mutated by this
-    function
-    """
-    unsigned = pdu_dict["unsigned"]
-
-    if not isinstance(unsigned, dict):
-        pdu_dict["unsigned"] = {}
-
-    if pdu_dict["type"] == "m.room.member":
-        whitelist = ["knock_room_state", "invite_room_state", "age"]
-    else:
-        whitelist = ["age"]
-
-    filtered_unsigned = {k: v for k, v in unsigned.items() if k in whitelist}
-    pdu_dict["unsigned"] = filtered_unsigned

synapse/federation/federation_client.py

@@ -56,6 +56,7 @@ from synapse.api.room_versions import (
 from synapse.events import EventBase, builder
 from synapse.federation.federation_base import FederationBase, event_from_pdu_json
 from synapse.federation.transport.client import SendJoinResponse
+from synapse.logging.utils import log_function
 from synapse.types import JsonDict, get_domain_from_id
 from synapse.util.async_helpers import concurrently_execute
 from synapse.util.caches.expiringcache import ExpiringCache
@@ -118,8 +119,7 @@ class FederationClient(FederationBase):
         # It is a map of (room ID, suggested-only) -> the response of
         # get_room_hierarchy.
         self._get_room_hierarchy_cache: ExpiringCache[
-            Tuple[str, bool],
-            Tuple[JsonDict, Sequence[JsonDict], Sequence[JsonDict], Sequence[str]],
+            Tuple[str, bool], Tuple[JsonDict, Sequence[JsonDict], Sequence[str]]
         ] = ExpiringCache(
             cache_name="get_room_hierarchy_cache",
             clock=self._clock,
@@ -144,6 +144,7 @@ class FederationClient(FederationBase):
             if destination_dict:
                 self.pdu_destination_tried[event_id] = destination_dict
 
+    @log_function
     async def make_query(
         self,
         destination: str,
@@ -177,6 +178,7 @@ class FederationClient(FederationBase):
             ignore_backoff=ignore_backoff,
         )
 
+    @log_function
     async def query_client_keys(
         self, destination: str, content: JsonDict, timeout: int
     ) -> JsonDict:
@@ -194,6 +196,7 @@ class FederationClient(FederationBase):
             destination, content, timeout
         )
 
+    @log_function
     async def query_user_devices(
         self, destination: str, user_id: str, timeout: int = 30000
     ) -> JsonDict:
@@ -205,6 +208,7 @@ class FederationClient(FederationBase):
             destination, user_id, timeout
         )
 
+    @log_function
     async def claim_client_keys(
         self, destination: str, content: JsonDict, timeout: int
     ) -> JsonDict:
@@ -1334,7 +1338,7 @@ class FederationClient(FederationBase):
         destinations: Iterable[str],
         room_id: str,
         suggested_only: bool,
-    ) -> Tuple[JsonDict, Sequence[JsonDict], Sequence[JsonDict], Sequence[str]]:
+    ) -> Tuple[JsonDict, Sequence[JsonDict], Sequence[str]]:
         """
         Call other servers to get a hierarchy of the given room.
 
@@ -1349,8 +1353,7 @@ class FederationClient(FederationBase):
 
         Returns:
             A tuple of:
-                The room as a JSON dictionary, without a "children_state" key.
-                A list of `m.space.child` state events.
+                The room as a JSON dictionary.
                 A list of children rooms, as JSON dictionaries.
                 A list of inaccessible children room IDs.
 
@@ -1365,7 +1368,7 @@ class FederationClient(FederationBase):
 
         async def send_request(
             destination: str,
-        ) -> Tuple[JsonDict, Sequence[JsonDict], Sequence[JsonDict], Sequence[str]]:
+        ) -> Tuple[JsonDict, Sequence[JsonDict], Sequence[str]]:
             try:
                 res = await self.transport_layer.get_room_hierarchy(
                     destination=destination,
@@ -1394,7 +1397,7 @@ class FederationClient(FederationBase):
                 raise InvalidResponseError("'room' must be a dict")
 
             # Validate children_state of the room.
-            children_state = room.pop("children_state", [])
+            children_state = room.get("children_state", [])
             if not isinstance(children_state, Sequence):
                 raise InvalidResponseError("'room.children_state' must be a list")
             if any(not isinstance(e, dict) for e in children_state):
@@ -1423,7 +1426,7 @@ class FederationClient(FederationBase):
                     "Invalid room ID in 'inaccessible_children' list"
                 )
 
-            return room, children_state, children, inaccessible_children
+            return room, children, inaccessible_children
 
         try:
             result = await self._try_destination_list(
@@ -1471,6 +1474,8 @@ class FederationClient(FederationBase):
                 if event.room_id == room_id:
                     children_events.append(event.data)
                     children_room_ids.add(event.state_key)
+            # And add them under the requested room.
+            requested_room["children_state"] = children_events
 
             # Find the children rooms.
             children = []
@@ -1480,7 +1485,7 @@ class FederationClient(FederationBase):
 
             # It isn't clear from the response whether some of the rooms are
             # not accessible.
-            result = (requested_room, children_events, children, ())
+            result = (requested_room, children, ())
 
         # Cache the result to avoid fetching data over federation every time.
         self._get_room_hierarchy_cache[(room_id, suggested_only)] = result

synapse/federation/federation_server.py

@@ -58,6 +58,7 @@ from synapse.logging.context import (
     run_in_background,
 )
 from synapse.logging.opentracing import log_kv, start_active_span_from_edu, trace
+from synapse.logging.utils import log_function
 from synapse.metrics.background_process_metrics import wrap_as_background_process
 from synapse.replication.http.federation import (
     ReplicationFederationSendEduRestServlet,
@@ -858,6 +859,7 @@ class FederationServer(FederationBase):
             res = {"auth_chain": [a.get_pdu_json(time_now) for a in auth_pdus]}
         return 200, res
 
+    @log_function
     async def on_query_client_keys(
         self, origin: str, content: Dict[str, str]
     ) -> Tuple[int, Dict[str, Any]]:
@@ -938,6 +940,7 @@ class FederationServer(FederationBase):
 
         return {"events": [ev.get_pdu_json(time_now) for ev in missing_events]}
 
+    @log_function
     async def on_openid_userinfo(self, token: str) -> Optional[str]:
         ts_now_ms = self._clock.time_msec()
         return await self.store.get_user_id_for_open_id_token(token, ts_now_ms)

synapse/federation/persistence.py

@@ -23,6 +23,7 @@ import logging
 from typing import Optional, Tuple
 
 from synapse.federation.units import Transaction
+from synapse.logging.utils import log_function
 from synapse.storage.databases.main import DataStore
 from synapse.types import JsonDict
 
@@ -35,6 +36,7 @@ class TransactionActions:
     def __init__(self, datastore: DataStore):
         self.store = datastore
 
+    @log_function
     async def have_responded(
         self, origin: str, transaction: Transaction
     ) -> Optional[Tuple[int, JsonDict]]:
@@ -51,6 +53,7 @@ class TransactionActions:
 
         return await self.store.get_received_txn_response(transaction_id, origin)
 
+    @log_function
     async def set_response(
         self, origin: str, transaction: Transaction, code: int, response: JsonDict
     ) -> None:

synapse/federation/sender/per_destination_queue.py

@@ -607,18 +607,18 @@ class PerDestinationQueue:
         self._pending_pdus = []
 
 
-@attr.s(slots=True, auto_attribs=True)
+@attr.s(slots=True)
 class _TransactionQueueManager:
     """A helper async context manager for pulling stuff off the queues and
     tracking what was last successfully sent, etc.
     """
 
-    queue: PerDestinationQueue
+    queue = attr.ib(type=PerDestinationQueue)
 
-    _device_stream_id: Optional[int] = None
-    _device_list_id: Optional[int] = None
-    _last_stream_ordering: Optional[int] = None
-    _pdus: List[EventBase] = attr.Factory(list)
+    _device_stream_id = attr.ib(type=Optional[int], default=None)
+    _device_list_id = attr.ib(type=Optional[int], default=None)
+    _last_stream_ordering = attr.ib(type=Optional[int], default=None)
+    _pdus = attr.ib(type=List[EventBase], factory=list)
 
     async def __aenter__(self) -> Tuple[List[EventBase], List[Edu]]:
         # First we calculate the EDUs we want to send, if any.

synapse/federation/sender/transaction_manager.py

@@ -35,7 +35,6 @@ if TYPE_CHECKING:
     import synapse.server
 
 logger = logging.getLogger(__name__)
-issue_8631_logger = logging.getLogger("synapse.8631_debug")
 
 last_pdu_ts_metric = Gauge(
     "synapse_federation_last_sent_pdu_time",
@@ -125,17 +124,6 @@ class TransactionManager:
                 len(pdus),
                 len(edus),
             )
-            if issue_8631_logger.isEnabledFor(logging.DEBUG):
-                DEVICE_UPDATE_EDUS = {"m.device_list_update", "m.signing_key_update"}
-                device_list_updates = [
-                    edu.content for edu in edus if edu.edu_type in DEVICE_UPDATE_EDUS
-                ]
-                if device_list_updates:
-                    issue_8631_logger.debug(
-                        "about to send txn [%s] including device list updates: %s",
-                        transaction.transaction_id,
-                        device_list_updates,
-                    )
 
             # Actually send the transaction
 

synapse/federation/transport/client.py

@@ -44,6 +44,7 @@ from synapse.api.urls import (
 from synapse.events import EventBase, make_event_from_dict
 from synapse.federation.units import Transaction
 from synapse.http.matrixfederationclient import ByteParser
+from synapse.logging.utils import log_function
 from synapse.types import JsonDict
 
 logger = logging.getLogger(__name__)
@@ -61,6 +62,7 @@ class TransportLayerClient:
         self.server_name = hs.hostname
         self.client = hs.get_federation_http_client()
 
+    @log_function
     async def get_room_state_ids(
         self, destination: str, room_id: str, event_id: str
     ) -> JsonDict:
@@ -86,6 +88,7 @@ class TransportLayerClient:
             try_trailing_slash_on_400=True,
         )
 
+    @log_function
     async def get_event(
         self, destination: str, event_id: str, timeout: Optional[int] = None
     ) -> JsonDict:
@@ -108,6 +111,7 @@ class TransportLayerClient:
             destination, path=path, timeout=timeout, try_trailing_slash_on_400=True
         )
 
+    @log_function
     async def backfill(
         self, destination: str, room_id: str, event_tuples: Collection[str], limit: int
     ) -> Optional[JsonDict]:
@@ -145,6 +149,7 @@ class TransportLayerClient:
             destination, path=path, args=args, try_trailing_slash_on_400=True
         )
 
+    @log_function
     async def timestamp_to_event(
         self, destination: str, room_id: str, timestamp: int, direction: str
     ) -> Union[JsonDict, List]:
@@ -180,6 +185,7 @@ class TransportLayerClient:
 
         return remote_response
 
+    @log_function
     async def send_transaction(
         self,
         transaction: Transaction,
@@ -228,6 +234,7 @@ class TransportLayerClient:
             try_trailing_slash_on_400=True,
         )
 
+    @log_function
     async def make_query(
         self,
         destination: str,
@@ -247,6 +254,7 @@ class TransportLayerClient:
             ignore_backoff=ignore_backoff,
         )
 
+    @log_function
     async def make_membership_event(
         self,
         destination: str,
@@ -309,6 +317,7 @@ class TransportLayerClient:
             ignore_backoff=ignore_backoff,
         )
 
+    @log_function
     async def send_join_v1(
         self,
         room_version: RoomVersion,
@@ -327,6 +336,7 @@ class TransportLayerClient:
             max_response_size=MAX_RESPONSE_SIZE_SEND_JOIN,
         )
 
+    @log_function
     async def send_join_v2(
         self,
         room_version: RoomVersion,
@@ -345,6 +355,7 @@ class TransportLayerClient:
             max_response_size=MAX_RESPONSE_SIZE_SEND_JOIN,
         )
 
+    @log_function
     async def send_leave_v1(
         self, destination: str, room_id: str, event_id: str, content: JsonDict
     ) -> Tuple[int, JsonDict]:
@@ -361,6 +372,7 @@ class TransportLayerClient:
             ignore_backoff=True,
         )
 
+    @log_function
     async def send_leave_v2(
         self, destination: str, room_id: str, event_id: str, content: JsonDict
     ) -> JsonDict:
@@ -377,6 +389,7 @@ class TransportLayerClient:
             ignore_backoff=True,
         )
 
+    @log_function
     async def send_knock_v1(
         self,
         destination: str,
@@ -410,6 +423,7 @@ class TransportLayerClient:
             destination=destination, path=path, data=content
         )
 
+    @log_function
     async def send_invite_v1(
         self, destination: str, room_id: str, event_id: str, content: JsonDict
     ) -> Tuple[int, JsonDict]:
@@ -419,6 +433,7 @@ class TransportLayerClient:
             destination=destination, path=path, data=content, ignore_backoff=True
         )
 
+    @log_function
     async def send_invite_v2(
         self, destination: str, room_id: str, event_id: str, content: JsonDict
     ) -> JsonDict:
@@ -428,6 +443,7 @@ class TransportLayerClient:
             destination=destination, path=path, data=content, ignore_backoff=True
         )
 
+    @log_function
     async def get_public_rooms(
         self,
         remote_server: str,
@@ -500,6 +516,7 @@ class TransportLayerClient:
 
         return response
 
+    @log_function
     async def exchange_third_party_invite(
         self, destination: str, room_id: str, event_dict: JsonDict
     ) -> JsonDict:
@@ -509,6 +526,7 @@ class TransportLayerClient:
             destination=destination, path=path, data=event_dict
         )
 
+    @log_function
     async def get_event_auth(
         self, destination: str, room_id: str, event_id: str
     ) -> JsonDict:
@@ -516,6 +534,7 @@ class TransportLayerClient:
 
         return await self.client.get_json(destination=destination, path=path)
 
+    @log_function
     async def query_client_keys(
         self, destination: str, query_content: JsonDict, timeout: int
     ) -> JsonDict:
@@ -557,6 +576,7 @@ class TransportLayerClient:
             destination=destination, path=path, data=query_content, timeout=timeout
         )
 
+    @log_function
     async def query_user_devices(
         self, destination: str, user_id: str, timeout: int
     ) -> JsonDict:
@@ -596,6 +616,7 @@ class TransportLayerClient:
             destination=destination, path=path, timeout=timeout
         )
 
+    @log_function
     async def claim_client_keys(
         self, destination: str, query_content: JsonDict, timeout: int
     ) -> JsonDict:
@@ -634,6 +655,7 @@ class TransportLayerClient:
             destination=destination, path=path, data=query_content, timeout=timeout
         )
 
+    @log_function
     async def get_missing_events(
         self,
         destination: str,
@@ -658,6 +680,7 @@ class TransportLayerClient:
             timeout=timeout,
         )
 
+    @log_function
     async def get_group_profile(
         self, destination: str, group_id: str, requester_user_id: str
     ) -> JsonDict:
@@ -671,6 +694,7 @@ class TransportLayerClient:
             ignore_backoff=True,
         )
 
+    @log_function
     async def update_group_profile(
         self, destination: str, group_id: str, requester_user_id: str, content: JsonDict
     ) -> JsonDict:
@@ -692,6 +716,7 @@ class TransportLayerClient:
             ignore_backoff=True,
         )
 
+    @log_function
     async def get_group_summary(
         self, destination: str, group_id: str, requester_user_id: str
     ) -> JsonDict:
@@ -705,6 +730,7 @@ class TransportLayerClient:
             ignore_backoff=True,
         )
 
+    @log_function
     async def get_rooms_in_group(
         self, destination: str, group_id: str, requester_user_id: str
     ) -> JsonDict:
@@ -772,6 +798,7 @@ class TransportLayerClient:
             ignore_backoff=True,
         )
 
+    @log_function
     async def get_users_in_group(
         self, destination: str, group_id: str, requester_user_id: str
     ) -> JsonDict:
@@ -785,6 +812,7 @@ class TransportLayerClient:
             ignore_backoff=True,
         )
 
+    @log_function
     async def get_invited_users_in_group(
         self, destination: str, group_id: str, requester_user_id: str
     ) -> JsonDict:
@@ -798,6 +826,7 @@ class TransportLayerClient:
             ignore_backoff=True,
         )
 
+    @log_function
     async def accept_group_invite(
         self, destination: str, group_id: str, user_id: str, content: JsonDict
     ) -> JsonDict:
@@ -808,6 +837,7 @@ class TransportLayerClient:
             destination=destination, path=path, data=content, ignore_backoff=True
         )
 
+    @log_function
     def join_group(
         self, destination: str, group_id: str, user_id: str, content: JsonDict
     ) -> Awaitable[JsonDict]:
@@ -818,6 +848,7 @@ class TransportLayerClient:
             destination=destination, path=path, data=content, ignore_backoff=True
         )
 
+    @log_function
     async def invite_to_group(
         self,
         destination: str,
@@ -837,6 +868,7 @@ class TransportLayerClient:
             ignore_backoff=True,
         )
 
+    @log_function
     async def invite_to_group_notification(
         self, destination: str, group_id: str, user_id: str, content: JsonDict
     ) -> JsonDict:
@@ -850,6 +882,7 @@ class TransportLayerClient:
             destination=destination, path=path, data=content, ignore_backoff=True
         )
 
+    @log_function
     async def remove_user_from_group(
         self,
         destination: str,
@@ -869,6 +902,7 @@ class TransportLayerClient:
             ignore_backoff=True,
         )
 
+    @log_function
     async def remove_user_from_group_notification(
         self, destination: str, group_id: str, user_id: str, content: JsonDict
     ) -> JsonDict:
@@ -882,6 +916,7 @@ class TransportLayerClient:
             destination=destination, path=path, data=content, ignore_backoff=True
         )
 
+    @log_function
     async def renew_group_attestation(
         self, destination: str, group_id: str, user_id: str, content: JsonDict
     ) -> JsonDict:
@@ -895,6 +930,7 @@ class TransportLayerClient:
             destination=destination, path=path, data=content, ignore_backoff=True
         )
 
+    @log_function
     async def update_group_summary_room(
         self,
         destination: str,
@@ -923,6 +959,7 @@ class TransportLayerClient:
             ignore_backoff=True,
         )
 
+    @log_function
     async def delete_group_summary_room(
         self,
         destination: str,
@@ -949,6 +986,7 @@ class TransportLayerClient:
             ignore_backoff=True,
         )
 
+    @log_function
     async def get_group_categories(
         self, destination: str, group_id: str, requester_user_id: str
     ) -> JsonDict:
@@ -962,6 +1000,7 @@ class TransportLayerClient:
             ignore_backoff=True,
         )
 
+    @log_function
     async def get_group_category(
         self, destination: str, group_id: str, requester_user_id: str, category_id: str
     ) -> JsonDict:
@@ -975,6 +1014,7 @@ class TransportLayerClient:
             ignore_backoff=True,
         )
 
+    @log_function
     async def update_group_category(
         self,
         destination: str,
@@ -994,6 +1034,7 @@ class TransportLayerClient:
             ignore_backoff=True,
         )
 
+    @log_function
     async def delete_group_category(
         self, destination: str, group_id: str, requester_user_id: str, category_id: str
     ) -> JsonDict:
@@ -1007,6 +1048,7 @@ class TransportLayerClient:
             ignore_backoff=True,
         )
 
+    @log_function
     async def get_group_roles(
         self, destination: str, group_id: str, requester_user_id: str
     ) -> JsonDict:
@@ -1020,6 +1062,7 @@ class TransportLayerClient:
             ignore_backoff=True,
         )
 
+    @log_function
     async def get_group_role(
         self, destination: str, group_id: str, requester_user_id: str, role_id: str
     ) -> JsonDict:
@@ -1033,6 +1076,7 @@ class TransportLayerClient:
             ignore_backoff=True,
         )
 
+    @log_function
     async def update_group_role(
         self,
         destination: str,
@@ -1052,6 +1096,7 @@ class TransportLayerClient:
             ignore_backoff=True,
         )
 
+    @log_function
     async def delete_group_role(
         self, destination: str, group_id: str, requester_user_id: str, role_id: str
     ) -> JsonDict:
@@ -1065,6 +1110,7 @@ class TransportLayerClient:
             ignore_backoff=True,
         )
 
+    @log_function
     async def update_group_summary_user(
         self,
         destination: str,
@@ -1090,6 +1136,7 @@ class TransportLayerClient:
             ignore_backoff=True,
         )
 
+    @log_function
     async def set_group_join_policy(
         self, destination: str, group_id: str, requester_user_id: str, content: JsonDict
     ) -> JsonDict:
@@ -1104,6 +1151,7 @@ class TransportLayerClient:
             ignore_backoff=True,
         )
 
+    @log_function
     async def delete_group_summary_user(
         self,
         destination: str,

synapse/federation/transport/server/__init__.py

@@ -13,7 +13,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 import logging
-from typing import TYPE_CHECKING, Dict, Iterable, List, Optional, Tuple, Type
+from typing import Dict, Iterable, List, Optional, Tuple, Type
 
 from typing_extensions import Literal
 
@@ -36,19 +36,17 @@ from synapse.http.servlet import (
     parse_integer_from_args,
     parse_string_from_args,
 )
+from synapse.server import HomeServer
 from synapse.types import JsonDict, ThirdPartyInstanceID
 from synapse.util.ratelimitutils import FederationRateLimiter
 
-if TYPE_CHECKING:
-    from synapse.server import HomeServer
-
 logger = logging.getLogger(__name__)
 
 
 class TransportLayerServer(JsonResource):
     """Handles incoming federation HTTP requests"""
 
-    def __init__(self, hs: "HomeServer", servlet_groups: Optional[List[str]] = None):
+    def __init__(self, hs: HomeServer, servlet_groups: Optional[List[str]] = None):
         """Initialize the TransportLayerServer
 
         Will by default register all servlets. For custom behaviour, pass in
@@ -115,7 +113,7 @@ class PublicRoomList(BaseFederationServlet):
 
     def __init__(
         self,
-        hs: "HomeServer",
+        hs: HomeServer,
         authenticator: Authenticator,
         ratelimiter: FederationRateLimiter,
         server_name: str,
@@ -205,7 +203,7 @@ class FederationGroupsRenewAttestaionServlet(BaseFederationServlet):
 
     def __init__(
         self,
-        hs: "HomeServer",
+        hs: HomeServer,
         authenticator: Authenticator,
         ratelimiter: FederationRateLimiter,
         server_name: str,
@@ -253,7 +251,7 @@ class OpenIdUserInfo(BaseFederationServlet):
 
     def __init__(
         self,
-        hs: "HomeServer",
+        hs: HomeServer,
         authenticator: Authenticator,
         ratelimiter: FederationRateLimiter,
         server_name: str,
@@ -299,7 +297,7 @@ DEFAULT_SERVLET_GROUPS: Dict[str, Iterable[Type[BaseFederationServlet]]] = {
 
 
 def register_servlets(
-    hs: "HomeServer",
+    hs: HomeServer,
     resource: HttpServer,
     authenticator: Authenticator,
     ratelimiter: FederationRateLimiter,

synapse/federation/transport/server/_base.py

@@ -15,7 +15,7 @@
 import functools
 import logging
 import re
-from typing import TYPE_CHECKING, Any, Awaitable, Callable, Optional, Tuple, cast
+from typing import Any, Awaitable, Callable, Optional, Tuple, cast
 
 from synapse.api.errors import Codes, FederationDeniedError, SynapseError
 from synapse.api.urls import FEDERATION_V1_PREFIX
@@ -29,13 +29,11 @@ from synapse.logging.opentracing import (
     start_active_span_follows_from,
     whitelisted_homeserver,
 )
+from synapse.server import HomeServer
 from synapse.types import JsonDict
 from synapse.util.ratelimitutils import FederationRateLimiter
 from synapse.util.stringutils import parse_and_validate_server_name
 
-if TYPE_CHECKING:
-    from synapse.server import HomeServer
-
 logger = logging.getLogger(__name__)
 
 
@@ -48,7 +46,7 @@ class NoAuthenticationError(AuthenticationError):
 
 
 class Authenticator:
-    def __init__(self, hs: "HomeServer"):
+    def __init__(self, hs: HomeServer):
         self._clock = hs.get_clock()
         self.keyring = hs.get_keyring()
         self.server_name = hs.hostname
@@ -116,11 +114,11 @@ class Authenticator:
         # alive
         retry_timings = await self.store.get_destination_retry_timings(origin)
         if retry_timings and retry_timings.retry_last_ts:
-            run_in_background(self.reset_retry_timings, origin)
+            run_in_background(self._reset_retry_timings, origin)
 
         return origin
 
-    async def reset_retry_timings(self, origin: str) -> None:
+    async def _reset_retry_timings(self, origin: str) -> None:
         try:
             logger.info("Marking origin %r as up", origin)
             await self.store.set_destination_retry_timings(origin, None, 0, 0)
@@ -229,7 +227,7 @@ class BaseFederationServlet:
 
     def __init__(
         self,
-        hs: "HomeServer",
+        hs: HomeServer,
         authenticator: Authenticator,
         ratelimiter: FederationRateLimiter,
         server_name: str,

synapse/federation/transport/server/federation.py

@@ -12,17 +12,7 @@
 #  See the License for the specific language governing permissions and
 #  limitations under the License.
 import logging
-from typing import (
-    TYPE_CHECKING,
-    Dict,
-    List,
-    Mapping,
-    Optional,
-    Sequence,
-    Tuple,
-    Type,
-    Union,
-)
+from typing import Dict, List, Mapping, Optional, Sequence, Tuple, Type, Union
 
 from typing_extensions import Literal
 
@@ -40,15 +30,12 @@ from synapse.http.servlet import (
     parse_string_from_args,
     parse_strings_from_args,
 )
+from synapse.server import HomeServer
 from synapse.types import JsonDict
 from synapse.util.ratelimitutils import FederationRateLimiter
 from synapse.util.versionstring import get_version_string
 
-if TYPE_CHECKING:
-    from synapse.server import HomeServer
-
 logger = logging.getLogger(__name__)
-issue_8631_logger = logging.getLogger("synapse.8631_debug")
 
 
 class BaseFederationServerServlet(BaseFederationServlet):
@@ -59,7 +46,7 @@ class BaseFederationServerServlet(BaseFederationServlet):
 
     def __init__(
         self,
-        hs: "HomeServer",
+        hs: HomeServer,
         authenticator: Authenticator,
         ratelimiter: FederationRateLimiter,
         server_name: str,
@@ -108,20 +95,6 @@ class FederationSendServlet(BaseFederationServerServlet):
                 len(transaction_data.get("edus", [])),
             )
 
-            if issue_8631_logger.isEnabledFor(logging.DEBUG):
-                DEVICE_UPDATE_EDUS = {"m.device_list_update", "m.signing_key_update"}
-                device_list_updates = [
-                    edu.content
-                    for edu in transaction_data.get("edus", [])
-                    if edu.edu_type in DEVICE_UPDATE_EDUS
-                ]
-                if device_list_updates:
-                    issue_8631_logger.debug(
-                        "received transaction [%s] including device list updates: %s",
-                        transaction_id,
-                        device_list_updates,
-                    )
-
         except Exception as e:
             logger.exception(e)
             return 400, {"error": "Invalid transaction"}
@@ -608,7 +581,7 @@ class FederationSpaceSummaryServlet(BaseFederationServlet):
 
     def __init__(
         self,
-        hs: "HomeServer",
+        hs: HomeServer,
         authenticator: Authenticator,
         ratelimiter: FederationRateLimiter,
         server_name: str,
@@ -682,7 +655,7 @@ class FederationRoomHierarchyServlet(BaseFederationServlet):
 
     def __init__(
         self,
-        hs: "HomeServer",
+        hs: HomeServer,
         authenticator: Authenticator,
         ratelimiter: FederationRateLimiter,
         server_name: str,
@@ -718,7 +691,7 @@ class RoomComplexityServlet(BaseFederationServlet):
 
     def __init__(
         self,
-        hs: "HomeServer",
+        hs: HomeServer,
         authenticator: Authenticator,
         ratelimiter: FederationRateLimiter,
         server_name: str,

synapse/federation/transport/server/groups_local.py

@@ -11,7 +11,7 @@
 #  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 #  See the License for the specific language governing permissions and
 #  limitations under the License.
-from typing import TYPE_CHECKING, Dict, List, Tuple, Type
+from typing import Dict, List, Tuple, Type
 
 from synapse.api.errors import SynapseError
 from synapse.federation.transport.server._base import (
@@ -19,12 +19,10 @@ from synapse.federation.transport.server._base import (
     BaseFederationServlet,
 )
 from synapse.handlers.groups_local import GroupsLocalHandler
+from synapse.server import HomeServer
 from synapse.types import JsonDict, get_domain_from_id
 from synapse.util.ratelimitutils import FederationRateLimiter
 
-if TYPE_CHECKING:
-    from synapse.server import HomeServer
-
 
 class BaseGroupsLocalServlet(BaseFederationServlet):
     """Abstract base class for federation servlet classes which provides a groups local handler.
@@ -34,7 +32,7 @@ class BaseGroupsLocalServlet(BaseFederationServlet):
 
     def __init__(
         self,
-        hs: "HomeServer",
+        hs: HomeServer,
         authenticator: Authenticator,
         ratelimiter: FederationRateLimiter,
         server_name: str,

synapse/federation/transport/server/groups_server.py

@@ -11,7 +11,7 @@
 #  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 #  See the License for the specific language governing permissions and
 #  limitations under the License.
-from typing import TYPE_CHECKING, Dict, List, Tuple, Type
+from typing import Dict, List, Tuple, Type
 
 from typing_extensions import Literal
 
@@ -22,12 +22,10 @@ from synapse.federation.transport.server._base import (
     BaseFederationServlet,
 )
 from synapse.http.servlet import parse_string_from_args
+from synapse.server import HomeServer
 from synapse.types import JsonDict, get_domain_from_id
 from synapse.util.ratelimitutils import FederationRateLimiter
 
-if TYPE_CHECKING:
-    from synapse.server import HomeServer
-
 
 class BaseGroupsServerServlet(BaseFederationServlet):
     """Abstract base class for federation servlet classes which provides a groups server handler.
@@ -37,7 +35,7 @@ class BaseGroupsServerServlet(BaseFederationServlet):
 
     def __init__(
         self,
-        hs: "HomeServer",
+        hs: HomeServer,
         authenticator: Authenticator,
         ratelimiter: FederationRateLimiter,
         server_name: str,

synapse/handlers/account_data.py

@@ -77,7 +77,7 @@ class AccountDataHandler:
     async def add_account_data_for_user(
         self, user_id: str, account_data_type: str, content: JsonDict
     ) -> int:
-        """Add some global account_data for a user.
+        """Add some account_data to a room for a user.
 
         Args:
             user_id: The user to add a tag for.

synapse/handlers/admin.py

@@ -55,47 +55,21 @@ class AdminHandler:
 
     async def get_user(self, user: UserID) -> Optional[JsonDict]:
         """Function to get user details"""
-        user_info_dict = await self.store.get_user_by_id(user.to_string())
-        if user_info_dict is None:
-            return None
-
-        # Restrict returned information to a known set of fields. This prevents additional
-        # fields added to get_user_by_id from modifying Synapse's external API surface.
-        user_info_to_return = {
-            "name",
-            "admin",
-            "deactivated",
-            "shadow_banned",
-            "creation_ts",
-            "appservice_id",
-            "consent_server_notice_sent",
-            "consent_version",
-            "user_type",
-            "is_guest",
-        }
-
-        # Restrict returned keys to a known set.
-        user_info_dict = {
-            key: value
-            for key, value in user_info_dict.items()
-            if key in user_info_to_return
-        }
-
-        # Add additional user metadata
-        profile = await self.store.get_profileinfo(user.localpart)
-        threepids = await self.store.user_get_threepids(user.to_string())
-        external_ids = [
-            ({"auth_provider": auth_provider, "external_id": external_id})
-            for auth_provider, external_id in await self.store.get_external_ids_by_user(
-                user.to_string()
-            )
-        ]
-        user_info_dict["displayname"] = profile.display_name
-        user_info_dict["avatar_url"] = profile.avatar_url
-        user_info_dict["threepids"] = threepids
-        user_info_dict["external_ids"] = external_ids
-
-        return user_info_dict
+        ret = await self.store.get_user_by_id(user.to_string())
+        if ret:
+            profile = await self.store.get_profileinfo(user.localpart)
+            threepids = await self.store.user_get_threepids(user.to_string())
+            external_ids = [
+                ({"auth_provider": auth_provider, "external_id": external_id})
+                for auth_provider, external_id in await self.store.get_external_ids_by_user(
+                    user.to_string()
+                )
+            ]
+            ret["displayname"] = profile.display_name
+            ret["avatar_url"] = profile.avatar_url
+            ret["threepids"] = threepids
+            ret["external_ids"] = external_ids
+        return ret
 
     async def export_user_data(self, user_id: str, writer: "ExfiltrationWriter") -> Any:
         """Write all data we have on the user to the given writer.

synapse/handlers/auth.py

@@ -168,25 +168,25 @@ def login_id_phone_to_thirdparty(identifier: JsonDict) -> Dict[str, str]:
     }
 
 
-@attr.s(slots=True, auto_attribs=True)
+@attr.s(slots=True)
 class SsoLoginExtraAttributes:
     """Data we track about SAML2 sessions"""
 
     # time the session was created, in milliseconds
-    creation_time: int
-    extra_attributes: JsonDict
+    creation_time = attr.ib(type=int)
+    extra_attributes = attr.ib(type=JsonDict)
 
 
-@attr.s(slots=True, frozen=True, auto_attribs=True)
+@attr.s(slots=True, frozen=True)
 class LoginTokenAttributes:
     """Data we store in a short-term login token"""
 
-    user_id: str
+    user_id = attr.ib(type=str)
 
-    auth_provider_id: str
+    auth_provider_id = attr.ib(type=str)
     """The SSO Identity Provider that the user authenticated with, to get this token."""
 
-    auth_provider_session_id: Optional[str]
+    auth_provider_session_id = attr.ib(type=Optional[str])
     """The session ID advertised by the SSO Identity Provider."""
 
 
@@ -2060,10 +2060,6 @@ CHECK_AUTH_CALLBACK = Callable[
         Optional[Tuple[str, Optional[Callable[["LoginResponse"], Awaitable[None]]]]]
     ],
 ]
-GET_USERNAME_FOR_REGISTRATION_CALLBACK = Callable[
-    [JsonDict, JsonDict],
-    Awaitable[Optional[str]],
-]
 
 
 class PasswordAuthProvider:
@@ -2076,9 +2072,6 @@ class PasswordAuthProvider:
         # lists of callbacks
         self.check_3pid_auth_callbacks: List[CHECK_3PID_AUTH_CALLBACK] = []
         self.on_logged_out_callbacks: List[ON_LOGGED_OUT_CALLBACK] = []
-        self.get_username_for_registration_callbacks: List[
-            GET_USERNAME_FOR_REGISTRATION_CALLBACK
-        ] = []
 
         # Mapping from login type to login parameters
         self._supported_login_types: Dict[str, Iterable[str]] = {}
@@ -2093,9 +2086,6 @@ class PasswordAuthProvider:
         auth_checkers: Optional[
             Dict[Tuple[str, Tuple[str, ...]], CHECK_AUTH_CALLBACK]
         ] = None,
-        get_username_for_registration: Optional[
-            GET_USERNAME_FOR_REGISTRATION_CALLBACK
-        ] = None,
     ) -> None:
         # Register check_3pid_auth callback
         if check_3pid_auth is not None:
@@ -2140,11 +2130,6 @@ class PasswordAuthProvider:
                 # Add the new method to the list of auth_checker_callbacks for this login type
                 self.auth_checker_callbacks.setdefault(login_type, []).append(callback)
 
-        if get_username_for_registration is not None:
-            self.get_username_for_registration_callbacks.append(
-                get_username_for_registration,
-            )
-
     def get_supported_login_types(self) -> Mapping[str, Iterable[str]]:
         """Get the login types supported by this password provider
 
@@ -2296,50 +2281,7 @@ class PasswordAuthProvider:
         # call all of the on_logged_out callbacks
         for callback in self.on_logged_out_callbacks:
             try:
-                await callback(user_id, device_id, access_token)
+                callback(user_id, device_id, access_token)
             except Exception as e:
                 logger.warning("Failed to run module API callback %s: %s", callback, e)
                 continue
-
-    async def get_username_for_registration(
-        self,
-        uia_results: JsonDict,
-        params: JsonDict,
-    ) -> Optional[str]:
-        """Defines the username to use when registering the user, using the credentials
-        and parameters provided during the UIA flow.
-
-        Stops at the first callback that returns a string.
-
-        Args:
-            uia_results: The credentials provided during the UIA flow.
-            params: The parameters provided by the registration request.
-
-        Returns:
-            The localpart to use when registering this user, or None if no module
-            returned a localpart.
-        """
-        for callback in self.get_username_for_registration_callbacks:
-            try:
-                res = await callback(uia_results, params)
-
-                if isinstance(res, str):
-                    return res
-                elif res is not None:
-                    # mypy complains that this line is unreachable because it assumes the
-                    # data returned by the module fits the expected type. We just want
-                    # to make sure this is the case.
-                    logger.warning(  # type: ignore[unreachable]
-                        "Ignoring non-string value returned by"
-                        " get_username_for_registration callback %s: %s",
-                        callback,
-                        res,
-                    )
-            except Exception as e:
-                logger.error(
-                    "Module raised an exception in get_username_for_registration: %s",
-                    e,
-                )
-                raise SynapseError(code=500, msg="Internal Server Error")
-
-        return None

synapse/handlers/deactivate_account.py

@@ -157,9 +157,6 @@ class DeactivateAccountHandler:
         # Mark the user as deactivated.
         await self.store.set_user_deactivated_status(user_id, True)
 
-        # Remove account data (including ignored users and push rules).
-        await self.store.purge_account_data_for_user(user_id)
-
         return identity_server_supports_unbinding
 
     async def _reject_pending_invites_for_user(self, user_id: str) -> None:

synapse/handlers/device.py

@@ -948,16 +948,8 @@ class DeviceListUpdater:
             devices = []
             ignore_devices = True
         else:
-            prev_stream_id = await self.store.get_device_list_last_stream_id_for_remote(
-                user_id
-            )
             cached_devices = await self.store.get_cached_devices_for_user(user_id)
-
-            # To ensure that a user with no devices is cached, we skip the resync only
-            # if we have a stream_id from previously writing a cache entry.
-            if prev_stream_id is not None and cached_devices == {
-                d["device_id"]: d for d in devices
-            }:
+            if cached_devices == {d["device_id"]: d for d in devices}:
                 logging.info(
                     "Skipping device list resync for %s, as our cache matches already",
                     user_id,

synapse/handlers/e2e_keys.py

@@ -1321,14 +1321,14 @@ def _one_time_keys_match(old_key_json: str, new_key: JsonDict) -> bool:
     return old_key == new_key_copy
 
 
-@attr.s(slots=True, auto_attribs=True)
+@attr.s(slots=True)
 class SignatureListItem:
     """An item in the signature list as used by upload_signatures_for_device_keys."""
 
-    signing_key_id: str
-    target_user_id: str
-    target_device_id: str
-    signature: JsonDict
+    signing_key_id = attr.ib(type=str)
+    target_user_id = attr.ib(type=str)
+    target_device_id = attr.ib(type=str)
+    signature = attr.ib(type=JsonDict)
 
 
 class SigningKeyEduUpdater:

synapse/handlers/events.py

@@ -20,6 +20,7 @@ from synapse.api.constants import EduTypes, EventTypes, Membership
 from synapse.api.errors import AuthError, SynapseError
 from synapse.events import EventBase
 from synapse.handlers.presence import format_user_presence_state
+from synapse.logging.utils import log_function
 from synapse.streams.config import PaginationConfig
 from synapse.types import JsonDict, UserID
 from synapse.visibility import filter_events_for_client
@@ -42,6 +43,7 @@ class EventStreamHandler:
         self._server_notices_sender = hs.get_server_notices_sender()
         self._event_serializer = hs.get_event_client_serializer()
 
+    @log_function
     async def get_stream(
         self,
         auth_user_id: str,
@@ -117,7 +119,7 @@ class EventStreamHandler:
 
             events.extend(to_add)
 
-            chunks = self._event_serializer.serialize_events(
+            chunks = await self._event_serializer.serialize_events(
                 events,
                 time_now,
                 as_client_event=as_client_event,

synapse/handlers/federation.py

@@ -51,6 +51,7 @@ from synapse.logging.context import (
     preserve_fn,
     run_in_background,
 )
+from synapse.logging.utils import log_function
 from synapse.replication.http.federation import (
     ReplicationCleanRoomRestServlet,
     ReplicationStoreRoomOnOutlierMembershipRestServlet,
@@ -555,6 +556,7 @@ class FederationHandler:
 
             run_in_background(self._handle_queued_pdus, room_queue)
 
+    @log_function
     async def do_knock(
         self,
         target_hosts: List[str],
@@ -926,6 +928,7 @@ class FederationHandler:
 
         return event
 
+    @log_function
     async def on_make_knock_request(
         self, origin: str, room_id: str, user_id: str
     ) -> EventBase:
@@ -1036,6 +1039,7 @@ class FederationHandler:
         else:
             return []
 
+    @log_function
     async def on_backfill_request(
         self, origin: str, room_id: str, pdu_list: List[str], limit: int
     ) -> List[EventBase]:
@@ -1052,6 +1056,7 @@ class FederationHandler:
 
         return events
 
+    @log_function
     async def get_persisted_pdu(
         self, origin: str, event_id: str
     ) -> Optional[EventBase]:
@@ -1113,6 +1118,7 @@ class FederationHandler:
 
         return missing_events
 
+    @log_function
     async def exchange_third_party_invite(
         self, sender_user_id: str, target_user_id: str, room_id: str, signed: JsonDict
     ) -> None:

synapse/handlers/federation_event.py

@@ -56,6 +56,7 @@ from synapse.events import EventBase
 from synapse.events.snapshot import EventContext
 from synapse.federation.federation_client import InvalidResponseError
 from synapse.logging.context import nested_logging_context, run_in_background
+from synapse.logging.utils import log_function
 from synapse.metrics.background_process_metrics import run_as_background_process
 from synapse.replication.http.devices import ReplicationUserDevicesResyncRestServlet
 from synapse.replication.http.federation import (
@@ -274,6 +275,7 @@ class FederationEventHandler:
 
         await self._process_received_pdu(origin, pdu, state=None)
 
+    @log_function
     async def on_send_membership_event(
         self, origin: str, event: EventBase
     ) -> Tuple[EventBase, EventContext]:
@@ -470,6 +472,7 @@ class FederationEventHandler:
 
             return await self.persist_events_and_notify(room_id, [(event, context)])
 
+    @log_function
     async def backfill(
         self, dest: str, room_id: str, limit: int, extremities: Collection[str]
     ) -> None:

synapse/handlers/initial_sync.py

@@ -170,7 +170,7 @@ class InitialSyncHandler:
                 d["inviter"] = event.sender
 
                 invite_event = await self.store.get_event(event.event_id)
-                d["invite"] = self._event_serializer.serialize_event(
+                d["invite"] = await self._event_serializer.serialize_event(
                     invite_event,
                     time_now,
                     as_client_event=as_client_event,
@@ -222,7 +222,7 @@ class InitialSyncHandler:
 
                 d["messages"] = {
                     "chunk": (
-                        self._event_serializer.serialize_events(
+                        await self._event_serializer.serialize_events(
                             messages,
                             time_now=time_now,
                             as_client_event=as_client_event,
@@ -232,7 +232,7 @@ class InitialSyncHandler:
                     "end": await end_token.to_string(self.store),
                 }
 
-                d["state"] = self._event_serializer.serialize_events(
+                d["state"] = await self._event_serializer.serialize_events(
                     current_state.values(),
                     time_now=time_now,
                     as_client_event=as_client_event,
@@ -376,14 +376,16 @@ class InitialSyncHandler:
             "messages": {
                 "chunk": (
                     # Don't bundle aggregations as this is a deprecated API.
-                    self._event_serializer.serialize_events(messages, time_now)
+                    await self._event_serializer.serialize_events(messages, time_now)
                 ),
                 "start": await start_token.to_string(self.store),
                 "end": await end_token.to_string(self.store),
             },
             "state": (
                 # Don't bundle aggregations as this is a deprecated API.
-                self._event_serializer.serialize_events(room_state.values(), time_now)
+                await self._event_serializer.serialize_events(
+                    room_state.values(), time_now
+                )
             ),
             "presence": [],
             "receipts": [],
@@ -402,7 +404,7 @@ class InitialSyncHandler:
         # TODO: These concurrently
         time_now = self.clock.time_msec()
         # Don't bundle aggregations as this is a deprecated API.
-        state = self._event_serializer.serialize_events(
+        state = await self._event_serializer.serialize_events(
             current_state.values(), time_now
         )
 
@@ -478,7 +480,7 @@ class InitialSyncHandler:
             "messages": {
                 "chunk": (
                     # Don't bundle aggregations as this is a deprecated API.
-                    self._event_serializer.serialize_events(messages, time_now)
+                    await self._event_serializer.serialize_events(messages, time_now)
                 ),
                 "start": await start_token.to_string(self.store),
                 "end": await end_token.to_string(self.store),

synapse/handlers/message.py

@@ -246,7 +246,7 @@ class MessageHandler:
                 room_state = room_state_events[membership_event_id]
 
         now = self.clock.time_msec()
-        events = self._event_serializer.serialize_events(room_state.values(), now)
+        events = await self._event_serializer.serialize_events(room_state.values(), now)
         return events
 
     async def get_joined_members(self, requester: Requester, room_id: str) -> dict:
@@ -277,7 +277,7 @@ class MessageHandler:
         # If this is an AS, double check that they are allowed to see the members.
         # This can either be because the AS user is in the room or because there
         # is a user in the room that the AS is "interested in"
-        if requester.app_service and user_id not in users_with_profile:
+        if False and requester.app_service and user_id not in users_with_profile:
             for uid in users_with_profile:
                 if requester.app_service.is_interested_in_user(uid):
                     break

synapse/handlers/pagination.py

@@ -537,16 +537,14 @@ class PaginationHandler:
                 state_dict = await self.store.get_events(list(state_ids.values()))
                 state = state_dict.values()
 
-        aggregations = await self.store.get_bundled_aggregations(events, user_id)
-
         time_now = self.clock.time_msec()
 
         chunk = {
             "chunk": (
-                self._event_serializer.serialize_events(
+                await self._event_serializer.serialize_events(
                     events,
                     time_now,
-                    bundle_aggregations=aggregations,
+                    bundle_aggregations=True,
                     as_client_event=as_client_event,
                 )
             ),
@@ -555,7 +553,7 @@ class PaginationHandler:
         }
 
         if state:
-            chunk["state"] = self._event_serializer.serialize_events(
+            chunk["state"] = await self._event_serializer.serialize_events(
                 state, time_now, as_client_event=as_client_event
             )
 

synapse/handlers/presence.py

@@ -55,6 +55,7 @@ from synapse.api.presence import UserPresenceState
 from synapse.appservice import ApplicationService
 from synapse.events.presence_router import PresenceRouter
 from synapse.logging.context import run_in_background
+from synapse.logging.utils import log_function
 from synapse.metrics import LaterGauge
 from synapse.metrics.background_process_metrics import run_as_background_process
 from synapse.replication.http.presence import (
@@ -1541,6 +1542,7 @@ class PresenceEventSource(EventSource[int, UserPresenceState]):
         self.clock = hs.get_clock()
         self.store = hs.get_datastore()
 
+    @log_function
     async def get_new_events(
         self,
         user: UserID,

synapse/handlers/profile.py

@@ -31,8 +31,6 @@ from synapse.types import (
     create_requester,
     get_domain_from_id,
 )
-from synapse.util.caches.descriptors import cached
-from synapse.util.stringutils import parse_and_validate_mxc_uri
 
 if TYPE_CHECKING:
     from synapse.server import HomeServer
@@ -66,11 +64,6 @@ class ProfileHandler:
         self.user_directory_handler = hs.get_user_directory_handler()
         self.request_ratelimiter = hs.get_request_ratelimiter()
 
-        self.max_avatar_size = hs.config.server.max_avatar_size
-        self.allowed_avatar_mimetypes = hs.config.server.allowed_avatar_mimetypes
-
-        self.server_name = hs.config.server.server_name
-
         if hs.config.worker.run_background_tasks:
             self.clock.looping_call(
                 self._update_remote_profile_cache, self.PROFILE_UPDATE_MS
@@ -293,9 +286,6 @@ class ProfileHandler:
                 400, "Avatar URL is too long (max %i)" % (MAX_AVATAR_URL_LEN,)
             )
 
-        if not await self.check_avatar_size_and_mime_type(new_avatar_url):
-            raise SynapseError(403, "This avatar is not allowed", Codes.FORBIDDEN)
-
         avatar_url_to_set: Optional[str] = new_avatar_url
         if new_avatar_url == "":
             avatar_url_to_set = None
@@ -317,63 +307,6 @@ class ProfileHandler:
 
         await self._update_join_states(requester, target_user)
 
-    @cached()
-    async def check_avatar_size_and_mime_type(self, mxc: str) -> bool:
-        """Check that the size and content type of the avatar at the given MXC URI are
-        within the configured limits.
-
-        Args:
-            mxc: The MXC URI at which the avatar can be found.
-
-        Returns:
-             A boolean indicating whether the file can be allowed to be set as an avatar.
-        """
-        if not self.max_avatar_size and not self.allowed_avatar_mimetypes:
-            return True
-
-        server_name, _, media_id = parse_and_validate_mxc_uri(mxc)
-
-        if server_name == self.server_name:
-            media_info = await self.store.get_local_media(media_id)
-        else:
-            media_info = await self.store.get_cached_remote_media(server_name, media_id)
-
-        if media_info is None:
-            # Both configuration options need to access the file's metadata, and
-            # retrieving remote avatars just for this becomes a bit of a faff, especially
-            # if e.g. the file is too big. It's also generally safe to assume most files
-            # used as avatar are uploaded locally, or if the upload didn't happen as part
-            # of a PUT request on /avatar_url that the file was at least previewed by the
-            # user locally (and therefore downloaded to the remote media cache).
-            logger.warning("Forbidding avatar change to %s: avatar not on server", mxc)
-            return False
-
-        if self.max_avatar_size:
-            # Ensure avatar does not exceed max allowed avatar size
-            if media_info["media_length"] > self.max_avatar_size:
-                logger.warning(
-                    "Forbidding avatar change to %s: %d bytes is above the allowed size "
-                    "limit",
-                    mxc,
-                    media_info["media_length"],
-                )
-                return False
-
-        if self.allowed_avatar_mimetypes:
-            # Ensure the avatar's file type is allowed
-            if (
-                self.allowed_avatar_mimetypes
-                and media_info["media_type"] not in self.allowed_avatar_mimetypes
-            ):
-                logger.warning(
-                    "Forbidding avatar change to %s: mimetype %s not allowed",
-                    mxc,
-                    media_info["media_type"],
-                )
-                return False
-
-        return True
-
     async def on_profile_query(self, args: JsonDict) -> JsonDict:
         """Handles federation profile query requests."""
 

synapse/handlers/register.py

@@ -132,7 +132,6 @@ class RegistrationHandler:
         localpart: str,
         guest_access_token: Optional[str] = None,
         assigned_user_id: Optional[str] = None,
-        inhibit_user_in_use_error: bool = False,
     ) -> None:
         if types.contains_invalid_mxid_characters(localpart):
             raise SynapseError(
@@ -172,22 +171,21 @@ class RegistrationHandler:
 
         users = await self.store.get_users_by_id_case_insensitive(user_id)
         if users:
-            if not inhibit_user_in_use_error and not guest_access_token:
+            if not guest_access_token:
                 raise SynapseError(
                     400, "User ID already taken.", errcode=Codes.USER_IN_USE
                 )
-            if guest_access_token:
-                user_data = await self.auth.get_user_by_access_token(guest_access_token)
-                if (
-                    not user_data.is_guest
-                    or UserID.from_string(user_data.user_id).localpart != localpart
-                ):
-                    raise AuthError(
-                        403,
-                        "Cannot register taken user ID without valid guest "
-                        "credentials for that user.",
-                        errcode=Codes.FORBIDDEN,
-                    )
+            user_data = await self.auth.get_user_by_access_token(guest_access_token)
+            if (
+                not user_data.is_guest
+                or UserID.from_string(user_data.user_id).localpart != localpart
+            ):
+                raise AuthError(
+                    403,
+                    "Cannot register taken user ID without valid guest "
+                    "credentials for that user.",
+                    errcode=Codes.FORBIDDEN,
+                )
 
         if guest_access_token is None:
             try:

synapse/handlers/room.py

@@ -30,7 +30,6 @@ from typing import (
     Tuple,
 )
 
-import attr
 from typing_extensions import TypedDict
 
 from synapse.api.constants import (
@@ -61,7 +60,6 @@ from synapse.events.utils import copy_power_levels_contents
 from synapse.federation.federation_client import InvalidResponseError
 from synapse.handlers.federation import get_domains_from_state
 from synapse.rest.admin._base import assert_user_is_admin
-from synapse.storage.databases.main.relations import BundledAggregations
 from synapse.storage.state import StateFilter
 from synapse.streams import EventSource
 from synapse.types import (
@@ -92,17 +90,6 @@ id_server_scheme = "https://"
 FIVE_MINUTES_IN_MS = 5 * 60 * 1000
 
 
-@attr.s(slots=True, frozen=True, auto_attribs=True)
-class EventContext:
-    events_before: List[EventBase]
-    event: EventBase
-    events_after: List[EventBase]
-    state: List[EventBase]
-    aggregations: Dict[str, BundledAggregations]
-    start: str
-    end: str
-
-
 class RoomCreationHandler:
     def __init__(self, hs: "HomeServer"):
         self.store = hs.get_datastore()
@@ -406,9 +393,7 @@ class RoomCreationHandler:
         user_id = requester.user.to_string()
 
         if not await self.spam_checker.user_may_create_room(user_id):
-            raise SynapseError(
-                403, "You are not permitted to create rooms", Codes.FORBIDDEN
-            )
+            raise SynapseError(403, "You are not permitted to create rooms")
 
         creation_content: JsonDict = {
             "room_version": new_room_version.identifier,
@@ -700,9 +685,7 @@ class RoomCreationHandler:
                 invite_3pid_list,
             )
         ):
-            raise SynapseError(
-                403, "You are not permitted to create rooms", Codes.FORBIDDEN
-            )
+            raise SynapseError(403, "You are not permitted to create rooms")
 
         if ratelimit:
             await self.request_ratelimiter.ratelimit(requester)
@@ -1132,7 +1115,7 @@ class RoomContextHandler:
         limit: int,
         event_filter: Optional[Filter],
         use_admin_priviledge: bool = False,
-    ) -> Optional[EventContext]:
+    ) -> Optional[JsonDict]:
         """Retrieves events, pagination tokens and state around a given event
         in a room.
 
@@ -1180,28 +1163,22 @@ class RoomContextHandler:
         results = await self.store.get_events_around(
             room_id, event_id, before_limit, after_limit, event_filter
         )
-        events_before = results.events_before
-        events_after = results.events_after
 
         if event_filter:
-            events_before = await event_filter.filter(events_before)
-            events_after = await event_filter.filter(events_after)
+            results["events_before"] = await event_filter.filter(
+                results["events_before"]
+            )
+            results["events_after"] = await event_filter.filter(results["events_after"])
 
-        events_before = await filter_evts(events_before)
-        events_after = await filter_evts(events_after)
+        results["events_before"] = await filter_evts(results["events_before"])
+        results["events_after"] = await filter_evts(results["events_after"])
         # filter_evts can return a pruned event in case the user is allowed to see that
         # there's something there but not see the content, so use the event that's in
         # `filtered` rather than the event we retrieved from the datastore.
-        event = filtered[0]
+        results["event"] = filtered[0]
 
-        # Fetch the aggregations.
-        aggregations = await self.store.get_bundled_aggregations(
-            itertools.chain(events_before, (event,), events_after),
-            user.to_string(),
-        )
-
-        if events_after:
-            last_event_id = events_after[-1].event_id
+        if results["events_after"]:
+            last_event_id = results["events_after"][-1].event_id
         else:
             last_event_id = event_id
 
@@ -1209,9 +1186,9 @@ class RoomContextHandler:
             state_filter = StateFilter.from_lazy_load_member_list(
                 ev.sender
                 for ev in itertools.chain(
-                    events_before,
-                    (event,),
-                    events_after,
+                    results["events_before"],
+                    (results["event"],),
+                    results["events_after"],
                 )
             )
         else:
@@ -1229,23 +1206,21 @@ class RoomContextHandler:
         if event_filter:
             state_events = await event_filter.filter(state_events)
 
+        results["state"] = await filter_evts(state_events)
+
         # We use a dummy token here as we only care about the room portion of
         # the token, which we replace.
         token = StreamToken.START
 
-        return EventContext(
-            events_before=events_before,
-            event=event,
-            events_after=events_after,
-            state=await filter_evts(state_events),
-            aggregations=aggregations,
-            start=await token.copy_and_replace("room_key", results.start).to_string(
-                self.store
-            ),
-            end=await token.copy_and_replace("room_key", results.end).to_string(
-                self.store
-            ),
-        )
+        results["start"] = await token.copy_and_replace(
+            "room_key", results["start"]
+        ).to_string(self.store)
+
+        results["end"] = await token.copy_and_replace(
+            "room_key", results["end"]
+        ).to_string(self.store)
+
+        return results
 
 
 class TimestampLookupHandler:

synapse/handlers/room_member.py

@@ -82,6 +82,7 @@ class RoomMemberHandler(metaclass=abc.ABCMeta):
         self.event_auth_handler = hs.get_event_auth_handler()
 
         self.member_linearizer: Linearizer = Linearizer(name="member")
+        self.member_limiter = Linearizer(max_count=10, name="member_as_limiter")
 
         self.clock = hs.get_clock()
         self.spam_checker = hs.get_spam_checker()
@@ -482,24 +483,43 @@ class RoomMemberHandler(metaclass=abc.ABCMeta):
 
         key = (room_id,)
 
-        with (await self.member_linearizer.queue(key)):
-            result = await self.update_membership_locked(
-                requester,
-                target,
-                room_id,
-                action,
-                txn_id=txn_id,
-                remote_room_hosts=remote_room_hosts,
-                third_party_signed=third_party_signed,
-                ratelimit=ratelimit,
-                content=content,
-                new_room=new_room,
-                require_consent=require_consent,
-                outlier=outlier,
-                historical=historical,
-                prev_event_ids=prev_event_ids,
-                auth_event_ids=auth_event_ids,
-            )
+        as_id = object()
+        if requester.app_service:
+            as_id = requester.app_service.id
+
+        then = self.clock.time_msec()
+
+        with (await self.member_limiter.queue(as_id)):
+            diff = self.clock.time_msec() - then
+
+            if diff > 80 * 1000:
+                # haproxy would have timed the request out anyway...
+                raise SynapseError(504, "took to long to process")
+
+            with (await self.member_linearizer.queue(key)):
+                diff = self.clock.time_msec() - then
+
+                if diff > 80 * 1000:
+                    # haproxy would have timed the request out anyway...
+                    raise SynapseError(504, "took to long to process")
+
+                result = await self.update_membership_locked(
+                    requester,
+                    target,
+                    room_id,
+                    action,
+                    txn_id=txn_id,
+                    remote_room_hosts=remote_room_hosts,
+                    third_party_signed=third_party_signed,
+                    ratelimit=ratelimit,
+                    content=content,
+                    new_room=new_room,
+                    require_consent=require_consent,
+                    outlier=outlier,
+                    historical=historical,
+                    prev_event_ids=prev_event_ids,
+                    auth_event_ids=auth_event_ids,
+                )
 
         return result
 
@@ -590,12 +610,6 @@ class RoomMemberHandler(metaclass=abc.ABCMeta):
                 errcode=Codes.BAD_JSON,
             )
 
-        if "avatar_url" in content:
-            if not await self.profile_handler.check_avatar_size_and_mime_type(
-                content["avatar_url"],
-            ):
-                raise SynapseError(403, "This avatar is not allowed", Codes.FORBIDDEN)
-
         # The event content should *not* include the authorising user as
         # it won't be properly signed. Strip it out since it might come
         # back from a client updating a display name / avatar.

synapse/handlers/room_summary.py

@@ -153,9 +153,6 @@ class RoomSummaryHandler:
         rooms_result: List[JsonDict] = []
         events_result: List[JsonDict] = []
 
-        if max_rooms_per_space is None or max_rooms_per_space > MAX_ROOMS_PER_SPACE:
-            max_rooms_per_space = MAX_ROOMS_PER_SPACE
-
         while room_queue and len(rooms_result) < MAX_ROOMS:
             queue_entry = room_queue.popleft()
             room_id = queue_entry.room_id
@@ -170,7 +167,7 @@ class RoomSummaryHandler:
             # The client-specified max_rooms_per_space limit doesn't apply to the
             # room_id specified in the request, so we ignore it if this is the
             # first room we are processing.
-            max_children = max_rooms_per_space if processed_rooms else MAX_ROOMS
+            max_children = max_rooms_per_space if processed_rooms else None
 
             if is_in_room:
                 room_entry = await self._summarize_local_room(
@@ -212,7 +209,7 @@ class RoomSummaryHandler:
                         # Before returning to the client, remove the allowed_room_ids
                         # and allowed_spaces keys.
                         room.pop("allowed_room_ids", None)
-                        room.pop("allowed_spaces", None)  # historical
+                        room.pop("allowed_spaces", None)
 
                         rooms_result.append(room)
                         events.extend(room_entry.children_state_events)
@@ -398,7 +395,7 @@ class RoomSummaryHandler:
                     None,
                     room_id,
                     suggested_only,
-                    # Do not limit the maximum children.
+                    # TODO Handle max children.
                     max_children=None,
                 )
 
@@ -528,10 +525,6 @@ class RoomSummaryHandler:
         rooms_result: List[JsonDict] = []
         events_result: List[JsonDict] = []
 
-        # Set a limit on the number of rooms to return.
-        if max_rooms_per_space is None or max_rooms_per_space > MAX_ROOMS_PER_SPACE:
-            max_rooms_per_space = MAX_ROOMS_PER_SPACE
-
         while room_queue and len(rooms_result) < MAX_ROOMS:
             room_id = room_queue.popleft()
             if room_id in processed_rooms:
@@ -590,9 +583,7 @@ class RoomSummaryHandler:
 
         # Iterate through each child and potentially add it, but not its children,
         # to the response.
-        for child_room in itertools.islice(
-            root_room_entry.children_state_events, MAX_ROOMS_PER_SPACE
-        ):
+        for child_room in root_room_entry.children_state_events:
             room_id = child_room.get("state_key")
             assert isinstance(room_id, str)
             # If the room is unknown, skip it.
@@ -642,8 +633,8 @@ class RoomSummaryHandler:
             suggested_only: True if only suggested children should be returned.
                 Otherwise, all children are returned.
             max_children:
-                The maximum number of children rooms to include. A value of None
-                means no limit.
+                The maximum number of children rooms to include. This is capped
+                to a server-set limit.
 
         Returns:
             A room entry if the room should be returned. None, otherwise.
@@ -665,13 +656,8 @@ class RoomSummaryHandler:
             # we only care about suggested children
             child_events = filter(_is_suggested_child_event, child_events)
 
-        # TODO max_children is legacy code for the /spaces endpoint.
-        if max_children is not None:
-            child_iter: Iterable[EventBase] = itertools.islice(
-                child_events, max_children
-            )
-        else:
-            child_iter = child_events
+        if max_children is None or max_children > MAX_ROOMS_PER_SPACE:
+            max_children = MAX_ROOMS_PER_SPACE
 
         stripped_events: List[JsonDict] = [
             {
@@ -682,7 +668,7 @@ class RoomSummaryHandler:
                 "sender": e.sender,
                 "origin_server_ts": e.origin_server_ts,
             }
-            for e in child_iter
+            for e in itertools.islice(child_events, max_children)
         ]
         return _RoomEntry(room_id, room_entry, stripped_events)
 
@@ -780,7 +766,6 @@ class RoomSummaryHandler:
         try:
             (
                 room_response,
-                children_state_events,
                 children,
                 inaccessible_children,
             ) = await self._federation_client.get_room_hierarchy(
@@ -805,7 +790,7 @@ class RoomSummaryHandler:
         }
 
         return (
-            _RoomEntry(room_id, room_response, children_state_events),
+            _RoomEntry(room_id, room_response, room_response.pop("children_state", ())),
             children_by_room_id,
             set(inaccessible_children),
         )
@@ -1003,14 +988,12 @@ class RoomSummaryHandler:
             "canonical_alias": stats["canonical_alias"],
             "num_joined_members": stats["joined_members"],
             "avatar_url": stats["avatar"],
-            # plural join_rules is a documentation error but kept for historical
-            # purposes. Should match /publicRooms.
             "join_rules": stats["join_rules"],
-            "join_rule": stats["join_rules"],
             "world_readable": (
                 stats["history_visibility"] == HistoryVisibility.WORLD_READABLE
             ),
             "guest_can_join": stats["guest_access"] == "can_join",
+            "creation_ts": create_event.origin_server_ts,
             "room_type": create_event.content.get(EventContentFields.ROOM_TYPE),
         }
 

synapse/handlers/search.py

@@ -361,37 +361,36 @@ class SearchHandler:
 
                 logger.info(
                     "Context for search returned %d and %d events",
-                    len(res.events_before),
-                    len(res.events_after),
+                    len(res["events_before"]),
+                    len(res["events_after"]),
                 )
 
-                events_before = await filter_events_for_client(
-                    self.storage, user.to_string(), res.events_before
+                res["events_before"] = await filter_events_for_client(
+                    self.storage, user.to_string(), res["events_before"]
                 )
 
-                events_after = await filter_events_for_client(
-                    self.storage, user.to_string(), res.events_after
+                res["events_after"] = await filter_events_for_client(
+                    self.storage, user.to_string(), res["events_after"]
                 )
 
-                context = {
-                    "events_before": events_before,
-                    "events_after": events_after,
-                    "start": await now_token.copy_and_replace(
-                        "room_key", res.start
-                    ).to_string(self.store),
-                    "end": await now_token.copy_and_replace(
-                        "room_key", res.end
-                    ).to_string(self.store),
-                }
+                res["start"] = await now_token.copy_and_replace(
+                    "room_key", res["start"]
+                ).to_string(self.store)
+
+                res["end"] = await now_token.copy_and_replace(
+                    "room_key", res["end"]
+                ).to_string(self.store)
 
                 if include_profile:
                     senders = {
                         ev.sender
-                        for ev in itertools.chain(events_before, [event], events_after)
+                        for ev in itertools.chain(
+                            res["events_before"], [event], res["events_after"]
+                        )
                     }
 
-                    if events_after:
-                        last_event_id = events_after[-1].event_id
+                    if res["events_after"]:
+                        last_event_id = res["events_after"][-1].event_id
                     else:
                         last_event_id = event.event_id
 
@@ -403,7 +402,7 @@ class SearchHandler:
                         last_event_id, state_filter
                     )
 
-                    context["profile_info"] = {
+                    res["profile_info"] = {
                         s.state_key: {
                             "displayname": s.content.get("displayname", None),
                             "avatar_url": s.content.get("avatar_url", None),
@@ -412,7 +411,7 @@ class SearchHandler:
                         if s.type == EventTypes.Member and s.state_key in senders
                     }
 
-                contexts[event.event_id] = context
+                contexts[event.event_id] = res
         else:
             contexts = {}
 
@@ -421,11 +420,11 @@ class SearchHandler:
         time_now = self.clock.time_msec()
 
         for context in contexts.values():
-            context["events_before"] = self._event_serializer.serialize_events(
-                context["events_before"], time_now  # type: ignore[arg-type]
+            context["events_before"] = await self._event_serializer.serialize_events(
+                context["events_before"], time_now
             )
-            context["events_after"] = self._event_serializer.serialize_events(
-                context["events_after"], time_now  # type: ignore[arg-type]
+            context["events_after"] = await self._event_serializer.serialize_events(
+                context["events_after"], time_now
             )
 
         state_results = {}
@@ -442,7 +441,9 @@ class SearchHandler:
             results.append(
                 {
                     "rank": rank_map[e.event_id],
-                    "result": self._event_serializer.serialize_event(e, time_now),
+                    "result": (
+                        await self._event_serializer.serialize_event(e, time_now)
+                    ),
                     "context": contexts.get(e.event_id, {}),
                 }
             )
@@ -456,7 +457,7 @@ class SearchHandler:
         if state_results:
             s = {}
             for room_id, state_events in state_results.items():
-                s[room_id] = self._event_serializer.serialize_events(
+                s[room_id] = await self._event_serializer.serialize_events(
                     state_events, time_now
                 )
 

synapse/handlers/sso.py

@@ -126,45 +126,45 @@ class SsoIdentityProvider(Protocol):
         raise NotImplementedError()
 
 
-@attr.s(auto_attribs=True)
+@attr.s
 class UserAttributes:
     # the localpart of the mxid that the mapper has assigned to the user.
     # if `None`, the mapper has not picked a userid, and the user should be prompted to
     # enter one.
-    localpart: Optional[str]
-    display_name: Optional[str] = None
-    emails: Collection[str] = attr.Factory(list)
+    localpart = attr.ib(type=Optional[str])
+    display_name = attr.ib(type=Optional[str], default=None)
+    emails = attr.ib(type=Collection[str], default=attr.Factory(list))
 
 
-@attr.s(slots=True, auto_attribs=True)
+@attr.s(slots=True)
 class UsernameMappingSession:
     """Data we track about SSO sessions"""
 
     # A unique identifier for this SSO provider, e.g.  "oidc" or "saml".
-    auth_provider_id: str
+    auth_provider_id = attr.ib(type=str)
 
     # user ID on the IdP server
-    remote_user_id: str
+    remote_user_id = attr.ib(type=str)
 
     # attributes returned by the ID mapper
-    display_name: Optional[str]
-    emails: Collection[str]
+    display_name = attr.ib(type=Optional[str])
+    emails = attr.ib(type=Collection[str])
 
     # An optional dictionary of extra attributes to be provided to the client in the
     # login response.
-    extra_login_attributes: Optional[JsonDict]
+    extra_login_attributes = attr.ib(type=Optional[JsonDict])
 
     # where to redirect the client back to
-    client_redirect_url: str
+    client_redirect_url = attr.ib(type=str)
 
     # expiry time for the session, in milliseconds
-    expiry_time_ms: int
+    expiry_time_ms = attr.ib(type=int)
 
     # choices made by the user
-    chosen_localpart: Optional[str] = None
-    use_display_name: bool = True
-    emails_to_use: Collection[str] = ()
-    terms_accepted_version: Optional[str] = None
+    chosen_localpart = attr.ib(type=Optional[str], default=None)
+    use_display_name = attr.ib(type=bool, default=True)
+    emails_to_use = attr.ib(type=Collection[str], default=())
+    terms_accepted_version = attr.ib(type=Optional[str], default=None)
 
 
 # the HTTP cookie used to track the mapping session id

synapse/handlers/sync.py

@@ -37,7 +37,6 @@ from synapse.logging.context import current_context
 from synapse.logging.opentracing import SynapseTags, log_kv, set_tag, start_active_span
 from synapse.push.clientformat import format_push_rules_for_user
 from synapse.storage.databases.main.event_push_actions import NotifCounts
-from synapse.storage.databases.main.relations import BundledAggregations
 from synapse.storage.roommember import MemberSummary
 from synapse.storage.state import StateFilter
 from synapse.types import (
@@ -61,6 +60,10 @@ if TYPE_CHECKING:
 
 logger = logging.getLogger(__name__)
 
+# Debug logger for https://github.com/matrix-org/synapse/issues/4422
+issue4422_logger = logging.getLogger("synapse.handler.sync.4422_debug")
+
+
 # Counts the number of times we returned a non-empty sync. `type` is one of
 # "initial_sync", "full_state_sync" or "incremental_sync", `lazy_loaded` is
 # "true" or "false" depending on if the request asked for lazy loaded members or
@@ -99,9 +102,6 @@ class TimelineBatch:
     prev_batch: StreamToken
     events: List[EventBase]
     limited: bool
-    # A mapping of event ID to the bundled aggregations for the above events.
-    # This is only calculated if limited is true.
-    bundled_aggregations: Optional[Dict[str, BundledAggregations]] = None
 
     def __bool__(self) -> bool:
         """Make the result appear empty if there are no updates. This is used
@@ -634,19 +634,10 @@ class SyncHandler:
 
             prev_batch_token = now_token.copy_and_replace("room_key", room_key)
 
-        # Don't bother to bundle aggregations if the timeline is unlimited,
-        # as clients will have all the necessary information.
-        bundled_aggregations = None
-        if limited or newly_joined_room:
-            bundled_aggregations = await self.store.get_bundled_aggregations(
-                recents, sync_config.user.to_string()
-            )
-
         return TimelineBatch(
             events=recents,
             prev_batch=prev_batch_token,
             limited=limited or newly_joined_room,
-            bundled_aggregations=bundled_aggregations,
         )
 
     async def get_state_after_event(
@@ -1170,8 +1161,13 @@ class SyncHandler:
 
         num_events = 0
 
-        # debug for https://github.com/matrix-org/synapse/issues/9424
+        # debug for https://github.com/matrix-org/synapse/issues/4422
         for joined_room in sync_result_builder.joined:
+            room_id = joined_room.room_id
+            if room_id in newly_joined_rooms:
+                issue4422_logger.debug(
+                    "Sync result for newly joined room %s: %r", room_id, joined_room
+                )
             num_events += len(joined_room.timeline.events)
 
         log_kv(
@@ -1620,7 +1616,7 @@ class SyncHandler:
         # TODO: Can we `SELECT ignored_user_id FROM ignored_users WHERE ignorer_user_id=?;` instead?
         ignored_account_data = (
             await self.store.get_global_account_data_by_type_for_user(
-                user_id=user_id, data_type=AccountDataTypes.IGNORED_USER_LIST
+                AccountDataTypes.IGNORED_USER_LIST, user_id=user_id
             )
         )
 
@@ -1744,6 +1740,18 @@ class SyncHandler:
                         old_mem_ev_id, allow_none=True
                     )
 
+                # debug for #4422
+                if has_join:
+                    prev_membership = None
+                    if old_mem_ev:
+                        prev_membership = old_mem_ev.membership
+                    issue4422_logger.debug(
+                        "Previous membership for room %s with join: %s (event %s)",
+                        room_id,
+                        prev_membership,
+                        old_mem_ev_id,
+                    )
+
                 if not old_mem_ev or old_mem_ev.membership != Membership.JOIN:
                     newly_joined_rooms.append(room_id)
 
@@ -1885,6 +1893,13 @@ class SyncHandler:
                     upto_token=since_token,
                 )
 
+            if newly_joined:
+                # debugging for https://github.com/matrix-org/synapse/issues/4422
+                issue4422_logger.debug(
+                    "RoomSyncResultBuilder events for newly joined room %s: %r",
+                    room_id,
+                    entry.events,
+                )
             room_entries.append(entry)
 
         return _RoomChanges(
@@ -2062,6 +2077,14 @@ class SyncHandler:
             # `_load_filtered_recents` can't find any events the user should see
             # (e.g. due to having ignored the sender of the last 50 events).
 
+            if newly_joined:
+                # debug for https://github.com/matrix-org/synapse/issues/4422
+                issue4422_logger.debug(
+                    "Timeline events after filtering in newly-joined room %s: %r",
+                    room_id,
+                    batch,
+                )
+
             # When we join the room (or the client requests full_state), we should
             # send down any existing tags. Usually the user won't have tags in a
             # newly joined room, unless either a) they've joined before or b) the

synapse/http/client.py

@@ -731,24 +731,15 @@ class SimpleHttpClient:
         # straight back in again
 
         try:
-            d = read_body_with_max_size(response, output_stream, max_size)
-
-            # Ensure that the body is not read forever.
-            d = timeout_deferred(d, 30, self.hs.get_reactor())
-
-            length = await make_deferred_yieldable(d)
+            length = await make_deferred_yieldable(
+                read_body_with_max_size(response, output_stream, max_size)
+            )
         except BodyExceededMaxSize:
             raise SynapseError(
                 HTTPStatus.BAD_GATEWAY,
                 "Requested file is too large > %r bytes" % (max_size,),
                 Codes.TOO_LARGE,
             )
-        except defer.TimeoutError:
-            raise SynapseError(
-                HTTPStatus.BAD_GATEWAY,
-                "Requested file took too long to download",
-                Codes.TOO_LARGE,
-            )
         except Exception as e:
             raise SynapseError(
                 HTTPStatus.BAD_GATEWAY, ("Failed to download remote body: %s" % e)

synapse/http/connectproxyclient.py

@@ -32,9 +32,9 @@ class ProxyConnectError(ConnectError):
     pass
 
 
-@attr.s(auto_attribs=True)
+@attr.s
 class ProxyCredentials:
-    username_password: bytes
+    username_password = attr.ib(type=bytes)
 
     def as_proxy_authorization_value(self) -> bytes:
         """