WIP: debug

.github/workflows/dependabot_changelog.yml

@@ -0,0 +1,49 @@
+name: Write changelog for dependabot PR
+on:
+  pull_request:
+    types:
+      - opened
+      - reopened  # For debugging!
+
+permissions:
+  # Needed to be able to push the commit. See
+  #     https://docs.github.com/en/code-security/dependabot/working-with-dependabot/automating-dependabot-with-github-actions#enable-auto-merge-on-a-pull-request
+  # for a similar example
+  contents: write
+
+jobs:
+  add-changelog:
+    runs-on: 'ubuntu-latest'
+    if: ${{ github.actor == 'dependabot[bot]' }}
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          ref: ${{ github.event.pull_request.head.ref }}
+      - name: Write, commit and push changelog
+        env:
+          PR_TITLE: ${{ github.event.pull_request.title }}
+          PR_NUMBER: ${{ github.event.pull_request.number }}
+        run: |
+          echo "${PR_TITLE}." > "changelog.d/${PR_NUMBER}".misc
+          git add changelog.d
+          git config user.email "github-actions[bot]@users.noreply.github.com"
+          git config user.name "GitHub Actions"
+          git commit -m "Changelog"
+          git push
+        shell: bash
+      # The `git push` above does not trigger CI on the dependabot PR.
+      #
+      # By default, workflows can't trigger other workflows when they're just using the
+      # default `GITHUB_TOKEN` access token. (This is intended to stop you from writing
+      # recursive workflow loops by accident, because that'll get very expensive very
+      # quickly.) Instead, you have to manually call out to another workflow, or else
+      # make your changes (i.e. the `git push` above) using a personal access token.
+      # See
+      # https://docs.github.com/en/actions/using-workflows/triggering-a-workflow#triggering-a-workflow-from-a-workflow
+      #
+      # I have tried and failed to find a way to trigger CI on the "merge ref" of the PR.
+      # See git commit history for previous attempts. If anyone desperately wants to try
+      # again in the future, make a matrix-bot account and use its access token to git push.
+
+  # THIS WORKFLOW HAS WRITE PERMISSIONS---do not add other jobs here unless they
+  # are sufficiently locked down to dependabot only as above.

.github/workflows/latest_deps.yml

@@ -22,21 +22,7 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
-  check_repo:
-    # Prevent this workflow from running on any fork of Synapse other than matrix-org/synapse, as it is
-    # only useful to the Synapse core team.
-    # All other workflow steps depend on this one, thus if 'should_run_workflow' is not 'true', the rest
-    # of the workflow will be skipped as well.
-    runs-on: ubuntu-latest
-    outputs:
-      should_run_workflow: ${{ steps.check_condition.outputs.should_run_workflow }}
-    steps:
-      - id: check_condition
-        run: echo "should_run_workflow=${{ github.repository == 'matrix-org/synapse' }}" >> "$GITHUB_OUTPUT"
-
   mypy:
-    needs: check_repo
-    if: needs.check_repo.outputs.should_run_workflow == 'true'
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3
@@ -61,8 +47,6 @@ jobs:
         run: sed '/warn_unused_ignores = True/d' -i mypy.ini
       - run: poetry run mypy
   trial:
-    needs: check_repo
-    if: needs.check_repo.outputs.should_run_workflow == 'true'
     runs-on: ubuntu-latest
     strategy:
       matrix:
@@ -121,8 +105,6 @@ jobs:
 
 
   sytest:
-    needs: check_repo
-    if: needs.check_repo.outputs.should_run_workflow == 'true'
     runs-on: ubuntu-latest
     container:
       image: matrixdotorg/sytest-synapse:testing
@@ -174,8 +156,7 @@ jobs:
 
 
   complement:
-    needs: check_repo
-    if: "!failure() && !cancelled() && needs.check_repo.outputs.should_run_workflow == 'true'"
+    if: "${{ !failure() && !cancelled() }}"
     runs-on: ubuntu-latest
 
     strategy:
@@ -211,7 +192,7 @@ jobs:
   # Open an issue if the build fails, so we know about it.
   # Only do this if we're not experimenting with this action in a PR.
   open-issue:
-    if: "failure() && github.event_name != 'push' && github.event_name != 'pull_request' && needs.check_repo.outputs.should_run_workflow == 'true'"
+    if: "failure() && github.event_name != 'push' && github.event_name != 'pull_request'"
     needs:
       # TODO: should mypy be included here? It feels more brittle than the others.
       - mypy

.github/workflows/release-artifacts.yml

@@ -34,7 +34,6 @@ jobs:
       - id: set-distros
         run: |
           # if we're running from a tag, get the full list of distros; otherwise just use debian:sid
-          # NOTE: inside the actual Dockerfile-dhvirtualenv, the image name is expanded into its full image path
           dists='["debian:sid"]'
           if [[ $GITHUB_REF == refs/tags/* ]]; then
               dists=$(scripts-dev/build_debian_packages.py --show-dists-json)

.github/workflows/tests.yml

@@ -35,7 +35,7 @@ jobs:
     steps:
       - uses: actions/checkout@v3
       - name: Install Rust
-        uses: dtolnay/rust-toolchain@1.60.0
+        uses: dtolnay/rust-toolchain@1.58.1
       - uses: Swatinem/rust-cache@v2
       - uses: matrix-org/setup-python-poetry@v1
         with:
@@ -92,10 +92,6 @@ jobs:
       - name: Checkout repository
         uses: actions/checkout@v3
 
-      - name: Install Rust
-        uses: dtolnay/rust-toolchain@1.60.0
-      - uses: Swatinem/rust-cache@v2
-
       - name: Setup Poetry
         uses: matrix-org/setup-python-poetry@v1
         with:
@@ -107,6 +103,10 @@ jobs:
           # To make CI green, err towards caution and install the project.
           install-project: "true"
 
+      - name: Install Rust
+        uses: dtolnay/rust-toolchain@1.58.1
+      - uses: Swatinem/rust-cache@v2
+
       # Cribbed from
       # https://github.com/AustinScola/mypy-cache-github-action/blob/85ea4f2972abed39b33bd02c36e341b28ca59213/src/restore.ts#L10-L17
       - name: Restore/persist mypy's cache
@@ -150,7 +150,7 @@ jobs:
         with:
           ref: ${{ github.event.pull_request.head.sha }}
       - name: Install Rust
-        uses: dtolnay/rust-toolchain@1.60.0
+        uses: dtolnay/rust-toolchain@1.58.1
       - uses: Swatinem/rust-cache@v2
       - uses: matrix-org/setup-python-poetry@v1
         with:
@@ -167,7 +167,7 @@ jobs:
       - uses: actions/checkout@v3
 
       - name: Install Rust
-        uses: dtolnay/rust-toolchain@1.60.0
+        uses: dtolnay/rust-toolchain@1.58.1
         with:
             components: clippy
       - uses: Swatinem/rust-cache@v2
@@ -268,7 +268,7 @@ jobs:
             postgres:${{ matrix.job.postgres-version }}
 
       - name: Install Rust
-        uses: dtolnay/rust-toolchain@1.60.0
+        uses: dtolnay/rust-toolchain@1.58.1
       - uses: Swatinem/rust-cache@v2
 
       - uses: matrix-org/setup-python-poetry@v1
@@ -308,13 +308,13 @@ jobs:
       - uses: actions/checkout@v3
 
       - name: Install Rust
-        uses: dtolnay/rust-toolchain@1.60.0
+        uses: dtolnay/rust-toolchain@1.58.1
       - uses: Swatinem/rust-cache@v2
 
       # There aren't wheels for some of the older deps, so we need to install
       # their build dependencies
       - run: |
-          sudo apt-get -qq update
+          sudo apt update
           sudo apt-get -qq install build-essential libffi-dev python-dev \
             libxml2-dev libxslt-dev xmlsec1 zlib1g-dev libjpeg-dev libwebp-dev
 
@@ -416,7 +416,7 @@ jobs:
         run: cat sytest-blacklist .ci/worker-blacklist > synapse-blacklist-with-workers
 
       - name: Install Rust
-        uses: dtolnay/rust-toolchain@1.60.0
+        uses: dtolnay/rust-toolchain@1.58.1
       - uses: Swatinem/rust-cache@v2
 
       - name: Run SyTest
@@ -556,7 +556,7 @@ jobs:
           path: synapse
 
       - name: Install Rust
-        uses: dtolnay/rust-toolchain@1.60.0
+        uses: dtolnay/rust-toolchain@1.58.1
       - uses: Swatinem/rust-cache@v2
 
       - uses: actions/setup-go@v4
@@ -584,7 +584,7 @@ jobs:
       - uses: actions/checkout@v3
 
       - name: Install Rust
-        uses: dtolnay/rust-toolchain@1.60.0
+        uses: dtolnay/rust-toolchain@1.58.1
       - uses: Swatinem/rust-cache@v2
 
       - run: cargo test

.github/workflows/twisted_trunk.yml

@@ -18,22 +18,7 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
-  check_repo:
-    # Prevent this workflow from running on any fork of Synapse other than matrix-org/synapse, as it is
-    # only useful to the Synapse core team.
-    # All other workflow steps depend on this one, thus if 'should_run_workflow' is not 'true', the rest
-    # of the workflow will be skipped as well.
-    if: github.repository == 'matrix-org/synapse'
-    runs-on: ubuntu-latest
-    outputs:
-      should_run_workflow: ${{ steps.check_condition.outputs.should_run_workflow }}
-    steps:
-      - id: check_condition
-        run: echo "should_run_workflow=${{ github.repository == 'matrix-org/synapse' }}" >> "$GITHUB_OUTPUT"
-
   mypy:
-    needs: check_repo
-    if: needs.check_repo.outputs.should_run_workflow == 'true'
     runs-on: ubuntu-latest
 
     steps:
@@ -56,8 +41,6 @@ jobs:
       - run: poetry run mypy
 
   trial:
-    needs: check_repo
-    if: needs.check_repo.outputs.should_run_workflow == 'true'
     runs-on: ubuntu-latest
 
     steps:
@@ -92,8 +75,6 @@ jobs:
           || true
 
   sytest:
-    needs: check_repo
-    if: needs.check_repo.outputs.should_run_workflow == 'true'
     runs-on: ubuntu-latest
     container:
       image: matrixdotorg/sytest-synapse:buster
@@ -138,8 +119,7 @@ jobs:
             /logs/**/*.log*
 
   complement:
-    needs: check_repo
-    if: "!failure() && !cancelled() && needs.check_repo.outputs.should_run_workflow == 'true'"
+    if: "${{ !failure() && !cancelled() }}"
     runs-on: ubuntu-latest
 
     strategy:
@@ -186,7 +166,7 @@ jobs:
 
   # open an issue if the build fails, so we know about it.
   open-issue:
-    if: failure() && needs.check_repo.outputs.should_run_workflow == 'true'
+    if: failure()
     needs:
       - mypy
       - trial

CHANGES.md

@@ -1,249 +1,3 @@
-Synapse 1.86.0rc2 (2023-06-14)
-==============================
-
-Bugfixes
---------
-
-- Fix an error when having workers of different versions running. ([\#15774](https://github.com/matrix-org/synapse/issues/15774))
-
-
-Synapse 1.86.0rc1 (2023-06-13)
-==============================
-
-This version was tagged but never released.
-
-Features
---------
-
-- Stable support for [MSC3882](https://github.com/matrix-org/matrix-spec-proposals/pull/3882) to allow an existing device/session to generate a login token for use on a new device/session. ([\#15388](https://github.com/matrix-org/synapse/issues/15388))
-- Support resolving a room's [canonical alias](https://spec.matrix.org/v1.7/client-server-api/#mroomcanonical_alias) via the module API. ([\#15450](https://github.com/matrix-org/synapse/issues/15450))
-- Enable support for [MSC3952](https://github.com/matrix-org/matrix-spec-proposals/pull/3952): intentional mentions. ([\#15520](https://github.com/matrix-org/synapse/issues/15520))
-- Experimental [MSC3861](https://github.com/matrix-org/matrix-spec-proposals/pull/3861) support: delegate auth to an OIDC provider. ([\#15582](https://github.com/matrix-org/synapse/issues/15582))
-- Add Synapse version deploy annotations to Grafana dashboard which enables easy correlation between behavior changes witnessed in a graph to a certain Synapse version and nail down regressions. ([\#15674](https://github.com/matrix-org/synapse/issues/15674))
-- Add a catch-all * to the supported relation types when redacting an event and its related events. This is an update to [MSC3912](https://github.com/matrix-org/matrix-spec-proposals/pull/3861) implementation. ([\#15705](https://github.com/matrix-org/synapse/issues/15705))
-- Speed up `/messages` by backfilling in the background when there are no backward extremities where we are directly paginating. ([\#15710](https://github.com/matrix-org/synapse/issues/15710))
-- Expose a metric reporting the database background update status. ([\#15740](https://github.com/matrix-org/synapse/issues/15740))
-
-
-Bugfixes
---------
-
-- Correctly clear caches when we delete a room. ([\#15609](https://github.com/matrix-org/synapse/issues/15609))
-- Check permissions for enabling encryption earlier during room creation to avoid creating broken rooms. ([\#15695](https://github.com/matrix-org/synapse/issues/15695))
-
-
-Improved Documentation
-----------------------
-
-- Simplify query to find participating servers in a room. ([\#15732](https://github.com/matrix-org/synapse/issues/15732))
-
-
-Internal Changes
-----------------
-
-- Log when events are (maybe unexpectedly) filtered out of responses in tests. ([\#14213](https://github.com/matrix-org/synapse/issues/14213))
-- Read from column `full_user_id` rather than `user_id` of tables `profiles` and `user_filters`. ([\#15649](https://github.com/matrix-org/synapse/issues/15649))
-- Add support for tracing functions which return `Awaitable`s. ([\#15650](https://github.com/matrix-org/synapse/issues/15650))
-- Cache requests for user's devices over federation. ([\#15675](https://github.com/matrix-org/synapse/issues/15675))
-- Add fully qualified docker image names to Dockerfiles. ([\#15689](https://github.com/matrix-org/synapse/issues/15689))
-- Remove some unused code. ([\#15690](https://github.com/matrix-org/synapse/issues/15690))
-- Improve type hints. ([\#15694](https://github.com/matrix-org/synapse/issues/15694), [\#15697](https://github.com/matrix-org/synapse/issues/15697))
-- Update docstring and traces on `maybe_backfill()` functions. ([\#15709](https://github.com/matrix-org/synapse/issues/15709))
-- Add context for when/why to use the `long_retries` option when sending Federation requests. ([\#15721](https://github.com/matrix-org/synapse/issues/15721))
-- Removed some unused fields. ([\#15723](https://github.com/matrix-org/synapse/issues/15723))
-- Update federation error to more plainly explain we can only authorize our own membership events. ([\#15725](https://github.com/matrix-org/synapse/issues/15725))
-- Prevent the `latest_deps` and `twisted_trunk` daily GitHub Actions workflows from running on forks of the codebase. ([\#15726](https://github.com/matrix-org/synapse/issues/15726))
-- Improve performance of user directory search. ([\#15729](https://github.com/matrix-org/synapse/issues/15729))
-- Remove redundant table join with `room_memberships` when doing a `is_host_joined()`/`is_host_invited()` call (`membership` is already part of the `current_state_events`). ([\#15731](https://github.com/matrix-org/synapse/issues/15731))
-- Remove superfluous `room_memberships` join from background update. ([\#15733](https://github.com/matrix-org/synapse/issues/15733))
-- Speed up typechecking CI. ([\#15752](https://github.com/matrix-org/synapse/issues/15752))
-- Bump minimum supported Rust version to 1.60.0. ([\#15768](https://github.com/matrix-org/synapse/issues/15768))
-
-### Updates to locked dependencies
-
-* Bump importlib-metadata from 6.1.0 to 6.6.0. ([\#15711](https://github.com/matrix-org/synapse/issues/15711))
-* Bump library/redis from 6-bullseye to 7-bullseye in /docker. ([\#15712](https://github.com/matrix-org/synapse/issues/15712))
-* Bump log from 0.4.18 to 0.4.19. ([\#15761](https://github.com/matrix-org/synapse/issues/15761))
-* Bump phonenumbers from 8.13.11 to 8.13.13. ([\#15763](https://github.com/matrix-org/synapse/issues/15763))
-* Bump pyasn1 from 0.4.8 to 0.5.0. ([\#15713](https://github.com/matrix-org/synapse/issues/15713))
-* Bump pydantic from 1.10.8 to 1.10.9. ([\#15762](https://github.com/matrix-org/synapse/issues/15762))
-* Bump pyo3-log from 0.8.1 to 0.8.2. ([\#15759](https://github.com/matrix-org/synapse/issues/15759))
-* Bump pyopenssl from 23.1.1 to 23.2.0. ([\#15765](https://github.com/matrix-org/synapse/issues/15765))
-* Bump regex from 1.7.3 to 1.8.4. ([\#15769](https://github.com/matrix-org/synapse/issues/15769))
-* Bump sentry-sdk from 1.22.1 to 1.25.0. ([\#15714](https://github.com/matrix-org/synapse/issues/15714))
-* Bump sentry-sdk from 1.25.0 to 1.25.1. ([\#15764](https://github.com/matrix-org/synapse/issues/15764))
-* Bump serde from 1.0.163 to 1.0.164. ([\#15760](https://github.com/matrix-org/synapse/issues/15760))
-* Bump types-jsonschema from 4.17.0.7 to 4.17.0.8. ([\#15716](https://github.com/matrix-org/synapse/issues/15716))
-* Bump types-pyopenssl from 23.1.0.2 to 23.2.0.0. ([\#15766](https://github.com/matrix-org/synapse/issues/15766))
-* Bump types-requests from 2.31.0.0 to 2.31.0.1. ([\#15715](https://github.com/matrix-org/synapse/issues/15715))
-
-Synapse 1.85.2 (2023-06-08)
-===========================
-
-Bugfixes
---------
-
-- Fix regression where using TLS for HTTP replication between workers did not work. Introduced in v1.85.0. ([\#15746](https://github.com/matrix-org/synapse/issues/15746))
-
-
-Synapse 1.85.1 (2023-06-07)
-===========================
-
-Note: this release only fixes a bug that stopped some deployments from upgrading to v1.85.0. There is no need to upgrade to v1.85.1 if successfully running v1.85.0.
-
-Bugfixes
---------
-
-- Fix bug in schema delta that broke upgrades for some deployments. Introduced in v1.85.0. ([\#15738](https://github.com/matrix-org/synapse/issues/15738), [\#15739](https://github.com/matrix-org/synapse/issues/15739))
-
-
-Synapse 1.85.0 (2023-06-06)
-===========================
-
-No significant changes since 1.85.0rc2.
-
-
-## Security advisory
-
-The following issues are fixed in 1.85.0 (and RCs).
-
-- [GHSA-26c5-ppr8-f33p](https://github.com/matrix-org/synapse/security/advisories/GHSA-26c5-ppr8-f33p) / [CVE-2023-32682](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2023-32682) — Low Severity
-
-  It may be possible for a deactivated user to login when using uncommon configurations.
-
-- [GHSA-98px-6486-j7qc](https://github.com/matrix-org/synapse/security/advisories/GHSA-98px-6486-j7qc) / [CVE-2023-32683](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2023-32683) — Low Severity
-
-  A discovered oEmbed or image URL can bypass the `url_preview_url_blacklist` setting potentially allowing server side request forgery or bypassing network policies. Impact is limited to IP addresses allowed by the `url_preview_ip_range_blacklist` setting (by default this only allows public IPs).
-
-See the advisories for more details. If you have any questions, email security@matrix.org.
-
-
-Synapse 1.85.0rc2 (2023-06-01)
-==============================
-
-Bugfixes
---------
-
-- Fix a performance issue introduced in Synapse v1.83.0 which meant that purging rooms was very slow and database-intensive. ([\#15693](https://github.com/matrix-org/synapse/issues/15693))
-
-
-Deprecations and Removals
--------------------------
-
-- Deprecate calling the `/register` endpoint with an unspecced `user` property for application services. ([\#15703](https://github.com/matrix-org/synapse/issues/15703))
-
-
-Internal Changes
-----------------
-
-- Speed up background jobs `populate_full_user_id_user_filters` and `populate_full_user_id_profiles`. ([\#15700](https://github.com/matrix-org/synapse/issues/15700))
-
-
-Synapse 1.85.0rc1 (2023-05-30)
-==============================
-
-Features
---------
-
-- Improve performance of backfill requests by performing backfill of previously failed requests in the background. ([\#15585](https://github.com/matrix-org/synapse/issues/15585))
-- Add a new [admin API](https://matrix-org.github.io/synapse/v1.85/usage/administration/admin_api/index.html) to [create a new device for a user](https://matrix-org.github.io/synapse/v1.85/admin_api/user_admin_api.html#create-a-device). ([\#15611](https://github.com/matrix-org/synapse/issues/15611))
-- Add Unix socket support for Redis connections. Contributed by Jason Little. ([\#15644](https://github.com/matrix-org/synapse/issues/15644))
-
-
-Bugfixes
---------
-
-- Fix a long-standing bug where setting the read marker could fail when using message retention. Contributed by Nick @ Beeper (@fizzadar). ([\#15464](https://github.com/matrix-org/synapse/issues/15464))
-- Fix a long-standing bug where the `url_preview_url_blacklist` configuration setting was not applied to oEmbed or image URLs found while previewing a URL. ([\#15601](https://github.com/matrix-org/synapse/issues/15601))
-- Fix a long-standing bug where filters with multiple backslashes were rejected. ([\#15607](https://github.com/matrix-org/synapse/issues/15607))
-- Fix a bug introduced in Synapse 1.82.0 where the error message displayed when validation of the `app_service_config_files` config option fails would be incorrectly formatted. ([\#15614](https://github.com/matrix-org/synapse/issues/15614))
-- Fix a long-standing bug where deactivated users were still able to login using the custom `org.matrix.login.jwt` login type (if enabled). ([\#15624](https://github.com/matrix-org/synapse/issues/15624))
-- Fix a long-standing bug where deactivated users were able to login in uncommon situations. ([\#15634](https://github.com/matrix-org/synapse/issues/15634))
-
-
-Improved Documentation
-----------------------
-
-- Warn users that at least 3.75GB of space is needed for the nix Synapse development environment. ([\#15613](https://github.com/matrix-org/synapse/issues/15613))
-- Remove outdated comment from the generated and sample homeserver log configs. ([\#15648](https://github.com/matrix-org/synapse/issues/15648))
-- Improve contributor docs to make it more clear that Rust is a necessary prerequisite. Contributed by @grantm. ([\#15668](https://github.com/matrix-org/synapse/issues/15668))
-
-
-Deprecations and Removals
--------------------------
-
-- Remove the old version of the R30 (30-day retained users) phone-home metric. ([\#10428](https://github.com/matrix-org/synapse/issues/10428))
-
-
-Internal Changes
-----------------
-
-- Create dependabot changelogs at release time. ([\#15481](https://github.com/matrix-org/synapse/issues/15481))
-- Add not null constraint to column `full_user_id` of tables `profiles` and `user_filters`. ([\#15537](https://github.com/matrix-org/synapse/issues/15537))
-- Allow connecting to HTTP Replication Endpoints by using `worker_name` when constructing the request. ([\#15578](https://github.com/matrix-org/synapse/issues/15578))
-- Make the `thread_id` column on `event_push_actions`, `event_push_actions_staging`, and `event_push_summary` non-null. ([\#15597](https://github.com/matrix-org/synapse/issues/15597))
-- Run mypy type checking with the minimum supported Python version to catch new usage that isn't backwards-compatible. ([\#15602](https://github.com/matrix-org/synapse/issues/15602))
-- Fix subscriptable type usage in Python <3.9. ([\#15604](https://github.com/matrix-org/synapse/issues/15604))
-- Update internal terminology. ([\#15606](https://github.com/matrix-org/synapse/issues/15606), [\#15620](https://github.com/matrix-org/synapse/issues/15620))
-- Instrument `state` and `state_group` storage-related operations to better picture what's happening when tracing. ([\#15610](https://github.com/matrix-org/synapse/issues/15610), [\#15647](https://github.com/matrix-org/synapse/issues/15647))
-- Trace how many new events from the backfill response we need to process. ([\#15633](https://github.com/matrix-org/synapse/issues/15633))
-- Re-type config paths in `ConfigError`s to be `StrSequence`s instead of `Iterable[str]`s. ([\#15615](https://github.com/matrix-org/synapse/issues/15615))
-- Update Mutual Rooms ([MSC2666](https://github.com/matrix-org/matrix-spec-proposals/pull/2666)) implementation to match new proposal text. ([\#15621](https://github.com/matrix-org/synapse/issues/15621))
-- Remove the unstable identifiers from faster joins ([MSC3706](https://github.com/matrix-org/matrix-spec-proposals/pull/3706)). ([\#15625](https://github.com/matrix-org/synapse/issues/15625))
-- Fix the olddeps CI. ([\#15626](https://github.com/matrix-org/synapse/issues/15626))
-- Remove duplicate timestamp from test logs (`_trial_temp/test.log`). ([\#15636](https://github.com/matrix-org/synapse/issues/15636))
-- Fix two memory leaks in `trial` test runs. ([\#15630](https://github.com/matrix-org/synapse/issues/15630))
-- Limit the size of the `HomeServerConfig` cache in trial test runs. ([\#15646](https://github.com/matrix-org/synapse/issues/15646))
-- Improve type hints. ([\#15658](https://github.com/matrix-org/synapse/issues/15658), [\#15659](https://github.com/matrix-org/synapse/issues/15659))
-- Add requesting user id parameter to key claim methods in `TransportLayerClient`. ([\#15663](https://github.com/matrix-org/synapse/issues/15663))
-- Speed up rebuilding of the user directory for local users. ([\#15665](https://github.com/matrix-org/synapse/issues/15665))
-- Implement "option 2" for [MSC3820](https://github.com/matrix-org/matrix-spec-proposals/pull/3820): Room version 11. ([\#15666](https://github.com/matrix-org/synapse/issues/15666), [\#15678](https://github.com/matrix-org/synapse/issues/15678))
-
-### Updates to locked dependencies
-
-* Bump furo from 2023.3.27 to 2023.5.20. ([\#15642](https://github.com/matrix-org/synapse/issues/15642))
-* Bump log from 0.4.17 to 0.4.18. ([\#15681](https://github.com/matrix-org/synapse/issues/15681))
-* Bump prometheus-client from 0.16.0 to 0.17.0. ([\#15682](https://github.com/matrix-org/synapse/issues/15682))
-* Bump pydantic from 1.10.7 to 1.10.8. ([\#15685](https://github.com/matrix-org/synapse/issues/15685))
-* Bump pygithub from 1.58.1 to 1.58.2. ([\#15643](https://github.com/matrix-org/synapse/issues/15643))
-* Bump requests from 2.28.2 to 2.31.0. ([\#15651](https://github.com/matrix-org/synapse/issues/15651))
-* Bump sphinx from 6.1.3 to 6.2.1. ([\#15641](https://github.com/matrix-org/synapse/issues/15641))
-* Bump types-bleach from 6.0.0.1 to 6.0.0.3. ([\#15686](https://github.com/matrix-org/synapse/issues/15686))
-* Bump types-pillow from 9.5.0.2 to 9.5.0.4. ([\#15640](https://github.com/matrix-org/synapse/issues/15640))
-* Bump types-pyyaml from 6.0.12.9 to 6.0.12.10. ([\#15683](https://github.com/matrix-org/synapse/issues/15683))
-* Bump types-requests from 2.30.0.0 to 2.31.0.0. ([\#15684](https://github.com/matrix-org/synapse/issues/15684))
-* Bump types-setuptools from 67.7.0.2 to 67.8.0.0. ([\#15639](https://github.com/matrix-org/synapse/issues/15639))
-
-Synapse 1.84.1 (2023-05-26)
-===========================
-
-This patch release fixes a major issue with homeservers that do not have an `instance_map` defined but which do use workers.
-If you have already upgraded to Synapse 1.84.0 and your homeserver is working normally, then there is no need to update to this patch release.
-
-
-Bugfixes
---------
-
-- Fix a bug introduced in Synapse v1.84.0 where workers do not start up when no `instance_map` was provided. ([\#15672](https://github.com/matrix-org/synapse/issues/15672))
-
-
-Internal Changes
-----------------
-
-- Add `dch` and `notify-send` to the development Nix flake so that the release script can be used. ([\#15673](https://github.com/matrix-org/synapse/issues/15673))
-
-
-Synapse 1.84.0 (2023-05-23)
-===========================
-
-The `worker_replication_*` configuration settings have been deprecated in favour of configuring the main process consistently with other instances in the `instance_map`. The deprecated settings will be removed in Synapse v1.88.0, but changing your configuration in advance is recommended. See the [upgrade notes](https://github.com/matrix-org/synapse/blob/release-v1.84/docs/upgrade.md#upgrading-to-v1840) for more information.
-
-Bugfixes
---------
-
-- Fix a bug introduced in Synapse 1.84.0rc1 where errors during startup were not reported correctly on Python < 3.10. ([\#15599](https://github.com/matrix-org/synapse/issues/15599))
-
-
 Synapse 1.84.0rc1 (2023-05-16)
 ==============================
 
@@ -269,12 +23,6 @@ Bugfixes
 - Require at least poetry-core v1.1.0. ([\#15566](https://github.com/matrix-org/synapse/issues/15566), [\#15571](https://github.com/matrix-org/synapse/issues/15571))
 
 
-Deprecations and Removals
--------------------------
-
-- Remove need for `worker_replication_*` based settings in worker configuration yaml by placing this data directly on the `instance_map` instead. ([\#15491](https://github.com/matrix-org/synapse/issues/15491))
-
-
 Updates to the Docker image
 ---------------------------
 
@@ -294,6 +42,7 @@ Internal Changes
 
 - Use oEmbed to generate URL previews for YouTube Shorts. ([\#15025](https://github.com/matrix-org/synapse/issues/15025))
 - Create new `Client` for use with HTTP Replication between workers. Contributed by Jason Little. ([\#15470](https://github.com/matrix-org/synapse/issues/15470))
+- Remove need for `worker_replication_*` based settings in worker configuration yaml by placing this data directly on the `instance_map` instead. ([\#15491](https://github.com/matrix-org/synapse/issues/15491))
 - Bump pyicu from 2.10.2 to 2.11. ([\#15509](https://github.com/matrix-org/synapse/issues/15509))
 - Remove references to supporting per-user flag for [MSC2654](https://github.com/matrix-org/matrix-spec-proposals/pull/2654). ([\#15522](https://github.com/matrix-org/synapse/issues/15522))
 - Don't use a trusted key server when running the demo scripts. ([\#15527](https://github.com/matrix-org/synapse/issues/15527))

Cargo.lock

@@ -4,9 +4,9 @@ version = 3
 
 [[package]]
 name = "aho-corasick"
-version = "1.0.2"
+version = "0.7.19"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "43f6cb1bf222025340178f382c426f13757b2960e89779dfcb319c32542a5a41"
+checksum = "b4f55bd91a0978cbfd91c457a164bab8b4001c833b7f323132c0a4e1922dd44e"
 dependencies = [
  "memchr",
 ]
@@ -132,9 +132,12 @@ dependencies = [
 
 [[package]]
 name = "log"
-version = "0.4.19"
+version = "0.4.17"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b06a4cde4c0f271a446782e3eff8de789548ce57dbc8eca9292c27f4a42004b4"
+checksum = "abb12e687cfb44aa40f41fc3978ef76448f9b6038cad6aef4259d3c095a2382e"
+dependencies = [
+ "cfg-if",
+]
 
 [[package]]
 name = "memchr"
@@ -229,9 +232,9 @@ dependencies = [
 
 [[package]]
 name = "pyo3-log"
-version = "0.8.2"
+version = "0.8.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c94ff6535a6bae58d7d0b85e60d4c53f7f84d0d0aa35d6a28c3f3e70bfe51444"
+checksum = "f9c8b57fe71fb5dcf38970ebedc2b1531cf1c14b1b9b4c560a182a57e115575c"
 dependencies = [
  "arc-swap",
  "log",
@@ -291,9 +294,9 @@ dependencies = [
 
 [[package]]
 name = "regex"
-version = "1.8.4"
+version = "1.7.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d0ab3ca65655bb1e41f2a8c8cd662eb4fb035e67c3f78da1d61dffe89d07300f"
+checksum = "8b1f693b24f6ac912f4893ef08244d70b6067480d2f1a46e950c9691e6749d1d"
 dependencies = [
  "aho-corasick",
  "memchr",
@@ -302,9 +305,9 @@ dependencies = [
 
 [[package]]
 name = "regex-syntax"
-version = "0.7.2"
+version = "0.6.29"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "436b050e76ed2903236f032a59761c1eb99e1b0aead2c257922771dab1fc8c78"
+checksum = "f162c6dd7b008981e4d40210aca20b4bd0f9b60ca9271061b07f78537722f2e1"
 
 [[package]]
 name = "ryu"
@@ -320,18 +323,18 @@ checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd"
 
 [[package]]
 name = "serde"
-version = "1.0.164"
+version = "1.0.163"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9e8c8cf938e98f769bc164923b06dce91cea1751522f46f8466461af04c9027d"
+checksum = "2113ab51b87a539ae008b5c6c02dc020ffa39afd2d83cffcb3f4eb2722cebec2"
 dependencies = [
  "serde_derive",
 ]
 
 [[package]]
 name = "serde_derive"
-version = "1.0.164"
+version = "1.0.163"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d9735b638ccc51c28bf6914d90a2e9725b377144fc612c49a611fddd1b631d68"
+checksum = "8c805777e3930c8883389c602315a24224bcc738b63905ef87cd1420353ea93e"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -340,9 +343,9 @@ dependencies = [
 
 [[package]]
 name = "serde_json"
-version = "1.0.97"
+version = "1.0.96"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bdf3bf93142acad5821c99197022e170842cdbc1c30482b98750c688c640842a"
+checksum = "057d394a50403bcac12672b2b18fb387ab6d289d957dab67dd201875391e52f1"
 dependencies = [
  "itoa",
  "ryu",

changelog.d/10428.removal

@@ -0,0 +1 @@
+Remove the old version of the R30 (30-day retained users) phone-home metric.

changelog.d/15233.misc

@@ -1 +0,0 @@
-Replace `EventContext` fields `prev_group` and `delta_ids` with field `state_group_deltas`.

changelog.d/15464.bugfix

@@ -0,0 +1 @@
+Fix a long-standing bug where setting the read marker could fail when using message retention. Contributed by Nick @ Beeper (@fizzadar).

changelog.d/15537.misc

@@ -0,0 +1 @@
+Add not null constraint to column full_user_id of tables profiles and user_filters.

changelog.d/15599.bugfix

@@ -0,0 +1 @@
+Print full error and stack-trace of any exception that occurs during startup/initialization.

changelog.d/15601.bugfix

@@ -0,0 +1 @@
+Fix a long-standing bug where the `url_preview_url_blacklist` configuration setting was not applied to oEmbed or image URLs found while previewing a URL.

changelog.d/15602.misc

@@ -0,0 +1 @@
+Run mypy type checking with the minimum supported Python version to catch new usage that isn't backwards-compatible.

changelog.d/15604.misc

@@ -0,0 +1 @@
+Fix subscriptable type usage in Python <3.9.

changelog.d/15606.misc

@@ -0,0 +1 @@
+Update internal terminology.

changelog.d/15610.misc

@@ -0,0 +1 @@
+Instrument `state` and `state_group` storage-related operations to better picture what's happening when tracing.

changelog.d/15611.feature

@@ -0,0 +1 @@
+Add a new admin API to create a new device for a user.

changelog.d/15613.doc

@@ -0,0 +1 @@
+Warn users that at least 3.75GB of space is needed for the nix Synapse development environment.

changelog.d/15614.bugfix

@@ -0,0 +1 @@
+Fix a bug introduced in Synapse 1.82.0 where the error message displayed when validation of the `app_service_config_files` config option fails would be incorrectly formatted.

changelog.d/15615.misc

@@ -0,0 +1 @@
+Re-type config paths in `ConfigError`s to be `StrSequence`s instead of `Iterable[str]`s.

changelog.d/15620.misc

@@ -0,0 +1 @@
+Update internal terminology.

changelog.d/15621.misc

@@ -0,0 +1 @@
+Update Mutual Rooms (MSC2666) implementation to match new proposal text.

changelog.d/15624.bugfix

@@ -0,0 +1 @@
+Fix a long-standing bug where deactivated users were still able to login using the custom `org.matrix.login.jwt` login type (if enabled).

changelog.d/15625.misc

@@ -0,0 +1 @@
+Remove the unstable identifiers from faster joins ([MSC3706](https://github.com/matrix-org/matrix-spec-proposals/pull/3706).

changelog.d/15626.misc

@@ -0,0 +1 @@
+Fix the olddeps CI.

changelog.d/15630.misc

@@ -0,0 +1 @@
+Fix two memory leaks in `trial` test runs.

changelog.d/15633.misc

@@ -0,0 +1 @@
+Trace how many new events from the backfill response we need to process.

changelog.d/15639.misc

@@ -0,0 +1 @@
+Bump types-setuptools from 67.7.0.2 to 67.8.0.0.

changelog.d/15640.misc

@@ -0,0 +1 @@
+Bump types-pillow from 9.5.0.2 to 9.5.0.4.

changelog.d/15641.misc

@@ -0,0 +1 @@
+Bump sphinx from 6.1.3 to 6.2.1.

changelog.d/15642.misc

@@ -0,0 +1 @@
+Bump furo from 2023.3.27 to 2023.5.20.

changelog.d/15646.misc

@@ -0,0 +1 @@
+Limit the size of the `HomeServerConfig` cache in trial test runs.

changelog.d/15680.bugfix

@@ -1 +0,0 @@
-Fix a long-standing bug where media files were served in an unsafe manner. Contributed by @joshqou.

changelog.d/15737.feature

@@ -1 +0,0 @@
-Improve `/messages` response time by avoiding backfill when we already have messages to return.

changelog.d/15743.misc

@@ -1 +0,0 @@
-Regularly try to send transactions to other servers after they failed instead of waiting for a new event to be available before trying.

changelog.d/15748.removal

@@ -1 +0,0 @@
-Remove experimental [MSC2716](https://github.com/matrix-org/matrix-spec-proposals/pull/2716) implementation to incrementally import history into existing rooms.

changelog.d/15755.misc

@@ -1 +0,0 @@
-Fix requesting multiple keys at once over federation, related to [MSC3983](https://github.com/matrix-org/matrix-spec-proposals/pull/3983).

changelog.d/15758.bugfix

@@ -1 +0,0 @@
-Avoid invalidating a cache that was just prefilled.

changelog.d/15770.bugfix

@@ -1 +0,0 @@
-Fix requesting multiple keys at once over federation, related to [MSC3983](https://github.com/matrix-org/matrix-spec-proposals/pull/3983).

changelog.d/15772.doc

@@ -1 +0,0 @@
-Document `looping_call()` functionality that will wait for the given function to finish before scheduling another.

changelog.d/15776.bugfix

@@ -1 +0,0 @@
-Fix joining rooms through aliases where the alias server isn't a real homeserver. Contributed by @tulir @ Beeper.

changelog.d/15781.bugfix

@@ -1 +0,0 @@
-Fix a bug in push rules handling leading to an invalid (per spec) `is_user_mention` rule sent to clients. Also fix wrong rule names for `is_user_mention` and `is_room_mention`.

changelog.d/15788.bugfix

@@ -1 +0,0 @@
-Fix a bug introduced in 1.57.0 where the wrong table would be locked on updating database rows when using SQLite as the database backend.

changelog.d/15805.doc

@@ -1 +0,0 @@
-Fix a typo in the [Admin API](https://matrix-org.github.io/synapse/latest/usage/administration/admin_api/index.html).

changelog.d/15806.misc

@@ -1 +0,0 @@
-Switch from `matrix://` to `matrix-federation://` scheme for internal Synapse routing of outbound federation traffic.

changelog.d/15808.misc

@@ -1 +0,0 @@
-Ignore key request if the device inbox is already big.

contrib/lnav/synapse-log-format.json

@@ -29,7 +29,7 @@
         "level": "error"
       },
       {
-        "line": "my-matrix-server-federation-sender-1 | 2023-01-25 20:56:20,995 - synapse.http.matrixfederationclient - 709 - WARNING - federation_transaction_transmission_loop-3 - {PUT-O-3} [example.com] Request failed: PUT matrix-federation://example.com/_matrix/federation/v1/send/1674680155797: HttpResponseException('403: Forbidden')",
+        "line": "my-matrix-server-federation-sender-1 | 2023-01-25 20:56:20,995 - synapse.http.matrixfederationclient - 709 - WARNING - federation_transaction_transmission_loop-3 - {PUT-O-3} [example.com] Request failed: PUT matrix://example.com/_matrix/federation/v1/send/1674680155797: HttpResponseException('403: Forbidden')",
         "level": "warning"
       },
       {

debian/changelog

@@ -1,57 +1,3 @@
-matrix-synapse-py3 (1.86.0~rc2) stable; urgency=medium
-
-  * New Synapse release 1.86.0rc2.
-
- -- Synapse Packaging team <packages@matrix.org>  Wed, 14 Jun 2023 12:16:27 +0200
-
-matrix-synapse-py3 (1.86.0~rc1) stable; urgency=medium
-
-  * New Synapse release 1.86.0rc1.
-
- -- Synapse Packaging team <packages@matrix.org>  Tue, 13 Jun 2023 14:30:45 +0200
-
-matrix-synapse-py3 (1.85.2) stable; urgency=medium
-
-  * New Synapse release 1.85.2.
-
- -- Synapse Packaging team <packages@matrix.org>  Thu, 08 Jun 2023 13:04:18 +0100
-
-matrix-synapse-py3 (1.85.1) stable; urgency=medium
-
-  * New Synapse release 1.85.1.
-
- -- Synapse Packaging team <packages@matrix.org>  Wed, 07 Jun 2023 10:51:12 +0100
-
-matrix-synapse-py3 (1.85.0) stable; urgency=medium
-
-  * New Synapse release 1.85.0.
-
- -- Synapse Packaging team <packages@matrix.org>  Tue, 06 Jun 2023 09:39:29 +0100
-
-matrix-synapse-py3 (1.85.0~rc2) stable; urgency=medium
-
-  * New Synapse release 1.85.0rc2.
-
- -- Synapse Packaging team <packages@matrix.org>  Thu, 01 Jun 2023 09:16:18 -0700
-
-matrix-synapse-py3 (1.85.0~rc1) stable; urgency=medium
-
-  * New Synapse release 1.85.0rc1.
-
- -- Synapse Packaging team <packages@matrix.org>  Tue, 30 May 2023 13:56:54 +0100
-
-matrix-synapse-py3 (1.84.1) stable; urgency=medium
-
-  * New Synapse release 1.84.1.
-
- -- Synapse Packaging team <packages@matrix.org>  Fri, 26 May 2023 16:15:30 +0100
-
-matrix-synapse-py3 (1.84.0) stable; urgency=medium
-
-  * New Synapse release 1.84.0.
-
- -- Synapse Packaging team <packages@matrix.org>  Tue, 23 May 2023 10:57:22 +0100
-
 matrix-synapse-py3 (1.84.0~rc1) stable; urgency=medium
 
   * New Synapse release 1.84.0rc1.

docker/Dockerfile

@@ -27,7 +27,7 @@ ARG PYTHON_VERSION=3.11
 ###
 # We hardcode the use of Debian bullseye here because this could change upstream
 # and other Dockerfiles used for testing are expecting bullseye.
-FROM docker.io/library/python:${PYTHON_VERSION}-slim-bullseye as requirements
+FROM docker.io/python:${PYTHON_VERSION}-slim-bullseye as requirements
 
 # RUN --mount is specific to buildkit and is documented at
 # https://github.com/moby/buildkit/blob/master/frontend/dockerfile/docs/syntax.md#build-mounts-run---mount.
@@ -87,7 +87,7 @@ RUN if [ -z "$TEST_ONLY_IGNORE_POETRY_LOCKFILE" ]; then \
 ###
 ### Stage 1: builder
 ###
-FROM docker.io/library/python:${PYTHON_VERSION}-slim-bullseye as builder
+FROM docker.io/python:${PYTHON_VERSION}-slim-bullseye as builder
 
 # install the OS build deps
 RUN \
@@ -158,7 +158,7 @@ RUN --mount=type=cache,target=/synapse/target,sharing=locked \
 ### Stage 2: runtime
 ###
 
-FROM docker.io/library/python:${PYTHON_VERSION}-slim-bullseye
+FROM docker.io/python:${PYTHON_VERSION}-slim-bullseye
 
 LABEL org.opencontainers.image.url='https://matrix.org/docs/projects/server/synapse'
 LABEL org.opencontainers.image.documentation='https://github.com/matrix-org/synapse/blob/master/docker/README.md'

docker/Dockerfile-dhvirtualenv

@@ -24,7 +24,7 @@ ARG distro=""
 # https://launchpad.net/~jyrki-pulliainen/+archive/ubuntu/dh-virtualenv, but
 # it's not obviously easier to use that than to build our own.)
 
-FROM docker.io/library/${distro} as builder
+FROM ${distro} as builder
 
 RUN apt-get update -qq -o Acquire::Languages=none
 RUN env DEBIAN_FRONTEND=noninteractive apt-get install \
@@ -55,7 +55,7 @@ RUN cd /dh-virtualenv && DEB_BUILD_OPTIONS=nodoc dpkg-buildpackage -us -uc -b
 ###
 ### Stage 1
 ###
-FROM docker.io/library/${distro}
+FROM ${distro}
 
 # Get the distro we want to pull from as a dynamic build variable
 # (We need to define it in each build stage)

docker/Dockerfile-workers

@@ -7,7 +7,7 @@ ARG FROM=matrixdotorg/synapse:$SYNAPSE_VERSION
 # target image. For repeated rebuilds, this is much faster than apt installing
 # each time.
 
-FROM docker.io/library/debian:bullseye-slim AS deps_base
+FROM debian:bullseye-slim AS deps_base
     RUN \
        --mount=type=cache,target=/var/cache/apt,sharing=locked \
        --mount=type=cache,target=/var/lib/apt,sharing=locked \
@@ -21,7 +21,7 @@ FROM docker.io/library/debian:bullseye-slim AS deps_base
 # which makes it much easier to copy (but we need to make sure we use an image
 # based on the same debian version as the synapse image, to make sure we get
 # the expected version of libc.
-FROM docker.io/library/redis:7-bullseye AS redis_base
+FROM redis:6-bullseye AS redis_base
 
 # now build the final image, based on the the regular Synapse docker image
 FROM $FROM

docker/README.md

@@ -73,8 +73,7 @@ The following environment variables are supported in `generate` mode:
   will log sensitive information such as access tokens.
   This should not be needed unless you are a developer attempting to debug something
   particularly tricky.
-* `SYNAPSE_LOG_TESTING`: if set, Synapse will log additional information useful
-  for testing.
+
 
 ## Postgres
 

docker/complement/Dockerfile

@@ -7,7 +7,6 @@
 # https://github.com/matrix-org/synapse/blob/develop/docker/README-testing.md#testing-with-postgresql-and-single-or-multi-process-synapse
 
 ARG SYNAPSE_VERSION=latest
-# This is an intermediate image, to be built locally (not pulled from a registry).
 ARG FROM=matrixdotorg/synapse-workers:$SYNAPSE_VERSION
 
 FROM $FROM
@@ -20,8 +19,8 @@ FROM $FROM
     # the same debian version as Synapse's docker image (so the versions of the
     # shared libraries match).
     RUN adduser --system --uid 999 postgres --home /var/lib/postgresql
-    COPY --from=docker.io/library/postgres:13-bullseye /usr/lib/postgresql /usr/lib/postgresql
-    COPY --from=docker.io/library/postgres:13-bullseye /usr/share/postgresql /usr/share/postgresql
+    COPY --from=postgres:13-bullseye /usr/lib/postgresql /usr/lib/postgresql
+    COPY --from=postgres:13-bullseye /usr/share/postgresql /usr/share/postgresql
     RUN mkdir /var/run/postgresql && chown postgres /var/run/postgresql
     ENV PATH="${PATH}:/usr/lib/postgresql/13/bin"
     ENV PGDATA=/var/lib/postgresql/data

docker/complement/conf/workers-shared-extra.yaml.j2

@@ -92,6 +92,8 @@ allow_device_name_lookup_over_federation: true
 ## Experimental Features ##
 
 experimental_features:
+  # Enable history backfilling support
+  msc2716_enabled: true
   # client-side support for partial state in /send_join responses
   faster_joins: true
   # Enable support for polls

docker/conf/log.config

@@ -49,35 +49,17 @@ handlers:
     class: logging.StreamHandler
     formatter: precise
 
+{% if not SYNAPSE_LOG_SENSITIVE %}
+{#
+  If SYNAPSE_LOG_SENSITIVE is unset, then override synapse.storage.SQL to INFO
+  so that DEBUG entries (containing sensitive information) are not emitted.
+#}
 loggers:
-    # This is just here so we can leave `loggers` in the config regardless of whether
-    # we configure other loggers below (avoid empty yaml dict error).
-    _placeholder:
-        level: "INFO"
-
-    {% if not SYNAPSE_LOG_SENSITIVE %}
-    {#
-      If SYNAPSE_LOG_SENSITIVE is unset, then override synapse.storage.SQL to INFO
-      so that DEBUG entries (containing sensitive information) are not emitted.
-    #}
     synapse.storage.SQL:
         # beware: increasing this to DEBUG will make synapse log sensitive
         # information such as access tokens.
         level: INFO
-    {% endif %}
-
-    {% if SYNAPSE_LOG_TESTING %}
-    {#
-      If Synapse is under test, log a few more useful things for a developer
-      attempting to debug something particularly tricky.
-
-      With `synapse.visibility.filtered_event_debug`, it logs when events are (maybe
-      unexpectedly) filtered out of responses in tests. It's just nice to be able to
-      look at the CI log and figure out why an event isn't being returned.
-    #}
-    synapse.visibility.filtered_event_debug:
-        level: DEBUG
-    {% endif %}
+{% endif %}
 
 root:
     level: {{ SYNAPSE_LOG_LEVEL or "INFO" }}

docker/configure_workers_and_start.py

@@ -40,8 +40,6 @@
 #         log level. INFO is the default.
 #   * SYNAPSE_LOG_SENSITIVE: If unset, SQL and SQL values won't be logged,
 #         regardless of the SYNAPSE_LOG_LEVEL setting.
-#   * SYNAPSE_LOG_TESTING: if set, Synapse will log additional information useful
-#     for testing.
 #
 # NOTE: According to Complement's ENTRYPOINT expectations for a homeserver image (as defined
 # in the project's README), this script may be run multiple times, and functionality should
@@ -244,6 +242,7 @@ WORKERS_CONFIG: Dict[str, Dict[str, Any]] = {
             "^/_matrix/client/(api/v1|r0|v3|unstable)/join/",
             "^/_matrix/client/(api/v1|r0|v3|unstable)/knock/",
             "^/_matrix/client/(api/v1|r0|v3|unstable)/profile/",
+            "^/_matrix/client/(v1|unstable/org.matrix.msc2716)/rooms/.*/batch_send",
         ],
         "shared_extra_conf": {},
         "worker_extra_conf": "",
@@ -948,7 +947,6 @@ def generate_worker_log_config(
     extra_log_template_args["SYNAPSE_LOG_SENSITIVE"] = environ.get(
         "SYNAPSE_LOG_SENSITIVE"
     )
-    extra_log_template_args["SYNAPSE_LOG_TESTING"] = environ.get("SYNAPSE_LOG_TESTING")
 
     # Render and write the file
     log_config_filepath = f"/conf/workers/{worker_name}.log.config"

docker/editable.Dockerfile

@@ -10,7 +10,7 @@ ARG PYTHON_VERSION=3.9
 ###
 # We hardcode the use of Debian bullseye here because this could change upstream
 # and other Dockerfiles used for testing are expecting bullseye.
-FROM docker.io/library/python:${PYTHON_VERSION}-slim-bullseye
+FROM docker.io/python:${PYTHON_VERSION}-slim-bullseye
 
 # Install Rust and other dependencies (stolen from normal Dockerfile)
 # install the OS build deps

docs/admin_api/rooms.md

@@ -419,7 +419,7 @@ The following query parameters are available:
 
 * `from` (required) - The token to start returning events from. This token can be obtained from a prev_batch
   or next_batch token returned by the /sync endpoint, or from an end token returned by a previous request to this endpoint.
-* `to` - The token to stop returning events at.
+* `to` - The token to spot returning events at.
 * `limit` - The maximum number of events to return. Defaults to `10`.
 * `filter` - A JSON RoomEventFilter to filter returned events with.
 * `dir` - The direction to return events from. Either `f` for forwards or `b` for backwards. Setting

docs/development/contributing_guide.md

@@ -22,9 +22,6 @@ on Windows is not officially supported.
 
 The code of Synapse is written in Python 3. To do pretty much anything, you'll need [a recent version of Python 3](https://www.python.org/downloads/). Your Python also needs support for [virtual environments](https://docs.python.org/3/library/venv.html). This is usually built-in, but some Linux distributions like Debian and Ubuntu split it out into its own package. Running `sudo apt install python3-venv` should be enough.
 
-A recent version of the Rust compiler is needed to build the native modules. The
-easiest way of installing the latest version is to use [rustup](https://rustup.rs/).
-
 Synapse can connect to PostgreSQL via the [psycopg2](https://pypi.org/project/psycopg2/) Python library. Building this library from source requires access to PostgreSQL's C header files. On Debian or Ubuntu Linux, these can be installed with `sudo apt install libpq-dev`.
 
 Synapse has an optional, improved user search with better Unicode support. For that you need the development package of `libicu`. On Debian or Ubuntu Linux, this can be installed with `sudo apt install libicu-dev`.
@@ -33,6 +30,9 @@ The source code of Synapse is hosted on GitHub. You will also need [a recent ver
 
 For some tests, you will need [a recent version of Docker](https://docs.docker.com/get-docker/).
 
+A recent version of the Rust compiler is needed to build the native modules. The
+easiest way of installing the latest version is to use [rustup](https://rustup.rs/).
+
 
 # 3. Get the source.
 
@@ -53,11 +53,6 @@ can find many good git tutorials on the web.
 
 # 4. Install the dependencies
 
-
-Before installing the Python dependencies, make sure you have installed a recent version
-of Rust (see the "What do I need?" section above). The easiest way of installing the
-latest version is to use [rustup](https://rustup.rs/).
-
 Synapse uses the [poetry](https://python-poetry.org/) project to manage its dependencies
 and development environment. Once you have installed Python 3 and added the
 source, you should install `poetry`.
@@ -81,8 +76,7 @@ cd path/where/you/have/cloned/the/repository
 poetry install --extras all
 ```
 
-This will install the runtime and developer dependencies for the project.  Be sure to check
-that the `poetry install` step completed cleanly.
+This will install the runtime and developer dependencies for the project.
 
 ## Running Synapse via poetry
 
@@ -90,31 +84,14 @@ To start a local instance of Synapse in the locked poetry environment, create a
 
 ```sh
 cp docs/sample_config.yaml homeserver.yaml
-cp docs/sample_log_config.yaml log_config.yaml
 ```
 
-Now edit `homeserver.yaml`, things you might want to change include:
-
-- Set a `server_name`
-- Adjusting paths to be correct for your system like the `log_config` to point to the log config you just copied
-- Using a [PostgreSQL database instead of SQLite](https://matrix-org.github.io/synapse/latest/usage/configuration/config_documentation.html#database)
-- Adding a [`registration_shared_secret`](https://matrix-org.github.io/synapse/latest/usage/configuration/config_documentation.html#registration_shared_secret) so you can use [`register_new_matrix_user` command](https://matrix-org.github.io/synapse/latest/setup/installation.html#registering-a-user).
-
-And then run Synapse with the following command:
+Now edit homeserver.yaml, and run Synapse with:
 
 ```sh
 poetry run python -m synapse.app.homeserver -c homeserver.yaml
 ```
 
-If you get an error like the following:
-
-```
-importlib.metadata.PackageNotFoundError: matrix-synapse
-```
-
-this probably indicates that the `poetry install` step did not complete cleanly - go back and
-resolve any issues and re-run until successful.
-
 # 5. Get in touch.
 
 Join our developer community on Matrix: [#synapse-dev:matrix.org](https://matrix.to/#/#synapse-dev:matrix.org)!

docs/development/dependencies.md

@@ -260,17 +260,15 @@ doesn't require poetry. (It's what we use in CI too). However, you could try
 
 ## ...handle a Dependabot pull request?
 
-Synapse uses Dependabot to keep the `poetry.lock` and `Cargo.lock` file 
-up-to-date with the latest releases of our dependencies. The changelog check is
-omitted for Dependabot PRs; the release script will include them in the 
-changelog.
-
-When reviewing a dependabot PR, ensure that:
+Synapse uses Dependabot to keep the `poetry.lock` file up-to-date. When it
+creates a pull request a GitHub Action will run to automatically create a changelog
+file. Ensure that:
 
 * the lockfile changes look reasonable;
 * the upstream changelog file (linked in the description) doesn't include any
   breaking changes;
-* continuous integration passes.
+* continuous integration passes (due to permissions, the GitHub Actions run on
+  the changelog commit will fail, look at the initial commit of the pull request);
 
 In particular, any updates to the type hints (usually packages which start with `types-`)
 should be safe to merge if linting passes.

docs/modules/password_auth_provider_callbacks.md

@@ -46,9 +46,6 @@ instead.
 
 If the authentication is unsuccessful, the module must return `None`.
 
-Note that the user is not automatically registered, the `register_user(..)` method of
-the [module API](writing_a_module.html) can be used to lazily create users.
-
 If multiple modules register an auth checker for the same login type but with different
 fields, Synapse will refuse to start.
 

docs/sample_log_config.yaml

@@ -68,7 +68,9 @@ root:
     # Write logs to the `buffer` handler, which will buffer them together in memory,
     # then write them to a file.
     #
-    # Replace "buffer" with "console" to log to stderr instead.
+    # Replace "buffer" with "console" to log to stderr instead. (Note that you'll
+    # also need to update the configuration for the `twisted` logger above, in
+    # this case.)
     #
     handlers: [buffer]
 

docs/upgrade.md

@@ -87,46 +87,20 @@ process, for example:
     wget https://packages.matrix.org/debian/pool/main/m/matrix-synapse-py3/matrix-synapse-py3_1.3.0+stretch1_amd64.deb
     dpkg -i matrix-synapse-py3_1.3.0+stretch1_amd64.deb
     ```
-# Upgrading to v1.86.0
-
-## Minimum supported Rust version
-
-The minimum supported Rust version has been increased from v1.58.1 to v1.60.0.
-Users building from source will need to ensure their `rustc` version is up to
-date.
-
-
-# Upgrading to v1.85.0
-
-## Application service registration with "user" property deprecation
-
-Application services should ensure they call the `/register` endpoint with a
-`username` property. The legacy `user` property is considered deprecated and
-should no longer be included.
-
-A future version of Synapse (v1.88.0 or later) will remove support for legacy
-application service login.
 
 # Upgrading to v1.84.0
 
 ## Deprecation of `worker_replication_*` configuration settings
 
-When using workers,
-
+When using workers, 
 * `worker_replication_host`
 * `worker_replication_http_port`
 * `worker_replication_http_tls`
  
-should now be removed from individual worker YAML configurations and the main process should instead be added to the `instance_map`
-in the shared YAML configuration, using the name `main`.
-
-The old `worker_replication_*` settings are now considered deprecated and are expected to be removed in Synapse v1.88.0.
-
-
-### Example change
-
-#### Before:
+can now be removed from individual worker YAML configuration ***if*** you add the main process to the `instance_map` in the shared YAML configuration,
+using the name `main`.
 
+### Before:
 Shared YAML
 ```yaml
 instance_map:
@@ -135,7 +109,6 @@ instance_map:
     port: 5678
     tls: false
 ```
-
 Worker YAML
 ```yaml
 worker_app: synapse.app.generic_worker
@@ -157,10 +130,7 @@ worker_listeners:
 
 worker_log_config: /etc/matrix-synapse/generic-worker-log.yaml
 ```
-
-
-#### After:
-
+### After:
 Shared YAML
 ```yaml
 instance_map:
@@ -173,7 +143,6 @@ instance_map:
     port: 5678
     tls: false
 ```
-
 Worker YAML
 ```yaml
 worker_app: synapse.app.generic_worker
@@ -196,6 +165,7 @@ Notes:
 * `tls` is optional but mirrors the functionality of `worker_replication_http_tls`
 
 
+
 # Upgrading to v1.81.0
 
 ## Application service path & authentication deprecations

docs/usage/administration/admin_faq.md

@@ -27,8 +27,9 @@ What servers are currently participating in this room?
 Run this sql query on your db:
 ```sql
 SELECT DISTINCT split_part(state_key, ':', 2)
-FROM current_state_events
-WHERE room_id = '!cURbafjkfsMDVwdRDQ:matrix.org' AND membership = 'join';
+    FROM current_state_events AS c
+    INNER JOIN room_memberships AS m USING (room_id, event_id)
+    WHERE room_id = '!cURbafjkfsMDVwdRDQ:matrix.org' AND membership = 'join';
 ```
 
 What users are registered on my server?

docs/usage/configuration/config_documentation.md

@@ -2570,50 +2570,7 @@ Example configuration:
 ```yaml
 nonrefreshable_access_token_lifetime: 24h
 ```
----
-### `ui_auth`
 
-The amount of time to allow a user-interactive authentication session to be active.
-
-This defaults to 0, meaning the user is queried for their credentials
-before every action, but this can be overridden to allow a single
-validation to be re-used.  This weakens the protections afforded by
-the user-interactive authentication process, by allowing for multiple
-(and potentially different) operations to use the same validation session.
-
-This is ignored for potentially "dangerous" operations (including
-deactivating an account, modifying an account password, adding a 3PID,
-and minting additional login tokens).
-
-Use the `session_timeout` sub-option here to change the time allowed for credential validation.
-
-Example configuration:
-```yaml
-ui_auth:
-    session_timeout: "15s"
-```
----
-### `login_via_existing_session`
-
-Matrix supports the ability of an existing session to mint a login token for
-another client.
-
-Synapse disables this by default as it has security ramifications -- a malicious
-client could use the mechanism to spawn more than one session.
-
-The duration of time the generated token is valid for can be configured with the
-`token_timeout` sub-option.
-
-User-interactive authentication is required when this is enabled unless the
-`require_ui_auth` sub-option is set to `False`.
-
-Example configuration:
-```yaml
-login_via_existing_session:
-    enabled: true
-    require_ui_auth: false
-    token_timeout: "5m"
-```
 ---
 ## Metrics
 Config options related to metrics.
@@ -3458,6 +3415,28 @@ password_config:
       require_uppercase: true
 ```
 ---
+### `ui_auth`
+
+The amount of time to allow a user-interactive authentication session to be active.
+
+This defaults to 0, meaning the user is queried for their credentials
+before every action, but this can be overridden to allow a single
+validation to be re-used.  This weakens the protections afforded by
+the user-interactive authentication process, by allowing for multiple
+(and potentially different) operations to use the same validation session.
+
+This is ignored for potentially "dangerous" operations (including
+deactivating an account, modifying an account password, and
+adding a 3PID).
+
+Use the `session_timeout` sub-option here to change the time allowed for credential validation.
+
+Example configuration:
+```yaml
+ui_auth:
+    session_timeout: "15s"
+```
+---
 ## Push
 Configuration settings related to push notifications
 
@@ -4000,8 +3979,6 @@ This setting has the following sub-options:
 * `enabled`: whether to use Redis support. Defaults to false.
 * `host` and `port`: Optional host and port to use to connect to redis. Defaults to
    localhost and 6379
-* `path`: The full path to a local Unix socket file. **If this is used, `host` and
- `port` are ignored.** Defaults to `/tmp/redis.sock'
 * `password`: Optional password if configured on the Redis instance.
 * `dbid`: Optional redis dbid if needs to connect to specific redis logical db.
 * `use_tls`: Whether to use tls connection. Defaults to false.
@@ -4014,8 +3991,6 @@ This setting has the following sub-options:
 
   _Changed in Synapse 1.84.0: Added use\_tls, certificate\_file, private\_key\_file, ca\_file and ca\_path attributes_
 
-  _Changed in Synapse 1.85.0: Added path option to use a local Unix socket_
-
 Example configuration:
 ```yaml
 redis:

docs/workers.md

@@ -232,6 +232,7 @@ information.
     ^/_matrix/client/v1/rooms/.*/hierarchy$
     ^/_matrix/client/(v1|unstable)/rooms/.*/relations/
     ^/_matrix/client/v1/rooms/.*/threads$
+    ^/_matrix/client/unstable/org.matrix.msc2716/rooms/.*/batch_send$
     ^/_matrix/client/unstable/im.nheko.summary/rooms/.*/summary$
     ^/_matrix/client/(r0|v3|unstable)/account/3pid$
     ^/_matrix/client/(r0|v3|unstable)/account/whoami$

flake.nix

@@ -95,10 +95,6 @@
 
                   # For building the Synapse documentation website.
                   mdbook
-
-                  # For releasing Synapse
-                  debian-devscripts # (`dch` for manipulating the Debian changelog)
-                  libnotify # (the release script uses `notify-send` to tell you when CI jobs are done)
                 ];
 
                 # Install Python and manage a virtualenv with Poetry.

mypy.ini

@@ -2,29 +2,17 @@
 namespace_packages = True
 plugins = pydantic.mypy, mypy_zope:plugin, scripts-dev/mypy_synapse_plugin.py
 follow_imports = normal
+check_untyped_defs = True
 show_error_codes = True
 show_traceback = True
 mypy_path = stubs
 warn_unreachable = True
+warn_unused_ignores = True
 local_partial_types = True
 no_implicit_optional = True
-
-# Strict checks, see mypy --help
-warn_unused_configs = True
-# disallow_any_generics = True
-disallow_subclassing_any = True
-# disallow_untyped_calls = True
 disallow_untyped_defs = True
-disallow_incomplete_defs = True
-# check_untyped_defs = True
-# disallow_untyped_decorators = True
-warn_redundant_casts = True
-warn_unused_ignores = True
-# warn_return_any = True
-# no_implicit_reexport = True
 strict_equality = True
-strict_concatenate = True
-
+warn_redundant_casts = True
 # Run mypy type checking with the minimum supported Python version to catch new usage
 # that isn't backwards-compatible (types, overloads, etc).
 python_version = 3.8
@@ -43,7 +31,9 @@ warn_unused_ignores = False
 
 [mypy-synapse.util.caches.treecache]
 disallow_untyped_defs = False
-disallow_incomplete_defs = False
+
+[mypy-tests.util.caches.test_descriptors]
+disallow_untyped_defs = False
 
 ;; Dependencies without annotations
 ;; Before ignoring a module, check to see if type stubs are available.
@@ -53,18 +43,18 @@ disallow_incomplete_defs = False
 ;; which we can pull in as a dev dependency by adding to `pyproject.toml`'s
 ;; `[tool.poetry.dev-dependencies]` list.
 
-# https://github.com/lepture/authlib/issues/460
 [mypy-authlib.*]
 ignore_missing_imports = True
 
 [mypy-ijson.*]
 ignore_missing_imports = True
 
-# https://github.com/msgpack/msgpack-python/issues/448
+[mypy-lxml]
+ignore_missing_imports = True
+
 [mypy-msgpack]
 ignore_missing_imports = True
 
-# https://github.com/wolever/parameterized/issues/143
 [mypy-parameterized.*]
 ignore_missing_imports = True
 
@@ -86,7 +76,6 @@ ignore_missing_imports = True
 [mypy-srvlookup.*]
 ignore_missing_imports = True
 
-# https://github.com/twisted/treq/pull/366
 [mypy-treq.*]
 ignore_missing_imports = True
 

pyproject.toml

@@ -89,7 +89,7 @@ manifest-path = "rust/Cargo.toml"
 
 [tool.poetry]
 name = "matrix-synapse"
-version = "1.86.0rc2"
+version = "1.84.0rc1"
 description = "Homeserver for the Matrix decentralised comms protocol"
 authors = ["Matrix.org Team and Contributors <packages@matrix.org>"]
 license = "Apache-2.0"
@@ -311,10 +311,9 @@ all = [
 # We pin black so that our tests don't start failing on new releases.
 isort = ">=5.10.1"
 black = ">=22.3.0"
-ruff = "0.0.272"
+ruff = "0.0.265"
 
 # Typechecking
-lxml-stubs = ">=0.4.0"
 mypy = "*"
 mypy-zope = "*"
 types-bleach = ">=4.1.0"

rust/Cargo.toml

@@ -7,7 +7,7 @@ name = "synapse"
 version = "0.1.0"
 
 edition = "2021"
-rust-version = "1.60.0"
+rust-version = "1.58.1"
 
 [lib]
 name = "synapse"

rust/benches/evaluator.rs

@@ -13,6 +13,8 @@
 // limitations under the License.
 
 #![feature(test)]
+use std::collections::BTreeSet;
+
 use synapse::push::{
     evaluator::PushRuleEvaluator, Condition, EventMatchCondition, FilteredPushRules, JsonValue,
     PushRules, SimpleJsonValue,
@@ -195,6 +197,7 @@ fn bench_eval_message(b: &mut Bencher) {
         false,
         false,
         false,
+        false,
     );
 
     b.iter(|| eval.run(&rules, Some("bob"), Some("person")));

rust/src/push/base_rules.rs

@@ -142,11 +142,11 @@ pub const BASE_APPEND_OVERRIDE_RULES: &[PushRule] = &[
         default_enabled: true,
     },
     PushRule {
-        rule_id: Cow::Borrowed("global/override/.m.rule.is_user_mention"),
+        rule_id: Cow::Borrowed(".org.matrix.msc3952.is_user_mention"),
         priority_class: 5,
         conditions: Cow::Borrowed(&[Condition::Known(
             KnownCondition::ExactEventPropertyContainsType(EventPropertyIsTypeCondition {
-                key: Cow::Borrowed("content.m\\.mentions.user_ids"),
+                key: Cow::Borrowed("content.org\\.matrix\\.msc3952\\.mentions.user_ids"),
                 value_type: Cow::Borrowed(&EventMatchPatternType::UserId),
             }),
         )]),
@@ -163,11 +163,11 @@ pub const BASE_APPEND_OVERRIDE_RULES: &[PushRule] = &[
         default_enabled: true,
     },
     PushRule {
-        rule_id: Cow::Borrowed("global/override/.m.rule.is_room_mention"),
+        rule_id: Cow::Borrowed(".org.matrix.msc3952.is_room_mention"),
         priority_class: 5,
         conditions: Cow::Borrowed(&[
             Condition::Known(KnownCondition::EventPropertyIs(EventPropertyIsCondition {
-                key: Cow::Borrowed("content.m\\.mentions.room"),
+                key: Cow::Borrowed("content.org\\.matrix\\.msc3952\\.mentions.room"),
                 value: Cow::Borrowed(&SimpleJsonValue::Bool(true)),
             })),
             Condition::Known(KnownCondition::SenderNotificationPermission {

rust/src/push/evaluator.rs

@@ -70,9 +70,7 @@ pub struct PushRuleEvaluator {
     /// The "content.body", if any.
     body: String,
 
-    /// True if the event has a m.mentions property. (Note that this is a separate
-    /// flag instead of checking flattened_keys since the m.mentions property
-    /// might be an empty map and not appear in flattened_keys.
+    /// True if the event has a mentions property and MSC3952 support is enabled.
     has_mentions: bool,
 
     /// The number of users in the room.
@@ -157,7 +155,9 @@ impl PushRuleEvaluator {
             let rule_id = &push_rule.rule_id().to_string();
 
             // For backwards-compatibility the legacy mention rules are disabled
-            // if the event contains the 'm.mentions' property.
+            // if the event contains the 'm.mentions' property (and if the
+            // experimental feature is enabled, both of these are represented
+            // by the has_mentions flag).
             if self.has_mentions
                 && (rule_id == "global/override/.m.rule.contains_display_name"
                     || rule_id == "global/content/.m.rule.contains_user_name"
@@ -562,7 +562,7 @@ fn test_requires_room_version_supports_condition() {
     };
     let rules = PushRules::new(vec![custom_rule]);
     result = evaluator.run(
-        &FilteredPushRules::py_new(rules, BTreeMap::new(), true, false, true, false),
+        &FilteredPushRules::py_new(rules, BTreeMap::new(), true, false, true, false, false),
         None,
         None,
     );

rust/src/push/mod.rs

@@ -527,6 +527,7 @@ pub struct FilteredPushRules {
     msc1767_enabled: bool,
     msc3381_polls_enabled: bool,
     msc3664_enabled: bool,
+    msc3952_intentional_mentions: bool,
     msc3958_suppress_edits_enabled: bool,
 }
 
@@ -539,6 +540,7 @@ impl FilteredPushRules {
         msc1767_enabled: bool,
         msc3381_polls_enabled: bool,
         msc3664_enabled: bool,
+        msc3952_intentional_mentions: bool,
         msc3958_suppress_edits_enabled: bool,
     ) -> Self {
         Self {
@@ -547,6 +549,7 @@ impl FilteredPushRules {
             msc1767_enabled,
             msc3381_polls_enabled,
             msc3664_enabled,
+            msc3952_intentional_mentions,
             msc3958_suppress_edits_enabled,
         }
     }
@@ -584,6 +587,10 @@ impl FilteredPushRules {
                     return false;
                 }
 
+                if !self.msc3952_intentional_mentions && rule.rule_id.contains("org.matrix.msc3952")
+                {
+                    return false;
+                }
                 if !self.msc3958_suppress_edits_enabled
                     && rule.rule_id == "global/override/.com.beeper.suppress_edits"
                 {

scripts-dev/build_debian_packages.py

@@ -20,8 +20,6 @@ from concurrent.futures import ThreadPoolExecutor
 from types import FrameType
 from typing import Collection, Optional, Sequence, Set
 
-# These are expanded inside the dockerfile to be a fully qualified image name.
-# e.g. docker.io/library/debian:bullseye
 DISTS = (
     "debian:buster",  # oldstable: EOL 2022-08
     "debian:bullseye",

scripts-dev/complement.sh

@@ -246,6 +246,10 @@ else
   else
     export PASS_SYNAPSE_COMPLEMENT_DATABASE=sqlite
   fi
+
+  # The tests for importing historical messages (MSC2716)
+  # only pass with monoliths, currently.
+  test_tags="$test_tags,msc2716"
 fi
 
 if [[ -n "$ASYNCIO_REACTOR" ]]; then
@@ -265,10 +269,6 @@ if [[ -n "$SYNAPSE_TEST_LOG_LEVEL" ]]; then
   export PASS_SYNAPSE_LOG_SENSITIVE=1
 fi
 
-# Log a few more useful things for a developer attempting to debug something
-# particularly tricky.
-export PASS_SYNAPSE_LOG_TESTING=1
-
 # Run the tests!
 echo "Images built; running complement"
 cd "$COMPLEMENT_DIR"

scripts-dev/federation_client.py

@@ -136,11 +136,11 @@ def request(
         authorization_headers.append(header)
         print("Authorization: %s" % header, file=sys.stderr)
 
-    dest = "matrix-federation://%s%s" % (destination, path)
+    dest = "matrix://%s%s" % (destination, path)
     print("Requesting %s" % dest, file=sys.stderr)
 
     s = requests.Session()
-    s.mount("matrix-federation://", MatrixConnectionAdapter())
+    s.mount("matrix://", MatrixConnectionAdapter())
 
     headers: Dict[str, str] = {
         "Authorization": authorization_headers[0],

scripts-dev/mypy_synapse_plugin.py

@@ -18,11 +18,10 @@ can crop up, e.g the cache descriptors.
 
 from typing import Callable, Optional, Type
 
-from mypy.erasetype import remove_instance_last_known_values
 from mypy.nodes import ARG_NAMED_OPT
 from mypy.plugin import MethodSigContext, Plugin
 from mypy.typeops import bind_self
-from mypy.types import CallableType, Instance, NoneType, UnionType
+from mypy.types import CallableType, NoneType, UnionType
 
 
 class SynapsePlugin(Plugin):
@@ -93,41 +92,10 @@ def cached_function_method_signature(ctx: MethodSigContext) -> CallableType:
     arg_names.append("on_invalidate")
     arg_kinds.append(ARG_NAMED_OPT)  # Arg is an optional kwarg.
 
-    # Finally we ensure the return type is a Deferred.
-    if (
-        isinstance(signature.ret_type, Instance)
-        and signature.ret_type.type.fullname == "twisted.internet.defer.Deferred"
-    ):
-        # If it is already a Deferred, nothing to do.
-        ret_type = signature.ret_type
-    else:
-        ret_arg = None
-        if isinstance(signature.ret_type, Instance):
-            # If a coroutine, wrap the coroutine's return type in a Deferred.
-            if signature.ret_type.type.fullname == "typing.Coroutine":
-                ret_arg = signature.ret_type.args[2]
-
-            # If an awaitable, wrap the awaitable's final value in a Deferred.
-            elif signature.ret_type.type.fullname == "typing.Awaitable":
-                ret_arg = signature.ret_type.args[0]
-
-        # Otherwise, wrap the return value in a Deferred.
-        if ret_arg is None:
-            ret_arg = signature.ret_type
-
-        # This should be able to use ctx.api.named_generic_type, but that doesn't seem
-        # to find the correct symbol for anything more than 1 module deep.
-        #
-        # modules is not part of CheckerPluginInterface. The following is a combination
-        # of TypeChecker.named_generic_type and TypeChecker.lookup_typeinfo.
-        sym = ctx.api.modules["twisted.internet.defer"].names.get("Deferred")  # type: ignore[attr-defined]
-        ret_type = Instance(sym.node, [remove_instance_last_known_values(ret_arg)])
-
     signature = signature.copy_modified(
         arg_types=arg_types,
         arg_names=arg_names,
         arg_kinds=arg_kinds,
-        ret_type=ret_type,
     )
 
     return signature

scripts-dev/release.py

@@ -27,7 +27,7 @@ import time
 import urllib.request
 from os import path
 from tempfile import TemporaryDirectory
-from typing import Any, List, Match, Optional, Union
+from typing import Any, List, Optional
 
 import attr
 import click
@@ -233,7 +233,7 @@ def _prepare() -> None:
     subprocess.check_output(["poetry", "version", new_version])
 
     # Generate changelogs.
-    generate_and_write_changelog(synapse_repo, current_version, new_version)
+    generate_and_write_changelog(current_version, new_version)
 
     # Generate debian changelogs
     if parsed_new_version.pre is not None:
@@ -814,7 +814,7 @@ def get_changes_for_version(wanted_version: version.Version) -> str:
 
 
 def generate_and_write_changelog(
-    repo: Repo, current_version: version.Version, new_version: str
+    current_version: version.Version, new_version: str
 ) -> None:
     # We do this by getting a draft so that we can edit it before writing to the
     # changelog.
@@ -827,10 +827,6 @@ def generate_and_write_changelog(
     new_changes = new_changes.replace(
         "No significant changes.", f"No significant changes since {current_version}."
     )
-    new_changes += build_dependabot_changelog(
-        repo,
-        current_version,
-    )
 
     # Prepend changes to changelog
     with open("CHANGES.md", "r+") as f:
@@ -845,49 +841,5 @@ def generate_and_write_changelog(
         os.remove(filename)
 
 
-def build_dependabot_changelog(repo: Repo, current_version: version.Version) -> str:
-    """Summarise dependabot commits between `current_version` and `release_branch`.
-
-    Returns an empty string if there have been no such commits; otherwise outputs a
-    third-level markdown header followed by an unordered list."""
-    last_release_commit = repo.tag("v" + str(current_version)).commit
-    rev_spec = f"{last_release_commit.hexsha}.."
-    commits = list(git.objects.Commit.iter_items(repo, rev_spec))
-    messages = []
-    for commit in reversed(commits):
-        if commit.author.name == "dependabot[bot]":
-            message: Union[str, bytes] = commit.message
-            if isinstance(message, bytes):
-                message = message.decode("utf-8")
-            messages.append(message.split("\n", maxsplit=1)[0])
-
-    if not messages:
-        print(f"No dependabot commits in range {rev_spec}", file=sys.stderr)
-        return ""
-
-    messages.sort()
-
-    def replacer(match: Match[str]) -> str:
-        desc = match.group(1)
-        number = match.group(2)
-        return f"* {desc}. ([\\#{number}](https://github.com/matrix-org/synapse/issues/{number}))"
-
-    for i, message in enumerate(messages):
-        messages[i] = re.sub(r"(.*) \(#(\d+)\)$", replacer, message)
-    messages.insert(0, "### Updates to locked dependencies\n")
-    # Add an extra blank line to the bottom of the section
-    messages.append("")
-    return "\n".join(messages)
-
-
-@cli.command()
-@click.argument("since")
-def test_dependabot_changelog(since: str) -> None:
-    """Test building the dependabot changelog.
-
-    Summarises all dependabot commits between the SINCE tag and the current git HEAD."""
-    print(build_dependabot_changelog(git.Repo("."), version.Version(since)))
-
-
 if __name__ == "__main__":
     cli()

stubs/synapse/synapse_rust/push.pyi

@@ -46,6 +46,7 @@ class FilteredPushRules:
         msc1767_enabled: bool,
         msc3381_polls_enabled: bool,
         msc3664_enabled: bool,
+        msc3952_intentional_mentions: bool,
         msc3958_suppress_edits_enabled: bool,
     ): ...
     def rules(self) -> Collection[Tuple[PushRule, bool]]: ...

stubs/txredisapi.pyi

@@ -61,9 +61,6 @@ def lazyConnection(
 # most methods to it via ConnectionHandler.__getattr__.
 class ConnectionHandler(RedisProtocol):
     def disconnect(self) -> "Deferred[None]": ...
-    def __repr__(self) -> str: ...
-
-class UnixConnectionHandler(ConnectionHandler): ...
 
 class RedisFactory(protocol.ReconnectingClientFactory):
     continueTrying: bool

synapse/api/auth.py

@@ -1,4 +1,4 @@
-# Copyright 2023 The Matrix.org Foundation.
+# Copyright 2014 - 2016 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -14,6 +14,7 @@
 import logging
 from typing import TYPE_CHECKING, Optional, Tuple
 
+import pymacaroons
 from netaddr import IPAddress
 
 from twisted.web.server import Request
@@ -23,11 +24,19 @@ from synapse.api.constants import EventTypes, HistoryVisibility, Membership
 from synapse.api.errors import (
     AuthError,
     Codes,
+    InvalidClientTokenError,
     MissingClientTokenError,
     UnstableSpecAuthError,
 )
 from synapse.appservice import ApplicationService
-from synapse.logging.opentracing import trace
+from synapse.http import get_request_user_agent
+from synapse.http.site import SynapseRequest
+from synapse.logging.opentracing import (
+    active_span,
+    force_tracing,
+    start_active_span,
+    trace,
+)
 from synapse.types import Requester, create_requester
 from synapse.util.cancellation import cancellable
 
@@ -37,13 +46,26 @@ if TYPE_CHECKING:
 logger = logging.getLogger(__name__)
 
 
-class BaseAuth:
-    """Common base class for all auth implementations."""
+# guests always get this device id.
+GUEST_DEVICE_ID = "guest_device"
+
+
+class Auth:
+    """
+    This class contains functions for authenticating users of our client-server API.
+    """
 
     def __init__(self, hs: "HomeServer"):
         self.hs = hs
+        self.clock = hs.get_clock()
         self.store = hs.get_datastores().main
+        self._account_validity_handler = hs.get_account_validity_handler()
         self._storage_controllers = hs.get_storage_controllers()
+        self._macaroon_generator = hs.get_macaroon_generator()
+
+        self._track_appservice_user_ips = hs.config.appservice.track_appservice_user_ips
+        self._track_puppeted_user_ips = hs.config.api.track_puppeted_user_ips
+        self._force_tracing_for_users = hs.config.tracing.force_tracing_for_users
 
     async def check_user_in_room(
         self,
@@ -97,49 +119,139 @@ class BaseAuth:
             errcode=Codes.NOT_JOINED,
         )
 
-    @trace
-    async def check_user_in_room_or_world_readable(
-        self, room_id: str, requester: Requester, allow_departed_users: bool = False
-    ) -> Tuple[str, Optional[str]]:
-        """Checks that the user is or was in the room or the room is world
-        readable. If it isn't then an exception is raised.
+    @cancellable
+    async def get_user_by_req(
+        self,
+        request: SynapseRequest,
+        allow_guest: bool = False,
+        allow_expired: bool = False,
+    ) -> Requester:
+        """Get a registered user's ID.
 
         Args:
-            room_id: room to check
-            user_id: user to check
-            allow_departed_users: if True, accept users that were previously
-                members but have now departed
+            request: An HTTP request with an access_token query parameter.
+            allow_guest: If False, will raise an AuthError if the user making the
+                request is a guest.
+            allow_expired: If True, allow the request through even if the account
+                is expired, or session token lifetime has ended. Note that
+                /login will deliver access tokens regardless of expiration.
 
         Returns:
-            Resolves to the current membership of the user in the room and the
-            membership event ID of the user. If the user is not in the room and
-            never has been, then `(Membership.JOIN, None)` is returned.
+            Resolves to the requester
+        Raises:
+            InvalidClientCredentialsError if no user by that token exists or the token
+                is invalid.
+            AuthError if access is denied for the user in the access token
         """
+        parent_span = active_span()
+        with start_active_span("get_user_by_req"):
+            requester = await self._wrapped_get_user_by_req(
+                request, allow_guest, allow_expired
+            )
 
+            if parent_span:
+                if requester.authenticated_entity in self._force_tracing_for_users:
+                    # request tracing is enabled for this user, so we need to force it
+                    # tracing on for the parent span (which will be the servlet span).
+                    #
+                    # It's too late for the get_user_by_req span to inherit the setting,
+                    # so we also force it on for that.
+                    force_tracing()
+                    force_tracing(parent_span)
+                parent_span.set_tag(
+                    "authenticated_entity", requester.authenticated_entity
+                )
+                parent_span.set_tag("user_id", requester.user.to_string())
+                if requester.device_id is not None:
+                    parent_span.set_tag("device_id", requester.device_id)
+                if requester.app_service is not None:
+                    parent_span.set_tag("appservice_id", requester.app_service.id)
+            return requester
+
+    @cancellable
+    async def _wrapped_get_user_by_req(
+        self,
+        request: SynapseRequest,
+        allow_guest: bool,
+        allow_expired: bool,
+    ) -> Requester:
+        """Helper for get_user_by_req
+
+        Once get_user_by_req has set up the opentracing span, this does the actual work.
+        """
         try:
-            # check_user_in_room will return the most recent membership
-            # event for the user if:
-            #  * The user is a non-guest user, and was ever in the room
-            #  * The user is a guest user, and has joined the room
-            # else it will throw.
-            return await self.check_user_in_room(
-                room_id, requester, allow_departed_users=allow_departed_users
-            )
-        except AuthError:
-            visibility = await self._storage_controllers.state.get_current_state_event(
-                room_id, EventTypes.RoomHistoryVisibility, ""
-            )
-            if (
-                visibility
-                and visibility.content.get("history_visibility")
-                == HistoryVisibility.WORLD_READABLE
+            ip_addr = request.getClientAddress().host
+            user_agent = get_request_user_agent(request)
+
+            access_token = self.get_access_token_from_request(request)
+
+            # First check if it could be a request from an appservice
+            requester = await self._get_appservice_user(request)
+            if not requester:
+                # If not, it should be from a regular user
+                requester = await self.get_user_by_access_token(
+                    access_token, allow_expired=allow_expired
+                )
+
+                # Deny the request if the user account has expired.
+                # This check is only done for regular users, not appservice ones.
+                if not allow_expired:
+                    if await self._account_validity_handler.is_user_expired(
+                        requester.user.to_string()
+                    ):
+                        # Raise the error if either an account validity module has determined
+                        # the account has expired, or the legacy account validity
+                        # implementation is enabled and determined the account has expired
+                        raise AuthError(
+                            403,
+                            "User account has expired",
+                            errcode=Codes.EXPIRED_ACCOUNT,
+                        )
+
+            if ip_addr and (
+                not requester.app_service or self._track_appservice_user_ips
             ):
-                return Membership.JOIN, None
-            raise AuthError(
-                403,
-                "User %r not in room %s, and room previews are disabled"
-                % (requester.user, room_id),
-            )
+                # XXX(quenting): I'm 95% confident that we could skip setting the
+                # device_id to "dummy-device" for appservices, and that the only impact
+                # would be some rows which whould not deduplicate in the 'user_ips'
+                # table during the transition
+                recorded_device_id = (
+                    "dummy-device"
+                    if requester.device_id is None and requester.app_service is not None
+                    else requester.device_id
+                )
+                await self.store.insert_client_ip(
+                    user_id=requester.authenticated_entity,
+                    access_token=access_token,
+                    ip=ip_addr,
+                    user_agent=user_agent,
+                    device_id=recorded_device_id,
+                )
+
+                # Track also the puppeted user client IP if enabled and the user is puppeting
+                if (
+                    requester.user.to_string() != requester.authenticated_entity
+                    and self._track_puppeted_user_ips
+                ):
+                    await self.store.insert_client_ip(
+                        user_id=requester.user.to_string(),
+                        access_token=access_token,
+                        ip=ip_addr,
+                        user_agent=user_agent,
+                        device_id=requester.device_id,
+                    )
+
+            if requester.is_guest and not allow_guest:
+                raise AuthError(
+                    403,
+                    "Guest access not allowed",
+                    errcode=Codes.GUEST_ACCESS_FORBIDDEN,
+                )
+
+            request.requester = requester
+            return requester
+        except KeyError:
+            raise MissingClientTokenError()
 
     async def validate_appservice_can_control_user_id(
         self, app_service: ApplicationService, user_id: str
@@ -172,16 +284,184 @@ class BaseAuth:
                 403, "Application service has not registered this user (%s)" % user_id
             )
 
+    @cancellable
+    async def _get_appservice_user(self, request: Request) -> Optional[Requester]:
+        """
+        Given a request, reads the request parameters to determine:
+        - whether it's an application service that's making this request
+        - what user the application service should be treated as controlling
+          (the user_id URI parameter allows an application service to masquerade
+          any applicable user in its namespace)
+        - what device the application service should be treated as controlling
+          (the device_id[^1] URI parameter allows an application service to masquerade
+          as any device that exists for the relevant user)
+
+        [^1] Unstable and provided by MSC3202.
+             Must use `org.matrix.msc3202.device_id` in place of `device_id` for now.
+
+        Returns:
+            the application service `Requester` of that request
+
+        Postconditions:
+        - The `app_service` field in the returned `Requester` is set
+        - The `user_id` field in the returned `Requester` is either the application
+          service sender or the controlled user set by the `user_id` URI parameter
+        - The returned application service is permitted to control the returned user ID.
+        - The returned device ID, if present, has been checked to be a valid device ID
+          for the returned user ID.
+        """
+        DEVICE_ID_ARG_NAME = b"org.matrix.msc3202.device_id"
+
+        app_service = self.store.get_app_service_by_token(
+            self.get_access_token_from_request(request)
+        )
+        if app_service is None:
+            return None
+
+        if app_service.ip_range_whitelist:
+            ip_address = IPAddress(request.getClientAddress().host)
+            if ip_address not in app_service.ip_range_whitelist:
+                return None
+
+        # This will always be set by the time Twisted calls us.
+        assert request.args is not None
+
+        if b"user_id" in request.args:
+            effective_user_id = request.args[b"user_id"][0].decode("utf8")
+            await self.validate_appservice_can_control_user_id(
+                app_service, effective_user_id
+            )
+        else:
+            effective_user_id = app_service.sender
+
+        effective_device_id: Optional[str] = None
+
+        if (
+            self.hs.config.experimental.msc3202_device_masquerading_enabled
+            and DEVICE_ID_ARG_NAME in request.args
+        ):
+            effective_device_id = request.args[DEVICE_ID_ARG_NAME][0].decode("utf8")
+            # We only just set this so it can't be None!
+            assert effective_device_id is not None
+            device_opt = await self.store.get_device(
+                effective_user_id, effective_device_id
+            )
+            if device_opt is None:
+                # For now, use 400 M_EXCLUSIVE if the device doesn't exist.
+                # This is an open thread of discussion on MSC3202 as of 2021-12-09.
+                raise AuthError(
+                    400,
+                    f"Application service trying to use a device that doesn't exist ('{effective_device_id}' for {effective_user_id})",
+                    Codes.EXCLUSIVE,
+                )
+
+        return create_requester(
+            effective_user_id, app_service=app_service, device_id=effective_device_id
+        )
+
+    async def get_user_by_access_token(
+        self,
+        token: str,
+        allow_expired: bool = False,
+    ) -> Requester:
+        """Validate access token and get user_id from it
+
+        Args:
+            token: The access token to get the user by
+            allow_expired: If False, raises an InvalidClientTokenError
+                if the token is expired
+
+        Raises:
+            InvalidClientTokenError if a user by that token exists, but the token is
+                expired
+            InvalidClientCredentialsError if no user by that token exists or the token
+                is invalid
+        """
+
+        # First look in the database to see if the access token is present
+        # as an opaque token.
+        user_info = await self.store.get_user_by_access_token(token)
+        if user_info:
+            valid_until_ms = user_info.valid_until_ms
+            if (
+                not allow_expired
+                and valid_until_ms is not None
+                and valid_until_ms < self.clock.time_msec()
+            ):
+                # there was a valid access token, but it has expired.
+                # soft-logout the user.
+                raise InvalidClientTokenError(
+                    msg="Access token has expired", soft_logout=True
+                )
+
+            # Mark the token as used. This is used to invalidate old refresh
+            # tokens after some time.
+            await self.store.mark_access_token_as_used(user_info.token_id)
+
+            requester = create_requester(
+                user_id=user_info.user_id,
+                access_token_id=user_info.token_id,
+                is_guest=user_info.is_guest,
+                shadow_banned=user_info.shadow_banned,
+                device_id=user_info.device_id,
+                authenticated_entity=user_info.token_owner,
+            )
+
+            return requester
+
+        # If the token isn't found in the database, then it could still be a
+        # macaroon for a guest, so we check that here.
+        try:
+            user_id = self._macaroon_generator.verify_guest_token(token)
+
+            # Guest access tokens are not stored in the database (there can
+            # only be one access token per guest, anyway).
+            #
+            # In order to prevent guest access tokens being used as regular
+            # user access tokens (and hence getting around the invalidation
+            # process), we look up the user id and check that it is indeed
+            # a guest user.
+            #
+            # It would of course be much easier to store guest access
+            # tokens in the database as well, but that would break existing
+            # guest tokens.
+            stored_user = await self.store.get_user_by_id(user_id)
+            if not stored_user:
+                raise InvalidClientTokenError("Unknown user_id %s" % user_id)
+            if not stored_user["is_guest"]:
+                raise InvalidClientTokenError(
+                    "Guest access token used for regular user"
+                )
+
+            return create_requester(
+                user_id=user_id,
+                is_guest=True,
+                # all guests get the same device id
+                device_id=GUEST_DEVICE_ID,
+                authenticated_entity=user_id,
+            )
+        except (
+            pymacaroons.exceptions.MacaroonException,
+            TypeError,
+            ValueError,
+        ) as e:
+            logger.warning(
+                "Invalid access token in auth: %s %s.",
+                type(e),
+                e,
+            )
+            raise InvalidClientTokenError("Invalid access token passed.")
+
     async def is_server_admin(self, requester: Requester) -> bool:
         """Check if the given user is a local server admin.
 
         Args:
-            requester: user to check
+            requester: The user making the request, according to the access token.
 
         Returns:
             True if the user is an admin
         """
-        raise NotImplementedError()
+        return await self.store.is_server_admin(requester.user)
 
     async def check_can_change_room_list(
         self, room_id: str, requester: Requester
@@ -190,8 +470,8 @@ class BaseAuth:
         published room list.
 
         Args:
-            room_id
-            user
+            room_id: The room to check.
+            requester: The user making the request, according to the access token.
         """
 
         is_admin = await self.is_server_admin(requester)
@@ -238,6 +518,7 @@ class BaseAuth:
         return bool(query_params) or bool(auth_headers)
 
     @staticmethod
+    @cancellable
     def get_access_token_from_request(request: Request) -> str:
         """Extracts the access_token from the request.
 
@@ -275,77 +556,47 @@ class BaseAuth:
 
             return query_params[0].decode("ascii")
 
-    @cancellable
-    async def get_appservice_user(
-        self, request: Request, access_token: str
-    ) -> Optional[Requester]:
-        """
-        Given a request, reads the request parameters to determine:
-        - whether it's an application service that's making this request
-        - what user the application service should be treated as controlling
-          (the user_id URI parameter allows an application service to masquerade
-          any applicable user in its namespace)
-        - what device the application service should be treated as controlling
-          (the device_id[^1] URI parameter allows an application service to masquerade
-          as any device that exists for the relevant user)
+    @trace
+    async def check_user_in_room_or_world_readable(
+        self, room_id: str, requester: Requester, allow_departed_users: bool = False
+    ) -> Tuple[str, Optional[str]]:
+        """Checks that the user is or was in the room or the room is world
+        readable. If it isn't then an exception is raised.
 
-        [^1] Unstable and provided by MSC3202.
-             Must use `org.matrix.msc3202.device_id` in place of `device_id` for now.
+        Args:
+            room_id: The room to check.
+            requester: The user making the request, according to the access token.
+            allow_departed_users: If True, accept users that were previously
+                members but have now departed.
 
         Returns:
-            the application service `Requester` of that request
-
-        Postconditions:
-        - The `app_service` field in the returned `Requester` is set
-        - The `user_id` field in the returned `Requester` is either the application
-          service sender or the controlled user set by the `user_id` URI parameter
-        - The returned application service is permitted to control the returned user ID.
-        - The returned device ID, if present, has been checked to be a valid device ID
-          for the returned user ID.
+            Resolves to the current membership of the user in the room and the
+            membership event ID of the user. If the user is not in the room and
+            never has been, then `(Membership.JOIN, None)` is returned.
         """
-        DEVICE_ID_ARG_NAME = b"org.matrix.msc3202.device_id"
 
-        app_service = self.store.get_app_service_by_token(access_token)
-        if app_service is None:
-            return None
-
-        if app_service.ip_range_whitelist:
-            ip_address = IPAddress(request.getClientAddress().host)
-            if ip_address not in app_service.ip_range_whitelist:
-                return None
-
-        # This will always be set by the time Twisted calls us.
-        assert request.args is not None
-
-        if b"user_id" in request.args:
-            effective_user_id = request.args[b"user_id"][0].decode("utf8")
-            await self.validate_appservice_can_control_user_id(
-                app_service, effective_user_id
+        try:
+            # check_user_in_room will return the most recent membership
+            # event for the user if:
+            #  * The user is a non-guest user, and was ever in the room
+            #  * The user is a guest user, and has joined the room
+            # else it will throw.
+            return await self.check_user_in_room(
+                room_id, requester, allow_departed_users=allow_departed_users
             )
-        else:
-            effective_user_id = app_service.sender
-
-        effective_device_id: Optional[str] = None
-
-        if (
-            self.hs.config.experimental.msc3202_device_masquerading_enabled
-            and DEVICE_ID_ARG_NAME in request.args
-        ):
-            effective_device_id = request.args[DEVICE_ID_ARG_NAME][0].decode("utf8")
-            # We only just set this so it can't be None!
-            assert effective_device_id is not None
-            device_opt = await self.store.get_device(
-                effective_user_id, effective_device_id
+        except AuthError:
+            visibility = await self._storage_controllers.state.get_current_state_event(
+                room_id, EventTypes.RoomHistoryVisibility, ""
+            )
+            if (
+                visibility
+                and visibility.content.get("history_visibility")
+                == HistoryVisibility.WORLD_READABLE
+            ):
+                return Membership.JOIN, None
+            raise UnstableSpecAuthError(
+                403,
+                "User %s not in room %s, and room previews are disabled"
+                % (requester.user, room_id),
+                errcode=Codes.NOT_JOINED,
             )
-            if device_opt is None:
-                # For now, use 400 M_EXCLUSIVE if the device doesn't exist.
-                # This is an open thread of discussion on MSC3202 as of 2021-12-09.
-                raise AuthError(
-                    400,
-                    f"Application service trying to use a device that doesn't exist ('{effective_device_id}' for {effective_user_id})",
-                    Codes.EXCLUSIVE,
-                )
-
-        return create_requester(
-            effective_user_id, app_service=app_service, device_id=effective_device_id
-        )

synapse/api/auth/__init__.py

@@ -1,175 +0,0 @@
-# Copyright 2023 The Matrix.org Foundation.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-from typing import Optional, Tuple
-
-from typing_extensions import Protocol
-
-from twisted.web.server import Request
-
-from synapse.appservice import ApplicationService
-from synapse.http.site import SynapseRequest
-from synapse.types import Requester
-
-# guests always get this device id.
-GUEST_DEVICE_ID = "guest_device"
-
-
-class Auth(Protocol):
-    """The interface that an auth provider must implement."""
-
-    async def check_user_in_room(
-        self,
-        room_id: str,
-        requester: Requester,
-        allow_departed_users: bool = False,
-    ) -> Tuple[str, Optional[str]]:
-        """Check if the user is in the room, or was at some point.
-        Args:
-            room_id: The room to check.
-
-            user_id: The user to check.
-
-            current_state: Optional map of the current state of the room.
-                If provided then that map is used to check whether they are a
-                member of the room. Otherwise the current membership is
-                loaded from the database.
-
-            allow_departed_users: if True, accept users that were previously
-                members but have now departed.
-
-        Raises:
-            AuthError if the user is/was not in the room.
-        Returns:
-            The current membership of the user in the room and the
-            membership event ID of the user.
-        """
-
-    async def get_user_by_req(
-        self,
-        request: SynapseRequest,
-        allow_guest: bool = False,
-        allow_expired: bool = False,
-    ) -> Requester:
-        """Get a registered user's ID.
-
-        Args:
-            request: An HTTP request with an access_token query parameter.
-            allow_guest: If False, will raise an AuthError if the user making the
-                request is a guest.
-            allow_expired: If True, allow the request through even if the account
-                is expired, or session token lifetime has ended. Note that
-                /login will deliver access tokens regardless of expiration.
-
-        Returns:
-            Resolves to the requester
-        Raises:
-            InvalidClientCredentialsError if no user by that token exists or the token
-                is invalid.
-            AuthError if access is denied for the user in the access token
-        """
-
-    async def validate_appservice_can_control_user_id(
-        self, app_service: ApplicationService, user_id: str
-    ) -> None:
-        """Validates that the app service is allowed to control
-        the given user.
-
-        Args:
-            app_service: The app service that controls the user
-            user_id: The author MXID that the app service is controlling
-
-        Raises:
-            AuthError: If the application service is not allowed to control the user
-                (user namespace regex does not match, wrong homeserver, etc)
-                or if the user has not been registered yet.
-        """
-
-    async def get_user_by_access_token(
-        self,
-        token: str,
-        allow_expired: bool = False,
-    ) -> Requester:
-        """Validate access token and get user_id from it
-
-        Args:
-            token: The access token to get the user by
-            allow_expired: If False, raises an InvalidClientTokenError
-                if the token is expired
-
-        Raises:
-            InvalidClientTokenError if a user by that token exists, but the token is
-                expired
-            InvalidClientCredentialsError if no user by that token exists or the token
-                is invalid
-        """
-
-    async def is_server_admin(self, requester: Requester) -> bool:
-        """Check if the given user is a local server admin.
-
-        Args:
-            requester: user to check
-
-        Returns:
-            True if the user is an admin
-        """
-
-    async def check_can_change_room_list(
-        self, room_id: str, requester: Requester
-    ) -> bool:
-        """Determine whether the user is allowed to edit the room's entry in the
-        published room list.
-
-        Args:
-            room_id
-            user
-        """
-
-    @staticmethod
-    def has_access_token(request: Request) -> bool:
-        """Checks if the request has an access_token.
-
-        Returns:
-            False if no access_token was given, True otherwise.
-        """
-
-    @staticmethod
-    def get_access_token_from_request(request: Request) -> str:
-        """Extracts the access_token from the request.
-
-        Args:
-            request: The http request.
-        Returns:
-            The access_token
-        Raises:
-            MissingClientTokenError: If there isn't a single access_token in the
-                request
-        """
-
-    async def check_user_in_room_or_world_readable(
-        self, room_id: str, requester: Requester, allow_departed_users: bool = False
-    ) -> Tuple[str, Optional[str]]:
-        """Checks that the user is or was in the room or the room is world
-        readable. If it isn't then an exception is raised.
-
-        Args:
-            room_id: room to check
-            user_id: user to check
-            allow_departed_users: if True, accept users that were previously
-                members but have now departed
-
-        Returns:
-            Resolves to the current membership of the user in the room and the
-            membership event ID of the user. If the user is not in the room and
-            never has been, then `(Membership.JOIN, None)` is returned.
-        """

synapse/api/auth/internal.py

@@ -1,291 +0,0 @@
-# Copyright 2023 The Matrix.org Foundation.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-import logging
-from typing import TYPE_CHECKING
-
-import pymacaroons
-
-from synapse.api.errors import (
-    AuthError,
-    Codes,
-    InvalidClientTokenError,
-    MissingClientTokenError,
-)
-from synapse.http import get_request_user_agent
-from synapse.http.site import SynapseRequest
-from synapse.logging.opentracing import active_span, force_tracing, start_active_span
-from synapse.types import Requester, create_requester
-from synapse.util.cancellation import cancellable
-
-from . import GUEST_DEVICE_ID
-from .base import BaseAuth
-
-if TYPE_CHECKING:
-    from synapse.server import HomeServer
-
-logger = logging.getLogger(__name__)
-
-
-class InternalAuth(BaseAuth):
-    """
-    This class contains functions for authenticating users of our client-server API.
-    """
-
-    def __init__(self, hs: "HomeServer"):
-        super().__init__(hs)
-        self.clock = hs.get_clock()
-        self._account_validity_handler = hs.get_account_validity_handler()
-        self._macaroon_generator = hs.get_macaroon_generator()
-
-        self._track_appservice_user_ips = hs.config.appservice.track_appservice_user_ips
-        self._track_puppeted_user_ips = hs.config.api.track_puppeted_user_ips
-        self._force_tracing_for_users = hs.config.tracing.force_tracing_for_users
-
-    @cancellable
-    async def get_user_by_req(
-        self,
-        request: SynapseRequest,
-        allow_guest: bool = False,
-        allow_expired: bool = False,
-    ) -> Requester:
-        """Get a registered user's ID.
-
-        Args:
-            request: An HTTP request with an access_token query parameter.
-            allow_guest: If False, will raise an AuthError if the user making the
-                request is a guest.
-            allow_expired: If True, allow the request through even if the account
-                is expired, or session token lifetime has ended. Note that
-                /login will deliver access tokens regardless of expiration.
-
-        Returns:
-            Resolves to the requester
-        Raises:
-            InvalidClientCredentialsError if no user by that token exists or the token
-                is invalid.
-            AuthError if access is denied for the user in the access token
-        """
-        parent_span = active_span()
-        with start_active_span("get_user_by_req"):
-            requester = await self._wrapped_get_user_by_req(
-                request, allow_guest, allow_expired
-            )
-
-            if parent_span:
-                if requester.authenticated_entity in self._force_tracing_for_users:
-                    # request tracing is enabled for this user, so we need to force it
-                    # tracing on for the parent span (which will be the servlet span).
-                    #
-                    # It's too late for the get_user_by_req span to inherit the setting,
-                    # so we also force it on for that.
-                    force_tracing()
-                    force_tracing(parent_span)
-                parent_span.set_tag(
-                    "authenticated_entity", requester.authenticated_entity
-                )
-                parent_span.set_tag("user_id", requester.user.to_string())
-                if requester.device_id is not None:
-                    parent_span.set_tag("device_id", requester.device_id)
-                if requester.app_service is not None:
-                    parent_span.set_tag("appservice_id", requester.app_service.id)
-            return requester
-
-    @cancellable
-    async def _wrapped_get_user_by_req(
-        self,
-        request: SynapseRequest,
-        allow_guest: bool,
-        allow_expired: bool,
-    ) -> Requester:
-        """Helper for get_user_by_req
-
-        Once get_user_by_req has set up the opentracing span, this does the actual work.
-        """
-        try:
-            ip_addr = request.getClientAddress().host
-            user_agent = get_request_user_agent(request)
-
-            access_token = self.get_access_token_from_request(request)
-
-            # First check if it could be a request from an appservice
-            requester = await self.get_appservice_user(request, access_token)
-            if not requester:
-                # If not, it should be from a regular user
-                requester = await self.get_user_by_access_token(
-                    access_token, allow_expired=allow_expired
-                )
-
-                # Deny the request if the user account has expired.
-                # This check is only done for regular users, not appservice ones.
-                if not allow_expired:
-                    if await self._account_validity_handler.is_user_expired(
-                        requester.user.to_string()
-                    ):
-                        # Raise the error if either an account validity module has determined
-                        # the account has expired, or the legacy account validity
-                        # implementation is enabled and determined the account has expired
-                        raise AuthError(
-                            403,
-                            "User account has expired",
-                            errcode=Codes.EXPIRED_ACCOUNT,
-                        )
-
-            if ip_addr and (
-                not requester.app_service or self._track_appservice_user_ips
-            ):
-                # XXX(quenting): I'm 95% confident that we could skip setting the
-                # device_id to "dummy-device" for appservices, and that the only impact
-                # would be some rows which whould not deduplicate in the 'user_ips'
-                # table during the transition
-                recorded_device_id = (
-                    "dummy-device"
-                    if requester.device_id is None and requester.app_service is not None
-                    else requester.device_id
-                )
-                await self.store.insert_client_ip(
-                    user_id=requester.authenticated_entity,
-                    access_token=access_token,
-                    ip=ip_addr,
-                    user_agent=user_agent,
-                    device_id=recorded_device_id,
-                )
-
-                # Track also the puppeted user client IP if enabled and the user is puppeting
-                if (
-                    requester.user.to_string() != requester.authenticated_entity
-                    and self._track_puppeted_user_ips
-                ):
-                    await self.store.insert_client_ip(
-                        user_id=requester.user.to_string(),
-                        access_token=access_token,
-                        ip=ip_addr,
-                        user_agent=user_agent,
-                        device_id=requester.device_id,
-                    )
-
-            if requester.is_guest and not allow_guest:
-                raise AuthError(
-                    403,
-                    "Guest access not allowed",
-                    errcode=Codes.GUEST_ACCESS_FORBIDDEN,
-                )
-
-            request.requester = requester
-            return requester
-        except KeyError:
-            raise MissingClientTokenError()
-
-    async def get_user_by_access_token(
-        self,
-        token: str,
-        allow_expired: bool = False,
-    ) -> Requester:
-        """Validate access token and get user_id from it
-
-        Args:
-            token: The access token to get the user by
-            allow_expired: If False, raises an InvalidClientTokenError
-                if the token is expired
-
-        Raises:
-            InvalidClientTokenError if a user by that token exists, but the token is
-                expired
-            InvalidClientCredentialsError if no user by that token exists or the token
-                is invalid
-        """
-
-        # First look in the database to see if the access token is present
-        # as an opaque token.
-        user_info = await self.store.get_user_by_access_token(token)
-        if user_info:
-            valid_until_ms = user_info.valid_until_ms
-            if (
-                not allow_expired
-                and valid_until_ms is not None
-                and valid_until_ms < self.clock.time_msec()
-            ):
-                # there was a valid access token, but it has expired.
-                # soft-logout the user.
-                raise InvalidClientTokenError(
-                    msg="Access token has expired", soft_logout=True
-                )
-
-            # Mark the token as used. This is used to invalidate old refresh
-            # tokens after some time.
-            await self.store.mark_access_token_as_used(user_info.token_id)
-
-            requester = create_requester(
-                user_id=user_info.user_id,
-                access_token_id=user_info.token_id,
-                is_guest=user_info.is_guest,
-                shadow_banned=user_info.shadow_banned,
-                device_id=user_info.device_id,
-                authenticated_entity=user_info.token_owner,
-            )
-
-            return requester
-
-        # If the token isn't found in the database, then it could still be a
-        # macaroon for a guest, so we check that here.
-        try:
-            user_id = self._macaroon_generator.verify_guest_token(token)
-
-            # Guest access tokens are not stored in the database (there can
-            # only be one access token per guest, anyway).
-            #
-            # In order to prevent guest access tokens being used as regular
-            # user access tokens (and hence getting around the invalidation
-            # process), we look up the user id and check that it is indeed
-            # a guest user.
-            #
-            # It would of course be much easier to store guest access
-            # tokens in the database as well, but that would break existing
-            # guest tokens.
-            stored_user = await self.store.get_user_by_id(user_id)
-            if not stored_user:
-                raise InvalidClientTokenError("Unknown user_id %s" % user_id)
-            if not stored_user["is_guest"]:
-                raise InvalidClientTokenError(
-                    "Guest access token used for regular user"
-                )
-
-            return create_requester(
-                user_id=user_id,
-                is_guest=True,
-                # all guests get the same device id
-                device_id=GUEST_DEVICE_ID,
-                authenticated_entity=user_id,
-            )
-        except (
-            pymacaroons.exceptions.MacaroonException,
-            TypeError,
-            ValueError,
-        ) as e:
-            logger.warning(
-                "Invalid access token in auth: %s %s.",
-                type(e),
-                e,
-            )
-            raise InvalidClientTokenError("Invalid access token passed.")
-
-    async def is_server_admin(self, requester: Requester) -> bool:
-        """Check if the given user is a local server admin.
-
-        Args:
-            requester: The user making the request, according to the access token.
-
-        Returns:
-            True if the user is an admin
-        """
-        return await self.store.is_server_admin(requester.user)

synapse/api/auth/msc3861_delegated.py

@@ -1,352 +0,0 @@
-# Copyright 2023 The Matrix.org Foundation.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-import logging
-from typing import TYPE_CHECKING, Any, Dict, List, Optional
-from urllib.parse import urlencode
-
-from authlib.oauth2 import ClientAuth
-from authlib.oauth2.auth import encode_client_secret_basic, encode_client_secret_post
-from authlib.oauth2.rfc7523 import ClientSecretJWT, PrivateKeyJWT, private_key_jwt_sign
-from authlib.oauth2.rfc7662 import IntrospectionToken
-from authlib.oidc.discovery import OpenIDProviderMetadata, get_well_known_url
-
-from twisted.web.client import readBody
-from twisted.web.http_headers import Headers
-
-from synapse.api.auth.base import BaseAuth
-from synapse.api.errors import (
-    AuthError,
-    HttpResponseException,
-    InvalidClientTokenError,
-    OAuthInsufficientScopeError,
-    StoreError,
-    SynapseError,
-)
-from synapse.http.site import SynapseRequest
-from synapse.logging.context import make_deferred_yieldable
-from synapse.types import Requester, UserID, create_requester
-from synapse.util import json_decoder
-from synapse.util.caches.cached_call import RetryOnExceptionCachedCall
-
-if TYPE_CHECKING:
-    from synapse.server import HomeServer
-
-logger = logging.getLogger(__name__)
-
-# Scope as defined by MSC2967
-# https://github.com/matrix-org/matrix-spec-proposals/pull/2967
-SCOPE_MATRIX_API = "urn:matrix:org.matrix.msc2967.client:api:*"
-SCOPE_MATRIX_GUEST = "urn:matrix:org.matrix.msc2967.client:api:guest"
-SCOPE_MATRIX_DEVICE_PREFIX = "urn:matrix:org.matrix.msc2967.client:device:"
-
-# Scope which allows access to the Synapse admin API
-SCOPE_SYNAPSE_ADMIN = "urn:synapse:admin:*"
-
-
-def scope_to_list(scope: str) -> List[str]:
-    """Convert a scope string to a list of scope tokens"""
-    return scope.strip().split(" ")
-
-
-class PrivateKeyJWTWithKid(PrivateKeyJWT):  # type: ignore[misc]
-    """An implementation of the private_key_jwt client auth method that includes a kid header.
-
-    This is needed because some providers (Keycloak) require the kid header to figure
-    out which key to use to verify the signature.
-    """
-
-    def sign(self, auth: Any, token_endpoint: str) -> bytes:
-        return private_key_jwt_sign(
-            auth.client_secret,
-            client_id=auth.client_id,
-            token_endpoint=token_endpoint,
-            claims=self.claims,
-            header={"kid": auth.client_secret["kid"]},
-        )
-
-
-class MSC3861DelegatedAuth(BaseAuth):
-    AUTH_METHODS = {
-        "client_secret_post": encode_client_secret_post,
-        "client_secret_basic": encode_client_secret_basic,
-        "client_secret_jwt": ClientSecretJWT(),
-        "private_key_jwt": PrivateKeyJWTWithKid(),
-    }
-
-    EXTERNAL_ID_PROVIDER = "oauth-delegated"
-
-    def __init__(self, hs: "HomeServer"):
-        super().__init__(hs)
-
-        self._config = hs.config.experimental.msc3861
-        auth_method = MSC3861DelegatedAuth.AUTH_METHODS.get(
-            self._config.client_auth_method.value, None
-        )
-        # Those assertions are already checked when parsing the config
-        assert self._config.enabled, "OAuth delegation is not enabled"
-        assert self._config.issuer, "No issuer provided"
-        assert self._config.client_id, "No client_id provided"
-        assert auth_method is not None, "Invalid client_auth_method provided"
-
-        self._http_client = hs.get_proxied_http_client()
-        self._hostname = hs.hostname
-        self._admin_token = self._config.admin_token
-
-        self._issuer_metadata = RetryOnExceptionCachedCall(self._load_metadata)
-
-        if isinstance(auth_method, PrivateKeyJWTWithKid):
-            # Use the JWK as the client secret when using the private_key_jwt method
-            assert self._config.jwk, "No JWK provided"
-            self._client_auth = ClientAuth(
-                self._config.client_id, self._config.jwk, auth_method
-            )
-        else:
-            # Else use the client secret
-            assert self._config.client_secret, "No client_secret provided"
-            self._client_auth = ClientAuth(
-                self._config.client_id, self._config.client_secret, auth_method
-            )
-
-    async def _load_metadata(self) -> OpenIDProviderMetadata:
-        if self._config.issuer_metadata is not None:
-            return OpenIDProviderMetadata(**self._config.issuer_metadata)
-        url = get_well_known_url(self._config.issuer, external=True)
-        response = await self._http_client.get_json(url)
-        metadata = OpenIDProviderMetadata(**response)
-        # metadata.validate_introspection_endpoint()
-        return metadata
-
-    async def _introspect_token(self, token: str) -> IntrospectionToken:
-        """
-        Send a token to the introspection endpoint and returns the introspection response
-
-        Parameters:
-            token: The token to introspect
-
-        Raises:
-            HttpResponseException: If the introspection endpoint returns a non-2xx response
-            ValueError: If the introspection endpoint returns an invalid JSON response
-            JSONDecodeError: If the introspection endpoint returns a non-JSON response
-            Exception: If the HTTP request fails
-
-        Returns:
-            The introspection response
-        """
-        metadata = await self._issuer_metadata.get()
-        introspection_endpoint = metadata.get("introspection_endpoint")
-        raw_headers: Dict[str, str] = {
-            "Content-Type": "application/x-www-form-urlencoded",
-            "User-Agent": str(self._http_client.user_agent, "utf-8"),
-            "Accept": "application/json",
-        }
-
-        args = {"token": token, "token_type_hint": "access_token"}
-        body = urlencode(args, True)
-
-        # Fill the body/headers with credentials
-        uri, raw_headers, body = self._client_auth.prepare(
-            method="POST", uri=introspection_endpoint, headers=raw_headers, body=body
-        )
-        headers = Headers({k: [v] for (k, v) in raw_headers.items()})
-
-        # Do the actual request
-        # We're not using the SimpleHttpClient util methods as we don't want to
-        # check the HTTP status code, and we do the body encoding ourselves.
-        response = await self._http_client.request(
-            method="POST",
-            uri=uri,
-            data=body.encode("utf-8"),
-            headers=headers,
-        )
-
-        resp_body = await make_deferred_yieldable(readBody(response))
-
-        if response.code < 200 or response.code >= 300:
-            raise HttpResponseException(
-                response.code,
-                response.phrase.decode("ascii", errors="replace"),
-                resp_body,
-            )
-
-        resp = json_decoder.decode(resp_body.decode("utf-8"))
-
-        if not isinstance(resp, dict):
-            raise ValueError(
-                "The introspection endpoint returned an invalid JSON response."
-            )
-
-        return IntrospectionToken(**resp)
-
-    async def is_server_admin(self, requester: Requester) -> bool:
-        return "urn:synapse:admin:*" in requester.scope
-
-    async def get_user_by_req(
-        self,
-        request: SynapseRequest,
-        allow_guest: bool = False,
-        allow_expired: bool = False,
-    ) -> Requester:
-        access_token = self.get_access_token_from_request(request)
-
-        requester = await self.get_appservice_user(request, access_token)
-        if not requester:
-            # TODO: we probably want to assert the allow_guest inside this call
-            # so that we don't provision the user if they don't have enough permission:
-            requester = await self.get_user_by_access_token(access_token, allow_expired)
-
-        if not allow_guest and requester.is_guest:
-            raise OAuthInsufficientScopeError([SCOPE_MATRIX_API])
-
-        request.requester = requester
-
-        return requester
-
-    async def get_user_by_access_token(
-        self,
-        token: str,
-        allow_expired: bool = False,
-    ) -> Requester:
-        if self._admin_token is not None and token == self._admin_token:
-            # XXX: This is a temporary solution so that the admin API can be called by
-            # the OIDC provider. This will be removed once we have OIDC client
-            # credentials grant support in matrix-authentication-service.
-            logging.info("Admin toked used")
-            # XXX: that user doesn't exist and won't be provisioned.
-            # This is mostly fine for admin calls, but we should also think about doing
-            # requesters without a user_id.
-            admin_user = UserID("__oidc_admin", self._hostname)
-            return create_requester(
-                user_id=admin_user,
-                scope=["urn:synapse:admin:*"],
-            )
-
-        try:
-            introspection_result = await self._introspect_token(token)
-        except Exception:
-            logger.exception("Failed to introspect token")
-            raise SynapseError(503, "Unable to introspect the access token")
-
-        logger.info(f"Introspection result: {introspection_result!r}")
-
-        # TODO: introspection verification should be more extensive, especially:
-        #   - verify the audience
-        if not introspection_result.get("active"):
-            raise InvalidClientTokenError("Token is not active")
-
-        # Let's look at the scope
-        scope: List[str] = scope_to_list(introspection_result.get("scope", ""))
-
-        # Determine type of user based on presence of particular scopes
-        has_user_scope = SCOPE_MATRIX_API in scope
-        has_guest_scope = SCOPE_MATRIX_GUEST in scope
-
-        if not has_user_scope and not has_guest_scope:
-            raise InvalidClientTokenError("No scope in token granting user rights")
-
-        # Match via the sub claim
-        sub: Optional[str] = introspection_result.get("sub")
-        if sub is None:
-            raise InvalidClientTokenError(
-                "Invalid sub claim in the introspection result"
-            )
-
-        user_id_str = await self.store.get_user_by_external_id(
-            MSC3861DelegatedAuth.EXTERNAL_ID_PROVIDER, sub
-        )
-        if user_id_str is None:
-            # If we could not find a user via the external_id, it either does not exist,
-            # or the external_id was never recorded
-
-            # TODO: claim mapping should be configurable
-            username: Optional[str] = introspection_result.get("username")
-            if username is None or not isinstance(username, str):
-                raise AuthError(
-                    500,
-                    "Invalid username claim in the introspection result",
-                )
-            user_id = UserID(username, self._hostname)
-
-            # First try to find a user from the username claim
-            user_info = await self.store.get_userinfo_by_id(user_id=user_id.to_string())
-            if user_info is None:
-                # If the user does not exist, we should create it on the fly
-                # TODO: we could use SCIM to provision users ahead of time and listen
-                # for SCIM SET events if those ever become standard:
-                # https://datatracker.ietf.org/doc/html/draft-hunt-scim-notify-00
-
-                # TODO: claim mapping should be configurable
-                # If present, use the name claim as the displayname
-                name: Optional[str] = introspection_result.get("name")
-
-                await self.store.register_user(
-                    user_id=user_id.to_string(), create_profile_with_displayname=name
-                )
-
-            # And record the sub as external_id
-            await self.store.record_user_external_id(
-                MSC3861DelegatedAuth.EXTERNAL_ID_PROVIDER, sub, user_id.to_string()
-            )
-        else:
-            user_id = UserID.from_string(user_id_str)
-
-        # Find device_ids in scope
-        # We only allow a single device_id in the scope, so we find them all in the
-        # scope list, and raise if there are more than one. The OIDC server should be
-        # the one enforcing valid scopes, so we raise a 500 if we find an invalid scope.
-        device_ids = [
-            tok[len(SCOPE_MATRIX_DEVICE_PREFIX) :]
-            for tok in scope
-            if tok.startswith(SCOPE_MATRIX_DEVICE_PREFIX)
-        ]
-
-        if len(device_ids) > 1:
-            raise AuthError(
-                500,
-                "Multiple device IDs in scope",
-            )
-
-        device_id = device_ids[0] if device_ids else None
-        if device_id is not None:
-            # Sanity check the device_id
-            if len(device_id) > 255 or len(device_id) < 1:
-                raise AuthError(
-                    500,
-                    "Invalid device ID in scope",
-                )
-
-            # Create the device on the fly if it does not exist
-            try:
-                await self.store.get_device(
-                    user_id=user_id.to_string(), device_id=device_id
-                )
-            except StoreError:
-                await self.store.store_device(
-                    user_id=user_id.to_string(),
-                    device_id=device_id,
-                    initial_device_display_name="OIDC-native client",
-                )
-
-        # TODO: there is a few things missing in the requester here, which still need
-        # to be figured out, like:
-        #   - impersonation, with the `authenticated_entity`, which is used for
-        #     rate-limiting, MAU limits, etc.
-        #   - shadow-banning, with the `shadow_banned` flag
-        #   - a proper solution for appservices, which still needs to be figured out in
-        #     the context of MSC3861
-        return create_requester(
-            user_id=user_id,
-            device_id=device_id,
-            scope=scope,
-            is_guest=(has_guest_scope and not has_user_scope),
-        )

synapse/api/constants.py

@@ -123,6 +123,10 @@ class EventTypes:
     SpaceChild: Final = "m.space.child"
     SpaceParent: Final = "m.space.parent"
 
+    MSC2716_INSERTION: Final = "org.matrix.msc2716.insertion"
+    MSC2716_BATCH: Final = "org.matrix.msc2716.batch"
+    MSC2716_MARKER: Final = "org.matrix.msc2716.marker"
+
     Reaction: Final = "m.reaction"
 
 
@@ -218,11 +222,21 @@ class EventContentFields:
     # Used in m.room.guest_access events.
     GUEST_ACCESS: Final = "guest_access"
 
+    # Used on normal messages to indicate they were historically imported after the fact
+    MSC2716_HISTORICAL: Final = "org.matrix.msc2716.historical"
+    # For "insertion" events to indicate what the next batch ID should be in
+    # order to connect to it
+    MSC2716_NEXT_BATCH_ID: Final = "next_batch_id"
+    # Used on "batch" events to indicate which insertion event it connects to
+    MSC2716_BATCH_ID: Final = "batch_id"
+    # For "marker" events
+    MSC2716_INSERTION_EVENT_REFERENCE: Final = "insertion_event_reference"
+
     # The authorising user for joining a restricted room.
     AUTHORISING_USER: Final = "join_authorised_via_users_server"
 
     # Use for mentioning users.
-    MENTIONS: Final = "m.mentions"
+    MSC3952_MENTIONS: Final = "org.matrix.msc3952.mentions"
 
     # an unspecced field added to to-device messages to identify them uniquely-ish
     TO_DEVICE_MSGID: Final = "org.matrix.msgid"

synapse/api/errors.py

@@ -119,20 +119,14 @@ class Codes(str, Enum):
 
 
 class CodeMessageException(RuntimeError):
-    """An exception with integer code, a message string attributes and optional headers.
+    """An exception with integer code and message string attributes.
 
     Attributes:
         code: HTTP error code
         msg: string describing the error
-        headers: optional response headers to send
     """
 
-    def __init__(
-        self,
-        code: Union[int, HTTPStatus],
-        msg: str,
-        headers: Optional[Dict[str, str]] = None,
-    ):
+    def __init__(self, code: Union[int, HTTPStatus], msg: str):
         super().__init__("%d: %s" % (code, msg))
 
         # Some calls to this method pass instances of http.HTTPStatus for `code`.
@@ -143,7 +137,6 @@ class CodeMessageException(RuntimeError):
         # To eliminate this behaviour, we convert them to their integer equivalents here.
         self.code = int(code)
         self.msg = msg
-        self.headers = headers
 
 
 class RedirectException(CodeMessageException):
@@ -189,7 +182,6 @@ class SynapseError(CodeMessageException):
         msg: str,
         errcode: str = Codes.UNKNOWN,
         additional_fields: Optional[Dict] = None,
-        headers: Optional[Dict[str, str]] = None,
     ):
         """Constructs a synapse error.
 
@@ -198,7 +190,7 @@ class SynapseError(CodeMessageException):
             msg: The human-readable error message.
             errcode: The matrix error code e.g 'M_FORBIDDEN'
         """
-        super().__init__(code, msg, headers)
+        super().__init__(code, msg)
         self.errcode = errcode
         if additional_fields is None:
             self._additional_fields: Dict = {}
@@ -343,20 +335,6 @@ class AuthError(SynapseError):
         super().__init__(code, msg, errcode, additional_fields)
 
 
-class OAuthInsufficientScopeError(SynapseError):
-    """An error raised when the caller does not have sufficient scope to perform the requested action"""
-
-    def __init__(
-        self,
-        required_scopes: List[str],
-    ):
-        headers = {
-            "WWW-Authenticate": 'Bearer error="insufficient_scope", scope="%s"'
-            % (" ".join(required_scopes))
-        }
-        super().__init__(401, "Insufficient scope", Codes.FORBIDDEN, None, headers)
-
-
 class UnstableSpecAuthError(AuthError):
     """An error raised when a new error code is being proposed to replace a previous one.
     This error will return a "org.matrix.unstable.errcode" property with the new error code,

synapse/api/filtering.py

@@ -128,7 +128,20 @@ USER_FILTER_SCHEMA = {
         "account_data": {"$ref": "#/definitions/filter"},
         "room": {"$ref": "#/definitions/room_filter"},
         "event_format": {"type": "string", "enum": ["client", "federation"]},
-        "event_fields": {"type": "array", "items": {"type": "string"}},
+        "event_fields": {
+            "type": "array",
+            "items": {
+                "type": "string",
+                # Don't allow '\\' in event field filters. This makes matching
+                # events a lot easier as we can then use a negative lookbehind
+                # assertion to split '\.' If we allowed \\ then it would
+                # incorrectly split '\\.' See synapse.events.utils.serialize_event
+                #
+                # Note that because this is a regular expression, we have to escape
+                # each backslash in the pattern.
+                "pattern": r"^((?!\\\\).)*$",
+            },
+        },
     },
     "additionalProperties": True,  # Allow new fields for forward compatibility
 }
@@ -152,9 +165,9 @@ class Filtering:
         self.DEFAULT_FILTER_COLLECTION = FilterCollection(hs, {})
 
     async def get_user_filter(
-        self, user_id: UserID, filter_id: Union[int, str]
+        self, user_localpart: str, filter_id: Union[int, str]
     ) -> "FilterCollection":
-        result = await self.store.get_user_filter(user_id, filter_id)
+        result = await self.store.get_user_filter(user_localpart, filter_id)
         return FilterCollection(self._hs, result)
 
     def add_user_filter(self, user_id: UserID, user_filter: JsonDict) -> Awaitable[int]:

synapse/api/room_versions.py

@@ -91,6 +91,11 @@ class RoomVersion:
     # MSC2403: Allows join_rules to be set to 'knock', changes auth rules to allow sending
     # m.room.membership event with membership 'knock'.
     msc2403_knocking: bool
+    # MSC2716: Adds m.room.power_levels -> content.historical field to control
+    # whether "insertion", "chunk", "marker" events can be sent
+    msc2716_historical: bool
+    # MSC2716: Adds support for redacting "insertion", "chunk", and "marker" events
+    msc2716_redactions: bool
     # MSC3389: Protect relation information from redaction.
     msc3389_relation_redactions: bool
     # MSC3787: Adds support for a `knock_restricted` join rule, mixing concepts of
@@ -125,6 +130,8 @@ class RoomVersions:
         msc3083_join_rules=False,
         msc3375_redaction_rules=False,
         msc2403_knocking=False,
+        msc2716_historical=False,
+        msc2716_redactions=False,
         msc3389_relation_redactions=False,
         msc3787_knock_restricted_join_rule=False,
         msc3667_int_only_power_levels=False,
@@ -146,6 +153,8 @@ class RoomVersions:
         msc3083_join_rules=False,
         msc3375_redaction_rules=False,
         msc2403_knocking=False,
+        msc2716_historical=False,
+        msc2716_redactions=False,
         msc3389_relation_redactions=False,
         msc3787_knock_restricted_join_rule=False,
         msc3667_int_only_power_levels=False,
@@ -167,6 +176,8 @@ class RoomVersions:
         msc3083_join_rules=False,
         msc3375_redaction_rules=False,
         msc2403_knocking=False,
+        msc2716_historical=False,
+        msc2716_redactions=False,
         msc3389_relation_redactions=False,
         msc3787_knock_restricted_join_rule=False,
         msc3667_int_only_power_levels=False,
@@ -188,6 +199,8 @@ class RoomVersions:
         msc3083_join_rules=False,
         msc3375_redaction_rules=False,
         msc2403_knocking=False,
+        msc2716_historical=False,
+        msc2716_redactions=False,
         msc3389_relation_redactions=False,
         msc3787_knock_restricted_join_rule=False,
         msc3667_int_only_power_levels=False,
@@ -209,6 +222,8 @@ class RoomVersions:
         msc3083_join_rules=False,
         msc3375_redaction_rules=False,
         msc2403_knocking=False,
+        msc2716_historical=False,
+        msc2716_redactions=False,
         msc3389_relation_redactions=False,
         msc3787_knock_restricted_join_rule=False,
         msc3667_int_only_power_levels=False,
@@ -230,6 +245,8 @@ class RoomVersions:
         msc3083_join_rules=False,
         msc3375_redaction_rules=False,
         msc2403_knocking=False,
+        msc2716_historical=False,
+        msc2716_redactions=False,
         msc3389_relation_redactions=False,
         msc3787_knock_restricted_join_rule=False,
         msc3667_int_only_power_levels=False,
@@ -251,6 +268,8 @@ class RoomVersions:
         msc3083_join_rules=False,
         msc3375_redaction_rules=False,
         msc2403_knocking=False,
+        msc2716_historical=False,
+        msc2716_redactions=False,
         msc3389_relation_redactions=False,
         msc3787_knock_restricted_join_rule=False,
         msc3667_int_only_power_levels=False,
@@ -272,6 +291,8 @@ class RoomVersions:
         msc3083_join_rules=False,
         msc3375_redaction_rules=False,
         msc2403_knocking=True,
+        msc2716_historical=False,
+        msc2716_redactions=False,
         msc3389_relation_redactions=False,
         msc3787_knock_restricted_join_rule=False,
         msc3667_int_only_power_levels=False,
@@ -293,6 +314,8 @@ class RoomVersions:
         msc3083_join_rules=True,
         msc3375_redaction_rules=False,
         msc2403_knocking=True,
+        msc2716_historical=False,
+        msc2716_redactions=False,
         msc3389_relation_redactions=False,
         msc3787_knock_restricted_join_rule=False,
         msc3667_int_only_power_levels=False,
@@ -314,6 +337,8 @@ class RoomVersions:
         msc3083_join_rules=True,
         msc3375_redaction_rules=True,
         msc2403_knocking=True,
+        msc2716_historical=False,
+        msc2716_redactions=False,
         msc3389_relation_redactions=False,
         msc3787_knock_restricted_join_rule=False,
         msc3667_int_only_power_levels=False,
@@ -335,6 +360,8 @@ class RoomVersions:
         msc3083_join_rules=True,
         msc3375_redaction_rules=True,
         msc2403_knocking=True,
+        msc2716_historical=False,
+        msc2716_redactions=False,
         msc3389_relation_redactions=False,
         msc3787_knock_restricted_join_rule=True,
         msc3667_int_only_power_levels=False,
@@ -356,6 +383,8 @@ class RoomVersions:
         msc3083_join_rules=True,
         msc3375_redaction_rules=True,
         msc2403_knocking=True,
+        msc2716_historical=False,
+        msc2716_redactions=False,
         msc3389_relation_redactions=False,
         msc3787_knock_restricted_join_rule=False,
         msc3667_int_only_power_levels=False,
@@ -377,6 +406,8 @@ class RoomVersions:
         msc3083_join_rules=True,
         msc3375_redaction_rules=True,
         msc2403_knocking=True,
+        msc2716_historical=False,
+        msc2716_redactions=False,
         msc3389_relation_redactions=False,
         msc3787_knock_restricted_join_rule=True,
         msc3667_int_only_power_levels=True,
@@ -384,6 +415,29 @@ class RoomVersions:
         msc3931_push_features=(),
         msc3989_redaction_rules=False,
     )
+    MSC2716v4 = RoomVersion(
+        "org.matrix.msc2716v4",
+        RoomDisposition.UNSTABLE,
+        EventFormatVersions.ROOM_V4_PLUS,
+        StateResolutionVersions.V2,
+        enforce_key_validity=True,
+        special_case_aliases_auth=False,
+        strict_canonicaljson=True,
+        limit_notifications_power_levels=True,
+        msc2175_implicit_room_creator=False,
+        msc2176_redaction_rules=False,
+        msc3083_join_rules=False,
+        msc3375_redaction_rules=False,
+        msc2403_knocking=True,
+        msc2716_historical=True,
+        msc2716_redactions=True,
+        msc3389_relation_redactions=False,
+        msc3787_knock_restricted_join_rule=False,
+        msc3667_int_only_power_levels=False,
+        msc3821_redaction_rules=False,
+        msc3931_push_features=(),
+        msc3989_redaction_rules=False,
+    )
     MSC1767v10 = RoomVersion(
         # MSC1767 (Extensible Events) based on room version "10"
         "org.matrix.msc1767.10",
@@ -399,6 +453,8 @@ class RoomVersions:
         msc3083_join_rules=True,
         msc3375_redaction_rules=True,
         msc2403_knocking=True,
+        msc2716_historical=False,
+        msc2716_redactions=False,
         msc3389_relation_redactions=False,
         msc3787_knock_restricted_join_rule=True,
         msc3667_int_only_power_levels=True,
@@ -420,6 +476,8 @@ class RoomVersions:
         msc3083_join_rules=True,
         msc3375_redaction_rules=True,
         msc2403_knocking=True,
+        msc2716_historical=False,
+        msc2716_redactions=False,
         msc3389_relation_redactions=False,
         msc3787_knock_restricted_join_rule=True,
         msc3667_int_only_power_levels=True,
@@ -427,28 +485,6 @@ class RoomVersions:
         msc3931_push_features=(),
         msc3989_redaction_rules=True,
     )
-    MSC3820opt2 = RoomVersion(
-        # Based upon v10
-        "org.matrix.msc3820.opt2",
-        RoomDisposition.UNSTABLE,
-        EventFormatVersions.ROOM_V4_PLUS,
-        StateResolutionVersions.V2,
-        enforce_key_validity=True,
-        special_case_aliases_auth=False,
-        strict_canonicaljson=True,
-        limit_notifications_power_levels=True,
-        msc2175_implicit_room_creator=True,  # Used by MSC3820
-        msc2176_redaction_rules=True,  # Used by MSC3820
-        msc3083_join_rules=True,
-        msc3375_redaction_rules=True,
-        msc2403_knocking=True,
-        msc3389_relation_redactions=False,
-        msc3787_knock_restricted_join_rule=True,
-        msc3667_int_only_power_levels=True,
-        msc3821_redaction_rules=True,  # Used by MSC3820
-        msc3931_push_features=(),
-        msc3989_redaction_rules=True,  # Used by MSC3820
-    )
 
 
 KNOWN_ROOM_VERSIONS: Dict[str, RoomVersion] = {
@@ -466,8 +502,8 @@ KNOWN_ROOM_VERSIONS: Dict[str, RoomVersion] = {
         RoomVersions.V9,
         RoomVersions.MSC3787,
         RoomVersions.V10,
+        RoomVersions.MSC2716v4,
         RoomVersions.MSC3989,
-        RoomVersions.MSC3820opt2,
     )
 }
 

synapse/app/generic_worker.py

@@ -83,6 +83,7 @@ from synapse.storage.databases.main.receipts import ReceiptsWorkerStore
 from synapse.storage.databases.main.registration import RegistrationWorkerStore
 from synapse.storage.databases.main.relations import RelationsWorkerStore
 from synapse.storage.databases.main.room import RoomWorkerStore
+from synapse.storage.databases.main.room_batch import RoomBatchStore
 from synapse.storage.databases.main.roommember import RoomMemberWorkerStore
 from synapse.storage.databases.main.search import SearchStore
 from synapse.storage.databases.main.session import SessionStore
@@ -119,6 +120,7 @@ class GenericWorkerStore(
     # the races it creates aren't too bad.
     KeyStore,
     RoomWorkerStore,
+    RoomBatchStore,
     DirectoryWorkerStore,
     PushRulesWorkerStore,
     ApplicationServiceTransactionWorkerStore,

synapse/appservice/__init__.py

@@ -86,7 +86,6 @@ class ApplicationService:
             url.rstrip("/") if isinstance(url, str) else None
         )  # url must not end with a slash
         self.hs_token = hs_token
-        # The full Matrix ID for this application service's sender.
         self.sender = sender
         self.namespaces = self._check_namespaces(namespaces)
         self.id = id
@@ -213,7 +212,7 @@ class ApplicationService:
             True if the application service is interested in the user, False if not.
         """
         return (
-            # User is the appservice's configured sender_localpart user
+            # User is the appservice's sender_localpart user
             user_id == self.sender
             # User is in the appservice's user namespace
             or self.is_user_in_namespace(user_id)

synapse/config/auth.py

@@ -29,14 +29,7 @@ class AuthConfig(Config):
         if password_config is None:
             password_config = {}
 
-        # The default value of password_config.enabled is True, unless msc3861 is enabled.
-        msc3861_enabled = (
-            config.get("experimental_features", {})
-            .get("msc3861", {})
-            .get("enabled", False)
-        )
-        passwords_enabled = password_config.get("enabled", not msc3861_enabled)
-
+        passwords_enabled = password_config.get("enabled", True)
         # 'only_for_reauth' allows users who have previously set a password to use it,
         # even though passwords would otherwise be disabled.
         passwords_for_reauth_only = passwords_enabled == "only_for_reauth"
@@ -60,13 +53,3 @@ class AuthConfig(Config):
         self.ui_auth_session_timeout = self.parse_duration(
             ui_auth.get("session_timeout", 0)
         )
-
-        # Logging in with an existing session.
-        login_via_existing = config.get("login_via_existing_session", {})
-        self.login_via_existing_enabled = login_via_existing.get("enabled", False)
-        self.login_via_existing_require_ui_auth = login_via_existing.get(
-            "require_ui_auth", True
-        )
-        self.login_via_existing_token_timeout = self.parse_duration(
-            login_via_existing.get("token_timeout", "5m")
-        )

synapse/config/experimental.py

@@ -12,216 +12,15 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import enum
-from typing import TYPE_CHECKING, Any, Optional
+from typing import Any, Optional
 
 import attr
-import attr.validators
 
 from synapse.api.room_versions import KNOWN_ROOM_VERSIONS, RoomVersions
 from synapse.config import ConfigError
-from synapse.config._base import Config, RootConfig
+from synapse.config._base import Config
 from synapse.types import JsonDict
 
-# Determine whether authlib is installed.
-try:
-    import authlib  # noqa: F401
-
-    HAS_AUTHLIB = True
-except ImportError:
-    HAS_AUTHLIB = False
-
-if TYPE_CHECKING:
-    # Only import this if we're type checking, as it might not be installed at runtime.
-    from authlib.jose.rfc7517 import JsonWebKey
-
-
-class ClientAuthMethod(enum.Enum):
-    """List of supported client auth methods."""
-
-    CLIENT_SECRET_POST = "client_secret_post"
-    CLIENT_SECRET_BASIC = "client_secret_basic"
-    CLIENT_SECRET_JWT = "client_secret_jwt"
-    PRIVATE_KEY_JWT = "private_key_jwt"
-
-
-def _parse_jwks(jwks: Optional[JsonDict]) -> Optional["JsonWebKey"]:
-    """A helper function to parse a JWK dict into a JsonWebKey."""
-
-    if jwks is None:
-        return None
-
-    from authlib.jose.rfc7517 import JsonWebKey
-
-    return JsonWebKey.import_key(jwks)
-
-
-@attr.s(slots=True, frozen=True)
-class MSC3861:
-    """Configuration for MSC3861: Matrix architecture change to delegate authentication via OIDC"""
-
-    enabled: bool = attr.ib(default=False, validator=attr.validators.instance_of(bool))
-    """Whether to enable MSC3861 auth delegation."""
-
-    @enabled.validator
-    def _check_enabled(self, attribute: attr.Attribute, value: bool) -> None:
-        # Only allow enabling MSC3861 if authlib is installed
-        if value and not HAS_AUTHLIB:
-            raise ConfigError(
-                "MSC3861 is enabled but authlib is not installed. "
-                "Please install authlib to use MSC3861.",
-                ("experimental", "msc3861", "enabled"),
-            )
-
-    issuer: str = attr.ib(default="", validator=attr.validators.instance_of(str))
-    """The URL of the OIDC Provider."""
-
-    issuer_metadata: Optional[JsonDict] = attr.ib(default=None)
-    """The issuer metadata to use, otherwise discovered from /.well-known/openid-configuration as per MSC2965."""
-
-    client_id: str = attr.ib(
-        default="",
-        validator=attr.validators.instance_of(str),
-    )
-    """The client ID to use when calling the introspection endpoint."""
-
-    client_auth_method: ClientAuthMethod = attr.ib(
-        default=ClientAuthMethod.CLIENT_SECRET_POST, converter=ClientAuthMethod
-    )
-    """The auth method used when calling the introspection endpoint."""
-
-    client_secret: Optional[str] = attr.ib(
-        default=None,
-        validator=attr.validators.optional(attr.validators.instance_of(str)),
-    )
-    """
-    The client secret to use when calling the introspection endpoint,
-    when using any of the client_secret_* client auth methods.
-    """
-
-    jwk: Optional["JsonWebKey"] = attr.ib(default=None, converter=_parse_jwks)
-    """
-    The JWKS to use when calling the introspection endpoint,
-    when using the private_key_jwt client auth method.
-    """
-
-    @client_auth_method.validator
-    def _check_client_auth_method(
-        self, attribute: attr.Attribute, value: ClientAuthMethod
-    ) -> None:
-        # Check that the right client credentials are provided for the client auth method.
-        if not self.enabled:
-            return
-
-        if value == ClientAuthMethod.PRIVATE_KEY_JWT and self.jwk is None:
-            raise ConfigError(
-                "A JWKS must be provided when using the private_key_jwt client auth method",
-                ("experimental", "msc3861", "client_auth_method"),
-            )
-
-        if (
-            value
-            in (
-                ClientAuthMethod.CLIENT_SECRET_POST,
-                ClientAuthMethod.CLIENT_SECRET_BASIC,
-                ClientAuthMethod.CLIENT_SECRET_JWT,
-            )
-            and self.client_secret is None
-        ):
-            raise ConfigError(
-                f"A client secret must be provided when using the {value} client auth method",
-                ("experimental", "msc3861", "client_auth_method"),
-            )
-
-    account_management_url: Optional[str] = attr.ib(
-        default=None,
-        validator=attr.validators.optional(attr.validators.instance_of(str)),
-    )
-    """The URL of the My Account page on the OIDC Provider as per MSC2965."""
-
-    admin_token: Optional[str] = attr.ib(
-        default=None,
-        validator=attr.validators.optional(attr.validators.instance_of(str)),
-    )
-    """
-    A token that should be considered as an admin token.
-    This is used by the OIDC provider, to make admin calls to Synapse.
-    """
-
-    def check_config_conflicts(self, root: RootConfig) -> None:
-        """Checks for any configuration conflicts with other parts of Synapse.
-
-        Raises:
-            ConfigError: If there are any configuration conflicts.
-        """
-
-        if not self.enabled:
-            return
-
-        if (
-            root.auth.password_enabled_for_reauth
-            or root.auth.password_enabled_for_login
-        ):
-            raise ConfigError(
-                "Password auth cannot be enabled when OAuth delegation is enabled",
-                ("password_config", "enabled"),
-            )
-
-        if root.registration.enable_registration:
-            raise ConfigError(
-                "Registration cannot be enabled when OAuth delegation is enabled",
-                ("enable_registration",),
-            )
-
-        if (
-            root.oidc.oidc_enabled
-            or root.saml2.saml2_enabled
-            or root.cas.cas_enabled
-            or root.jwt.jwt_enabled
-        ):
-            raise ConfigError("SSO cannot be enabled when OAuth delegation is enabled")
-
-        if bool(root.authproviders.password_providers):
-            raise ConfigError(
-                "Password auth providers cannot be enabled when OAuth delegation is enabled"
-            )
-
-        if root.captcha.enable_registration_captcha:
-            raise ConfigError(
-                "CAPTCHA cannot be enabled when OAuth delegation is enabled",
-                ("captcha", "enable_registration_captcha"),
-            )
-
-        if root.auth.login_via_existing_enabled:
-            raise ConfigError(
-                "Login via existing session cannot be enabled when OAuth delegation is enabled",
-                ("login_via_existing_session", "enabled"),
-            )
-
-        if root.registration.refresh_token_lifetime:
-            raise ConfigError(
-                "refresh_token_lifetime cannot be set when OAuth delegation is enabled",
-                ("refresh_token_lifetime",),
-            )
-
-        if root.registration.nonrefreshable_access_token_lifetime:
-            raise ConfigError(
-                "nonrefreshable_access_token_lifetime cannot be set when OAuth delegation is enabled",
-                ("nonrefreshable_access_token_lifetime",),
-            )
-
-        if root.registration.session_lifetime:
-            raise ConfigError(
-                "session_lifetime cannot be set when OAuth delegation is enabled",
-                ("session_lifetime",),
-            )
-
-        if not root.experimental.msc3970_enabled:
-            raise ConfigError(
-                "experimental_features.msc3970_enabled must be 'true' when OAuth delegation is enabled",
-                ("experimental_features", "msc3970_enabled"),
-            )
-
 
 @attr.s(auto_attribs=True, frozen=True, slots=True)
 class MSC3866Config:
@@ -247,6 +46,9 @@ class ExperimentalConfig(Config):
         # MSC3026 (busy presence state)
         self.msc3026_enabled: bool = experimental.get("msc3026_enabled", False)
 
+        # MSC2716 (importing historical messages)
+        self.msc2716_enabled: bool = experimental.get("msc2716_enabled", False)
+
         # MSC3244 (room version capabilities)
         self.msc3244_enabled: bool = experimental.get("msc3244_enabled", True)
 
@@ -316,6 +118,13 @@ class ExperimentalConfig(Config):
         # MSC3881: Remotely toggle push notifications for another client
         self.msc3881_enabled: bool = experimental.get("msc3881_enabled", False)
 
+        # MSC3882: Allow an existing session to sign in a new session
+        self.msc3882_enabled: bool = experimental.get("msc3882_enabled", False)
+        self.msc3882_ui_auth: bool = experimental.get("msc3882_ui_auth", True)
+        self.msc3882_token_timeout = self.parse_duration(
+            experimental.get("msc3882_token_timeout", "5m")
+        )
+
         # MSC3874: Filtering /messages with rel_types / not_rel_types.
         self.msc3874_enabled: bool = experimental.get("msc3874_enabled", False)
 
@@ -355,6 +164,11 @@ class ExperimentalConfig(Config):
         # MSC3391: Removing account data.
         self.msc3391_enabled = experimental.get("msc3391_enabled", False)
 
+        # MSC3952: Intentional mentions, this depends on MSC3966.
+        self.msc3952_intentional_mentions = experimental.get(
+            "msc3952_intentional_mentions", False
+        )
+
         # MSC3959: Do not generate notifications for edits.
         self.msc3958_supress_edit_notifs = experimental.get(
             "msc3958_supress_edit_notifs", False
@@ -368,19 +182,8 @@ class ExperimentalConfig(Config):
             "msc3981_recurse_relations", False
         )
 
-        # MSC3861: Matrix architecture change to delegate authentication via OIDC
-        try:
-            self.msc3861 = MSC3861(**experimental.get("msc3861", {}))
-        except ValueError as exc:
-            raise ConfigError(
-                "Invalid MSC3861 configuration", ("experimental", "msc3861")
-            ) from exc
-
         # MSC3970: Scope transaction IDs to devices
-        self.msc3970_enabled = experimental.get("msc3970_enabled", self.msc3861.enabled)
-
-        # Check that none of the other config options conflict with MSC3861 when enabled
-        self.msc3861.check_config_conflicts(self.root)
+        self.msc3970_enabled = experimental.get("msc3970_enabled", False)
 
         # MSC4009: E.164 Matrix IDs
         self.msc4009_e164_mxids = experimental.get("msc4009_e164_mxids", False)

synapse/config/logger.py

@@ -117,7 +117,9 @@ root:
     # Write logs to the `buffer` handler, which will buffer them together in memory,
     # then write them to a file.
     #
-    # Replace "buffer" with "console" to log to stderr instead.
+    # Replace "buffer" with "console" to log to stderr instead. (Note that you'll
+    # also need to update the configuration for the `twisted` logger above, in
+    # this case.)
     #
     handlers: [buffer]
 

synapse/config/redis.py

@@ -33,7 +33,6 @@ class RedisConfig(Config):
 
         self.redis_host = redis_config.get("host", "localhost")
         self.redis_port = redis_config.get("port", 6379)
-        self.redis_path = redis_config.get("path", None)
         self.redis_dbid = redis_config.get("dbid", None)
         self.redis_password = redis_config.get("password")
 

synapse/config/workers.py

@@ -222,7 +222,7 @@ class WorkerConfig(Config):
         # itself doesn't need this data as it would never have to talk to itself.
         instance_map: Dict[str, Any] = config.get("instance_map", {})
 
-        if self.instance_name is not MAIN_PROCESS_INSTANCE_NAME:
+        if instance_map and self.instance_name is not MAIN_PROCESS_INSTANCE_NAME:
             # The host used to connect to the main synapse
             main_host = config.get("worker_replication_host", None)
 

synapse/event_auth.py

@@ -339,6 +339,13 @@ def check_state_dependent_auth_rules(
     if event.type == EventTypes.Redaction:
         check_redaction(event.room_version, event, auth_dict)
 
+    if (
+        event.type == EventTypes.MSC2716_INSERTION
+        or event.type == EventTypes.MSC2716_BATCH
+        or event.type == EventTypes.MSC2716_MARKER
+    ):
+        check_historical(event.room_version, event, auth_dict)
+
     logger.debug("Allowing! %s", event)
 
 
@@ -358,6 +365,7 @@ LENIENT_EVENT_BYTE_LIMITS_ROOM_VERSIONS = {
     RoomVersions.V9,
     RoomVersions.MSC3787,
     RoomVersions.V10,
+    RoomVersions.MSC2716v4,
     RoomVersions.MSC1767v10,
 }
 
@@ -815,6 +823,38 @@ def check_redaction(
     raise AuthError(403, "You don't have permission to redact events")
 
 
+def check_historical(
+    room_version_obj: RoomVersion,
+    event: "EventBase",
+    auth_events: StateMap["EventBase"],
+) -> None:
+    """Check whether the event sender is allowed to send historical related
+    events like "insertion", "batch", and "marker".
+
+    Returns:
+        None
+
+    Raises:
+        AuthError if the event sender is not allowed to send historical related events
+        ("insertion", "batch", and "marker").
+    """
+    # Ignore the auth checks in room versions that do not support historical
+    # events
+    if not room_version_obj.msc2716_historical:
+        return
+
+    user_level = get_user_power_level(event.user_id, auth_events)
+
+    historical_level = get_named_level(auth_events, "historical", 100)
+
+    if user_level < historical_level:
+        raise UnstableSpecAuthError(
+            403,
+            'You don\'t have permission to send send historical related events ("insertion", "batch", and "marker")',
+            errcode=Codes.INSUFFICIENT_POWER,
+        )
+
+
 def _check_power_levels(
     room_version_obj: RoomVersion,
     event: "EventBase",