Speed up typechecking CI

.github/workflows/tests.yml

@@ -35,7 +35,7 @@ jobs:
     steps:
       - uses: actions/checkout@v3
       - name: Install Rust
-        uses: dtolnay/rust-toolchain@1.60.0
+        uses: dtolnay/rust-toolchain@1.58.1
       - uses: Swatinem/rust-cache@v2
       - uses: matrix-org/setup-python-poetry@v1
         with:
@@ -93,8 +93,11 @@ jobs:
         uses: actions/checkout@v3
 
       - name: Install Rust
-        uses: dtolnay/rust-toolchain@1.60.0
+        uses: dtolnay/rust-toolchain@1.58.1
       - uses: Swatinem/rust-cache@v2
+        with:
+          # We want to avoid rebuilding the top-level crate each time.
+          cache-all-crates: "true"
 
       - name: Setup Poetry
         uses: matrix-org/setup-python-poetry@v1
@@ -150,7 +153,7 @@ jobs:
         with:
           ref: ${{ github.event.pull_request.head.sha }}
       - name: Install Rust
-        uses: dtolnay/rust-toolchain@1.60.0
+        uses: dtolnay/rust-toolchain@1.58.1
       - uses: Swatinem/rust-cache@v2
       - uses: matrix-org/setup-python-poetry@v1
         with:
@@ -167,7 +170,7 @@ jobs:
       - uses: actions/checkout@v3
 
       - name: Install Rust
-        uses: dtolnay/rust-toolchain@1.60.0
+        uses: dtolnay/rust-toolchain@1.58.1
         with:
             components: clippy
       - uses: Swatinem/rust-cache@v2
@@ -268,7 +271,7 @@ jobs:
             postgres:${{ matrix.job.postgres-version }}
 
       - name: Install Rust
-        uses: dtolnay/rust-toolchain@1.60.0
+        uses: dtolnay/rust-toolchain@1.58.1
       - uses: Swatinem/rust-cache@v2
 
       - uses: matrix-org/setup-python-poetry@v1
@@ -308,7 +311,7 @@ jobs:
       - uses: actions/checkout@v3
 
       - name: Install Rust
-        uses: dtolnay/rust-toolchain@1.60.0
+        uses: dtolnay/rust-toolchain@1.58.1
       - uses: Swatinem/rust-cache@v2
 
       # There aren't wheels for some of the older deps, so we need to install
@@ -399,8 +402,8 @@ jobs:
       env:
         SYTEST_BRANCH: ${{ github.head_ref }}
         POSTGRES: ${{ matrix.job.postgres && 1}}
-        MULTI_POSTGRES: ${{ (matrix.job.postgres == 'multi-postgres') || '' }}
-        ASYNCIO_REACTOR: ${{ (matrix.job.reactor == 'asyncio') || '' }}
+        MULTI_POSTGRES: ${{ (matrix.job.postgres == 'multi-postgres') && 1}}
+        ASYNCIO_REACTOR: ${{ (matrix.job.reactor == 'asyncio') && 1 }}
         WORKERS: ${{ matrix.job.workers && 1 }}
         BLACKLIST: ${{ matrix.job.workers && 'synapse-blacklist-with-workers' }}
         TOP: ${{ github.workspace }}
@@ -416,7 +419,7 @@ jobs:
         run: cat sytest-blacklist .ci/worker-blacklist > synapse-blacklist-with-workers
 
       - name: Install Rust
-        uses: dtolnay/rust-toolchain@1.60.0
+        uses: dtolnay/rust-toolchain@1.58.1
       - uses: Swatinem/rust-cache@v2
 
       - name: Run SyTest
@@ -556,7 +559,7 @@ jobs:
           path: synapse
 
       - name: Install Rust
-        uses: dtolnay/rust-toolchain@1.60.0
+        uses: dtolnay/rust-toolchain@1.58.1
       - uses: Swatinem/rust-cache@v2
 
       - uses: actions/setup-go@v4
@@ -584,7 +587,7 @@ jobs:
       - uses: actions/checkout@v3
 
       - name: Install Rust
-        uses: dtolnay/rust-toolchain@1.60.0
+        uses: dtolnay/rust-toolchain@1.58.1
       - uses: Swatinem/rust-cache@v2
 
       - run: cargo test

Cargo.lock

@@ -4,9 +4,9 @@ version = 3
 
 [[package]]
 name = "aho-corasick"
-version = "1.0.2"
+version = "0.7.19"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "43f6cb1bf222025340178f382c426f13757b2960e89779dfcb319c32542a5a41"
+checksum = "b4f55bd91a0978cbfd91c457a164bab8b4001c833b7f323132c0a4e1922dd44e"
 dependencies = [
  "memchr",
 ]
@@ -132,9 +132,9 @@ dependencies = [
 
 [[package]]
 name = "log"
-version = "0.4.19"
+version = "0.4.18"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b06a4cde4c0f271a446782e3eff8de789548ce57dbc8eca9292c27f4a42004b4"
+checksum = "518ef76f2f87365916b142844c16d8fefd85039bc5699050210a7778ee1cd1de"
 
 [[package]]
 name = "memchr"
@@ -229,9 +229,9 @@ dependencies = [
 
 [[package]]
 name = "pyo3-log"
-version = "0.8.2"
+version = "0.8.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c94ff6535a6bae58d7d0b85e60d4c53f7f84d0d0aa35d6a28c3f3e70bfe51444"
+checksum = "f9c8b57fe71fb5dcf38970ebedc2b1531cf1c14b1b9b4c560a182a57e115575c"
 dependencies = [
  "arc-swap",
  "log",
@@ -291,9 +291,9 @@ dependencies = [
 
 [[package]]
 name = "regex"
-version = "1.8.4"
+version = "1.7.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d0ab3ca65655bb1e41f2a8c8cd662eb4fb035e67c3f78da1d61dffe89d07300f"
+checksum = "8b1f693b24f6ac912f4893ef08244d70b6067480d2f1a46e950c9691e6749d1d"
 dependencies = [
  "aho-corasick",
  "memchr",
@@ -302,9 +302,9 @@ dependencies = [
 
 [[package]]
 name = "regex-syntax"
-version = "0.7.2"
+version = "0.6.29"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "436b050e76ed2903236f032a59761c1eb99e1b0aead2c257922771dab1fc8c78"
+checksum = "f162c6dd7b008981e4d40210aca20b4bd0f9b60ca9271061b07f78537722f2e1"
 
 [[package]]
 name = "ryu"
@@ -320,18 +320,18 @@ checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd"
 
 [[package]]
 name = "serde"
-version = "1.0.164"
+version = "1.0.163"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9e8c8cf938e98f769bc164923b06dce91cea1751522f46f8466461af04c9027d"
+checksum = "2113ab51b87a539ae008b5c6c02dc020ffa39afd2d83cffcb3f4eb2722cebec2"
 dependencies = [
  "serde_derive",
 ]
 
 [[package]]
 name = "serde_derive"
-version = "1.0.164"
+version = "1.0.163"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d9735b638ccc51c28bf6914d90a2e9725b377144fc612c49a611fddd1b631d68"
+checksum = "8c805777e3930c8883389c602315a24224bcc738b63905ef87cd1420353ea93e"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -340,9 +340,9 @@ dependencies = [
 
 [[package]]
 name = "serde_json"
-version = "1.0.99"
+version = "1.0.96"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "46266871c240a00b8f503b877622fe33430b3c7d963bdc0f2adc511e54a1eae3"
+checksum = "057d394a50403bcac12672b2b18fb387ab6d289d957dab67dd201875391e52f1"
 dependencies = [
  "itoa",
  "ryu",

book.toml

@@ -34,14 +34,6 @@ additional-css = [
     "docs/website_files/table-of-contents.css",
     "docs/website_files/remove-nav-buttons.css",
     "docs/website_files/indent-section-headers.css",
-    "docs/website_files/version-picker.css",
 ]
-additional-js = [
-    "docs/website_files/table-of-contents.js",
-    "docs/website_files/version-picker.js",
-    "docs/website_files/version.js",
-]
-theme = "docs/website_files/theme"
-
-[preprocessor.schema_versions]
-command = "./scripts-dev/schema_versions.py"
+additional-js = ["docs/website_files/table-of-contents.js"]
+theme = "docs/website_files/theme"

changelog.d/12504.misc

@@ -0,0 +1 @@
+Allow for the configuration of max request retries and min/max retry delays in the matrix federation client.

changelog.d/14213.misc

@@ -0,0 +1 @@
+Log when events are (maybe unexpectedly) filtered out of responses in tests.

changelog.d/15388.feature

@@ -0,0 +1 @@
+Stable support for [MSC3882](https://github.com/matrix-org/matrix-spec-proposals/pull/3882) to allow an existing device/session to generate a login token for use on a new device/session.

changelog.d/15450.feature

@@ -0,0 +1 @@
+Support resolving a room's [canonical alias](https://spec.matrix.org/v1.7/client-server-api/#mroomcanonical_alias) via the module API.

changelog.d/15520.feature

@@ -0,0 +1 @@
+Enable support for [MSC3952](https://github.com/matrix-org/matrix-spec-proposals/pull/3952): intentional mentions.

changelog.d/15582.feature

@@ -0,0 +1 @@
+Experimental [MSC3861](https://github.com/matrix-org/matrix-spec-proposals/pull/3861) support: delegate auth to an OIDC provider.

changelog.d/15609.bugfix

@@ -0,0 +1 @@
+Correctly clear caches when we delete a room.

changelog.d/15649.misc

@@ -0,0 +1 @@
+Read from column `full_user_id` rather than `user_id` of tables `profiles` and `user_filters`.

changelog.d/15650.misc

@@ -0,0 +1 @@
+Add support for tracing functions which return `Awaitable`s.

changelog.d/15674.feature

@@ -0,0 +1 @@
+Add Syanpse version deploy annotations to Grafana dashboard which enables easy correlation between behavior changes witnessed in a graph to a certain Synapse version and nail down regressions.

changelog.d/15675.misc

@@ -0,0 +1 @@
+Cache requests for user's devices over federation.

changelog.d/15689.misc

@@ -0,0 +1 @@
+Add fully qualified docker image names to Dockerfiles.

changelog.d/15690.misc

@@ -0,0 +1 @@
+Remove some unused code.

changelog.d/15694.misc

@@ -0,0 +1 @@
+Improve type hints.

changelog.d/15695.bugfix

@@ -0,0 +1 @@
+Check permissions for enabling encryption earlier during room creation to avoid creating broken rooms.

changelog.d/15697.misc

@@ -0,0 +1 @@
+Improve type hints.

changelog.d/15705.feature

@@ -0,0 +1 @@
+Add a catch-all * to the supported relation types when redacting an event and its related events. This is an update to [MSC3912](https://github.com/matrix-org/matrix-spec-proposals/pull/3861) implementation.

changelog.d/15709.misc

@@ -0,0 +1 @@
+Update docstring and traces on `maybe_backfill()` functions.

changelog.d/15721.misc

@@ -0,0 +1 @@
+Add context for when/why to use the `long_retries` option when sending Federation requests.

changelog.d/15723.misc

@@ -0,0 +1 @@
+Removed some unused fields.

changelog.d/15725.misc

@@ -0,0 +1 @@
+Update federation error to more plainly explain we can only authorize our own membership events.

changelog.d/15726.misc

@@ -0,0 +1 @@
+Prevent the `latest_deps` and `twisted_trunk` daily GitHub Actions workflows from running on forks of the codebase.

changelog.d/15729.misc

@@ -0,0 +1 @@
+Improve performance of user directory search.

changelog.d/15731.misc

@@ -0,0 +1 @@
+Remove redundant table join with `room_memberships` when doing a `is_host_joined()`/`is_host_invited()` call (`membership` is already part of the `current_state_events`).

changelog.d/15732.doc

@@ -0,0 +1 @@
+Simplify query to find participating servers in a room.

changelog.d/15733.misc

@@ -0,0 +1 @@
+Remove superfluous `room_memberships` join from background update.

changelog.d/15740.feature

@@ -0,0 +1 @@
+Expose a metric reporting the database background update status.

changelog.d/15752.misc

@@ -0,0 +1 @@
+Speed up typechecking CI.

contrib/lnav/synapse-log-format.json

@@ -29,7 +29,7 @@
         "level": "error"
       },
       {
-        "line": "my-matrix-server-federation-sender-1 | 2023-01-25 20:56:20,995 - synapse.http.matrixfederationclient - 709 - WARNING - federation_transaction_transmission_loop-3 - {PUT-O-3} [example.com] Request failed: PUT matrix-federation://example.com/_matrix/federation/v1/send/1674680155797: HttpResponseException('403: Forbidden')",
+        "line": "my-matrix-server-federation-sender-1 | 2023-01-25 20:56:20,995 - synapse.http.matrixfederationclient - 709 - WARNING - federation_transaction_transmission_loop-3 - {PUT-O-3} [example.com] Request failed: PUT matrix://example.com/_matrix/federation/v1/send/1674680155797: HttpResponseException('403: Forbidden')",
         "level": "warning"
       },
       {

debian/changelog

@@ -1,33 +1,3 @@
-matrix-synapse-py3 (1.87.0) stable; urgency=medium
-
-  * New Synapse release 1.87.0.
-
- -- Synapse Packaging team <packages@matrix.org>  Tue, 04 Jul 2023 16:24:00 +0100
-
-matrix-synapse-py3 (1.87.0~rc1) stable; urgency=medium
-
-  * New synapse release 1.87.0rc1.
-
- -- Synapse Packaging team <packages@matrix.org>  Tue, 27 Jun 2023 15:27:04 +0000
-
-matrix-synapse-py3 (1.86.0) stable; urgency=medium
-
-  * New Synapse release 1.86.0.
-
- -- Synapse Packaging team <packages@matrix.org>  Tue, 20 Jun 2023 17:22:46 +0200
-
-matrix-synapse-py3 (1.86.0~rc2) stable; urgency=medium
-
-  * New Synapse release 1.86.0rc2.
-
- -- Synapse Packaging team <packages@matrix.org>  Wed, 14 Jun 2023 12:16:27 +0200
-
-matrix-synapse-py3 (1.86.0~rc1) stable; urgency=medium
-
-  * New Synapse release 1.86.0rc1.
-
- -- Synapse Packaging team <packages@matrix.org>  Tue, 13 Jun 2023 14:30:45 +0200
-
 matrix-synapse-py3 (1.85.2) stable; urgency=medium
 
   * New Synapse release 1.85.2.

docker/complement/conf/workers-shared-extra.yaml.j2

@@ -92,6 +92,8 @@ allow_device_name_lookup_over_federation: true
 ## Experimental Features ##
 
 experimental_features:
+  # Enable history backfilling support
+  msc2716_enabled: true
   # client-side support for partial state in /send_join responses
   faster_joins: true
   # Enable support for polls

docker/configure_workers_and_start.py

@@ -244,6 +244,7 @@ WORKERS_CONFIG: Dict[str, Dict[str, Any]] = {
             "^/_matrix/client/(api/v1|r0|v3|unstable)/join/",
             "^/_matrix/client/(api/v1|r0|v3|unstable)/knock/",
             "^/_matrix/client/(api/v1|r0|v3|unstable)/profile/",
+            "^/_matrix/client/(v1|unstable/org.matrix.msc2716)/rooms/.*/batch_send",
         ],
         "shared_extra_conf": {},
         "worker_extra_conf": "",

docs/admin_api/rooms.md

@@ -419,7 +419,7 @@ The following query parameters are available:
 
 * `from` (required) - The token to start returning events from. This token can be obtained from a prev_batch
   or next_batch token returned by the /sync endpoint, or from an end token returned by a previous request to this endpoint.
-* `to` - The token to stop returning events at.
+* `to` - The token to spot returning events at.
 * `limit` - The maximum number of events to return. Defaults to `10`.
 * `filter` - A JSON RoomEventFilter to filter returned events with.
 * `dir` - The direction to return events from. Either `f` for forwards or `b` for backwards. Setting

docs/development/synapse_architecture/faster_joins.md

@@ -6,7 +6,7 @@ This is a work-in-progress set of notes with two goals:
 
 See also [MSC3902](https://github.com/matrix-org/matrix-spec-proposals/pull/3902).
 
-The key idea is described by [MSC3706](https://github.com/matrix-org/matrix-spec-proposals/pull/3706). This allows servers to
+The key idea is described by [MSC706](https://github.com/matrix-org/matrix-spec-proposals/pull/3902). This allows servers to
 request a lightweight response to the federation `/send_join` endpoint.
 This is called a **faster join**, also known as a **partial join**. In these
 notes we'll usually use the word "partial" as it matches the database schema.

docs/modules/spam_checker_callbacks.md

@@ -348,42 +348,6 @@ callback returns `False`, Synapse falls through to the next one. The value of th
 callback that does not return `False` will be used. If this happens, Synapse will not call
 any of the subsequent implementations of this callback.
 
-
-### `check_login_for_spam`
-
-_First introduced in Synapse v1.87.0_
-
-```python
-async def check_login_for_spam(
-    user_id: str,
-    device_id: Optional[str],
-    initial_display_name: Optional[str],
-    request_info: Collection[Tuple[Optional[str], str]],
-    auth_provider_id: Optional[str] = None,
-) -> Union["synapse.module_api.NOT_SPAM", "synapse.module_api.errors.Codes"]
-```
-
-Called when a user logs in.
-
-The arguments passed to this callback are:
-
-* `user_id`: The user ID the user is logging in with
-* `device_id`: The device ID the user is re-logging into.
-* `initial_display_name`: The device display name, if any.
-* `request_info`: A collection of tuples, which first item is a user agent, and which
-  second item is an IP address. These user agents and IP addresses are the ones that were
-  used during the login process.
-* `auth_provider_id`: The identifier of the SSO authentication provider, if any.
-
-If multiple modules implement this callback, they will be considered in order. If a
-callback returns `synapse.module_api.NOT_SPAM`, Synapse falls through to the next one.
-The value of the first callback that does not return `synapse.module_api.NOT_SPAM` will
-be used. If this happens, Synapse will not call any of the subsequent implementations of
-this callback.
-
-*Note:* This will not be called when a user registers.
-
-
 ## Example
 
 The example below is a module that implements the spam checker callback

docs/upgrade.md

@@ -87,14 +87,6 @@ process, for example:
     wget https://packages.matrix.org/debian/pool/main/m/matrix-synapse-py3/matrix-synapse-py3_1.3.0+stretch1_amd64.deb
     dpkg -i matrix-synapse-py3_1.3.0+stretch1_amd64.deb
     ```
-# Upgrading to v1.86.0
-
-## Minimum supported Rust version
-
-The minimum supported Rust version has been increased from v1.58.1 to v1.60.0.
-Users building from source will need to ensure their `rustc` version is up to
-date.
-
 
 # Upgrading to v1.85.0
 

docs/usage/configuration/config_documentation.md

@@ -1206,18 +1206,18 @@ Short retry algorithm is used when something or someone will wait for the reques
 answer, while long retry is used for requests that happen in the background,
 like sending a federation transaction.
 
-* `client_timeout`: timeout for the federation requests. Default to 60s.
-* `max_short_retry_delay`: maximum delay to be used for the short retry algo. Default to 2s.
-* `max_long_retry_delay`: maximum delay to be used for the short retry algo. Default to 60s.
+* `client_timeout`: timeout for the federation requests in seconds. Default to 60s.
+* `max_short_retry_delay`: maximum delay to be used for the short retry algo in seconds. Default to 2s.
+* `max_long_retry_delay`: maximum delay to be used for the short retry algo in seconds. Default to 60s.
 * `max_short_retries`: maximum number of retries for the short retry algo. Default to 3 attempts.
 * `max_long_retries`: maximum number of retries for the long retry algo. Default to 10 attempts.
 
 Example configuration:
 ```yaml
 federation:
-  client_timeout: 180s
-  max_short_retry_delay: 7s
-  max_long_retry_delay: 100s
+  client_timeout: 180
+  max_short_retry_delay: 7
+  max_long_retry_delay: 100
   max_short_retries: 5
   max_long_retries: 20
 ```

docs/website_files/README.md

@@ -24,11 +24,6 @@ Finally, we also stylise the chapter titles in the left sidebar by indenting the
 slightly so that they are more visually distinguishable from the section headers
 (the bold titles). This is done through the `indent-section-headers.css` file.
 
-In addition to these modifications, we have added a version picker to the documentation.
-Users can switch between documentations for different versions of Synapse.
-This functionality was implemented through the `version-picker.js` and
-`version-picker.css` files.
-
 More information can be found in mdbook's official documentation for
 [injecting page JS/CSS](https://rust-lang.github.io/mdBook/format/config.html)
 and

docs/website_files/theme/index.hbs

@@ -131,18 +131,6 @@
                             <i class="fa fa-search"></i>
                         </button>
                         {{/if}}
-                        <div class="version-picker">
-                            <div class="dropdown">
-                                <div class="select">
-                                    <span></span>
-                                    <i class="fa fa-chevron-down"></i>
-                                </div>
-                                <input type="hidden" name="version">
-                                <ul class="dropdown-menu">
-                                    <!-- Versions will be added dynamically in version-picker.js -->
-                                </ul>
-                            </div>
-                        </div>      
                     </div>
 
                     <h1 class="menu-title">{{ book_title }}</h1>
@@ -321,4 +309,4 @@
         {{/if}}
 
     </body>
-</html>
+</html>

docs/website_files/version-picker.css

@@ -1,78 +0,0 @@
-.version-picker {
-    display: flex;
-    align-items: center;
-}
-
-.version-picker .dropdown {
-    width: 130px;
-    max-height: 29px;
-    margin-left: 10px;
-    display: inline-block;
-    border-radius: 4px;
-    border: 1px solid var(--theme-popup-border);
-    position: relative;
-    font-size: 13px;
-    color: var(--fg);
-    height: 100%;
-    text-align: left;
-}
-.version-picker .dropdown .select {
-    cursor: pointer;
-    display: block;
-    padding: 5px 2px 5px 15px;
-}
-.version-picker .dropdown .select > i {
-    font-size: 10px;
-    color: var(--fg);
-    cursor: pointer;
-    float: right;
-    line-height: 20px !important;
-}
-.version-picker .dropdown:hover {
-    border: 1px solid var(--theme-popup-border);
-}
-.version-picker .dropdown:active {
-    background-color: var(--theme-popup-bg);
-}
-.version-picker .dropdown.active:hover,
-.version-picker .dropdown.active {
-    border: 1px solid var(--theme-popup-border);
-    border-radius: 2px 2px 0 0;
-    background-color: var(--theme-popup-bg);
-}
-.version-picker .dropdown.active .select > i {
-    transform: rotate(-180deg);
-}
-.version-picker .dropdown .dropdown-menu {
-    position: absolute;
-    background-color: var(--theme-popup-bg);
-    width: 100%;
-    left: -1px;
-    right: 1px;
-    margin-top: 1px;
-    border: 1px solid var(--theme-popup-border);
-    border-radius: 0 0 4px 4px;
-    overflow: hidden;
-    display: none;
-    max-height: 300px;
-    overflow-y: auto;
-    z-index: 9;
-}
-.version-picker .dropdown .dropdown-menu li {
-    font-size: 12px;
-    padding: 6px 20px;
-    cursor: pointer;
-} 
-.version-picker .dropdown .dropdown-menu {
-    padding: 0;
-    list-style: none;
-}
-.version-picker .dropdown .dropdown-menu li:hover {
-    background-color: var(--theme-hover);
-}
-.version-picker .dropdown .dropdown-menu li.active::before {
-    display: inline-block;
-    content: "✓";
-    margin-inline-start: -14px;
-    width: 14px;
-}

docs/website_files/version-picker.js

@@ -1,127 +0,0 @@
-
-const dropdown = document.querySelector('.version-picker .dropdown');
-const dropdownMenu = dropdown.querySelector('.dropdown-menu');
-
-fetchVersions(dropdown, dropdownMenu).then(() => {
-    initializeVersionDropdown(dropdown, dropdownMenu);
-});
-
-/**
- * Initialize the dropdown functionality for version selection.
- * 
- * @param {Element} dropdown - The dropdown element.
- * @param {Element} dropdownMenu - The dropdown menu element.
- */
-function initializeVersionDropdown(dropdown, dropdownMenu) {
-    // Toggle the dropdown menu on click
-    dropdown.addEventListener('click', function () {
-        this.setAttribute('tabindex', 1);
-        this.classList.toggle('active');
-        dropdownMenu.style.display = (dropdownMenu.style.display === 'block') ? 'none' : 'block';
-    });
-  
-    // Remove the 'active' class and hide the dropdown menu on focusout
-    dropdown.addEventListener('focusout', function () {
-        this.classList.remove('active');
-        dropdownMenu.style.display = 'none';
-    });
-  
-    // Handle item selection within the dropdown menu
-    const dropdownMenuItems = dropdownMenu.querySelectorAll('li');    
-    dropdownMenuItems.forEach(function (item) {
-        item.addEventListener('click', function () {
-            dropdownMenuItems.forEach(function (item) {
-                item.classList.remove('active');
-            });
-            this.classList.add('active');
-            dropdown.querySelector('span').textContent = this.textContent;
-            dropdown.querySelector('input').value = this.getAttribute('id');
-
-            window.location.href = changeVersion(window.location.href, this.textContent);
-        });
-    });
-};
-
-/**
- * This function fetches the available versions from a GitHub repository
- * and inserts them into the version picker.
- * 
- * @param {Element} dropdown - The dropdown element.
- * @param {Element} dropdownMenu - The dropdown menu element.
- * @returns {Promise<Array<string>>} A promise that resolves with an array of available versions.
- */
-function fetchVersions(dropdown, dropdownMenu) {
-    return new Promise((resolve, reject) => {
-        window.addEventListener("load", () => {
-
-            fetch("https://api.github.com/repos/matrix-org/synapse/git/trees/gh-pages", {
-                cache: "force-cache",
-            }).then(res => 
-                res.json()
-            ).then(resObject => {
-                const excluded = ['dev-docs', 'v1.91.0', 'v1.80.0', 'v1.69.0'];
-                const tree = resObject.tree.filter(item => item.type === "tree" && !excluded.includes(item.path));
-                const versions = tree.map(item => item.path).sort(sortVersions);
-
-                // Create a list of <li> items for versions
-                versions.forEach((version) => {
-                    const li = document.createElement("li");
-                    li.textContent = version;
-                    li.id = version;
-    
-                    if (window.SYNAPSE_VERSION === version) {
-                        li.classList.add('active');
-                        dropdown.querySelector('span').textContent = version;
-                        dropdown.querySelector('input').value = version;
-                    }
-    
-                    dropdownMenu.appendChild(li);
-                });
-
-                resolve(versions);
-
-            }).catch(ex => {
-                console.error("Failed to fetch version data", ex);
-                reject(ex);
-            })
-        });
-    });
-}
-
-/**
- * Custom sorting function to sort an array of version strings.
- *
- * @param {string} a - The first version string to compare.
- * @param {string} b - The second version string to compare.
- * @returns {number} - A negative number if a should come before b, a positive number if b should come before a, or 0 if they are equal.
- */
-function sortVersions(a, b) {
-    // Put 'develop' and 'latest' at the top
-    if (a === 'develop' || a === 'latest') return -1;
-    if (b === 'develop' || b === 'latest') return 1;
-
-    const versionA = (a.match(/v\d+(\.\d+)+/) || [])[0];
-    const versionB = (b.match(/v\d+(\.\d+)+/) || [])[0];
-
-    return versionB.localeCompare(versionA);
-}
-
-/**
- * Change the version in a URL path.
- *
- * @param {string} url - The original URL to be modified.
- * @param {string} newVersion - The new version to replace the existing version in the URL.
- * @returns {string} The updated URL with the new version.
- */
-function changeVersion(url, newVersion) {
-    const parsedURL = new URL(url);
-    const pathSegments = parsedURL.pathname.split('/');
-  
-    // Modify the version
-    pathSegments[2] = newVersion;
-
-    // Reconstruct the URL
-    parsedURL.pathname = pathSegments.join('/');
-  
-    return parsedURL.href;
-}

docs/website_files/version.js

@@ -1 +0,0 @@
-window.SYNAPSE_VERSION = 'v1.87';

docs/workers.md

@@ -232,6 +232,7 @@ information.
     ^/_matrix/client/v1/rooms/.*/hierarchy$
     ^/_matrix/client/(v1|unstable)/rooms/.*/relations/
     ^/_matrix/client/v1/rooms/.*/threads$
+    ^/_matrix/client/unstable/org.matrix.msc2716/rooms/.*/batch_send$
     ^/_matrix/client/unstable/im.nheko.summary/rooms/.*/summary$
     ^/_matrix/client/(r0|v3|unstable)/account/3pid$
     ^/_matrix/client/(r0|v3|unstable)/account/whoami$

pyproject.toml

@@ -89,7 +89,7 @@ manifest-path = "rust/Cargo.toml"
 
 [tool.poetry]
 name = "matrix-synapse"
-version = "1.87.0"
+version = "1.85.2"
 description = "Homeserver for the Matrix decentralised comms protocol"
 authors = ["Matrix.org Team and Contributors <packages@matrix.org>"]
 license = "Apache-2.0"
@@ -207,8 +207,7 @@ packaging = ">=16.1"
 # which shipped in Python 3.8. This corresponds to version 1.4 of the backport.
 importlib_metadata = { version = ">=1.4", python = "<3.8" }
 # This is the most recent version of Pydantic with available on common distros.
-# We are currently incompatible with >=2.0.0: (https://github.com/matrix-org/synapse/issues/15858)
-pydantic = "^1.7.4"
+pydantic = ">=1.7.4"
 
 # This is for building the rust components during "poetry install", which
 # currently ignores the `build-system.requires` directive (c.f.
@@ -312,7 +311,7 @@ all = [
 # We pin black so that our tests don't start failing on new releases.
 isort = ">=5.10.1"
 black = ">=22.3.0"
-ruff = "0.0.275"
+ruff = "0.0.265"
 
 # Typechecking
 lxml-stubs = ">=0.4.0"

rust/Cargo.toml

@@ -7,7 +7,7 @@ name = "synapse"
 version = "0.1.0"
 
 edition = "2021"
-rust-version = "1.60.0"
+rust-version = "1.58.1"
 
 [lib]
 name = "synapse"

rust/src/push/base_rules.rs

@@ -142,7 +142,7 @@ pub const BASE_APPEND_OVERRIDE_RULES: &[PushRule] = &[
         default_enabled: true,
     },
     PushRule {
-        rule_id: Cow::Borrowed("global/override/.m.rule.is_user_mention"),
+        rule_id: Cow::Borrowed("global/override/.m.is_user_mention"),
         priority_class: 5,
         conditions: Cow::Borrowed(&[Condition::Known(
             KnownCondition::ExactEventPropertyContainsType(EventPropertyIsTypeCondition {
@@ -163,7 +163,7 @@ pub const BASE_APPEND_OVERRIDE_RULES: &[PushRule] = &[
         default_enabled: true,
     },
     PushRule {
-        rule_id: Cow::Borrowed("global/override/.m.rule.is_room_mention"),
+        rule_id: Cow::Borrowed("global/override/.m.is_room_mention"),
         priority_class: 5,
         conditions: Cow::Borrowed(&[
             Condition::Known(KnownCondition::EventPropertyIs(EventPropertyIsCondition {

scripts-dev/complement.sh

@@ -246,6 +246,10 @@ else
   else
     export PASS_SYNAPSE_COMPLEMENT_DATABASE=sqlite
   fi
+
+  # The tests for importing historical messages (MSC2716)
+  # only pass with monoliths, currently.
+  test_tags="$test_tags,msc2716"
 fi
 
 if [[ -n "$ASYNCIO_REACTOR" ]]; then

scripts-dev/federation_client.py

@@ -136,11 +136,11 @@ def request(
         authorization_headers.append(header)
         print("Authorization: %s" % header, file=sys.stderr)
 
-    dest = "matrix-federation://%s%s" % (destination, path)
+    dest = "matrix://%s%s" % (destination, path)
     print("Requesting %s" % dest, file=sys.stderr)
 
     s = requests.Session()
-    s.mount("matrix-federation://", MatrixConnectionAdapter())
+    s.mount("matrix://", MatrixConnectionAdapter())
 
     headers: Dict[str, str] = {
         "Authorization": authorization_headers[0],

synapse/_scripts/synapse_port_db.py

@@ -1369,9 +1369,6 @@ def main() -> None:
         sys.stderr.write("Database must use the 'psycopg2' connector.\n")
         sys.exit(3)
 
-    # Don't run the background tasks that get started by the data stores.
-    hs_config["run_background_tasks_on"] = "some_other_process"
-
     config = HomeServerConfig()
     config.parse_config_dict(hs_config, "", "")
 

synapse/api/constants.py

@@ -123,6 +123,10 @@ class EventTypes:
     SpaceChild: Final = "m.space.child"
     SpaceParent: Final = "m.space.parent"
 
+    MSC2716_INSERTION: Final = "org.matrix.msc2716.insertion"
+    MSC2716_BATCH: Final = "org.matrix.msc2716.batch"
+    MSC2716_MARKER: Final = "org.matrix.msc2716.marker"
+
     Reaction: Final = "m.reaction"
 
 
@@ -218,6 +222,16 @@ class EventContentFields:
     # Used in m.room.guest_access events.
     GUEST_ACCESS: Final = "guest_access"
 
+    # Used on normal messages to indicate they were historically imported after the fact
+    MSC2716_HISTORICAL: Final = "org.matrix.msc2716.historical"
+    # For "insertion" events to indicate what the next batch ID should be in
+    # order to connect to it
+    MSC2716_NEXT_BATCH_ID: Final = "next_batch_id"
+    # Used on "batch" events to indicate which insertion event it connects to
+    MSC2716_BATCH_ID: Final = "batch_id"
+    # For "marker" events
+    MSC2716_INSERTION_EVENT_REFERENCE: Final = "insertion_event_reference"
+
     # The authorising user for joining a restricted room.
     AUTHORISING_USER: Final = "join_authorised_via_users_server"
 

synapse/api/room_versions.py

@@ -91,6 +91,11 @@ class RoomVersion:
     # MSC2403: Allows join_rules to be set to 'knock', changes auth rules to allow sending
     # m.room.membership event with membership 'knock'.
     msc2403_knocking: bool
+    # MSC2716: Adds m.room.power_levels -> content.historical field to control
+    # whether "insertion", "chunk", "marker" events can be sent
+    msc2716_historical: bool
+    # MSC2716: Adds support for redacting "insertion", "chunk", and "marker" events
+    msc2716_redactions: bool
     # MSC3389: Protect relation information from redaction.
     msc3389_relation_redactions: bool
     # MSC3787: Adds support for a `knock_restricted` join rule, mixing concepts of
@@ -125,6 +130,8 @@ class RoomVersions:
         msc3083_join_rules=False,
         msc3375_redaction_rules=False,
         msc2403_knocking=False,
+        msc2716_historical=False,
+        msc2716_redactions=False,
         msc3389_relation_redactions=False,
         msc3787_knock_restricted_join_rule=False,
         msc3667_int_only_power_levels=False,
@@ -146,6 +153,8 @@ class RoomVersions:
         msc3083_join_rules=False,
         msc3375_redaction_rules=False,
         msc2403_knocking=False,
+        msc2716_historical=False,
+        msc2716_redactions=False,
         msc3389_relation_redactions=False,
         msc3787_knock_restricted_join_rule=False,
         msc3667_int_only_power_levels=False,
@@ -167,6 +176,8 @@ class RoomVersions:
         msc3083_join_rules=False,
         msc3375_redaction_rules=False,
         msc2403_knocking=False,
+        msc2716_historical=False,
+        msc2716_redactions=False,
         msc3389_relation_redactions=False,
         msc3787_knock_restricted_join_rule=False,
         msc3667_int_only_power_levels=False,
@@ -188,6 +199,8 @@ class RoomVersions:
         msc3083_join_rules=False,
         msc3375_redaction_rules=False,
         msc2403_knocking=False,
+        msc2716_historical=False,
+        msc2716_redactions=False,
         msc3389_relation_redactions=False,
         msc3787_knock_restricted_join_rule=False,
         msc3667_int_only_power_levels=False,
@@ -209,6 +222,8 @@ class RoomVersions:
         msc3083_join_rules=False,
         msc3375_redaction_rules=False,
         msc2403_knocking=False,
+        msc2716_historical=False,
+        msc2716_redactions=False,
         msc3389_relation_redactions=False,
         msc3787_knock_restricted_join_rule=False,
         msc3667_int_only_power_levels=False,
@@ -230,6 +245,8 @@ class RoomVersions:
         msc3083_join_rules=False,
         msc3375_redaction_rules=False,
         msc2403_knocking=False,
+        msc2716_historical=False,
+        msc2716_redactions=False,
         msc3389_relation_redactions=False,
         msc3787_knock_restricted_join_rule=False,
         msc3667_int_only_power_levels=False,
@@ -251,6 +268,8 @@ class RoomVersions:
         msc3083_join_rules=False,
         msc3375_redaction_rules=False,
         msc2403_knocking=False,
+        msc2716_historical=False,
+        msc2716_redactions=False,
         msc3389_relation_redactions=False,
         msc3787_knock_restricted_join_rule=False,
         msc3667_int_only_power_levels=False,
@@ -272,6 +291,8 @@ class RoomVersions:
         msc3083_join_rules=False,
         msc3375_redaction_rules=False,
         msc2403_knocking=True,
+        msc2716_historical=False,
+        msc2716_redactions=False,
         msc3389_relation_redactions=False,
         msc3787_knock_restricted_join_rule=False,
         msc3667_int_only_power_levels=False,
@@ -293,6 +314,8 @@ class RoomVersions:
         msc3083_join_rules=True,
         msc3375_redaction_rules=False,
         msc2403_knocking=True,
+        msc2716_historical=False,
+        msc2716_redactions=False,
         msc3389_relation_redactions=False,
         msc3787_knock_restricted_join_rule=False,
         msc3667_int_only_power_levels=False,
@@ -314,6 +337,8 @@ class RoomVersions:
         msc3083_join_rules=True,
         msc3375_redaction_rules=True,
         msc2403_knocking=True,
+        msc2716_historical=False,
+        msc2716_redactions=False,
         msc3389_relation_redactions=False,
         msc3787_knock_restricted_join_rule=False,
         msc3667_int_only_power_levels=False,
@@ -335,6 +360,8 @@ class RoomVersions:
         msc3083_join_rules=True,
         msc3375_redaction_rules=True,
         msc2403_knocking=True,
+        msc2716_historical=False,
+        msc2716_redactions=False,
         msc3389_relation_redactions=False,
         msc3787_knock_restricted_join_rule=True,
         msc3667_int_only_power_levels=False,
@@ -356,6 +383,8 @@ class RoomVersions:
         msc3083_join_rules=True,
         msc3375_redaction_rules=True,
         msc2403_knocking=True,
+        msc2716_historical=False,
+        msc2716_redactions=False,
         msc3389_relation_redactions=False,
         msc3787_knock_restricted_join_rule=False,
         msc3667_int_only_power_levels=False,
@@ -377,6 +406,8 @@ class RoomVersions:
         msc3083_join_rules=True,
         msc3375_redaction_rules=True,
         msc2403_knocking=True,
+        msc2716_historical=False,
+        msc2716_redactions=False,
         msc3389_relation_redactions=False,
         msc3787_knock_restricted_join_rule=True,
         msc3667_int_only_power_levels=True,
@@ -384,6 +415,29 @@ class RoomVersions:
         msc3931_push_features=(),
         msc3989_redaction_rules=False,
     )
+    MSC2716v4 = RoomVersion(
+        "org.matrix.msc2716v4",
+        RoomDisposition.UNSTABLE,
+        EventFormatVersions.ROOM_V4_PLUS,
+        StateResolutionVersions.V2,
+        enforce_key_validity=True,
+        special_case_aliases_auth=False,
+        strict_canonicaljson=True,
+        limit_notifications_power_levels=True,
+        msc2175_implicit_room_creator=False,
+        msc2176_redaction_rules=False,
+        msc3083_join_rules=False,
+        msc3375_redaction_rules=False,
+        msc2403_knocking=True,
+        msc2716_historical=True,
+        msc2716_redactions=True,
+        msc3389_relation_redactions=False,
+        msc3787_knock_restricted_join_rule=False,
+        msc3667_int_only_power_levels=False,
+        msc3821_redaction_rules=False,
+        msc3931_push_features=(),
+        msc3989_redaction_rules=False,
+    )
     MSC1767v10 = RoomVersion(
         # MSC1767 (Extensible Events) based on room version "10"
         "org.matrix.msc1767.10",
@@ -399,6 +453,8 @@ class RoomVersions:
         msc3083_join_rules=True,
         msc3375_redaction_rules=True,
         msc2403_knocking=True,
+        msc2716_historical=False,
+        msc2716_redactions=False,
         msc3389_relation_redactions=False,
         msc3787_knock_restricted_join_rule=True,
         msc3667_int_only_power_levels=True,
@@ -420,6 +476,8 @@ class RoomVersions:
         msc3083_join_rules=True,
         msc3375_redaction_rules=True,
         msc2403_knocking=True,
+        msc2716_historical=False,
+        msc2716_redactions=False,
         msc3389_relation_redactions=False,
         msc3787_knock_restricted_join_rule=True,
         msc3667_int_only_power_levels=True,
@@ -442,6 +500,8 @@ class RoomVersions:
         msc3083_join_rules=True,
         msc3375_redaction_rules=True,
         msc2403_knocking=True,
+        msc2716_historical=False,
+        msc2716_redactions=False,
         msc3389_relation_redactions=False,
         msc3787_knock_restricted_join_rule=True,
         msc3667_int_only_power_levels=True,
@@ -466,6 +526,7 @@ KNOWN_ROOM_VERSIONS: Dict[str, RoomVersion] = {
         RoomVersions.V9,
         RoomVersions.MSC3787,
         RoomVersions.V10,
+        RoomVersions.MSC2716v4,
         RoomVersions.MSC3989,
         RoomVersions.MSC3820opt2,
     )

synapse/app/generic_worker.py

@@ -83,6 +83,7 @@ from synapse.storage.databases.main.receipts import ReceiptsWorkerStore
 from synapse.storage.databases.main.registration import RegistrationWorkerStore
 from synapse.storage.databases.main.relations import RelationsWorkerStore
 from synapse.storage.databases.main.room import RoomWorkerStore
+from synapse.storage.databases.main.room_batch import RoomBatchStore
 from synapse.storage.databases.main.roommember import RoomMemberWorkerStore
 from synapse.storage.databases.main.search import SearchStore
 from synapse.storage.databases.main.session import SessionStore
@@ -119,6 +120,7 @@ class GenericWorkerStore(
     # the races it creates aren't too bad.
     KeyStore,
     RoomWorkerStore,
+    RoomBatchStore,
     DirectoryWorkerStore,
     PushRulesWorkerStore,
     ApplicationServiceTransactionWorkerStore,

synapse/config/experimental.py

@@ -247,6 +247,9 @@ class ExperimentalConfig(Config):
         # MSC3026 (busy presence state)
         self.msc3026_enabled: bool = experimental.get("msc3026_enabled", False)
 
+        # MSC2716 (importing historical messages)
+        self.msc2716_enabled: bool = experimental.get("msc2716_enabled", False)
+
         # MSC3244 (room version capabilities)
         self.msc3244_enabled: bool = experimental.get("msc3244_enabled", True)
 

synapse/config/federation.py

@@ -53,15 +53,9 @@ class FederationConfig(Config):
 
         # Allow for the configuration of timeout, max request retries
         # and min/max retry delays in the matrix federation client.
-        self.client_timeout_ms = Config.parse_duration(
-            federation_config.get("client_timeout", "60s")
-        )
-        self.max_long_retry_delay_ms = Config.parse_duration(
-            federation_config.get("max_long_retry_delay", "60s")
-        )
-        self.max_short_retry_delay_ms = Config.parse_duration(
-            federation_config.get("max_short_retry_delay", "2s")
-        )
+        self.client_timeout = federation_config.get("client_timeout", 60)
+        self.max_long_retry_delay = federation_config.get("max_long_retry_delay", 60)
+        self.max_short_retry_delay = federation_config.get("max_short_retry_delay", 2)
         self.max_long_retries = federation_config.get("max_long_retries", 10)
         self.max_short_retries = federation_config.get("max_short_retries", 3)
 

synapse/event_auth.py

@@ -339,6 +339,13 @@ def check_state_dependent_auth_rules(
     if event.type == EventTypes.Redaction:
         check_redaction(event.room_version, event, auth_dict)
 
+    if (
+        event.type == EventTypes.MSC2716_INSERTION
+        or event.type == EventTypes.MSC2716_BATCH
+        or event.type == EventTypes.MSC2716_MARKER
+    ):
+        check_historical(event.room_version, event, auth_dict)
+
     logger.debug("Allowing! %s", event)
 
 
@@ -358,6 +365,7 @@ LENIENT_EVENT_BYTE_LIMITS_ROOM_VERSIONS = {
     RoomVersions.V9,
     RoomVersions.MSC3787,
     RoomVersions.V10,
+    RoomVersions.MSC2716v4,
     RoomVersions.MSC1767v10,
 }
 
@@ -815,6 +823,38 @@ def check_redaction(
     raise AuthError(403, "You don't have permission to redact events")
 
 
+def check_historical(
+    room_version_obj: RoomVersion,
+    event: "EventBase",
+    auth_events: StateMap["EventBase"],
+) -> None:
+    """Check whether the event sender is allowed to send historical related
+    events like "insertion", "batch", and "marker".
+
+    Returns:
+        None
+
+    Raises:
+        AuthError if the event sender is not allowed to send historical related events
+        ("insertion", "batch", and "marker").
+    """
+    # Ignore the auth checks in room versions that do not support historical
+    # events
+    if not room_version_obj.msc2716_historical:
+        return
+
+    user_level = get_user_power_level(event.user_id, auth_events)
+
+    historical_level = get_named_level(auth_events, "historical", 100)
+
+    if user_level < historical_level:
+        raise UnstableSpecAuthError(
+            403,
+            'You don\'t have permission to send send historical related events ("insertion", "batch", and "marker")',
+            errcode=Codes.INSUFFICIENT_POWER,
+        )
+
+
 def _check_power_levels(
     room_version_obj: RoomVersion,
     event: "EventBase",

synapse/events/__init__.py

@@ -198,6 +198,7 @@ class _EventInternalMetadata:
     soft_failed: DictProperty[bool] = DictProperty("soft_failed")
     proactively_send: DictProperty[bool] = DictProperty("proactively_send")
     redacted: DictProperty[bool] = DictProperty("redacted")
+    historical: DictProperty[bool] = DictProperty("historical")
 
     txn_id: DictProperty[str] = DictProperty("txn_id")
     """The transaction ID, if it was set when the event was created."""
@@ -287,6 +288,14 @@ class _EventInternalMetadata:
         """
         return self._dict.get("redacted", False)
 
+    def is_historical(self) -> bool:
+        """Whether this is a historical message.
+        This is used by the batchsend historical message endpoint and
+        is needed to and mark the event as backfilled and skip some checks
+        like push notifications.
+        """
+        return self._dict.get("historical", False)
+
     def is_notifiable(self) -> bool:
         """Whether this event can trigger a push notification"""
         return not self.is_outlier() or self.is_out_of_band_membership()

synapse/events/snapshot.py

@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 from abc import ABC, abstractmethod
-from typing import TYPE_CHECKING, Dict, List, Optional, Tuple
+from typing import TYPE_CHECKING, List, Optional, Tuple
 
 import attr
 from immutabledict import immutabledict
@@ -107,32 +107,33 @@ class EventContext(UnpersistedEventContextBase):
         state_delta_due_to_event: If `state_group` and `state_group_before_event` are not None
             then this is the delta of the state between the two groups.
 
-        state_group_deltas: If not empty, this is a dict collecting a mapping of the state
-            difference between state groups.
+        prev_group: If it is known, ``state_group``'s prev_group. Note that this being
+            None does not necessarily mean that ``state_group`` does not have
+            a prev_group!
 
-            The keys are a tuple of two integers: the initial group and final state group.
-            The corresponding value is a state map representing the state delta between
-            these state groups.
+            If the event is a state event, this is normally the same as
+            ``state_group_before_event``.
 
-            The dictionary is expected to have at most two entries with state groups of:
+            If ``state_group`` is None (ie, the event is an outlier), ``prev_group``
+            will always also be ``None``.
 
-            1. The state group before the event and after the event.
-            2. The state group preceding the state group before the event and the
-               state group before the event.
+            Note that this *not* (necessarily) the state group associated with
+            ``_prev_state_ids``.
 
-            This information is collected and stored as part of an optimization for persisting
-            events.
+        delta_ids: If ``prev_group`` is not None, the state delta between ``prev_group``
+            and ``state_group``.
 
         partial_state: if True, we may be storing this event with a temporary,
             incomplete state.
     """
 
     _storage: "StorageControllers"
-    state_group_deltas: Dict[Tuple[int, int], StateMap[str]]
     rejected: Optional[str] = None
     _state_group: Optional[int] = None
     state_group_before_event: Optional[int] = None
     _state_delta_due_to_event: Optional[StateMap[str]] = None
+    prev_group: Optional[int] = None
+    delta_ids: Optional[StateMap[str]] = None
     app_service: Optional[ApplicationService] = None
 
     partial_state: bool = False
@@ -144,14 +145,16 @@ class EventContext(UnpersistedEventContextBase):
         state_group_before_event: Optional[int],
         state_delta_due_to_event: Optional[StateMap[str]],
         partial_state: bool,
-        state_group_deltas: Dict[Tuple[int, int], StateMap[str]],
+        prev_group: Optional[int] = None,
+        delta_ids: Optional[StateMap[str]] = None,
     ) -> "EventContext":
         return EventContext(
             storage=storage,
             state_group=state_group,
             state_group_before_event=state_group_before_event,
             state_delta_due_to_event=state_delta_due_to_event,
-            state_group_deltas=state_group_deltas,
+            prev_group=prev_group,
+            delta_ids=delta_ids,
             partial_state=partial_state,
         )
 
@@ -160,7 +163,7 @@ class EventContext(UnpersistedEventContextBase):
         storage: "StorageControllers",
     ) -> "EventContext":
         """Return an EventContext instance suitable for persisting an outlier event"""
-        return EventContext(storage=storage, state_group_deltas={})
+        return EventContext(storage=storage)
 
     async def persist(self, event: EventBase) -> "EventContext":
         return self
@@ -180,15 +183,13 @@ class EventContext(UnpersistedEventContextBase):
             "state_group": self._state_group,
             "state_group_before_event": self.state_group_before_event,
             "rejected": self.rejected,
-            "state_group_deltas": _encode_state_group_delta(self.state_group_deltas),
+            "prev_group": self.prev_group,
             "state_delta_due_to_event": _encode_state_dict(
                 self._state_delta_due_to_event
             ),
+            "delta_ids": _encode_state_dict(self.delta_ids),
             "app_service_id": self.app_service.id if self.app_service else None,
             "partial_state": self.partial_state,
-            # add dummy delta_ids and prev_group for backwards compatibility
-            "delta_ids": None,
-            "prev_group": None,
         }
 
     @staticmethod
@@ -203,24 +204,17 @@ class EventContext(UnpersistedEventContextBase):
         Returns:
             The event context.
         """
-        # workaround for backwards/forwards compatibility: if the input doesn't have a value
-        # for "state_group_deltas" just assign an empty dict
-        state_group_deltas = input.get("state_group_deltas", None)
-        if state_group_deltas:
-            state_group_deltas = _decode_state_group_delta(state_group_deltas)
-        else:
-            state_group_deltas = {}
-
         context = EventContext(
             # We use the state_group and prev_state_id stuff to pull the
             # current_state_ids out of the DB and construct prev_state_ids.
             storage=storage,
             state_group=input["state_group"],
             state_group_before_event=input["state_group_before_event"],
-            state_group_deltas=state_group_deltas,
+            prev_group=input["prev_group"],
             state_delta_due_to_event=_decode_state_dict(
                 input["state_delta_due_to_event"]
             ),
+            delta_ids=_decode_state_dict(input["delta_ids"]),
             rejected=input["rejected"],
             partial_state=input.get("partial_state", False),
         )
@@ -355,7 +349,7 @@ class UnpersistedEventContext(UnpersistedEventContextBase):
     _storage: "StorageControllers"
     state_group_before_event: Optional[int]
     state_group_after_event: Optional[int]
-    state_delta_due_to_event: Optional[StateMap[str]]
+    state_delta_due_to_event: Optional[dict]
     prev_group_for_state_group_before_event: Optional[int]
     delta_ids_to_state_group_before_event: Optional[StateMap[str]]
     partial_state: bool
@@ -386,16 +380,26 @@ class UnpersistedEventContext(UnpersistedEventContextBase):
 
         events_and_persisted_context = []
         for event, unpersisted_context in amended_events_and_context:
-            state_group_deltas = unpersisted_context._build_state_group_deltas()
-
-            context = EventContext(
-                storage=unpersisted_context._storage,
-                state_group=unpersisted_context.state_group_after_event,
-                state_group_before_event=unpersisted_context.state_group_before_event,
-                state_delta_due_to_event=unpersisted_context.state_delta_due_to_event,
-                partial_state=unpersisted_context.partial_state,
-                state_group_deltas=state_group_deltas,
-            )
+            if event.is_state():
+                context = EventContext(
+                    storage=unpersisted_context._storage,
+                    state_group=unpersisted_context.state_group_after_event,
+                    state_group_before_event=unpersisted_context.state_group_before_event,
+                    state_delta_due_to_event=unpersisted_context.state_delta_due_to_event,
+                    partial_state=unpersisted_context.partial_state,
+                    prev_group=unpersisted_context.state_group_before_event,
+                    delta_ids=unpersisted_context.state_delta_due_to_event,
+                )
+            else:
+                context = EventContext(
+                    storage=unpersisted_context._storage,
+                    state_group=unpersisted_context.state_group_after_event,
+                    state_group_before_event=unpersisted_context.state_group_before_event,
+                    state_delta_due_to_event=unpersisted_context.state_delta_due_to_event,
+                    partial_state=unpersisted_context.partial_state,
+                    prev_group=unpersisted_context.prev_group_for_state_group_before_event,
+                    delta_ids=unpersisted_context.delta_ids_to_state_group_before_event,
+                )
             events_and_persisted_context.append((event, context))
         return events_and_persisted_context
 
@@ -448,11 +452,11 @@ class UnpersistedEventContext(UnpersistedEventContextBase):
 
         # if the event isn't a state event the state group doesn't change
         if not self.state_delta_due_to_event:
-            self.state_group_after_event = self.state_group_before_event
+            state_group_after_event = self.state_group_before_event
 
         # otherwise if it is a state event we need to get a state group for it
         else:
-            self.state_group_after_event = await self._storage.state.store_state_group(
+            state_group_after_event = await self._storage.state.store_state_group(
                 event.event_id,
                 event.room_id,
                 prev_group=self.state_group_before_event,
@@ -460,81 +464,16 @@ class UnpersistedEventContext(UnpersistedEventContextBase):
                 current_state_ids=None,
             )
 
-        state_group_deltas = self._build_state_group_deltas()
-
         return EventContext.with_state(
             storage=self._storage,
-            state_group=self.state_group_after_event,
+            state_group=state_group_after_event,
             state_group_before_event=self.state_group_before_event,
             state_delta_due_to_event=self.state_delta_due_to_event,
-            state_group_deltas=state_group_deltas,
             partial_state=self.partial_state,
+            prev_group=self.state_group_before_event,
+            delta_ids=self.state_delta_due_to_event,
         )
 
-    def _build_state_group_deltas(self) -> Dict[Tuple[int, int], StateMap]:
-        """
-        Collect deltas between the state groups associated with this context
-        """
-        state_group_deltas = {}
-
-        # if we know the state group before the event and after the event, add them and the
-        # state delta between them to state_group_deltas
-        if self.state_group_before_event and self.state_group_after_event:
-            # if we have the state groups we should have the delta
-            assert self.state_delta_due_to_event is not None
-            state_group_deltas[
-                (
-                    self.state_group_before_event,
-                    self.state_group_after_event,
-                )
-            ] = self.state_delta_due_to_event
-
-        # the state group before the event may also have a state group which precedes it, if
-        # we have that and the state group before the event, add them and the state
-        # delta between them to state_group_deltas
-        if (
-            self.prev_group_for_state_group_before_event
-            and self.state_group_before_event
-        ):
-            # if we have both state groups we should have the delta between them
-            assert self.delta_ids_to_state_group_before_event is not None
-            state_group_deltas[
-                (
-                    self.prev_group_for_state_group_before_event,
-                    self.state_group_before_event,
-                )
-            ] = self.delta_ids_to_state_group_before_event
-
-        return state_group_deltas
-
-
-def _encode_state_group_delta(
-    state_group_delta: Dict[Tuple[int, int], StateMap[str]]
-) -> List[Tuple[int, int, Optional[List[Tuple[str, str, str]]]]]:
-    if not state_group_delta:
-        return []
-
-    state_group_delta_encoded = []
-    for key, value in state_group_delta.items():
-        state_group_delta_encoded.append((key[0], key[1], _encode_state_dict(value)))
-
-    return state_group_delta_encoded
-
-
-def _decode_state_group_delta(
-    input: List[Tuple[int, int, List[Tuple[str, str, str]]]]
-) -> Dict[Tuple[int, int], StateMap[str]]:
-    if not input:
-        return {}
-
-    state_group_deltas = {}
-    for state_group_1, state_group_2, state_dict in input:
-        state_map = _decode_state_dict(state_dict)
-        assert state_map is not None
-        state_group_deltas[(state_group_1, state_group_2)] = state_map
-
-    return state_group_deltas
-
 
 def _encode_state_dict(
     state_dict: Optional[StateMap[str]],

synapse/events/utils.py

@@ -164,12 +164,21 @@ def prune_event_dict(room_version: RoomVersion, event_dict: JsonDict) -> JsonDic
         if room_version.msc2176_redaction_rules:
             add_fields("invite")
 
+        if room_version.msc2716_historical:
+            add_fields("historical")
+
     elif event_type == EventTypes.Aliases and room_version.special_case_aliases_auth:
         add_fields("aliases")
     elif event_type == EventTypes.RoomHistoryVisibility:
         add_fields("history_visibility")
     elif event_type == EventTypes.Redaction and room_version.msc2176_redaction_rules:
         add_fields("redacts")
+    elif room_version.msc2716_redactions and event_type == EventTypes.MSC2716_INSERTION:
+        add_fields(EventContentFields.MSC2716_NEXT_BATCH_ID)
+    elif room_version.msc2716_redactions and event_type == EventTypes.MSC2716_BATCH:
+        add_fields(EventContentFields.MSC2716_BATCH_ID)
+    elif room_version.msc2716_redactions and event_type == EventTypes.MSC2716_MARKER:
+        add_fields(EventContentFields.MSC2716_INSERTION_EVENT_REFERENCE)
 
     # Protect the rel_type and event_id fields under the m.relates_to field.
     if room_version.msc3389_relation_redactions:

synapse/federation/federation_client.py

@@ -260,9 +260,7 @@ class FederationClient(FederationBase):
         use_unstable = False
         for user_id, one_time_keys in query.items():
             for device_id, algorithms in one_time_keys.items():
-                # If more than one algorithm is requested, attempt to use the unstable
-                # endpoint.
-                if sum(algorithms.values()) > 1:
+                if any(count > 1 for count in algorithms.values()):
                     use_unstable = True
                 if algorithms:
                     # For the stable query, choose only the first algorithm.
@@ -298,7 +296,6 @@ class FederationClient(FederationBase):
         else:
             logger.debug("Skipping unstable claim client keys API")
 
-        # TODO Potentially attempt multiple queries and combine the results?
         return await self.transport_layer.claim_client_keys(
             user, destination, content, timeout
         )

synapse/federation/federation_server.py

@@ -1016,9 +1016,7 @@ class FederationServer(FederationBase):
             for user_id, device_keys in result.items():
                 for device_id, keys in device_keys.items():
                     for key_id, key in keys.items():
-                        json_result.setdefault(user_id, {}).setdefault(device_id, {})[
-                            key_id
-                        ] = key
+                        json_result.setdefault(user_id, {})[device_id] = {key_id: key}
 
         logger.info(
             "Claimed one-time-keys: %s",

synapse/federation/sender/__init__.py

@@ -109,8 +109,10 @@ was enabled*, Catch-Up Mode is exited and we return to `_transaction_transmissio
 
 If a remote server is unreachable over federation, we back off from that server,
 with an exponentially-increasing retry interval.
-We automatically retry after the retry interval expires (roughly, the logic to do so
-being triggered every minute).
+Whilst we don't automatically retry after the interval, we prevent making new attempts
+until such time as the back-off has cleared.
+Once the back-off is cleared and a new PDU or EDU arrives for transmission, the transmission
+loop resumes and empties the queue by making federation requests.
 
 If the backoff grows too large (> 1 hour), the in-memory queue is emptied (to prevent
 unbounded growth) and Catch-Up Mode is entered.
@@ -143,6 +145,7 @@ from prometheus_client import Counter
 from typing_extensions import Literal
 
 from twisted.internet import defer
+from twisted.internet.interfaces import IDelayedCall
 
 import synapse.metrics
 from synapse.api.presence import UserPresenceState
@@ -181,18 +184,14 @@ sent_pdus_destination_dist_total = Counter(
     "Total number of PDUs queued for sending across all destinations",
 )
 
-# Time (in s) to wait before trying to wake up destinations that have
-# catch-up outstanding. This will also be the delay applied at startup
-# before trying the same.
-# Please note that rate limiting still applies, so while the loop is
-# executed every X seconds the destinations may not be wake up because
-# they are being rate limited following previous attempt failures.
-WAKEUP_RETRY_PERIOD_SEC = 60
+# Time (in s) after Synapse's startup that we will begin to wake up destinations
+# that have catch-up outstanding.
+CATCH_UP_STARTUP_DELAY_SEC = 15
 
 # Time (in s) to wait in between waking up each destination, i.e. one destination
-# will be woken up every <x> seconds until we have woken every destination
-# has outstanding catch-up.
-WAKEUP_INTERVAL_BETWEEN_DESTINATIONS_SEC = 5
+# will be woken up every <x> seconds after Synapse's startup until we have woken
+# every destination has outstanding catch-up.
+CATCH_UP_STARTUP_INTERVAL_SEC = 5
 
 
 class AbstractFederationSender(metaclass=abc.ABCMeta):
@@ -416,10 +415,12 @@ class FederationSender(AbstractFederationSender):
             / hs.config.ratelimiting.federation_rr_transactions_per_room_per_second
         )
 
-        # Regularly wake up destinations that have outstanding PDUs to be caught up
-        self.clock.looping_call(
+        # wake up destinations that have outstanding PDUs to be caught up
+        self._catchup_after_startup_timer: Optional[
+            IDelayedCall
+        ] = self.clock.call_later(
+            CATCH_UP_STARTUP_DELAY_SEC,
             run_as_background_process,
-            WAKEUP_RETRY_PERIOD_SEC * 1000.0,
             "wake_destinations_needing_catchup",
             self._wake_destinations_needing_catchup,
         )
@@ -965,6 +966,7 @@ class FederationSender(AbstractFederationSender):
 
             if not destinations_to_wake:
                 # finished waking all destinations!
+                self._catchup_after_startup_timer = None
                 break
 
             last_processed = destinations_to_wake[-1]
@@ -981,4 +983,4 @@ class FederationSender(AbstractFederationSender):
                     last_processed,
                 )
                 self.wake_destination(destination)
-                await self.clock.sleep(WAKEUP_INTERVAL_BETWEEN_DESTINATIONS_SEC)
+                await self.clock.sleep(CATCH_UP_STARTUP_INTERVAL_SEC)

synapse/handlers/federation.py

@@ -105,12 +105,14 @@ backfill_processing_before_timer = Histogram(
 )
 
 
-# TODO: We can refactor this away now that there is only one backfill point again
 class _BackfillPointType(Enum):
     # a regular backwards extremity (ie, an event which we don't yet have, but which
     # is referred to by other events in the DAG)
     BACKWARDS_EXTREMITY = enum.auto()
 
+    # an MSC2716 "insertion event"
+    INSERTION_PONT = enum.auto()
+
 
 @attr.s(slots=True, auto_attribs=True, frozen=True)
 class _BackfillPoint:
@@ -271,10 +273,32 @@ class FederationHandler:
             )
         ]
 
+        insertion_events_to_be_backfilled: List[_BackfillPoint] = []
+        if self.hs.config.experimental.msc2716_enabled:
+            insertion_events_to_be_backfilled = [
+                _BackfillPoint(event_id, depth, _BackfillPointType.INSERTION_PONT)
+                for event_id, depth in await self.store.get_insertion_event_backward_extremities_in_room(
+                    room_id=room_id,
+                    current_depth=current_depth,
+                    # We only need to end up with 5 extremities combined with
+                    # the backfill points to make the `/backfill` request ...
+                    # (see the other comment above for more context).
+                    limit=50,
+                )
+            ]
+        logger.debug(
+            "_maybe_backfill_inner: backwards_extremities=%s insertion_events_to_be_backfilled=%s",
+            backwards_extremities,
+            insertion_events_to_be_backfilled,
+        )
+
         # we now have a list of potential places to backpaginate from. We prefer to
         # start with the most recent (ie, max depth), so let's sort the list.
         sorted_backfill_points: List[_BackfillPoint] = sorted(
-            backwards_extremities,
+            itertools.chain(
+                backwards_extremities,
+                insertion_events_to_be_backfilled,
+            ),
             key=lambda e: -int(e.depth),
         )
 
@@ -296,21 +320,14 @@ class FederationHandler:
             str(len(sorted_backfill_points)),
         )
 
-        # If we have no backfill points lower than the `current_depth` then either we
-        # can a) bail or b) still attempt to backfill. We opt to try backfilling anyway
-        # just in case we do get relevant events. This is good for eventual consistency
-        # sake but we don't need to block the client for something that is just as
-        # likely not to return anything relevant so we backfill in the background. The
-        # only way, this could return something relevant is if we discover a new branch
-        # of history that extends all the way back to where we are currently paginating
-        # and it's within the 100 events that are returned from `/backfill`.
+        # If we have no backfill points lower than the `current_depth` then
+        # either we can a) bail or b) still attempt to backfill. We opt to try
+        # backfilling anyway just in case we do get relevant events.
         if not sorted_backfill_points and current_depth != MAX_DEPTH:
             logger.debug(
                 "_maybe_backfill_inner: all backfill points are *after* current depth. Trying again with later backfill points."
             )
-            run_as_background_process(
-                "_maybe_backfill_inner_anyway_with_max_depth",
-                self._maybe_backfill_inner,
+            return await self._maybe_backfill_inner(
                 room_id=room_id,
                 # We use `MAX_DEPTH` so that we find all backfill points next
                 # time (all events are below the `MAX_DEPTH`)
@@ -321,9 +338,6 @@ class FederationHandler:
                 # overall otherwise the smaller one will throw off the results.
                 processing_start_time=None,
             )
-            # We return `False` because we're backfilling in the background and there is
-            # no new events immediately for the caller to know about yet.
-            return False
 
         # Even after recursing with `MAX_DEPTH`, we didn't find any
         # backward extremities to backfill from.
@@ -387,7 +401,10 @@ class FederationHandler:
             #   event but not anything before it. This would require looking at the
             #   state *before* the event, ignoring the special casing certain event
             #   types have.
-            event_ids_to_check = await self.store.get_successor_events(bp.event_id)
+            if bp.type == _BackfillPointType.INSERTION_PONT:
+                event_ids_to_check = [bp.event_id]
+            else:
+                event_ids_to_check = await self.store.get_successor_events(bp.event_id)
 
             events_to_check = await self.store.get_events_as_list(
                 event_ids_to_check,

synapse/handlers/federation_event.py

@@ -601,6 +601,18 @@ class FederationEventHandler:
                 room_id, [(event, context)]
             )
 
+            # If we're joining the room again, check if there is new marker
+            # state indicating that there is new history imported somewhere in
+            # the DAG. Multiple markers can exist in the current state with
+            # unique state_keys.
+            #
+            # Do this after the state from the remote join was persisted (via
+            # `persist_events_and_notify`). Otherwise we can run into a
+            # situation where the create event doesn't exist yet in the
+            # `current_state_events`
+            for e in state:
+                await self._handle_marker_event(origin, e)
+
             return stream_id_after_persist
 
     async def update_state_for_partial_state_event(
@@ -903,6 +915,13 @@ class FederationEventHandler:
             )
         )
 
+        # We construct the event lists in source order from `/backfill` response because
+        # it's a) easiest, but also b) the order in which we process things matters for
+        # MSC2716 historical batches because many historical events are all at the same
+        # `depth` and we rely on the tenuous sort that the other server gave us and hope
+        # they're doing their best. The brittle nature of this ordering for historical
+        # messages over federation is one of the reasons why we don't want to continue
+        # on MSC2716 until we have online topological ordering.
         events_with_failed_pull_attempts, fresh_events = partition(
             new_events, lambda e: e.event_id in event_ids_with_failed_pull_attempts
         )
@@ -1441,6 +1460,8 @@ class FederationEventHandler:
 
         await self._run_push_actions_and_persist_event(event, context, backfilled)
 
+        await self._handle_marker_event(origin, event)
+
         if backfilled or context.rejected:
             return
 
@@ -1538,6 +1559,94 @@ class FederationEventHandler:
         except Exception:
             logger.exception("Failed to resync device for %s", sender)
 
+    @trace
+    async def _handle_marker_event(self, origin: str, marker_event: EventBase) -> None:
+        """Handles backfilling the insertion event when we receive a marker
+        event that points to one.
+
+        Args:
+            origin: Origin of the event. Will be called to get the insertion event
+            marker_event: The event to process
+        """
+
+        if marker_event.type != EventTypes.MSC2716_MARKER:
+            # Not a marker event
+            return
+
+        if marker_event.rejected_reason is not None:
+            # Rejected event
+            return
+
+        # Skip processing a marker event if the room version doesn't
+        # support it or the event is not from the room creator.
+        room_version = await self._store.get_room_version(marker_event.room_id)
+        create_event = await self._store.get_create_event_for_room(marker_event.room_id)
+        if not room_version.msc2175_implicit_room_creator:
+            room_creator = create_event.content.get(EventContentFields.ROOM_CREATOR)
+        else:
+            room_creator = create_event.sender
+        if not room_version.msc2716_historical and (
+            not self._config.experimental.msc2716_enabled
+            or marker_event.sender != room_creator
+        ):
+            return
+
+        logger.debug("_handle_marker_event: received %s", marker_event)
+
+        insertion_event_id = marker_event.content.get(
+            EventContentFields.MSC2716_INSERTION_EVENT_REFERENCE
+        )
+
+        if insertion_event_id is None:
+            # Nothing to retrieve then (invalid marker)
+            return
+
+        already_seen_insertion_event = await self._store.have_seen_event(
+            marker_event.room_id, insertion_event_id
+        )
+        if already_seen_insertion_event:
+            # No need to process a marker again if we have already seen the
+            # insertion event that it was pointing to
+            return
+
+        logger.debug(
+            "_handle_marker_event: backfilling insertion event %s", insertion_event_id
+        )
+
+        await self._get_events_and_persist(
+            origin,
+            marker_event.room_id,
+            [insertion_event_id],
+        )
+
+        insertion_event = await self._store.get_event(
+            insertion_event_id, allow_none=True
+        )
+        if insertion_event is None:
+            logger.warning(
+                "_handle_marker_event: server %s didn't return insertion event %s for marker %s",
+                origin,
+                insertion_event_id,
+                marker_event.event_id,
+            )
+            return
+
+        logger.debug(
+            "_handle_marker_event: succesfully backfilled insertion event %s from marker event %s",
+            insertion_event,
+            marker_event,
+        )
+
+        await self._store.insert_insertion_extremity(
+            insertion_event_id, marker_event.room_id
+        )
+
+        logger.debug(
+            "_handle_marker_event: insertion extremity added for %s from marker event %s",
+            insertion_event,
+            marker_event,
+        )
+
     async def backfill_event_id(
         self, destinations: List[str], room_id: str, event_id: str
     ) -> PulledPduInfo:

synapse/handlers/message.py

@@ -60,6 +60,7 @@ from synapse.replication.http.send_event import ReplicationSendEventRestServlet
 from synapse.replication.http.send_events import ReplicationSendEventsRestServlet
 from synapse.storage.databases.main.events_worker import EventRedactBehaviour
 from synapse.types import (
+    MutableStateMap,
     PersistedEventPosition,
     Requester,
     RoomAlias,
@@ -572,6 +573,7 @@ class EventCreationHandler:
         state_event_ids: Optional[List[str]] = None,
         require_consent: bool = True,
         outlier: bool = False,
+        historical: bool = False,
         depth: Optional[int] = None,
         state_map: Optional[StateMap[str]] = None,
         for_batch: bool = False,
@@ -597,7 +599,7 @@ class EventCreationHandler:
             allow_no_prev_events: Whether to allow this event to be created an empty
                 list of prev_events. Normally this is prohibited just because most
                 events should have a prev_event and we should only use this in special
-                cases (previously useful for MSC2716).
+                cases like MSC2716.
             prev_event_ids:
                 the forward extremities to use as the prev_events for the
                 new event.
@@ -612,10 +614,13 @@ class EventCreationHandler:
                 If non-None, prev_event_ids must also be provided.
 
             state_event_ids:
-                The full state at a given event. This was previously used particularly
-                by the MSC2716 /batch_send endpoint. This should normally be left as
-                None, which will cause the auth_event_ids to be calculated based on the
-                room state at the prev_events.
+                The full state at a given event. This is used particularly by the MSC2716
+                /batch_send endpoint. One use case is with insertion events which float at
+                the beginning of a historical batch and don't have any `prev_events` to
+                derive from; we add all of these state events as the explicit state so the
+                rest of the historical batch can inherit the same state and state_group.
+                This should normally be left as None, which will cause the auth_event_ids
+                to be calculated based on the room state at the prev_events.
 
             require_consent: Whether to check if the requester has
                 consented to the privacy policy.
@@ -624,6 +629,10 @@ class EventCreationHandler:
                 it's from an arbitrary point and floating in the DAG as
                 opposed to being inline with the current DAG.
 
+            historical: Indicates whether the message is being inserted
+                back in time around some existing events. This is used to skip
+                a few checks and mark the event as backfilled.
+
             depth: Override the depth used to order the event in the DAG.
                 Should normally be set to None, which will cause the depth to be calculated
                 based on the prev_events.
@@ -708,6 +717,8 @@ class EventCreationHandler:
 
         builder.internal_metadata.outlier = outlier
 
+        builder.internal_metadata.historical = historical
+
         event, unpersisted_context = await self.create_new_client_event(
             builder=builder,
             requester=requester,
@@ -936,6 +947,7 @@ class EventCreationHandler:
         txn_id: Optional[str] = None,
         ignore_shadow_ban: bool = False,
         outlier: bool = False,
+        historical: bool = False,
         depth: Optional[int] = None,
     ) -> Tuple[EventBase, int]:
         """
@@ -949,16 +961,19 @@ class EventCreationHandler:
             allow_no_prev_events: Whether to allow this event to be created an empty
                 list of prev_events. Normally this is prohibited just because most
                 events should have a prev_event and we should only use this in special
-                cases (previously useful for MSC2716).
+                cases like MSC2716.
             prev_event_ids:
                 The event IDs to use as the prev events.
                 Should normally be left as None to automatically request them
                 from the database.
             state_event_ids:
-                The full state at a given event. This was previously used particularly
-                by the MSC2716 /batch_send endpoint. This should normally be left as
-                None, which will cause the auth_event_ids to be calculated based on the
-                room state at the prev_events.
+                The full state at a given event. This is used particularly by the MSC2716
+                /batch_send endpoint. One use case is with insertion events which float at
+                the beginning of a historical batch and don't have any `prev_events` to
+                derive from; we add all of these state events as the explicit state so the
+                rest of the historical batch can inherit the same state and state_group.
+                This should normally be left as None, which will cause the auth_event_ids
+                to be calculated based on the room state at the prev_events.
             ratelimit: Whether to rate limit this send.
             txn_id: The transaction ID.
             ignore_shadow_ban: True if shadow-banned users should be allowed to
@@ -966,6 +981,9 @@ class EventCreationHandler:
             outlier: Indicates whether the event is an `outlier`, i.e. if
                 it's from an arbitrary point and floating in the DAG as
                 opposed to being inline with the current DAG.
+            historical: Indicates whether the message is being inserted
+                back in time around some existing events. This is used to skip
+                a few checks and mark the event as backfilled.
             depth: Override the depth used to order the event in the DAG.
                 Should normally be set to None, which will cause the depth to be calculated
                 based on the prev_events.
@@ -1035,6 +1053,7 @@ class EventCreationHandler:
                     prev_event_ids=prev_event_ids,
                     state_event_ids=state_event_ids,
                     outlier=outlier,
+                    historical=historical,
                     depth=depth,
                 )
                 context = await unpersisted_context.persist(event)
@@ -1126,7 +1145,7 @@ class EventCreationHandler:
             allow_no_prev_events: Whether to allow this event to be created an empty
                 list of prev_events. Normally this is prohibited just because most
                 events should have a prev_event and we should only use this in special
-                cases (previously useful for MSC2716).
+                cases like MSC2716.
             prev_event_ids:
                 the forward extremities to use as the prev_events for the
                 new event.
@@ -1139,10 +1158,13 @@ class EventCreationHandler:
                 based on the room state at the prev_events.
 
             state_event_ids:
-                The full state at a given event. This was previously used particularly
-                by the MSC2716 /batch_send endpoint. This should normally be left as
-                None, which will cause the auth_event_ids to be calculated based on the
-                room state at the prev_events.
+                The full state at a given event. This is used particularly by the MSC2716
+                /batch_send endpoint. One use case is with insertion events which float at
+                the beginning of a historical batch and don't have any `prev_events` to
+                derive from; we add all of these state events as the explicit state so the
+                rest of the historical batch can inherit the same state and state_group.
+                This should normally be left as None, which will cause the auth_event_ids
+                to be calculated based on the room state at the prev_events.
 
             depth: Override the depth used to order the event in the DAG.
                 Should normally be set to None, which will cause the depth to be calculated
@@ -1239,6 +1261,52 @@ class EventCreationHandler:
             if builder.internal_metadata.outlier:
                 event.internal_metadata.outlier = True
                 context = EventContext.for_outlier(self._storage_controllers)
+            elif (
+                event.type == EventTypes.MSC2716_INSERTION
+                and state_event_ids
+                and builder.internal_metadata.is_historical()
+            ):
+                # Add explicit state to the insertion event so it has state to derive
+                # from even though it's floating with no `prev_events`. The rest of
+                # the batch can derive from this state and state_group.
+                #
+                # TODO(faster_joins): figure out how this works, and make sure that the
+                #   old state is complete.
+                #   https://github.com/matrix-org/synapse/issues/13003
+                metadata = await self.store.get_metadata_for_events(state_event_ids)
+
+                state_map_for_event: MutableStateMap[str] = {}
+                for state_id in state_event_ids:
+                    data = metadata.get(state_id)
+                    if data is None:
+                        # We're trying to persist a new historical batch of events
+                        # with the given state, e.g. via
+                        # `RoomBatchSendEventRestServlet`. The state can be inferred
+                        # by Synapse or set directly by the client.
+                        #
+                        # Either way, we should have persisted all the state before
+                        # getting here.
+                        raise Exception(
+                            f"State event {state_id} not found in DB,"
+                            " Synapse should have persisted it before using it."
+                        )
+
+                    if data.state_key is None:
+                        raise Exception(
+                            f"Trying to set non-state event {state_id} as state"
+                        )
+
+                    state_map_for_event[(data.event_type, data.state_key)] = state_id
+
+                # TODO(faster_joins): check how MSC2716 works and whether we can have
+                #   partial state here
+                #   https://github.com/matrix-org/synapse/issues/13003
+                context = await self.state.calculate_context_info(
+                    event,
+                    state_ids_before_event=state_map_for_event,
+                    partial_state=False,
+                )
+
             else:
                 context = await self.state.calculate_context_info(event)
 
@@ -1808,6 +1876,28 @@ class EventCreationHandler:
                             403, "Redacting server ACL events is not permitted"
                         )
 
+                    # Add a little safety stop-gap to prevent people from trying to
+                    # redact MSC2716 related events when they're in a room version
+                    # which does not support it yet. We allow people to use MSC2716
+                    # events in existing room versions but only from the room
+                    # creator since it does not require any changes to the auth
+                    # rules and in effect, the redaction algorithm . In the
+                    # supported room version, we add the `historical` power level to
+                    # auth the MSC2716 related events and adjust the redaction
+                    # algorthim to keep the `historical` field around (redacting an
+                    # event should only strip fields which don't affect the
+                    # structural protocol level).
+                    is_msc2716_event = (
+                        original_event.type == EventTypes.MSC2716_INSERTION
+                        or original_event.type == EventTypes.MSC2716_BATCH
+                        or original_event.type == EventTypes.MSC2716_MARKER
+                    )
+                    if not room_version_obj.msc2716_historical and is_msc2716_event:
+                        raise AuthError(
+                            403,
+                            "Redacting MSC2716 events is not supported in this room version",
+                        )
+
                 event_types = event_auth.auth_types_for_event(event.room_version, event)
                 prev_state_ids = await context.get_prev_state_ids(
                     StateFilter.from_types(event_types)
@@ -1845,12 +1935,58 @@ class EventCreationHandler:
                 if prev_state_ids:
                     raise AuthError(403, "Changing the room create event is forbidden")
 
+            if event.type == EventTypes.MSC2716_INSERTION:
+                room_version = await self.store.get_room_version_id(event.room_id)
+                room_version_obj = KNOWN_ROOM_VERSIONS[room_version]
+
+                create_event = await self.store.get_create_event_for_room(event.room_id)
+                if not room_version_obj.msc2175_implicit_room_creator:
+                    room_creator = create_event.content.get(
+                        EventContentFields.ROOM_CREATOR
+                    )
+                else:
+                    room_creator = create_event.sender
+
+                # Only check an insertion event if the room version
+                # supports it or the event is from the room creator.
+                if room_version_obj.msc2716_historical or (
+                    self.config.experimental.msc2716_enabled
+                    and event.sender == room_creator
+                ):
+                    next_batch_id = event.content.get(
+                        EventContentFields.MSC2716_NEXT_BATCH_ID
+                    )
+                    conflicting_insertion_event_id = None
+                    if next_batch_id:
+                        conflicting_insertion_event_id = (
+                            await self.store.get_insertion_event_id_by_batch_id(
+                                event.room_id, next_batch_id
+                            )
+                        )
+                    if conflicting_insertion_event_id is not None:
+                        # The current insertion event that we're processing is invalid
+                        # because an insertion event already exists in the room with the
+                        # same next_batch_id. We can't allow multiple because the batch
+                        # pointing will get weird, e.g. we can't determine which insertion
+                        # event the batch event is pointing to.
+                        raise SynapseError(
+                            HTTPStatus.BAD_REQUEST,
+                            "Another insertion event already exists with the same next_batch_id",
+                            errcode=Codes.INVALID_PARAM,
+                        )
+
+            # Mark any `m.historical` messages as backfilled so they don't appear
+            # in `/sync` and have the proper decrementing `stream_ordering` as we import
+            backfilled = False
+            if event.internal_metadata.is_historical():
+                backfilled = True
+
         assert self._storage_controllers.persistence is not None
         (
             persisted_events,
             max_stream_token,
         ) = await self._storage_controllers.persistence.persist_events(
-            events_and_context,
+            events_and_context, backfilled=backfilled
         )
 
         events_and_pos = []

synapse/handlers/pagination.py

@@ -40,11 +40,6 @@ if TYPE_CHECKING:
 
 logger = logging.getLogger(__name__)
 
-# How many single event gaps we tolerate returning in a `/messages` response before we
-# backfill and try to fill in the history. This is an arbitrarily picked number so feel
-# free to tune it in the future.
-BACKFILL_BECAUSE_TOO_MANY_GAPS_THRESHOLD = 3
-
 
 @attr.s(slots=True, auto_attribs=True)
 class PurgeStatus:
@@ -491,35 +486,35 @@ class PaginationHandler:
                         room_id, room_token.stream
                     )
 
-            # If they have left the room then clamp the token to be before
-            # they left the room, to save the effort of loading from the
-            # database.
-            if (
-                pagin_config.direction == Direction.BACKWARDS
-                and not use_admin_priviledge
-                and membership == Membership.LEAVE
-            ):
-                # This is only None if the room is world_readable, in which case
-                # "Membership.JOIN" would have been returned and we should never hit
-                # this branch.
-                assert member_event_id
+                if not use_admin_priviledge and membership == Membership.LEAVE:
+                    # If they have left the room then clamp the token to be before
+                    # they left the room, to save the effort of loading from the
+                    # database.
 
-                leave_token = await self.store.get_topological_token_for_event(
-                    member_event_id
-                )
-                assert leave_token.topological is not None
+                    # This is only None if the room is world_readable, in which
+                    # case "JOIN" would have been returned.
+                    assert member_event_id
 
-                if leave_token.topological < curr_topo:
-                    from_token = from_token.copy_and_replace(
-                        StreamKeyType.ROOM, leave_token
+                    leave_token = await self.store.get_topological_token_for_event(
+                        member_event_id
                     )
+                    assert leave_token.topological is not None
+
+                    if leave_token.topological < curr_topo:
+                        from_token = from_token.copy_and_replace(
+                            StreamKeyType.ROOM, leave_token
+                        )
+
+                await self.hs.get_federation_handler().maybe_backfill(
+                    room_id,
+                    curr_topo,
+                    limit=pagin_config.limit,
+                )
 
             to_room_key = None
             if pagin_config.to_token:
                 to_room_key = pagin_config.to_token.room_key
 
-            # Initially fetch the events from the database. With any luck, we can return
-            # these without blocking on backfill (handled below).
             events, next_key = await self.store.paginate_room_events(
                 room_id=room_id,
                 from_key=from_token.room_key,
@@ -529,94 +524,6 @@ class PaginationHandler:
                 event_filter=event_filter,
             )
 
-            if pagin_config.direction == Direction.BACKWARDS:
-                # We use a `Set` because there can be multiple events at a given depth
-                # and we only care about looking at the unique continum of depths to
-                # find gaps.
-                event_depths: Set[int] = {event.depth for event in events}
-                sorted_event_depths = sorted(event_depths)
-
-                # Inspect the depths of the returned events to see if there are any gaps
-                found_big_gap = False
-                number_of_gaps = 0
-                previous_event_depth = (
-                    sorted_event_depths[0] if len(sorted_event_depths) > 0 else 0
-                )
-                for event_depth in sorted_event_depths:
-                    # We don't expect a negative depth but we'll just deal with it in
-                    # any case by taking the absolute value to get the true gap between
-                    # any two integers.
-                    depth_gap = abs(event_depth - previous_event_depth)
-                    # A `depth_gap` of 1 is a normal continuous chain to the next event
-                    # (1 <-- 2 <-- 3) so anything larger indicates a missing event (it's
-                    # also possible there is no event at a given depth but we can't ever
-                    # know that for sure)
-                    if depth_gap > 1:
-                        number_of_gaps += 1
-
-                    # We only tolerate a small number single-event long gaps in the
-                    # returned events because those are most likely just events we've
-                    # failed to pull in the past. Anything longer than that is probably
-                    # a sign that we're missing a decent chunk of history and we should
-                    # try to backfill it.
-                    #
-                    # XXX: It's possible we could tolerate longer gaps if we checked
-                    # that a given events `prev_events` is one that has failed pull
-                    # attempts and we could just treat it like a dead branch of history
-                    # for now or at least something that we don't need the block the
-                    # client on to try pulling.
-                    #
-                    # XXX: If we had something like MSC3871 to indicate gaps in the
-                    # timeline to the client, we could also get away with any sized gap
-                    # and just have the client refetch the holes as they see fit.
-                    if depth_gap > 2:
-                        found_big_gap = True
-                        break
-                    previous_event_depth = event_depth
-
-                # Backfill in the foreground if we found a big gap, have too many holes,
-                # or we don't have enough events to fill the limit that the client asked
-                # for.
-                missing_too_many_events = (
-                    number_of_gaps > BACKFILL_BECAUSE_TOO_MANY_GAPS_THRESHOLD
-                )
-                not_enough_events_to_fill_response = len(events) < pagin_config.limit
-                if (
-                    found_big_gap
-                    or missing_too_many_events
-                    or not_enough_events_to_fill_response
-                ):
-                    did_backfill = (
-                        await self.hs.get_federation_handler().maybe_backfill(
-                            room_id,
-                            curr_topo,
-                            limit=pagin_config.limit,
-                        )
-                    )
-
-                    # If we did backfill something, refetch the events from the database to
-                    # catch anything new that might have been added since we last fetched.
-                    if did_backfill:
-                        events, next_key = await self.store.paginate_room_events(
-                            room_id=room_id,
-                            from_key=from_token.room_key,
-                            to_key=to_room_key,
-                            direction=pagin_config.direction,
-                            limit=pagin_config.limit,
-                            event_filter=event_filter,
-                        )
-                else:
-                    # Otherwise, we can backfill in the background for eventual
-                    # consistency's sake but we don't need to block the client waiting
-                    # for a costly federation call and processing.
-                    run_as_background_process(
-                        "maybe_backfill_in_the_background",
-                        self.hs.get_federation_handler().maybe_backfill,
-                        room_id,
-                        curr_topo,
-                        limit=pagin_config.limit,
-                    )
-
             next_token = from_token.copy_and_replace(StreamKeyType.ROOM, next_key)
 
         # if no events are returned from pagination, that implies

synapse/handlers/room_batch.py

@@ -0,0 +1,466 @@
+import logging
+from typing import TYPE_CHECKING, List, Tuple
+
+from synapse.api.constants import EventContentFields, EventTypes
+from synapse.appservice import ApplicationService
+from synapse.http.servlet import assert_params_in_dict
+from synapse.types import JsonDict, Requester, UserID, create_requester
+from synapse.util.stringutils import random_string
+
+if TYPE_CHECKING:
+    from synapse.server import HomeServer
+
+logger = logging.getLogger(__name__)
+
+
+class RoomBatchHandler:
+    def __init__(self, hs: "HomeServer"):
+        self.hs = hs
+        self.store = hs.get_datastores().main
+        self._state_storage_controller = hs.get_storage_controllers().state
+        self.event_creation_handler = hs.get_event_creation_handler()
+        self.room_member_handler = hs.get_room_member_handler()
+        self.auth = hs.get_auth()
+
+    async def inherit_depth_from_prev_ids(self, prev_event_ids: List[str]) -> int:
+        """Finds the depth which would sort it after the most-recent
+        prev_event_id but before the successors of those events. If no
+        successors are found, we assume it's an historical extremity part of the
+        current batch and use the same depth of the prev_event_ids.
+
+        Args:
+            prev_event_ids: List of prev event IDs
+
+        Returns:
+            Inherited depth
+        """
+        (
+            most_recent_prev_event_id,
+            most_recent_prev_event_depth,
+        ) = await self.store.get_max_depth_of(prev_event_ids)
+
+        # We want to insert the historical event after the `prev_event` but before the successor event
+        #
+        # We inherit depth from the successor event instead of the `prev_event`
+        # because events returned from `/messages` are first sorted by `topological_ordering`
+        # which is just the `depth` and then tie-break with `stream_ordering`.
+        #
+        # We mark these inserted historical events as "backfilled" which gives them a
+        # negative `stream_ordering`. If we use the same depth as the `prev_event`,
+        # then our historical event will tie-break and be sorted before the `prev_event`
+        # when it should come after.
+        #
+        # We want to use the successor event depth so they appear after `prev_event` because
+        # it has a larger `depth` but before the successor event because the `stream_ordering`
+        # is negative before the successor event.
+        assert most_recent_prev_event_id is not None
+        successor_event_ids = await self.store.get_successor_events(
+            most_recent_prev_event_id
+        )
+
+        # If we can't find any successor events, then it's a forward extremity of
+        # historical messages and we can just inherit from the previous historical
+        # event which we can already assume has the correct depth where we want
+        # to insert into.
+        if not successor_event_ids:
+            depth = most_recent_prev_event_depth
+        else:
+            (
+                _,
+                oldest_successor_depth,
+            ) = await self.store.get_min_depth_of(successor_event_ids)
+
+            depth = oldest_successor_depth
+
+        return depth
+
+    def create_insertion_event_dict(
+        self, sender: str, room_id: str, origin_server_ts: int
+    ) -> JsonDict:
+        """Creates an event dict for an "insertion" event with the proper fields
+        and a random batch ID.
+
+        Args:
+            sender: The event author MXID
+            room_id: The room ID that the event belongs to
+            origin_server_ts: Timestamp when the event was sent
+
+        Returns:
+            The new event dictionary to insert.
+        """
+
+        next_batch_id = random_string(8)
+        insertion_event = {
+            "type": EventTypes.MSC2716_INSERTION,
+            "sender": sender,
+            "room_id": room_id,
+            "content": {
+                EventContentFields.MSC2716_NEXT_BATCH_ID: next_batch_id,
+                EventContentFields.MSC2716_HISTORICAL: True,
+            },
+            "origin_server_ts": origin_server_ts,
+        }
+
+        return insertion_event
+
+    async def create_requester_for_user_id_from_app_service(
+        self, user_id: str, app_service: ApplicationService
+    ) -> Requester:
+        """Creates a new requester for the given user_id
+        and validates that the app service is allowed to control
+        the given user.
+
+        Args:
+            user_id: The author MXID that the app service is controlling
+            app_service: The app service that controls the user
+
+        Returns:
+            Requester object
+        """
+
+        await self.auth.validate_appservice_can_control_user_id(app_service, user_id)
+
+        return create_requester(user_id, app_service=app_service)
+
+    async def get_most_recent_full_state_ids_from_event_id_list(
+        self, event_ids: List[str]
+    ) -> List[str]:
+        """Find the most recent event_id and grab the full state at that event.
+        We will use this as a base to auth our historical messages against.
+
+        Args:
+            event_ids: List of event ID's to look at
+
+        Returns:
+            List of event ID's
+        """
+
+        (
+            most_recent_event_id,
+            _,
+        ) = await self.store.get_max_depth_of(event_ids)
+        # mapping from (type, state_key) -> state_event_id
+        assert most_recent_event_id is not None
+        prev_state_map = await self._state_storage_controller.get_state_ids_for_event(
+            most_recent_event_id
+        )
+        # List of state event ID's
+        full_state_ids = list(prev_state_map.values())
+
+        return full_state_ids
+
+    async def persist_state_events_at_start(
+        self,
+        state_events_at_start: List[JsonDict],
+        room_id: str,
+        initial_state_event_ids: List[str],
+        app_service_requester: Requester,
+    ) -> List[str]:
+        """Takes all `state_events_at_start` event dictionaries and creates/persists
+        them in a floating state event chain which don't resolve into the current room
+        state. They are floating because they reference no prev_events which disconnects
+        them from the normal DAG.
+
+        Args:
+            state_events_at_start:
+            room_id: Room where you want the events persisted in.
+            initial_state_event_ids:
+                The base set of state for the historical batch which the floating
+                state chain will derive from. This should probably be the state
+                from the `prev_event` defined by `/batch_send?prev_event_id=$abc`.
+            app_service_requester: The requester of an application service.
+
+        Returns:
+            List of state event ID's we just persisted
+        """
+        assert app_service_requester.app_service
+
+        state_event_ids_at_start = []
+        state_event_ids = initial_state_event_ids.copy()
+
+        # Make the state events float off on their own by specifying no
+        # prev_events for the first one in the chain so we don't have a bunch of
+        # `@mxid joined the room` noise between each batch.
+        prev_event_ids_for_state_chain: List[str] = []
+
+        for index, state_event in enumerate(state_events_at_start):
+            assert_params_in_dict(
+                state_event, ["type", "origin_server_ts", "content", "sender"]
+            )
+
+            logger.debug(
+                "RoomBatchSendEventRestServlet inserting state_event=%s", state_event
+            )
+
+            event_dict = {
+                "type": state_event["type"],
+                "origin_server_ts": state_event["origin_server_ts"],
+                "content": state_event["content"],
+                "room_id": room_id,
+                "sender": state_event["sender"],
+                "state_key": state_event["state_key"],
+            }
+
+            # Mark all events as historical
+            event_dict["content"][EventContentFields.MSC2716_HISTORICAL] = True
+
+            # TODO: This is pretty much the same as some other code to handle inserting state in this file
+            if event_dict["type"] == EventTypes.Member:
+                membership = event_dict["content"].get("membership", None)
+                event_id, _ = await self.room_member_handler.update_membership(
+                    await self.create_requester_for_user_id_from_app_service(
+                        state_event["sender"], app_service_requester.app_service
+                    ),
+                    target=UserID.from_string(event_dict["state_key"]),
+                    room_id=room_id,
+                    action=membership,
+                    content=event_dict["content"],
+                    historical=True,
+                    # Only the first event in the state chain should be floating.
+                    # The rest should hang off each other in a chain.
+                    allow_no_prev_events=index == 0,
+                    prev_event_ids=prev_event_ids_for_state_chain,
+                    # The first event in the state chain is floating with no
+                    # `prev_events` which means it can't derive state from
+                    # anywhere automatically. So we need to set some state
+                    # explicitly.
+                    #
+                    # Make sure to use a copy of this list because we modify it
+                    # later in the loop here. Otherwise it will be the same
+                    # reference and also update in the event when we append
+                    # later.
+                    state_event_ids=state_event_ids.copy(),
+                )
+            else:
+                (
+                    event,
+                    _,
+                ) = await self.event_creation_handler.create_and_send_nonmember_event(
+                    await self.create_requester_for_user_id_from_app_service(
+                        state_event["sender"], app_service_requester.app_service
+                    ),
+                    event_dict,
+                    historical=True,
+                    # Only the first event in the state chain should be floating.
+                    # The rest should hang off each other in a chain.
+                    allow_no_prev_events=index == 0,
+                    prev_event_ids=prev_event_ids_for_state_chain,
+                    # The first event in the state chain is floating with no
+                    # `prev_events` which means it can't derive state from
+                    # anywhere automatically. So we need to set some state
+                    # explicitly.
+                    #
+                    # Make sure to use a copy of this list because we modify it
+                    # later in the loop here. Otherwise it will be the same
+                    # reference and also update in the event when we append later.
+                    state_event_ids=state_event_ids.copy(),
+                )
+                event_id = event.event_id
+
+            state_event_ids_at_start.append(event_id)
+            state_event_ids.append(event_id)
+            # Connect all the state in a floating chain
+            prev_event_ids_for_state_chain = [event_id]
+
+        return state_event_ids_at_start
+
+    async def persist_historical_events(
+        self,
+        events_to_create: List[JsonDict],
+        room_id: str,
+        inherited_depth: int,
+        initial_state_event_ids: List[str],
+        app_service_requester: Requester,
+    ) -> List[str]:
+        """Create and persists all events provided sequentially. Handles the
+        complexity of creating events in chronological order so they can
+        reference each other by prev_event but still persists in
+        reverse-chronoloical order so they have the correct
+        (topological_ordering, stream_ordering) and sort correctly from
+        /messages.
+
+        Args:
+            events_to_create: List of historical events to create in JSON
+                dictionary format.
+            room_id: Room where you want the events persisted in.
+            inherited_depth: The depth to create the events at (you will
+                probably by calling inherit_depth_from_prev_ids(...)).
+            initial_state_event_ids:
+                This is used to set explicit state for the insertion event at
+                the start of the historical batch since it's floating with no
+                prev_events to derive state from automatically.
+            app_service_requester: The requester of an application service.
+
+        Returns:
+            List of persisted event IDs
+        """
+        assert app_service_requester.app_service
+
+        # We expect the first event in a historical batch to be an insertion event
+        assert events_to_create[0]["type"] == EventTypes.MSC2716_INSERTION
+        # We expect the last event in a historical batch to be an batch event
+        assert events_to_create[-1]["type"] == EventTypes.MSC2716_BATCH
+
+        # Make the historical event chain float off on its own by specifying no
+        # prev_events for the first event in the chain which causes the HS to
+        # ask for the state at the start of the batch later.
+        prev_event_ids: List[str] = []
+
+        event_ids = []
+        events_to_persist = []
+        for index, ev in enumerate(events_to_create):
+            assert_params_in_dict(ev, ["type", "origin_server_ts", "content", "sender"])
+
+            assert self.hs.is_mine_id(ev["sender"]), "User must be our own: %s" % (
+                ev["sender"],
+            )
+
+            event_dict = {
+                "type": ev["type"],
+                "origin_server_ts": ev["origin_server_ts"],
+                "content": ev["content"],
+                "room_id": room_id,
+                "sender": ev["sender"],  # requester.user.to_string(),
+                "prev_events": prev_event_ids.copy(),
+            }
+
+            # Mark all events as historical
+            event_dict["content"][EventContentFields.MSC2716_HISTORICAL] = True
+
+            event, unpersisted_context = await self.event_creation_handler.create_event(
+                await self.create_requester_for_user_id_from_app_service(
+                    ev["sender"], app_service_requester.app_service
+                ),
+                event_dict,
+                # Only the first event (which is the insertion event) in the
+                # chain should be floating. The rest should hang off each other
+                # in a chain.
+                allow_no_prev_events=index == 0,
+                prev_event_ids=event_dict.get("prev_events"),
+                # Since the first event (which is the insertion event) in the
+                # chain is floating with no `prev_events`, it can't derive state
+                # from anywhere automatically. So we need to set some state
+                # explicitly.
+                state_event_ids=initial_state_event_ids if index == 0 else None,
+                historical=True,
+                depth=inherited_depth,
+            )
+            context = await unpersisted_context.persist(event)
+            assert context._state_group
+
+            # Normally this is done when persisting the event but we have to
+            # pre-emptively do it here because we create all the events first,
+            # then persist them in another pass below. And we want to share
+            # state_groups across the whole batch so this lookup needs to work
+            # for the next event in the batch in this loop.
+            await self.store.store_state_group_id_for_event_id(
+                event_id=event.event_id,
+                state_group_id=context._state_group,
+            )
+
+            logger.debug(
+                "RoomBatchSendEventRestServlet inserting event=%s, prev_event_ids=%s",
+                event,
+                prev_event_ids,
+            )
+
+            events_to_persist.append((event, context))
+            event_id = event.event_id
+
+            event_ids.append(event_id)
+            prev_event_ids = [event_id]
+
+        # Persist events in reverse-chronological order so they have the
+        # correct stream_ordering as they are backfilled (which decrements).
+        # Events are sorted by (topological_ordering, stream_ordering)
+        # where topological_ordering is just depth.
+        for event, context in reversed(events_to_persist):
+            # This call can't raise `PartialStateConflictError` since we forbid
+            # use of the historical batch API during partial state
+            await self.event_creation_handler.handle_new_client_event(
+                await self.create_requester_for_user_id_from_app_service(
+                    event.sender, app_service_requester.app_service
+                ),
+                events_and_context=[(event, context)],
+            )
+
+        return event_ids
+
+    async def handle_batch_of_events(
+        self,
+        events_to_create: List[JsonDict],
+        room_id: str,
+        batch_id_to_connect_to: str,
+        inherited_depth: int,
+        initial_state_event_ids: List[str],
+        app_service_requester: Requester,
+    ) -> Tuple[List[str], str]:
+        """
+        Handles creating and persisting all of the historical events as well as
+        insertion and batch meta events to make the batch navigable in the DAG.
+
+        Args:
+            events_to_create: List of historical events to create in JSON
+                dictionary format.
+            room_id: Room where you want the events created in.
+            batch_id_to_connect_to: The batch_id from the insertion event you
+                want this batch to connect to.
+            inherited_depth: The depth to create the events at (you will
+                probably by calling inherit_depth_from_prev_ids(...)).
+            initial_state_event_ids:
+                This is used to set explicit state for the insertion event at
+                the start of the historical batch since it's floating with no
+                prev_events to derive state from automatically. This should
+                probably be the state from the `prev_event` defined by
+                `/batch_send?prev_event_id=$abc` plus the outcome of
+                `persist_state_events_at_start`
+            app_service_requester: The requester of an application service.
+
+        Returns:
+            Tuple containing a list of created events and the next_batch_id
+        """
+
+        # Connect this current batch to the insertion event from the previous batch
+        last_event_in_batch = events_to_create[-1]
+        batch_event = {
+            "type": EventTypes.MSC2716_BATCH,
+            "sender": app_service_requester.user.to_string(),
+            "room_id": room_id,
+            "content": {
+                EventContentFields.MSC2716_BATCH_ID: batch_id_to_connect_to,
+                EventContentFields.MSC2716_HISTORICAL: True,
+            },
+            # Since the batch event is put at the end of the batch,
+            # where the newest-in-time event is, copy the origin_server_ts from
+            # the last event we're inserting
+            "origin_server_ts": last_event_in_batch["origin_server_ts"],
+        }
+        # Add the batch event to the end of the batch (newest-in-time)
+        events_to_create.append(batch_event)
+
+        # Add an "insertion" event to the start of each batch (next to the oldest-in-time
+        # event in the batch) so the next batch can be connected to this one.
+        insertion_event = self.create_insertion_event_dict(
+            sender=app_service_requester.user.to_string(),
+            room_id=room_id,
+            # Since the insertion event is put at the start of the batch,
+            # where the oldest-in-time event is, copy the origin_server_ts from
+            # the first event we're inserting
+            origin_server_ts=events_to_create[0]["origin_server_ts"],
+        )
+        next_batch_id = insertion_event["content"][
+            EventContentFields.MSC2716_NEXT_BATCH_ID
+        ]
+        # Prepend the insertion event to the start of the batch (oldest-in-time)
+        events_to_create = [insertion_event] + events_to_create
+
+        # Create and persist all of the historical events
+        event_ids = await self.persist_historical_events(
+            events_to_create=events_to_create,
+            room_id=room_id,
+            inherited_depth=inherited_depth,
+            initial_state_event_ids=initial_state_event_ids,
+            app_service_requester=app_service_requester,
+        )
+
+        return event_ids, next_batch_id

synapse/handlers/room_member.py

@@ -362,6 +362,7 @@ class RoomMemberHandler(metaclass=abc.ABCMeta):
         content: Optional[dict] = None,
         require_consent: bool = True,
         outlier: bool = False,
+        historical: bool = False,
         origin_server_ts: Optional[int] = None,
     ) -> Tuple[str, int]:
         """
@@ -377,13 +378,16 @@ class RoomMemberHandler(metaclass=abc.ABCMeta):
             allow_no_prev_events: Whether to allow this event to be created an empty
                 list of prev_events. Normally this is prohibited just because most
                 events should have a prev_event and we should only use this in special
-                cases (previously useful for MSC2716).
+                cases like MSC2716.
             prev_event_ids: The event IDs to use as the prev events
             state_event_ids:
-                The full state at a given event. This was previously used particularly
-                by the MSC2716 /batch_send endpoint. This should normally be left as
-                None, which will cause the auth_event_ids to be calculated based on the
-                room state at the prev_events.
+                The full state at a given event. This is used particularly by the MSC2716
+                /batch_send endpoint. One use case is the historical `state_events_at_start`;
+                since each is marked as an `outlier`, the `EventContext.for_outlier()` won't
+                have any `state_ids` set and therefore can't derive any state even though the
+                prev_events are set so we need to set them ourself via this argument.
+                This should normally be left as None, which will cause the auth_event_ids
+                to be calculated based on the room state at the prev_events.
             depth: Override the depth used to order the event in the DAG.
                 Should normally be set to None, which will cause the depth to be calculated
                 based on the prev_events.
@@ -396,6 +400,9 @@ class RoomMemberHandler(metaclass=abc.ABCMeta):
             outlier: Indicates whether the event is an `outlier`, i.e. if
                 it's from an arbitrary point and floating in the DAG as
                 opposed to being inline with the current DAG.
+            historical: Indicates whether the message is being inserted
+                back in time around some existing events. This is used to skip
+                a few checks and mark the event as backfilled.
             origin_server_ts: The origin_server_ts to use if a new event is created. Uses
                 the current timestamp if set to None.
 
@@ -470,6 +477,7 @@ class RoomMemberHandler(metaclass=abc.ABCMeta):
                     depth=depth,
                     require_consent=require_consent,
                     outlier=outlier,
+                    historical=historical,
                 )
                 context = await unpersisted_context.persist(event)
                 prev_state_ids = await context.get_prev_state_ids(
@@ -577,6 +585,7 @@ class RoomMemberHandler(metaclass=abc.ABCMeta):
         new_room: bool = False,
         require_consent: bool = True,
         outlier: bool = False,
+        historical: bool = False,
         allow_no_prev_events: bool = False,
         prev_event_ids: Optional[List[str]] = None,
         state_event_ids: Optional[List[str]] = None,
@@ -601,16 +610,22 @@ class RoomMemberHandler(metaclass=abc.ABCMeta):
             outlier: Indicates whether the event is an `outlier`, i.e. if
                 it's from an arbitrary point and floating in the DAG as
                 opposed to being inline with the current DAG.
+            historical: Indicates whether the message is being inserted
+                back in time around some existing events. This is used to skip
+                a few checks and mark the event as backfilled.
             allow_no_prev_events: Whether to allow this event to be created an empty
                 list of prev_events. Normally this is prohibited just because most
                 events should have a prev_event and we should only use this in special
-                cases (previously useful for MSC2716).
+                cases like MSC2716.
             prev_event_ids: The event IDs to use as the prev events
             state_event_ids:
-                The full state at a given event. This was previously used particularly
-                by the MSC2716 /batch_send endpoint. This should normally be left as
-                None, which will cause the auth_event_ids to be calculated based on the
-                room state at the prev_events.
+                The full state at a given event. This is used particularly by the MSC2716
+                /batch_send endpoint. One use case is the historical `state_events_at_start`;
+                since each is marked as an `outlier`, the `EventContext.for_outlier()` won't
+                have any `state_ids` set and therefore can't derive any state even though the
+                prev_events are set so we need to set them ourself via this argument.
+                This should normally be left as None, which will cause the auth_event_ids
+                to be calculated based on the room state at the prev_events.
             depth: Override the depth used to order the event in the DAG.
                 Should normally be set to None, which will cause the depth to be calculated
                 based on the prev_events.
@@ -652,6 +667,7 @@ class RoomMemberHandler(metaclass=abc.ABCMeta):
                         new_room=new_room,
                         require_consent=require_consent,
                         outlier=outlier,
+                        historical=historical,
                         allow_no_prev_events=allow_no_prev_events,
                         prev_event_ids=prev_event_ids,
                         state_event_ids=state_event_ids,
@@ -675,6 +691,7 @@ class RoomMemberHandler(metaclass=abc.ABCMeta):
         new_room: bool = False,
         require_consent: bool = True,
         outlier: bool = False,
+        historical: bool = False,
         allow_no_prev_events: bool = False,
         prev_event_ids: Optional[List[str]] = None,
         state_event_ids: Optional[List[str]] = None,
@@ -701,16 +718,22 @@ class RoomMemberHandler(metaclass=abc.ABCMeta):
             outlier: Indicates whether the event is an `outlier`, i.e. if
                 it's from an arbitrary point and floating in the DAG as
                 opposed to being inline with the current DAG.
+            historical: Indicates whether the message is being inserted
+                back in time around some existing events. This is used to skip
+                a few checks and mark the event as backfilled.
             allow_no_prev_events: Whether to allow this event to be created an empty
                 list of prev_events. Normally this is prohibited just because most
                 events should have a prev_event and we should only use this in special
-                cases (previously useful for MSC2716).
+                cases like MSC2716.
             prev_event_ids: The event IDs to use as the prev events
             state_event_ids:
-                The full state at a given event. This was previously used particularly
-                by the MSC2716 /batch_send endpoint. This should normally be left as
-                None, which will cause the auth_event_ids to be calculated based on the
-                room state at the prev_events.
+                The full state at a given event. This is used particularly by the MSC2716
+                /batch_send endpoint. One use case is the historical `state_events_at_start`;
+                since each is marked as an `outlier`, the `EventContext.for_outlier()` won't
+                have any `state_ids` set and therefore can't derive any state even though the
+                prev_events are set so we need to set them ourself via this argument.
+                This should normally be left as None, which will cause the auth_event_ids
+                to be calculated based on the room state at the prev_events.
             depth: Override the depth used to order the event in the DAG.
                 Should normally be set to None, which will cause the depth to be calculated
                 based on the prev_events.
@@ -854,6 +877,7 @@ class RoomMemberHandler(metaclass=abc.ABCMeta):
                 content=content,
                 require_consent=require_consent,
                 outlier=outlier,
+                historical=historical,
                 origin_server_ts=origin_server_ts,
             )
 
@@ -1474,7 +1498,7 @@ class RoomMemberHandler(metaclass=abc.ABCMeta):
         # put the server which owns the alias at the front of the server list.
         if room_alias.domain in servers:
             servers.remove(room_alias.domain)
-            servers.insert(0, room_alias.domain)
+        servers.insert(0, room_alias.domain)
 
         return RoomID.from_string(room_id), servers
 

synapse/http/federation/matrix_federation_agent.py

@@ -51,10 +51,8 @@ logger = logging.getLogger(__name__)
 @implementer(IAgent)
 class MatrixFederationAgent:
     """An Agent-like thing which provides a `request` method which correctly
-    handles resolving matrix server names when using `matrix-federation://`. Handles
-    standard https URIs as normal. The `matrix-federation://` scheme is internal to
-    Synapse and we purposely want to avoid colliding with the `matrix://` URL scheme
-    which is now specced.
+    handles resolving matrix server names when using matrix://. Handles standard
+    https URIs as normal.
 
     Doesn't implement any retries. (Those are done in MatrixFederationHttpClient.)
 
@@ -169,14 +167,14 @@ class MatrixFederationAgent:
         # There must be a valid hostname.
         assert parsed_uri.hostname
 
-        # If this is a matrix-federation:// URI check if the server has delegated matrix
+        # If this is a matrix:// URI check if the server has delegated matrix
         # traffic using well-known delegation.
         #
         # We have to do this here and not in the endpoint as we need to rewrite
         # the host header with the delegated server name.
         delegated_server = None
         if (
-            parsed_uri.scheme == b"matrix-federation"
+            parsed_uri.scheme == b"matrix"
             and not _is_ip_literal(parsed_uri.hostname)
             and not parsed_uri.port
         ):
@@ -252,7 +250,7 @@ class MatrixHostnameEndpointFactory:
 
 @implementer(IStreamClientEndpoint)
 class MatrixHostnameEndpoint:
-    """An endpoint that resolves matrix-federation:// URLs using Matrix server name
+    """An endpoint that resolves matrix:// URLs using Matrix server name
     resolution (i.e. via SRV). Does not check for well-known delegation.
 
     Args:
@@ -381,7 +379,7 @@ class MatrixHostnameEndpoint:
         connect to.
         """
 
-        if self._parsed_uri.scheme != b"matrix-federation":
+        if self._parsed_uri.scheme != b"matrix":
             return [Server(host=self._parsed_uri.host, port=self._parsed_uri.port)]
 
         # Note: We don't do well-known lookup as that needs to have happened

synapse/http/matrixfederationclient.py

@@ -172,14 +172,7 @@ class MatrixFederationRequest:
 
         # The object is frozen so we can pre-compute this.
         uri = urllib.parse.urlunparse(
-            (
-                b"matrix-federation",
-                destination_bytes,
-                path_bytes,
-                None,
-                query_bytes,
-                b"",
-            )
+            (b"matrix", destination_bytes, path_bytes, None, query_bytes, b"")
         )
         object.__setattr__(self, "uri", uri)
 
@@ -411,14 +404,10 @@ class MatrixFederationHttpClient:
         self.clock = hs.get_clock()
         self._store = hs.get_datastores().main
         self.version_string_bytes = hs.version_string.encode("ascii")
-        self.default_timeout_seconds = hs.config.federation.client_timeout_ms / 1000
+        self.default_timeout = hs.config.federation.client_timeout
 
-        self.max_long_retry_delay_seconds = (
-            hs.config.federation.max_long_retry_delay_ms / 1000
-        )
-        self.max_short_retry_delay_seconds = (
-            hs.config.federation.max_short_retry_delay_ms / 1000
-        )
+        self.max_long_retry_delay = hs.config.federation.max_long_retry_delay
+        self.max_short_retry_delay = hs.config.federation.max_short_retry_delay
         self.max_long_retries = hs.config.federation.max_long_retries
         self.max_short_retries = hs.config.federation.max_short_retries
 
@@ -549,10 +538,10 @@ class MatrixFederationHttpClient:
             logger.exception(f"Invalid destination: {request.destination}.")
             raise FederationDeniedError(request.destination)
 
-        if timeout is not None:
+        if timeout:
             _sec_timeout = timeout / 1000
         else:
-            _sec_timeout = self.default_timeout_seconds
+            _sec_timeout = self.default_timeout
 
         if (
             self.hs.config.federation.federation_domain_whitelist is not None
@@ -744,34 +733,24 @@ class MatrixFederationHttpClient:
 
                     if retries_left and not timeout:
                         if long_retries:
-                            delay_seconds = 4 ** (
-                                self.max_long_retries + 1 - retries_left
-                            )
-                            delay_seconds = min(
-                                delay_seconds, self.max_long_retry_delay_seconds
-                            )
-                            delay_seconds *= random.uniform(0.8, 1.4)
+                            delay = 4 ** (self.max_long_retries + 1 - retries_left)
+                            delay = min(delay, self.max_long_retry_delay)
+                            delay *= random.uniform(0.8, 1.4)
                         else:
-                            delay_seconds = 0.5 * 2 ** (
-                                self.max_short_retries - retries_left
-                            )
-                            delay_seconds = min(
-                                delay_seconds, self.max_short_retry_delay_seconds
-                            )
-                            delay_seconds *= random.uniform(0.8, 1.4)
+                            delay = 0.5 * 2 ** (self.max_short_retries - retries_left)
+                            delay = min(delay, self.max_short_retry_delay)
+                            delay *= random.uniform(0.8, 1.4)
 
                         logger.debug(
                             "{%s} [%s] Waiting %ss before re-sending...",
                             request.txn_id,
                             request.destination,
-                            delay_seconds,
+                            delay,
                         )
 
                         # Sleep for the calculated delay, or wake up immediately
                         # if we get notified that the server is back up.
-                        await self._sleeper.sleep(
-                            request.destination, delay_seconds * 1000
-                        )
+                        await self._sleeper.sleep(request.destination, delay * 1000)
                         retries_left -= 1
                     else:
                         raise
@@ -970,7 +949,7 @@ class MatrixFederationHttpClient:
         if timeout is not None:
             _sec_timeout = timeout / 1000
         else:
-            _sec_timeout = self.default_timeout_seconds
+            _sec_timeout = self.default_timeout
 
         if parser is None:
             parser = cast(ByteParser[T], JsonParser())
@@ -1048,10 +1027,10 @@ class MatrixFederationHttpClient:
             ignore_backoff=ignore_backoff,
         )
 
-        if timeout is not None:
+        if timeout:
             _sec_timeout = timeout / 1000
         else:
-            _sec_timeout = self.default_timeout_seconds
+            _sec_timeout = self.default_timeout
 
         body = await _handle_response(
             self.reactor, _sec_timeout, request, response, start_ms, parser=JsonParser()
@@ -1159,7 +1138,7 @@ class MatrixFederationHttpClient:
         if timeout is not None:
             _sec_timeout = timeout / 1000
         else:
-            _sec_timeout = self.default_timeout_seconds
+            _sec_timeout = self.default_timeout
 
         if parser is None:
             parser = cast(ByteParser[T], JsonParser())
@@ -1235,7 +1214,7 @@ class MatrixFederationHttpClient:
         if timeout is not None:
             _sec_timeout = timeout / 1000
         else:
-            _sec_timeout = self.default_timeout_seconds
+            _sec_timeout = self.default_timeout
 
         body = await _handle_response(
             self.reactor, _sec_timeout, request, response, start_ms, parser=JsonParser()
@@ -1287,7 +1266,7 @@ class MatrixFederationHttpClient:
 
         try:
             d = read_body_with_max_size(response, output_stream, max_size)
-            d.addTimeout(self.default_timeout_seconds, self.reactor)
+            d.addTimeout(self.default_timeout, self.reactor)
             length = await make_deferred_yieldable(d)
         except BodyExceededMaxSize:
             msg = "Requested file is too large > %r bytes" % (max_size,)

synapse/http/site.py

@@ -521,11 +521,6 @@ class SynapseRequest(Request):
         else:
             return self.getClientAddress().host
 
-    def request_info(self) -> "RequestInfo":
-        h = self.getHeader(b"User-Agent")
-        user_agent = h.decode("ascii", "replace") if h else None
-        return RequestInfo(user_agent=user_agent, ip=self.get_client_ip_if_available())
-
 
 class XForwardedForRequest(SynapseRequest):
     """Request object which honours proxy headers
@@ -666,9 +661,3 @@ class SynapseSite(Site):
 
     def log(self, request: SynapseRequest) -> None:
         pass
-
-
-@attr.s(auto_attribs=True, frozen=True, slots=True)
-class RequestInfo:
-    user_agent: Optional[str]
-    ip: str

synapse/media/_base.py

@@ -152,9 +152,6 @@ def add_file_headers(
         content_type = media_type
 
     request.setHeader(b"Content-Type", content_type.encode("UTF-8"))
-
-    # Use a Content-Disposition of attachment to force download of media.
-    disposition = "attachment"
     if upload_name:
         # RFC6266 section 4.1 [1] defines both `filename` and `filename*`.
         #
@@ -176,17 +173,11 @@ def add_file_headers(
         # correctly interpret those as of 0.99.2 and (b) they are a bit of a pain and we
         # may as well just do the filename* version.
         if _can_encode_filename_as_token(upload_name):
-            disposition = "%s; filename=%s" % (
-                disposition,
-                upload_name,
-            )
+            disposition = "inline; filename=%s" % (upload_name,)
         else:
-            disposition = "%s; filename*=utf-8''%s" % (
-                disposition,
-                _quote(upload_name),
-            )
+            disposition = "inline; filename*=utf-8''%s" % (_quote(upload_name),)
 
-    request.setHeader(b"Content-Disposition", disposition.encode("ascii"))
+        request.setHeader(b"Content-Disposition", disposition.encode("ascii"))
 
     # cache for at least a day.
     # XXX: we might want to turn this off for data we don't want to

synapse/module_api/__init__.py

@@ -80,7 +80,6 @@ from synapse.module_api.callbacks.account_validity_callbacks import (
 )
 from synapse.module_api.callbacks.spamchecker_callbacks import (
     CHECK_EVENT_FOR_SPAM_CALLBACK,
-    CHECK_LOGIN_FOR_SPAM_CALLBACK,
     CHECK_MEDIA_FILE_FOR_SPAM_CALLBACK,
     CHECK_REGISTRATION_FOR_SPAM_CALLBACK,
     CHECK_USERNAME_FOR_SPAM_CALLBACK,
@@ -303,7 +302,6 @@ class ModuleApi:
             CHECK_REGISTRATION_FOR_SPAM_CALLBACK
         ] = None,
         check_media_file_for_spam: Optional[CHECK_MEDIA_FILE_FOR_SPAM_CALLBACK] = None,
-        check_login_for_spam: Optional[CHECK_LOGIN_FOR_SPAM_CALLBACK] = None,
     ) -> None:
         """Registers callbacks for spam checking capabilities.
 
@@ -321,7 +319,6 @@ class ModuleApi:
             check_username_for_spam=check_username_for_spam,
             check_registration_for_spam=check_registration_for_spam,
             check_media_file_for_spam=check_media_file_for_spam,
-            check_login_for_spam=check_login_for_spam,
         )
 
     def register_account_validity_callbacks(

synapse/module_api/callbacks/spamchecker_callbacks.py

@@ -196,26 +196,6 @@ CHECK_MEDIA_FILE_FOR_SPAM_CALLBACK = Callable[
         ]
     ],
 ]
-CHECK_LOGIN_FOR_SPAM_CALLBACK = Callable[
-    [
-        str,
-        Optional[str],
-        Optional[str],
-        Collection[Tuple[Optional[str], str]],
-        Optional[str],
-    ],
-    Awaitable[
-        Union[
-            Literal["NOT_SPAM"],
-            Codes,
-            # Highly experimental, not officially part of the spamchecker API, may
-            # disappear without warning depending on the results of ongoing
-            # experiments.
-            # Use this to return additional information as part of an error.
-            Tuple[Codes, JsonDict],
-        ]
-    ],
-]
 
 
 def load_legacy_spam_checkers(hs: "synapse.server.HomeServer") -> None:
@@ -335,7 +315,6 @@ class SpamCheckerModuleApiCallbacks:
         self._check_media_file_for_spam_callbacks: List[
             CHECK_MEDIA_FILE_FOR_SPAM_CALLBACK
         ] = []
-        self._check_login_for_spam_callbacks: List[CHECK_LOGIN_FOR_SPAM_CALLBACK] = []
 
     def register_callbacks(
         self,
@@ -356,7 +335,6 @@ class SpamCheckerModuleApiCallbacks:
             CHECK_REGISTRATION_FOR_SPAM_CALLBACK
         ] = None,
         check_media_file_for_spam: Optional[CHECK_MEDIA_FILE_FOR_SPAM_CALLBACK] = None,
-        check_login_for_spam: Optional[CHECK_LOGIN_FOR_SPAM_CALLBACK] = None,
     ) -> None:
         """Register callbacks from module for each hook."""
         if check_event_for_spam is not None:
@@ -400,9 +378,6 @@ class SpamCheckerModuleApiCallbacks:
         if check_media_file_for_spam is not None:
             self._check_media_file_for_spam_callbacks.append(check_media_file_for_spam)
 
-        if check_login_for_spam is not None:
-            self._check_login_for_spam_callbacks.append(check_login_for_spam)
-
     @trace
     async def check_event_for_spam(
         self, event: "synapse.events.EventBase"
@@ -844,58 +819,3 @@ class SpamCheckerModuleApiCallbacks:
                     return synapse.api.errors.Codes.FORBIDDEN, {}
 
         return self.NOT_SPAM
-
-    async def check_login_for_spam(
-        self,
-        user_id: str,
-        device_id: Optional[str],
-        initial_display_name: Optional[str],
-        request_info: Collection[Tuple[Optional[str], str]],
-        auth_provider_id: Optional[str] = None,
-    ) -> Union[Tuple[Codes, dict], Literal["NOT_SPAM"]]:
-        """Checks if we should allow the given registration request.
-
-        Args:
-            user_id: The request user ID
-            request_info: List of tuples of user agent and IP that
-                were used during the registration process.
-            auth_provider_id: The SSO IdP the user used, e.g "oidc", "saml",
-                "cas". If any. Note this does not include users registered
-                via a password provider.
-
-        Returns:
-            Enum for how the request should be handled
-        """
-
-        for callback in self._check_login_for_spam_callbacks:
-            with Measure(
-                self.clock, "{}.{}".format(callback.__module__, callback.__qualname__)
-            ):
-                res = await delay_cancellation(
-                    callback(
-                        user_id,
-                        device_id,
-                        initial_display_name,
-                        request_info,
-                        auth_provider_id,
-                    )
-                )
-                # Normalize return values to `Codes` or `"NOT_SPAM"`.
-                if res is self.NOT_SPAM:
-                    continue
-                elif isinstance(res, synapse.api.errors.Codes):
-                    return res, {}
-                elif (
-                    isinstance(res, tuple)
-                    and len(res) == 2
-                    and isinstance(res[0], synapse.api.errors.Codes)
-                    and isinstance(res[1], dict)
-                ):
-                    return res
-                else:
-                    logger.warning(
-                        "Module returned invalid value, rejecting login as spam"
-                    )
-                    return synapse.api.errors.Codes.FORBIDDEN, {}
-
-        return self.NOT_SPAM

synapse/push/bulk_push_rule_evaluator.py

@@ -322,6 +322,7 @@ class BulkPushRuleEvaluator:
     ) -> None:
         if (
             not event.internal_metadata.is_notifiable()
+            or event.internal_metadata.is_historical()
             or event.room_id in self.hs.config.server.rooms_to_exclude_from_sync
         ):
             # Push rules for events that aren't notifiable can't be processed by this and

synapse/push/clientformat.py

@@ -41,7 +41,12 @@ def format_push_rules_for_user(
 
         rulearray.append(template_rule)
 
-        _convert_type_to_value(template_rule, user)
+        for type_key in ("pattern", "value"):
+            type_value = template_rule.pop(f"{type_key}_type", None)
+            if type_value == "user_id":
+                template_rule[type_key] = user.to_string()
+            elif type_value == "user_localpart":
+                template_rule[type_key] = user.localpart
 
         template_rule["enabled"] = enabled
 
@@ -58,20 +63,19 @@ def format_push_rules_for_user(
         for c in template_rule["conditions"]:
             c.pop("_cache_key", None)
 
-            _convert_type_to_value(c, user)
+            pattern_type = c.pop("pattern_type", None)
+            if pattern_type == "user_id":
+                c["pattern"] = user.to_string()
+            elif pattern_type == "user_localpart":
+                c["pattern"] = user.localpart
+
+            sender_type = c.pop("sender_type", None)
+            if sender_type == "user_id":
+                c["sender"] = user.to_string()
 
     return rules
 
 
-def _convert_type_to_value(rule_or_cond: Dict[str, Any], user: UserID) -> None:
-    for type_key in ("pattern", "value"):
-        type_value = rule_or_cond.pop(f"{type_key}_type", None)
-        if type_value == "user_id":
-            rule_or_cond[type_key] = user.to_string()
-        elif type_value == "user_localpart":
-            rule_or_cond[type_key] = user.localpart
-
-
 def _add_empty_priority_class_arrays(d: Dict[str, list]) -> Dict[str, list]:
     for pc in PRIORITY_CLASS_MAP.keys():
         d[pc] = []

synapse/rest/__init__.py

@@ -48,6 +48,7 @@ from synapse.rest.client import (
     rendezvous,
     report_event,
     room,
+    room_batch,
     room_keys,
     room_upgrade_rest_servlet,
     sendtodevice,
@@ -131,6 +132,7 @@ class ClientRestResource(JsonResource):
         user_directory.register_servlets(hs, client_resource)
         if is_main_process:
             room_upgrade_rest_servlet.register_servlets(hs, client_resource)
+        room_batch.register_servlets(hs, client_resource)
         capabilities.register_servlets(hs, client_resource)
         if is_main_process:
             account_validity.register_servlets(hs, client_resource)

synapse/rest/client/login.py

@@ -50,7 +50,7 @@ from synapse.http.servlet import (
     parse_json_object_from_request,
     parse_string,
 )
-from synapse.http.site import RequestInfo, SynapseRequest
+from synapse.http.site import SynapseRequest
 from synapse.rest.client._base import client_patterns
 from synapse.rest.well_known import WellKnownBuilder
 from synapse.types import JsonDict, UserID
@@ -114,7 +114,6 @@ class LoginRestServlet(RestServlet):
         self.auth_handler = self.hs.get_auth_handler()
         self.registration_handler = hs.get_registration_handler()
         self._sso_handler = hs.get_sso_handler()
-        self._spam_checker = hs.get_module_api_callbacks().spam_checker
 
         self._well_known_builder = WellKnownBuilder(hs)
         self._address_ratelimiter = Ratelimiter(
@@ -198,8 +197,6 @@ class LoginRestServlet(RestServlet):
             self._refresh_tokens_enabled and client_requested_refresh_token
         )
 
-        request_info = request.request_info()
-
         try:
             if login_submission["type"] == LoginRestServlet.APPSERVICE_TYPE:
                 requester = await self.auth.get_user_by_req(request)
@@ -219,7 +216,6 @@ class LoginRestServlet(RestServlet):
                     login_submission,
                     appservice,
                     should_issue_refresh_token=should_issue_refresh_token,
-                    request_info=request_info,
                 )
             elif (
                 self.jwt_enabled
@@ -231,7 +227,6 @@ class LoginRestServlet(RestServlet):
                 result = await self._do_jwt_login(
                     login_submission,
                     should_issue_refresh_token=should_issue_refresh_token,
-                    request_info=request_info,
                 )
             elif login_submission["type"] == LoginRestServlet.TOKEN_TYPE:
                 await self._address_ratelimiter.ratelimit(
@@ -240,7 +235,6 @@ class LoginRestServlet(RestServlet):
                 result = await self._do_token_login(
                     login_submission,
                     should_issue_refresh_token=should_issue_refresh_token,
-                    request_info=request_info,
                 )
             else:
                 await self._address_ratelimiter.ratelimit(
@@ -249,7 +243,6 @@ class LoginRestServlet(RestServlet):
                 result = await self._do_other_login(
                     login_submission,
                     should_issue_refresh_token=should_issue_refresh_token,
-                    request_info=request_info,
                 )
         except KeyError:
             raise SynapseError(400, "Missing JSON keys.")
@@ -272,8 +265,6 @@ class LoginRestServlet(RestServlet):
         login_submission: JsonDict,
         appservice: ApplicationService,
         should_issue_refresh_token: bool = False,
-        *,
-        request_info: RequestInfo,
     ) -> LoginResponse:
         identifier = login_submission.get("identifier")
         logger.info("Got appservice login request with identifier: %r", identifier)
@@ -309,15 +300,10 @@ class LoginRestServlet(RestServlet):
             # The user represented by an appservice's configured sender_localpart
             # is not actually created in Synapse.
             should_check_deactivated=qualified_user_id != appservice.sender,
-            request_info=request_info,
         )
 
     async def _do_other_login(
-        self,
-        login_submission: JsonDict,
-        should_issue_refresh_token: bool = False,
-        *,
-        request_info: RequestInfo,
+        self, login_submission: JsonDict, should_issue_refresh_token: bool = False
     ) -> LoginResponse:
         """Handle non-token/saml/jwt logins
 
@@ -347,7 +333,6 @@ class LoginRestServlet(RestServlet):
             login_submission,
             callback,
             should_issue_refresh_token=should_issue_refresh_token,
-            request_info=request_info,
         )
         return result
 
@@ -362,8 +347,6 @@ class LoginRestServlet(RestServlet):
         should_issue_refresh_token: bool = False,
         auth_provider_session_id: Optional[str] = None,
         should_check_deactivated: bool = True,
-        *,
-        request_info: RequestInfo,
     ) -> LoginResponse:
         """Called when we've successfully authed the user and now need to
         actually login them in (e.g. create devices). This gets called on
@@ -388,7 +371,6 @@ class LoginRestServlet(RestServlet):
 
                 This exists purely for appservice's configured sender_localpart
                 which doesn't have an associated user in the database.
-            request_info: The user agent/IP address of the user.
 
         Returns:
             Dictionary of account information after successful login.
@@ -435,22 +417,6 @@ class LoginRestServlet(RestServlet):
                 )
 
         initial_display_name = login_submission.get("initial_device_display_name")
-        spam_check = await self._spam_checker.check_login_for_spam(
-            user_id,
-            device_id=device_id,
-            initial_display_name=initial_display_name,
-            request_info=[(request_info.user_agent, request_info.ip)],
-            auth_provider_id=auth_provider_id,
-        )
-        if spam_check != self._spam_checker.NOT_SPAM:
-            logger.info("Blocking login due to spam checker")
-            raise SynapseError(
-                403,
-                msg="Login was blocked by the server",
-                errcode=spam_check[0],
-                additional_fields=spam_check[1],
-            )
-
         (
             device_id,
             access_token,
@@ -485,11 +451,7 @@ class LoginRestServlet(RestServlet):
         return result
 
     async def _do_token_login(
-        self,
-        login_submission: JsonDict,
-        should_issue_refresh_token: bool = False,
-        *,
-        request_info: RequestInfo,
+        self, login_submission: JsonDict, should_issue_refresh_token: bool = False
     ) -> LoginResponse:
         """
         Handle token login.
@@ -512,15 +474,10 @@ class LoginRestServlet(RestServlet):
             auth_provider_id=res.auth_provider_id,
             should_issue_refresh_token=should_issue_refresh_token,
             auth_provider_session_id=res.auth_provider_session_id,
-            request_info=request_info,
         )
 
     async def _do_jwt_login(
-        self,
-        login_submission: JsonDict,
-        should_issue_refresh_token: bool = False,
-        *,
-        request_info: RequestInfo,
+        self, login_submission: JsonDict, should_issue_refresh_token: bool = False
     ) -> LoginResponse:
         """
         Handle the custom JWT login.
@@ -539,7 +496,6 @@ class LoginRestServlet(RestServlet):
             login_submission,
             create_non_existent_users=True,
             should_issue_refresh_token=should_issue_refresh_token,
-            request_info=request_info,
         )
 
 

synapse/rest/client/room_batch.py

@@ -0,0 +1,254 @@
+# Copyright 2016 OpenMarket Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import logging
+import re
+from http import HTTPStatus
+from typing import TYPE_CHECKING, Tuple
+
+from synapse.api.constants import EventContentFields
+from synapse.api.errors import AuthError, Codes, SynapseError
+from synapse.http.server import HttpServer
+from synapse.http.servlet import (
+    RestServlet,
+    assert_params_in_dict,
+    parse_json_object_from_request,
+    parse_string,
+    parse_strings_from_args,
+)
+from synapse.http.site import SynapseRequest
+from synapse.types import JsonDict
+
+if TYPE_CHECKING:
+    from synapse.server import HomeServer
+
+logger = logging.getLogger(__name__)
+
+
+class RoomBatchSendEventRestServlet(RestServlet):
+    """
+    API endpoint which can insert a batch of events historically back in time
+    next to the given `prev_event`.
+
+    `batch_id` comes from `next_batch_id `in the response of the batch send
+    endpoint and is derived from the "insertion" events added to each batch.
+    It's not required for the first batch send.
+
+    `state_events_at_start` is used to define the historical state events
+    needed to auth the events like join events. These events will float
+    outside of the normal DAG as outlier's and won't be visible in the chat
+    history which also allows us to insert multiple batches without having a bunch
+    of `@mxid joined the room` noise between each batch.
+
+    `events` is chronological list of events you want to insert.
+    There is a reverse-chronological constraint on batches so once you insert
+    some messages, you can only insert older ones after that.
+    tldr; Insert batches from your most recent history -> oldest history.
+
+    POST /_matrix/client/unstable/org.matrix.msc2716/rooms/<roomID>/batch_send?prev_event_id=<eventID>&batch_id=<batchID>
+    {
+        "events": [ ... ],
+        "state_events_at_start": [ ... ]
+    }
+    """
+
+    PATTERNS = (
+        re.compile(
+            "^/_matrix/client/unstable/org.matrix.msc2716"
+            "/rooms/(?P<room_id>[^/]*)/batch_send$"
+        ),
+    )
+    CATEGORY = "Client API requests"
+
+    def __init__(self, hs: "HomeServer"):
+        super().__init__()
+        self.store = hs.get_datastores().main
+        self.event_creation_handler = hs.get_event_creation_handler()
+        self.auth = hs.get_auth()
+        self.room_batch_handler = hs.get_room_batch_handler()
+
+    async def on_POST(
+        self, request: SynapseRequest, room_id: str
+    ) -> Tuple[int, JsonDict]:
+        requester = await self.auth.get_user_by_req(request, allow_guest=False)
+
+        if not requester.app_service:
+            raise AuthError(
+                HTTPStatus.FORBIDDEN,
+                "Only application services can use the /batchsend endpoint",
+            )
+
+        body = parse_json_object_from_request(request)
+        assert_params_in_dict(body, ["state_events_at_start", "events"])
+
+        assert request.args is not None
+        prev_event_ids_from_query = parse_strings_from_args(
+            request.args, "prev_event_id"
+        )
+        batch_id_from_query = parse_string(request, "batch_id")
+
+        if prev_event_ids_from_query is None:
+            raise SynapseError(
+                HTTPStatus.BAD_REQUEST,
+                "prev_event query parameter is required when inserting historical messages back in time",
+                errcode=Codes.MISSING_PARAM,
+            )
+
+        if await self.store.is_partial_state_room(room_id):
+            raise SynapseError(
+                HTTPStatus.BAD_REQUEST,
+                "Cannot insert history batches until we have fully joined the room",
+                errcode=Codes.UNABLE_DUE_TO_PARTIAL_STATE,
+            )
+
+        # Verify the batch_id_from_query corresponds to an actual insertion event
+        # and have the batch connected.
+        if batch_id_from_query:
+            corresponding_insertion_event_id = (
+                await self.store.get_insertion_event_id_by_batch_id(
+                    room_id, batch_id_from_query
+                )
+            )
+            if corresponding_insertion_event_id is None:
+                raise SynapseError(
+                    HTTPStatus.BAD_REQUEST,
+                    "No insertion event corresponds to the given ?batch_id",
+                    errcode=Codes.INVALID_PARAM,
+                )
+
+        # Make sure that the prev_event_ids exist and aren't outliers - ie, they are
+        # regular parts of the room DAG where we know the state.
+        non_outlier_prev_events = await self.store.have_events_in_timeline(
+            prev_event_ids_from_query
+        )
+        for prev_event_id in prev_event_ids_from_query:
+            if prev_event_id not in non_outlier_prev_events:
+                raise SynapseError(
+                    HTTPStatus.BAD_REQUEST,
+                    "prev_event %s does not exist, or is an outlier" % (prev_event_id,),
+                    errcode=Codes.INVALID_PARAM,
+                )
+
+        # For the event we are inserting next to (`prev_event_ids_from_query`),
+        # find the most recent state events that allowed that message to be
+        # sent. We will use that as a base to auth our historical messages
+        # against.
+        state_event_ids = await self.room_batch_handler.get_most_recent_full_state_ids_from_event_id_list(
+            prev_event_ids_from_query
+        )
+
+        state_event_ids_at_start = []
+        # Create and persist all of the state events that float off on their own
+        # before the batch. These will most likely be all of the invite/member
+        # state events used to auth the upcoming historical messages.
+        if body["state_events_at_start"]:
+            state_event_ids_at_start = (
+                await self.room_batch_handler.persist_state_events_at_start(
+                    state_events_at_start=body["state_events_at_start"],
+                    room_id=room_id,
+                    initial_state_event_ids=state_event_ids,
+                    app_service_requester=requester,
+                )
+            )
+            # Update our ongoing auth event ID list with all of the new state we
+            # just created
+            state_event_ids.extend(state_event_ids_at_start)
+
+        inherited_depth = await self.room_batch_handler.inherit_depth_from_prev_ids(
+            prev_event_ids_from_query
+        )
+
+        events_to_create = body["events"]
+
+        # Figure out which batch to connect to. If they passed in
+        # batch_id_from_query let's use it. The batch ID passed in comes
+        # from the batch_id in the "insertion" event from the previous batch.
+        last_event_in_batch = events_to_create[-1]
+        base_insertion_event = None
+        if batch_id_from_query:
+            batch_id_to_connect_to = batch_id_from_query
+        # Otherwise, create an insertion event to act as a starting point.
+        #
+        # We don't always have an insertion event to start hanging more history
+        # off of (ideally there would be one in the main DAG, but that's not the
+        # case if we're wanting to add history to e.g. existing rooms without
+        # an insertion event), in which case we just create a new insertion event
+        # that can then get pointed to by a "marker" event later.
+        else:
+            base_insertion_event_dict = (
+                self.room_batch_handler.create_insertion_event_dict(
+                    sender=requester.user.to_string(),
+                    room_id=room_id,
+                    origin_server_ts=last_event_in_batch["origin_server_ts"],
+                )
+            )
+            base_insertion_event_dict["prev_events"] = prev_event_ids_from_query.copy()
+
+            (
+                base_insertion_event,
+                _,
+            ) = await self.event_creation_handler.create_and_send_nonmember_event(
+                await self.room_batch_handler.create_requester_for_user_id_from_app_service(
+                    base_insertion_event_dict["sender"],
+                    requester.app_service,
+                ),
+                base_insertion_event_dict,
+                prev_event_ids=base_insertion_event_dict.get("prev_events"),
+                # Also set the explicit state here because we want to resolve
+                # any `state_events_at_start` here too. It's not strictly
+                # necessary to accomplish anything but if someone asks for the
+                # state at this point, we probably want to show them the
+                # historical state that was part of this batch.
+                state_event_ids=state_event_ids,
+                historical=True,
+                depth=inherited_depth,
+            )
+
+            batch_id_to_connect_to = base_insertion_event.content[
+                EventContentFields.MSC2716_NEXT_BATCH_ID
+            ]
+
+        # Create and persist all of the historical events as well as insertion
+        # and batch meta events to make the batch navigable in the DAG.
+        event_ids, next_batch_id = await self.room_batch_handler.handle_batch_of_events(
+            events_to_create=events_to_create,
+            room_id=room_id,
+            batch_id_to_connect_to=batch_id_to_connect_to,
+            inherited_depth=inherited_depth,
+            initial_state_event_ids=state_event_ids,
+            app_service_requester=requester,
+        )
+
+        insertion_event_id = event_ids[0]
+        batch_event_id = event_ids[-1]
+        historical_event_ids = event_ids[1:-1]
+
+        response_dict = {
+            "state_event_ids": state_event_ids_at_start,
+            "event_ids": historical_event_ids,
+            "next_batch_id": next_batch_id,
+            "insertion_event_id": insertion_event_id,
+            "batch_event_id": batch_event_id,
+        }
+        if base_insertion_event is not None:
+            response_dict["base_insertion_event_id"] = base_insertion_event.event_id
+
+        return HTTPStatus.OK, response_dict
+
+
+def register_servlets(hs: "HomeServer", http_server: HttpServer) -> None:
+    msc2716_enabled = hs.config.experimental.msc2716_enabled
+
+    if msc2716_enabled:
+        RoomBatchSendEventRestServlet(hs).register(http_server)

synapse/rest/client/versions.py

@@ -102,6 +102,8 @@ class VersionsRestServlet(RestServlet):
                     "org.matrix.msc2285.stable": True,  # TODO: Remove when MSC2285 becomes a part of the spec
                     # Supports filtering of /publicRooms by room type as per MSC3827
                     "org.matrix.msc3827.stable": True,
+                    # Adds support for importing historical messages as per MSC2716
+                    "org.matrix.msc2716": self.config.experimental.msc2716_enabled,
                     # Adds support for thread relations, per MSC3440.
                     "org.matrix.msc3440.stable": True,  # TODO: remove when "v1.3" is added above
                     # Support for thread read receipts & notification counts.

synapse/server.py

@@ -91,6 +91,7 @@ from synapse.handlers.room import (
     RoomShutdownHandler,
     TimestampLookupHandler,
 )
+from synapse.handlers.room_batch import RoomBatchHandler
 from synapse.handlers.room_list import RoomListHandler
 from synapse.handlers.room_member import (
     RoomForgetterHandler,
@@ -491,6 +492,10 @@ class HomeServer(metaclass=abc.ABCMeta):
     def get_room_creation_handler(self) -> RoomCreationHandler:
         return RoomCreationHandler(self)
 
+    @cache_in_self
+    def get_room_batch_handler(self) -> RoomBatchHandler:
+        return RoomBatchHandler(self)
+
     @cache_in_self
     def get_room_shutdown_handler(self) -> RoomShutdownHandler:
         return RoomShutdownHandler(self)

synapse/storage/controllers/persist_events.py

@@ -839,8 +839,9 @@ class EventsPersistenceStorageController:
                         "group" % (ev.event_id,)
                     )
                 continue
-            if ctx.state_group_deltas:
-                state_group_deltas.update(ctx.state_group_deltas)
+
+            if ctx.prev_group:
+                state_group_deltas[(ctx.prev_group, ctx.state_group)] = ctx.delta_ids
 
         # We need to map the event_ids to their state groups. First, let's
         # check if the event is one we're persisting, in which case we can

synapse/storage/database.py

@@ -1529,7 +1529,7 @@ class DatabasePool:
         # Lock the table just once, to prevent it being done once per row.
         # Note that, according to Postgres' documentation, once obtained,
         # the lock is held for the remainder of the current transaction.
-        self.engine.lock_table(txn, table)
+        self.engine.lock_table(txn, "user_ips")
 
         for keyv, valv in zip(key_values, value_values):
             _keys = dict(zip(key_names, keyv))

synapse/storage/databases/main/__init__.py

@@ -61,6 +61,7 @@ from .registration import RegistrationStore
 from .rejections import RejectionsStore
 from .relations import RelationsStore
 from .room import RoomStore
+from .room_batch import RoomBatchStore
 from .roommember import RoomMemberStore
 from .search import SearchStore
 from .session import SessionStore
@@ -86,6 +87,7 @@ class DataStore(
     DeviceStore,
     RoomMemberStore,
     RoomStore,
+    RoomBatchStore,
     RegistrationStore,
     ProfileStore,
     PresenceStore,

synapse/storage/databases/main/cache.py

@@ -289,17 +289,6 @@ class CacheInvalidationWorkerStore(SQLBaseStore):
             )
             self._attempt_to_invalidate_cache("get_rooms_for_user", (state_key,))
 
-            self._attempt_to_invalidate_cache(
-                "did_forget",
-                (
-                    state_key,
-                    room_id,
-                ),
-            )
-            self._attempt_to_invalidate_cache(
-                "get_forgotten_rooms_for_user", (state_key,)
-            )
-
         if relates_to:
             self._attempt_to_invalidate_cache("get_relations_for_event", (relates_to,))
             self._attempt_to_invalidate_cache("get_references_for_event", (relates_to,))
@@ -347,8 +336,6 @@ class CacheInvalidationWorkerStore(SQLBaseStore):
             "get_rooms_for_user_with_stream_ordering", None
         )
         self._attempt_to_invalidate_cache("get_rooms_for_user", None)
-        self._attempt_to_invalidate_cache("did_forget", None)
-        self._attempt_to_invalidate_cache("get_forgotten_rooms_for_user", None)
         self._attempt_to_invalidate_cache("get_references_for_event", None)
         self._attempt_to_invalidate_cache("get_thread_summary", None)
         self._attempt_to_invalidate_cache("get_thread_participated", None)

synapse/storage/databases/main/event_federation.py

@@ -31,7 +31,7 @@ from typing import (
 import attr
 from prometheus_client import Counter, Gauge
 
-from synapse.api.constants import MAX_DEPTH
+from synapse.api.constants import MAX_DEPTH, EventTypes
 from synapse.api.errors import StoreError
 from synapse.api.room_versions import EventFormatVersions, RoomVersion
 from synapse.events import EventBase, make_event_from_dict
@@ -891,6 +891,124 @@ class EventFederationWorkerStore(SignatureWorkerStore, EventsWorkerStore, SQLBas
             room_id,
         )
 
+    @trace
+    async def get_insertion_event_backward_extremities_in_room(
+        self,
+        room_id: str,
+        current_depth: int,
+        limit: int,
+    ) -> List[Tuple[str, int]]:
+        """
+        Get the insertion events we know about that we haven't backfilled yet
+        along with the approximate depth. Only returns insertion events that are
+        at a depth lower than or equal to the `current_depth`. Sorted by depth,
+        highest to lowest (descending) so the closest events to the
+        `current_depth` are first in the list.
+
+        We ignore insertion events that are newer than the user's current scroll
+        position (ie, those with depth greater than `current_depth`) as:
+            1. we don't really care about getting events that have happened
+               after our current position; and
+            2. by the nature of paginating and scrolling back, we have likely
+               previously tried and failed to backfill from that insertion event, so
+               to avoid getting "stuck" requesting the same backfill repeatedly
+               we drop those insertion event.
+
+        Args:
+            room_id: Room where we want to find the oldest events
+            current_depth: The depth at the user's current scrollback position
+            limit: The max number of insertion event extremities to return
+
+        Returns:
+            List of (event_id, depth) tuples. Sorted by depth, highest to lowest
+            (descending) so the closest events to the `current_depth` are first
+            in the list.
+        """
+
+        def get_insertion_event_backward_extremities_in_room_txn(
+            txn: LoggingTransaction, room_id: str
+        ) -> List[Tuple[str, int]]:
+            if isinstance(self.database_engine, PostgresEngine):
+                least_function = "LEAST"
+            elif isinstance(self.database_engine, Sqlite3Engine):
+                least_function = "MIN"
+            else:
+                raise RuntimeError("Unknown database engine")
+
+            sql = f"""
+                SELECT
+                    insertion_event_extremity.event_id, event.depth
+                /* We only want insertion events that are also marked as backwards extremities */
+                FROM insertion_event_extremities AS insertion_event_extremity
+                /* Get the depth of the insertion event from the events table */
+                INNER JOIN events AS event USING (event_id)
+                /**
+                 * We use this info to make sure we don't retry to use a backfill point
+                 * if we've already attempted to backfill from it recently.
+                 */
+                LEFT JOIN event_failed_pull_attempts AS failed_backfill_attempt_info
+                ON
+                    failed_backfill_attempt_info.room_id = insertion_event_extremity.room_id
+                    AND failed_backfill_attempt_info.event_id = insertion_event_extremity.event_id
+                WHERE
+                    insertion_event_extremity.room_id = ?
+                    /**
+                     * We only want extremities that are older than or at
+                     * the same position of the given `current_depth` (where older
+                     * means less than the given depth) because we're looking backwards
+                     * from the `current_depth` when backfilling.
+                     *
+                     *                         current_depth (ignore events that come after this, ignore 2-4)
+                     *                         |
+                     *                         ▼
+                     * <oldest-in-time> [0]<--[1]<--[2]<--[3]<--[4] <newest-in-time>
+                     */
+                    AND event.depth <= ? /* current_depth */
+                    /**
+                     * Exponential back-off (up to the upper bound) so we don't retry the
+                     * same backfill point over and over. ex. 2hr, 4hr, 8hr, 16hr, etc
+                     *
+                     * We use `1 << n` as a power of 2 equivalent for compatibility
+                     * with older SQLites. The left shift equivalent only works with
+                     * powers of 2 because left shift is a binary operation (base-2).
+                     * Otherwise, we would use `power(2, n)` or the power operator, `2^n`.
+                     */
+                    AND (
+                        failed_backfill_attempt_info.event_id IS NULL
+                        OR ? /* current_time */ >= failed_backfill_attempt_info.last_attempt_ts + (
+                            (1 << {least_function}(failed_backfill_attempt_info.num_attempts, ? /* max doubling steps */))
+                            * ? /* step */
+                        )
+                    )
+                /**
+                 * Sort from highest (closest to the `current_depth`) to the lowest depth
+                 * because the closest are most relevant to backfill from first.
+                 * Then tie-break on alphabetical order of the event_ids so we get a
+                 * consistent ordering which is nice when asserting things in tests.
+                 */
+                ORDER BY event.depth DESC, insertion_event_extremity.event_id DESC
+                LIMIT ?
+            """
+
+            txn.execute(
+                sql,
+                (
+                    room_id,
+                    current_depth,
+                    self._clock.time_msec(),
+                    BACKFILL_EVENT_EXPONENTIAL_BACKOFF_MAXIMUM_DOUBLING_STEPS,
+                    BACKFILL_EVENT_EXPONENTIAL_BACKOFF_STEP_MILLISECONDS,
+                    limit,
+                ),
+            )
+            return cast(List[Tuple[str, int]], txn.fetchall())
+
+        return await self.db_pool.runInteraction(
+            "get_insertion_event_backward_extremities_in_room",
+            get_insertion_event_backward_extremities_in_room_txn,
+            room_id,
+        )
+
     async def get_max_depth_of(
         self, event_ids: Collection[str]
     ) -> Tuple[Optional[str], int]:
@@ -1162,6 +1280,50 @@ class EventFederationWorkerStore(SignatureWorkerStore, EventsWorkerStore, SQLBas
 
         return event_ids
 
+    def _get_connected_batch_event_backfill_results_txn(
+        self, txn: LoggingTransaction, insertion_event_id: str, limit: int
+    ) -> List[BackfillQueueNavigationItem]:
+        """
+        Find any batch connections of a given insertion event.
+        A batch event points at a insertion event via:
+        batch_event.content[MSC2716_BATCH_ID] -> insertion_event.content[MSC2716_NEXT_BATCH_ID]
+
+        Args:
+            txn: The database transaction to use
+            insertion_event_id: The event ID to navigate from. We will find
+                batch events that point back at this insertion event.
+            limit: Max number of event ID's to query for and return
+
+        Returns:
+            List of batch events that the backfill queue can process
+        """
+        batch_connection_query = """
+            SELECT e.depth, e.stream_ordering, c.event_id, e.type FROM insertion_events AS i
+            /* Find the batch that connects to the given insertion event */
+            INNER JOIN batch_events AS c
+            ON i.next_batch_id = c.batch_id
+            /* Get the depth of the batch start event from the events table */
+            INNER JOIN events AS e ON c.event_id = e.event_id
+            /* Find an insertion event which matches the given event_id */
+            WHERE i.event_id = ?
+            LIMIT ?
+        """
+
+        # Find any batch connections for the given insertion event
+        txn.execute(
+            batch_connection_query,
+            (insertion_event_id, limit),
+        )
+        return [
+            BackfillQueueNavigationItem(
+                depth=row[0],
+                stream_ordering=row[1],
+                event_id=row[2],
+                type=row[3],
+            )
+            for row in txn
+        ]
+
     def _get_connected_prev_event_backfill_results_txn(
         self, txn: LoggingTransaction, event_id: str, limit: int
     ) -> List[BackfillQueueNavigationItem]:
@@ -1310,6 +1472,40 @@ class EventFederationWorkerStore(SignatureWorkerStore, EventsWorkerStore, SQLBas
 
             event_id_results.add(event_id)
 
+            # Try and find any potential historical batches of message history.
+            if self.hs.config.experimental.msc2716_enabled:
+                # We need to go and try to find any batch events connected
+                # to a given insertion event (by batch_id). If we find any, we'll
+                # add them to the queue and navigate up the DAG like normal in the
+                # next iteration of the loop.
+                if event_type == EventTypes.MSC2716_INSERTION:
+                    # Find any batch connections for the given insertion event
+                    connected_batch_event_backfill_results = (
+                        self._get_connected_batch_event_backfill_results_txn(
+                            txn, event_id, limit - len(event_id_results)
+                        )
+                    )
+                    logger.debug(
+                        "_get_backfill_events(room_id=%s): connected_batch_event_backfill_results=%s",
+                        room_id,
+                        connected_batch_event_backfill_results,
+                    )
+                    for (
+                        connected_batch_event_backfill_item
+                    ) in connected_batch_event_backfill_results:
+                        if (
+                            connected_batch_event_backfill_item.event_id
+                            not in event_id_results
+                        ):
+                            queue.put(
+                                (
+                                    -connected_batch_event_backfill_item.depth,
+                                    -connected_batch_event_backfill_item.stream_ordering,
+                                    connected_batch_event_backfill_item.event_id,
+                                    connected_batch_event_backfill_item.type,
+                                )
+                            )
+
             # Now we just look up the DAG by prev_events as normal
             connected_prev_event_backfill_results = (
                 self._get_connected_prev_event_backfill_results_txn(
@@ -1552,6 +1748,19 @@ class EventFederationWorkerStore(SignatureWorkerStore, EventsWorkerStore, SQLBas
             _delete_old_forward_extrem_cache_txn,
         )
 
+    @trace
+    async def insert_insertion_extremity(self, event_id: str, room_id: str) -> None:
+        await self.db_pool.simple_upsert(
+            table="insertion_event_extremities",
+            keyvalues={"event_id": event_id},
+            values={
+                "event_id": event_id,
+                "room_id": room_id,
+            },
+            insertion_values={},
+            desc="insert_insertion_extremity",
+        )
+
     async def insert_received_event_to_staging(
         self, origin: str, event: EventBase
     ) -> None:

synapse/storage/databases/main/events.py

@@ -1664,6 +1664,9 @@ class PersistEventsStore:
 
             self._handle_event_relations(txn, event)
 
+            self._handle_insertion_event(txn, event)
+            self._handle_batch_event(txn, event)
+
             # Store the labels for this event.
             labels = event.content.get(EventContentFields.LABELS)
             if labels:
@@ -1726,22 +1729,13 @@ class PersistEventsStore:
             if not row["rejects"] and not row["redacts"]:
                 to_prefill.append(EventCacheEntry(event=event, redacted_event=None))
 
-        async def external_prefill() -> None:
+        async def prefill() -> None:
             for cache_entry in to_prefill:
-                await self.store._get_event_cache.set_external(
+                await self.store._get_event_cache.set(
                     (cache_entry.event.event_id,), cache_entry
                 )
 
-        def local_prefill() -> None:
-            for cache_entry in to_prefill:
-                self.store._get_event_cache.set_local(
-                    (cache_entry.event.event_id,), cache_entry
-                )
-
-        # The order these are called here is not as important as knowing that after the
-        # transaction is finished, the async_call_after will run before the call_after.
-        txn.async_call_after(external_prefill)
-        txn.call_after(local_prefill)
+        txn.async_call_after(prefill)
 
     def _store_redaction(self, txn: LoggingTransaction, event: EventBase) -> None:
         assert event.redacts is not None
@@ -1924,6 +1918,128 @@ class PersistEventsStore:
                 ),
             )
 
+    def _handle_insertion_event(
+        self, txn: LoggingTransaction, event: EventBase
+    ) -> None:
+        """Handles keeping track of insertion events and edges/connections.
+        Part of MSC2716.
+
+        Args:
+            txn: The database transaction object
+            event: The event to process
+        """
+
+        if event.type != EventTypes.MSC2716_INSERTION:
+            # Not a insertion event
+            return
+
+        # Skip processing an insertion event if the room version doesn't
+        # support it or the event is not from the room creator.
+        room_version = self.store.get_room_version_txn(txn, event.room_id)
+        room_creator = self.db_pool.simple_select_one_onecol_txn(
+            txn,
+            table="rooms",
+            keyvalues={"room_id": event.room_id},
+            retcol="creator",
+            allow_none=True,
+        )
+        if not room_version.msc2716_historical and (
+            not self.hs.config.experimental.msc2716_enabled
+            or event.sender != room_creator
+        ):
+            return
+
+        next_batch_id = event.content.get(EventContentFields.MSC2716_NEXT_BATCH_ID)
+        if next_batch_id is None:
+            # Invalid insertion event without next batch ID
+            return
+
+        logger.debug(
+            "_handle_insertion_event (next_batch_id=%s) %s", next_batch_id, event
+        )
+
+        # Keep track of the insertion event and the batch ID
+        self.db_pool.simple_insert_txn(
+            txn,
+            table="insertion_events",
+            values={
+                "event_id": event.event_id,
+                "room_id": event.room_id,
+                "next_batch_id": next_batch_id,
+            },
+        )
+
+        # Insert an edge for every prev_event connection
+        for prev_event_id in event.prev_event_ids():
+            self.db_pool.simple_insert_txn(
+                txn,
+                table="insertion_event_edges",
+                values={
+                    "event_id": event.event_id,
+                    "room_id": event.room_id,
+                    "insertion_prev_event_id": prev_event_id,
+                },
+            )
+
+    def _handle_batch_event(self, txn: LoggingTransaction, event: EventBase) -> None:
+        """Handles inserting the batch edges/connections between the batch event
+        and an insertion event. Part of MSC2716.
+
+        Args:
+            txn: The database transaction object
+            event: The event to process
+        """
+
+        if event.type != EventTypes.MSC2716_BATCH:
+            # Not a batch event
+            return
+
+        # Skip processing a batch event if the room version doesn't
+        # support it or the event is not from the room creator.
+        room_version = self.store.get_room_version_txn(txn, event.room_id)
+        room_creator = self.db_pool.simple_select_one_onecol_txn(
+            txn,
+            table="rooms",
+            keyvalues={"room_id": event.room_id},
+            retcol="creator",
+            allow_none=True,
+        )
+        if not room_version.msc2716_historical and (
+            not self.hs.config.experimental.msc2716_enabled
+            or event.sender != room_creator
+        ):
+            return
+
+        batch_id = event.content.get(EventContentFields.MSC2716_BATCH_ID)
+        if batch_id is None:
+            # Invalid batch event without a batch ID
+            return
+
+        logger.debug("_handle_batch_event batch_id=%s %s", batch_id, event)
+
+        # Keep track of the insertion event and the batch ID
+        self.db_pool.simple_insert_txn(
+            txn,
+            table="batch_events",
+            values={
+                "event_id": event.event_id,
+                "room_id": event.room_id,
+                "batch_id": batch_id,
+            },
+        )
+
+        # When we receive an event with a `batch_id` referencing the
+        # `next_batch_id` of the insertion event, we can remove it from the
+        # `insertion_event_extremities` table.
+        sql = """
+            DELETE FROM insertion_event_extremities WHERE event_id IN (
+                SELECT event_id FROM insertion_events
+                WHERE next_batch_id = ?
+            )
+        """
+
+        txn.execute(sql, (batch_id,))
+
     def _handle_redact_relations(
         self, txn: LoggingTransaction, room_id: str, redacted_event_id: str
     ) -> None:

synapse/storage/databases/main/events_worker.py

@@ -883,7 +883,7 @@ class EventsWorkerStore(SQLBaseStore):
 
     async def _invalidate_async_get_event_cache(self, event_id: str) -> None:
         """
-        Invalidates an event in the asynchronous get event cache, which may be remote.
+        Invalidates an event in the asyncronous get event cache, which may be remote.
 
         Arguments:
             event_id: the event ID to invalidate

synapse/storage/databases/main/room_batch.py

@@ -0,0 +1,47 @@
+# Copyright 2021 The Matrix.org Foundation C.I.C.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from typing import Optional
+
+from synapse.storage._base import SQLBaseStore
+
+
+class RoomBatchStore(SQLBaseStore):
+    async def get_insertion_event_id_by_batch_id(
+        self, room_id: str, batch_id: str
+    ) -> Optional[str]:
+        """Retrieve a insertion event ID.
+
+        Args:
+            batch_id: The batch ID of the insertion event to retrieve.
+
+        Returns:
+            The event_id of an insertion event, or None if there is no known
+            insertion event for the given insertion event.
+        """
+        return await self.db_pool.simple_select_one_onecol(
+            table="insertion_events",
+            keyvalues={"room_id": room_id, "next_batch_id": batch_id},
+            retcol="event_id",
+            allow_none=True,
+        )
+
+    async def store_state_group_id_for_event_id(
+        self, event_id: str, state_group_id: int
+    ) -> None:
+        await self.db_pool.simple_upsert(
+            table="event_to_state_groups",
+            keyvalues={"event_id": event_id},
+            values={"state_group": state_group_id, "event_id": event_id},
+        )

synapse/storage/schema/main/delta/78/02_validate_and_update_user_filters.py

@@ -61,7 +61,9 @@ def run_upgrade(
             full_user_id text NOT NULL,
             user_id text NOT NULL,
             filter_id bigint NOT NULL,
-            filter_json bytea NOT NULL
+            filter_json bytea NOT NULL,
+            UNIQUE (full_user_id),
+            UNIQUE (user_id)
         )
         """
         cur.execute(create_sql)

synapse/storage/schema/main/delta/78/03_remove_unused_indexes_user_filters.py

@@ -1,65 +0,0 @@
-# Copyright 2023 The Matrix.org Foundation C.I.C
-#
-#  Licensed under the Apache License, Version 2.0 (the "License");
-#  you may not use this file except in compliance with the License.
-#  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-#  Unless required by applicable law or agreed to in writing, software
-#  distributed under the License is distributed on an "AS IS" BASIS,
-#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#  See the License for the specific language governing permissions and
-#  limitations under the License.
-from synapse.config.homeserver import HomeServerConfig
-from synapse.storage.database import LoggingTransaction
-from synapse.storage.engines import BaseDatabaseEngine, Sqlite3Engine
-
-
-def run_update(
-    cur: LoggingTransaction,
-    database_engine: BaseDatabaseEngine,
-    config: HomeServerConfig,
-) -> None:
-    """
-    Fix to drop unused indexes caused by incorrectly adding UNIQUE constraint to
-    columns `user_id` and `full_user_id` of table `user_filters` in previous migration.
-    """
-
-    if isinstance(database_engine, Sqlite3Engine):
-        cur.execute("DROP TABLE IF EXISTS temp_user_filters")
-        create_sql = """
-        CREATE TABLE temp_user_filters (
-            full_user_id text NOT NULL,
-            user_id text NOT NULL,
-            filter_id bigint NOT NULL,
-            filter_json bytea NOT NULL
-        )
-        """
-        cur.execute(create_sql)
-
-        copy_sql = """
-        INSERT INTO temp_user_filters (
-            user_id,
-            filter_id,
-            filter_json,
-            full_user_id)
-            SELECT user_id, filter_id, filter_json, full_user_id FROM user_filters
-        """
-        cur.execute(copy_sql)
-
-        drop_sql = """
-        DROP TABLE user_filters
-        """
-        cur.execute(drop_sql)
-
-        rename_sql = """
-        ALTER TABLE temp_user_filters RENAME to user_filters
-        """
-        cur.execute(rename_sql)
-
-        index_sql = """
-        CREATE UNIQUE INDEX IF NOT EXISTS user_filters_unique ON
-        user_filters (user_id, filter_id)
-        """
-        cur.execute(index_sql)

synapse/storage/schema/main/delta/78/04_add_full_user_id_index_user_filters.py

@@ -1,25 +0,0 @@
-# Copyright 2023 The Matrix.org Foundation C.I.C
-#
-#  Licensed under the Apache License, Version 2.0 (the "License");
-#  you may not use this file except in compliance with the License.
-#  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-#  Unless required by applicable law or agreed to in writing, software
-#  distributed under the License is distributed on an "AS IS" BASIS,
-#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#  See the License for the specific language governing permissions and
-#  limitations under the License.
-
-from synapse.storage.database import LoggingTransaction
-from synapse.storage.engines import BaseDatabaseEngine, Sqlite3Engine
-
-
-def run_create(cur: LoggingTransaction, database_engine: BaseDatabaseEngine) -> None:
-    if isinstance(database_engine, Sqlite3Engine):
-        idx_sql = """
-        CREATE UNIQUE INDEX IF NOT EXISTS user_filters_full_user_id_unique ON
-        user_filters (full_user_id, filter_id)
-        """
-        cur.execute(idx_sql)

synapse/types/__init__.py

@@ -177,7 +177,7 @@ class Requester:
             user=UserID.from_string(input["user_id"]),
             access_token_id=input["access_token_id"],
             is_guest=input["is_guest"],
-            scope=set(input.get("scope", [])),
+            scope=set(input["scope"]),
             shadow_banned=input["shadow_banned"],
             device_id=input["device_id"],
             app_service=appservice,

synapse/util/__init__.py

@@ -116,11 +116,6 @@ class Clock:
 
         Waits `msec` initially before calling `f` for the first time.
 
-        If the function given to `looping_call` returns an awaitable/deferred, the next
-        call isn't scheduled until after the returned awaitable has finished. We get
-        this functionality thanks to this function being a thin wrapper around
-        `twisted.internet.task.LoopingCall`.
-
         Note that the function will be called with no logcontext, so if it is anything
         other than trivial, you probably want to wrap it in run_as_background_process.
 

synapse/util/caches/lrucache.py

@@ -842,13 +842,7 @@ class AsyncLruCache(Generic[KT, VT]):
         return self._lru_cache.get(key, update_metrics=update_metrics)
 
     async def set(self, key: KT, value: VT) -> None:
-        # This will add the entries in the correct order, local first external second
-        self.set_local(key, value)
-        await self.set_external(key, value)
-
-    async def set_external(self, key: KT, value: VT) -> None:
-        # This method should add an entry to any configured external cache, in this case noop.
-        pass
+        self._lru_cache.set(key, value)
 
     def set_local(self, key: KT, value: VT) -> None:
         self._lru_cache.set(key, value)