From 8771b1337da9faa3b60cf0ec0a128a7de856f19e Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Thu, 6 May 2021 15:54:07 +0100
Subject: [PATCH 01/15] Export jemalloc stats to prometheus when used (#9882)

---
 changelog.d/9882.misc       |   1 +
 synapse/app/_base.py        |   2 +
 synapse/metrics/__init__.py |   1 +
 synapse/metrics/jemalloc.py | 196 ++++++++++++++++++++++++++++++++++++
 4 files changed, 200 insertions(+)
 create mode 100644 changelog.d/9882.misc
 create mode 100644 synapse/metrics/jemalloc.py

diff --git a/changelog.d/9882.misc b/changelog.d/9882.misc
new file mode 100644
index 0000000000..facfa31f38
--- /dev/null
+++ b/changelog.d/9882.misc
@@ -0,0 +1 @@
+Export jemalloc stats to Prometheus if it is being used.
diff --git a/synapse/app/_base.py b/synapse/app/_base.py
index 638e01c1b2..59918d789e 100644
--- a/synapse/app/_base.py
+++ b/synapse/app/_base.py
@@ -37,6 +37,7 @@ from synapse.config.homeserver import HomeServerConfig
 from synapse.crypto import context_factory
 from synapse.logging.context import PreserveLoggingContext
 from synapse.metrics.background_process_metrics import wrap_as_background_process
+from synapse.metrics.jemalloc import setup_jemalloc_stats
 from synapse.util.async_helpers import Linearizer
 from synapse.util.daemonize import daemonize_process
 from synapse.util.rlimit import change_resource_limit
@@ -115,6 +116,7 @@ def start_reactor(
 
     def run():
         logger.info("Running")
+        setup_jemalloc_stats()
         change_resource_limit(soft_file_limit)
         if gc_thresholds:
             gc.set_threshold(*gc_thresholds)
diff --git a/synapse/metrics/__init__.py b/synapse/metrics/__init__.py
index e671da26d5..fef2846669 100644
--- a/synapse/metrics/__init__.py
+++ b/synapse/metrics/__init__.py
@@ -629,6 +629,7 @@ try:
 except AttributeError:
     pass
 
+
 __all__ = [
     "MetricsResource",
     "generate_latest",
diff --git a/synapse/metrics/jemalloc.py b/synapse/metrics/jemalloc.py
new file mode 100644
index 0000000000..29ab6c0229
--- /dev/null
+++ b/synapse/metrics/jemalloc.py
@@ -0,0 +1,196 @@
+# Copyright 2021 The Matrix.org Foundation C.I.C.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import ctypes
+import logging
+import os
+import re
+from typing import Optional
+
+from synapse.metrics import REGISTRY, GaugeMetricFamily
+
+logger = logging.getLogger(__name__)
+
+
+def _setup_jemalloc_stats():
+    """Checks to see if jemalloc is loaded, and hooks up a collector to record
+    statistics exposed by jemalloc.
+    """
+
+    # Try to find the loaded jemalloc shared library, if any. We need to
+    # introspect into what is loaded, rather than loading whatever is on the
+    # path, as if we load a *different* jemalloc version things will seg fault.
+
+    # We look in `/proc/self/maps`, which only exists on linux.
+    if not os.path.exists("/proc/self/maps"):
+        logger.debug("Not looking for jemalloc as no /proc/self/maps exist")
+        return
+
+    # We're looking for a path at the end of the line that includes
+    # "libjemalloc".
+    regex = re.compile(r"/\S+/libjemalloc.*$")
+
+    jemalloc_path = None
+    with open("/proc/self/maps") as f:
+        for line in f:
+            match = regex.search(line.strip())
+            if match:
+                jemalloc_path = match.group()
+
+    if not jemalloc_path:
+        # No loaded jemalloc was found.
+        logger.debug("jemalloc not found")
+        return
+
+    logger.debug("Found jemalloc at %s", jemalloc_path)
+
+    jemalloc = ctypes.CDLL(jemalloc_path)
+
+    def _mallctl(
+        name: str, read: bool = True, write: Optional[int] = None
+    ) -> Optional[int]:
+        """Wrapper around `mallctl` for reading and writing integers to
+        jemalloc.
+
+        Args:
+            name: The name of the option to read from/write to.
+            read: Whether to try and read the value.
+            write: The value to write, if given.
+
+        Returns:
+            The value read if `read` is True, otherwise None.
+
+        Raises:
+            An exception if `mallctl` returns a non-zero error code.
+        """
+
+        input_var = None
+        input_var_ref = None
+        input_len_ref = None
+        if read:
+            input_var = ctypes.c_size_t(0)
+            input_len = ctypes.c_size_t(ctypes.sizeof(input_var))
+
+            input_var_ref = ctypes.byref(input_var)
+            input_len_ref = ctypes.byref(input_len)
+
+        write_var_ref = None
+        write_len = ctypes.c_size_t(0)
+        if write is not None:
+            write_var = ctypes.c_size_t(write)
+            write_len = ctypes.c_size_t(ctypes.sizeof(write_var))
+
+            write_var_ref = ctypes.byref(write_var)
+
+        # The interface is:
+        #
+        #   int mallctl(
+        #       const char *name,
+        #       void *oldp,
+        #       size_t *oldlenp,
+        #       void *newp,
+        #       size_t newlen
+        #   )
+        #
+        # Where oldp/oldlenp is a buffer where the old value will be written to
+        # (if not null), and newp/newlen is the buffer with the new value to set
+        # (if not null). Note that they're all references *except* newlen.
+        result = jemalloc.mallctl(
+            name.encode("ascii"),
+            input_var_ref,
+            input_len_ref,
+            write_var_ref,
+            write_len,
+        )
+
+        if result != 0:
+            raise Exception("Failed to call mallctl")
+
+        if input_var is None:
+            return None
+
+        return input_var.value
+
+    def _jemalloc_refresh_stats() -> None:
+        """Request that jemalloc updates its internal statistics. This needs to
+        be called before querying for stats, otherwise it will return stale
+        values.
+        """
+        try:
+            _mallctl("epoch", read=False, write=1)
+        except Exception as e:
+            logger.warning("Failed to reload jemalloc stats: %s", e)
+
+    class JemallocCollector:
+        """Metrics for internal jemalloc stats."""
+
+        def collect(self):
+            _jemalloc_refresh_stats()
+
+            g = GaugeMetricFamily(
+                "jemalloc_stats_app_memory_bytes",
+                "The stats reported by jemalloc",
+                labels=["type"],
+            )
+
+            # Read the relevant global stats from jemalloc. Note that these may
+            # not be accurate if python is configured to use its internal small
+            # object allocator (which is on by default, disable by setting the
+            # env `PYTHONMALLOC=malloc`).
+            #
+            # See the jemalloc manpage for details about what each value means,
+            # roughly:
+            #   - allocated ─ Total number of bytes allocated by the app
+            #   - active ─ Total number of bytes in active pages allocated by
+            #     the application, this is bigger than `allocated`.
+            #   - resident ─ Maximum number of bytes in physically resident data
+            #     pages mapped by the allocator, comprising all pages dedicated
+            #     to allocator metadata, pages backing active allocations, and
+            #     unused dirty pages. This is bigger than `active`.
+            #   - mapped ─ Total number of bytes in active extents mapped by the
+            #     allocator.
+            #   - metadata ─ Total number of bytes dedicated to jemalloc
+            #     metadata.
+            for t in (
+                "allocated",
+                "active",
+                "resident",
+                "mapped",
+                "metadata",
+            ):
+                try:
+                    value = _mallctl(f"stats.{t}")
+                except Exception as e:
+                    # There was an error fetching the value, skip.
+                    logger.warning("Failed to read jemalloc stats.%s: %s", t, e)
+                    continue
+
+                g.add_metric([t], value=value)
+
+            yield g
+
+    REGISTRY.register(JemallocCollector())
+
+    logger.debug("Added jemalloc stats")
+
+
+def setup_jemalloc_stats():
+    """Try to setup jemalloc stats, if jemalloc is loaded."""
+
+    try:
+        _setup_jemalloc_stats()
+    except Exception as e:
+        # This should only happen if we find the loaded jemalloc library, but
+        # fail to load it somehow (e.g. we somehow picked the wrong version).
+        logger.info("Failed to setup collector to record jemalloc stats: %s", e)

From 25f43faa70f7cc58493b636c2702ae63395779dc Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <1389908+richvdh@users.noreply.github.com>
Date: Fri, 7 May 2021 10:22:05 +0100
Subject: [PATCH 02/15] Reorganise the database schema directories (#9932)

The hope here is that by moving all the schema files into synapse/storage/schema, it gets a bit easier for newcomers to navigate.

It certainly got easier for me to write a helpful README. There's more to do on that front, but I'll follow up with other PRs for that.
---
 changelog.d/9932.misc                         |  1 +
 .../main/schema/full_schemas/README.md        | 21 --------
 synapse/storage/prepare_database.py           | 48 ++++++++++---------
 synapse/storage/schema/README.md              | 37 ++++++++++++++
 synapse/storage/schema/__init__.py            | 17 +++++++
 .../delta/25/00background_updates.sql         |  0
 .../delta/35/00background_updates_add_col.sql |  0
 .../delta/58/00background_update_ordering.sql |  0
 .../{ => common}/full_schemas/54/full.sql     |  0
 .../schema/{ => common}/schema_version.sql    |  0
 .../schema => schema/main}/delta/12/v12.sql   |  0
 .../schema => schema/main}/delta/13/v13.sql   |  0
 .../schema => schema/main}/delta/14/v14.sql   |  0
 .../main}/delta/15/appservice_txns.sql        |  0
 .../main}/delta/15/presence_indices.sql       |  0
 .../schema => schema/main}/delta/15/v15.sql   |  0
 .../main}/delta/16/events_order_index.sql     |  0
 .../delta/16/remote_media_cache_index.sql     |  0
 .../main}/delta/16/remove_duplicates.sql      |  0
 .../main}/delta/16/room_alias_index.sql       |  0
 .../main}/delta/16/unique_constraints.sql     |  0
 .../schema => schema/main}/delta/16/users.sql |  0
 .../main}/delta/17/drop_indexes.sql           |  0
 .../main}/delta/17/server_keys.sql            |  0
 .../main}/delta/17/user_threepids.sql         |  0
 .../delta/18/server_keys_bigger_ints.sql      |  0
 .../main}/delta/19/event_index.sql            |  0
 .../schema => schema/main}/delta/20/dummy.sql |  0
 .../main}/delta/20/pushers.py                 |  0
 .../main}/delta/21/end_to_end_keys.sql        |  0
 .../main}/delta/21/receipts.sql               |  0
 .../main}/delta/22/receipts_index.sql         |  0
 .../main}/delta/22/user_threepids_unique.sql  |  0
 .../main}/delta/24/stats_reporting.sql        |  0
 .../schema => schema/main}/delta/25/fts.py    |  0
 .../main}/delta/25/guest_access.sql           |  0
 .../main}/delta/25/history_visibility.sql     |  0
 .../schema => schema/main}/delta/25/tags.sql  |  0
 .../main}/delta/26/account_data.sql           |  0
 .../main}/delta/27/account_data.sql           |  0
 .../main}/delta/27/forgotten_memberships.sql  |  0
 .../schema => schema/main}/delta/27/ts.py     |  0
 .../main}/delta/28/event_push_actions.sql     |  0
 .../main}/delta/28/events_room_stream.sql     |  0
 .../main}/delta/28/public_roms_index.sql      |  0
 .../main}/delta/28/receipts_user_id_index.sql |  0
 .../main}/delta/28/upgrade_times.sql          |  0
 .../main}/delta/28/users_is_guest.sql         |  0
 .../main}/delta/29/push_actions.sql           |  0
 .../main}/delta/30/alias_creator.sql          |  0
 .../main}/delta/30/as_users.py                |  0
 .../main}/delta/30/deleted_pushers.sql        |  0
 .../main}/delta/30/presence_stream.sql        |  0
 .../main}/delta/30/public_rooms.sql           |  0
 .../main}/delta/30/push_rule_stream.sql       |  0
 .../delta/30/threepid_guest_access_tokens.sql |  0
 .../main}/delta/31/invites.sql                |  0
 .../31/local_media_repository_url_cache.sql   |  0
 .../main}/delta/31/pushers.py                 |  0
 .../main}/delta/31/pushers_index.sql          |  0
 .../main}/delta/31/search_update.py           |  0
 .../main}/delta/32/events.sql                 |  0
 .../main}/delta/32/openid.sql                 |  0
 .../main}/delta/32/pusher_throttle.sql        |  0
 .../main}/delta/32/remove_indices.sql         |  0
 .../main}/delta/32/reports.sql                |  0
 .../delta/33/access_tokens_device_index.sql   |  0
 .../main}/delta/33/devices.sql                |  0
 .../main}/delta/33/devices_for_e2e_keys.sql   |  0
 ...ices_for_e2e_keys_clear_unknown_device.sql |  0
 .../main}/delta/33/event_fields.py            |  0
 .../main}/delta/33/remote_media_ts.py         |  0
 .../main}/delta/33/user_ips_index.sql         |  0
 .../main}/delta/34/appservice_stream.sql      |  0
 .../main}/delta/34/cache_stream.py            |  0
 .../main}/delta/34/device_inbox.sql           |  0
 .../delta/34/push_display_name_rename.sql     |  0
 .../main}/delta/34/received_txn_purge.py      |  0
 .../main}/delta/35/contains_url.sql           |  0
 .../main}/delta/35/device_outbox.sql          |  0
 .../main}/delta/35/device_stream_id.sql       |  0
 .../delta/35/event_push_actions_index.sql     |  0
 .../35/public_room_list_change_stream.sql     |  0
 .../main}/delta/35/stream_order_to_extrem.sql |  0
 .../main}/delta/36/readd_public_rooms.sql     |  0
 .../main}/delta/37/remove_auth_idx.py         |  0
 .../main}/delta/37/user_threepids.sql         |  0
 .../main}/delta/38/postgres_fts_gist.sql      |  0
 .../main}/delta/39/appservice_room_list.sql   |  0
 .../delta/39/device_federation_stream_idx.sql |  0
 .../main}/delta/39/event_push_index.sql       |  0
 .../delta/39/federation_out_position.sql      |  0
 .../main}/delta/39/membership_profile.sql     |  0
 .../main}/delta/40/current_state_idx.sql      |  0
 .../main}/delta/40/device_inbox.sql           |  0
 .../main}/delta/40/device_list_streams.sql    |  0
 .../main}/delta/40/event_push_summary.sql     |  0
 .../main}/delta/40/pushers.sql                |  0
 .../main}/delta/41/device_list_stream_idx.sql |  0
 .../main}/delta/41/device_outbound_index.sql  |  0
 .../delta/41/event_search_event_id_idx.sql    |  0
 .../main}/delta/41/ratelimit.sql              |  0
 .../main}/delta/42/current_state_delta.sql    |  0
 .../main}/delta/42/device_list_last_id.sql    |  0
 .../main}/delta/42/event_auth_state_only.sql  |  0
 .../main}/delta/42/user_dir.py                |  0
 .../main}/delta/43/blocked_rooms.sql          |  0
 .../main}/delta/43/quarantine_media.sql       |  0
 .../main}/delta/43/url_cache.sql              |  0
 .../main}/delta/43/user_share.sql             |  0
 .../main}/delta/44/expire_url_cache.sql       |  0
 .../main}/delta/45/group_server.sql           |  0
 .../main}/delta/45/profile_cache.sql          |  0
 .../main}/delta/46/drop_refresh_tokens.sql    |  0
 .../delta/46/drop_unique_deleted_pushers.sql  |  0
 .../main}/delta/46/group_server.sql           |  0
 .../46/local_media_repository_url_idx.sql     |  0
 .../main}/delta/46/user_dir_null_room_ids.sql |  0
 .../main}/delta/46/user_dir_typos.sql         |  0
 .../main}/delta/47/last_access_media.sql      |  0
 .../main}/delta/47/postgres_fts_gin.sql       |  0
 .../main}/delta/47/push_actions_staging.sql   |  0
 .../main}/delta/48/add_user_consent.sql       |  0
 .../delta/48/add_user_ips_last_seen_index.sql |  0
 .../main}/delta/48/deactivated_users.sql      |  0
 .../main}/delta/48/group_unique_indexes.py    |  0
 .../main}/delta/48/groups_joinable.sql        |  0
 .../add_user_consent_server_notice_sent.sql   |  0
 .../main}/delta/49/add_user_daily_visits.sql  |  0
 .../49/add_user_ips_last_seen_only_index.sql  |  0
 .../delta/50/add_creation_ts_users_index.sql  |  0
 .../main}/delta/50/erasure_store.sql          |  0
 .../delta/50/make_event_content_nullable.py   |  0
 .../main}/delta/51/e2e_room_keys.sql          |  0
 .../main}/delta/51/monthly_active_users.sql   |  0
 .../52/add_event_to_state_group_index.sql     |  0
 .../52/device_list_streams_unique_idx.sql     |  0
 .../main}/delta/52/e2e_room_keys.sql          |  0
 .../main}/delta/53/add_user_type_to_users.sql |  0
 .../main}/delta/53/drop_sent_transactions.sql |  0
 .../main}/delta/53/event_format_version.sql   |  0
 .../main}/delta/53/user_dir_populate.sql      |  0
 .../main}/delta/53/user_ips_index.sql         |  0
 .../main}/delta/53/user_share.sql             |  0
 .../main}/delta/53/user_threepid_id.sql       |  0
 .../main}/delta/53/users_in_public_rooms.sql  |  0
 .../54/account_validity_with_renewal.sql      |  0
 .../delta/54/add_validity_to_server_keys.sql  |  0
 .../delta/54/delete_forward_extremities.sql   |  0
 .../main}/delta/54/drop_legacy_tables.sql     |  0
 .../main}/delta/54/drop_presence_list.sql     |  0
 .../main}/delta/54/relations.sql              |  0
 .../schema => schema/main}/delta/54/stats.sql |  0
 .../main}/delta/54/stats2.sql                 |  0
 .../main}/delta/55/access_token_expiry.sql    |  0
 .../delta/55/track_threepid_validations.sql   |  0
 .../delta/55/users_alter_deactivated.sql      |  0
 .../delta/56/add_spans_to_device_lists.sql    |  0
 .../56/current_state_events_membership.sql    |  0
 .../current_state_events_membership_mk2.sql   |  0
 .../56/delete_keys_from_deleted_backups.sql   |  0
 .../delta/56/destinations_failure_ts.sql      |  0
 ...tinations_retry_interval_type.sql.postgres |  0
 .../delta/56/device_stream_id_insert.sql      |  0
 .../main}/delta/56/devices_last_seen.sql      |  0
 .../delta/56/drop_unused_event_tables.sql     |  0
 .../main}/delta/56/event_expiry.sql           |  0
 .../main}/delta/56/event_labels.sql           |  0
 .../56/event_labels_background_update.sql     |  0
 .../main}/delta/56/fix_room_keys_index.sql    |  0
 .../main}/delta/56/hidden_devices.sql         |  0
 .../delta/56/hidden_devices_fix.sql.sqlite    |  0
 .../56/nuke_empty_communities_from_db.sql     |  0
 .../main}/delta/56/public_room_list_idx.sql   |  0
 .../main}/delta/56/redaction_censor.sql       |  0
 .../main}/delta/56/redaction_censor2.sql      |  0
 .../redaction_censor3_fix_update.sql.postgres |  0
 .../main}/delta/56/redaction_censor4.sql      |  0
 ...remove_tombstoned_rooms_from_directory.sql |  0
 .../main}/delta/56/room_key_etag.sql          |  0
 .../main}/delta/56/room_membership_idx.sql    |  0
 .../main}/delta/56/room_retention.sql         |  0
 .../main}/delta/56/signing_keys.sql           |  0
 .../56/signing_keys_nonunique_signatures.sql  |  0
 .../main}/delta/56/stats_separated.sql        |  0
 .../delta/56/unique_user_filter_index.py      |  0
 .../main}/delta/56/user_external_ids.sql      |  0
 .../delta/56/users_in_public_rooms_idx.sql    |  0
 .../57/delete_old_current_state_events.sql    |  0
 .../57/device_list_remote_cache_stale.sql     |  0
 .../delta/57/local_current_membership.py      |  0
 .../delta/57/remove_sent_outbound_pokes.sql   |  0
 .../main}/delta/57/rooms_version_column.sql   |  0
 .../57/rooms_version_column_2.sql.postgres    |  0
 .../57/rooms_version_column_2.sql.sqlite      |  0
 .../57/rooms_version_column_3.sql.postgres    |  0
 .../57/rooms_version_column_3.sql.sqlite      |  0
 .../delta/58/02remove_dup_outbound_pokes.sql  |  0
 .../main}/delta/58/03persist_ui_auth.sql      |  0
 .../delta/58/05cache_instance.sql.postgres    |  0
 .../main}/delta/58/06dlols_unique_idx.py      |  0
 ...ethod_to_thumbnail_constraint.sql.postgres |  0
 ..._method_to_thumbnail_constraint.sql.sqlite |  0
 .../main}/delta/58/07persist_ui_auth_ips.sql  |  0
 ...08_media_safe_from_quarantine.sql.postgres |  0
 .../08_media_safe_from_quarantine.sql.sqlite  |  0
 .../main}/delta/58/09shadow_ban.sql           |  0
 .../10_pushrules_enabled_delete_obsolete.sql  |  0
 .../58/10drop_local_rejections_stream.sql     |  0
 .../58/10federation_pos_instance_name.sql     |  0
 .../main}/delta/58/11dehydration.sql          |  0
 .../main}/delta/58/11fallback.sql             |  0
 .../main}/delta/58/11user_id_seq.py           |  0
 .../main}/delta/58/12room_stats.sql           |  0
 .../58/13remove_presence_allow_inbound.sql    |  0
 .../main}/delta/58/14events_instance_name.sql |  0
 .../58/14events_instance_name.sql.postgres    |  0
 .../delta/58/15_catchup_destination_rooms.sql |  0
 .../main}/delta/58/15unread_count.sql         |  0
 .../58/16populate_stats_process_rooms_fix.sql |  0
 .../delta/58/17_catchup_last_successful.sql   |  0
 .../main}/delta/58/18stream_positions.sql     |  0
 .../delta/58/19instance_map.sql.postgres      |  0
 .../main}/delta/58/19txn_id.sql               |  0
 .../delta/58/20instance_name_event_tables.sql |  0
 .../main}/delta/58/20user_daily_visits.sql    |  0
 .../main}/delta/58/21as_device_stream.sql     |  0
 .../delta/58/21drop_device_max_stream_id.sql  |  0
 .../main}/delta/58/22puppet_token.sql         |  0
 .../delta/58/22users_have_local_media.sql     |  0
 .../delta/58/23e2e_cross_signing_keys_idx.sql |  0
 .../delta/58/24drop_event_json_index.sql      |  0
 .../58/25user_external_ids_user_id_idx.sql    |  0
 .../58/26access_token_last_validated.sql      |  0
 .../main}/delta/58/27local_invites.sql        |  0
 .../58/28drop_last_used_column.sql.postgres   |  0
 .../58/28drop_last_used_column.sql.sqlite     |  0
 .../main}/delta/59/01ignored_user.py          |  0
 .../main}/delta/59/02shard_send_to_device.sql |  0
 ...shard_send_to_device_sequence.sql.postgres |  0
 .../main}/delta/59/04_event_auth_chains.sql   |  0
 .../59/04_event_auth_chains.sql.postgres      |  0
 .../main}/delta/59/04drop_account_data.sql    |  0
 .../main}/delta/59/05cache_invalidation.sql   |  0
 .../main}/delta/59/06chain_cover_index.sql    |  0
 .../main}/delta/59/06shard_account_data.sql   |  0
 .../59/06shard_account_data.sql.postgres      |  0
 .../delta/59/07shard_account_data_fix.sql     |  0
 ...elete_pushers_for_deactivated_accounts.sql |  0
 .../main}/delta/59/08delete_stale_pushers.sql |  0
 .../delta/59/09rejected_events_metadata.sql   |  0
 .../delta/59/10delete_purged_chain_cover.sql  |  0
 .../11drop_thumbnail_constraint.sql.postgres  |  0
 .../12account_validity_token_used_ts_ms.sql   |  0
 .../delta/59/12presence_stream_instance.sql   |  0
 ...2presence_stream_instance_seq.sql.postgres |  0
 .../full_schemas/16/application_services.sql  |  0
 .../main}/full_schemas/16/event_edges.sql     |  0
 .../full_schemas/16/event_signatures.sql      |  0
 .../main}/full_schemas/16/im.sql              |  0
 .../main}/full_schemas/16/keys.sql            |  0
 .../full_schemas/16/media_repository.sql      |  0
 .../main}/full_schemas/16/presence.sql        |  0
 .../main}/full_schemas/16/profiles.sql        |  0
 .../main}/full_schemas/16/push.sql            |  0
 .../main}/full_schemas/16/redactions.sql      |  0
 .../main}/full_schemas/16/room_aliases.sql    |  0
 .../main}/full_schemas/16/state.sql           |  0
 .../main}/full_schemas/16/transactions.sql    |  0
 .../main}/full_schemas/16/users.sql           |  0
 .../main}/full_schemas/54/full.sql.postgres   |  0
 .../main}/full_schemas/54/full.sql.sqlite     |  0
 .../full_schemas/54/stream_positions.sql      |  0
 .../state}/delta/23/drop_state_index.sql      |  0
 .../state}/delta/30/state_stream.sql          |  0
 .../state}/delta/32/remove_state_indices.sql  |  0
 .../state}/delta/35/add_state_index.sql       |  0
 .../state}/delta/35/state.sql                 |  0
 .../state}/delta/35/state_dedupe.sql          |  0
 .../state}/delta/47/state_group_seq.py        |  0
 .../state}/delta/56/state_group_room_idx.sql  |  0
 .../state}/full_schemas/54/full.sql           |  0
 .../full_schemas/54/sequence.sql.postgres     |  0
 tests/storage/test_cleanup_extrems.py         |  4 +-
 284 files changed, 81 insertions(+), 47 deletions(-)
 create mode 100644 changelog.d/9932.misc
 delete mode 100644 synapse/storage/databases/main/schema/full_schemas/README.md
 create mode 100644 synapse/storage/schema/README.md
 create mode 100644 synapse/storage/schema/__init__.py
 rename synapse/storage/schema/{ => common}/delta/25/00background_updates.sql (100%)
 rename synapse/storage/schema/{ => common}/delta/35/00background_updates_add_col.sql (100%)
 rename synapse/storage/schema/{ => common}/delta/58/00background_update_ordering.sql (100%)
 rename synapse/storage/schema/{ => common}/full_schemas/54/full.sql (100%)
 rename synapse/storage/schema/{ => common}/schema_version.sql (100%)
 rename synapse/storage/{databases/main/schema => schema/main}/delta/12/v12.sql (100%)
 rename synapse/storage/{databases/main/schema => schema/main}/delta/13/v13.sql (100%)
 rename synapse/storage/{databases/main/schema => schema/main}/delta/14/v14.sql (100%)
 rename synapse/storage/{databases/main/schema => schema/main}/delta/15/appservice_txns.sql (100%)
 rename synapse/storage/{databases/main/schema => schema/main}/delta/15/presence_indices.sql (100%)
 rename synapse/storage/{databases/main/schema => schema/main}/delta/15/v15.sql (100%)
 rename synapse/storage/{databases/main/schema => schema/main}/delta/16/events_order_index.sql (100%)
 rename synapse/storage/{databases/main/schema => schema/main}/delta/16/remote_media_cache_index.sql (100%)
 rename synapse/storage/{databases/main/schema => schema/main}/delta/16/remove_duplicates.sql (100%)
 rename synapse/storage/{databases/main/schema => schema/main}/delta/16/room_alias_index.sql (100%)
 rename synapse/storage/{databases/main/schema => schema/main}/delta/16/unique_constraints.sql (100%)
 rename synapse/storage/{databases/main/schema => schema/main}/delta/16/users.sql (100%)
 rename synapse/storage/{databases/main/schema => schema/main}/delta/17/drop_indexes.sql (100%)
 rename synapse/storage/{databases/main/schema => schema/main}/delta/17/server_keys.sql (100%)
 rename synapse/storage/{databases/main/schema => schema/main}/delta/17/user_threepids.sql (100%)
 rename synapse/storage/{databases/main/schema => schema/main}/delta/18/server_keys_bigger_ints.sql (100%)
 rename synapse/storage/{databases/main/schema => schema/main}/delta/19/event_index.sql (100%)
 rename synapse/storage/{databases/main/schema => schema/main}/delta/20/dummy.sql (100%)
 rename synapse/storage/{databases/main/schema => schema/main}/delta/20/pushers.py (100%)
 rename synapse/storage/{databases/main/schema => schema/main}/delta/21/end_to_end_keys.sql (100%)
 rename synapse/storage/{databases/main/schema => schema/main}/delta/21/receipts.sql (100%)
 rename synapse/storage/{databases/main/schema => schema/main}/delta/22/receipts_index.sql (100%)
 rename synapse/storage/{databases/main/schema => schema/main}/delta/22/user_threepids_unique.sql (100%)
 rename synapse/storage/{databases/main/schema => schema/main}/delta/24/stats_reporting.sql (100%)
 rename synapse/storage/{databases/main/schema => schema/main}/delta/25/fts.py (100%)
 rename synapse/storage/{databases/main/schema => schema/main}/delta/25/guest_access.sql (100%)
 rename synapse/storage/{databases/main/schema => schema/main}/delta/25/history_visibility.sql (100%)
 rename synapse/storage/{databases/main/schema => schema/main}/delta/25/tags.sql (100%)
 rename synapse/storage/{databases/main/schema => schema/main}/delta/26/account_data.sql (100%)
 rename synapse/storage/{databases/main/schema => schema/main}/delta/27/account_data.sql (100%)
 rename synapse/storage/{databases/main/schema => schema/main}/delta/27/forgotten_memberships.sql (100%)
 rename synapse/storage/{databases/main/schema => schema/main}/delta/27/ts.py (100%)
 rename synapse/storage/{databases/main/schema => schema/main}/delta/28/event_push_actions.sql (100%)
 rename synapse/storage/{databases/main/schema => schema/main}/delta/28/events_room_stream.sql (100%)
 rename synapse/storage/{databases/main/schema => schema/main}/delta/28/public_roms_index.sql (100%)
 rename synapse/storage/{databases/main/schema => schema/main}/delta/28/receipts_user_id_index.sql (100%)
 rename synapse/storage/{databases/main/schema => schema/main}/delta/28/upgrade_times.sql (100%)
 rename synapse/storage/{databases/main/schema => schema/main}/delta/28/users_is_guest.sql (100%)
 rename synapse/storage/{databases/main/schema => schema/main}/delta/29/push_actions.sql (100%)
 rename synapse/storage/{databases/main/schema => schema/main}/delta/30/alias_creator.sql (100%)
 rename synapse/storage/{databases/main/schema => schema/main}/delta/30/as_users.py (100%)
 rename synapse/storage/{databases/main/schema => schema/main}/delta/30/deleted_pushers.sql (100%)
 rename synapse/storage/{databases/main/schema => schema/main}/delta/30/presence_stream.sql (100%)
 rename synapse/storage/{databases/main/schema => schema/main}/delta/30/public_rooms.sql (100%)
 rename synapse/storage/{databases/main/schema => schema/main}/delta/30/push_rule_stream.sql (100%)
 rename synapse/storage/{databases/main/schema => schema/main}/delta/30/threepid_guest_access_tokens.sql (100%)
 rename synapse/storage/{databases/main/schema => schema/main}/delta/31/invites.sql (100%)
 rename synapse/storage/{databases/main/schema => schema/main}/delta/31/local_media_repository_url_cache.sql (100%)
 rename synapse/storage/{databases/main/schema => schema/main}/delta/31/pushers.py (100%)
 rename synapse/storage/{databases/main/schema => schema/main}/delta/31/pushers_index.sql (100%)
 rename synapse/storage/{databases/main/schema => schema/main}/delta/31/search_update.py (100%)
 rename synapse/storage/{databases/main/schema => schema/main}/delta/32/events.sql (100%)
 rename synapse/storage/{databases/main/schema => schema/main}/delta/32/openid.sql (100%)
 rename synapse/storage/{databases/main/schema => schema/main}/delta/32/pusher_throttle.sql (100%)
 rename synapse/storage/{databases/main/schema => schema/main}/delta/32/remove_indices.sql (100%)
 rename synapse/storage/{databases/main/schema => schema/main}/delta/32/reports.sql (100%)
 rename synapse/storage/{databases/main/schema => schema/main}/delta/33/access_tokens_device_index.sql (100%)
 rename synapse/storage/{databases/main/schema => schema/main}/delta/33/devices.sql (100%)
 rename synapse/storage/{databases/main/schema => schema/main}/delta/33/devices_for_e2e_keys.sql (100%)
 rename synapse/storage/{databases/main/schema => schema/main}/delta/33/devices_for_e2e_keys_clear_unknown_device.sql (100%)
 rename synapse/storage/{databases/main/schema => schema/main}/delta/33/event_fields.py (100%)
 rename synapse/storage/{databases/main/schema => schema/main}/delta/33/remote_media_ts.py (100%)
 rename synapse/storage/{databases/main/schema => schema/main}/delta/33/user_ips_index.sql (100%)
 rename synapse/storage/{databases/main/schema => schema/main}/delta/34/appservice_stream.sql (100%)
 rename synapse/storage/{databases/main/schema => schema/main}/delta/34/cache_stream.py (100%)
 rename synapse/storage/{databases/main/schema => schema/main}/delta/34/device_inbox.sql (100%)
 rename synapse/storage/{databases/main/schema => schema/main}/delta/34/push_display_name_rename.sql (100%)
 rename synapse/storage/{databases/main/schema => schema/main}/delta/34/received_txn_purge.py (100%)
 rename synapse/storage/{databases/main/schema => schema/main}/delta/35/contains_url.sql (100%)
 rename synapse/storage/{databases/main/schema => schema/main}/delta/35/device_outbox.sql (100%)
 rename synapse/storage/{databases/main/schema => schema/main}/delta/35/device_stream_id.sql (100%)
 rename synapse/storage/{databases/main/schema => schema/main}/delta/35/event_push_actions_index.sql (100%)
 rename synapse/storage/{databases/main/schema => schema/main}/delta/35/public_room_list_change_stream.sql (100%)
 rename synapse/storage/{databases/main/schema => schema/main}/delta/35/stream_order_to_extrem.sql (100%)
 rename synapse/storage/{databases/main/schema => schema/main}/delta/36/readd_public_rooms.sql (100%)
 rename synapse/storage/{databases/main/schema => schema/main}/delta/37/remove_auth_idx.py (100%)
 rename synapse/storage/{databases/main/schema => schema/main}/delta/37/user_threepids.sql (100%)
 rename synapse/storage/{databases/main/schema => schema/main}/delta/38/postgres_fts_gist.sql (100%)
 rename synapse/storage/{databases/main/schema => schema/main}/delta/39/appservice_room_list.sql (100%)
 rename synapse/storage/{databases/main/schema => schema/main}/delta/39/device_federation_stream_idx.sql (100%)
 rename synapse/storage/{databases/main/schema => schema/main}/delta/39/event_push_index.sql (100%)
 rename synapse/storage/{databases/main/schema => schema/main}/delta/39/federation_out_position.sql (100%)
 rename synapse/storage/{databases/main/schema => schema/main}/delta/39/membership_profile.sql (100%)
 rename synapse/storage/{databases/main/schema => schema/main}/delta/40/current_state_idx.sql (100%)
 rename synapse/storage/{databases/main/schema => schema/main}/delta/40/device_inbox.sql (100%)
 rename synapse/storage/{databases/main/schema => schema/main}/delta/40/device_list_streams.sql (100%)
 rename synapse/storage/{databases/main/schema => schema/main}/delta/40/event_push_summary.sql (100%)
 rename synapse/storage/{databases/main/schema => schema/main}/delta/40/pushers.sql (100%)
 rename synapse/storage/{databases/main/schema => schema/main}/delta/41/device_list_stream_idx.sql (100%)
 rename synapse/storage/{databases/main/schema => schema/main}/delta/41/device_outbound_index.sql (100%)
 rename synapse/storage/{databases/main/schema => schema/main}/delta/41/event_search_event_id_idx.sql (100%)
 rename synapse/storage/{databases/main/schema => schema/main}/delta/41/ratelimit.sql (100%)
 rename synapse/storage/{databases/main/schema => schema/main}/delta/42/current_state_delta.sql (100%)
 rename synapse/storage/{databases/main/schema => schema/main}/delta/42/device_list_last_id.sql (100%)
 rename synapse/storage/{databases/main/schema => schema/main}/delta/42/event_auth_state_only.sql (100%)
 rename synapse/storage/{databases/main/schema => schema/main}/delta/42/user_dir.py (100%)
 rename synapse/storage/{databases/main/schema => schema/main}/delta/43/blocked_rooms.sql (100%)
 rename synapse/storage/{databases/main/schema => schema/main}/delta/43/quarantine_media.sql (100%)
 rename synapse/storage/{databases/main/schema => schema/main}/delta/43/url_cache.sql (100%)
 rename synapse/storage/{databases/main/schema => schema/main}/delta/43/user_share.sql (100%)
 rename synapse/storage/{databases/main/schema => schema/main}/delta/44/expire_url_cache.sql (100%)
 rename synapse/storage/{databases/main/schema => schema/main}/delta/45/group_server.sql (100%)
 rename synapse/storage/{databases/main/schema => schema/main}/delta/45/profile_cache.sql (100%)
 rename synapse/storage/{databases/main/schema => schema/main}/delta/46/drop_refresh_tokens.sql (100%)
 rename synapse/storage/{databases/main/schema => schema/main}/delta/46/drop_unique_deleted_pushers.sql (100%)
 rename synapse/storage/{databases/main/schema => schema/main}/delta/46/group_server.sql (100%)
 rename synapse/storage/{databases/main/schema => schema/main}/delta/46/local_media_repository_url_idx.sql (100%)
 rename synapse/storage/{databases/main/schema => schema/main}/delta/46/user_dir_null_room_ids.sql (100%)
 rename synapse/storage/{databases/main/schema => schema/main}/delta/46/user_dir_typos.sql (100%)
 rename synapse/storage/{databases/main/schema => schema/main}/delta/47/last_access_media.sql (100%)
 rename synapse/storage/{databases/main/schema => schema/main}/delta/47/postgres_fts_gin.sql (100%)
 rename synapse/storage/{databases/main/schema => schema/main}/delta/47/push_actions_staging.sql (100%)
 rename synapse/storage/{databases/main/schema => schema/main}/delta/48/add_user_consent.sql (100%)
 rename synapse/storage/{databases/main/schema => schema/main}/delta/48/add_user_ips_last_seen_index.sql (100%)
 rename synapse/storage/{databases/main/schema => schema/main}/delta/48/deactivated_users.sql (100%)
 rename synapse/storage/{databases/main/schema => schema/main}/delta/48/group_unique_indexes.py (100%)
 rename synapse/storage/{databases/main/schema => schema/main}/delta/48/groups_joinable.sql (100%)
 rename synapse/storage/{databases/main/schema => schema/main}/delta/49/add_user_consent_server_notice_sent.sql (100%)
 rename synapse/storage/{databases/main/schema => schema/main}/delta/49/add_user_daily_visits.sql (100%)
 rename synapse/storage/{databases/main/schema => schema/main}/delta/49/add_user_ips_last_seen_only_index.sql (100%)
 rename synapse/storage/{databases/main/schema => schema/main}/delta/50/add_creation_ts_users_index.sql (100%)
 rename synapse/storage/{databases/main/schema => schema/main}/delta/50/erasure_store.sql (100%)
 rename synapse/storage/{databases/main/schema => schema/main}/delta/50/make_event_content_nullable.py (100%)
 rename synapse/storage/{databases/main/schema => schema/main}/delta/51/e2e_room_keys.sql (100%)
 rename synapse/storage/{databases/main/schema => schema/main}/delta/51/monthly_active_users.sql (100%)
 rename synapse/storage/{databases/main/schema => schema/main}/delta/52/add_event_to_state_group_index.sql (100%)
 rename synapse/storage/{databases/main/schema => schema/main}/delta/52/device_list_streams_unique_idx.sql (100%)
 rename synapse/storage/{databases/main/schema => schema/main}/delta/52/e2e_room_keys.sql (100%)
 rename synapse/storage/{databases/main/schema => schema/main}/delta/53/add_user_type_to_users.sql (100%)
 rename synapse/storage/{databases/main/schema => schema/main}/delta/53/drop_sent_transactions.sql (100%)
 rename synapse/storage/{databases/main/schema => schema/main}/delta/53/event_format_version.sql (100%)
 rename synapse/storage/{databases/main/schema => schema/main}/delta/53/user_dir_populate.sql (100%)
 rename synapse/storage/{databases/main/schema => schema/main}/delta/53/user_ips_index.sql (100%)
 rename synapse/storage/{databases/main/schema => schema/main}/delta/53/user_share.sql (100%)
 rename synapse/storage/{databases/main/schema => schema/main}/delta/53/user_threepid_id.sql (100%)
 rename synapse/storage/{databases/main/schema => schema/main}/delta/53/users_in_public_rooms.sql (100%)
 rename synapse/storage/{databases/main/schema => schema/main}/delta/54/account_validity_with_renewal.sql (100%)
 rename synapse/storage/{databases/main/schema => schema/main}/delta/54/add_validity_to_server_keys.sql (100%)
 rename synapse/storage/{databases/main/schema => schema/main}/delta/54/delete_forward_extremities.sql (100%)
 rename synapse/storage/{databases/main/schema => schema/main}/delta/54/drop_legacy_tables.sql (100%)
 rename synapse/storage/{databases/main/schema => schema/main}/delta/54/drop_presence_list.sql (100%)
 rename synapse/storage/{databases/main/schema => schema/main}/delta/54/relations.sql (100%)
 rename synapse/storage/{databases/main/schema => schema/main}/delta/54/stats.sql (100%)
 rename synapse/storage/{databases/main/schema => schema/main}/delta/54/stats2.sql (100%)
 rename synapse/storage/{databases/main/schema => schema/main}/delta/55/access_token_expiry.sql (100%)
 rename synapse/storage/{databases/main/schema => schema/main}/delta/55/track_threepid_validations.sql (100%)
 rename synapse/storage/{databases/main/schema => schema/main}/delta/55/users_alter_deactivated.sql (100%)
 rename synapse/storage/{databases/main/schema => schema/main}/delta/56/add_spans_to_device_lists.sql (100%)
 rename synapse/storage/{databases/main/schema => schema/main}/delta/56/current_state_events_membership.sql (100%)
 rename synapse/storage/{databases/main/schema => schema/main}/delta/56/current_state_events_membership_mk2.sql (100%)
 rename synapse/storage/{databases/main/schema => schema/main}/delta/56/delete_keys_from_deleted_backups.sql (100%)
 rename synapse/storage/{databases/main/schema => schema/main}/delta/56/destinations_failure_ts.sql (100%)
 rename synapse/storage/{databases/main/schema => schema/main}/delta/56/destinations_retry_interval_type.sql.postgres (100%)
 rename synapse/storage/{databases/main/schema => schema/main}/delta/56/device_stream_id_insert.sql (100%)
 rename synapse/storage/{databases/main/schema => schema/main}/delta/56/devices_last_seen.sql (100%)
 rename synapse/storage/{databases/main/schema => schema/main}/delta/56/drop_unused_event_tables.sql (100%)
 rename synapse/storage/{databases/main/schema => schema/main}/delta/56/event_expiry.sql (100%)
 rename synapse/storage/{databases/main/schema => schema/main}/delta/56/event_labels.sql (100%)
 rename synapse/storage/{databases/main/schema => schema/main}/delta/56/event_labels_background_update.sql (100%)
 rename synapse/storage/{databases/main/schema => schema/main}/delta/56/fix_room_keys_index.sql (100%)
 rename synapse/storage/{databases/main/schema => schema/main}/delta/56/hidden_devices.sql (100%)
 rename synapse/storage/{databases/main/schema => schema/main}/delta/56/hidden_devices_fix.sql.sqlite (100%)
 rename synapse/storage/{databases/main/schema => schema/main}/delta/56/nuke_empty_communities_from_db.sql (100%)
 rename synapse/storage/{databases/main/schema => schema/main}/delta/56/public_room_list_idx.sql (100%)
 rename synapse/storage/{databases/main/schema => schema/main}/delta/56/redaction_censor.sql (100%)
 rename synapse/storage/{databases/main/schema => schema/main}/delta/56/redaction_censor2.sql (100%)
 rename synapse/storage/{databases/main/schema => schema/main}/delta/56/redaction_censor3_fix_update.sql.postgres (100%)
 rename synapse/storage/{databases/main/schema => schema/main}/delta/56/redaction_censor4.sql (100%)
 rename synapse/storage/{databases/main/schema => schema/main}/delta/56/remove_tombstoned_rooms_from_directory.sql (100%)
 rename synapse/storage/{databases/main/schema => schema/main}/delta/56/room_key_etag.sql (100%)
 rename synapse/storage/{databases/main/schema => schema/main}/delta/56/room_membership_idx.sql (100%)
 rename synapse/storage/{databases/main/schema => schema/main}/delta/56/room_retention.sql (100%)
 rename synapse/storage/{databases/main/schema => schema/main}/delta/56/signing_keys.sql (100%)
 rename synapse/storage/{databases/main/schema => schema/main}/delta/56/signing_keys_nonunique_signatures.sql (100%)
 rename synapse/storage/{databases/main/schema => schema/main}/delta/56/stats_separated.sql (100%)
 rename synapse/storage/{databases/main/schema => schema/main}/delta/56/unique_user_filter_index.py (100%)
 rename synapse/storage/{databases/main/schema => schema/main}/delta/56/user_external_ids.sql (100%)
 rename synapse/storage/{databases/main/schema => schema/main}/delta/56/users_in_public_rooms_idx.sql (100%)
 rename synapse/storage/{databases/main/schema => schema/main}/delta/57/delete_old_current_state_events.sql (100%)
 rename synapse/storage/{databases/main/schema => schema/main}/delta/57/device_list_remote_cache_stale.sql (100%)
 rename synapse/storage/{databases/main/schema => schema/main}/delta/57/local_current_membership.py (100%)
 rename synapse/storage/{databases/main/schema => schema/main}/delta/57/remove_sent_outbound_pokes.sql (100%)
 rename synapse/storage/{databases/main/schema => schema/main}/delta/57/rooms_version_column.sql (100%)
 rename synapse/storage/{databases/main/schema => schema/main}/delta/57/rooms_version_column_2.sql.postgres (100%)
 rename synapse/storage/{databases/main/schema => schema/main}/delta/57/rooms_version_column_2.sql.sqlite (100%)
 rename synapse/storage/{databases/main/schema => schema/main}/delta/57/rooms_version_column_3.sql.postgres (100%)
 rename synapse/storage/{databases/main/schema => schema/main}/delta/57/rooms_version_column_3.sql.sqlite (100%)
 rename synapse/storage/{databases/main/schema => schema/main}/delta/58/02remove_dup_outbound_pokes.sql (100%)
 rename synapse/storage/{databases/main/schema => schema/main}/delta/58/03persist_ui_auth.sql (100%)
 rename synapse/storage/{databases/main/schema => schema/main}/delta/58/05cache_instance.sql.postgres (100%)
 rename synapse/storage/{databases/main/schema => schema/main}/delta/58/06dlols_unique_idx.py (100%)
 rename synapse/storage/{databases/main/schema => schema/main}/delta/58/07add_method_to_thumbnail_constraint.sql.postgres (100%)
 rename synapse/storage/{databases/main/schema => schema/main}/delta/58/07add_method_to_thumbnail_constraint.sql.sqlite (100%)
 rename synapse/storage/{databases/main/schema => schema/main}/delta/58/07persist_ui_auth_ips.sql (100%)
 rename synapse/storage/{databases/main/schema => schema/main}/delta/58/08_media_safe_from_quarantine.sql.postgres (100%)
 rename synapse/storage/{databases/main/schema => schema/main}/delta/58/08_media_safe_from_quarantine.sql.sqlite (100%)
 rename synapse/storage/{databases/main/schema => schema/main}/delta/58/09shadow_ban.sql (100%)
 rename synapse/storage/{databases/main/schema => schema/main}/delta/58/10_pushrules_enabled_delete_obsolete.sql (100%)
 rename synapse/storage/{databases/main/schema => schema/main}/delta/58/10drop_local_rejections_stream.sql (100%)
 rename synapse/storage/{databases/main/schema => schema/main}/delta/58/10federation_pos_instance_name.sql (100%)
 rename synapse/storage/{databases/main/schema => schema/main}/delta/58/11dehydration.sql (100%)
 rename synapse/storage/{databases/main/schema => schema/main}/delta/58/11fallback.sql (100%)
 rename synapse/storage/{databases/main/schema => schema/main}/delta/58/11user_id_seq.py (100%)
 rename synapse/storage/{databases/main/schema => schema/main}/delta/58/12room_stats.sql (100%)
 rename synapse/storage/{databases/main/schema => schema/main}/delta/58/13remove_presence_allow_inbound.sql (100%)
 rename synapse/storage/{databases/main/schema => schema/main}/delta/58/14events_instance_name.sql (100%)
 rename synapse/storage/{databases/main/schema => schema/main}/delta/58/14events_instance_name.sql.postgres (100%)
 rename synapse/storage/{databases/main/schema => schema/main}/delta/58/15_catchup_destination_rooms.sql (100%)
 rename synapse/storage/{databases/main/schema => schema/main}/delta/58/15unread_count.sql (100%)
 rename synapse/storage/{databases/main/schema => schema/main}/delta/58/16populate_stats_process_rooms_fix.sql (100%)
 rename synapse/storage/{databases/main/schema => schema/main}/delta/58/17_catchup_last_successful.sql (100%)
 rename synapse/storage/{databases/main/schema => schema/main}/delta/58/18stream_positions.sql (100%)
 rename synapse/storage/{databases/main/schema => schema/main}/delta/58/19instance_map.sql.postgres (100%)
 rename synapse/storage/{databases/main/schema => schema/main}/delta/58/19txn_id.sql (100%)
 rename synapse/storage/{databases/main/schema => schema/main}/delta/58/20instance_name_event_tables.sql (100%)
 rename synapse/storage/{databases/main/schema => schema/main}/delta/58/20user_daily_visits.sql (100%)
 rename synapse/storage/{databases/main/schema => schema/main}/delta/58/21as_device_stream.sql (100%)
 rename synapse/storage/{databases/main/schema => schema/main}/delta/58/21drop_device_max_stream_id.sql (100%)
 rename synapse/storage/{databases/main/schema => schema/main}/delta/58/22puppet_token.sql (100%)
 rename synapse/storage/{databases/main/schema => schema/main}/delta/58/22users_have_local_media.sql (100%)
 rename synapse/storage/{databases/main/schema => schema/main}/delta/58/23e2e_cross_signing_keys_idx.sql (100%)
 rename synapse/storage/{databases/main/schema => schema/main}/delta/58/24drop_event_json_index.sql (100%)
 rename synapse/storage/{databases/main/schema => schema/main}/delta/58/25user_external_ids_user_id_idx.sql (100%)
 rename synapse/storage/{databases/main/schema => schema/main}/delta/58/26access_token_last_validated.sql (100%)
 rename synapse/storage/{databases/main/schema => schema/main}/delta/58/27local_invites.sql (100%)
 rename synapse/storage/{databases/main/schema => schema/main}/delta/58/28drop_last_used_column.sql.postgres (100%)
 rename synapse/storage/{databases/main/schema => schema/main}/delta/58/28drop_last_used_column.sql.sqlite (100%)
 rename synapse/storage/{databases/main/schema => schema/main}/delta/59/01ignored_user.py (100%)
 rename synapse/storage/{databases/main/schema => schema/main}/delta/59/02shard_send_to_device.sql (100%)
 rename synapse/storage/{databases/main/schema => schema/main}/delta/59/03shard_send_to_device_sequence.sql.postgres (100%)
 rename synapse/storage/{databases/main/schema => schema/main}/delta/59/04_event_auth_chains.sql (100%)
 rename synapse/storage/{databases/main/schema => schema/main}/delta/59/04_event_auth_chains.sql.postgres (100%)
 rename synapse/storage/{databases/main/schema => schema/main}/delta/59/04drop_account_data.sql (100%)
 rename synapse/storage/{databases/main/schema => schema/main}/delta/59/05cache_invalidation.sql (100%)
 rename synapse/storage/{databases/main/schema => schema/main}/delta/59/06chain_cover_index.sql (100%)
 rename synapse/storage/{databases/main/schema => schema/main}/delta/59/06shard_account_data.sql (100%)
 rename synapse/storage/{databases/main/schema => schema/main}/delta/59/06shard_account_data.sql.postgres (100%)
 rename synapse/storage/{databases/main/schema => schema/main}/delta/59/07shard_account_data_fix.sql (100%)
 rename synapse/storage/{databases/main/schema => schema/main}/delta/59/08delete_pushers_for_deactivated_accounts.sql (100%)
 rename synapse/storage/{databases/main/schema => schema/main}/delta/59/08delete_stale_pushers.sql (100%)
 rename synapse/storage/{databases/main/schema => schema/main}/delta/59/09rejected_events_metadata.sql (100%)
 rename synapse/storage/{databases/main/schema => schema/main}/delta/59/10delete_purged_chain_cover.sql (100%)
 rename synapse/storage/{databases/main/schema => schema/main}/delta/59/11drop_thumbnail_constraint.sql.postgres (100%)
 rename synapse/storage/{databases/main/schema => schema/main}/delta/59/12account_validity_token_used_ts_ms.sql (100%)
 rename synapse/storage/{databases/main/schema => schema/main}/delta/59/12presence_stream_instance.sql (100%)
 rename synapse/storage/{databases/main/schema => schema/main}/delta/59/12presence_stream_instance_seq.sql.postgres (100%)
 rename synapse/storage/{databases/main/schema => schema/main}/full_schemas/16/application_services.sql (100%)
 rename synapse/storage/{databases/main/schema => schema/main}/full_schemas/16/event_edges.sql (100%)
 rename synapse/storage/{databases/main/schema => schema/main}/full_schemas/16/event_signatures.sql (100%)
 rename synapse/storage/{databases/main/schema => schema/main}/full_schemas/16/im.sql (100%)
 rename synapse/storage/{databases/main/schema => schema/main}/full_schemas/16/keys.sql (100%)
 rename synapse/storage/{databases/main/schema => schema/main}/full_schemas/16/media_repository.sql (100%)
 rename synapse/storage/{databases/main/schema => schema/main}/full_schemas/16/presence.sql (100%)
 rename synapse/storage/{databases/main/schema => schema/main}/full_schemas/16/profiles.sql (100%)
 rename synapse/storage/{databases/main/schema => schema/main}/full_schemas/16/push.sql (100%)
 rename synapse/storage/{databases/main/schema => schema/main}/full_schemas/16/redactions.sql (100%)
 rename synapse/storage/{databases/main/schema => schema/main}/full_schemas/16/room_aliases.sql (100%)
 rename synapse/storage/{databases/main/schema => schema/main}/full_schemas/16/state.sql (100%)
 rename synapse/storage/{databases/main/schema => schema/main}/full_schemas/16/transactions.sql (100%)
 rename synapse/storage/{databases/main/schema => schema/main}/full_schemas/16/users.sql (100%)
 rename synapse/storage/{databases/main/schema => schema/main}/full_schemas/54/full.sql.postgres (100%)
 rename synapse/storage/{databases/main/schema => schema/main}/full_schemas/54/full.sql.sqlite (100%)
 rename synapse/storage/{databases/main/schema => schema/main}/full_schemas/54/stream_positions.sql (100%)
 rename synapse/storage/{databases/state/schema => schema/state}/delta/23/drop_state_index.sql (100%)
 rename synapse/storage/{databases/state/schema => schema/state}/delta/30/state_stream.sql (100%)
 rename synapse/storage/{databases/state/schema => schema/state}/delta/32/remove_state_indices.sql (100%)
 rename synapse/storage/{databases/state/schema => schema/state}/delta/35/add_state_index.sql (100%)
 rename synapse/storage/{databases/state/schema => schema/state}/delta/35/state.sql (100%)
 rename synapse/storage/{databases/state/schema => schema/state}/delta/35/state_dedupe.sql (100%)
 rename synapse/storage/{databases/state/schema => schema/state}/delta/47/state_group_seq.py (100%)
 rename synapse/storage/{databases/state/schema => schema/state}/delta/56/state_group_room_idx.sql (100%)
 rename synapse/storage/{databases/state/schema => schema/state}/full_schemas/54/full.sql (100%)
 rename synapse/storage/{databases/state/schema => schema/state}/full_schemas/54/sequence.sql.postgres (100%)

diff --git a/changelog.d/9932.misc b/changelog.d/9932.misc
new file mode 100644
index 0000000000..9e16a36173
--- /dev/null
+++ b/changelog.d/9932.misc
@@ -0,0 +1 @@
+Move database schema files into a common directory.
diff --git a/synapse/storage/databases/main/schema/full_schemas/README.md b/synapse/storage/databases/main/schema/full_schemas/README.md
deleted file mode 100644
index c00f287190..0000000000
--- a/synapse/storage/databases/main/schema/full_schemas/README.md
+++ /dev/null
@@ -1,21 +0,0 @@
-# Synapse Database Schemas
-
-These schemas are used as a basis to create brand new Synapse databases, on both
-SQLite3 and Postgres.
-
-## Building full schema dumps
-
-If you want to recreate these schemas, they need to be made from a database that
-has had all background updates run.
-
-To do so, use `scripts-dev/make_full_schema.sh`. This will produce new
-`full.sql.postgres ` and `full.sql.sqlite` files. 
-
-Ensure postgres is installed and your user has the ability to run bash commands
-such as `createdb`, then call
-
-    ./scripts-dev/make_full_schema.sh -p postgres_username -o output_dir/
-
-There are currently two folders with full-schema snapshots. `16` is a snapshot
-from 2015, for historical reference. The other contains the most recent full
-schema snapshot.
diff --git a/synapse/storage/prepare_database.py b/synapse/storage/prepare_database.py
index 7a2cbee426..3799d46734 100644
--- a/synapse/storage/prepare_database.py
+++ b/synapse/storage/prepare_database.py
@@ -26,16 +26,13 @@ from synapse.config.homeserver import HomeServerConfig
 from synapse.storage.database import LoggingDatabaseConnection
 from synapse.storage.engines import BaseDatabaseEngine
 from synapse.storage.engines.postgres import PostgresEngine
+from synapse.storage.schema import SCHEMA_VERSION
 from synapse.storage.types import Cursor
 
 logger = logging.getLogger(__name__)
 
 
-# Remember to update this number every time a change is made to database
-# schema files, so the users will be informed on server restarts.
-SCHEMA_VERSION = 59
-
-dir_path = os.path.abspath(os.path.dirname(__file__))
+schema_path = os.path.join(os.path.abspath(os.path.dirname(__file__)), "schema")
 
 
 class PrepareDatabaseException(Exception):
@@ -167,7 +164,14 @@ def _setup_new_database(
 
     Example directory structure:
 
-        schema/
+    schema/
+        common/
+            delta/
+                ...
+            full_schemas/
+                11/
+                    foo.sql
+        main/
             delta/
                 ...
             full_schemas/
@@ -175,15 +179,14 @@ def _setup_new_database(
                     test.sql
                     ...
                 11/
-                    foo.sql
                     bar.sql
                 ...
 
     In the example foo.sql and bar.sql would be run, and then any delta files
     for versions strictly greater than 11.
 
-    Note: we apply the full schemas and deltas from the top level `schema/`
-    folder as well those in the data stores specified.
+    Note: we apply the full schemas and deltas from the `schema/common`
+    folder as well those in the databases specified.
 
     Args:
         cur: a database cursor
@@ -195,12 +198,12 @@ def _setup_new_database(
     # configured to our liking.
     database_engine.check_new_database(cur)
 
-    current_dir = os.path.join(dir_path, "schema", "full_schemas")
+    full_schemas_dir = os.path.join(schema_path, "common", "full_schemas")
 
     # First we find the highest full schema version we have
     valid_versions = []
 
-    for filename in os.listdir(current_dir):
+    for filename in os.listdir(full_schemas_dir):
         try:
             ver = int(filename)
         except ValueError:
@@ -218,15 +221,13 @@ def _setup_new_database(
 
     logger.debug("Initialising schema v%d", max_current_ver)
 
-    # Now lets find all the full schema files, both in the global schema and
-    # in data store schemas.
-    directories = [os.path.join(current_dir, str(max_current_ver))]
+    # Now let's find all the full schema files, both in the common schema and
+    # in database schemas.
+    directories = [os.path.join(full_schemas_dir, str(max_current_ver))]
     directories.extend(
         os.path.join(
-            dir_path,
-            "databases",
+            schema_path,
             database,
-            "schema",
             "full_schemas",
             str(max_current_ver),
         )
@@ -357,6 +358,9 @@ def _upgrade_existing_database(
         check_database_before_upgrade(cur, database_engine, config)
 
     start_ver = current_version
+
+    # if we got to this schema version by running a full_schema rather than a series
+    # of deltas, we should not run the deltas for this version.
     if not upgraded:
         start_ver += 1
 
@@ -385,12 +389,10 @@ def _upgrade_existing_database(
         # directories for schema updates.
 
         # First we find the directories to search in
-        delta_dir = os.path.join(dir_path, "schema", "delta", str(v))
+        delta_dir = os.path.join(schema_path, "common", "delta", str(v))
         directories = [delta_dir]
         for database in databases:
-            directories.append(
-                os.path.join(dir_path, "databases", database, "schema", "delta", str(v))
-            )
+            directories.append(os.path.join(schema_path, database, "delta", str(v)))
 
         # Used to check if we have any duplicate file names
         file_name_counter = Counter()  # type: CounterType[str]
@@ -621,8 +623,8 @@ def _get_or_create_schema_state(
     txn: Cursor, database_engine: BaseDatabaseEngine
 ) -> Optional[Tuple[int, List[str], bool]]:
     # Bluntly try creating the schema_version tables.
-    schema_path = os.path.join(dir_path, "schema", "schema_version.sql")
-    executescript(txn, schema_path)
+    sql_path = os.path.join(schema_path, "common", "schema_version.sql")
+    executescript(txn, sql_path)
 
     txn.execute("SELECT version, upgraded FROM schema_version")
     row = txn.fetchone()
diff --git a/synapse/storage/schema/README.md b/synapse/storage/schema/README.md
new file mode 100644
index 0000000000..030153db64
--- /dev/null
+++ b/synapse/storage/schema/README.md
@@ -0,0 +1,37 @@
+# Synapse Database Schemas
+
+This directory contains the schema files used to build Synapse databases.
+
+Synapse supports splitting its datastore across multiple physical databases (which can
+be useful for large installations), and the schema files are therefore split according
+to the logical database they are apply to.
+
+At the time of writing, the following "logical" databases are supported:
+
+* `state` - used to store Matrix room state (more specifically, `state_groups`,
+  their relationships and contents.)
+* `main` - stores everything else.
+
+Addionally, the `common` directory contains schema files for tables which must be
+present on *all* physical databases.
+
+## Full schema dumps
+
+In the `full_schemas` directories, only the most recently-numbered snapshot is useful
+(`54` at the time of writing). Older snapshots (eg, `16`) are present for historical
+reference only.
+
+## Building full schema dumps
+
+If you want to recreate these schemas, they need to be made from a database that
+has had all background updates run.
+
+To do so, use `scripts-dev/make_full_schema.sh`. This will produce new
+`full.sql.postgres` and `full.sql.sqlite` files.
+
+Ensure postgres is installed, then run:
+
+    ./scripts-dev/make_full_schema.sh -p postgres_username -o output_dir/
+
+NB at the time of writing, this script predates the split into separate `state`/`main`
+databases so will require updates to handle that correctly.
diff --git a/synapse/storage/schema/__init__.py b/synapse/storage/schema/__init__.py
new file mode 100644
index 0000000000..f0d9f23167
--- /dev/null
+++ b/synapse/storage/schema/__init__.py
@@ -0,0 +1,17 @@
+# Copyright 2021 The Matrix.org Foundation C.I.C.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Remember to update this number every time a change is made to database
+# schema files, so the users will be informed on server restarts.
+SCHEMA_VERSION = 59
diff --git a/synapse/storage/schema/delta/25/00background_updates.sql b/synapse/storage/schema/common/delta/25/00background_updates.sql
similarity index 100%
rename from synapse/storage/schema/delta/25/00background_updates.sql
rename to synapse/storage/schema/common/delta/25/00background_updates.sql
diff --git a/synapse/storage/schema/delta/35/00background_updates_add_col.sql b/synapse/storage/schema/common/delta/35/00background_updates_add_col.sql
similarity index 100%
rename from synapse/storage/schema/delta/35/00background_updates_add_col.sql
rename to synapse/storage/schema/common/delta/35/00background_updates_add_col.sql
diff --git a/synapse/storage/schema/delta/58/00background_update_ordering.sql b/synapse/storage/schema/common/delta/58/00background_update_ordering.sql
similarity index 100%
rename from synapse/storage/schema/delta/58/00background_update_ordering.sql
rename to synapse/storage/schema/common/delta/58/00background_update_ordering.sql
diff --git a/synapse/storage/schema/full_schemas/54/full.sql b/synapse/storage/schema/common/full_schemas/54/full.sql
similarity index 100%
rename from synapse/storage/schema/full_schemas/54/full.sql
rename to synapse/storage/schema/common/full_schemas/54/full.sql
diff --git a/synapse/storage/schema/schema_version.sql b/synapse/storage/schema/common/schema_version.sql
similarity index 100%
rename from synapse/storage/schema/schema_version.sql
rename to synapse/storage/schema/common/schema_version.sql
diff --git a/synapse/storage/databases/main/schema/delta/12/v12.sql b/synapse/storage/schema/main/delta/12/v12.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/12/v12.sql
rename to synapse/storage/schema/main/delta/12/v12.sql
diff --git a/synapse/storage/databases/main/schema/delta/13/v13.sql b/synapse/storage/schema/main/delta/13/v13.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/13/v13.sql
rename to synapse/storage/schema/main/delta/13/v13.sql
diff --git a/synapse/storage/databases/main/schema/delta/14/v14.sql b/synapse/storage/schema/main/delta/14/v14.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/14/v14.sql
rename to synapse/storage/schema/main/delta/14/v14.sql
diff --git a/synapse/storage/databases/main/schema/delta/15/appservice_txns.sql b/synapse/storage/schema/main/delta/15/appservice_txns.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/15/appservice_txns.sql
rename to synapse/storage/schema/main/delta/15/appservice_txns.sql
diff --git a/synapse/storage/databases/main/schema/delta/15/presence_indices.sql b/synapse/storage/schema/main/delta/15/presence_indices.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/15/presence_indices.sql
rename to synapse/storage/schema/main/delta/15/presence_indices.sql
diff --git a/synapse/storage/databases/main/schema/delta/15/v15.sql b/synapse/storage/schema/main/delta/15/v15.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/15/v15.sql
rename to synapse/storage/schema/main/delta/15/v15.sql
diff --git a/synapse/storage/databases/main/schema/delta/16/events_order_index.sql b/synapse/storage/schema/main/delta/16/events_order_index.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/16/events_order_index.sql
rename to synapse/storage/schema/main/delta/16/events_order_index.sql
diff --git a/synapse/storage/databases/main/schema/delta/16/remote_media_cache_index.sql b/synapse/storage/schema/main/delta/16/remote_media_cache_index.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/16/remote_media_cache_index.sql
rename to synapse/storage/schema/main/delta/16/remote_media_cache_index.sql
diff --git a/synapse/storage/databases/main/schema/delta/16/remove_duplicates.sql b/synapse/storage/schema/main/delta/16/remove_duplicates.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/16/remove_duplicates.sql
rename to synapse/storage/schema/main/delta/16/remove_duplicates.sql
diff --git a/synapse/storage/databases/main/schema/delta/16/room_alias_index.sql b/synapse/storage/schema/main/delta/16/room_alias_index.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/16/room_alias_index.sql
rename to synapse/storage/schema/main/delta/16/room_alias_index.sql
diff --git a/synapse/storage/databases/main/schema/delta/16/unique_constraints.sql b/synapse/storage/schema/main/delta/16/unique_constraints.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/16/unique_constraints.sql
rename to synapse/storage/schema/main/delta/16/unique_constraints.sql
diff --git a/synapse/storage/databases/main/schema/delta/16/users.sql b/synapse/storage/schema/main/delta/16/users.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/16/users.sql
rename to synapse/storage/schema/main/delta/16/users.sql
diff --git a/synapse/storage/databases/main/schema/delta/17/drop_indexes.sql b/synapse/storage/schema/main/delta/17/drop_indexes.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/17/drop_indexes.sql
rename to synapse/storage/schema/main/delta/17/drop_indexes.sql
diff --git a/synapse/storage/databases/main/schema/delta/17/server_keys.sql b/synapse/storage/schema/main/delta/17/server_keys.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/17/server_keys.sql
rename to synapse/storage/schema/main/delta/17/server_keys.sql
diff --git a/synapse/storage/databases/main/schema/delta/17/user_threepids.sql b/synapse/storage/schema/main/delta/17/user_threepids.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/17/user_threepids.sql
rename to synapse/storage/schema/main/delta/17/user_threepids.sql
diff --git a/synapse/storage/databases/main/schema/delta/18/server_keys_bigger_ints.sql b/synapse/storage/schema/main/delta/18/server_keys_bigger_ints.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/18/server_keys_bigger_ints.sql
rename to synapse/storage/schema/main/delta/18/server_keys_bigger_ints.sql
diff --git a/synapse/storage/databases/main/schema/delta/19/event_index.sql b/synapse/storage/schema/main/delta/19/event_index.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/19/event_index.sql
rename to synapse/storage/schema/main/delta/19/event_index.sql
diff --git a/synapse/storage/databases/main/schema/delta/20/dummy.sql b/synapse/storage/schema/main/delta/20/dummy.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/20/dummy.sql
rename to synapse/storage/schema/main/delta/20/dummy.sql
diff --git a/synapse/storage/databases/main/schema/delta/20/pushers.py b/synapse/storage/schema/main/delta/20/pushers.py
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/20/pushers.py
rename to synapse/storage/schema/main/delta/20/pushers.py
diff --git a/synapse/storage/databases/main/schema/delta/21/end_to_end_keys.sql b/synapse/storage/schema/main/delta/21/end_to_end_keys.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/21/end_to_end_keys.sql
rename to synapse/storage/schema/main/delta/21/end_to_end_keys.sql
diff --git a/synapse/storage/databases/main/schema/delta/21/receipts.sql b/synapse/storage/schema/main/delta/21/receipts.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/21/receipts.sql
rename to synapse/storage/schema/main/delta/21/receipts.sql
diff --git a/synapse/storage/databases/main/schema/delta/22/receipts_index.sql b/synapse/storage/schema/main/delta/22/receipts_index.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/22/receipts_index.sql
rename to synapse/storage/schema/main/delta/22/receipts_index.sql
diff --git a/synapse/storage/databases/main/schema/delta/22/user_threepids_unique.sql b/synapse/storage/schema/main/delta/22/user_threepids_unique.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/22/user_threepids_unique.sql
rename to synapse/storage/schema/main/delta/22/user_threepids_unique.sql
diff --git a/synapse/storage/databases/main/schema/delta/24/stats_reporting.sql b/synapse/storage/schema/main/delta/24/stats_reporting.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/24/stats_reporting.sql
rename to synapse/storage/schema/main/delta/24/stats_reporting.sql
diff --git a/synapse/storage/databases/main/schema/delta/25/fts.py b/synapse/storage/schema/main/delta/25/fts.py
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/25/fts.py
rename to synapse/storage/schema/main/delta/25/fts.py
diff --git a/synapse/storage/databases/main/schema/delta/25/guest_access.sql b/synapse/storage/schema/main/delta/25/guest_access.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/25/guest_access.sql
rename to synapse/storage/schema/main/delta/25/guest_access.sql
diff --git a/synapse/storage/databases/main/schema/delta/25/history_visibility.sql b/synapse/storage/schema/main/delta/25/history_visibility.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/25/history_visibility.sql
rename to synapse/storage/schema/main/delta/25/history_visibility.sql
diff --git a/synapse/storage/databases/main/schema/delta/25/tags.sql b/synapse/storage/schema/main/delta/25/tags.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/25/tags.sql
rename to synapse/storage/schema/main/delta/25/tags.sql
diff --git a/synapse/storage/databases/main/schema/delta/26/account_data.sql b/synapse/storage/schema/main/delta/26/account_data.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/26/account_data.sql
rename to synapse/storage/schema/main/delta/26/account_data.sql
diff --git a/synapse/storage/databases/main/schema/delta/27/account_data.sql b/synapse/storage/schema/main/delta/27/account_data.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/27/account_data.sql
rename to synapse/storage/schema/main/delta/27/account_data.sql
diff --git a/synapse/storage/databases/main/schema/delta/27/forgotten_memberships.sql b/synapse/storage/schema/main/delta/27/forgotten_memberships.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/27/forgotten_memberships.sql
rename to synapse/storage/schema/main/delta/27/forgotten_memberships.sql
diff --git a/synapse/storage/databases/main/schema/delta/27/ts.py b/synapse/storage/schema/main/delta/27/ts.py
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/27/ts.py
rename to synapse/storage/schema/main/delta/27/ts.py
diff --git a/synapse/storage/databases/main/schema/delta/28/event_push_actions.sql b/synapse/storage/schema/main/delta/28/event_push_actions.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/28/event_push_actions.sql
rename to synapse/storage/schema/main/delta/28/event_push_actions.sql
diff --git a/synapse/storage/databases/main/schema/delta/28/events_room_stream.sql b/synapse/storage/schema/main/delta/28/events_room_stream.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/28/events_room_stream.sql
rename to synapse/storage/schema/main/delta/28/events_room_stream.sql
diff --git a/synapse/storage/databases/main/schema/delta/28/public_roms_index.sql b/synapse/storage/schema/main/delta/28/public_roms_index.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/28/public_roms_index.sql
rename to synapse/storage/schema/main/delta/28/public_roms_index.sql
diff --git a/synapse/storage/databases/main/schema/delta/28/receipts_user_id_index.sql b/synapse/storage/schema/main/delta/28/receipts_user_id_index.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/28/receipts_user_id_index.sql
rename to synapse/storage/schema/main/delta/28/receipts_user_id_index.sql
diff --git a/synapse/storage/databases/main/schema/delta/28/upgrade_times.sql b/synapse/storage/schema/main/delta/28/upgrade_times.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/28/upgrade_times.sql
rename to synapse/storage/schema/main/delta/28/upgrade_times.sql
diff --git a/synapse/storage/databases/main/schema/delta/28/users_is_guest.sql b/synapse/storage/schema/main/delta/28/users_is_guest.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/28/users_is_guest.sql
rename to synapse/storage/schema/main/delta/28/users_is_guest.sql
diff --git a/synapse/storage/databases/main/schema/delta/29/push_actions.sql b/synapse/storage/schema/main/delta/29/push_actions.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/29/push_actions.sql
rename to synapse/storage/schema/main/delta/29/push_actions.sql
diff --git a/synapse/storage/databases/main/schema/delta/30/alias_creator.sql b/synapse/storage/schema/main/delta/30/alias_creator.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/30/alias_creator.sql
rename to synapse/storage/schema/main/delta/30/alias_creator.sql
diff --git a/synapse/storage/databases/main/schema/delta/30/as_users.py b/synapse/storage/schema/main/delta/30/as_users.py
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/30/as_users.py
rename to synapse/storage/schema/main/delta/30/as_users.py
diff --git a/synapse/storage/databases/main/schema/delta/30/deleted_pushers.sql b/synapse/storage/schema/main/delta/30/deleted_pushers.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/30/deleted_pushers.sql
rename to synapse/storage/schema/main/delta/30/deleted_pushers.sql
diff --git a/synapse/storage/databases/main/schema/delta/30/presence_stream.sql b/synapse/storage/schema/main/delta/30/presence_stream.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/30/presence_stream.sql
rename to synapse/storage/schema/main/delta/30/presence_stream.sql
diff --git a/synapse/storage/databases/main/schema/delta/30/public_rooms.sql b/synapse/storage/schema/main/delta/30/public_rooms.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/30/public_rooms.sql
rename to synapse/storage/schema/main/delta/30/public_rooms.sql
diff --git a/synapse/storage/databases/main/schema/delta/30/push_rule_stream.sql b/synapse/storage/schema/main/delta/30/push_rule_stream.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/30/push_rule_stream.sql
rename to synapse/storage/schema/main/delta/30/push_rule_stream.sql
diff --git a/synapse/storage/databases/main/schema/delta/30/threepid_guest_access_tokens.sql b/synapse/storage/schema/main/delta/30/threepid_guest_access_tokens.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/30/threepid_guest_access_tokens.sql
rename to synapse/storage/schema/main/delta/30/threepid_guest_access_tokens.sql
diff --git a/synapse/storage/databases/main/schema/delta/31/invites.sql b/synapse/storage/schema/main/delta/31/invites.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/31/invites.sql
rename to synapse/storage/schema/main/delta/31/invites.sql
diff --git a/synapse/storage/databases/main/schema/delta/31/local_media_repository_url_cache.sql b/synapse/storage/schema/main/delta/31/local_media_repository_url_cache.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/31/local_media_repository_url_cache.sql
rename to synapse/storage/schema/main/delta/31/local_media_repository_url_cache.sql
diff --git a/synapse/storage/databases/main/schema/delta/31/pushers.py b/synapse/storage/schema/main/delta/31/pushers.py
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/31/pushers.py
rename to synapse/storage/schema/main/delta/31/pushers.py
diff --git a/synapse/storage/databases/main/schema/delta/31/pushers_index.sql b/synapse/storage/schema/main/delta/31/pushers_index.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/31/pushers_index.sql
rename to synapse/storage/schema/main/delta/31/pushers_index.sql
diff --git a/synapse/storage/databases/main/schema/delta/31/search_update.py b/synapse/storage/schema/main/delta/31/search_update.py
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/31/search_update.py
rename to synapse/storage/schema/main/delta/31/search_update.py
diff --git a/synapse/storage/databases/main/schema/delta/32/events.sql b/synapse/storage/schema/main/delta/32/events.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/32/events.sql
rename to synapse/storage/schema/main/delta/32/events.sql
diff --git a/synapse/storage/databases/main/schema/delta/32/openid.sql b/synapse/storage/schema/main/delta/32/openid.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/32/openid.sql
rename to synapse/storage/schema/main/delta/32/openid.sql
diff --git a/synapse/storage/databases/main/schema/delta/32/pusher_throttle.sql b/synapse/storage/schema/main/delta/32/pusher_throttle.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/32/pusher_throttle.sql
rename to synapse/storage/schema/main/delta/32/pusher_throttle.sql
diff --git a/synapse/storage/databases/main/schema/delta/32/remove_indices.sql b/synapse/storage/schema/main/delta/32/remove_indices.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/32/remove_indices.sql
rename to synapse/storage/schema/main/delta/32/remove_indices.sql
diff --git a/synapse/storage/databases/main/schema/delta/32/reports.sql b/synapse/storage/schema/main/delta/32/reports.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/32/reports.sql
rename to synapse/storage/schema/main/delta/32/reports.sql
diff --git a/synapse/storage/databases/main/schema/delta/33/access_tokens_device_index.sql b/synapse/storage/schema/main/delta/33/access_tokens_device_index.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/33/access_tokens_device_index.sql
rename to synapse/storage/schema/main/delta/33/access_tokens_device_index.sql
diff --git a/synapse/storage/databases/main/schema/delta/33/devices.sql b/synapse/storage/schema/main/delta/33/devices.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/33/devices.sql
rename to synapse/storage/schema/main/delta/33/devices.sql
diff --git a/synapse/storage/databases/main/schema/delta/33/devices_for_e2e_keys.sql b/synapse/storage/schema/main/delta/33/devices_for_e2e_keys.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/33/devices_for_e2e_keys.sql
rename to synapse/storage/schema/main/delta/33/devices_for_e2e_keys.sql
diff --git a/synapse/storage/databases/main/schema/delta/33/devices_for_e2e_keys_clear_unknown_device.sql b/synapse/storage/schema/main/delta/33/devices_for_e2e_keys_clear_unknown_device.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/33/devices_for_e2e_keys_clear_unknown_device.sql
rename to synapse/storage/schema/main/delta/33/devices_for_e2e_keys_clear_unknown_device.sql
diff --git a/synapse/storage/databases/main/schema/delta/33/event_fields.py b/synapse/storage/schema/main/delta/33/event_fields.py
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/33/event_fields.py
rename to synapse/storage/schema/main/delta/33/event_fields.py
diff --git a/synapse/storage/databases/main/schema/delta/33/remote_media_ts.py b/synapse/storage/schema/main/delta/33/remote_media_ts.py
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/33/remote_media_ts.py
rename to synapse/storage/schema/main/delta/33/remote_media_ts.py
diff --git a/synapse/storage/databases/main/schema/delta/33/user_ips_index.sql b/synapse/storage/schema/main/delta/33/user_ips_index.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/33/user_ips_index.sql
rename to synapse/storage/schema/main/delta/33/user_ips_index.sql
diff --git a/synapse/storage/databases/main/schema/delta/34/appservice_stream.sql b/synapse/storage/schema/main/delta/34/appservice_stream.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/34/appservice_stream.sql
rename to synapse/storage/schema/main/delta/34/appservice_stream.sql
diff --git a/synapse/storage/databases/main/schema/delta/34/cache_stream.py b/synapse/storage/schema/main/delta/34/cache_stream.py
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/34/cache_stream.py
rename to synapse/storage/schema/main/delta/34/cache_stream.py
diff --git a/synapse/storage/databases/main/schema/delta/34/device_inbox.sql b/synapse/storage/schema/main/delta/34/device_inbox.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/34/device_inbox.sql
rename to synapse/storage/schema/main/delta/34/device_inbox.sql
diff --git a/synapse/storage/databases/main/schema/delta/34/push_display_name_rename.sql b/synapse/storage/schema/main/delta/34/push_display_name_rename.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/34/push_display_name_rename.sql
rename to synapse/storage/schema/main/delta/34/push_display_name_rename.sql
diff --git a/synapse/storage/databases/main/schema/delta/34/received_txn_purge.py b/synapse/storage/schema/main/delta/34/received_txn_purge.py
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/34/received_txn_purge.py
rename to synapse/storage/schema/main/delta/34/received_txn_purge.py
diff --git a/synapse/storage/databases/main/schema/delta/35/contains_url.sql b/synapse/storage/schema/main/delta/35/contains_url.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/35/contains_url.sql
rename to synapse/storage/schema/main/delta/35/contains_url.sql
diff --git a/synapse/storage/databases/main/schema/delta/35/device_outbox.sql b/synapse/storage/schema/main/delta/35/device_outbox.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/35/device_outbox.sql
rename to synapse/storage/schema/main/delta/35/device_outbox.sql
diff --git a/synapse/storage/databases/main/schema/delta/35/device_stream_id.sql b/synapse/storage/schema/main/delta/35/device_stream_id.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/35/device_stream_id.sql
rename to synapse/storage/schema/main/delta/35/device_stream_id.sql
diff --git a/synapse/storage/databases/main/schema/delta/35/event_push_actions_index.sql b/synapse/storage/schema/main/delta/35/event_push_actions_index.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/35/event_push_actions_index.sql
rename to synapse/storage/schema/main/delta/35/event_push_actions_index.sql
diff --git a/synapse/storage/databases/main/schema/delta/35/public_room_list_change_stream.sql b/synapse/storage/schema/main/delta/35/public_room_list_change_stream.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/35/public_room_list_change_stream.sql
rename to synapse/storage/schema/main/delta/35/public_room_list_change_stream.sql
diff --git a/synapse/storage/databases/main/schema/delta/35/stream_order_to_extrem.sql b/synapse/storage/schema/main/delta/35/stream_order_to_extrem.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/35/stream_order_to_extrem.sql
rename to synapse/storage/schema/main/delta/35/stream_order_to_extrem.sql
diff --git a/synapse/storage/databases/main/schema/delta/36/readd_public_rooms.sql b/synapse/storage/schema/main/delta/36/readd_public_rooms.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/36/readd_public_rooms.sql
rename to synapse/storage/schema/main/delta/36/readd_public_rooms.sql
diff --git a/synapse/storage/databases/main/schema/delta/37/remove_auth_idx.py b/synapse/storage/schema/main/delta/37/remove_auth_idx.py
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/37/remove_auth_idx.py
rename to synapse/storage/schema/main/delta/37/remove_auth_idx.py
diff --git a/synapse/storage/databases/main/schema/delta/37/user_threepids.sql b/synapse/storage/schema/main/delta/37/user_threepids.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/37/user_threepids.sql
rename to synapse/storage/schema/main/delta/37/user_threepids.sql
diff --git a/synapse/storage/databases/main/schema/delta/38/postgres_fts_gist.sql b/synapse/storage/schema/main/delta/38/postgres_fts_gist.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/38/postgres_fts_gist.sql
rename to synapse/storage/schema/main/delta/38/postgres_fts_gist.sql
diff --git a/synapse/storage/databases/main/schema/delta/39/appservice_room_list.sql b/synapse/storage/schema/main/delta/39/appservice_room_list.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/39/appservice_room_list.sql
rename to synapse/storage/schema/main/delta/39/appservice_room_list.sql
diff --git a/synapse/storage/databases/main/schema/delta/39/device_federation_stream_idx.sql b/synapse/storage/schema/main/delta/39/device_federation_stream_idx.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/39/device_federation_stream_idx.sql
rename to synapse/storage/schema/main/delta/39/device_federation_stream_idx.sql
diff --git a/synapse/storage/databases/main/schema/delta/39/event_push_index.sql b/synapse/storage/schema/main/delta/39/event_push_index.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/39/event_push_index.sql
rename to synapse/storage/schema/main/delta/39/event_push_index.sql
diff --git a/synapse/storage/databases/main/schema/delta/39/federation_out_position.sql b/synapse/storage/schema/main/delta/39/federation_out_position.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/39/federation_out_position.sql
rename to synapse/storage/schema/main/delta/39/federation_out_position.sql
diff --git a/synapse/storage/databases/main/schema/delta/39/membership_profile.sql b/synapse/storage/schema/main/delta/39/membership_profile.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/39/membership_profile.sql
rename to synapse/storage/schema/main/delta/39/membership_profile.sql
diff --git a/synapse/storage/databases/main/schema/delta/40/current_state_idx.sql b/synapse/storage/schema/main/delta/40/current_state_idx.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/40/current_state_idx.sql
rename to synapse/storage/schema/main/delta/40/current_state_idx.sql
diff --git a/synapse/storage/databases/main/schema/delta/40/device_inbox.sql b/synapse/storage/schema/main/delta/40/device_inbox.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/40/device_inbox.sql
rename to synapse/storage/schema/main/delta/40/device_inbox.sql
diff --git a/synapse/storage/databases/main/schema/delta/40/device_list_streams.sql b/synapse/storage/schema/main/delta/40/device_list_streams.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/40/device_list_streams.sql
rename to synapse/storage/schema/main/delta/40/device_list_streams.sql
diff --git a/synapse/storage/databases/main/schema/delta/40/event_push_summary.sql b/synapse/storage/schema/main/delta/40/event_push_summary.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/40/event_push_summary.sql
rename to synapse/storage/schema/main/delta/40/event_push_summary.sql
diff --git a/synapse/storage/databases/main/schema/delta/40/pushers.sql b/synapse/storage/schema/main/delta/40/pushers.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/40/pushers.sql
rename to synapse/storage/schema/main/delta/40/pushers.sql
diff --git a/synapse/storage/databases/main/schema/delta/41/device_list_stream_idx.sql b/synapse/storage/schema/main/delta/41/device_list_stream_idx.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/41/device_list_stream_idx.sql
rename to synapse/storage/schema/main/delta/41/device_list_stream_idx.sql
diff --git a/synapse/storage/databases/main/schema/delta/41/device_outbound_index.sql b/synapse/storage/schema/main/delta/41/device_outbound_index.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/41/device_outbound_index.sql
rename to synapse/storage/schema/main/delta/41/device_outbound_index.sql
diff --git a/synapse/storage/databases/main/schema/delta/41/event_search_event_id_idx.sql b/synapse/storage/schema/main/delta/41/event_search_event_id_idx.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/41/event_search_event_id_idx.sql
rename to synapse/storage/schema/main/delta/41/event_search_event_id_idx.sql
diff --git a/synapse/storage/databases/main/schema/delta/41/ratelimit.sql b/synapse/storage/schema/main/delta/41/ratelimit.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/41/ratelimit.sql
rename to synapse/storage/schema/main/delta/41/ratelimit.sql
diff --git a/synapse/storage/databases/main/schema/delta/42/current_state_delta.sql b/synapse/storage/schema/main/delta/42/current_state_delta.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/42/current_state_delta.sql
rename to synapse/storage/schema/main/delta/42/current_state_delta.sql
diff --git a/synapse/storage/databases/main/schema/delta/42/device_list_last_id.sql b/synapse/storage/schema/main/delta/42/device_list_last_id.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/42/device_list_last_id.sql
rename to synapse/storage/schema/main/delta/42/device_list_last_id.sql
diff --git a/synapse/storage/databases/main/schema/delta/42/event_auth_state_only.sql b/synapse/storage/schema/main/delta/42/event_auth_state_only.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/42/event_auth_state_only.sql
rename to synapse/storage/schema/main/delta/42/event_auth_state_only.sql
diff --git a/synapse/storage/databases/main/schema/delta/42/user_dir.py b/synapse/storage/schema/main/delta/42/user_dir.py
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/42/user_dir.py
rename to synapse/storage/schema/main/delta/42/user_dir.py
diff --git a/synapse/storage/databases/main/schema/delta/43/blocked_rooms.sql b/synapse/storage/schema/main/delta/43/blocked_rooms.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/43/blocked_rooms.sql
rename to synapse/storage/schema/main/delta/43/blocked_rooms.sql
diff --git a/synapse/storage/databases/main/schema/delta/43/quarantine_media.sql b/synapse/storage/schema/main/delta/43/quarantine_media.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/43/quarantine_media.sql
rename to synapse/storage/schema/main/delta/43/quarantine_media.sql
diff --git a/synapse/storage/databases/main/schema/delta/43/url_cache.sql b/synapse/storage/schema/main/delta/43/url_cache.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/43/url_cache.sql
rename to synapse/storage/schema/main/delta/43/url_cache.sql
diff --git a/synapse/storage/databases/main/schema/delta/43/user_share.sql b/synapse/storage/schema/main/delta/43/user_share.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/43/user_share.sql
rename to synapse/storage/schema/main/delta/43/user_share.sql
diff --git a/synapse/storage/databases/main/schema/delta/44/expire_url_cache.sql b/synapse/storage/schema/main/delta/44/expire_url_cache.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/44/expire_url_cache.sql
rename to synapse/storage/schema/main/delta/44/expire_url_cache.sql
diff --git a/synapse/storage/databases/main/schema/delta/45/group_server.sql b/synapse/storage/schema/main/delta/45/group_server.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/45/group_server.sql
rename to synapse/storage/schema/main/delta/45/group_server.sql
diff --git a/synapse/storage/databases/main/schema/delta/45/profile_cache.sql b/synapse/storage/schema/main/delta/45/profile_cache.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/45/profile_cache.sql
rename to synapse/storage/schema/main/delta/45/profile_cache.sql
diff --git a/synapse/storage/databases/main/schema/delta/46/drop_refresh_tokens.sql b/synapse/storage/schema/main/delta/46/drop_refresh_tokens.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/46/drop_refresh_tokens.sql
rename to synapse/storage/schema/main/delta/46/drop_refresh_tokens.sql
diff --git a/synapse/storage/databases/main/schema/delta/46/drop_unique_deleted_pushers.sql b/synapse/storage/schema/main/delta/46/drop_unique_deleted_pushers.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/46/drop_unique_deleted_pushers.sql
rename to synapse/storage/schema/main/delta/46/drop_unique_deleted_pushers.sql
diff --git a/synapse/storage/databases/main/schema/delta/46/group_server.sql b/synapse/storage/schema/main/delta/46/group_server.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/46/group_server.sql
rename to synapse/storage/schema/main/delta/46/group_server.sql
diff --git a/synapse/storage/databases/main/schema/delta/46/local_media_repository_url_idx.sql b/synapse/storage/schema/main/delta/46/local_media_repository_url_idx.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/46/local_media_repository_url_idx.sql
rename to synapse/storage/schema/main/delta/46/local_media_repository_url_idx.sql
diff --git a/synapse/storage/databases/main/schema/delta/46/user_dir_null_room_ids.sql b/synapse/storage/schema/main/delta/46/user_dir_null_room_ids.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/46/user_dir_null_room_ids.sql
rename to synapse/storage/schema/main/delta/46/user_dir_null_room_ids.sql
diff --git a/synapse/storage/databases/main/schema/delta/46/user_dir_typos.sql b/synapse/storage/schema/main/delta/46/user_dir_typos.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/46/user_dir_typos.sql
rename to synapse/storage/schema/main/delta/46/user_dir_typos.sql
diff --git a/synapse/storage/databases/main/schema/delta/47/last_access_media.sql b/synapse/storage/schema/main/delta/47/last_access_media.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/47/last_access_media.sql
rename to synapse/storage/schema/main/delta/47/last_access_media.sql
diff --git a/synapse/storage/databases/main/schema/delta/47/postgres_fts_gin.sql b/synapse/storage/schema/main/delta/47/postgres_fts_gin.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/47/postgres_fts_gin.sql
rename to synapse/storage/schema/main/delta/47/postgres_fts_gin.sql
diff --git a/synapse/storage/databases/main/schema/delta/47/push_actions_staging.sql b/synapse/storage/schema/main/delta/47/push_actions_staging.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/47/push_actions_staging.sql
rename to synapse/storage/schema/main/delta/47/push_actions_staging.sql
diff --git a/synapse/storage/databases/main/schema/delta/48/add_user_consent.sql b/synapse/storage/schema/main/delta/48/add_user_consent.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/48/add_user_consent.sql
rename to synapse/storage/schema/main/delta/48/add_user_consent.sql
diff --git a/synapse/storage/databases/main/schema/delta/48/add_user_ips_last_seen_index.sql b/synapse/storage/schema/main/delta/48/add_user_ips_last_seen_index.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/48/add_user_ips_last_seen_index.sql
rename to synapse/storage/schema/main/delta/48/add_user_ips_last_seen_index.sql
diff --git a/synapse/storage/databases/main/schema/delta/48/deactivated_users.sql b/synapse/storage/schema/main/delta/48/deactivated_users.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/48/deactivated_users.sql
rename to synapse/storage/schema/main/delta/48/deactivated_users.sql
diff --git a/synapse/storage/databases/main/schema/delta/48/group_unique_indexes.py b/synapse/storage/schema/main/delta/48/group_unique_indexes.py
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/48/group_unique_indexes.py
rename to synapse/storage/schema/main/delta/48/group_unique_indexes.py
diff --git a/synapse/storage/databases/main/schema/delta/48/groups_joinable.sql b/synapse/storage/schema/main/delta/48/groups_joinable.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/48/groups_joinable.sql
rename to synapse/storage/schema/main/delta/48/groups_joinable.sql
diff --git a/synapse/storage/databases/main/schema/delta/49/add_user_consent_server_notice_sent.sql b/synapse/storage/schema/main/delta/49/add_user_consent_server_notice_sent.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/49/add_user_consent_server_notice_sent.sql
rename to synapse/storage/schema/main/delta/49/add_user_consent_server_notice_sent.sql
diff --git a/synapse/storage/databases/main/schema/delta/49/add_user_daily_visits.sql b/synapse/storage/schema/main/delta/49/add_user_daily_visits.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/49/add_user_daily_visits.sql
rename to synapse/storage/schema/main/delta/49/add_user_daily_visits.sql
diff --git a/synapse/storage/databases/main/schema/delta/49/add_user_ips_last_seen_only_index.sql b/synapse/storage/schema/main/delta/49/add_user_ips_last_seen_only_index.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/49/add_user_ips_last_seen_only_index.sql
rename to synapse/storage/schema/main/delta/49/add_user_ips_last_seen_only_index.sql
diff --git a/synapse/storage/databases/main/schema/delta/50/add_creation_ts_users_index.sql b/synapse/storage/schema/main/delta/50/add_creation_ts_users_index.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/50/add_creation_ts_users_index.sql
rename to synapse/storage/schema/main/delta/50/add_creation_ts_users_index.sql
diff --git a/synapse/storage/databases/main/schema/delta/50/erasure_store.sql b/synapse/storage/schema/main/delta/50/erasure_store.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/50/erasure_store.sql
rename to synapse/storage/schema/main/delta/50/erasure_store.sql
diff --git a/synapse/storage/databases/main/schema/delta/50/make_event_content_nullable.py b/synapse/storage/schema/main/delta/50/make_event_content_nullable.py
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/50/make_event_content_nullable.py
rename to synapse/storage/schema/main/delta/50/make_event_content_nullable.py
diff --git a/synapse/storage/databases/main/schema/delta/51/e2e_room_keys.sql b/synapse/storage/schema/main/delta/51/e2e_room_keys.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/51/e2e_room_keys.sql
rename to synapse/storage/schema/main/delta/51/e2e_room_keys.sql
diff --git a/synapse/storage/databases/main/schema/delta/51/monthly_active_users.sql b/synapse/storage/schema/main/delta/51/monthly_active_users.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/51/monthly_active_users.sql
rename to synapse/storage/schema/main/delta/51/monthly_active_users.sql
diff --git a/synapse/storage/databases/main/schema/delta/52/add_event_to_state_group_index.sql b/synapse/storage/schema/main/delta/52/add_event_to_state_group_index.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/52/add_event_to_state_group_index.sql
rename to synapse/storage/schema/main/delta/52/add_event_to_state_group_index.sql
diff --git a/synapse/storage/databases/main/schema/delta/52/device_list_streams_unique_idx.sql b/synapse/storage/schema/main/delta/52/device_list_streams_unique_idx.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/52/device_list_streams_unique_idx.sql
rename to synapse/storage/schema/main/delta/52/device_list_streams_unique_idx.sql
diff --git a/synapse/storage/databases/main/schema/delta/52/e2e_room_keys.sql b/synapse/storage/schema/main/delta/52/e2e_room_keys.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/52/e2e_room_keys.sql
rename to synapse/storage/schema/main/delta/52/e2e_room_keys.sql
diff --git a/synapse/storage/databases/main/schema/delta/53/add_user_type_to_users.sql b/synapse/storage/schema/main/delta/53/add_user_type_to_users.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/53/add_user_type_to_users.sql
rename to synapse/storage/schema/main/delta/53/add_user_type_to_users.sql
diff --git a/synapse/storage/databases/main/schema/delta/53/drop_sent_transactions.sql b/synapse/storage/schema/main/delta/53/drop_sent_transactions.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/53/drop_sent_transactions.sql
rename to synapse/storage/schema/main/delta/53/drop_sent_transactions.sql
diff --git a/synapse/storage/databases/main/schema/delta/53/event_format_version.sql b/synapse/storage/schema/main/delta/53/event_format_version.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/53/event_format_version.sql
rename to synapse/storage/schema/main/delta/53/event_format_version.sql
diff --git a/synapse/storage/databases/main/schema/delta/53/user_dir_populate.sql b/synapse/storage/schema/main/delta/53/user_dir_populate.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/53/user_dir_populate.sql
rename to synapse/storage/schema/main/delta/53/user_dir_populate.sql
diff --git a/synapse/storage/databases/main/schema/delta/53/user_ips_index.sql b/synapse/storage/schema/main/delta/53/user_ips_index.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/53/user_ips_index.sql
rename to synapse/storage/schema/main/delta/53/user_ips_index.sql
diff --git a/synapse/storage/databases/main/schema/delta/53/user_share.sql b/synapse/storage/schema/main/delta/53/user_share.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/53/user_share.sql
rename to synapse/storage/schema/main/delta/53/user_share.sql
diff --git a/synapse/storage/databases/main/schema/delta/53/user_threepid_id.sql b/synapse/storage/schema/main/delta/53/user_threepid_id.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/53/user_threepid_id.sql
rename to synapse/storage/schema/main/delta/53/user_threepid_id.sql
diff --git a/synapse/storage/databases/main/schema/delta/53/users_in_public_rooms.sql b/synapse/storage/schema/main/delta/53/users_in_public_rooms.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/53/users_in_public_rooms.sql
rename to synapse/storage/schema/main/delta/53/users_in_public_rooms.sql
diff --git a/synapse/storage/databases/main/schema/delta/54/account_validity_with_renewal.sql b/synapse/storage/schema/main/delta/54/account_validity_with_renewal.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/54/account_validity_with_renewal.sql
rename to synapse/storage/schema/main/delta/54/account_validity_with_renewal.sql
diff --git a/synapse/storage/databases/main/schema/delta/54/add_validity_to_server_keys.sql b/synapse/storage/schema/main/delta/54/add_validity_to_server_keys.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/54/add_validity_to_server_keys.sql
rename to synapse/storage/schema/main/delta/54/add_validity_to_server_keys.sql
diff --git a/synapse/storage/databases/main/schema/delta/54/delete_forward_extremities.sql b/synapse/storage/schema/main/delta/54/delete_forward_extremities.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/54/delete_forward_extremities.sql
rename to synapse/storage/schema/main/delta/54/delete_forward_extremities.sql
diff --git a/synapse/storage/databases/main/schema/delta/54/drop_legacy_tables.sql b/synapse/storage/schema/main/delta/54/drop_legacy_tables.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/54/drop_legacy_tables.sql
rename to synapse/storage/schema/main/delta/54/drop_legacy_tables.sql
diff --git a/synapse/storage/databases/main/schema/delta/54/drop_presence_list.sql b/synapse/storage/schema/main/delta/54/drop_presence_list.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/54/drop_presence_list.sql
rename to synapse/storage/schema/main/delta/54/drop_presence_list.sql
diff --git a/synapse/storage/databases/main/schema/delta/54/relations.sql b/synapse/storage/schema/main/delta/54/relations.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/54/relations.sql
rename to synapse/storage/schema/main/delta/54/relations.sql
diff --git a/synapse/storage/databases/main/schema/delta/54/stats.sql b/synapse/storage/schema/main/delta/54/stats.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/54/stats.sql
rename to synapse/storage/schema/main/delta/54/stats.sql
diff --git a/synapse/storage/databases/main/schema/delta/54/stats2.sql b/synapse/storage/schema/main/delta/54/stats2.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/54/stats2.sql
rename to synapse/storage/schema/main/delta/54/stats2.sql
diff --git a/synapse/storage/databases/main/schema/delta/55/access_token_expiry.sql b/synapse/storage/schema/main/delta/55/access_token_expiry.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/55/access_token_expiry.sql
rename to synapse/storage/schema/main/delta/55/access_token_expiry.sql
diff --git a/synapse/storage/databases/main/schema/delta/55/track_threepid_validations.sql b/synapse/storage/schema/main/delta/55/track_threepid_validations.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/55/track_threepid_validations.sql
rename to synapse/storage/schema/main/delta/55/track_threepid_validations.sql
diff --git a/synapse/storage/databases/main/schema/delta/55/users_alter_deactivated.sql b/synapse/storage/schema/main/delta/55/users_alter_deactivated.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/55/users_alter_deactivated.sql
rename to synapse/storage/schema/main/delta/55/users_alter_deactivated.sql
diff --git a/synapse/storage/databases/main/schema/delta/56/add_spans_to_device_lists.sql b/synapse/storage/schema/main/delta/56/add_spans_to_device_lists.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/56/add_spans_to_device_lists.sql
rename to synapse/storage/schema/main/delta/56/add_spans_to_device_lists.sql
diff --git a/synapse/storage/databases/main/schema/delta/56/current_state_events_membership.sql b/synapse/storage/schema/main/delta/56/current_state_events_membership.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/56/current_state_events_membership.sql
rename to synapse/storage/schema/main/delta/56/current_state_events_membership.sql
diff --git a/synapse/storage/databases/main/schema/delta/56/current_state_events_membership_mk2.sql b/synapse/storage/schema/main/delta/56/current_state_events_membership_mk2.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/56/current_state_events_membership_mk2.sql
rename to synapse/storage/schema/main/delta/56/current_state_events_membership_mk2.sql
diff --git a/synapse/storage/databases/main/schema/delta/56/delete_keys_from_deleted_backups.sql b/synapse/storage/schema/main/delta/56/delete_keys_from_deleted_backups.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/56/delete_keys_from_deleted_backups.sql
rename to synapse/storage/schema/main/delta/56/delete_keys_from_deleted_backups.sql
diff --git a/synapse/storage/databases/main/schema/delta/56/destinations_failure_ts.sql b/synapse/storage/schema/main/delta/56/destinations_failure_ts.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/56/destinations_failure_ts.sql
rename to synapse/storage/schema/main/delta/56/destinations_failure_ts.sql
diff --git a/synapse/storage/databases/main/schema/delta/56/destinations_retry_interval_type.sql.postgres b/synapse/storage/schema/main/delta/56/destinations_retry_interval_type.sql.postgres
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/56/destinations_retry_interval_type.sql.postgres
rename to synapse/storage/schema/main/delta/56/destinations_retry_interval_type.sql.postgres
diff --git a/synapse/storage/databases/main/schema/delta/56/device_stream_id_insert.sql b/synapse/storage/schema/main/delta/56/device_stream_id_insert.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/56/device_stream_id_insert.sql
rename to synapse/storage/schema/main/delta/56/device_stream_id_insert.sql
diff --git a/synapse/storage/databases/main/schema/delta/56/devices_last_seen.sql b/synapse/storage/schema/main/delta/56/devices_last_seen.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/56/devices_last_seen.sql
rename to synapse/storage/schema/main/delta/56/devices_last_seen.sql
diff --git a/synapse/storage/databases/main/schema/delta/56/drop_unused_event_tables.sql b/synapse/storage/schema/main/delta/56/drop_unused_event_tables.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/56/drop_unused_event_tables.sql
rename to synapse/storage/schema/main/delta/56/drop_unused_event_tables.sql
diff --git a/synapse/storage/databases/main/schema/delta/56/event_expiry.sql b/synapse/storage/schema/main/delta/56/event_expiry.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/56/event_expiry.sql
rename to synapse/storage/schema/main/delta/56/event_expiry.sql
diff --git a/synapse/storage/databases/main/schema/delta/56/event_labels.sql b/synapse/storage/schema/main/delta/56/event_labels.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/56/event_labels.sql
rename to synapse/storage/schema/main/delta/56/event_labels.sql
diff --git a/synapse/storage/databases/main/schema/delta/56/event_labels_background_update.sql b/synapse/storage/schema/main/delta/56/event_labels_background_update.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/56/event_labels_background_update.sql
rename to synapse/storage/schema/main/delta/56/event_labels_background_update.sql
diff --git a/synapse/storage/databases/main/schema/delta/56/fix_room_keys_index.sql b/synapse/storage/schema/main/delta/56/fix_room_keys_index.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/56/fix_room_keys_index.sql
rename to synapse/storage/schema/main/delta/56/fix_room_keys_index.sql
diff --git a/synapse/storage/databases/main/schema/delta/56/hidden_devices.sql b/synapse/storage/schema/main/delta/56/hidden_devices.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/56/hidden_devices.sql
rename to synapse/storage/schema/main/delta/56/hidden_devices.sql
diff --git a/synapse/storage/databases/main/schema/delta/56/hidden_devices_fix.sql.sqlite b/synapse/storage/schema/main/delta/56/hidden_devices_fix.sql.sqlite
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/56/hidden_devices_fix.sql.sqlite
rename to synapse/storage/schema/main/delta/56/hidden_devices_fix.sql.sqlite
diff --git a/synapse/storage/databases/main/schema/delta/56/nuke_empty_communities_from_db.sql b/synapse/storage/schema/main/delta/56/nuke_empty_communities_from_db.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/56/nuke_empty_communities_from_db.sql
rename to synapse/storage/schema/main/delta/56/nuke_empty_communities_from_db.sql
diff --git a/synapse/storage/databases/main/schema/delta/56/public_room_list_idx.sql b/synapse/storage/schema/main/delta/56/public_room_list_idx.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/56/public_room_list_idx.sql
rename to synapse/storage/schema/main/delta/56/public_room_list_idx.sql
diff --git a/synapse/storage/databases/main/schema/delta/56/redaction_censor.sql b/synapse/storage/schema/main/delta/56/redaction_censor.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/56/redaction_censor.sql
rename to synapse/storage/schema/main/delta/56/redaction_censor.sql
diff --git a/synapse/storage/databases/main/schema/delta/56/redaction_censor2.sql b/synapse/storage/schema/main/delta/56/redaction_censor2.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/56/redaction_censor2.sql
rename to synapse/storage/schema/main/delta/56/redaction_censor2.sql
diff --git a/synapse/storage/databases/main/schema/delta/56/redaction_censor3_fix_update.sql.postgres b/synapse/storage/schema/main/delta/56/redaction_censor3_fix_update.sql.postgres
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/56/redaction_censor3_fix_update.sql.postgres
rename to synapse/storage/schema/main/delta/56/redaction_censor3_fix_update.sql.postgres
diff --git a/synapse/storage/databases/main/schema/delta/56/redaction_censor4.sql b/synapse/storage/schema/main/delta/56/redaction_censor4.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/56/redaction_censor4.sql
rename to synapse/storage/schema/main/delta/56/redaction_censor4.sql
diff --git a/synapse/storage/databases/main/schema/delta/56/remove_tombstoned_rooms_from_directory.sql b/synapse/storage/schema/main/delta/56/remove_tombstoned_rooms_from_directory.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/56/remove_tombstoned_rooms_from_directory.sql
rename to synapse/storage/schema/main/delta/56/remove_tombstoned_rooms_from_directory.sql
diff --git a/synapse/storage/databases/main/schema/delta/56/room_key_etag.sql b/synapse/storage/schema/main/delta/56/room_key_etag.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/56/room_key_etag.sql
rename to synapse/storage/schema/main/delta/56/room_key_etag.sql
diff --git a/synapse/storage/databases/main/schema/delta/56/room_membership_idx.sql b/synapse/storage/schema/main/delta/56/room_membership_idx.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/56/room_membership_idx.sql
rename to synapse/storage/schema/main/delta/56/room_membership_idx.sql
diff --git a/synapse/storage/databases/main/schema/delta/56/room_retention.sql b/synapse/storage/schema/main/delta/56/room_retention.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/56/room_retention.sql
rename to synapse/storage/schema/main/delta/56/room_retention.sql
diff --git a/synapse/storage/databases/main/schema/delta/56/signing_keys.sql b/synapse/storage/schema/main/delta/56/signing_keys.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/56/signing_keys.sql
rename to synapse/storage/schema/main/delta/56/signing_keys.sql
diff --git a/synapse/storage/databases/main/schema/delta/56/signing_keys_nonunique_signatures.sql b/synapse/storage/schema/main/delta/56/signing_keys_nonunique_signatures.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/56/signing_keys_nonunique_signatures.sql
rename to synapse/storage/schema/main/delta/56/signing_keys_nonunique_signatures.sql
diff --git a/synapse/storage/databases/main/schema/delta/56/stats_separated.sql b/synapse/storage/schema/main/delta/56/stats_separated.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/56/stats_separated.sql
rename to synapse/storage/schema/main/delta/56/stats_separated.sql
diff --git a/synapse/storage/databases/main/schema/delta/56/unique_user_filter_index.py b/synapse/storage/schema/main/delta/56/unique_user_filter_index.py
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/56/unique_user_filter_index.py
rename to synapse/storage/schema/main/delta/56/unique_user_filter_index.py
diff --git a/synapse/storage/databases/main/schema/delta/56/user_external_ids.sql b/synapse/storage/schema/main/delta/56/user_external_ids.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/56/user_external_ids.sql
rename to synapse/storage/schema/main/delta/56/user_external_ids.sql
diff --git a/synapse/storage/databases/main/schema/delta/56/users_in_public_rooms_idx.sql b/synapse/storage/schema/main/delta/56/users_in_public_rooms_idx.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/56/users_in_public_rooms_idx.sql
rename to synapse/storage/schema/main/delta/56/users_in_public_rooms_idx.sql
diff --git a/synapse/storage/databases/main/schema/delta/57/delete_old_current_state_events.sql b/synapse/storage/schema/main/delta/57/delete_old_current_state_events.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/57/delete_old_current_state_events.sql
rename to synapse/storage/schema/main/delta/57/delete_old_current_state_events.sql
diff --git a/synapse/storage/databases/main/schema/delta/57/device_list_remote_cache_stale.sql b/synapse/storage/schema/main/delta/57/device_list_remote_cache_stale.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/57/device_list_remote_cache_stale.sql
rename to synapse/storage/schema/main/delta/57/device_list_remote_cache_stale.sql
diff --git a/synapse/storage/databases/main/schema/delta/57/local_current_membership.py b/synapse/storage/schema/main/delta/57/local_current_membership.py
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/57/local_current_membership.py
rename to synapse/storage/schema/main/delta/57/local_current_membership.py
diff --git a/synapse/storage/databases/main/schema/delta/57/remove_sent_outbound_pokes.sql b/synapse/storage/schema/main/delta/57/remove_sent_outbound_pokes.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/57/remove_sent_outbound_pokes.sql
rename to synapse/storage/schema/main/delta/57/remove_sent_outbound_pokes.sql
diff --git a/synapse/storage/databases/main/schema/delta/57/rooms_version_column.sql b/synapse/storage/schema/main/delta/57/rooms_version_column.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/57/rooms_version_column.sql
rename to synapse/storage/schema/main/delta/57/rooms_version_column.sql
diff --git a/synapse/storage/databases/main/schema/delta/57/rooms_version_column_2.sql.postgres b/synapse/storage/schema/main/delta/57/rooms_version_column_2.sql.postgres
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/57/rooms_version_column_2.sql.postgres
rename to synapse/storage/schema/main/delta/57/rooms_version_column_2.sql.postgres
diff --git a/synapse/storage/databases/main/schema/delta/57/rooms_version_column_2.sql.sqlite b/synapse/storage/schema/main/delta/57/rooms_version_column_2.sql.sqlite
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/57/rooms_version_column_2.sql.sqlite
rename to synapse/storage/schema/main/delta/57/rooms_version_column_2.sql.sqlite
diff --git a/synapse/storage/databases/main/schema/delta/57/rooms_version_column_3.sql.postgres b/synapse/storage/schema/main/delta/57/rooms_version_column_3.sql.postgres
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/57/rooms_version_column_3.sql.postgres
rename to synapse/storage/schema/main/delta/57/rooms_version_column_3.sql.postgres
diff --git a/synapse/storage/databases/main/schema/delta/57/rooms_version_column_3.sql.sqlite b/synapse/storage/schema/main/delta/57/rooms_version_column_3.sql.sqlite
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/57/rooms_version_column_3.sql.sqlite
rename to synapse/storage/schema/main/delta/57/rooms_version_column_3.sql.sqlite
diff --git a/synapse/storage/databases/main/schema/delta/58/02remove_dup_outbound_pokes.sql b/synapse/storage/schema/main/delta/58/02remove_dup_outbound_pokes.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/58/02remove_dup_outbound_pokes.sql
rename to synapse/storage/schema/main/delta/58/02remove_dup_outbound_pokes.sql
diff --git a/synapse/storage/databases/main/schema/delta/58/03persist_ui_auth.sql b/synapse/storage/schema/main/delta/58/03persist_ui_auth.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/58/03persist_ui_auth.sql
rename to synapse/storage/schema/main/delta/58/03persist_ui_auth.sql
diff --git a/synapse/storage/databases/main/schema/delta/58/05cache_instance.sql.postgres b/synapse/storage/schema/main/delta/58/05cache_instance.sql.postgres
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/58/05cache_instance.sql.postgres
rename to synapse/storage/schema/main/delta/58/05cache_instance.sql.postgres
diff --git a/synapse/storage/databases/main/schema/delta/58/06dlols_unique_idx.py b/synapse/storage/schema/main/delta/58/06dlols_unique_idx.py
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/58/06dlols_unique_idx.py
rename to synapse/storage/schema/main/delta/58/06dlols_unique_idx.py
diff --git a/synapse/storage/databases/main/schema/delta/58/07add_method_to_thumbnail_constraint.sql.postgres b/synapse/storage/schema/main/delta/58/07add_method_to_thumbnail_constraint.sql.postgres
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/58/07add_method_to_thumbnail_constraint.sql.postgres
rename to synapse/storage/schema/main/delta/58/07add_method_to_thumbnail_constraint.sql.postgres
diff --git a/synapse/storage/databases/main/schema/delta/58/07add_method_to_thumbnail_constraint.sql.sqlite b/synapse/storage/schema/main/delta/58/07add_method_to_thumbnail_constraint.sql.sqlite
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/58/07add_method_to_thumbnail_constraint.sql.sqlite
rename to synapse/storage/schema/main/delta/58/07add_method_to_thumbnail_constraint.sql.sqlite
diff --git a/synapse/storage/databases/main/schema/delta/58/07persist_ui_auth_ips.sql b/synapse/storage/schema/main/delta/58/07persist_ui_auth_ips.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/58/07persist_ui_auth_ips.sql
rename to synapse/storage/schema/main/delta/58/07persist_ui_auth_ips.sql
diff --git a/synapse/storage/databases/main/schema/delta/58/08_media_safe_from_quarantine.sql.postgres b/synapse/storage/schema/main/delta/58/08_media_safe_from_quarantine.sql.postgres
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/58/08_media_safe_from_quarantine.sql.postgres
rename to synapse/storage/schema/main/delta/58/08_media_safe_from_quarantine.sql.postgres
diff --git a/synapse/storage/databases/main/schema/delta/58/08_media_safe_from_quarantine.sql.sqlite b/synapse/storage/schema/main/delta/58/08_media_safe_from_quarantine.sql.sqlite
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/58/08_media_safe_from_quarantine.sql.sqlite
rename to synapse/storage/schema/main/delta/58/08_media_safe_from_quarantine.sql.sqlite
diff --git a/synapse/storage/databases/main/schema/delta/58/09shadow_ban.sql b/synapse/storage/schema/main/delta/58/09shadow_ban.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/58/09shadow_ban.sql
rename to synapse/storage/schema/main/delta/58/09shadow_ban.sql
diff --git a/synapse/storage/databases/main/schema/delta/58/10_pushrules_enabled_delete_obsolete.sql b/synapse/storage/schema/main/delta/58/10_pushrules_enabled_delete_obsolete.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/58/10_pushrules_enabled_delete_obsolete.sql
rename to synapse/storage/schema/main/delta/58/10_pushrules_enabled_delete_obsolete.sql
diff --git a/synapse/storage/databases/main/schema/delta/58/10drop_local_rejections_stream.sql b/synapse/storage/schema/main/delta/58/10drop_local_rejections_stream.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/58/10drop_local_rejections_stream.sql
rename to synapse/storage/schema/main/delta/58/10drop_local_rejections_stream.sql
diff --git a/synapse/storage/databases/main/schema/delta/58/10federation_pos_instance_name.sql b/synapse/storage/schema/main/delta/58/10federation_pos_instance_name.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/58/10federation_pos_instance_name.sql
rename to synapse/storage/schema/main/delta/58/10federation_pos_instance_name.sql
diff --git a/synapse/storage/databases/main/schema/delta/58/11dehydration.sql b/synapse/storage/schema/main/delta/58/11dehydration.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/58/11dehydration.sql
rename to synapse/storage/schema/main/delta/58/11dehydration.sql
diff --git a/synapse/storage/databases/main/schema/delta/58/11fallback.sql b/synapse/storage/schema/main/delta/58/11fallback.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/58/11fallback.sql
rename to synapse/storage/schema/main/delta/58/11fallback.sql
diff --git a/synapse/storage/databases/main/schema/delta/58/11user_id_seq.py b/synapse/storage/schema/main/delta/58/11user_id_seq.py
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/58/11user_id_seq.py
rename to synapse/storage/schema/main/delta/58/11user_id_seq.py
diff --git a/synapse/storage/databases/main/schema/delta/58/12room_stats.sql b/synapse/storage/schema/main/delta/58/12room_stats.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/58/12room_stats.sql
rename to synapse/storage/schema/main/delta/58/12room_stats.sql
diff --git a/synapse/storage/databases/main/schema/delta/58/13remove_presence_allow_inbound.sql b/synapse/storage/schema/main/delta/58/13remove_presence_allow_inbound.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/58/13remove_presence_allow_inbound.sql
rename to synapse/storage/schema/main/delta/58/13remove_presence_allow_inbound.sql
diff --git a/synapse/storage/databases/main/schema/delta/58/14events_instance_name.sql b/synapse/storage/schema/main/delta/58/14events_instance_name.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/58/14events_instance_name.sql
rename to synapse/storage/schema/main/delta/58/14events_instance_name.sql
diff --git a/synapse/storage/databases/main/schema/delta/58/14events_instance_name.sql.postgres b/synapse/storage/schema/main/delta/58/14events_instance_name.sql.postgres
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/58/14events_instance_name.sql.postgres
rename to synapse/storage/schema/main/delta/58/14events_instance_name.sql.postgres
diff --git a/synapse/storage/databases/main/schema/delta/58/15_catchup_destination_rooms.sql b/synapse/storage/schema/main/delta/58/15_catchup_destination_rooms.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/58/15_catchup_destination_rooms.sql
rename to synapse/storage/schema/main/delta/58/15_catchup_destination_rooms.sql
diff --git a/synapse/storage/databases/main/schema/delta/58/15unread_count.sql b/synapse/storage/schema/main/delta/58/15unread_count.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/58/15unread_count.sql
rename to synapse/storage/schema/main/delta/58/15unread_count.sql
diff --git a/synapse/storage/databases/main/schema/delta/58/16populate_stats_process_rooms_fix.sql b/synapse/storage/schema/main/delta/58/16populate_stats_process_rooms_fix.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/58/16populate_stats_process_rooms_fix.sql
rename to synapse/storage/schema/main/delta/58/16populate_stats_process_rooms_fix.sql
diff --git a/synapse/storage/databases/main/schema/delta/58/17_catchup_last_successful.sql b/synapse/storage/schema/main/delta/58/17_catchup_last_successful.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/58/17_catchup_last_successful.sql
rename to synapse/storage/schema/main/delta/58/17_catchup_last_successful.sql
diff --git a/synapse/storage/databases/main/schema/delta/58/18stream_positions.sql b/synapse/storage/schema/main/delta/58/18stream_positions.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/58/18stream_positions.sql
rename to synapse/storage/schema/main/delta/58/18stream_positions.sql
diff --git a/synapse/storage/databases/main/schema/delta/58/19instance_map.sql.postgres b/synapse/storage/schema/main/delta/58/19instance_map.sql.postgres
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/58/19instance_map.sql.postgres
rename to synapse/storage/schema/main/delta/58/19instance_map.sql.postgres
diff --git a/synapse/storage/databases/main/schema/delta/58/19txn_id.sql b/synapse/storage/schema/main/delta/58/19txn_id.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/58/19txn_id.sql
rename to synapse/storage/schema/main/delta/58/19txn_id.sql
diff --git a/synapse/storage/databases/main/schema/delta/58/20instance_name_event_tables.sql b/synapse/storage/schema/main/delta/58/20instance_name_event_tables.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/58/20instance_name_event_tables.sql
rename to synapse/storage/schema/main/delta/58/20instance_name_event_tables.sql
diff --git a/synapse/storage/databases/main/schema/delta/58/20user_daily_visits.sql b/synapse/storage/schema/main/delta/58/20user_daily_visits.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/58/20user_daily_visits.sql
rename to synapse/storage/schema/main/delta/58/20user_daily_visits.sql
diff --git a/synapse/storage/databases/main/schema/delta/58/21as_device_stream.sql b/synapse/storage/schema/main/delta/58/21as_device_stream.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/58/21as_device_stream.sql
rename to synapse/storage/schema/main/delta/58/21as_device_stream.sql
diff --git a/synapse/storage/databases/main/schema/delta/58/21drop_device_max_stream_id.sql b/synapse/storage/schema/main/delta/58/21drop_device_max_stream_id.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/58/21drop_device_max_stream_id.sql
rename to synapse/storage/schema/main/delta/58/21drop_device_max_stream_id.sql
diff --git a/synapse/storage/databases/main/schema/delta/58/22puppet_token.sql b/synapse/storage/schema/main/delta/58/22puppet_token.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/58/22puppet_token.sql
rename to synapse/storage/schema/main/delta/58/22puppet_token.sql
diff --git a/synapse/storage/databases/main/schema/delta/58/22users_have_local_media.sql b/synapse/storage/schema/main/delta/58/22users_have_local_media.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/58/22users_have_local_media.sql
rename to synapse/storage/schema/main/delta/58/22users_have_local_media.sql
diff --git a/synapse/storage/databases/main/schema/delta/58/23e2e_cross_signing_keys_idx.sql b/synapse/storage/schema/main/delta/58/23e2e_cross_signing_keys_idx.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/58/23e2e_cross_signing_keys_idx.sql
rename to synapse/storage/schema/main/delta/58/23e2e_cross_signing_keys_idx.sql
diff --git a/synapse/storage/databases/main/schema/delta/58/24drop_event_json_index.sql b/synapse/storage/schema/main/delta/58/24drop_event_json_index.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/58/24drop_event_json_index.sql
rename to synapse/storage/schema/main/delta/58/24drop_event_json_index.sql
diff --git a/synapse/storage/databases/main/schema/delta/58/25user_external_ids_user_id_idx.sql b/synapse/storage/schema/main/delta/58/25user_external_ids_user_id_idx.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/58/25user_external_ids_user_id_idx.sql
rename to synapse/storage/schema/main/delta/58/25user_external_ids_user_id_idx.sql
diff --git a/synapse/storage/databases/main/schema/delta/58/26access_token_last_validated.sql b/synapse/storage/schema/main/delta/58/26access_token_last_validated.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/58/26access_token_last_validated.sql
rename to synapse/storage/schema/main/delta/58/26access_token_last_validated.sql
diff --git a/synapse/storage/databases/main/schema/delta/58/27local_invites.sql b/synapse/storage/schema/main/delta/58/27local_invites.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/58/27local_invites.sql
rename to synapse/storage/schema/main/delta/58/27local_invites.sql
diff --git a/synapse/storage/databases/main/schema/delta/58/28drop_last_used_column.sql.postgres b/synapse/storage/schema/main/delta/58/28drop_last_used_column.sql.postgres
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/58/28drop_last_used_column.sql.postgres
rename to synapse/storage/schema/main/delta/58/28drop_last_used_column.sql.postgres
diff --git a/synapse/storage/databases/main/schema/delta/58/28drop_last_used_column.sql.sqlite b/synapse/storage/schema/main/delta/58/28drop_last_used_column.sql.sqlite
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/58/28drop_last_used_column.sql.sqlite
rename to synapse/storage/schema/main/delta/58/28drop_last_used_column.sql.sqlite
diff --git a/synapse/storage/databases/main/schema/delta/59/01ignored_user.py b/synapse/storage/schema/main/delta/59/01ignored_user.py
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/59/01ignored_user.py
rename to synapse/storage/schema/main/delta/59/01ignored_user.py
diff --git a/synapse/storage/databases/main/schema/delta/59/02shard_send_to_device.sql b/synapse/storage/schema/main/delta/59/02shard_send_to_device.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/59/02shard_send_to_device.sql
rename to synapse/storage/schema/main/delta/59/02shard_send_to_device.sql
diff --git a/synapse/storage/databases/main/schema/delta/59/03shard_send_to_device_sequence.sql.postgres b/synapse/storage/schema/main/delta/59/03shard_send_to_device_sequence.sql.postgres
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/59/03shard_send_to_device_sequence.sql.postgres
rename to synapse/storage/schema/main/delta/59/03shard_send_to_device_sequence.sql.postgres
diff --git a/synapse/storage/databases/main/schema/delta/59/04_event_auth_chains.sql b/synapse/storage/schema/main/delta/59/04_event_auth_chains.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/59/04_event_auth_chains.sql
rename to synapse/storage/schema/main/delta/59/04_event_auth_chains.sql
diff --git a/synapse/storage/databases/main/schema/delta/59/04_event_auth_chains.sql.postgres b/synapse/storage/schema/main/delta/59/04_event_auth_chains.sql.postgres
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/59/04_event_auth_chains.sql.postgres
rename to synapse/storage/schema/main/delta/59/04_event_auth_chains.sql.postgres
diff --git a/synapse/storage/databases/main/schema/delta/59/04drop_account_data.sql b/synapse/storage/schema/main/delta/59/04drop_account_data.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/59/04drop_account_data.sql
rename to synapse/storage/schema/main/delta/59/04drop_account_data.sql
diff --git a/synapse/storage/databases/main/schema/delta/59/05cache_invalidation.sql b/synapse/storage/schema/main/delta/59/05cache_invalidation.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/59/05cache_invalidation.sql
rename to synapse/storage/schema/main/delta/59/05cache_invalidation.sql
diff --git a/synapse/storage/databases/main/schema/delta/59/06chain_cover_index.sql b/synapse/storage/schema/main/delta/59/06chain_cover_index.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/59/06chain_cover_index.sql
rename to synapse/storage/schema/main/delta/59/06chain_cover_index.sql
diff --git a/synapse/storage/databases/main/schema/delta/59/06shard_account_data.sql b/synapse/storage/schema/main/delta/59/06shard_account_data.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/59/06shard_account_data.sql
rename to synapse/storage/schema/main/delta/59/06shard_account_data.sql
diff --git a/synapse/storage/databases/main/schema/delta/59/06shard_account_data.sql.postgres b/synapse/storage/schema/main/delta/59/06shard_account_data.sql.postgres
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/59/06shard_account_data.sql.postgres
rename to synapse/storage/schema/main/delta/59/06shard_account_data.sql.postgres
diff --git a/synapse/storage/databases/main/schema/delta/59/07shard_account_data_fix.sql b/synapse/storage/schema/main/delta/59/07shard_account_data_fix.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/59/07shard_account_data_fix.sql
rename to synapse/storage/schema/main/delta/59/07shard_account_data_fix.sql
diff --git a/synapse/storage/databases/main/schema/delta/59/08delete_pushers_for_deactivated_accounts.sql b/synapse/storage/schema/main/delta/59/08delete_pushers_for_deactivated_accounts.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/59/08delete_pushers_for_deactivated_accounts.sql
rename to synapse/storage/schema/main/delta/59/08delete_pushers_for_deactivated_accounts.sql
diff --git a/synapse/storage/databases/main/schema/delta/59/08delete_stale_pushers.sql b/synapse/storage/schema/main/delta/59/08delete_stale_pushers.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/59/08delete_stale_pushers.sql
rename to synapse/storage/schema/main/delta/59/08delete_stale_pushers.sql
diff --git a/synapse/storage/databases/main/schema/delta/59/09rejected_events_metadata.sql b/synapse/storage/schema/main/delta/59/09rejected_events_metadata.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/59/09rejected_events_metadata.sql
rename to synapse/storage/schema/main/delta/59/09rejected_events_metadata.sql
diff --git a/synapse/storage/databases/main/schema/delta/59/10delete_purged_chain_cover.sql b/synapse/storage/schema/main/delta/59/10delete_purged_chain_cover.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/59/10delete_purged_chain_cover.sql
rename to synapse/storage/schema/main/delta/59/10delete_purged_chain_cover.sql
diff --git a/synapse/storage/databases/main/schema/delta/59/11drop_thumbnail_constraint.sql.postgres b/synapse/storage/schema/main/delta/59/11drop_thumbnail_constraint.sql.postgres
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/59/11drop_thumbnail_constraint.sql.postgres
rename to synapse/storage/schema/main/delta/59/11drop_thumbnail_constraint.sql.postgres
diff --git a/synapse/storage/databases/main/schema/delta/59/12account_validity_token_used_ts_ms.sql b/synapse/storage/schema/main/delta/59/12account_validity_token_used_ts_ms.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/59/12account_validity_token_used_ts_ms.sql
rename to synapse/storage/schema/main/delta/59/12account_validity_token_used_ts_ms.sql
diff --git a/synapse/storage/databases/main/schema/delta/59/12presence_stream_instance.sql b/synapse/storage/schema/main/delta/59/12presence_stream_instance.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/59/12presence_stream_instance.sql
rename to synapse/storage/schema/main/delta/59/12presence_stream_instance.sql
diff --git a/synapse/storage/databases/main/schema/delta/59/12presence_stream_instance_seq.sql.postgres b/synapse/storage/schema/main/delta/59/12presence_stream_instance_seq.sql.postgres
similarity index 100%
rename from synapse/storage/databases/main/schema/delta/59/12presence_stream_instance_seq.sql.postgres
rename to synapse/storage/schema/main/delta/59/12presence_stream_instance_seq.sql.postgres
diff --git a/synapse/storage/databases/main/schema/full_schemas/16/application_services.sql b/synapse/storage/schema/main/full_schemas/16/application_services.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/full_schemas/16/application_services.sql
rename to synapse/storage/schema/main/full_schemas/16/application_services.sql
diff --git a/synapse/storage/databases/main/schema/full_schemas/16/event_edges.sql b/synapse/storage/schema/main/full_schemas/16/event_edges.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/full_schemas/16/event_edges.sql
rename to synapse/storage/schema/main/full_schemas/16/event_edges.sql
diff --git a/synapse/storage/databases/main/schema/full_schemas/16/event_signatures.sql b/synapse/storage/schema/main/full_schemas/16/event_signatures.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/full_schemas/16/event_signatures.sql
rename to synapse/storage/schema/main/full_schemas/16/event_signatures.sql
diff --git a/synapse/storage/databases/main/schema/full_schemas/16/im.sql b/synapse/storage/schema/main/full_schemas/16/im.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/full_schemas/16/im.sql
rename to synapse/storage/schema/main/full_schemas/16/im.sql
diff --git a/synapse/storage/databases/main/schema/full_schemas/16/keys.sql b/synapse/storage/schema/main/full_schemas/16/keys.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/full_schemas/16/keys.sql
rename to synapse/storage/schema/main/full_schemas/16/keys.sql
diff --git a/synapse/storage/databases/main/schema/full_schemas/16/media_repository.sql b/synapse/storage/schema/main/full_schemas/16/media_repository.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/full_schemas/16/media_repository.sql
rename to synapse/storage/schema/main/full_schemas/16/media_repository.sql
diff --git a/synapse/storage/databases/main/schema/full_schemas/16/presence.sql b/synapse/storage/schema/main/full_schemas/16/presence.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/full_schemas/16/presence.sql
rename to synapse/storage/schema/main/full_schemas/16/presence.sql
diff --git a/synapse/storage/databases/main/schema/full_schemas/16/profiles.sql b/synapse/storage/schema/main/full_schemas/16/profiles.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/full_schemas/16/profiles.sql
rename to synapse/storage/schema/main/full_schemas/16/profiles.sql
diff --git a/synapse/storage/databases/main/schema/full_schemas/16/push.sql b/synapse/storage/schema/main/full_schemas/16/push.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/full_schemas/16/push.sql
rename to synapse/storage/schema/main/full_schemas/16/push.sql
diff --git a/synapse/storage/databases/main/schema/full_schemas/16/redactions.sql b/synapse/storage/schema/main/full_schemas/16/redactions.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/full_schemas/16/redactions.sql
rename to synapse/storage/schema/main/full_schemas/16/redactions.sql
diff --git a/synapse/storage/databases/main/schema/full_schemas/16/room_aliases.sql b/synapse/storage/schema/main/full_schemas/16/room_aliases.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/full_schemas/16/room_aliases.sql
rename to synapse/storage/schema/main/full_schemas/16/room_aliases.sql
diff --git a/synapse/storage/databases/main/schema/full_schemas/16/state.sql b/synapse/storage/schema/main/full_schemas/16/state.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/full_schemas/16/state.sql
rename to synapse/storage/schema/main/full_schemas/16/state.sql
diff --git a/synapse/storage/databases/main/schema/full_schemas/16/transactions.sql b/synapse/storage/schema/main/full_schemas/16/transactions.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/full_schemas/16/transactions.sql
rename to synapse/storage/schema/main/full_schemas/16/transactions.sql
diff --git a/synapse/storage/databases/main/schema/full_schemas/16/users.sql b/synapse/storage/schema/main/full_schemas/16/users.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/full_schemas/16/users.sql
rename to synapse/storage/schema/main/full_schemas/16/users.sql
diff --git a/synapse/storage/databases/main/schema/full_schemas/54/full.sql.postgres b/synapse/storage/schema/main/full_schemas/54/full.sql.postgres
similarity index 100%
rename from synapse/storage/databases/main/schema/full_schemas/54/full.sql.postgres
rename to synapse/storage/schema/main/full_schemas/54/full.sql.postgres
diff --git a/synapse/storage/databases/main/schema/full_schemas/54/full.sql.sqlite b/synapse/storage/schema/main/full_schemas/54/full.sql.sqlite
similarity index 100%
rename from synapse/storage/databases/main/schema/full_schemas/54/full.sql.sqlite
rename to synapse/storage/schema/main/full_schemas/54/full.sql.sqlite
diff --git a/synapse/storage/databases/main/schema/full_schemas/54/stream_positions.sql b/synapse/storage/schema/main/full_schemas/54/stream_positions.sql
similarity index 100%
rename from synapse/storage/databases/main/schema/full_schemas/54/stream_positions.sql
rename to synapse/storage/schema/main/full_schemas/54/stream_positions.sql
diff --git a/synapse/storage/databases/state/schema/delta/23/drop_state_index.sql b/synapse/storage/schema/state/delta/23/drop_state_index.sql
similarity index 100%
rename from synapse/storage/databases/state/schema/delta/23/drop_state_index.sql
rename to synapse/storage/schema/state/delta/23/drop_state_index.sql
diff --git a/synapse/storage/databases/state/schema/delta/30/state_stream.sql b/synapse/storage/schema/state/delta/30/state_stream.sql
similarity index 100%
rename from synapse/storage/databases/state/schema/delta/30/state_stream.sql
rename to synapse/storage/schema/state/delta/30/state_stream.sql
diff --git a/synapse/storage/databases/state/schema/delta/32/remove_state_indices.sql b/synapse/storage/schema/state/delta/32/remove_state_indices.sql
similarity index 100%
rename from synapse/storage/databases/state/schema/delta/32/remove_state_indices.sql
rename to synapse/storage/schema/state/delta/32/remove_state_indices.sql
diff --git a/synapse/storage/databases/state/schema/delta/35/add_state_index.sql b/synapse/storage/schema/state/delta/35/add_state_index.sql
similarity index 100%
rename from synapse/storage/databases/state/schema/delta/35/add_state_index.sql
rename to synapse/storage/schema/state/delta/35/add_state_index.sql
diff --git a/synapse/storage/databases/state/schema/delta/35/state.sql b/synapse/storage/schema/state/delta/35/state.sql
similarity index 100%
rename from synapse/storage/databases/state/schema/delta/35/state.sql
rename to synapse/storage/schema/state/delta/35/state.sql
diff --git a/synapse/storage/databases/state/schema/delta/35/state_dedupe.sql b/synapse/storage/schema/state/delta/35/state_dedupe.sql
similarity index 100%
rename from synapse/storage/databases/state/schema/delta/35/state_dedupe.sql
rename to synapse/storage/schema/state/delta/35/state_dedupe.sql
diff --git a/synapse/storage/databases/state/schema/delta/47/state_group_seq.py b/synapse/storage/schema/state/delta/47/state_group_seq.py
similarity index 100%
rename from synapse/storage/databases/state/schema/delta/47/state_group_seq.py
rename to synapse/storage/schema/state/delta/47/state_group_seq.py
diff --git a/synapse/storage/databases/state/schema/delta/56/state_group_room_idx.sql b/synapse/storage/schema/state/delta/56/state_group_room_idx.sql
similarity index 100%
rename from synapse/storage/databases/state/schema/delta/56/state_group_room_idx.sql
rename to synapse/storage/schema/state/delta/56/state_group_room_idx.sql
diff --git a/synapse/storage/databases/state/schema/full_schemas/54/full.sql b/synapse/storage/schema/state/full_schemas/54/full.sql
similarity index 100%
rename from synapse/storage/databases/state/schema/full_schemas/54/full.sql
rename to synapse/storage/schema/state/full_schemas/54/full.sql
diff --git a/synapse/storage/databases/state/schema/full_schemas/54/sequence.sql.postgres b/synapse/storage/schema/state/full_schemas/54/sequence.sql.postgres
similarity index 100%
rename from synapse/storage/databases/state/schema/full_schemas/54/sequence.sql.postgres
rename to synapse/storage/schema/state/full_schemas/54/sequence.sql.postgres
diff --git a/tests/storage/test_cleanup_extrems.py b/tests/storage/test_cleanup_extrems.py
index aa20588bbe..77c4fe721c 100644
--- a/tests/storage/test_cleanup_extrems.py
+++ b/tests/storage/test_cleanup_extrems.py
@@ -47,10 +47,8 @@ class CleanupExtremBackgroundUpdateStoreTestCase(HomeserverTestCase):
         )
 
         schema_path = os.path.join(
-            prepare_database.dir_path,
-            "databases",
+            prepare_database.schema_path,
             "main",
-            "schema",
             "delta",
             "54",
             "delete_forward_extremities.sql",

From 765473567cd7e9520bdb9f85491bb5fe719c360b Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <1389908+richvdh@users.noreply.github.com>
Date: Fri, 7 May 2021 14:01:57 +0100
Subject: [PATCH 03/15] Fix make_full_schema to create the db with the right
 options and user (#9931)

---
 changelog.d/9931.misc           |  1 +
 scripts-dev/make_full_schema.sh | 19 ++++++++++---------
 2 files changed, 11 insertions(+), 9 deletions(-)
 create mode 100644 changelog.d/9931.misc

diff --git a/changelog.d/9931.misc b/changelog.d/9931.misc
new file mode 100644
index 0000000000..326adc7f3c
--- /dev/null
+++ b/changelog.d/9931.misc
@@ -0,0 +1 @@
+Minor fixes to the `make_full_schema.sh` script.
diff --git a/scripts-dev/make_full_schema.sh b/scripts-dev/make_full_schema.sh
index bc8f978660..39bf30d258 100755
--- a/scripts-dev/make_full_schema.sh
+++ b/scripts-dev/make_full_schema.sh
@@ -6,7 +6,7 @@
 # It does so by having Synapse generate an up-to-date SQLite DB, then running
 # synapse_port_db to convert it to Postgres. It then dumps the contents of both.
 
-POSTGRES_HOST="localhost"
+export PGHOST="localhost"
 POSTGRES_DB_NAME="synapse_full_schema.$$"
 
 SQLITE_FULL_SCHEMA_OUTPUT_FILE="full.sql.sqlite"
@@ -32,7 +32,7 @@ usage() {
 while getopts "p:co:h" opt; do
   case $opt in
     p)
-      POSTGRES_USERNAME=$OPTARG
+      export PGUSER=$OPTARG
       ;;
     c)
       # Print all commands that are being executed
@@ -69,7 +69,7 @@ if [ ${#unsatisfied_requirements} -ne 0 ]; then
   exit 1
 fi
 
-if [ -z "$POSTGRES_USERNAME" ]; then
+if [ -z "$PGUSER" ]; then
   echo "No postgres username supplied"
   usage
   exit 1
@@ -84,8 +84,9 @@ fi
 # Create the output directory if it doesn't exist
 mkdir -p "$OUTPUT_DIR"
 
-read -rsp "Postgres password for '$POSTGRES_USERNAME': " POSTGRES_PASSWORD
+read -rsp "Postgres password for '$PGUSER': " PGPASSWORD
 echo ""
+export PGPASSWORD
 
 # Exit immediately if a command fails
 set -e
@@ -131,9 +132,9 @@ report_stats: false
 database:
   name: "psycopg2"
   args:
-    user: "$POSTGRES_USERNAME"
-    host: "$POSTGRES_HOST"
-    password: "$POSTGRES_PASSWORD"
+    user: "$PGUSER"
+    host: "$PGHOST"
+    password: "$PGPASSWORD"
     database: "$POSTGRES_DB_NAME"
 
 # Suppress the key server warning.
@@ -150,7 +151,7 @@ scripts-dev/update_database --database-config "$SQLITE_CONFIG"
 
 # Create the PostgreSQL database.
 echo "Creating postgres database..."
-createdb $POSTGRES_DB_NAME
+createdb --lc-collate=C --lc-ctype=C --template=template0 "$POSTGRES_DB_NAME"
 
 echo "Copying data from SQLite3 to Postgres with synapse_port_db..."
 if [ -z "$COVERAGE" ]; then
@@ -181,7 +182,7 @@ DROP TABLE user_directory_search_docsize;
 DROP TABLE user_directory_search_stat;
 "
 sqlite3 "$SQLITE_DB" <<< "$SQL"
-psql $POSTGRES_DB_NAME -U "$POSTGRES_USERNAME" -w <<< "$SQL"
+psql "$POSTGRES_DB_NAME" -w <<< "$SQL"
 
 echo "Dumping SQLite3 schema to '$OUTPUT_DIR/$SQLITE_FULL_SCHEMA_OUTPUT_FILE'..."
 sqlite3 "$SQLITE_DB" ".dump" > "$OUTPUT_DIR/$SQLITE_FULL_SCHEMA_OUTPUT_FILE"

From 6c847785494473c0a430f368a0d79c9202b8ddd0 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Fri, 7 May 2021 14:54:09 +0100
Subject: [PATCH 04/15] Always cache 'event_to_prev_state_group' (#9950)

Fixes regression in send PDU times introduced in #9905.
---
 changelog.d/9950.feature    |  1 +
 synapse/handlers/message.py | 13 +++++++------
 2 files changed, 8 insertions(+), 6 deletions(-)
 create mode 100644 changelog.d/9950.feature

diff --git a/changelog.d/9950.feature b/changelog.d/9950.feature
new file mode 100644
index 0000000000..96a0e7f09f
--- /dev/null
+++ b/changelog.d/9950.feature
@@ -0,0 +1 @@
+Improve performance of sending events for worker-based deployments using Redis.
diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py
index 8729332d4b..5afb7fc261 100644
--- a/synapse/handlers/message.py
+++ b/synapse/handlers/message.py
@@ -1050,6 +1050,13 @@ class EventCreationHandler:
         )
 
         if state_entry.state_group:
+            await self._external_cache.set(
+                "event_to_prev_state_group",
+                event.event_id,
+                state_entry.state_group,
+                expiry_ms=60 * 60 * 1000,
+            )
+
             if state_entry.state_group in self._external_cache_joined_hosts_updates:
                 return
 
@@ -1057,12 +1064,6 @@ class EventCreationHandler:
 
             # Note that the expiry times must be larger than the expiry time in
             # _external_cache_joined_hosts_updates.
-            await self._external_cache.set(
-                "event_to_prev_state_group",
-                event.event_id,
-                state_entry.state_group,
-                expiry_ms=60 * 60 * 1000,
-            )
             await self._external_cache.set(
                 "get_joined_hosts",
                 str(state_entry.state_group),

From 51065c44bb0875373fada2838e69e4bc5005a95d Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <1389908+richvdh@users.noreply.github.com>
Date: Mon, 10 May 2021 13:02:55 +0100
Subject: [PATCH 05/15] Fix port_db on empty db (#9930)

... and test it.
---
 .buildkite/scripts/create_postgres_db.py   | 36 ---------------------
 .buildkite/scripts/postgres_exec.py        | 31 ++++++++++++++++++
 .buildkite/scripts/test_synapse_port_db.sh | 37 ++++++++++++++++------
 .github/workflows/tests.yml                |  2 +-
 changelog.d/9930.bugfix                    |  1 +
 scripts/synapse_port_db                    | 18 ++++++-----
 6 files changed, 70 insertions(+), 55 deletions(-)
 delete mode 100755 .buildkite/scripts/create_postgres_db.py
 create mode 100755 .buildkite/scripts/postgres_exec.py
 create mode 100644 changelog.d/9930.bugfix

diff --git a/.buildkite/scripts/create_postgres_db.py b/.buildkite/scripts/create_postgres_db.py
deleted file mode 100755
index cc829db216..0000000000
--- a/.buildkite/scripts/create_postgres_db.py
+++ /dev/null
@@ -1,36 +0,0 @@
-#!/usr/bin/env python
-# Copyright 2019 The Matrix.org Foundation C.I.C.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import logging
-
-from synapse.storage.engines import create_engine
-
-logger = logging.getLogger("create_postgres_db")
-
-if __name__ == "__main__":
-    # Create a PostgresEngine.
-    db_engine = create_engine({"name": "psycopg2", "args": {}})
-
-    # Connect to postgres to create the base database.
-    # We use "postgres" as a database because it's bound to exist and the "synapse" one
-    # doesn't exist yet.
-    db_conn = db_engine.module.connect(
-        user="postgres", host="postgres", password="postgres", dbname="postgres"
-    )
-    db_conn.autocommit = True
-    cur = db_conn.cursor()
-    cur.execute("CREATE DATABASE synapse;")
-    cur.close()
-    db_conn.close()
diff --git a/.buildkite/scripts/postgres_exec.py b/.buildkite/scripts/postgres_exec.py
new file mode 100755
index 0000000000..086b391724
--- /dev/null
+++ b/.buildkite/scripts/postgres_exec.py
@@ -0,0 +1,31 @@
+#!/usr/bin/env python
+# Copyright 2019 The Matrix.org Foundation C.I.C.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import sys
+
+import psycopg2
+
+# a very simple replacment for `psql`, to make up for the lack of the postgres client
+# libraries in the synapse docker image.
+
+# We use "postgres" as a database because it's bound to exist and the "synapse" one
+# doesn't exist yet.
+db_conn = psycopg2.connect(
+    user="postgres", host="postgres", password="postgres", dbname="postgres"
+)
+db_conn.autocommit = True
+cur = db_conn.cursor()
+for c in sys.argv[1:]:
+    cur.execute(c)
diff --git a/.buildkite/scripts/test_synapse_port_db.sh b/.buildkite/scripts/test_synapse_port_db.sh
index 8914319e38..a7e2454769 100755
--- a/.buildkite/scripts/test_synapse_port_db.sh
+++ b/.buildkite/scripts/test_synapse_port_db.sh
@@ -1,10 +1,10 @@
 #!/usr/bin/env bash
 #
-# Test script for 'synapse_port_db', which creates a virtualenv, installs Synapse along
-# with additional dependencies needed for the test (such as coverage or the PostgreSQL
-# driver), update the schema of the test SQLite database and run background updates on it,
-# create an empty test database in PostgreSQL, then run the 'synapse_port_db' script to
-# test porting the SQLite database to the PostgreSQL database (with coverage).
+# Test script for 'synapse_port_db'.
+#   - sets up synapse and deps
+#   - runs the port script on a prepopulated test sqlite db
+#   - also runs it against an new sqlite db
+
 
 set -xe
 cd `dirname $0`/../..
@@ -22,15 +22,32 @@ echo "--- Generate the signing key"
 # Generate the server's signing key.
 python -m synapse.app.homeserver --generate-keys -c .buildkite/sqlite-config.yaml
 
-echo "--- Prepare the databases"
+echo "--- Prepare test database"
 
 # Make sure the SQLite3 database is using the latest schema and has no pending background update.
 scripts-dev/update_database --database-config .buildkite/sqlite-config.yaml
 
 # Create the PostgreSQL database.
-./.buildkite/scripts/create_postgres_db.py
+./.buildkite/scripts/postgres_exec.py "CREATE DATABASE synapse"
 
-echo "+++ Run synapse_port_db"
-
-# Run the script
+echo "+++ Run synapse_port_db against test database"
+coverage run scripts/synapse_port_db --sqlite-database .buildkite/test_db.db --postgres-config .buildkite/postgres-config.yaml
+
+#####
+
+# Now do the same again, on an empty database.
+
+echo "--- Prepare empty SQLite database"
+
+# we do this by deleting the sqlite db, and then doing the same again.
+rm .buildkite/test_db.db
+
+scripts-dev/update_database --database-config .buildkite/sqlite-config.yaml
+
+# re-create the PostgreSQL database.
+./.buildkite/scripts/postgres_exec.py \
+  "DROP DATABASE synapse" \
+  "CREATE DATABASE synapse"
+
+echo "+++ Run synapse_port_db against empty database"
 coverage run scripts/synapse_port_db --sqlite-database .buildkite/test_db.db --postgres-config .buildkite/postgres-config.yaml
diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
index 12c82ac620..e7f3be1b4e 100644
--- a/.github/workflows/tests.yml
+++ b/.github/workflows/tests.yml
@@ -273,7 +273,7 @@ jobs:
           python-version: ${{ matrix.python-version }}
       - name: Patch Buildkite-specific test scripts
         run: |
-          sed -i -e 's/host="postgres"/host="localhost"/' .buildkite/scripts/create_postgres_db.py
+          sed -i -e 's/host="postgres"/host="localhost"/' .buildkite/scripts/postgres_exec.py
           sed -i -e 's/host: postgres/host: localhost/' .buildkite/postgres-config.yaml
           sed -i -e 's|/src/||' .buildkite/{sqlite,postgres}-config.yaml
           sed -i -e 's/\$TOP/\$GITHUB_WORKSPACE/' .coveragerc
diff --git a/changelog.d/9930.bugfix b/changelog.d/9930.bugfix
new file mode 100644
index 0000000000..9b22ed4458
--- /dev/null
+++ b/changelog.d/9930.bugfix
@@ -0,0 +1 @@
+Fix bugs introduced in v1.23.0 which made the PostgreSQL port script fail when run with a newly-created SQLite database.
diff --git a/scripts/synapse_port_db b/scripts/synapse_port_db
index f0c93d5226..5fb5bb35f7 100755
--- a/scripts/synapse_port_db
+++ b/scripts/synapse_port_db
@@ -913,10 +913,11 @@ class Porter(object):
                     (curr_forward_id + 1,),
                 )
 
-            txn.execute(
-                "ALTER SEQUENCE events_backfill_stream_seq RESTART WITH %s",
-                (curr_backward_id + 1,),
-            )
+            if curr_backward_id:
+                txn.execute(
+                    "ALTER SEQUENCE events_backfill_stream_seq RESTART WITH %s",
+                    (curr_backward_id + 1,),
+                )
 
         await self.postgres_store.db_pool.runInteraction(
             "_setup_events_stream_seqs", _setup_events_stream_seqs_set_pos,
@@ -954,10 +955,11 @@ class Porter(object):
                 (curr_chain_id,),
             )
 
-        await self.postgres_store.db_pool.runInteraction(
-            "_setup_event_auth_chain_id", r,
-        )
-
+        if curr_chain_id is not None:
+            await self.postgres_store.db_pool.runInteraction(
+                "_setup_event_auth_chain_id",
+                r,
+            )
 
 
 ##############################################

From 2b2985b5cfc4267dfb0f6b900e1a0f69f3f2cdc5 Mon Sep 17 00:00:00 2001
From: Erik Johnston <erik@matrix.org>
Date: Mon, 10 May 2021 13:29:02 +0100
Subject: [PATCH 06/15] Improve performance of backfilling in large rooms.
 (#9935)

We were pulling the full auth chain for the room out of the DB each time
we backfilled, which can be *huge* for large rooms and is totally
unnecessary.
---
 changelog.d/9935.feature       |   1 +
 synapse/handlers/federation.py | 131 +++++++++++++++------------------
 2 files changed, 59 insertions(+), 73 deletions(-)
 create mode 100644 changelog.d/9935.feature

diff --git a/changelog.d/9935.feature b/changelog.d/9935.feature
new file mode 100644
index 0000000000..eeda5bf50e
--- /dev/null
+++ b/changelog.d/9935.feature
@@ -0,0 +1 @@
+Improve performance of backfilling in large rooms.
diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py
index e8330a2b50..798ed75b30 100644
--- a/synapse/handlers/federation.py
+++ b/synapse/handlers/federation.py
@@ -552,8 +552,12 @@ class FederationHandler(BaseHandler):
         destination: str,
         room_id: str,
         event_id: str,
-    ) -> Tuple[List[EventBase], List[EventBase]]:
-        """Requests all of the room state at a given event from a remote homeserver.
+    ) -> List[EventBase]:
+        """Requests all of the room state at a given event from a remote
+        homeserver.
+
+        Will also fetch any missing events reported in the `auth_chain_ids`
+        section of `/state_ids`.
 
         Args:
             destination: The remote homeserver to query for the state.
@@ -561,8 +565,7 @@ class FederationHandler(BaseHandler):
             event_id: The id of the event we want the state at.
 
         Returns:
-            A list of events in the state, not including the event itself, and
-            a list of events in the auth chain for the given event.
+            A list of events in the state, not including the event itself.
         """
         (
             state_event_ids,
@@ -571,68 +574,53 @@ class FederationHandler(BaseHandler):
             destination, room_id, event_id=event_id
         )
 
-        desired_events = set(state_event_ids + auth_event_ids)
-
-        event_map = await self._get_events_from_store_or_dest(
-            destination, room_id, desired_events
-        )
-
-        failed_to_fetch = desired_events - event_map.keys()
-        if failed_to_fetch:
-            logger.warning(
-                "Failed to fetch missing state/auth events for %s %s",
-                event_id,
-                failed_to_fetch,
-            )
-
-        remote_state = [
-            event_map[e_id] for e_id in state_event_ids if e_id in event_map
-        ]
-
-        auth_chain = [event_map[e_id] for e_id in auth_event_ids if e_id in event_map]
-        auth_chain.sort(key=lambda e: e.depth)
-
-        return remote_state, auth_chain
-
-    async def _get_events_from_store_or_dest(
-        self, destination: str, room_id: str, event_ids: Iterable[str]
-    ) -> Dict[str, EventBase]:
-        """Fetch events from a remote destination, checking if we already have them.
-
-        Persists any events we don't already have as outliers.
-
-        If we fail to fetch any of the events, a warning will be logged, and the event
-        will be omitted from the result. Likewise, any events which turn out not to
-        be in the given room.
-
-        This function *does not* automatically get missing auth events of the
-        newly fetched events. Callers must include the full auth chain of
-        of the missing events in the `event_ids` argument, to ensure that any
-        missing auth events are correctly fetched.
-
-        Returns:
-            map from event_id to event
-        """
-        fetched_events = await self.store.get_events(event_ids, allow_rejected=True)
-
-        missing_events = set(event_ids) - fetched_events.keys()
-
-        if missing_events:
-            logger.debug(
-                "Fetching unknown state/auth events %s for room %s",
-                missing_events,
-                room_id,
-            )
+        # Fetch the state events from the DB, and check we have the auth events.
+        event_map = await self.store.get_events(state_event_ids, allow_rejected=True)
+        auth_events_in_store = await self.store.have_seen_events(auth_event_ids)
 
+        # Check for missing events. We handle state and auth event seperately,
+        # as we want to pull the state from the DB, but we don't for the auth
+        # events. (Note: we likely won't use the majority of the auth chain, and
+        # it can be *huge* for large rooms, so it's worth ensuring that we don't
+        # unnecessarily pull it from the DB).
+        missing_state_events = set(state_event_ids) - set(event_map)
+        missing_auth_events = set(auth_event_ids) - set(auth_events_in_store)
+        if missing_state_events or missing_auth_events:
             await self._get_events_and_persist(
-                destination=destination, room_id=room_id, events=missing_events
+                destination=destination,
+                room_id=room_id,
+                events=missing_state_events | missing_auth_events,
             )
 
-            # we need to make sure we re-load from the database to get the rejected
-            # state correct.
-            fetched_events.update(
-                (await self.store.get_events(missing_events, allow_rejected=True))
-            )
+            if missing_state_events:
+                new_events = await self.store.get_events(
+                    missing_state_events, allow_rejected=True
+                )
+                event_map.update(new_events)
+
+                missing_state_events.difference_update(new_events)
+
+                if missing_state_events:
+                    logger.warning(
+                        "Failed to fetch missing state events for %s %s",
+                        event_id,
+                        missing_state_events,
+                    )
+
+            if missing_auth_events:
+                auth_events_in_store = await self.store.have_seen_events(
+                    missing_auth_events
+                )
+                missing_auth_events.difference_update(auth_events_in_store)
+
+                if missing_auth_events:
+                    logger.warning(
+                        "Failed to fetch missing auth events for %s %s",
+                        event_id,
+                        missing_auth_events,
+                    )
+
+        remote_state = list(event_map.values())
 
         # check for events which were in the wrong room.
         #
@@ -640,8 +628,8 @@ class FederationHandler(BaseHandler):
         # auth_events at an event in room A are actually events in room B
 
         bad_events = [
-            (event_id, event.room_id)
-            for event_id, event in fetched_events.items()
+            (event.event_id, event.room_id)
+            for event in remote_state
             if event.room_id != room_id
         ]
 
@@ -658,9 +646,10 @@ class FederationHandler(BaseHandler):
                 room_id,
             )
 
-            del fetched_events[bad_event_id]
+        if bad_events:
+            remote_state = [e for e in remote_state if e.room_id == room_id]
 
-        return fetched_events
+        return remote_state
 
     async def _get_state_after_missing_prev_event(
         self,
@@ -963,27 +952,23 @@ class FederationHandler(BaseHandler):
 
         # For each edge get the current state.
 
-        auth_events = {}
         state_events = {}
         events_to_state = {}
         for e_id in edges:
-            state, auth = await self._get_state_for_room(
+            state = await self._get_state_for_room(
                 destination=dest,
                 room_id=room_id,
                 event_id=e_id,
             )
-            auth_events.update({a.event_id: a for a in auth})
-            auth_events.update({s.event_id: s for s in state})
             state_events.update({s.event_id: s for s in state})
             events_to_state[e_id] = state
 
         required_auth = {
             a_id
-            for event in events
-            + list(state_events.values())
-            + list(auth_events.values())
+            for event in events + list(state_events.values())
             for a_id in event.auth_event_ids()
         }
+        auth_events = await self.store.get_events(required_auth, allow_rejected=True)
         auth_events.update(
             {e_id: event_map[e_id] for e_id in required_auth if e_id in event_map}
         )

From 7967b36efe6a033f46cd882d0b31a8c3eb18631c Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <1389908+richvdh@users.noreply.github.com>
Date: Tue, 11 May 2021 11:02:56 +0100
Subject: [PATCH 07/15] Fix `m.room_key_request` to-device messages (#9961)

fixes #9960
---
 changelog.d/9961.bugfix                 |  1 +
 synapse/api/constants.py                |  5 +++-
 synapse/federation/federation_server.py | 19 --------------
 synapse/handlers/devicemessage.py       | 33 ++++++++++++++++++++-----
 4 files changed, 32 insertions(+), 26 deletions(-)
 create mode 100644 changelog.d/9961.bugfix

diff --git a/changelog.d/9961.bugfix b/changelog.d/9961.bugfix
new file mode 100644
index 0000000000..e26d141a53
--- /dev/null
+++ b/changelog.d/9961.bugfix
@@ -0,0 +1 @@
+Fix a bug introduced in Synapse 1.29.0 which caused `m.room_key_request` to-device messages sent from one user to another to be dropped.
diff --git a/synapse/api/constants.py b/synapse/api/constants.py
index ab628b2be7..3940da5c88 100644
--- a/synapse/api/constants.py
+++ b/synapse/api/constants.py
@@ -116,9 +116,12 @@ class EventTypes:
     MSC1772_SPACE_PARENT = "org.matrix.msc1772.space.parent"
 
 
+class ToDeviceEventTypes:
+    RoomKeyRequest = "m.room_key_request"
+
+
 class EduTypes:
     Presence = "m.presence"
-    RoomKeyRequest = "m.room_key_request"
 
 
 class RejectedReason:
diff --git a/synapse/federation/federation_server.py b/synapse/federation/federation_server.py
index b729a69203..ace30aa450 100644
--- a/synapse/federation/federation_server.py
+++ b/synapse/federation/federation_server.py
@@ -44,7 +44,6 @@ from synapse.api.errors import (
     SynapseError,
     UnsupportedRoomVersionError,
 )
-from synapse.api.ratelimiting import Ratelimiter
 from synapse.api.room_versions import KNOWN_ROOM_VERSIONS
 from synapse.events import EventBase
 from synapse.federation.federation_base import FederationBase, event_from_pdu_json
@@ -865,14 +864,6 @@ class FederationHandlerRegistry:
         # EDU received.
         self._edu_type_to_instance = {}  # type: Dict[str, List[str]]
 
-        # A rate limiter for incoming room key requests per origin.
-        self._room_key_request_rate_limiter = Ratelimiter(
-            store=hs.get_datastore(),
-            clock=self.clock,
-            rate_hz=self.config.rc_key_requests.per_second,
-            burst_count=self.config.rc_key_requests.burst_count,
-        )
-
     def register_edu_handler(
         self, edu_type: str, handler: Callable[[str, JsonDict], Awaitable[None]]
     ) -> None:
@@ -926,16 +917,6 @@ class FederationHandlerRegistry:
         if not self.config.use_presence and edu_type == EduTypes.Presence:
             return
 
-        # If the incoming room key requests from a particular origin are over
-        # the limit, drop them.
-        if (
-            edu_type == EduTypes.RoomKeyRequest
-            and not await self._room_key_request_rate_limiter.can_do_action(
-                None, origin
-            )
-        ):
-            return
-
         # Check if we have a handler on this instance
         handler = self.edu_handlers.get(edu_type)
         if handler:
diff --git a/synapse/handlers/devicemessage.py b/synapse/handlers/devicemessage.py
index c5d631de07..580b941595 100644
--- a/synapse/handlers/devicemessage.py
+++ b/synapse/handlers/devicemessage.py
@@ -15,7 +15,7 @@
 import logging
 from typing import TYPE_CHECKING, Any, Dict
 
-from synapse.api.constants import EduTypes
+from synapse.api.constants import ToDeviceEventTypes
 from synapse.api.errors import SynapseError
 from synapse.api.ratelimiting import Ratelimiter
 from synapse.logging.context import run_in_background
@@ -79,6 +79,8 @@ class DeviceMessageHandler:
                 ReplicationUserDevicesResyncRestServlet.make_client(hs)
             )
 
+        # a rate limiter for room key requests.  The keys are
+        # (sending_user_id, sending_device_id).
         self._ratelimiter = Ratelimiter(
             store=self.store,
             clock=hs.get_clock(),
@@ -100,12 +102,25 @@ class DeviceMessageHandler:
         for user_id, by_device in content["messages"].items():
             # we use UserID.from_string to catch invalid user ids
             if not self.is_mine(UserID.from_string(user_id)):
-                logger.warning("Request for keys for non-local user %s", user_id)
+                logger.warning("To-device message to non-local user %s", user_id)
                 raise SynapseError(400, "Not a user here")
 
             if not by_device:
                 continue
 
+            # Ratelimit key requests by the sending user.
+            if message_type == ToDeviceEventTypes.RoomKeyRequest:
+                allowed, _ = await self._ratelimiter.can_do_action(
+                    None, (sender_user_id, None)
+                )
+                if not allowed:
+                    logger.info(
+                        "Dropping room_key_request from %s to %s due to rate limit",
+                        sender_user_id,
+                        user_id,
+                    )
+                    continue
+
             messages_by_device = {
                 device_id: {
                     "content": message_content,
@@ -192,13 +207,19 @@ class DeviceMessageHandler:
         for user_id, by_device in messages.items():
             # Ratelimit local cross-user key requests by the sending device.
             if (
-                message_type == EduTypes.RoomKeyRequest
+                message_type == ToDeviceEventTypes.RoomKeyRequest
                 and user_id != sender_user_id
-                and await self._ratelimiter.can_do_action(
+            ):
+                allowed, _ = await self._ratelimiter.can_do_action(
                     requester, (sender_user_id, requester.device_id)
                 )
-            ):
-                continue
+                if not allowed:
+                    logger.info(
+                        "Dropping room_key_request from %s to %s due to rate limit",
+                        sender_user_id,
+                        user_id,
+                    )
+                    continue
 
             # we use UserID.from_string to catch invalid user ids
             if self.is_mine(UserID.from_string(user_id)):

From b378d98c8f93412bc51f0d4b9c4aa2d1701cf523 Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <1389908+richvdh@users.noreply.github.com>
Date: Tue, 11 May 2021 11:04:03 +0100
Subject: [PATCH 08/15] Add debug logging for issue #9533 (#9959)

Hopefully this will help us track down where to-device messages are getting
lost/delayed.
---
 changelog.d/9959.misc                          |  1 +
 .../federation/sender/per_destination_queue.py |  9 +++++++++
 synapse/logging/__init__.py                    |  7 ++++++-
 synapse/notifier.py                            |  8 ++++++++
 synapse/replication/tcp/client.py              |  1 -
 synapse/storage/databases/main/deviceinbox.py  | 18 ++++++++++++++++++
 6 files changed, 42 insertions(+), 2 deletions(-)
 create mode 100644 changelog.d/9959.misc

diff --git a/changelog.d/9959.misc b/changelog.d/9959.misc
new file mode 100644
index 0000000000..7231f29d79
--- /dev/null
+++ b/changelog.d/9959.misc
@@ -0,0 +1 @@
+Add debug logging for lost/delayed to-device messages.
diff --git a/synapse/federation/sender/per_destination_queue.py b/synapse/federation/sender/per_destination_queue.py
index 3b053ebcfb..3a2efd56ee 100644
--- a/synapse/federation/sender/per_destination_queue.py
+++ b/synapse/federation/sender/per_destination_queue.py
@@ -28,6 +28,7 @@ from synapse.api.presence import UserPresenceState
 from synapse.events import EventBase
 from synapse.federation.units import Edu
 from synapse.handlers.presence import format_user_presence_state
+from synapse.logging import issue9533_logger
 from synapse.logging.opentracing import SynapseTags, set_tag
 from synapse.metrics import sent_transactions_counter
 from synapse.metrics.background_process_metrics import run_as_background_process
@@ -574,6 +575,14 @@ class PerDestinationQueue:
             for content in contents
         ]
 
+        if edus:
+            issue9533_logger.debug(
+                "Sending %i to-device messages to %s, up to stream id %i",
+                len(edus),
+                self._destination,
+                stream_id,
+            )
+
         return (edus, stream_id)
 
     def _start_catching_up(self) -> None:
diff --git a/synapse/logging/__init__.py b/synapse/logging/__init__.py
index e00969f8b1..b50a4f95eb 100644
--- a/synapse/logging/__init__.py
+++ b/synapse/logging/__init__.py
@@ -12,8 +12,13 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-# These are imported to allow for nicer logging configuration files.
+import logging
+
 from synapse.logging._remote import RemoteHandler
 from synapse.logging._terse_json import JsonFormatter, TerseJsonFormatter
 
+# These are imported to allow for nicer logging configuration files.
 __all__ = ["RemoteHandler", "JsonFormatter", "TerseJsonFormatter"]
+
+# Debug logger for https://github.com/matrix-org/synapse/issues/9533 etc
+issue9533_logger = logging.getLogger("synapse.9533_debug")
diff --git a/synapse/notifier.py b/synapse/notifier.py
index b9531007e2..24b4e6649f 100644
--- a/synapse/notifier.py
+++ b/synapse/notifier.py
@@ -38,6 +38,7 @@ from synapse.api.constants import EventTypes, HistoryVisibility, Membership
 from synapse.api.errors import AuthError
 from synapse.events import EventBase
 from synapse.handlers.presence import format_user_presence_state
+from synapse.logging import issue9533_logger
 from synapse.logging.context import PreserveLoggingContext
 from synapse.logging.opentracing import log_kv, start_active_span
 from synapse.logging.utils import log_function
@@ -426,6 +427,13 @@ class Notifier:
             for room in rooms:
                 user_streams |= self.room_to_user_streams.get(room, set())
 
+            if stream_key == "to_device_key":
+                issue9533_logger.debug(
+                    "to-device messages stream id %s, awaking streams for %s",
+                    new_token,
+                    users,
+                )
+
             time_now_ms = self.clock.time_msec()
             for user_stream in user_streams:
                 try:
diff --git a/synapse/replication/tcp/client.py b/synapse/replication/tcp/client.py
index 4f3c6a18b6..62d7809175 100644
--- a/synapse/replication/tcp/client.py
+++ b/synapse/replication/tcp/client.py
@@ -51,7 +51,6 @@ if TYPE_CHECKING:
 
 logger = logging.getLogger(__name__)
 
-
 # How long we allow callers to wait for replication updates before timing out.
 _WAIT_FOR_REPLICATION_TIMEOUT_SECONDS = 30
 
diff --git a/synapse/storage/databases/main/deviceinbox.py b/synapse/storage/databases/main/deviceinbox.py
index 7c9d1f744e..50e7ddd735 100644
--- a/synapse/storage/databases/main/deviceinbox.py
+++ b/synapse/storage/databases/main/deviceinbox.py
@@ -15,6 +15,7 @@
 import logging
 from typing import List, Optional, Tuple
 
+from synapse.logging import issue9533_logger
 from synapse.logging.opentracing import log_kv, set_tag, trace
 from synapse.replication.tcp.streams import ToDeviceStream
 from synapse.storage._base import SQLBaseStore, db_to_json
@@ -404,6 +405,13 @@ class DeviceInboxWorkerStore(SQLBaseStore):
                 ],
             )
 
+            if remote_messages_by_destination:
+                issue9533_logger.debug(
+                    "Queued outgoing to-device messages with stream_id %i for %s",
+                    stream_id,
+                    list(remote_messages_by_destination.keys()),
+                )
+
         async with self._device_inbox_id_gen.get_next() as stream_id:
             now_ms = self.clock.time_msec()
             await self.db_pool.runInteraction(
@@ -533,6 +541,16 @@ class DeviceInboxWorkerStore(SQLBaseStore):
             ],
         )
 
+        issue9533_logger.debug(
+            "Stored to-device messages with stream_id %i for %s",
+            stream_id,
+            [
+                (user_id, device_id)
+                for (user_id, messages_by_device) in local_by_user_then_device.items()
+                for device_id in messages_by_device.keys()
+            ],
+        )
+
 
 class DeviceInboxBackgroundUpdateStore(SQLBaseStore):
     DEVICE_INBOX_STREAM_ID = "device_inbox_stream_drop"

From 86fb71431ca562ba536c50620d1e367a9c6d4ac9 Mon Sep 17 00:00:00 2001
From: Brendan Abolivier <babolivier@matrix.org>
Date: Tue, 11 May 2021 13:53:49 +0100
Subject: [PATCH 09/15]  1.33.2

---
 CHANGES.md            | 16 ++++++++++++++++
 changelog.d/9946.misc |  1 -
 debian/changelog      |  6 ++++++
 synapse/__init__.py   |  2 +-
 4 files changed, 23 insertions(+), 2 deletions(-)
 delete mode 100644 changelog.d/9946.misc

diff --git a/CHANGES.md b/CHANGES.md
index a41abbefba..7ae0e7b3c1 100644
--- a/CHANGES.md
+++ b/CHANGES.md
@@ -1,3 +1,19 @@
+Synapse 1.33.2 (2021-05-11)
+===========================
+
+Due to the security issue highlighted below, server administrators are encouraged to update Synapse. We are not aware of these vulnerabilities being exploited in the wild.
+
+Security advisory
+-----------------
+
+This release fixes a denial of service attack ([CVE-2021-29471](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-29471)) against Synapse's push rules implementation. Server admins are encouraged to upgrade.
+
+Internal Changes
+----------------
+
+- Unpin attrs dependency. ([\#9946](https://github.com/matrix-org/synapse/issues/9946))
+
+
 Synapse 1.33.1 (2021-05-06)
 ===========================
 
diff --git a/changelog.d/9946.misc b/changelog.d/9946.misc
deleted file mode 100644
index 142ec5496f..0000000000
--- a/changelog.d/9946.misc
+++ /dev/null
@@ -1 +0,0 @@
-Unpin attrs dependency.
diff --git a/debian/changelog b/debian/changelog
index de50dd14ea..76b82c172e 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,3 +1,9 @@
+matrix-synapse-py3 (1.33.2) stable; urgency=medium
+
+  * New synapse release 1.33.2.
+
+ -- Synapse Packaging team <packages@matrix.org>  Tue, 11 May 2021 11:17:59 +0100
+
 matrix-synapse-py3 (1.33.1) stable; urgency=medium
 
   * New synapse release 1.33.1.
diff --git a/synapse/__init__.py b/synapse/__init__.py
index 441cd8b339..ce822ccb04 100644
--- a/synapse/__init__.py
+++ b/synapse/__init__.py
@@ -47,7 +47,7 @@ try:
 except ImportError:
     pass
 
-__version__ = "1.33.1"
+__version__ = "1.33.2"
 
 if bool(os.environ.get("SYNAPSE_TEST_PATCH_LOG_CONTEXTS", False)):
     # We import here so that we don't have to install a bunch of deps when

From dc6366a9bd370a0f772f376a2053c0ce48cb6607 Mon Sep 17 00:00:00 2001
From: Aaron Raimist <aaron@raim.ist>
Date: Tue, 11 May 2021 08:03:23 -0500
Subject: [PATCH 10/15] Add config option to hide device names over federation
 (#9945)

Now that cross signing exists there is much less of a need for other people to look at devices and verify them individually. This PR adds a config option to allow you to prevent device display names from being shared with other servers.

Signed-off-by: Aaron Raimist <aaron@raim.ist>
---
 changelog.d/9945.feature                          |  1 +
 docs/sample_config.yaml                           |  6 ++++++
 synapse/config/federation.py                      | 10 ++++++++++
 synapse/storage/databases/main/end_to_end_keys.py |  4 +++-
 4 files changed, 20 insertions(+), 1 deletion(-)
 create mode 100644 changelog.d/9945.feature

diff --git a/changelog.d/9945.feature b/changelog.d/9945.feature
new file mode 100644
index 0000000000..84308e8cce
--- /dev/null
+++ b/changelog.d/9945.feature
@@ -0,0 +1 @@
+Add a config option to allow you to prevent device display names from being shared over federation. Contributed by @aaronraimist.
diff --git a/docs/sample_config.yaml b/docs/sample_config.yaml
index f469d6e54f..7cf222d356 100644
--- a/docs/sample_config.yaml
+++ b/docs/sample_config.yaml
@@ -741,6 +741,12 @@ acme:
 #
 #allow_profile_lookup_over_federation: false
 
+# Uncomment to disable device display name lookup over federation. By default, the
+# Federation API allows other homeservers to obtain device display names of any user
+# on this homeserver. Defaults to 'true'.
+#
+#allow_device_name_lookup_over_federation: false
+
 
 ## Caching ##
 
diff --git a/synapse/config/federation.py b/synapse/config/federation.py
index 090ba047fa..cdd7a1ef05 100644
--- a/synapse/config/federation.py
+++ b/synapse/config/federation.py
@@ -44,6 +44,10 @@ class FederationConfig(Config):
             "allow_profile_lookup_over_federation", True
         )
 
+        self.allow_device_name_lookup_over_federation = config.get(
+            "allow_device_name_lookup_over_federation", True
+        )
+
     def generate_config_section(self, config_dir_path, server_name, **kwargs):
         return """\
         ## Federation ##
@@ -75,6 +79,12 @@ class FederationConfig(Config):
         # on this homeserver. Defaults to 'true'.
         #
         #allow_profile_lookup_over_federation: false
+
+        # Uncomment to disable device display name lookup over federation. By default, the
+        # Federation API allows other homeservers to obtain device display names of any user
+        # on this homeserver. Defaults to 'true'.
+        #
+        #allow_device_name_lookup_over_federation: false
         """
 
 
diff --git a/synapse/storage/databases/main/end_to_end_keys.py b/synapse/storage/databases/main/end_to_end_keys.py
index 88afe97c41..398d6b6acb 100644
--- a/synapse/storage/databases/main/end_to_end_keys.py
+++ b/synapse/storage/databases/main/end_to_end_keys.py
@@ -84,7 +84,9 @@ class EndToEndKeyWorkerStore(EndToEndKeyBackgroundStore):
                 if keys:
                     result["keys"] = keys
 
-                device_display_name = device.display_name
+                device_display_name = None
+                if self.hs.config.allow_device_name_lookup_over_federation:
+                    device_display_name = device.display_name
                 if device_display_name:
                     result["device_display_name"] = device_display_name
 

From d1473f7362e9b146dbd256076c8e3c7d163e7d94 Mon Sep 17 00:00:00 2001
From: Brendan Abolivier <babolivier@matrix.org>
Date: Tue, 11 May 2021 14:09:46 +0100
Subject: [PATCH 11/15] Use link to advisory rather than to the CVE repo

---
 CHANGES.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/CHANGES.md b/CHANGES.md
index 7ae0e7b3c1..93efa3ce56 100644
--- a/CHANGES.md
+++ b/CHANGES.md
@@ -6,7 +6,7 @@ Due to the security issue highlighted below, server administrators are encourage
 Security advisory
 -----------------
 
-This release fixes a denial of service attack ([CVE-2021-29471](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-29471)) against Synapse's push rules implementation. Server admins are encouraged to upgrade.
+This release fixes a denial of service attack ([CVE-2021-29471](https://github.com/matrix-org/synapse/security/advisories/GHSA-x345-32rc-8h85)) against Synapse's push rules implementation. Server admins are encouraged to upgrade.
 
 Internal Changes
 ----------------

From 28c68411028e15858819f2a5896313ce0e71c25b Mon Sep 17 00:00:00 2001
From: Patrick Cloke <clokep@users.noreply.github.com>
Date: Tue, 11 May 2021 10:58:58 -0400
Subject: [PATCH 12/15] Send the `m.room.create` stripped event with invites
 (support MSC1772). (#9966)

MSC1772 specifies the m.room.create event should be sent as part
of the invite_state. This was done optionally behind an experimental
flag, but is now done by default due to MSC1772 being approved.
---
 UPGRADE.rst              | 29 +++++++++++++++++++++++++++++
 changelog.d/9915.feature |  2 +-
 changelog.d/9966.feature |  1 +
 docs/sample_config.yaml  |  1 +
 synapse/config/api.py    |  6 ++----
 5 files changed, 34 insertions(+), 5 deletions(-)
 create mode 100644 changelog.d/9966.feature

diff --git a/UPGRADE.rst b/UPGRADE.rst
index e921e0c08a..606e357b6e 100644
--- a/UPGRADE.rst
+++ b/UPGRADE.rst
@@ -85,6 +85,35 @@ for example:
      wget https://packages.matrix.org/debian/pool/main/m/matrix-synapse-py3/matrix-synapse-py3_1.3.0+stretch1_amd64.deb
      dpkg -i matrix-synapse-py3_1.3.0+stretch1_amd64.deb
 
+Upgrading to v1.34.0
+====================
+
+`room_invite_state_types` configuration setting
+-----------------------------------------------
+
+The ``room_invite_state_types`` configuration setting has been deprecated and
+replaced with ``room_prejoin_state``. See the `sample configuration file <https://github.com/matrix-org/synapse/blob/v1.34.0/docs/sample_config.yaml#L1515>`_.
+
+If you have set ``room_invite_state_types`` to the default value you should simply
+remove it from your configuration file. The default value used to be:
+
+.. code:: yaml
+
+   room_invite_state_types:
+      - "m.room.join_rules"
+      - "m.room.canonical_alias"
+      - "m.room.avatar"
+      - "m.room.encryption"
+      - "m.room.name"
+
+If you have customised this value by adding addition state types, you should
+remove ``room_invite_state_types`` and configure ``additional_event_types`` with
+your customisations.
+
+If you have customised this value by removing state types, you should rename
+``room_invite_state_types`` to ``additional_event_types``, and set
+``disable_default_event_types`` to ``true``.
+
 Upgrading to v1.33.0
 ====================
 
diff --git a/changelog.d/9915.feature b/changelog.d/9915.feature
index 832916cb01..7b81faabea 100644
--- a/changelog.d/9915.feature
+++ b/changelog.d/9915.feature
@@ -1 +1 @@
-Support stable identifiers from [MSC1772](https://github.com/matrix-org/matrix-doc/pull/1772).
+Support stable identifiers for [MSC1772](https://github.com/matrix-org/matrix-doc/pull/1772) Spaces. `m.space.child` events will now be taken into account when populating the experimental spaces summary response. Please see `UPGRADE.rst` if you have customised `room_invite_state_types` in your configuration.
\ No newline at end of file
diff --git a/changelog.d/9966.feature b/changelog.d/9966.feature
new file mode 100644
index 0000000000..7b81faabea
--- /dev/null
+++ b/changelog.d/9966.feature
@@ -0,0 +1 @@
+Support stable identifiers for [MSC1772](https://github.com/matrix-org/matrix-doc/pull/1772) Spaces. `m.space.child` events will now be taken into account when populating the experimental spaces summary response. Please see `UPGRADE.rst` if you have customised `room_invite_state_types` in your configuration.
\ No newline at end of file
diff --git a/docs/sample_config.yaml b/docs/sample_config.yaml
index 7cf222d356..67ad57b1aa 100644
--- a/docs/sample_config.yaml
+++ b/docs/sample_config.yaml
@@ -1521,6 +1521,7 @@ room_prejoin_state:
    # - m.room.avatar
    # - m.room.encryption
    # - m.room.name
+   # - m.room.create
    #
    # Uncomment the following to disable these defaults (so that only the event
    # types listed in 'additional_event_types' are shared). Defaults to 'false'.
diff --git a/synapse/config/api.py b/synapse/config/api.py
index 55c038c0c4..b18044f982 100644
--- a/synapse/config/api.py
+++ b/synapse/config/api.py
@@ -88,10 +88,6 @@ class ApiConfig(Config):
         if not room_prejoin_state_config.get("disable_default_event_types"):
             yield from _DEFAULT_PREJOIN_STATE_TYPES
 
-            if self.spaces_enabled:
-                # MSC1772 suggests adding m.room.create to the prejoin state
-                yield EventTypes.Create
-
         yield from room_prejoin_state_config.get("additional_event_types", [])
 
 
@@ -109,6 +105,8 @@ _DEFAULT_PREJOIN_STATE_TYPES = [
     EventTypes.RoomAvatar,
     EventTypes.RoomEncryption,
     EventTypes.Name,
+    # Per MSC1772.
+    EventTypes.Create,
 ]
 
 

From f4833e0c062e189fcfd8186fc197d1fc1052814a Mon Sep 17 00:00:00 2001
From: Patrick Cloke <clokep@users.noreply.github.com>
Date: Tue, 11 May 2021 12:21:43 -0400
Subject: [PATCH 13/15] Support fetching the spaces summary via GET over
 federation. (#9947)

Per changes in MSC2946, the C-S and S-S APIs for spaces summary
should use GET requests.

Until this is stable, the POST endpoints still exist.

This does not switch federation requests to use the GET version yet
since it is newly added and already deployed servers might not support
it. When switching to the stable endpoint we should switch to GET
requests.
---
 changelog.d/9947.feature               |  1 +
 synapse/federation/transport/client.py |  1 +
 synapse/federation/transport/server.py | 26 ++++++++++++++++++++++++++
 synapse/rest/client/v1/room.py         |  1 +
 4 files changed, 29 insertions(+)
 create mode 100644 changelog.d/9947.feature

diff --git a/changelog.d/9947.feature b/changelog.d/9947.feature
new file mode 100644
index 0000000000..ce8874f810
--- /dev/null
+++ b/changelog.d/9947.feature
@@ -0,0 +1 @@
+Update support for [MSC2946](https://github.com/matrix-org/matrix-doc/pull/2946): Spaces Summary.
diff --git a/synapse/federation/transport/client.py b/synapse/federation/transport/client.py
index ada322a81e..497848a2b7 100644
--- a/synapse/federation/transport/client.py
+++ b/synapse/federation/transport/client.py
@@ -995,6 +995,7 @@ class TransportLayerClient:
                returned per space
             exclude_rooms: a list of any rooms we can skip
         """
+        # TODO When switching to the stable endpoint, use GET instead of POST.
         path = _create_path(
             FEDERATION_UNSTABLE_PREFIX, "/org.matrix.msc2946/spaces/%s", room_id
         )
diff --git a/synapse/federation/transport/server.py b/synapse/federation/transport/server.py
index a3759bdda1..e1b7462474 100644
--- a/synapse/federation/transport/server.py
+++ b/synapse/federation/transport/server.py
@@ -1376,6 +1376,32 @@ class FederationSpaceSummaryServlet(BaseFederationServlet):
     PREFIX = FEDERATION_UNSTABLE_PREFIX + "/org.matrix.msc2946"
     PATH = "/spaces/(?P<room_id>[^/]*)"
 
+    async def on_GET(
+        self,
+        origin: str,
+        content: JsonDict,
+        query: Mapping[bytes, Sequence[bytes]],
+        room_id: str,
+    ) -> Tuple[int, JsonDict]:
+        suggested_only = parse_boolean_from_args(query, "suggested_only", default=False)
+        max_rooms_per_space = parse_integer_from_args(query, "max_rooms_per_space")
+
+        exclude_rooms = []
+        if b"exclude_rooms" in query:
+            try:
+                exclude_rooms = [
+                    room_id.decode("ascii") for room_id in query[b"exclude_rooms"]
+                ]
+            except Exception:
+                raise SynapseError(
+                    400, "Bad query parameter for exclude_rooms", Codes.INVALID_PARAM
+                )
+
+        return 200, await self.handler.federation_space_summary(
+            room_id, suggested_only, max_rooms_per_space, exclude_rooms
+        )
+
+    # TODO When switching to the stable endpoint, remove the POST handler.
     async def on_POST(
         self,
         origin: str,
diff --git a/synapse/rest/client/v1/room.py b/synapse/rest/client/v1/room.py
index 5cab4d3c7b..51813cccbe 100644
--- a/synapse/rest/client/v1/room.py
+++ b/synapse/rest/client/v1/room.py
@@ -1020,6 +1020,7 @@ class RoomSpaceSummaryRestServlet(RestServlet):
             max_rooms_per_space=parse_integer(request, "max_rooms_per_space"),
         )
 
+    # TODO When switching to the stable endpoint, remove the POST handler.
     async def on_POST(
         self, request: SynapseRequest, room_id: str
     ) -> Tuple[int, JsonDict]:

From 27c375f812725bead5fa41a55aacf316f6e2376c Mon Sep 17 00:00:00 2001
From: Patrick Cloke <clokep@users.noreply.github.com>
Date: Tue, 11 May 2021 12:57:39 -0400
Subject: [PATCH 14/15] Sort child events according to MSC1772 for the spaces
 summary API. (#9954)

This should help ensure that equivalent results are achieved between
homeservers querying for the summary of a space.

This implements modified MSC1772 rules, according to MSC2946.
The different is that the origin_server_ts of the m.room.create event
is not used as a tie-breaker since this might not be known if the
homeserver is not part of the room.
---
 changelog.d/9954.feature             |  1 +
 synapse/handlers/space_summary.py    | 71 +++++++++++++++++++++++-
 tests/handlers/test_space_summary.py | 81 ++++++++++++++++++++++++++++
 3 files changed, 151 insertions(+), 2 deletions(-)
 create mode 100644 changelog.d/9954.feature
 create mode 100644 tests/handlers/test_space_summary.py

diff --git a/changelog.d/9954.feature b/changelog.d/9954.feature
new file mode 100644
index 0000000000..ce8874f810
--- /dev/null
+++ b/changelog.d/9954.feature
@@ -0,0 +1 @@
+Update support for [MSC2946](https://github.com/matrix-org/matrix-doc/pull/2946): Spaces Summary.
diff --git a/synapse/handlers/space_summary.py b/synapse/handlers/space_summary.py
index 2e997841f1..e35d91832b 100644
--- a/synapse/handlers/space_summary.py
+++ b/synapse/handlers/space_summary.py
@@ -14,6 +14,7 @@
 
 import itertools
 import logging
+import re
 from collections import deque
 from typing import TYPE_CHECKING, Iterable, List, Optional, Sequence, Set, Tuple, cast
 
@@ -226,6 +227,23 @@ class SpaceSummaryHandler:
         suggested_only: bool,
         max_children: Optional[int],
     ) -> Tuple[Sequence[JsonDict], Sequence[JsonDict]]:
+        """
+        Generate a room entry and a list of event entries for a given room.
+
+        Args:
+            requester: The requesting user, or None if this is over federation.
+            room_id: The room ID to summarize.
+            suggested_only: True if only suggested children should be returned.
+                Otherwise, all children are returned.
+            max_children: The maximum number of children to return for this node.
+
+        Returns:
+            A tuple of:
+                An iterable of a single value of the room.
+
+                An iterable of the sorted children events. This may be limited
+                to a maximum size or may include all children.
+        """
         if not await self._is_room_accessible(room_id, requester):
             return (), ()
 
@@ -357,6 +375,18 @@ class SpaceSummaryHandler:
         return room_entry
 
     async def _get_child_events(self, room_id: str) -> Iterable[EventBase]:
+        """
+        Get the child events for a given room.
+
+        The returned results are sorted for stability.
+
+        Args:
+            room_id: The room id to get the children of.
+
+        Returns:
+            An iterable of sorted child events.
+        """
+
         # look for child rooms/spaces.
         current_state_ids = await self._store.get_current_state_ids(room_id)
 
@@ -370,8 +400,9 @@ class SpaceSummaryHandler:
             ]
         )
 
-        # filter out any events without a "via" (which implies it has been redacted)
-        return (e for e in events if _has_valid_via(e))
+        # filter out any events without a "via" (which implies it has been redacted),
+        # and order to ensure we return stable results.
+        return sorted(filter(_has_valid_via, events), key=_child_events_comparison_key)
 
 
 @attr.s(frozen=True, slots=True)
@@ -397,3 +428,39 @@ def _is_suggested_child_event(edge_event: EventBase) -> bool:
         return True
     logger.debug("Ignorning not-suggested child %s", edge_event.state_key)
     return False
+
+
+# Order may only contain characters in the range of \x20 (space) to \x7F (~).
+_INVALID_ORDER_CHARS_RE = re.compile(r"[^\x20-\x7F]")
+
+
+def _child_events_comparison_key(child: EventBase) -> Tuple[bool, Optional[str], str]:
+    """
+    Generate a value for comparing two child events for ordering.
+
+    The rules for ordering are supposed to be:
+
+    1. The 'order' key, if it is valid.
+    2. The 'origin_server_ts' of the 'm.room.create' event.
+    3. The 'room_id'.
+
+    But we skip step 2 since we may not have any state from the room.
+
+    Args:
+        child: The event for generating a comparison key.
+
+    Returns:
+        The comparison key as a tuple of:
+            False if the ordering is valid.
+            The ordering field.
+            The room ID.
+    """
+    order = child.content.get("order")
+    # If order is not a string or doesn't meet the requirements, ignore it.
+    if not isinstance(order, str):
+        order = None
+    elif len(order) > 50 or _INVALID_ORDER_CHARS_RE.search(order):
+        order = None
+
+    # Items without an order come last.
+    return (order is None, order, child.room_id)
diff --git a/tests/handlers/test_space_summary.py b/tests/handlers/test_space_summary.py
new file mode 100644
index 0000000000..2c5e81531b
--- /dev/null
+++ b/tests/handlers/test_space_summary.py
@@ -0,0 +1,81 @@
+#  Copyright 2021 The Matrix.org Foundation C.I.C.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+from typing import Any, Optional
+from unittest import mock
+
+from synapse.handlers.space_summary import _child_events_comparison_key
+
+from tests import unittest
+
+
+def _create_event(room_id: str, order: Optional[Any] = None):
+    result = mock.Mock()
+    result.room_id = room_id
+    result.content = {}
+    if order is not None:
+        result.content["order"] = order
+    return result
+
+
+def _order(*events):
+    return sorted(events, key=_child_events_comparison_key)
+
+
+class TestSpaceSummarySort(unittest.TestCase):
+    def test_no_order_last(self):
+        """An event with no ordering is placed behind those with an ordering."""
+        ev1 = _create_event("!abc:test")
+        ev2 = _create_event("!xyz:test", "xyz")
+
+        self.assertEqual([ev2, ev1], _order(ev1, ev2))
+
+    def test_order(self):
+        """The ordering should be used."""
+        ev1 = _create_event("!abc:test", "xyz")
+        ev2 = _create_event("!xyz:test", "abc")
+
+        self.assertEqual([ev2, ev1], _order(ev1, ev2))
+
+    def test_order_room_id(self):
+        """Room ID is a tie-breaker for ordering."""
+        ev1 = _create_event("!abc:test", "abc")
+        ev2 = _create_event("!xyz:test", "abc")
+
+        self.assertEqual([ev1, ev2], _order(ev1, ev2))
+
+    def test_invalid_ordering_type(self):
+        """Invalid orderings are considered the same as missing."""
+        ev1 = _create_event("!abc:test", 1)
+        ev2 = _create_event("!xyz:test", "xyz")
+
+        self.assertEqual([ev2, ev1], _order(ev1, ev2))
+
+        ev1 = _create_event("!abc:test", {})
+        self.assertEqual([ev2, ev1], _order(ev1, ev2))
+
+        ev1 = _create_event("!abc:test", [])
+        self.assertEqual([ev2, ev1], _order(ev1, ev2))
+
+        ev1 = _create_event("!abc:test", True)
+        self.assertEqual([ev2, ev1], _order(ev1, ev2))
+
+    def test_invalid_ordering_value(self):
+        """Invalid orderings are considered the same as missing."""
+        ev1 = _create_event("!abc:test", "foo\n")
+        ev2 = _create_event("!xyz:test", "xyz")
+
+        self.assertEqual([ev2, ev1], _order(ev1, ev2))
+
+        ev1 = _create_event("!abc:test", "a" * 51)
+        self.assertEqual([ev2, ev1], _order(ev1, ev2))

From 63fb220e5f90b63acfa4bdfb61788f2c2d867c86 Mon Sep 17 00:00:00 2001
From: Richard van der Hoff <1389908+richvdh@users.noreply.github.com>
Date: Tue, 11 May 2021 18:01:11 +0100
Subject: [PATCH 15/15] Tests for to-device messages (#9965)

---
 changelog.d/9965.bugfix                       |   1 +
 .../rest/client/v2_alpha/test_sendtodevice.py | 201 ++++++++++++++++++
 2 files changed, 202 insertions(+)
 create mode 100644 changelog.d/9965.bugfix
 create mode 100644 tests/rest/client/v2_alpha/test_sendtodevice.py

diff --git a/changelog.d/9965.bugfix b/changelog.d/9965.bugfix
new file mode 100644
index 0000000000..e26d141a53
--- /dev/null
+++ b/changelog.d/9965.bugfix
@@ -0,0 +1 @@
+Fix a bug introduced in Synapse 1.29.0 which caused `m.room_key_request` to-device messages sent from one user to another to be dropped.
diff --git a/tests/rest/client/v2_alpha/test_sendtodevice.py b/tests/rest/client/v2_alpha/test_sendtodevice.py
new file mode 100644
index 0000000000..c9c99cc5d7
--- /dev/null
+++ b/tests/rest/client/v2_alpha/test_sendtodevice.py
@@ -0,0 +1,201 @@
+# Copyright 2021 The Matrix.org Foundation C.I.C.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from synapse.rest import admin
+from synapse.rest.client.v1 import login
+from synapse.rest.client.v2_alpha import sendtodevice, sync
+
+from tests.unittest import HomeserverTestCase, override_config
+
+
+class SendToDeviceTestCase(HomeserverTestCase):
+    servlets = [
+        admin.register_servlets,
+        login.register_servlets,
+        sendtodevice.register_servlets,
+        sync.register_servlets,
+    ]
+
+    def test_user_to_user(self):
+        """A to-device message from one user to another should get delivered"""
+
+        user1 = self.register_user("u1", "pass")
+        user1_tok = self.login("u1", "pass", "d1")
+
+        user2 = self.register_user("u2", "pass")
+        user2_tok = self.login("u2", "pass", "d2")
+
+        # send the message
+        test_msg = {"foo": "bar"}
+        chan = self.make_request(
+            "PUT",
+            "/_matrix/client/r0/sendToDevice/m.test/1234",
+            content={"messages": {user2: {"d2": test_msg}}},
+            access_token=user1_tok,
+        )
+        self.assertEqual(chan.code, 200, chan.result)
+
+        # check it appears
+        channel = self.make_request("GET", "/sync", access_token=user2_tok)
+        self.assertEqual(channel.code, 200, channel.result)
+        expected_result = {
+            "events": [
+                {
+                    "sender": user1,
+                    "type": "m.test",
+                    "content": test_msg,
+                }
+            ]
+        }
+        self.assertEqual(channel.json_body["to_device"], expected_result)
+
+        # it should re-appear if we do another sync
+        channel = self.make_request("GET", "/sync", access_token=user2_tok)
+        self.assertEqual(channel.code, 200, channel.result)
+        self.assertEqual(channel.json_body["to_device"], expected_result)
+
+        # it should *not* appear if we do an incremental sync
+        sync_token = channel.json_body["next_batch"]
+        channel = self.make_request(
+            "GET", f"/sync?since={sync_token}", access_token=user2_tok
+        )
+        self.assertEqual(channel.code, 200, channel.result)
+        self.assertEqual(channel.json_body.get("to_device", {}).get("events", []), [])
+
+    @override_config({"rc_key_requests": {"per_second": 10, "burst_count": 2}})
+    def test_local_room_key_request(self):
+        """m.room_key_request has special-casing; test from local user"""
+        user1 = self.register_user("u1", "pass")
+        user1_tok = self.login("u1", "pass", "d1")
+
+        user2 = self.register_user("u2", "pass")
+        user2_tok = self.login("u2", "pass", "d2")
+
+        # send three messages
+        for i in range(3):
+            chan = self.make_request(
+                "PUT",
+                f"/_matrix/client/r0/sendToDevice/m.room_key_request/{i}",
+                content={"messages": {user2: {"d2": {"idx": i}}}},
+                access_token=user1_tok,
+            )
+            self.assertEqual(chan.code, 200, chan.result)
+
+        # now sync: we should get two of the three
+        channel = self.make_request("GET", "/sync", access_token=user2_tok)
+        self.assertEqual(channel.code, 200, channel.result)
+        msgs = channel.json_body["to_device"]["events"]
+        self.assertEqual(len(msgs), 2)
+        for i in range(2):
+            self.assertEqual(
+                msgs[i],
+                {"sender": user1, "type": "m.room_key_request", "content": {"idx": i}},
+            )
+        sync_token = channel.json_body["next_batch"]
+
+        # ... time passes
+        self.reactor.advance(1)
+
+        # and we can send more messages
+        chan = self.make_request(
+            "PUT",
+            "/_matrix/client/r0/sendToDevice/m.room_key_request/3",
+            content={"messages": {user2: {"d2": {"idx": 3}}}},
+            access_token=user1_tok,
+        )
+        self.assertEqual(chan.code, 200, chan.result)
+
+        # ... which should arrive
+        channel = self.make_request(
+            "GET", f"/sync?since={sync_token}", access_token=user2_tok
+        )
+        self.assertEqual(channel.code, 200, channel.result)
+        msgs = channel.json_body["to_device"]["events"]
+        self.assertEqual(len(msgs), 1)
+        self.assertEqual(
+            msgs[0],
+            {"sender": user1, "type": "m.room_key_request", "content": {"idx": 3}},
+        )
+
+    @override_config({"rc_key_requests": {"per_second": 10, "burst_count": 2}})
+    def test_remote_room_key_request(self):
+        """m.room_key_request has special-casing; test from remote user"""
+        user2 = self.register_user("u2", "pass")
+        user2_tok = self.login("u2", "pass", "d2")
+
+        federation_registry = self.hs.get_federation_registry()
+
+        # send three messages
+        for i in range(3):
+            self.get_success(
+                federation_registry.on_edu(
+                    "m.direct_to_device",
+                    "remote_server",
+                    {
+                        "sender": "@user:remote_server",
+                        "type": "m.room_key_request",
+                        "messages": {user2: {"d2": {"idx": i}}},
+                        "message_id": f"{i}",
+                    },
+                )
+            )
+
+        # now sync: we should get two of the three
+        channel = self.make_request("GET", "/sync", access_token=user2_tok)
+        self.assertEqual(channel.code, 200, channel.result)
+        msgs = channel.json_body["to_device"]["events"]
+        self.assertEqual(len(msgs), 2)
+        for i in range(2):
+            self.assertEqual(
+                msgs[i],
+                {
+                    "sender": "@user:remote_server",
+                    "type": "m.room_key_request",
+                    "content": {"idx": i},
+                },
+            )
+        sync_token = channel.json_body["next_batch"]
+
+        # ... time passes
+        self.reactor.advance(1)
+
+        # and we can send more messages
+        self.get_success(
+            federation_registry.on_edu(
+                "m.direct_to_device",
+                "remote_server",
+                {
+                    "sender": "@user:remote_server",
+                    "type": "m.room_key_request",
+                    "messages": {user2: {"d2": {"idx": 3}}},
+                    "message_id": "3",
+                },
+            )
+        )
+
+        # ... which should arrive
+        channel = self.make_request(
+            "GET", f"/sync?since={sync_token}", access_token=user2_tok
+        )
+        self.assertEqual(channel.code, 200, channel.result)
+        msgs = channel.json_body["to_device"]["events"]
+        self.assertEqual(len(msgs), 1)
+        self.assertEqual(
+            msgs[0],
+            {
+                "sender": "@user:remote_server",
+                "type": "m.room_key_request",
+                "content": {"idx": 3},
+            },
+        )