From 56c4b47df36a99d2fc57a64013a4f482e25ee097 Mon Sep 17 00:00:00 2001 From: Dan Callahan Date: Fri, 30 Apr 2021 15:36:05 +0100 Subject: [PATCH 01/34] Build Debian packages for Ubuntu 21.04 Hirsute (#9909) Signed-off-by: Dan Callahan --- changelog.d/9909.feature | 1 + scripts-dev/build_debian_packages | 7 ++++--- 2 files changed, 5 insertions(+), 3 deletions(-) create mode 100644 changelog.d/9909.feature diff --git a/changelog.d/9909.feature b/changelog.d/9909.feature new file mode 100644 index 0000000000..41aba5cca6 --- /dev/null +++ b/changelog.d/9909.feature @@ -0,0 +1 @@ +Build Debian packages for Ubuntu 21.04 (Hirsute Hippo). diff --git a/scripts-dev/build_debian_packages b/scripts-dev/build_debian_packages index 3bb6e2c7ea..07d018db99 100755 --- a/scripts-dev/build_debian_packages +++ b/scripts-dev/build_debian_packages @@ -21,9 +21,10 @@ DISTS = ( "debian:buster", "debian:bullseye", "debian:sid", - "ubuntu:bionic", - "ubuntu:focal", - "ubuntu:groovy", + "ubuntu:bionic", # 18.04 LTS (our EOL forced by Py36 on 2021-12-23) + "ubuntu:focal", # 20.04 LTS (our EOL forced by Py38 on 2024-10-14) + "ubuntu:groovy", # 20.10 (EOL 2021-07-07) + "ubuntu:hirsute", # 21.04 (EOL 2022-01-05) ) DESC = '''\ From b85821aca2c3cfc1c732dfcc0c1d6758a263487a Mon Sep 17 00:00:00 2001 From: Andrew Morgan <1342360+anoadragon453@users.noreply.github.com> Date: Tue, 4 May 2021 13:28:59 +0100 Subject: [PATCH 02/34] Add port parameter to the sample config for psycopg2 args (#9911) Adds the `port` option with the default value to the sample config file. --- changelog.d/9911.doc | 1 + docs/sample_config.yaml | 1 + synapse/config/database.py | 1 + 3 files changed, 3 insertions(+) create mode 100644 changelog.d/9911.doc diff --git a/changelog.d/9911.doc b/changelog.d/9911.doc new file mode 100644 index 0000000000..f7fd9f1ba9 --- /dev/null +++ b/changelog.d/9911.doc @@ -0,0 +1 @@ +Add `port` argument to the Postgres database sample config section. \ No newline at end of file diff --git a/docs/sample_config.yaml b/docs/sample_config.yaml index e0350279ad..d013725cdc 100644 --- a/docs/sample_config.yaml +++ b/docs/sample_config.yaml @@ -810,6 +810,7 @@ caches: # password: secretpassword # database: synapse # host: localhost +# port: 5432 # cp_min: 5 # cp_max: 10 # diff --git a/synapse/config/database.py b/synapse/config/database.py index 79a02706b4..c76ef1e1de 100644 --- a/synapse/config/database.py +++ b/synapse/config/database.py @@ -58,6 +58,7 @@ DEFAULT_CONFIG = """\ # password: secretpassword # database: synapse # host: localhost +# port: 5432 # cp_min: 5 # cp_max: 10 # From e3bc4617fcd5c858ff02cf2d443b898db87ae8a5 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 4 May 2021 15:14:22 +0100 Subject: [PATCH 03/34] Time external cache response time (#9904) --- changelog.d/9904.misc | 1 + synapse/replication/tcp/external_cache.py | 36 ++++++++++++++++------- 2 files changed, 27 insertions(+), 10 deletions(-) create mode 100644 changelog.d/9904.misc diff --git a/changelog.d/9904.misc b/changelog.d/9904.misc new file mode 100644 index 0000000000..3db1e625ae --- /dev/null +++ b/changelog.d/9904.misc @@ -0,0 +1 @@ +Time response time for external cache requests. diff --git a/synapse/replication/tcp/external_cache.py b/synapse/replication/tcp/external_cache.py index 1a3b051e3c..b402f82810 100644 --- a/synapse/replication/tcp/external_cache.py +++ b/synapse/replication/tcp/external_cache.py @@ -15,7 +15,7 @@ import logging from typing import TYPE_CHECKING, Any, Optional -from prometheus_client import Counter +from prometheus_client import Counter, Histogram from synapse.logging.context import make_deferred_yieldable from synapse.util import json_decoder, json_encoder @@ -35,6 +35,20 @@ get_counter = Counter( labelnames=["cache_name", "hit"], ) +response_timer = Histogram( + "synapse_external_cache_response_time_seconds", + "Time taken to get a response from Redis for a cache get/set request", + labelnames=["method"], + buckets=( + 0.001, + 0.002, + 0.005, + 0.01, + 0.02, + 0.05, + ), +) + logger = logging.getLogger(__name__) @@ -72,13 +86,14 @@ class ExternalCache: logger.debug("Caching %s %s: %r", cache_name, key, encoded_value) - return await make_deferred_yieldable( - self._redis_connection.set( - self._get_redis_key(cache_name, key), - encoded_value, - pexpire=expiry_ms, + with response_timer.labels("set").time(): + return await make_deferred_yieldable( + self._redis_connection.set( + self._get_redis_key(cache_name, key), + encoded_value, + pexpire=expiry_ms, + ) ) - ) async def get(self, cache_name: str, key: str) -> Optional[Any]: """Look up a key/value in the named cache.""" @@ -86,9 +101,10 @@ class ExternalCache: if self._redis_connection is None: return None - result = await make_deferred_yieldable( - self._redis_connection.get(self._get_redis_key(cache_name, key)) - ) + with response_timer.labels("get").time(): + result = await make_deferred_yieldable( + self._redis_connection.get(self._get_redis_key(cache_name, key)) + ) logger.debug("Got cache result %s %s: %r", cache_name, key, result) From 0644ac0989d06fc0ba5d08a5412d65d75f7d8db2 Mon Sep 17 00:00:00 2001 From: Brendan Abolivier Date: Wed, 5 May 2021 14:15:54 +0100 Subject: [PATCH 04/34] 1.33.0 --- CHANGES.md | 9 +++++++++ changelog.d/9909.feature | 1 - debian/changelog | 6 ++++++ synapse/__init__.py | 2 +- 4 files changed, 16 insertions(+), 2 deletions(-) delete mode 100644 changelog.d/9909.feature diff --git a/CHANGES.md b/CHANGES.md index bdeb614b9e..206859cff7 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,3 +1,12 @@ +Synapse 1.33.0 (2021-05-05) +=========================== + +Features +-------- + +- Build Debian packages for Ubuntu 21.04 (Hirsute Hippo). ([\#9909](https://github.com/matrix-org/synapse/issues/9909)) + + Synapse 1.33.0rc2 (2021-04-29) ============================== diff --git a/changelog.d/9909.feature b/changelog.d/9909.feature deleted file mode 100644 index 41aba5cca6..0000000000 --- a/changelog.d/9909.feature +++ /dev/null @@ -1 +0,0 @@ -Build Debian packages for Ubuntu 21.04 (Hirsute Hippo). diff --git a/debian/changelog b/debian/changelog index fd33bfda5c..b54d3f2bc6 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,9 @@ +matrix-synapse-py3 (1.33.0) stable; urgency=medium + + * New synapse release 1.33.0. + + -- Synapse Packaging team Wed, 05 May 2021 14:15:27 +0100 + matrix-synapse-py3 (1.32.2) stable; urgency=medium * New synapse release 1.32.2. diff --git a/synapse/__init__.py b/synapse/__init__.py index 319c52be2c..5eac40730a 100644 --- a/synapse/__init__.py +++ b/synapse/__init__.py @@ -47,7 +47,7 @@ try: except ImportError: pass -__version__ = "1.33.0rc2" +__version__ = "1.33.0" if bool(os.environ.get("SYNAPSE_TEST_PATCH_LOG_CONTEXTS", False)): # We import here so that we don't have to install a bunch of deps when From e9eb3549d32a6f93d07de8dbd5e1ebe54c8d8278 Mon Sep 17 00:00:00 2001 From: "DeepBlueV7.X" Date: Wed, 5 May 2021 13:37:56 +0000 Subject: [PATCH 05/34] Leave out optional keys from /sync (#9919) This leaves out all optional keys from /sync. This should be fine for all clients tested against conduit already, but it may break some clients, as such we should check, that at least most of them don't break horribly and maybe back out some of the individual changes. (We can probably always leave out groups for example, while the others may cause more issues.) Signed-off-by: Nicolas Werner --- changelog.d/9919.feature | 1 + synapse/rest/client/v2_alpha/sync.py | 62 +++++++++++++------ tests/rest/client/v2_alpha/test_sync.py | 30 +-------- .../test_resource_limits_server_notices.py | 8 ++- 4 files changed, 51 insertions(+), 50 deletions(-) create mode 100644 changelog.d/9919.feature diff --git a/changelog.d/9919.feature b/changelog.d/9919.feature new file mode 100644 index 0000000000..07747505d2 --- /dev/null +++ b/changelog.d/9919.feature @@ -0,0 +1 @@ +Omit empty fields from the `/sync` response. Contributed by @deepbluev7. diff --git a/synapse/rest/client/v2_alpha/sync.py b/synapse/rest/client/v2_alpha/sync.py index 95ee3f1b84..5f85653330 100644 --- a/synapse/rest/client/v2_alpha/sync.py +++ b/synapse/rest/client/v2_alpha/sync.py @@ -14,6 +14,7 @@ import itertools import logging +from collections import defaultdict from typing import TYPE_CHECKING, Tuple from synapse.api.constants import PresenceState @@ -229,24 +230,49 @@ class SyncRestServlet(RestServlet): ) logger.debug("building sync response dict") - return { - "account_data": {"events": sync_result.account_data}, - "to_device": {"events": sync_result.to_device}, - "device_lists": { - "changed": list(sync_result.device_lists.changed), - "left": list(sync_result.device_lists.left), - }, - "presence": SyncRestServlet.encode_presence(sync_result.presence, time_now), - "rooms": {"join": joined, "invite": invited, "leave": archived}, - "groups": { - "join": sync_result.groups.join, - "invite": sync_result.groups.invite, - "leave": sync_result.groups.leave, - }, - "device_one_time_keys_count": sync_result.device_one_time_keys_count, - "org.matrix.msc2732.device_unused_fallback_key_types": sync_result.device_unused_fallback_key_types, - "next_batch": await sync_result.next_batch.to_string(self.store), - } + + response: dict = defaultdict(dict) + response["next_batch"] = await sync_result.next_batch.to_string(self.store) + + if sync_result.account_data: + response["account_data"] = {"events": sync_result.account_data} + if sync_result.presence: + response["presence"] = SyncRestServlet.encode_presence( + sync_result.presence, time_now + ) + + if sync_result.to_device: + response["to_device"] = {"events": sync_result.to_device} + + if sync_result.device_lists.changed: + response["device_lists"]["changed"] = list(sync_result.device_lists.changed) + if sync_result.device_lists.left: + response["device_lists"]["left"] = list(sync_result.device_lists.left) + + if sync_result.device_one_time_keys_count: + response[ + "device_one_time_keys_count" + ] = sync_result.device_one_time_keys_count + if sync_result.device_unused_fallback_key_types: + response[ + "org.matrix.msc2732.device_unused_fallback_key_types" + ] = sync_result.device_unused_fallback_key_types + + if joined: + response["rooms"]["join"] = joined + if invited: + response["rooms"]["invite"] = invited + if archived: + response["rooms"]["leave"] = archived + + if sync_result.groups.join: + response["groups"]["join"] = sync_result.groups.join + if sync_result.groups.invite: + response["groups"]["invite"] = sync_result.groups.invite + if sync_result.groups.leave: + response["groups"]["leave"] = sync_result.groups.leave + + return response @staticmethod def encode_presence(events, time_now): diff --git a/tests/rest/client/v2_alpha/test_sync.py b/tests/rest/client/v2_alpha/test_sync.py index dbcbdf159a..74be5176d0 100644 --- a/tests/rest/client/v2_alpha/test_sync.py +++ b/tests/rest/client/v2_alpha/test_sync.py @@ -37,35 +37,7 @@ class FilterTestCase(unittest.HomeserverTestCase): channel = self.make_request("GET", "/sync") self.assertEqual(channel.code, 200) - self.assertTrue( - { - "next_batch", - "rooms", - "presence", - "account_data", - "to_device", - "device_lists", - }.issubset(set(channel.json_body.keys())) - ) - - def test_sync_presence_disabled(self): - """ - When presence is disabled, the key does not appear in /sync. - """ - self.hs.config.use_presence = False - - channel = self.make_request("GET", "/sync") - - self.assertEqual(channel.code, 200) - self.assertTrue( - { - "next_batch", - "rooms", - "account_data", - "to_device", - "device_lists", - }.issubset(set(channel.json_body.keys())) - ) + self.assertIn("next_batch", channel.json_body) class SyncFilterTestCase(unittest.HomeserverTestCase): diff --git a/tests/server_notices/test_resource_limits_server_notices.py b/tests/server_notices/test_resource_limits_server_notices.py index d46521ccdc..3245aa91ca 100644 --- a/tests/server_notices/test_resource_limits_server_notices.py +++ b/tests/server_notices/test_resource_limits_server_notices.py @@ -306,8 +306,9 @@ class TestResourceLimitsServerNoticesWithRealRooms(unittest.HomeserverTestCase): channel = self.make_request("GET", "/sync?timeout=0", access_token=tok) - invites = channel.json_body["rooms"]["invite"] - self.assertEqual(len(invites), 0, invites) + self.assertNotIn( + "rooms", channel.json_body, "Got invites without server notice" + ) def test_invite_with_notice(self): """Tests that, if the MAU limit is hit, the server notices user invites each user @@ -364,7 +365,8 @@ class TestResourceLimitsServerNoticesWithRealRooms(unittest.HomeserverTestCase): # We could also pick another user and sync with it, which would return an # invite to a system notices room, but it doesn't matter which user we're # using so we use the last one because it saves us an extra sync. - invites = channel.json_body["rooms"]["invite"] + if "rooms" in channel.json_body: + invites = channel.json_body["rooms"]["invite"] # Make sure we have an invite to process. self.assertEqual(len(invites), 1, invites) From d5305000f1c5799ffb6fcd64ad27e7bfd8ba2113 Mon Sep 17 00:00:00 2001 From: Christopher May-Townsend Date: Wed, 5 May 2021 16:33:04 +0100 Subject: [PATCH 06/34] Docker healthcheck timings - add startup delay and changed interval (#9913) * Add healthcheck startup delay by 5secs and reduced interval check to 15s to reduce waiting time for docker aware edge routers bringing an instance online --- changelog.d/9913.docker | 1 + docker/Dockerfile | 2 +- docker/README.md | 17 ++++++++++++++--- 3 files changed, 16 insertions(+), 4 deletions(-) create mode 100644 changelog.d/9913.docker diff --git a/changelog.d/9913.docker b/changelog.d/9913.docker new file mode 100644 index 0000000000..93835e14cb --- /dev/null +++ b/changelog.d/9913.docker @@ -0,0 +1 @@ +Added startup_delay to docker healthcheck to reduce waiting time for coming online, updated readme for extra options, contributed by @Maquis196. diff --git a/docker/Dockerfile b/docker/Dockerfile index 4f5cd06d72..2bdc607e66 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -88,5 +88,5 @@ EXPOSE 8008/tcp 8009/tcp 8448/tcp ENTRYPOINT ["/start.py"] -HEALTHCHECK --interval=1m --timeout=5s \ +HEALTHCHECK --start-period=5s --interval=15s --timeout=5s \ CMD curl -fSs http://localhost:8008/health || exit 1 diff --git a/docker/README.md b/docker/README.md index a7d1e670fe..c8d3c4b3da 100644 --- a/docker/README.md +++ b/docker/README.md @@ -191,6 +191,16 @@ whilst running the above `docker run` commands. ``` --no-healthcheck ``` + +## Disabling the healthcheck in docker-compose file + +If you wish to disable the healthcheck via docker-compose, append the following to your service configuration. + +``` + healthcheck: + disable: true +``` + ## Setting custom healthcheck on docker run If you wish to point the healthcheck at a different port with docker command, add the following @@ -202,14 +212,15 @@ If you wish to point the healthcheck at a different port with docker command, ad ## Setting the healthcheck in docker-compose file You can add the following to set a custom healthcheck in a docker compose file. -You will need version >2.1 for this to work. +You will need docker-compose version >2.1 for this to work. ``` healthcheck: test: ["CMD", "curl", "-fSs", "http://localhost:8008/health"] - interval: 1m - timeout: 10s + interval: 15s + timeout: 5s retries: 3 + start_period: 5s ``` ## Using jemalloc From d0aee697ac0587c005bc1048f5036979331f1101 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 5 May 2021 16:49:34 +0100 Subject: [PATCH 07/34] Use get_current_users_in_room from store and not StateHandler (#9910) --- changelog.d/9910.bugfix | 1 + changelog.d/9910.feature | 1 + synapse/handlers/directory.py | 4 ++-- synapse/handlers/events.py | 2 +- synapse/handlers/message.py | 2 +- synapse/handlers/presence.py | 2 +- synapse/handlers/room.py | 2 +- synapse/handlers/sync.py | 6 +++--- synapse/state/__init__.py | 10 +++++++--- synapse/storage/_base.py | 1 + synapse/storage/databases/main/roommember.py | 8 ++++++-- synapse/storage/databases/main/user_directory.py | 4 +--- 12 files changed, 26 insertions(+), 17 deletions(-) create mode 100644 changelog.d/9910.bugfix create mode 100644 changelog.d/9910.feature diff --git a/changelog.d/9910.bugfix b/changelog.d/9910.bugfix new file mode 100644 index 0000000000..06d523fd46 --- /dev/null +++ b/changelog.d/9910.bugfix @@ -0,0 +1 @@ +Fix bug where user directory could get out of sync if room visibility and membership changed in quick succession. diff --git a/changelog.d/9910.feature b/changelog.d/9910.feature new file mode 100644 index 0000000000..54165cce18 --- /dev/null +++ b/changelog.d/9910.feature @@ -0,0 +1 @@ +Improve performance after joining a large room when presence is enabled. diff --git a/synapse/handlers/directory.py b/synapse/handlers/directory.py index de1b14cde3..4064a2b859 100644 --- a/synapse/handlers/directory.py +++ b/synapse/handlers/directory.py @@ -78,7 +78,7 @@ class DirectoryHandler(BaseHandler): # TODO(erikj): Add transactions. # TODO(erikj): Check if there is a current association. if not servers: - users = await self.state.get_current_users_in_room(room_id) + users = await self.store.get_users_in_room(room_id) servers = {get_domain_from_id(u) for u in users} if not servers: @@ -270,7 +270,7 @@ class DirectoryHandler(BaseHandler): Codes.NOT_FOUND, ) - users = await self.state.get_current_users_in_room(room_id) + users = await self.store.get_users_in_room(room_id) extra_servers = {get_domain_from_id(u) for u in users} servers = set(extra_servers) | set(servers) diff --git a/synapse/handlers/events.py b/synapse/handlers/events.py index d82144d7fa..f134f1e234 100644 --- a/synapse/handlers/events.py +++ b/synapse/handlers/events.py @@ -103,7 +103,7 @@ class EventStreamHandler(BaseHandler): # Send down presence. if event.state_key == auth_user_id: # Send down presence for everyone in the room. - users = await self.state.get_current_users_in_room( + users = await self.store.get_users_in_room( event.room_id ) # type: Iterable[str] else: diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py index 49f8aa25ea..393f17c3a3 100644 --- a/synapse/handlers/message.py +++ b/synapse/handlers/message.py @@ -258,7 +258,7 @@ class MessageHandler: "Getting joined members after leaving is not implemented" ) - users_with_profile = await self.state.get_current_users_in_room(room_id) + users_with_profile = await self.store.get_users_in_room_with_profiles(room_id) # If this is an AS, double check that they are allowed to see the members. # This can either be because the AS user is in the room or because there diff --git a/synapse/handlers/presence.py b/synapse/handlers/presence.py index ebbc234334..8e085dfbec 100644 --- a/synapse/handlers/presence.py +++ b/synapse/handlers/presence.py @@ -1293,7 +1293,7 @@ class PresenceHandler(BasePresenceHandler): remote_host = get_domain_from_id(user_id) - users = await self.state.get_current_users_in_room(room_id) + users = await self.store.get_users_in_room(room_id) user_ids = list(filter(self.is_mine_id, users)) states_d = await self.current_state_for_users(user_ids) diff --git a/synapse/handlers/room.py b/synapse/handlers/room.py index 5a888b7941..fb4823a5cc 100644 --- a/synapse/handlers/room.py +++ b/synapse/handlers/room.py @@ -1327,7 +1327,7 @@ class RoomShutdownHandler: new_room_id = None logger.info("Shutting down room %r", room_id) - users = await self.state.get_current_users_in_room(room_id) + users = await self.store.get_users_in_room(room_id) kicked_users = [] failed_to_kick_users = [] for user_id in users: diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py index a9a3ee05c3..0fcc1532da 100644 --- a/synapse/handlers/sync.py +++ b/synapse/handlers/sync.py @@ -1190,7 +1190,7 @@ class SyncHandler: # Step 1b, check for newly joined rooms for room_id in newly_joined_rooms: - joined_users = await self.state.get_current_users_in_room(room_id) + joined_users = await self.store.get_users_in_room(room_id) newly_joined_or_invited_users.update(joined_users) # TODO: Check that these users are actually new, i.e. either they @@ -1206,7 +1206,7 @@ class SyncHandler: # Now find users that we no longer track for room_id in newly_left_rooms: - left_users = await self.state.get_current_users_in_room(room_id) + left_users = await self.store.get_users_in_room(room_id) newly_left_users.update(left_users) # Remove any users that we still share a room with. @@ -1361,7 +1361,7 @@ class SyncHandler: extra_users_ids = set(newly_joined_or_invited_users) for room_id in newly_joined_rooms: - users = await self.state.get_current_users_in_room(room_id) + users = await self.store.get_users_in_room(room_id) extra_users_ids.update(users) extra_users_ids.discard(user.to_string()) diff --git a/synapse/state/__init__.py b/synapse/state/__init__.py index b3bd92d37c..a1770f620e 100644 --- a/synapse/state/__init__.py +++ b/synapse/state/__init__.py @@ -213,19 +213,23 @@ class StateHandler: return ret.state async def get_current_users_in_room( - self, room_id: str, latest_event_ids: Optional[List[str]] = None + self, room_id: str, latest_event_ids: List[str] ) -> Dict[str, ProfileInfo]: """ Get the users who are currently in a room. + Note: This is much slower than using the equivalent method + `DataStore.get_users_in_room` or `DataStore.get_users_in_room_with_profiles`, + so this should only be used when wanting the users at a particular point + in the room. + Args: room_id: The ID of the room. latest_event_ids: Precomputed list of latest event IDs. Will be computed if None. Returns: Dictionary of user IDs to their profileinfo. """ - if not latest_event_ids: - latest_event_ids = await self.store.get_latest_event_ids_in_room(room_id) + assert latest_event_ids is not None logger.debug("calling resolve_state_groups from get_current_users_in_room") diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index 6b68d8720c..3d98d3f5f8 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -69,6 +69,7 @@ class SQLBaseStore(metaclass=ABCMeta): self._attempt_to_invalidate_cache("is_host_joined", (room_id, host)) self._attempt_to_invalidate_cache("get_users_in_room", (room_id,)) + self._attempt_to_invalidate_cache("get_users_in_room_with_profiles", (room_id,)) self._attempt_to_invalidate_cache("get_room_summary", (room_id,)) self._attempt_to_invalidate_cache("get_current_state_ids", (room_id,)) diff --git a/synapse/storage/databases/main/roommember.py b/synapse/storage/databases/main/roommember.py index 2a8532f8c1..5fc3bb5a7d 100644 --- a/synapse/storage/databases/main/roommember.py +++ b/synapse/storage/databases/main/roommember.py @@ -205,8 +205,12 @@ class RoomMemberWorkerStore(EventsWorkerStore): def _get_users_in_room_with_profiles(txn) -> Dict[str, ProfileInfo]: sql = """ - SELECT user_id, display_name, avatar_url FROM room_memberships - WHERE room_id = ? AND membership = ? + SELECT state_key, display_name, avatar_url FROM room_memberships as m + INNER JOIN current_state_events as c + ON m.event_id = c.event_id + AND m.room_id = c.room_id + AND m.user_id = c.state_key + WHERE c.type = 'm.room.member' AND c.room_id = ? AND m.membership = ? """ txn.execute(sql, (room_id, Membership.JOIN)) diff --git a/synapse/storage/databases/main/user_directory.py b/synapse/storage/databases/main/user_directory.py index 7a082fdd21..a6bfb4902a 100644 --- a/synapse/storage/databases/main/user_directory.py +++ b/synapse/storage/databases/main/user_directory.py @@ -142,8 +142,6 @@ class UserDirectoryBackgroundUpdateStore(StateDeltasStore): batch_size (int): Maximum number of state events to process per cycle. """ - state = self.hs.get_state_handler() - # If we don't have progress filed, delete everything. if not progress: await self.delete_all_from_user_dir() @@ -197,7 +195,7 @@ class UserDirectoryBackgroundUpdateStore(StateDeltasStore): room_id ) - users_with_profile = await state.get_current_users_in_room(room_id) + users_with_profile = await self.get_users_in_room_with_profiles(room_id) user_ids = set(users_with_profile) # Update each user in the user directory. From de8f0a03a3cc3a2327dfd3058c99e48067965079 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 5 May 2021 16:53:22 +0100 Subject: [PATCH 08/34] Don't set the external cache if its been done recently (#9905) --- changelog.d/9905.feature | 1 + synapse/handlers/federation.py | 4 +++- synapse/handlers/message.py | 34 ++++++++++++++++++++++++++++++---- 3 files changed, 34 insertions(+), 5 deletions(-) create mode 100644 changelog.d/9905.feature diff --git a/changelog.d/9905.feature b/changelog.d/9905.feature new file mode 100644 index 0000000000..96a0e7f09f --- /dev/null +++ b/changelog.d/9905.feature @@ -0,0 +1 @@ +Improve performance of sending events for worker-based deployments using Redis. diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index 9d867aaf4d..e8330a2b50 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -2446,7 +2446,9 @@ class FederationHandler(BaseHandler): # If we are going to send this event over federation we precaclculate # the joined hosts. if event.internal_metadata.get_send_on_behalf_of(): - await self.event_creation_handler.cache_joined_hosts_for_event(event) + await self.event_creation_handler.cache_joined_hosts_for_event( + event, context + ) return context diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py index 393f17c3a3..8729332d4b 100644 --- a/synapse/handlers/message.py +++ b/synapse/handlers/message.py @@ -51,6 +51,7 @@ from synapse.storage.state import StateFilter from synapse.types import Requester, RoomAlias, StreamToken, UserID, create_requester from synapse.util import json_decoder, json_encoder from synapse.util.async_helpers import Linearizer +from synapse.util.caches.expiringcache import ExpiringCache from synapse.util.metrics import measure_func from synapse.visibility import filter_events_for_client @@ -457,6 +458,19 @@ class EventCreationHandler: self._external_cache = hs.get_external_cache() + # Stores the state groups we've recently added to the joined hosts + # external cache. Note that the timeout must be significantly less than + # the TTL on the external cache. + self._external_cache_joined_hosts_updates = ( + None + ) # type: Optional[ExpiringCache] + if self._external_cache.is_enabled(): + self._external_cache_joined_hosts_updates = ExpiringCache( + "_external_cache_joined_hosts_updates", + self.clock, + expiry_ms=30 * 60 * 1000, + ) + async def create_event( self, requester: Requester, @@ -967,7 +981,7 @@ class EventCreationHandler: await self.action_generator.handle_push_actions_for_event(event, context) - await self.cache_joined_hosts_for_event(event) + await self.cache_joined_hosts_for_event(event, context) try: # If we're a worker we need to hit out to the master. @@ -1008,7 +1022,9 @@ class EventCreationHandler: await self.store.remove_push_actions_from_staging(event.event_id) raise - async def cache_joined_hosts_for_event(self, event: EventBase) -> None: + async def cache_joined_hosts_for_event( + self, event: EventBase, context: EventContext + ) -> None: """Precalculate the joined hosts at the event, when using Redis, so that external federation senders don't have to recalculate it themselves. """ @@ -1016,6 +1032,9 @@ class EventCreationHandler: if not self._external_cache.is_enabled(): return + # If external cache is enabled we should always have this. + assert self._external_cache_joined_hosts_updates is not None + # We actually store two mappings, event ID -> prev state group, # state group -> joined hosts, which is much more space efficient # than event ID -> joined hosts. @@ -1023,16 +1042,21 @@ class EventCreationHandler: # Note: We have to cache event ID -> prev state group, as we don't # store that in the DB. # - # Note: We always set the state group -> joined hosts cache, even if - # we already set it, so that the expiry time is reset. + # Note: We set the state group -> joined hosts cache if it hasn't been + # set for a while, so that the expiry time is reset. state_entry = await self.state.resolve_state_groups_for_events( event.room_id, event_ids=event.prev_event_ids() ) if state_entry.state_group: + if state_entry.state_group in self._external_cache_joined_hosts_updates: + return + joined_hosts = await self.store.get_joined_hosts(event.room_id, state_entry) + # Note that the expiry times must be larger than the expiry time in + # _external_cache_joined_hosts_updates. await self._external_cache.set( "event_to_prev_state_group", event.event_id, @@ -1046,6 +1070,8 @@ class EventCreationHandler: expiry_ms=60 * 60 * 1000, ) + self._external_cache_joined_hosts_updates[state_entry.state_group] = None + async def _validate_canonical_alias( self, directory_handler, room_alias_str: str, expected_room_id: str ) -> None: From 1fb9a2d0bf2506ca6e5343cb340a441585ca1c07 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 5 May 2021 16:53:45 +0100 Subject: [PATCH 09/34] Limit how often GC happens by time. (#9902) Synapse can be quite memory intensive, and unless care is taken to tune the GC thresholds it can end up thrashing, causing noticable performance problems for large servers. We fix this by limiting how often we GC a given generation, regardless of current counts/thresholds. This does not help with the reverse problem where the thresholds are set too high, but that should only happen in situations where they've been manually configured. Adds a `gc_min_seconds_between` config option to override the defaults. Fixes #9890. --- changelog.d/9902.feature | 1 + docs/sample_config.yaml | 10 ++++++++++ synapse/app/generic_worker.py | 3 +++ synapse/app/homeserver.py | 3 +++ synapse/config/server.py | 31 ++++++++++++++++++++++++++++++- synapse/metrics/__init__.py | 18 ++++++++++++++++-- 6 files changed, 63 insertions(+), 3 deletions(-) create mode 100644 changelog.d/9902.feature diff --git a/changelog.d/9902.feature b/changelog.d/9902.feature new file mode 100644 index 0000000000..4d9f324d4e --- /dev/null +++ b/changelog.d/9902.feature @@ -0,0 +1 @@ +Add limits to how often Synapse will GC, ensuring that large servers do not end up GC thrashing if `gc_thresholds` has not been correctly set. diff --git a/docs/sample_config.yaml b/docs/sample_config.yaml index d013725cdc..f469d6e54f 100644 --- a/docs/sample_config.yaml +++ b/docs/sample_config.yaml @@ -152,6 +152,16 @@ presence: # #gc_thresholds: [700, 10, 10] +# The minimum time in seconds between each GC for a generation, regardless of +# the GC thresholds. This ensures that we don't do GC too frequently. +# +# A value of `[1s, 10s, 30s]` indicates that a second must pass between consecutive +# generation 0 GCs, etc. +# +# Defaults to `[1s, 10s, 30s]`. +# +#gc_min_interval: [0.5s, 30s, 1m] + # Set the limit on the returned events in the timeline in the get # and sync operations. The default value is 100. -1 means no upper limit. # diff --git a/synapse/app/generic_worker.py b/synapse/app/generic_worker.py index 1a15ceee81..a3fe9a3f38 100644 --- a/synapse/app/generic_worker.py +++ b/synapse/app/generic_worker.py @@ -455,6 +455,9 @@ def start(config_options): synapse.events.USE_FROZEN_DICTS = config.use_frozen_dicts + if config.server.gc_seconds: + synapse.metrics.MIN_TIME_BETWEEN_GCS = config.server.gc_seconds + hs = GenericWorkerServer( config.server_name, config=config, diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index 8e78134bbe..6a823da10d 100644 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -342,6 +342,9 @@ def setup(config_options): events.USE_FROZEN_DICTS = config.use_frozen_dicts + if config.server.gc_seconds: + synapse.metrics.MIN_TIME_BETWEEN_GCS = config.server.gc_seconds + hs = SynapseHomeServer( config.server_name, config=config, diff --git a/synapse/config/server.py b/synapse/config/server.py index 21ca7b33e3..c290a35a92 100644 --- a/synapse/config/server.py +++ b/synapse/config/server.py @@ -19,7 +19,7 @@ import logging import os.path import re from textwrap import indent -from typing import Any, Dict, Iterable, List, Optional, Set +from typing import Any, Dict, Iterable, List, Optional, Set, Tuple import attr import yaml @@ -572,6 +572,7 @@ class ServerConfig(Config): _warn_if_webclient_configured(self.listeners) self.gc_thresholds = read_gc_thresholds(config.get("gc_thresholds", None)) + self.gc_seconds = self.read_gc_intervals(config.get("gc_min_interval", None)) @attr.s class LimitRemoteRoomsConfig: @@ -917,6 +918,16 @@ class ServerConfig(Config): # #gc_thresholds: [700, 10, 10] + # The minimum time in seconds between each GC for a generation, regardless of + # the GC thresholds. This ensures that we don't do GC too frequently. + # + # A value of `[1s, 10s, 30s]` indicates that a second must pass between consecutive + # generation 0 GCs, etc. + # + # Defaults to `[1s, 10s, 30s]`. + # + #gc_min_interval: [0.5s, 30s, 1m] + # Set the limit on the returned events in the timeline in the get # and sync operations. The default value is 100. -1 means no upper limit. # @@ -1305,6 +1316,24 @@ class ServerConfig(Config): help="Turn on the twisted telnet manhole service on the given port.", ) + def read_gc_intervals(self, durations) -> Optional[Tuple[float, float, float]]: + """Reads the three durations for the GC min interval option, returning seconds.""" + if durations is None: + return None + + try: + if len(durations) != 3: + raise ValueError() + return ( + self.parse_duration(durations[0]) / 1000, + self.parse_duration(durations[1]) / 1000, + self.parse_duration(durations[2]) / 1000, + ) + except Exception: + raise ConfigError( + "Value of `gc_min_interval` must be a list of three durations if set" + ) + def is_threepid_reserved(reserved_threepids, threepid): """Check the threepid against the reserved threepid config diff --git a/synapse/metrics/__init__.py b/synapse/metrics/__init__.py index 31b7b3c256..e671da26d5 100644 --- a/synapse/metrics/__init__.py +++ b/synapse/metrics/__init__.py @@ -535,6 +535,13 @@ class ReactorLastSeenMetric: REGISTRY.register(ReactorLastSeenMetric()) +# The minimum time in seconds between GCs for each generation, regardless of the current GC +# thresholds and counts. +MIN_TIME_BETWEEN_GCS = (1.0, 10.0, 30.0) + +# The time (in seconds since the epoch) of the last time we did a GC for each generation. +_last_gc = [0.0, 0.0, 0.0] + def runUntilCurrentTimer(reactor, func): @functools.wraps(func) @@ -575,11 +582,16 @@ def runUntilCurrentTimer(reactor, func): return ret # Check if we need to do a manual GC (since its been disabled), and do - # one if necessary. + # one if necessary. Note we go in reverse order as e.g. a gen 1 GC may + # promote an object into gen 2, and we don't want to handle the same + # object multiple times. threshold = gc.get_threshold() counts = gc.get_count() for i in (2, 1, 0): - if threshold[i] < counts[i]: + # We check if we need to do one based on a straightforward + # comparison between the threshold and count. We also do an extra + # check to make sure that we don't a GC too often. + if threshold[i] < counts[i] and MIN_TIME_BETWEEN_GCS[i] < end - _last_gc[i]: if i == 0: logger.debug("Collecting gc %d", i) else: @@ -589,6 +601,8 @@ def runUntilCurrentTimer(reactor, func): unreachable = gc.collect(i) end = time.time() + _last_gc[i] = end + gc_time.labels(i).observe(end - start) gc_unreachable.labels(i).set(unreachable) From ef889c98a6cde0cfa95f7fdaf7f99ec3c1e9bb7f Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 5 May 2021 16:54:36 +0100 Subject: [PATCH 10/34] Optionally track memory usage of each LruCache (#9881) This will double count slightly in the presence of interned strings. It's off by default as it can consume a lot of resources. --- changelog.d/9881.feature | 1 + mypy.ini | 3 +++ synapse/app/generic_worker.py | 1 + synapse/app/homeserver.py | 1 + synapse/config/cache.py | 11 ++++++++ synapse/python_dependencies.py | 2 ++ synapse/util/caches/__init__.py | 31 +++++++++++++++++++++ synapse/util/caches/lrucache.py | 48 ++++++++++++++++++++++++++++++++- 8 files changed, 97 insertions(+), 1 deletion(-) create mode 100644 changelog.d/9881.feature diff --git a/changelog.d/9881.feature b/changelog.d/9881.feature new file mode 100644 index 0000000000..088a517e02 --- /dev/null +++ b/changelog.d/9881.feature @@ -0,0 +1 @@ +Add experimental option to track memory usage of the caches. diff --git a/mypy.ini b/mypy.ini index a40f705b76..ea655a0d4d 100644 --- a/mypy.ini +++ b/mypy.ini @@ -171,3 +171,6 @@ ignore_missing_imports = True [mypy-txacme.*] ignore_missing_imports = True + +[mypy-pympler.*] +ignore_missing_imports = True diff --git a/synapse/app/generic_worker.py b/synapse/app/generic_worker.py index a3fe9a3f38..f730cdbd78 100644 --- a/synapse/app/generic_worker.py +++ b/synapse/app/generic_worker.py @@ -454,6 +454,7 @@ def start(config_options): config.server.update_user_directory = False synapse.events.USE_FROZEN_DICTS = config.use_frozen_dicts + synapse.util.caches.TRACK_MEMORY_USAGE = config.caches.track_memory_usage if config.server.gc_seconds: synapse.metrics.MIN_TIME_BETWEEN_GCS = config.server.gc_seconds diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index 6a823da10d..b2501ee4d7 100644 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -341,6 +341,7 @@ def setup(config_options): sys.exit(0) events.USE_FROZEN_DICTS = config.use_frozen_dicts + synapse.util.caches.TRACK_MEMORY_USAGE = config.caches.track_memory_usage if config.server.gc_seconds: synapse.metrics.MIN_TIME_BETWEEN_GCS = config.server.gc_seconds diff --git a/synapse/config/cache.py b/synapse/config/cache.py index 41b9b3f51f..91165ee1ce 100644 --- a/synapse/config/cache.py +++ b/synapse/config/cache.py @@ -17,6 +17,8 @@ import re import threading from typing import Callable, Dict +from synapse.python_dependencies import DependencyException, check_requirements + from ._base import Config, ConfigError # The prefix for all cache factor-related environment variables @@ -189,6 +191,15 @@ class CacheConfig(Config): ) self.cache_factors[cache] = factor + self.track_memory_usage = cache_config.get("track_memory_usage", False) + if self.track_memory_usage: + try: + check_requirements("cache_memory") + except DependencyException as e: + raise ConfigError( + e.message # noqa: B306, DependencyException.message is a property + ) + # Resize all caches (if necessary) with the new factors we've loaded self.resize_all_caches() diff --git a/synapse/python_dependencies.py b/synapse/python_dependencies.py index 2de946f464..d58eeeaa74 100644 --- a/synapse/python_dependencies.py +++ b/synapse/python_dependencies.py @@ -116,6 +116,8 @@ CONDITIONAL_REQUIREMENTS = { # hiredis is not a *strict* dependency, but it makes things much faster. # (if it is not installed, we fall back to slow code.) "redis": ["txredisapi>=1.4.7", "hiredis"], + # Required to use experimental `caches.track_memory_usage` config option. + "cache_memory": ["pympler"], } ALL_OPTIONAL_REQUIREMENTS = set() # type: Set[str] diff --git a/synapse/util/caches/__init__.py b/synapse/util/caches/__init__.py index 46af7fa473..ca36f07c20 100644 --- a/synapse/util/caches/__init__.py +++ b/synapse/util/caches/__init__.py @@ -24,6 +24,11 @@ from synapse.config.cache import add_resizable_cache logger = logging.getLogger(__name__) + +# Whether to track estimated memory usage of the LruCaches. +TRACK_MEMORY_USAGE = False + + caches_by_name = {} # type: Dict[str, Sized] collectors_by_name = {} # type: Dict[str, CacheMetric] @@ -32,6 +37,11 @@ cache_hits = Gauge("synapse_util_caches_cache:hits", "", ["name"]) cache_evicted = Gauge("synapse_util_caches_cache:evicted_size", "", ["name"]) cache_total = Gauge("synapse_util_caches_cache:total", "", ["name"]) cache_max_size = Gauge("synapse_util_caches_cache_max_size", "", ["name"]) +cache_memory_usage = Gauge( + "synapse_util_caches_cache_size_bytes", + "Estimated memory usage of the caches", + ["name"], +) response_cache_size = Gauge("synapse_util_caches_response_cache:size", "", ["name"]) response_cache_hits = Gauge("synapse_util_caches_response_cache:hits", "", ["name"]) @@ -52,6 +62,7 @@ class CacheMetric: hits = attr.ib(default=0) misses = attr.ib(default=0) evicted_size = attr.ib(default=0) + memory_usage = attr.ib(default=None) def inc_hits(self): self.hits += 1 @@ -62,6 +73,19 @@ class CacheMetric: def inc_evictions(self, size=1): self.evicted_size += size + def inc_memory_usage(self, memory: int): + if self.memory_usage is None: + self.memory_usage = 0 + + self.memory_usage += memory + + def dec_memory_usage(self, memory: int): + self.memory_usage -= memory + + def clear_memory_usage(self): + if self.memory_usage is not None: + self.memory_usage = 0 + def describe(self): return [] @@ -81,6 +105,13 @@ class CacheMetric: cache_total.labels(self._cache_name).set(self.hits + self.misses) if getattr(self._cache, "max_size", None): cache_max_size.labels(self._cache_name).set(self._cache.max_size) + + if TRACK_MEMORY_USAGE: + # self.memory_usage can be None if nothing has been inserted + # into the cache yet. + cache_memory_usage.labels(self._cache_name).set( + self.memory_usage or 0 + ) if self._collect_callback: self._collect_callback() except Exception as e: diff --git a/synapse/util/caches/lrucache.py b/synapse/util/caches/lrucache.py index 10b0ec6b75..1be675e014 100644 --- a/synapse/util/caches/lrucache.py +++ b/synapse/util/caches/lrucache.py @@ -32,9 +32,36 @@ from typing import ( from typing_extensions import Literal from synapse.config import cache as cache_config +from synapse.util import caches from synapse.util.caches import CacheMetric, register_cache from synapse.util.caches.treecache import TreeCache +try: + from pympler.asizeof import Asizer + + def _get_size_of(val: Any, *, recurse=True) -> int: + """Get an estimate of the size in bytes of the object. + + Args: + val: The object to size. + recurse: If true will include referenced values in the size, + otherwise only sizes the given object. + """ + # Ignore singleton values when calculating memory usage. + if val in ((), None, ""): + return 0 + + sizer = Asizer() + sizer.exclude_refs((), None, "") + return sizer.asizeof(val, limit=100 if recurse else 0) + + +except ImportError: + + def _get_size_of(val: Any, *, recurse=True) -> int: + return 0 + + # Function type: the type used for invalidation callbacks FT = TypeVar("FT", bound=Callable[..., Any]) @@ -56,7 +83,7 @@ def enumerate_leaves(node, depth): class _Node: - __slots__ = ["prev_node", "next_node", "key", "value", "callbacks"] + __slots__ = ["prev_node", "next_node", "key", "value", "callbacks", "memory"] def __init__( self, @@ -84,6 +111,16 @@ class _Node: self.add_callbacks(callbacks) + self.memory = 0 + if caches.TRACK_MEMORY_USAGE: + self.memory = ( + _get_size_of(key) + + _get_size_of(value) + + _get_size_of(self.callbacks, recurse=False) + + _get_size_of(self, recurse=False) + ) + self.memory += _get_size_of(self.memory, recurse=False) + def add_callbacks(self, callbacks: Collection[Callable[[], None]]) -> None: """Add to stored list of callbacks, removing duplicates.""" @@ -233,6 +270,9 @@ class LruCache(Generic[KT, VT]): if size_callback: cached_cache_len[0] += size_callback(node.value) + if caches.TRACK_MEMORY_USAGE and metrics: + metrics.inc_memory_usage(node.memory) + def move_node_to_front(node): prev_node = node.prev_node next_node = node.next_node @@ -258,6 +298,9 @@ class LruCache(Generic[KT, VT]): node.run_and_clear_callbacks() + if caches.TRACK_MEMORY_USAGE and metrics: + metrics.dec_memory_usage(node.memory) + return deleted_len @overload @@ -373,6 +416,9 @@ class LruCache(Generic[KT, VT]): if size_callback: cached_cache_len[0] = 0 + if caches.TRACK_MEMORY_USAGE and metrics: + metrics.clear_memory_usage() + @synchronized def cache_contains(key: KT) -> bool: return key in cache From e2a443550e7b47bf8fe1b5fbd76f9ca95e81cbad Mon Sep 17 00:00:00 2001 From: Patrick Cloke Date: Wed, 5 May 2021 11:56:51 -0400 Subject: [PATCH 11/34] Support stable MSC1772 spaces identifiers. (#9915) Support both the unstable and stable identifiers. A future release will disable the unstable identifiers. --- changelog.d/9915.feature | 1 + synapse/api/constants.py | 3 +++ synapse/handlers/space_summary.py | 8 ++++++-- 3 files changed, 10 insertions(+), 2 deletions(-) create mode 100644 changelog.d/9915.feature diff --git a/changelog.d/9915.feature b/changelog.d/9915.feature new file mode 100644 index 0000000000..832916cb01 --- /dev/null +++ b/changelog.d/9915.feature @@ -0,0 +1 @@ +Support stable identifiers from [MSC1772](https://github.com/matrix-org/matrix-doc/pull/1772). diff --git a/synapse/api/constants.py b/synapse/api/constants.py index 936b6534b4..bff750e5fb 100644 --- a/synapse/api/constants.py +++ b/synapse/api/constants.py @@ -110,6 +110,8 @@ class EventTypes: Dummy = "org.matrix.dummy_event" + SpaceChild = "m.space.child" + SpaceParent = "m.space.parent" MSC1772_SPACE_CHILD = "org.matrix.msc1772.space.child" MSC1772_SPACE_PARENT = "org.matrix.msc1772.space.parent" @@ -174,6 +176,7 @@ class EventContentFields: SELF_DESTRUCT_AFTER = "org.matrix.self_destruct_after" # cf https://github.com/matrix-org/matrix-doc/pull/1772 + ROOM_TYPE = "m.type" MSC1772_ROOM_TYPE = "org.matrix.msc1772.type" diff --git a/synapse/handlers/space_summary.py b/synapse/handlers/space_summary.py index 01e3e050f9..d32452747c 100644 --- a/synapse/handlers/space_summary.py +++ b/synapse/handlers/space_summary.py @@ -288,6 +288,7 @@ class SpaceSummaryHandler: ev.data for ev in res.events if ev.event_type == EventTypes.MSC1772_SPACE_CHILD + or ev.event_type == EventTypes.SpaceChild ) async def _is_room_accessible(self, room_id: str, requester: Optional[str]) -> bool: @@ -331,7 +332,9 @@ class SpaceSummaryHandler: ) # TODO: update once MSC1772 lands - room_type = create_event.content.get(EventContentFields.MSC1772_ROOM_TYPE) + room_type = create_event.content.get(EventContentFields.ROOM_TYPE) + if not room_type: + room_type = create_event.content.get(EventContentFields.MSC1772_ROOM_TYPE) entry = { "room_id": stats["room_id"], @@ -360,8 +363,9 @@ class SpaceSummaryHandler: [ event_id for key, event_id in current_state_ids.items() - # TODO: update once MSC1772 lands + # TODO: update once MSC1772 has been FCP for a period of time. if key[0] == EventTypes.MSC1772_SPACE_CHILD + or key[0] == EventTypes.SpaceChild ] ) From 37623e33822d4e032ba6d1f523fb09b12fe27aab Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 5 May 2021 17:27:05 +0100 Subject: [PATCH 12/34] Increase perf of handling presence when joining large rooms. (#9916) --- changelog.d/9916.feature | 1 + synapse/handlers/presence.py | 148 +++++++++++++++++--------------- tests/handlers/test_presence.py | 14 +-- 3 files changed, 84 insertions(+), 79 deletions(-) create mode 100644 changelog.d/9916.feature diff --git a/changelog.d/9916.feature b/changelog.d/9916.feature new file mode 100644 index 0000000000..54165cce18 --- /dev/null +++ b/changelog.d/9916.feature @@ -0,0 +1 @@ +Improve performance after joining a large room when presence is enabled. diff --git a/synapse/handlers/presence.py b/synapse/handlers/presence.py index 8e085dfbec..6fd1f34289 100644 --- a/synapse/handlers/presence.py +++ b/synapse/handlers/presence.py @@ -1183,7 +1183,16 @@ class PresenceHandler(BasePresenceHandler): max_pos, deltas = await self.store.get_current_state_deltas( self._event_pos, room_max_stream_ordering ) - await self._handle_state_delta(deltas) + + # We may get multiple deltas for different rooms, but we want to + # handle them on a room by room basis, so we batch them up by + # room. + deltas_by_room: Dict[str, List[JsonDict]] = {} + for delta in deltas: + deltas_by_room.setdefault(delta["room_id"], []).append(delta) + + for room_id, deltas_for_room in deltas_by_room.items(): + await self._handle_state_delta(room_id, deltas_for_room) self._event_pos = max_pos @@ -1192,17 +1201,21 @@ class PresenceHandler(BasePresenceHandler): max_pos ) - async def _handle_state_delta(self, deltas: List[JsonDict]) -> None: - """Process current state deltas to find new joins that need to be - handled. + async def _handle_state_delta(self, room_id: str, deltas: List[JsonDict]) -> None: + """Process current state deltas for the room to find new joins that need + to be handled. """ - # A map of destination to a set of user state that they should receive - presence_destinations = {} # type: Dict[str, Set[UserPresenceState]] + + # Sets of newly joined users. Note that if the local server is + # joining a remote room for the first time we'll see both the joining + # user and all remote users as newly joined. + newly_joined_users = set() for delta in deltas: + assert room_id == delta["room_id"] + typ = delta["type"] state_key = delta["state_key"] - room_id = delta["room_id"] event_id = delta["event_id"] prev_event_id = delta["prev_event_id"] @@ -1231,72 +1244,55 @@ class PresenceHandler(BasePresenceHandler): # Ignore changes to join events. continue - # Retrieve any user presence state updates that need to be sent as a result, - # and the destinations that need to receive it - destinations, user_presence_states = await self._on_user_joined_room( - room_id, state_key - ) + newly_joined_users.add(state_key) - # Insert the destinations and respective updates into our destinations dict - for destination in destinations: - presence_destinations.setdefault(destination, set()).update( - user_presence_states - ) + if not newly_joined_users: + # If nobody has joined then there's nothing to do. + return - # Send out user presence updates for each destination - for destination, user_state_set in presence_destinations.items(): - self._federation_queue.send_presence_to_destinations( - destinations=[destination], states=user_state_set - ) + # We want to send: + # 1. presence states of all local users in the room to newly joined + # remote servers + # 2. presence states of newly joined users to all remote servers in + # the room. + # + # TODO: Only send presence states to remote hosts that don't already + # have them (because they already share rooms). - async def _on_user_joined_room( - self, room_id: str, user_id: str - ) -> Tuple[List[str], List[UserPresenceState]]: - """Called when we detect a user joining the room via the current state - delta stream. Returns the destinations that need to be updated and the - presence updates to send to them. + # Get all the users who were already in the room, by fetching the + # current users in the room and removing the newly joined users. + users = await self.store.get_users_in_room(room_id) + prev_users = set(users) - newly_joined_users - Args: - room_id: The ID of the room that the user has joined. - user_id: The ID of the user that has joined the room. + # Construct sets for all the local users and remote hosts that were + # already in the room + prev_local_users = [] + prev_remote_hosts = set() + for user_id in prev_users: + if self.is_mine_id(user_id): + prev_local_users.append(user_id) + else: + prev_remote_hosts.add(get_domain_from_id(user_id)) - Returns: - A tuple of destinations and presence updates to send to them. - """ - if self.is_mine_id(user_id): - # If this is a local user then we need to send their presence - # out to hosts in the room (who don't already have it) + # Similarly, construct sets for all the local users and remote hosts + # that were *not* already in the room. Care needs to be taken with the + # calculating the remote hosts, as a host may have already been in the + # room even if there is a newly joined user from that host. + newly_joined_local_users = [] + newly_joined_remote_hosts = set() + for user_id in newly_joined_users: + if self.is_mine_id(user_id): + newly_joined_local_users.append(user_id) + else: + host = get_domain_from_id(user_id) + if host not in prev_remote_hosts: + newly_joined_remote_hosts.add(host) - # TODO: We should be able to filter the hosts down to those that - # haven't previously seen the user - - remote_hosts = await self.state.get_current_hosts_in_room(room_id) - - # Filter out ourselves. - filtered_remote_hosts = [ - host for host in remote_hosts if host != self.server_name - ] - - state = await self.current_state_for_user(user_id) - return filtered_remote_hosts, [state] - else: - # A remote user has joined the room, so we need to: - # 1. Check if this is a new server in the room - # 2. If so send any presence they don't already have for - # local users in the room. - - # TODO: We should be able to filter the users down to those that - # the server hasn't previously seen - - # TODO: Check that this is actually a new server joining the - # room. - - remote_host = get_domain_from_id(user_id) - - users = await self.store.get_users_in_room(room_id) - user_ids = list(filter(self.is_mine_id, users)) - - states_d = await self.current_state_for_users(user_ids) + # Send presence states of all local users in the room to newly joined + # remote servers. (We actually only send states for local users already + # in the room, as we'll send states for newly joined local users below.) + if prev_local_users and newly_joined_remote_hosts: + local_states = await self.current_state_for_users(prev_local_users) # Filter out old presence, i.e. offline presence states where # the user hasn't been active for a week. We can change this @@ -1306,13 +1302,27 @@ class PresenceHandler(BasePresenceHandler): now = self.clock.time_msec() states = [ state - for state in states_d.values() + for state in local_states.values() if state.state != PresenceState.OFFLINE or now - state.last_active_ts < 7 * 24 * 60 * 60 * 1000 or state.status_msg is not None ] - return [remote_host], states + self._federation_queue.send_presence_to_destinations( + destinations=newly_joined_remote_hosts, + states=states, + ) + + # Send presence states of newly joined users to all remote servers in + # the room + if newly_joined_local_users and ( + prev_remote_hosts or newly_joined_remote_hosts + ): + local_states = await self.current_state_for_users(newly_joined_local_users) + self._federation_queue.send_presence_to_destinations( + destinations=prev_remote_hosts | newly_joined_remote_hosts, + states=list(local_states.values()), + ) def should_notify(old_state: UserPresenceState, new_state: UserPresenceState) -> bool: diff --git a/tests/handlers/test_presence.py b/tests/handlers/test_presence.py index ce330e79cc..1ffab709fc 100644 --- a/tests/handlers/test_presence.py +++ b/tests/handlers/test_presence.py @@ -729,7 +729,7 @@ class PresenceJoinTestCase(unittest.HomeserverTestCase): ) self.assertEqual(expected_state.state, PresenceState.ONLINE) self.federation_sender.send_presence_to_destinations.assert_called_once_with( - destinations=["server2"], states={expected_state} + destinations={"server2"}, states=[expected_state] ) # @@ -740,7 +740,7 @@ class PresenceJoinTestCase(unittest.HomeserverTestCase): self._add_new_user(room_id, "@bob:server3") self.federation_sender.send_presence_to_destinations.assert_called_once_with( - destinations=["server3"], states={expected_state} + destinations={"server3"}, states=[expected_state] ) def test_remote_gets_presence_when_local_user_joins(self): @@ -788,14 +788,8 @@ class PresenceJoinTestCase(unittest.HomeserverTestCase): self.presence_handler.current_state_for_user("@test2:server") ) self.assertEqual(expected_state.state, PresenceState.ONLINE) - self.assertEqual( - self.federation_sender.send_presence_to_destinations.call_count, 2 - ) - self.federation_sender.send_presence_to_destinations.assert_any_call( - destinations=["server3"], states={expected_state} - ) - self.federation_sender.send_presence_to_destinations.assert_any_call( - destinations=["server2"], states={expected_state} + self.federation_sender.send_presence_to_destinations.assert_called_once_with( + destinations={"server2", "server3"}, states=[expected_state] ) def _add_new_user(self, room_id, user_id): From d783880083733a694ed4c7b15ca53be00e06f8a7 Mon Sep 17 00:00:00 2001 From: Patrick Cloke Date: Wed, 5 May 2021 13:33:05 -0400 Subject: [PATCH 13/34] Include the time of the create event in Spaces Summary. (#9928) This is an update based on changes to MSC2946. The origin_server_ts of the m.room.create event is copied into the creation_ts field for each room returned from the spaces summary. --- changelog.d/9928.bugfix | 1 + synapse/handlers/space_summary.py | 1 + 2 files changed, 2 insertions(+) create mode 100644 changelog.d/9928.bugfix diff --git a/changelog.d/9928.bugfix b/changelog.d/9928.bugfix new file mode 100644 index 0000000000..7b74cd9fb6 --- /dev/null +++ b/changelog.d/9928.bugfix @@ -0,0 +1 @@ +Include the `origin_server_ts` property in the experimental [MSC2946](https://github.com/matrix-org/matrix-doc/pull/2946) support to allow clients to properly sort rooms. diff --git a/synapse/handlers/space_summary.py b/synapse/handlers/space_summary.py index d32452747c..2e997841f1 100644 --- a/synapse/handlers/space_summary.py +++ b/synapse/handlers/space_summary.py @@ -347,6 +347,7 @@ class SpaceSummaryHandler: stats["history_visibility"] == HistoryVisibility.WORLD_READABLE ), "guest_can_join": stats["guest_access"] == "can_join", + "creation_ts": create_event.origin_server_ts, "room_type": room_type, } From 70f0ffd2fcd815a065b4734ac606654a2e11dd28 Mon Sep 17 00:00:00 2001 From: Patrick Cloke Date: Wed, 5 May 2021 16:31:16 -0400 Subject: [PATCH 14/34] Follow-up to #9915 to correct the identifier for room types. --- synapse/api/constants.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/api/constants.py b/synapse/api/constants.py index bff750e5fb..ab628b2be7 100644 --- a/synapse/api/constants.py +++ b/synapse/api/constants.py @@ -176,7 +176,7 @@ class EventContentFields: SELF_DESTRUCT_AFTER = "org.matrix.self_destruct_after" # cf https://github.com/matrix-org/matrix-doc/pull/1772 - ROOM_TYPE = "m.type" + ROOM_TYPE = "type" MSC1772_ROOM_TYPE = "org.matrix.msc1772.type" From 24f07a83e604a7ee93214ef0bf7d7c56a14a534d Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 6 May 2021 14:06:06 +0100 Subject: [PATCH 15/34] Pin attrs to <21.1.0 (#9937) Fixes #9936 --- changelog.d/9937.bugfix | 1 + synapse/python_dependencies.py | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) create mode 100644 changelog.d/9937.bugfix diff --git a/changelog.d/9937.bugfix b/changelog.d/9937.bugfix new file mode 100644 index 0000000000..c1cc5ccb43 --- /dev/null +++ b/changelog.d/9937.bugfix @@ -0,0 +1 @@ +Fix bug where `/sync` would break if using the latest version of `attrs` dependency, by pinning to a previous version. diff --git a/synapse/python_dependencies.py b/synapse/python_dependencies.py index 2de946f464..7709361f16 100644 --- a/synapse/python_dependencies.py +++ b/synapse/python_dependencies.py @@ -78,7 +78,8 @@ REQUIREMENTS = [ # we use attr.validators.deep_iterable, which arrived in 19.1.0 (Note: # Fedora 31 only has 19.1, so if we want to upgrade we should wait until 33 # is out in November.) - "attrs>=19.1.0", + # Note: 21.1.0 broke `/sync`, see #9936 + "attrs>=19.1.0,<21.1.0", "netaddr>=0.7.18", "Jinja2>=2.9", "bleach>=1.4.3", From ac88aca7f7acc2ce909db230682f93bb4e2ff73b Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 6 May 2021 14:06:38 +0100 Subject: [PATCH 16/34] 1.33.1 --- CHANGES.md | 9 +++++++++ changelog.d/9937.bugfix | 1 - debian/changelog | 6 ++++++ synapse/__init__.py | 2 +- 4 files changed, 16 insertions(+), 2 deletions(-) delete mode 100644 changelog.d/9937.bugfix diff --git a/CHANGES.md b/CHANGES.md index 206859cff7..a41abbefba 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,3 +1,12 @@ +Synapse 1.33.1 (2021-05-06) +=========================== + +Bugfixes +-------- + +- Fix bug where `/sync` would break if using the latest version of `attrs` dependency, by pinning to a previous version. ([\#9937](https://github.com/matrix-org/synapse/issues/9937)) + + Synapse 1.33.0 (2021-05-05) =========================== diff --git a/changelog.d/9937.bugfix b/changelog.d/9937.bugfix deleted file mode 100644 index c1cc5ccb43..0000000000 --- a/changelog.d/9937.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix bug where `/sync` would break if using the latest version of `attrs` dependency, by pinning to a previous version. diff --git a/debian/changelog b/debian/changelog index b54d3f2bc6..de50dd14ea 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,9 @@ +matrix-synapse-py3 (1.33.1) stable; urgency=medium + + * New synapse release 1.33.1. + + -- Synapse Packaging team Thu, 06 May 2021 14:06:33 +0100 + matrix-synapse-py3 (1.33.0) stable; urgency=medium * New synapse release 1.33.0. diff --git a/synapse/__init__.py b/synapse/__init__.py index 5eac40730a..441cd8b339 100644 --- a/synapse/__init__.py +++ b/synapse/__init__.py @@ -47,7 +47,7 @@ try: except ImportError: pass -__version__ = "1.33.0" +__version__ = "1.33.1" if bool(os.environ.get("SYNAPSE_TEST_PATCH_LOG_CONTEXTS", False)): # We import here so that we don't have to install a bunch of deps when From eba431c539dbe0ca28794d89962d447d1f75938f Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 6 May 2021 15:06:35 +0100 Subject: [PATCH 17/34] Revert "Leave out optional keys from /sync (#9919)" (#9940) This reverts commit e9eb3549d32a6f93d07de8dbd5e1ebe54c8d8278. --- changelog.d/9919.feature | 1 - synapse/rest/client/v2_alpha/sync.py | 62 ++++++------------- tests/rest/client/v2_alpha/test_sync.py | 30 ++++++++- .../test_resource_limits_server_notices.py | 8 +-- 4 files changed, 50 insertions(+), 51 deletions(-) delete mode 100644 changelog.d/9919.feature diff --git a/changelog.d/9919.feature b/changelog.d/9919.feature deleted file mode 100644 index 07747505d2..0000000000 --- a/changelog.d/9919.feature +++ /dev/null @@ -1 +0,0 @@ -Omit empty fields from the `/sync` response. Contributed by @deepbluev7. diff --git a/synapse/rest/client/v2_alpha/sync.py b/synapse/rest/client/v2_alpha/sync.py index 5f85653330..95ee3f1b84 100644 --- a/synapse/rest/client/v2_alpha/sync.py +++ b/synapse/rest/client/v2_alpha/sync.py @@ -14,7 +14,6 @@ import itertools import logging -from collections import defaultdict from typing import TYPE_CHECKING, Tuple from synapse.api.constants import PresenceState @@ -230,49 +229,24 @@ class SyncRestServlet(RestServlet): ) logger.debug("building sync response dict") - - response: dict = defaultdict(dict) - response["next_batch"] = await sync_result.next_batch.to_string(self.store) - - if sync_result.account_data: - response["account_data"] = {"events": sync_result.account_data} - if sync_result.presence: - response["presence"] = SyncRestServlet.encode_presence( - sync_result.presence, time_now - ) - - if sync_result.to_device: - response["to_device"] = {"events": sync_result.to_device} - - if sync_result.device_lists.changed: - response["device_lists"]["changed"] = list(sync_result.device_lists.changed) - if sync_result.device_lists.left: - response["device_lists"]["left"] = list(sync_result.device_lists.left) - - if sync_result.device_one_time_keys_count: - response[ - "device_one_time_keys_count" - ] = sync_result.device_one_time_keys_count - if sync_result.device_unused_fallback_key_types: - response[ - "org.matrix.msc2732.device_unused_fallback_key_types" - ] = sync_result.device_unused_fallback_key_types - - if joined: - response["rooms"]["join"] = joined - if invited: - response["rooms"]["invite"] = invited - if archived: - response["rooms"]["leave"] = archived - - if sync_result.groups.join: - response["groups"]["join"] = sync_result.groups.join - if sync_result.groups.invite: - response["groups"]["invite"] = sync_result.groups.invite - if sync_result.groups.leave: - response["groups"]["leave"] = sync_result.groups.leave - - return response + return { + "account_data": {"events": sync_result.account_data}, + "to_device": {"events": sync_result.to_device}, + "device_lists": { + "changed": list(sync_result.device_lists.changed), + "left": list(sync_result.device_lists.left), + }, + "presence": SyncRestServlet.encode_presence(sync_result.presence, time_now), + "rooms": {"join": joined, "invite": invited, "leave": archived}, + "groups": { + "join": sync_result.groups.join, + "invite": sync_result.groups.invite, + "leave": sync_result.groups.leave, + }, + "device_one_time_keys_count": sync_result.device_one_time_keys_count, + "org.matrix.msc2732.device_unused_fallback_key_types": sync_result.device_unused_fallback_key_types, + "next_batch": await sync_result.next_batch.to_string(self.store), + } @staticmethod def encode_presence(events, time_now): diff --git a/tests/rest/client/v2_alpha/test_sync.py b/tests/rest/client/v2_alpha/test_sync.py index 74be5176d0..dbcbdf159a 100644 --- a/tests/rest/client/v2_alpha/test_sync.py +++ b/tests/rest/client/v2_alpha/test_sync.py @@ -37,7 +37,35 @@ class FilterTestCase(unittest.HomeserverTestCase): channel = self.make_request("GET", "/sync") self.assertEqual(channel.code, 200) - self.assertIn("next_batch", channel.json_body) + self.assertTrue( + { + "next_batch", + "rooms", + "presence", + "account_data", + "to_device", + "device_lists", + }.issubset(set(channel.json_body.keys())) + ) + + def test_sync_presence_disabled(self): + """ + When presence is disabled, the key does not appear in /sync. + """ + self.hs.config.use_presence = False + + channel = self.make_request("GET", "/sync") + + self.assertEqual(channel.code, 200) + self.assertTrue( + { + "next_batch", + "rooms", + "account_data", + "to_device", + "device_lists", + }.issubset(set(channel.json_body.keys())) + ) class SyncFilterTestCase(unittest.HomeserverTestCase): diff --git a/tests/server_notices/test_resource_limits_server_notices.py b/tests/server_notices/test_resource_limits_server_notices.py index 3245aa91ca..d46521ccdc 100644 --- a/tests/server_notices/test_resource_limits_server_notices.py +++ b/tests/server_notices/test_resource_limits_server_notices.py @@ -306,9 +306,8 @@ class TestResourceLimitsServerNoticesWithRealRooms(unittest.HomeserverTestCase): channel = self.make_request("GET", "/sync?timeout=0", access_token=tok) - self.assertNotIn( - "rooms", channel.json_body, "Got invites without server notice" - ) + invites = channel.json_body["rooms"]["invite"] + self.assertEqual(len(invites), 0, invites) def test_invite_with_notice(self): """Tests that, if the MAU limit is hit, the server notices user invites each user @@ -365,8 +364,7 @@ class TestResourceLimitsServerNoticesWithRealRooms(unittest.HomeserverTestCase): # We could also pick another user and sync with it, which would return an # invite to a system notices room, but it doesn't matter which user we're # using so we use the last one because it saves us an extra sync. - if "rooms" in channel.json_body: - invites = channel.json_body["rooms"]["invite"] + invites = channel.json_body["rooms"]["invite"] # Make sure we have an invite to process. self.assertEqual(len(invites), 1, invites) From 8771b1337da9faa3b60cf0ec0a128a7de856f19e Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 6 May 2021 15:54:07 +0100 Subject: [PATCH 18/34] Export jemalloc stats to prometheus when used (#9882) --- changelog.d/9882.misc | 1 + synapse/app/_base.py | 2 + synapse/metrics/__init__.py | 1 + synapse/metrics/jemalloc.py | 196 ++++++++++++++++++++++++++++++++++++ 4 files changed, 200 insertions(+) create mode 100644 changelog.d/9882.misc create mode 100644 synapse/metrics/jemalloc.py diff --git a/changelog.d/9882.misc b/changelog.d/9882.misc new file mode 100644 index 0000000000..facfa31f38 --- /dev/null +++ b/changelog.d/9882.misc @@ -0,0 +1 @@ +Export jemalloc stats to Prometheus if it is being used. diff --git a/synapse/app/_base.py b/synapse/app/_base.py index 638e01c1b2..59918d789e 100644 --- a/synapse/app/_base.py +++ b/synapse/app/_base.py @@ -37,6 +37,7 @@ from synapse.config.homeserver import HomeServerConfig from synapse.crypto import context_factory from synapse.logging.context import PreserveLoggingContext from synapse.metrics.background_process_metrics import wrap_as_background_process +from synapse.metrics.jemalloc import setup_jemalloc_stats from synapse.util.async_helpers import Linearizer from synapse.util.daemonize import daemonize_process from synapse.util.rlimit import change_resource_limit @@ -115,6 +116,7 @@ def start_reactor( def run(): logger.info("Running") + setup_jemalloc_stats() change_resource_limit(soft_file_limit) if gc_thresholds: gc.set_threshold(*gc_thresholds) diff --git a/synapse/metrics/__init__.py b/synapse/metrics/__init__.py index e671da26d5..fef2846669 100644 --- a/synapse/metrics/__init__.py +++ b/synapse/metrics/__init__.py @@ -629,6 +629,7 @@ try: except AttributeError: pass + __all__ = [ "MetricsResource", "generate_latest", diff --git a/synapse/metrics/jemalloc.py b/synapse/metrics/jemalloc.py new file mode 100644 index 0000000000..29ab6c0229 --- /dev/null +++ b/synapse/metrics/jemalloc.py @@ -0,0 +1,196 @@ +# Copyright 2021 The Matrix.org Foundation C.I.C. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import ctypes +import logging +import os +import re +from typing import Optional + +from synapse.metrics import REGISTRY, GaugeMetricFamily + +logger = logging.getLogger(__name__) + + +def _setup_jemalloc_stats(): + """Checks to see if jemalloc is loaded, and hooks up a collector to record + statistics exposed by jemalloc. + """ + + # Try to find the loaded jemalloc shared library, if any. We need to + # introspect into what is loaded, rather than loading whatever is on the + # path, as if we load a *different* jemalloc version things will seg fault. + + # We look in `/proc/self/maps`, which only exists on linux. + if not os.path.exists("/proc/self/maps"): + logger.debug("Not looking for jemalloc as no /proc/self/maps exist") + return + + # We're looking for a path at the end of the line that includes + # "libjemalloc". + regex = re.compile(r"/\S+/libjemalloc.*$") + + jemalloc_path = None + with open("/proc/self/maps") as f: + for line in f: + match = regex.search(line.strip()) + if match: + jemalloc_path = match.group() + + if not jemalloc_path: + # No loaded jemalloc was found. + logger.debug("jemalloc not found") + return + + logger.debug("Found jemalloc at %s", jemalloc_path) + + jemalloc = ctypes.CDLL(jemalloc_path) + + def _mallctl( + name: str, read: bool = True, write: Optional[int] = None + ) -> Optional[int]: + """Wrapper around `mallctl` for reading and writing integers to + jemalloc. + + Args: + name: The name of the option to read from/write to. + read: Whether to try and read the value. + write: The value to write, if given. + + Returns: + The value read if `read` is True, otherwise None. + + Raises: + An exception if `mallctl` returns a non-zero error code. + """ + + input_var = None + input_var_ref = None + input_len_ref = None + if read: + input_var = ctypes.c_size_t(0) + input_len = ctypes.c_size_t(ctypes.sizeof(input_var)) + + input_var_ref = ctypes.byref(input_var) + input_len_ref = ctypes.byref(input_len) + + write_var_ref = None + write_len = ctypes.c_size_t(0) + if write is not None: + write_var = ctypes.c_size_t(write) + write_len = ctypes.c_size_t(ctypes.sizeof(write_var)) + + write_var_ref = ctypes.byref(write_var) + + # The interface is: + # + # int mallctl( + # const char *name, + # void *oldp, + # size_t *oldlenp, + # void *newp, + # size_t newlen + # ) + # + # Where oldp/oldlenp is a buffer where the old value will be written to + # (if not null), and newp/newlen is the buffer with the new value to set + # (if not null). Note that they're all references *except* newlen. + result = jemalloc.mallctl( + name.encode("ascii"), + input_var_ref, + input_len_ref, + write_var_ref, + write_len, + ) + + if result != 0: + raise Exception("Failed to call mallctl") + + if input_var is None: + return None + + return input_var.value + + def _jemalloc_refresh_stats() -> None: + """Request that jemalloc updates its internal statistics. This needs to + be called before querying for stats, otherwise it will return stale + values. + """ + try: + _mallctl("epoch", read=False, write=1) + except Exception as e: + logger.warning("Failed to reload jemalloc stats: %s", e) + + class JemallocCollector: + """Metrics for internal jemalloc stats.""" + + def collect(self): + _jemalloc_refresh_stats() + + g = GaugeMetricFamily( + "jemalloc_stats_app_memory_bytes", + "The stats reported by jemalloc", + labels=["type"], + ) + + # Read the relevant global stats from jemalloc. Note that these may + # not be accurate if python is configured to use its internal small + # object allocator (which is on by default, disable by setting the + # env `PYTHONMALLOC=malloc`). + # + # See the jemalloc manpage for details about what each value means, + # roughly: + # - allocated ─ Total number of bytes allocated by the app + # - active ─ Total number of bytes in active pages allocated by + # the application, this is bigger than `allocated`. + # - resident ─ Maximum number of bytes in physically resident data + # pages mapped by the allocator, comprising all pages dedicated + # to allocator metadata, pages backing active allocations, and + # unused dirty pages. This is bigger than `active`. + # - mapped ─ Total number of bytes in active extents mapped by the + # allocator. + # - metadata ─ Total number of bytes dedicated to jemalloc + # metadata. + for t in ( + "allocated", + "active", + "resident", + "mapped", + "metadata", + ): + try: + value = _mallctl(f"stats.{t}") + except Exception as e: + # There was an error fetching the value, skip. + logger.warning("Failed to read jemalloc stats.%s: %s", t, e) + continue + + g.add_metric([t], value=value) + + yield g + + REGISTRY.register(JemallocCollector()) + + logger.debug("Added jemalloc stats") + + +def setup_jemalloc_stats(): + """Try to setup jemalloc stats, if jemalloc is loaded.""" + + try: + _setup_jemalloc_stats() + except Exception as e: + # This should only happen if we find the loaded jemalloc library, but + # fail to load it somehow (e.g. we somehow picked the wrong version). + logger.info("Failed to setup collector to record jemalloc stats: %s", e) From 25f43faa70f7cc58493b636c2702ae63395779dc Mon Sep 17 00:00:00 2001 From: Richard van der Hoff <1389908+richvdh@users.noreply.github.com> Date: Fri, 7 May 2021 10:22:05 +0100 Subject: [PATCH 19/34] Reorganise the database schema directories (#9932) The hope here is that by moving all the schema files into synapse/storage/schema, it gets a bit easier for newcomers to navigate. It certainly got easier for me to write a helpful README. There's more to do on that front, but I'll follow up with other PRs for that. --- changelog.d/9932.misc | 1 + .../main/schema/full_schemas/README.md | 21 -------- synapse/storage/prepare_database.py | 48 ++++++++++--------- synapse/storage/schema/README.md | 37 ++++++++++++++ synapse/storage/schema/__init__.py | 17 +++++++ .../delta/25/00background_updates.sql | 0 .../delta/35/00background_updates_add_col.sql | 0 .../delta/58/00background_update_ordering.sql | 0 .../{ => common}/full_schemas/54/full.sql | 0 .../schema/{ => common}/schema_version.sql | 0 .../schema => schema/main}/delta/12/v12.sql | 0 .../schema => schema/main}/delta/13/v13.sql | 0 .../schema => schema/main}/delta/14/v14.sql | 0 .../main}/delta/15/appservice_txns.sql | 0 .../main}/delta/15/presence_indices.sql | 0 .../schema => schema/main}/delta/15/v15.sql | 0 .../main}/delta/16/events_order_index.sql | 0 .../delta/16/remote_media_cache_index.sql | 0 .../main}/delta/16/remove_duplicates.sql | 0 .../main}/delta/16/room_alias_index.sql | 0 .../main}/delta/16/unique_constraints.sql | 0 .../schema => schema/main}/delta/16/users.sql | 0 .../main}/delta/17/drop_indexes.sql | 0 .../main}/delta/17/server_keys.sql | 0 .../main}/delta/17/user_threepids.sql | 0 .../delta/18/server_keys_bigger_ints.sql | 0 .../main}/delta/19/event_index.sql | 0 .../schema => schema/main}/delta/20/dummy.sql | 0 .../main}/delta/20/pushers.py | 0 .../main}/delta/21/end_to_end_keys.sql | 0 .../main}/delta/21/receipts.sql | 0 .../main}/delta/22/receipts_index.sql | 0 .../main}/delta/22/user_threepids_unique.sql | 0 .../main}/delta/24/stats_reporting.sql | 0 .../schema => schema/main}/delta/25/fts.py | 0 .../main}/delta/25/guest_access.sql | 0 .../main}/delta/25/history_visibility.sql | 0 .../schema => schema/main}/delta/25/tags.sql | 0 .../main}/delta/26/account_data.sql | 0 .../main}/delta/27/account_data.sql | 0 .../main}/delta/27/forgotten_memberships.sql | 0 .../schema => schema/main}/delta/27/ts.py | 0 .../main}/delta/28/event_push_actions.sql | 0 .../main}/delta/28/events_room_stream.sql | 0 .../main}/delta/28/public_roms_index.sql | 0 .../main}/delta/28/receipts_user_id_index.sql | 0 .../main}/delta/28/upgrade_times.sql | 0 .../main}/delta/28/users_is_guest.sql | 0 .../main}/delta/29/push_actions.sql | 0 .../main}/delta/30/alias_creator.sql | 0 .../main}/delta/30/as_users.py | 0 .../main}/delta/30/deleted_pushers.sql | 0 .../main}/delta/30/presence_stream.sql | 0 .../main}/delta/30/public_rooms.sql | 0 .../main}/delta/30/push_rule_stream.sql | 0 .../delta/30/threepid_guest_access_tokens.sql | 0 .../main}/delta/31/invites.sql | 0 .../31/local_media_repository_url_cache.sql | 0 .../main}/delta/31/pushers.py | 0 .../main}/delta/31/pushers_index.sql | 0 .../main}/delta/31/search_update.py | 0 .../main}/delta/32/events.sql | 0 .../main}/delta/32/openid.sql | 0 .../main}/delta/32/pusher_throttle.sql | 0 .../main}/delta/32/remove_indices.sql | 0 .../main}/delta/32/reports.sql | 0 .../delta/33/access_tokens_device_index.sql | 0 .../main}/delta/33/devices.sql | 0 .../main}/delta/33/devices_for_e2e_keys.sql | 0 ...ices_for_e2e_keys_clear_unknown_device.sql | 0 .../main}/delta/33/event_fields.py | 0 .../main}/delta/33/remote_media_ts.py | 0 .../main}/delta/33/user_ips_index.sql | 0 .../main}/delta/34/appservice_stream.sql | 0 .../main}/delta/34/cache_stream.py | 0 .../main}/delta/34/device_inbox.sql | 0 .../delta/34/push_display_name_rename.sql | 0 .../main}/delta/34/received_txn_purge.py | 0 .../main}/delta/35/contains_url.sql | 0 .../main}/delta/35/device_outbox.sql | 0 .../main}/delta/35/device_stream_id.sql | 0 .../delta/35/event_push_actions_index.sql | 0 .../35/public_room_list_change_stream.sql | 0 .../main}/delta/35/stream_order_to_extrem.sql | 0 .../main}/delta/36/readd_public_rooms.sql | 0 .../main}/delta/37/remove_auth_idx.py | 0 .../main}/delta/37/user_threepids.sql | 0 .../main}/delta/38/postgres_fts_gist.sql | 0 .../main}/delta/39/appservice_room_list.sql | 0 .../delta/39/device_federation_stream_idx.sql | 0 .../main}/delta/39/event_push_index.sql | 0 .../delta/39/federation_out_position.sql | 0 .../main}/delta/39/membership_profile.sql | 0 .../main}/delta/40/current_state_idx.sql | 0 .../main}/delta/40/device_inbox.sql | 0 .../main}/delta/40/device_list_streams.sql | 0 .../main}/delta/40/event_push_summary.sql | 0 .../main}/delta/40/pushers.sql | 0 .../main}/delta/41/device_list_stream_idx.sql | 0 .../main}/delta/41/device_outbound_index.sql | 0 .../delta/41/event_search_event_id_idx.sql | 0 .../main}/delta/41/ratelimit.sql | 0 .../main}/delta/42/current_state_delta.sql | 0 .../main}/delta/42/device_list_last_id.sql | 0 .../main}/delta/42/event_auth_state_only.sql | 0 .../main}/delta/42/user_dir.py | 0 .../main}/delta/43/blocked_rooms.sql | 0 .../main}/delta/43/quarantine_media.sql | 0 .../main}/delta/43/url_cache.sql | 0 .../main}/delta/43/user_share.sql | 0 .../main}/delta/44/expire_url_cache.sql | 0 .../main}/delta/45/group_server.sql | 0 .../main}/delta/45/profile_cache.sql | 0 .../main}/delta/46/drop_refresh_tokens.sql | 0 .../delta/46/drop_unique_deleted_pushers.sql | 0 .../main}/delta/46/group_server.sql | 0 .../46/local_media_repository_url_idx.sql | 0 .../main}/delta/46/user_dir_null_room_ids.sql | 0 .../main}/delta/46/user_dir_typos.sql | 0 .../main}/delta/47/last_access_media.sql | 0 .../main}/delta/47/postgres_fts_gin.sql | 0 .../main}/delta/47/push_actions_staging.sql | 0 .../main}/delta/48/add_user_consent.sql | 0 .../delta/48/add_user_ips_last_seen_index.sql | 0 .../main}/delta/48/deactivated_users.sql | 0 .../main}/delta/48/group_unique_indexes.py | 0 .../main}/delta/48/groups_joinable.sql | 0 .../add_user_consent_server_notice_sent.sql | 0 .../main}/delta/49/add_user_daily_visits.sql | 0 .../49/add_user_ips_last_seen_only_index.sql | 0 .../delta/50/add_creation_ts_users_index.sql | 0 .../main}/delta/50/erasure_store.sql | 0 .../delta/50/make_event_content_nullable.py | 0 .../main}/delta/51/e2e_room_keys.sql | 0 .../main}/delta/51/monthly_active_users.sql | 0 .../52/add_event_to_state_group_index.sql | 0 .../52/device_list_streams_unique_idx.sql | 0 .../main}/delta/52/e2e_room_keys.sql | 0 .../main}/delta/53/add_user_type_to_users.sql | 0 .../main}/delta/53/drop_sent_transactions.sql | 0 .../main}/delta/53/event_format_version.sql | 0 .../main}/delta/53/user_dir_populate.sql | 0 .../main}/delta/53/user_ips_index.sql | 0 .../main}/delta/53/user_share.sql | 0 .../main}/delta/53/user_threepid_id.sql | 0 .../main}/delta/53/users_in_public_rooms.sql | 0 .../54/account_validity_with_renewal.sql | 0 .../delta/54/add_validity_to_server_keys.sql | 0 .../delta/54/delete_forward_extremities.sql | 0 .../main}/delta/54/drop_legacy_tables.sql | 0 .../main}/delta/54/drop_presence_list.sql | 0 .../main}/delta/54/relations.sql | 0 .../schema => schema/main}/delta/54/stats.sql | 0 .../main}/delta/54/stats2.sql | 0 .../main}/delta/55/access_token_expiry.sql | 0 .../delta/55/track_threepid_validations.sql | 0 .../delta/55/users_alter_deactivated.sql | 0 .../delta/56/add_spans_to_device_lists.sql | 0 .../56/current_state_events_membership.sql | 0 .../current_state_events_membership_mk2.sql | 0 .../56/delete_keys_from_deleted_backups.sql | 0 .../delta/56/destinations_failure_ts.sql | 0 ...tinations_retry_interval_type.sql.postgres | 0 .../delta/56/device_stream_id_insert.sql | 0 .../main}/delta/56/devices_last_seen.sql | 0 .../delta/56/drop_unused_event_tables.sql | 0 .../main}/delta/56/event_expiry.sql | 0 .../main}/delta/56/event_labels.sql | 0 .../56/event_labels_background_update.sql | 0 .../main}/delta/56/fix_room_keys_index.sql | 0 .../main}/delta/56/hidden_devices.sql | 0 .../delta/56/hidden_devices_fix.sql.sqlite | 0 .../56/nuke_empty_communities_from_db.sql | 0 .../main}/delta/56/public_room_list_idx.sql | 0 .../main}/delta/56/redaction_censor.sql | 0 .../main}/delta/56/redaction_censor2.sql | 0 .../redaction_censor3_fix_update.sql.postgres | 0 .../main}/delta/56/redaction_censor4.sql | 0 ...remove_tombstoned_rooms_from_directory.sql | 0 .../main}/delta/56/room_key_etag.sql | 0 .../main}/delta/56/room_membership_idx.sql | 0 .../main}/delta/56/room_retention.sql | 0 .../main}/delta/56/signing_keys.sql | 0 .../56/signing_keys_nonunique_signatures.sql | 0 .../main}/delta/56/stats_separated.sql | 0 .../delta/56/unique_user_filter_index.py | 0 .../main}/delta/56/user_external_ids.sql | 0 .../delta/56/users_in_public_rooms_idx.sql | 0 .../57/delete_old_current_state_events.sql | 0 .../57/device_list_remote_cache_stale.sql | 0 .../delta/57/local_current_membership.py | 0 .../delta/57/remove_sent_outbound_pokes.sql | 0 .../main}/delta/57/rooms_version_column.sql | 0 .../57/rooms_version_column_2.sql.postgres | 0 .../57/rooms_version_column_2.sql.sqlite | 0 .../57/rooms_version_column_3.sql.postgres | 0 .../57/rooms_version_column_3.sql.sqlite | 0 .../delta/58/02remove_dup_outbound_pokes.sql | 0 .../main}/delta/58/03persist_ui_auth.sql | 0 .../delta/58/05cache_instance.sql.postgres | 0 .../main}/delta/58/06dlols_unique_idx.py | 0 ...ethod_to_thumbnail_constraint.sql.postgres | 0 ..._method_to_thumbnail_constraint.sql.sqlite | 0 .../main}/delta/58/07persist_ui_auth_ips.sql | 0 ...08_media_safe_from_quarantine.sql.postgres | 0 .../08_media_safe_from_quarantine.sql.sqlite | 0 .../main}/delta/58/09shadow_ban.sql | 0 .../10_pushrules_enabled_delete_obsolete.sql | 0 .../58/10drop_local_rejections_stream.sql | 0 .../58/10federation_pos_instance_name.sql | 0 .../main}/delta/58/11dehydration.sql | 0 .../main}/delta/58/11fallback.sql | 0 .../main}/delta/58/11user_id_seq.py | 0 .../main}/delta/58/12room_stats.sql | 0 .../58/13remove_presence_allow_inbound.sql | 0 .../main}/delta/58/14events_instance_name.sql | 0 .../58/14events_instance_name.sql.postgres | 0 .../delta/58/15_catchup_destination_rooms.sql | 0 .../main}/delta/58/15unread_count.sql | 0 .../58/16populate_stats_process_rooms_fix.sql | 0 .../delta/58/17_catchup_last_successful.sql | 0 .../main}/delta/58/18stream_positions.sql | 0 .../delta/58/19instance_map.sql.postgres | 0 .../main}/delta/58/19txn_id.sql | 0 .../delta/58/20instance_name_event_tables.sql | 0 .../main}/delta/58/20user_daily_visits.sql | 0 .../main}/delta/58/21as_device_stream.sql | 0 .../delta/58/21drop_device_max_stream_id.sql | 0 .../main}/delta/58/22puppet_token.sql | 0 .../delta/58/22users_have_local_media.sql | 0 .../delta/58/23e2e_cross_signing_keys_idx.sql | 0 .../delta/58/24drop_event_json_index.sql | 0 .../58/25user_external_ids_user_id_idx.sql | 0 .../58/26access_token_last_validated.sql | 0 .../main}/delta/58/27local_invites.sql | 0 .../58/28drop_last_used_column.sql.postgres | 0 .../58/28drop_last_used_column.sql.sqlite | 0 .../main}/delta/59/01ignored_user.py | 0 .../main}/delta/59/02shard_send_to_device.sql | 0 ...shard_send_to_device_sequence.sql.postgres | 0 .../main}/delta/59/04_event_auth_chains.sql | 0 .../59/04_event_auth_chains.sql.postgres | 0 .../main}/delta/59/04drop_account_data.sql | 0 .../main}/delta/59/05cache_invalidation.sql | 0 .../main}/delta/59/06chain_cover_index.sql | 0 .../main}/delta/59/06shard_account_data.sql | 0 .../59/06shard_account_data.sql.postgres | 0 .../delta/59/07shard_account_data_fix.sql | 0 ...elete_pushers_for_deactivated_accounts.sql | 0 .../main}/delta/59/08delete_stale_pushers.sql | 0 .../delta/59/09rejected_events_metadata.sql | 0 .../delta/59/10delete_purged_chain_cover.sql | 0 .../11drop_thumbnail_constraint.sql.postgres | 0 .../12account_validity_token_used_ts_ms.sql | 0 .../delta/59/12presence_stream_instance.sql | 0 ...2presence_stream_instance_seq.sql.postgres | 0 .../full_schemas/16/application_services.sql | 0 .../main}/full_schemas/16/event_edges.sql | 0 .../full_schemas/16/event_signatures.sql | 0 .../main}/full_schemas/16/im.sql | 0 .../main}/full_schemas/16/keys.sql | 0 .../full_schemas/16/media_repository.sql | 0 .../main}/full_schemas/16/presence.sql | 0 .../main}/full_schemas/16/profiles.sql | 0 .../main}/full_schemas/16/push.sql | 0 .../main}/full_schemas/16/redactions.sql | 0 .../main}/full_schemas/16/room_aliases.sql | 0 .../main}/full_schemas/16/state.sql | 0 .../main}/full_schemas/16/transactions.sql | 0 .../main}/full_schemas/16/users.sql | 0 .../main}/full_schemas/54/full.sql.postgres | 0 .../main}/full_schemas/54/full.sql.sqlite | 0 .../full_schemas/54/stream_positions.sql | 0 .../state}/delta/23/drop_state_index.sql | 0 .../state}/delta/30/state_stream.sql | 0 .../state}/delta/32/remove_state_indices.sql | 0 .../state}/delta/35/add_state_index.sql | 0 .../state}/delta/35/state.sql | 0 .../state}/delta/35/state_dedupe.sql | 0 .../state}/delta/47/state_group_seq.py | 0 .../state}/delta/56/state_group_room_idx.sql | 0 .../state}/full_schemas/54/full.sql | 0 .../full_schemas/54/sequence.sql.postgres | 0 tests/storage/test_cleanup_extrems.py | 4 +- 284 files changed, 81 insertions(+), 47 deletions(-) create mode 100644 changelog.d/9932.misc delete mode 100644 synapse/storage/databases/main/schema/full_schemas/README.md create mode 100644 synapse/storage/schema/README.md create mode 100644 synapse/storage/schema/__init__.py rename synapse/storage/schema/{ => common}/delta/25/00background_updates.sql (100%) rename synapse/storage/schema/{ => common}/delta/35/00background_updates_add_col.sql (100%) rename synapse/storage/schema/{ => common}/delta/58/00background_update_ordering.sql (100%) rename synapse/storage/schema/{ => common}/full_schemas/54/full.sql (100%) rename synapse/storage/schema/{ => common}/schema_version.sql (100%) rename synapse/storage/{databases/main/schema => schema/main}/delta/12/v12.sql (100%) rename synapse/storage/{databases/main/schema => schema/main}/delta/13/v13.sql (100%) rename synapse/storage/{databases/main/schema => schema/main}/delta/14/v14.sql (100%) rename synapse/storage/{databases/main/schema => schema/main}/delta/15/appservice_txns.sql (100%) rename synapse/storage/{databases/main/schema => schema/main}/delta/15/presence_indices.sql (100%) rename synapse/storage/{databases/main/schema => schema/main}/delta/15/v15.sql (100%) rename synapse/storage/{databases/main/schema => schema/main}/delta/16/events_order_index.sql (100%) rename synapse/storage/{databases/main/schema => schema/main}/delta/16/remote_media_cache_index.sql (100%) rename synapse/storage/{databases/main/schema => schema/main}/delta/16/remove_duplicates.sql (100%) rename synapse/storage/{databases/main/schema => schema/main}/delta/16/room_alias_index.sql (100%) rename synapse/storage/{databases/main/schema => schema/main}/delta/16/unique_constraints.sql (100%) rename synapse/storage/{databases/main/schema => schema/main}/delta/16/users.sql (100%) rename synapse/storage/{databases/main/schema => schema/main}/delta/17/drop_indexes.sql (100%) rename synapse/storage/{databases/main/schema => schema/main}/delta/17/server_keys.sql (100%) rename synapse/storage/{databases/main/schema => schema/main}/delta/17/user_threepids.sql (100%) rename synapse/storage/{databases/main/schema => schema/main}/delta/18/server_keys_bigger_ints.sql (100%) rename synapse/storage/{databases/main/schema => schema/main}/delta/19/event_index.sql (100%) rename synapse/storage/{databases/main/schema => schema/main}/delta/20/dummy.sql (100%) rename synapse/storage/{databases/main/schema => schema/main}/delta/20/pushers.py (100%) rename synapse/storage/{databases/main/schema => schema/main}/delta/21/end_to_end_keys.sql (100%) rename synapse/storage/{databases/main/schema => schema/main}/delta/21/receipts.sql (100%) rename synapse/storage/{databases/main/schema => schema/main}/delta/22/receipts_index.sql (100%) rename synapse/storage/{databases/main/schema => schema/main}/delta/22/user_threepids_unique.sql (100%) rename synapse/storage/{databases/main/schema => schema/main}/delta/24/stats_reporting.sql (100%) rename synapse/storage/{databases/main/schema => schema/main}/delta/25/fts.py (100%) rename synapse/storage/{databases/main/schema => schema/main}/delta/25/guest_access.sql (100%) rename synapse/storage/{databases/main/schema => schema/main}/delta/25/history_visibility.sql (100%) rename synapse/storage/{databases/main/schema => schema/main}/delta/25/tags.sql (100%) rename synapse/storage/{databases/main/schema => schema/main}/delta/26/account_data.sql (100%) rename synapse/storage/{databases/main/schema => schema/main}/delta/27/account_data.sql (100%) rename synapse/storage/{databases/main/schema => schema/main}/delta/27/forgotten_memberships.sql (100%) rename synapse/storage/{databases/main/schema => schema/main}/delta/27/ts.py (100%) rename synapse/storage/{databases/main/schema => schema/main}/delta/28/event_push_actions.sql (100%) rename synapse/storage/{databases/main/schema => schema/main}/delta/28/events_room_stream.sql (100%) rename synapse/storage/{databases/main/schema => schema/main}/delta/28/public_roms_index.sql (100%) rename synapse/storage/{databases/main/schema => schema/main}/delta/28/receipts_user_id_index.sql (100%) rename synapse/storage/{databases/main/schema => schema/main}/delta/28/upgrade_times.sql (100%) rename synapse/storage/{databases/main/schema => schema/main}/delta/28/users_is_guest.sql (100%) rename synapse/storage/{databases/main/schema => schema/main}/delta/29/push_actions.sql (100%) rename synapse/storage/{databases/main/schema => schema/main}/delta/30/alias_creator.sql (100%) rename synapse/storage/{databases/main/schema => schema/main}/delta/30/as_users.py (100%) rename synapse/storage/{databases/main/schema => schema/main}/delta/30/deleted_pushers.sql (100%) rename synapse/storage/{databases/main/schema => schema/main}/delta/30/presence_stream.sql (100%) rename synapse/storage/{databases/main/schema => schema/main}/delta/30/public_rooms.sql (100%) rename synapse/storage/{databases/main/schema => schema/main}/delta/30/push_rule_stream.sql (100%) rename synapse/storage/{databases/main/schema => schema/main}/delta/30/threepid_guest_access_tokens.sql (100%) rename synapse/storage/{databases/main/schema => schema/main}/delta/31/invites.sql (100%) rename synapse/storage/{databases/main/schema => schema/main}/delta/31/local_media_repository_url_cache.sql (100%) rename synapse/storage/{databases/main/schema => schema/main}/delta/31/pushers.py (100%) rename synapse/storage/{databases/main/schema => schema/main}/delta/31/pushers_index.sql (100%) rename synapse/storage/{databases/main/schema => schema/main}/delta/31/search_update.py (100%) rename synapse/storage/{databases/main/schema => schema/main}/delta/32/events.sql (100%) rename synapse/storage/{databases/main/schema => schema/main}/delta/32/openid.sql (100%) rename synapse/storage/{databases/main/schema => schema/main}/delta/32/pusher_throttle.sql (100%) rename synapse/storage/{databases/main/schema => schema/main}/delta/32/remove_indices.sql (100%) rename synapse/storage/{databases/main/schema => schema/main}/delta/32/reports.sql (100%) rename synapse/storage/{databases/main/schema => schema/main}/delta/33/access_tokens_device_index.sql (100%) rename synapse/storage/{databases/main/schema => schema/main}/delta/33/devices.sql (100%) rename synapse/storage/{databases/main/schema => schema/main}/delta/33/devices_for_e2e_keys.sql (100%) rename synapse/storage/{databases/main/schema => schema/main}/delta/33/devices_for_e2e_keys_clear_unknown_device.sql (100%) rename synapse/storage/{databases/main/schema => schema/main}/delta/33/event_fields.py (100%) rename synapse/storage/{databases/main/schema => schema/main}/delta/33/remote_media_ts.py (100%) rename synapse/storage/{databases/main/schema => schema/main}/delta/33/user_ips_index.sql (100%) rename synapse/storage/{databases/main/schema => schema/main}/delta/34/appservice_stream.sql (100%) rename synapse/storage/{databases/main/schema => schema/main}/delta/34/cache_stream.py (100%) rename synapse/storage/{databases/main/schema => schema/main}/delta/34/device_inbox.sql (100%) rename synapse/storage/{databases/main/schema => schema/main}/delta/34/push_display_name_rename.sql (100%) rename synapse/storage/{databases/main/schema => schema/main}/delta/34/received_txn_purge.py (100%) rename synapse/storage/{databases/main/schema => schema/main}/delta/35/contains_url.sql (100%) rename synapse/storage/{databases/main/schema => schema/main}/delta/35/device_outbox.sql (100%) rename synapse/storage/{databases/main/schema => schema/main}/delta/35/device_stream_id.sql (100%) rename synapse/storage/{databases/main/schema => schema/main}/delta/35/event_push_actions_index.sql (100%) rename synapse/storage/{databases/main/schema => schema/main}/delta/35/public_room_list_change_stream.sql (100%) rename synapse/storage/{databases/main/schema => schema/main}/delta/35/stream_order_to_extrem.sql (100%) rename synapse/storage/{databases/main/schema => schema/main}/delta/36/readd_public_rooms.sql (100%) rename synapse/storage/{databases/main/schema => schema/main}/delta/37/remove_auth_idx.py (100%) rename synapse/storage/{databases/main/schema => schema/main}/delta/37/user_threepids.sql (100%) rename synapse/storage/{databases/main/schema => schema/main}/delta/38/postgres_fts_gist.sql (100%) rename synapse/storage/{databases/main/schema => schema/main}/delta/39/appservice_room_list.sql (100%) rename synapse/storage/{databases/main/schema => schema/main}/delta/39/device_federation_stream_idx.sql (100%) rename synapse/storage/{databases/main/schema => schema/main}/delta/39/event_push_index.sql (100%) rename synapse/storage/{databases/main/schema => schema/main}/delta/39/federation_out_position.sql (100%) rename synapse/storage/{databases/main/schema => schema/main}/delta/39/membership_profile.sql (100%) rename synapse/storage/{databases/main/schema => schema/main}/delta/40/current_state_idx.sql (100%) rename synapse/storage/{databases/main/schema => schema/main}/delta/40/device_inbox.sql (100%) rename synapse/storage/{databases/main/schema => schema/main}/delta/40/device_list_streams.sql (100%) rename synapse/storage/{databases/main/schema => schema/main}/delta/40/event_push_summary.sql (100%) rename synapse/storage/{databases/main/schema => schema/main}/delta/40/pushers.sql (100%) rename synapse/storage/{databases/main/schema => schema/main}/delta/41/device_list_stream_idx.sql (100%) rename synapse/storage/{databases/main/schema => schema/main}/delta/41/device_outbound_index.sql (100%) rename synapse/storage/{databases/main/schema => schema/main}/delta/41/event_search_event_id_idx.sql (100%) rename synapse/storage/{databases/main/schema => schema/main}/delta/41/ratelimit.sql (100%) rename synapse/storage/{databases/main/schema => schema/main}/delta/42/current_state_delta.sql (100%) rename synapse/storage/{databases/main/schema => schema/main}/delta/42/device_list_last_id.sql (100%) rename synapse/storage/{databases/main/schema => schema/main}/delta/42/event_auth_state_only.sql (100%) rename synapse/storage/{databases/main/schema => schema/main}/delta/42/user_dir.py (100%) rename synapse/storage/{databases/main/schema => schema/main}/delta/43/blocked_rooms.sql (100%) rename synapse/storage/{databases/main/schema => schema/main}/delta/43/quarantine_media.sql (100%) rename synapse/storage/{databases/main/schema => schema/main}/delta/43/url_cache.sql (100%) rename synapse/storage/{databases/main/schema => schema/main}/delta/43/user_share.sql (100%) rename synapse/storage/{databases/main/schema => schema/main}/delta/44/expire_url_cache.sql (100%) rename synapse/storage/{databases/main/schema => schema/main}/delta/45/group_server.sql (100%) rename synapse/storage/{databases/main/schema => schema/main}/delta/45/profile_cache.sql (100%) rename synapse/storage/{databases/main/schema => schema/main}/delta/46/drop_refresh_tokens.sql (100%) rename synapse/storage/{databases/main/schema => schema/main}/delta/46/drop_unique_deleted_pushers.sql (100%) rename synapse/storage/{databases/main/schema => schema/main}/delta/46/group_server.sql (100%) rename synapse/storage/{databases/main/schema => schema/main}/delta/46/local_media_repository_url_idx.sql (100%) rename synapse/storage/{databases/main/schema => schema/main}/delta/46/user_dir_null_room_ids.sql (100%) rename synapse/storage/{databases/main/schema => schema/main}/delta/46/user_dir_typos.sql (100%) rename synapse/storage/{databases/main/schema => schema/main}/delta/47/last_access_media.sql (100%) rename synapse/storage/{databases/main/schema => schema/main}/delta/47/postgres_fts_gin.sql (100%) rename synapse/storage/{databases/main/schema => schema/main}/delta/47/push_actions_staging.sql (100%) rename synapse/storage/{databases/main/schema => schema/main}/delta/48/add_user_consent.sql (100%) rename synapse/storage/{databases/main/schema => schema/main}/delta/48/add_user_ips_last_seen_index.sql (100%) rename synapse/storage/{databases/main/schema => schema/main}/delta/48/deactivated_users.sql (100%) rename synapse/storage/{databases/main/schema => schema/main}/delta/48/group_unique_indexes.py (100%) rename synapse/storage/{databases/main/schema => schema/main}/delta/48/groups_joinable.sql (100%) rename synapse/storage/{databases/main/schema => schema/main}/delta/49/add_user_consent_server_notice_sent.sql (100%) rename synapse/storage/{databases/main/schema => schema/main}/delta/49/add_user_daily_visits.sql (100%) rename synapse/storage/{databases/main/schema => schema/main}/delta/49/add_user_ips_last_seen_only_index.sql (100%) rename synapse/storage/{databases/main/schema => schema/main}/delta/50/add_creation_ts_users_index.sql (100%) rename synapse/storage/{databases/main/schema => schema/main}/delta/50/erasure_store.sql (100%) rename synapse/storage/{databases/main/schema => schema/main}/delta/50/make_event_content_nullable.py (100%) rename synapse/storage/{databases/main/schema => schema/main}/delta/51/e2e_room_keys.sql (100%) rename synapse/storage/{databases/main/schema => schema/main}/delta/51/monthly_active_users.sql (100%) rename synapse/storage/{databases/main/schema => schema/main}/delta/52/add_event_to_state_group_index.sql (100%) rename synapse/storage/{databases/main/schema => schema/main}/delta/52/device_list_streams_unique_idx.sql (100%) rename synapse/storage/{databases/main/schema => schema/main}/delta/52/e2e_room_keys.sql (100%) rename synapse/storage/{databases/main/schema => schema/main}/delta/53/add_user_type_to_users.sql (100%) rename synapse/storage/{databases/main/schema => schema/main}/delta/53/drop_sent_transactions.sql (100%) rename synapse/storage/{databases/main/schema => schema/main}/delta/53/event_format_version.sql (100%) rename synapse/storage/{databases/main/schema => schema/main}/delta/53/user_dir_populate.sql (100%) rename synapse/storage/{databases/main/schema => schema/main}/delta/53/user_ips_index.sql (100%) rename synapse/storage/{databases/main/schema => schema/main}/delta/53/user_share.sql (100%) rename synapse/storage/{databases/main/schema => schema/main}/delta/53/user_threepid_id.sql (100%) rename synapse/storage/{databases/main/schema => schema/main}/delta/53/users_in_public_rooms.sql (100%) rename synapse/storage/{databases/main/schema => schema/main}/delta/54/account_validity_with_renewal.sql (100%) rename synapse/storage/{databases/main/schema => schema/main}/delta/54/add_validity_to_server_keys.sql (100%) rename synapse/storage/{databases/main/schema => schema/main}/delta/54/delete_forward_extremities.sql (100%) rename synapse/storage/{databases/main/schema => schema/main}/delta/54/drop_legacy_tables.sql (100%) rename synapse/storage/{databases/main/schema => schema/main}/delta/54/drop_presence_list.sql (100%) rename synapse/storage/{databases/main/schema => schema/main}/delta/54/relations.sql (100%) rename synapse/storage/{databases/main/schema => schema/main}/delta/54/stats.sql (100%) rename synapse/storage/{databases/main/schema => schema/main}/delta/54/stats2.sql (100%) rename synapse/storage/{databases/main/schema => schema/main}/delta/55/access_token_expiry.sql (100%) rename synapse/storage/{databases/main/schema => schema/main}/delta/55/track_threepid_validations.sql (100%) rename synapse/storage/{databases/main/schema => schema/main}/delta/55/users_alter_deactivated.sql (100%) rename synapse/storage/{databases/main/schema => schema/main}/delta/56/add_spans_to_device_lists.sql (100%) rename synapse/storage/{databases/main/schema => schema/main}/delta/56/current_state_events_membership.sql (100%) rename synapse/storage/{databases/main/schema => schema/main}/delta/56/current_state_events_membership_mk2.sql (100%) rename synapse/storage/{databases/main/schema => schema/main}/delta/56/delete_keys_from_deleted_backups.sql (100%) rename synapse/storage/{databases/main/schema => schema/main}/delta/56/destinations_failure_ts.sql (100%) rename synapse/storage/{databases/main/schema => schema/main}/delta/56/destinations_retry_interval_type.sql.postgres (100%) rename synapse/storage/{databases/main/schema => schema/main}/delta/56/device_stream_id_insert.sql (100%) rename synapse/storage/{databases/main/schema => schema/main}/delta/56/devices_last_seen.sql (100%) rename synapse/storage/{databases/main/schema => schema/main}/delta/56/drop_unused_event_tables.sql (100%) rename synapse/storage/{databases/main/schema => schema/main}/delta/56/event_expiry.sql (100%) rename synapse/storage/{databases/main/schema => schema/main}/delta/56/event_labels.sql (100%) rename synapse/storage/{databases/main/schema => schema/main}/delta/56/event_labels_background_update.sql (100%) rename synapse/storage/{databases/main/schema => schema/main}/delta/56/fix_room_keys_index.sql (100%) rename synapse/storage/{databases/main/schema => schema/main}/delta/56/hidden_devices.sql (100%) rename synapse/storage/{databases/main/schema => schema/main}/delta/56/hidden_devices_fix.sql.sqlite (100%) rename synapse/storage/{databases/main/schema => schema/main}/delta/56/nuke_empty_communities_from_db.sql (100%) rename synapse/storage/{databases/main/schema => schema/main}/delta/56/public_room_list_idx.sql (100%) rename synapse/storage/{databases/main/schema => schema/main}/delta/56/redaction_censor.sql (100%) rename synapse/storage/{databases/main/schema => schema/main}/delta/56/redaction_censor2.sql (100%) rename synapse/storage/{databases/main/schema => schema/main}/delta/56/redaction_censor3_fix_update.sql.postgres (100%) rename synapse/storage/{databases/main/schema => schema/main}/delta/56/redaction_censor4.sql (100%) rename synapse/storage/{databases/main/schema => schema/main}/delta/56/remove_tombstoned_rooms_from_directory.sql (100%) rename synapse/storage/{databases/main/schema => schema/main}/delta/56/room_key_etag.sql (100%) rename synapse/storage/{databases/main/schema => schema/main}/delta/56/room_membership_idx.sql (100%) rename synapse/storage/{databases/main/schema => schema/main}/delta/56/room_retention.sql (100%) rename synapse/storage/{databases/main/schema => schema/main}/delta/56/signing_keys.sql (100%) rename synapse/storage/{databases/main/schema => schema/main}/delta/56/signing_keys_nonunique_signatures.sql (100%) rename synapse/storage/{databases/main/schema => schema/main}/delta/56/stats_separated.sql (100%) rename synapse/storage/{databases/main/schema => schema/main}/delta/56/unique_user_filter_index.py (100%) rename synapse/storage/{databases/main/schema => schema/main}/delta/56/user_external_ids.sql (100%) rename synapse/storage/{databases/main/schema => schema/main}/delta/56/users_in_public_rooms_idx.sql (100%) rename synapse/storage/{databases/main/schema => schema/main}/delta/57/delete_old_current_state_events.sql (100%) rename synapse/storage/{databases/main/schema => schema/main}/delta/57/device_list_remote_cache_stale.sql (100%) rename synapse/storage/{databases/main/schema => schema/main}/delta/57/local_current_membership.py (100%) rename synapse/storage/{databases/main/schema => schema/main}/delta/57/remove_sent_outbound_pokes.sql (100%) rename synapse/storage/{databases/main/schema => schema/main}/delta/57/rooms_version_column.sql (100%) rename synapse/storage/{databases/main/schema => schema/main}/delta/57/rooms_version_column_2.sql.postgres (100%) rename synapse/storage/{databases/main/schema => schema/main}/delta/57/rooms_version_column_2.sql.sqlite (100%) rename synapse/storage/{databases/main/schema => schema/main}/delta/57/rooms_version_column_3.sql.postgres (100%) rename synapse/storage/{databases/main/schema => schema/main}/delta/57/rooms_version_column_3.sql.sqlite (100%) rename synapse/storage/{databases/main/schema => schema/main}/delta/58/02remove_dup_outbound_pokes.sql (100%) rename synapse/storage/{databases/main/schema => schema/main}/delta/58/03persist_ui_auth.sql (100%) rename synapse/storage/{databases/main/schema => schema/main}/delta/58/05cache_instance.sql.postgres (100%) rename synapse/storage/{databases/main/schema => schema/main}/delta/58/06dlols_unique_idx.py (100%) rename synapse/storage/{databases/main/schema => schema/main}/delta/58/07add_method_to_thumbnail_constraint.sql.postgres (100%) rename synapse/storage/{databases/main/schema => schema/main}/delta/58/07add_method_to_thumbnail_constraint.sql.sqlite (100%) rename synapse/storage/{databases/main/schema => schema/main}/delta/58/07persist_ui_auth_ips.sql (100%) rename synapse/storage/{databases/main/schema => schema/main}/delta/58/08_media_safe_from_quarantine.sql.postgres (100%) rename synapse/storage/{databases/main/schema => schema/main}/delta/58/08_media_safe_from_quarantine.sql.sqlite (100%) rename synapse/storage/{databases/main/schema => schema/main}/delta/58/09shadow_ban.sql (100%) rename synapse/storage/{databases/main/schema => schema/main}/delta/58/10_pushrules_enabled_delete_obsolete.sql (100%) rename synapse/storage/{databases/main/schema => schema/main}/delta/58/10drop_local_rejections_stream.sql (100%) rename synapse/storage/{databases/main/schema => schema/main}/delta/58/10federation_pos_instance_name.sql (100%) rename synapse/storage/{databases/main/schema => schema/main}/delta/58/11dehydration.sql (100%) rename synapse/storage/{databases/main/schema => schema/main}/delta/58/11fallback.sql (100%) rename synapse/storage/{databases/main/schema => schema/main}/delta/58/11user_id_seq.py (100%) rename synapse/storage/{databases/main/schema => schema/main}/delta/58/12room_stats.sql (100%) rename synapse/storage/{databases/main/schema => schema/main}/delta/58/13remove_presence_allow_inbound.sql (100%) rename synapse/storage/{databases/main/schema => schema/main}/delta/58/14events_instance_name.sql (100%) rename synapse/storage/{databases/main/schema => schema/main}/delta/58/14events_instance_name.sql.postgres (100%) rename synapse/storage/{databases/main/schema => schema/main}/delta/58/15_catchup_destination_rooms.sql (100%) rename synapse/storage/{databases/main/schema => schema/main}/delta/58/15unread_count.sql (100%) rename synapse/storage/{databases/main/schema => schema/main}/delta/58/16populate_stats_process_rooms_fix.sql (100%) rename synapse/storage/{databases/main/schema => schema/main}/delta/58/17_catchup_last_successful.sql (100%) rename synapse/storage/{databases/main/schema => schema/main}/delta/58/18stream_positions.sql (100%) rename synapse/storage/{databases/main/schema => schema/main}/delta/58/19instance_map.sql.postgres (100%) rename synapse/storage/{databases/main/schema => schema/main}/delta/58/19txn_id.sql (100%) rename synapse/storage/{databases/main/schema => schema/main}/delta/58/20instance_name_event_tables.sql (100%) rename synapse/storage/{databases/main/schema => schema/main}/delta/58/20user_daily_visits.sql (100%) rename synapse/storage/{databases/main/schema => schema/main}/delta/58/21as_device_stream.sql (100%) rename synapse/storage/{databases/main/schema => schema/main}/delta/58/21drop_device_max_stream_id.sql (100%) rename synapse/storage/{databases/main/schema => schema/main}/delta/58/22puppet_token.sql (100%) rename synapse/storage/{databases/main/schema => schema/main}/delta/58/22users_have_local_media.sql (100%) rename synapse/storage/{databases/main/schema => schema/main}/delta/58/23e2e_cross_signing_keys_idx.sql (100%) rename synapse/storage/{databases/main/schema => schema/main}/delta/58/24drop_event_json_index.sql (100%) rename synapse/storage/{databases/main/schema => schema/main}/delta/58/25user_external_ids_user_id_idx.sql (100%) rename synapse/storage/{databases/main/schema => schema/main}/delta/58/26access_token_last_validated.sql (100%) rename synapse/storage/{databases/main/schema => schema/main}/delta/58/27local_invites.sql (100%) rename synapse/storage/{databases/main/schema => schema/main}/delta/58/28drop_last_used_column.sql.postgres (100%) rename synapse/storage/{databases/main/schema => schema/main}/delta/58/28drop_last_used_column.sql.sqlite (100%) rename synapse/storage/{databases/main/schema => schema/main}/delta/59/01ignored_user.py (100%) rename synapse/storage/{databases/main/schema => schema/main}/delta/59/02shard_send_to_device.sql (100%) rename synapse/storage/{databases/main/schema => schema/main}/delta/59/03shard_send_to_device_sequence.sql.postgres (100%) rename synapse/storage/{databases/main/schema => schema/main}/delta/59/04_event_auth_chains.sql (100%) rename synapse/storage/{databases/main/schema => schema/main}/delta/59/04_event_auth_chains.sql.postgres (100%) rename synapse/storage/{databases/main/schema => schema/main}/delta/59/04drop_account_data.sql (100%) rename synapse/storage/{databases/main/schema => schema/main}/delta/59/05cache_invalidation.sql (100%) rename synapse/storage/{databases/main/schema => schema/main}/delta/59/06chain_cover_index.sql (100%) rename synapse/storage/{databases/main/schema => schema/main}/delta/59/06shard_account_data.sql (100%) rename synapse/storage/{databases/main/schema => schema/main}/delta/59/06shard_account_data.sql.postgres (100%) rename synapse/storage/{databases/main/schema => schema/main}/delta/59/07shard_account_data_fix.sql (100%) rename synapse/storage/{databases/main/schema => schema/main}/delta/59/08delete_pushers_for_deactivated_accounts.sql (100%) rename synapse/storage/{databases/main/schema => schema/main}/delta/59/08delete_stale_pushers.sql (100%) rename synapse/storage/{databases/main/schema => schema/main}/delta/59/09rejected_events_metadata.sql (100%) rename synapse/storage/{databases/main/schema => schema/main}/delta/59/10delete_purged_chain_cover.sql (100%) rename synapse/storage/{databases/main/schema => schema/main}/delta/59/11drop_thumbnail_constraint.sql.postgres (100%) rename synapse/storage/{databases/main/schema => schema/main}/delta/59/12account_validity_token_used_ts_ms.sql (100%) rename synapse/storage/{databases/main/schema => schema/main}/delta/59/12presence_stream_instance.sql (100%) rename synapse/storage/{databases/main/schema => schema/main}/delta/59/12presence_stream_instance_seq.sql.postgres (100%) rename synapse/storage/{databases/main/schema => schema/main}/full_schemas/16/application_services.sql (100%) rename synapse/storage/{databases/main/schema => schema/main}/full_schemas/16/event_edges.sql (100%) rename synapse/storage/{databases/main/schema => schema/main}/full_schemas/16/event_signatures.sql (100%) rename synapse/storage/{databases/main/schema => schema/main}/full_schemas/16/im.sql (100%) rename synapse/storage/{databases/main/schema => schema/main}/full_schemas/16/keys.sql (100%) rename synapse/storage/{databases/main/schema => schema/main}/full_schemas/16/media_repository.sql (100%) rename synapse/storage/{databases/main/schema => schema/main}/full_schemas/16/presence.sql (100%) rename synapse/storage/{databases/main/schema => schema/main}/full_schemas/16/profiles.sql (100%) rename synapse/storage/{databases/main/schema => schema/main}/full_schemas/16/push.sql (100%) rename synapse/storage/{databases/main/schema => schema/main}/full_schemas/16/redactions.sql (100%) rename synapse/storage/{databases/main/schema => schema/main}/full_schemas/16/room_aliases.sql (100%) rename synapse/storage/{databases/main/schema => schema/main}/full_schemas/16/state.sql (100%) rename synapse/storage/{databases/main/schema => schema/main}/full_schemas/16/transactions.sql (100%) rename synapse/storage/{databases/main/schema => schema/main}/full_schemas/16/users.sql (100%) rename synapse/storage/{databases/main/schema => schema/main}/full_schemas/54/full.sql.postgres (100%) rename synapse/storage/{databases/main/schema => schema/main}/full_schemas/54/full.sql.sqlite (100%) rename synapse/storage/{databases/main/schema => schema/main}/full_schemas/54/stream_positions.sql (100%) rename synapse/storage/{databases/state/schema => schema/state}/delta/23/drop_state_index.sql (100%) rename synapse/storage/{databases/state/schema => schema/state}/delta/30/state_stream.sql (100%) rename synapse/storage/{databases/state/schema => schema/state}/delta/32/remove_state_indices.sql (100%) rename synapse/storage/{databases/state/schema => schema/state}/delta/35/add_state_index.sql (100%) rename synapse/storage/{databases/state/schema => schema/state}/delta/35/state.sql (100%) rename synapse/storage/{databases/state/schema => schema/state}/delta/35/state_dedupe.sql (100%) rename synapse/storage/{databases/state/schema => schema/state}/delta/47/state_group_seq.py (100%) rename synapse/storage/{databases/state/schema => schema/state}/delta/56/state_group_room_idx.sql (100%) rename synapse/storage/{databases/state/schema => schema/state}/full_schemas/54/full.sql (100%) rename synapse/storage/{databases/state/schema => schema/state}/full_schemas/54/sequence.sql.postgres (100%) diff --git a/changelog.d/9932.misc b/changelog.d/9932.misc new file mode 100644 index 0000000000..9e16a36173 --- /dev/null +++ b/changelog.d/9932.misc @@ -0,0 +1 @@ +Move database schema files into a common directory. diff --git a/synapse/storage/databases/main/schema/full_schemas/README.md b/synapse/storage/databases/main/schema/full_schemas/README.md deleted file mode 100644 index c00f287190..0000000000 --- a/synapse/storage/databases/main/schema/full_schemas/README.md +++ /dev/null @@ -1,21 +0,0 @@ -# Synapse Database Schemas - -These schemas are used as a basis to create brand new Synapse databases, on both -SQLite3 and Postgres. - -## Building full schema dumps - -If you want to recreate these schemas, they need to be made from a database that -has had all background updates run. - -To do so, use `scripts-dev/make_full_schema.sh`. This will produce new -`full.sql.postgres ` and `full.sql.sqlite` files. - -Ensure postgres is installed and your user has the ability to run bash commands -such as `createdb`, then call - - ./scripts-dev/make_full_schema.sh -p postgres_username -o output_dir/ - -There are currently two folders with full-schema snapshots. `16` is a snapshot -from 2015, for historical reference. The other contains the most recent full -schema snapshot. diff --git a/synapse/storage/prepare_database.py b/synapse/storage/prepare_database.py index 7a2cbee426..3799d46734 100644 --- a/synapse/storage/prepare_database.py +++ b/synapse/storage/prepare_database.py @@ -26,16 +26,13 @@ from synapse.config.homeserver import HomeServerConfig from synapse.storage.database import LoggingDatabaseConnection from synapse.storage.engines import BaseDatabaseEngine from synapse.storage.engines.postgres import PostgresEngine +from synapse.storage.schema import SCHEMA_VERSION from synapse.storage.types import Cursor logger = logging.getLogger(__name__) -# Remember to update this number every time a change is made to database -# schema files, so the users will be informed on server restarts. -SCHEMA_VERSION = 59 - -dir_path = os.path.abspath(os.path.dirname(__file__)) +schema_path = os.path.join(os.path.abspath(os.path.dirname(__file__)), "schema") class PrepareDatabaseException(Exception): @@ -167,7 +164,14 @@ def _setup_new_database( Example directory structure: - schema/ + schema/ + common/ + delta/ + ... + full_schemas/ + 11/ + foo.sql + main/ delta/ ... full_schemas/ @@ -175,15 +179,14 @@ def _setup_new_database( test.sql ... 11/ - foo.sql bar.sql ... In the example foo.sql and bar.sql would be run, and then any delta files for versions strictly greater than 11. - Note: we apply the full schemas and deltas from the top level `schema/` - folder as well those in the data stores specified. + Note: we apply the full schemas and deltas from the `schema/common` + folder as well those in the databases specified. Args: cur: a database cursor @@ -195,12 +198,12 @@ def _setup_new_database( # configured to our liking. database_engine.check_new_database(cur) - current_dir = os.path.join(dir_path, "schema", "full_schemas") + full_schemas_dir = os.path.join(schema_path, "common", "full_schemas") # First we find the highest full schema version we have valid_versions = [] - for filename in os.listdir(current_dir): + for filename in os.listdir(full_schemas_dir): try: ver = int(filename) except ValueError: @@ -218,15 +221,13 @@ def _setup_new_database( logger.debug("Initialising schema v%d", max_current_ver) - # Now lets find all the full schema files, both in the global schema and - # in data store schemas. - directories = [os.path.join(current_dir, str(max_current_ver))] + # Now let's find all the full schema files, both in the common schema and + # in database schemas. + directories = [os.path.join(full_schemas_dir, str(max_current_ver))] directories.extend( os.path.join( - dir_path, - "databases", + schema_path, database, - "schema", "full_schemas", str(max_current_ver), ) @@ -357,6 +358,9 @@ def _upgrade_existing_database( check_database_before_upgrade(cur, database_engine, config) start_ver = current_version + + # if we got to this schema version by running a full_schema rather than a series + # of deltas, we should not run the deltas for this version. if not upgraded: start_ver += 1 @@ -385,12 +389,10 @@ def _upgrade_existing_database( # directories for schema updates. # First we find the directories to search in - delta_dir = os.path.join(dir_path, "schema", "delta", str(v)) + delta_dir = os.path.join(schema_path, "common", "delta", str(v)) directories = [delta_dir] for database in databases: - directories.append( - os.path.join(dir_path, "databases", database, "schema", "delta", str(v)) - ) + directories.append(os.path.join(schema_path, database, "delta", str(v))) # Used to check if we have any duplicate file names file_name_counter = Counter() # type: CounterType[str] @@ -621,8 +623,8 @@ def _get_or_create_schema_state( txn: Cursor, database_engine: BaseDatabaseEngine ) -> Optional[Tuple[int, List[str], bool]]: # Bluntly try creating the schema_version tables. - schema_path = os.path.join(dir_path, "schema", "schema_version.sql") - executescript(txn, schema_path) + sql_path = os.path.join(schema_path, "common", "schema_version.sql") + executescript(txn, sql_path) txn.execute("SELECT version, upgraded FROM schema_version") row = txn.fetchone() diff --git a/synapse/storage/schema/README.md b/synapse/storage/schema/README.md new file mode 100644 index 0000000000..030153db64 --- /dev/null +++ b/synapse/storage/schema/README.md @@ -0,0 +1,37 @@ +# Synapse Database Schemas + +This directory contains the schema files used to build Synapse databases. + +Synapse supports splitting its datastore across multiple physical databases (which can +be useful for large installations), and the schema files are therefore split according +to the logical database they are apply to. + +At the time of writing, the following "logical" databases are supported: + +* `state` - used to store Matrix room state (more specifically, `state_groups`, + their relationships and contents.) +* `main` - stores everything else. + +Addionally, the `common` directory contains schema files for tables which must be +present on *all* physical databases. + +## Full schema dumps + +In the `full_schemas` directories, only the most recently-numbered snapshot is useful +(`54` at the time of writing). Older snapshots (eg, `16`) are present for historical +reference only. + +## Building full schema dumps + +If you want to recreate these schemas, they need to be made from a database that +has had all background updates run. + +To do so, use `scripts-dev/make_full_schema.sh`. This will produce new +`full.sql.postgres` and `full.sql.sqlite` files. + +Ensure postgres is installed, then run: + + ./scripts-dev/make_full_schema.sh -p postgres_username -o output_dir/ + +NB at the time of writing, this script predates the split into separate `state`/`main` +databases so will require updates to handle that correctly. diff --git a/synapse/storage/schema/__init__.py b/synapse/storage/schema/__init__.py new file mode 100644 index 0000000000..f0d9f23167 --- /dev/null +++ b/synapse/storage/schema/__init__.py @@ -0,0 +1,17 @@ +# Copyright 2021 The Matrix.org Foundation C.I.C. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Remember to update this number every time a change is made to database +# schema files, so the users will be informed on server restarts. +SCHEMA_VERSION = 59 diff --git a/synapse/storage/schema/delta/25/00background_updates.sql b/synapse/storage/schema/common/delta/25/00background_updates.sql similarity index 100% rename from synapse/storage/schema/delta/25/00background_updates.sql rename to synapse/storage/schema/common/delta/25/00background_updates.sql diff --git a/synapse/storage/schema/delta/35/00background_updates_add_col.sql b/synapse/storage/schema/common/delta/35/00background_updates_add_col.sql similarity index 100% rename from synapse/storage/schema/delta/35/00background_updates_add_col.sql rename to synapse/storage/schema/common/delta/35/00background_updates_add_col.sql diff --git a/synapse/storage/schema/delta/58/00background_update_ordering.sql b/synapse/storage/schema/common/delta/58/00background_update_ordering.sql similarity index 100% rename from synapse/storage/schema/delta/58/00background_update_ordering.sql rename to synapse/storage/schema/common/delta/58/00background_update_ordering.sql diff --git a/synapse/storage/schema/full_schemas/54/full.sql b/synapse/storage/schema/common/full_schemas/54/full.sql similarity index 100% rename from synapse/storage/schema/full_schemas/54/full.sql rename to synapse/storage/schema/common/full_schemas/54/full.sql diff --git a/synapse/storage/schema/schema_version.sql b/synapse/storage/schema/common/schema_version.sql similarity index 100% rename from synapse/storage/schema/schema_version.sql rename to synapse/storage/schema/common/schema_version.sql diff --git a/synapse/storage/databases/main/schema/delta/12/v12.sql b/synapse/storage/schema/main/delta/12/v12.sql similarity index 100% rename from synapse/storage/databases/main/schema/delta/12/v12.sql rename to synapse/storage/schema/main/delta/12/v12.sql diff --git a/synapse/storage/databases/main/schema/delta/13/v13.sql b/synapse/storage/schema/main/delta/13/v13.sql similarity index 100% rename from synapse/storage/databases/main/schema/delta/13/v13.sql rename to synapse/storage/schema/main/delta/13/v13.sql diff --git a/synapse/storage/databases/main/schema/delta/14/v14.sql b/synapse/storage/schema/main/delta/14/v14.sql similarity index 100% rename from synapse/storage/databases/main/schema/delta/14/v14.sql rename to synapse/storage/schema/main/delta/14/v14.sql diff --git a/synapse/storage/databases/main/schema/delta/15/appservice_txns.sql b/synapse/storage/schema/main/delta/15/appservice_txns.sql similarity index 100% rename from synapse/storage/databases/main/schema/delta/15/appservice_txns.sql rename to synapse/storage/schema/main/delta/15/appservice_txns.sql diff --git a/synapse/storage/databases/main/schema/delta/15/presence_indices.sql b/synapse/storage/schema/main/delta/15/presence_indices.sql similarity index 100% rename from synapse/storage/databases/main/schema/delta/15/presence_indices.sql rename to synapse/storage/schema/main/delta/15/presence_indices.sql diff --git a/synapse/storage/databases/main/schema/delta/15/v15.sql b/synapse/storage/schema/main/delta/15/v15.sql similarity index 100% rename from synapse/storage/databases/main/schema/delta/15/v15.sql rename to synapse/storage/schema/main/delta/15/v15.sql diff --git a/synapse/storage/databases/main/schema/delta/16/events_order_index.sql b/synapse/storage/schema/main/delta/16/events_order_index.sql similarity index 100% rename from synapse/storage/databases/main/schema/delta/16/events_order_index.sql rename to synapse/storage/schema/main/delta/16/events_order_index.sql diff --git a/synapse/storage/databases/main/schema/delta/16/remote_media_cache_index.sql b/synapse/storage/schema/main/delta/16/remote_media_cache_index.sql similarity index 100% rename from synapse/storage/databases/main/schema/delta/16/remote_media_cache_index.sql rename to synapse/storage/schema/main/delta/16/remote_media_cache_index.sql diff --git a/synapse/storage/databases/main/schema/delta/16/remove_duplicates.sql b/synapse/storage/schema/main/delta/16/remove_duplicates.sql similarity index 100% rename from synapse/storage/databases/main/schema/delta/16/remove_duplicates.sql rename to synapse/storage/schema/main/delta/16/remove_duplicates.sql diff --git a/synapse/storage/databases/main/schema/delta/16/room_alias_index.sql b/synapse/storage/schema/main/delta/16/room_alias_index.sql similarity index 100% rename from synapse/storage/databases/main/schema/delta/16/room_alias_index.sql rename to synapse/storage/schema/main/delta/16/room_alias_index.sql diff --git a/synapse/storage/databases/main/schema/delta/16/unique_constraints.sql b/synapse/storage/schema/main/delta/16/unique_constraints.sql similarity index 100% rename from synapse/storage/databases/main/schema/delta/16/unique_constraints.sql rename to synapse/storage/schema/main/delta/16/unique_constraints.sql diff --git a/synapse/storage/databases/main/schema/delta/16/users.sql b/synapse/storage/schema/main/delta/16/users.sql similarity index 100% rename from synapse/storage/databases/main/schema/delta/16/users.sql rename to synapse/storage/schema/main/delta/16/users.sql diff --git a/synapse/storage/databases/main/schema/delta/17/drop_indexes.sql b/synapse/storage/schema/main/delta/17/drop_indexes.sql similarity index 100% rename from synapse/storage/databases/main/schema/delta/17/drop_indexes.sql rename to synapse/storage/schema/main/delta/17/drop_indexes.sql diff --git a/synapse/storage/databases/main/schema/delta/17/server_keys.sql b/synapse/storage/schema/main/delta/17/server_keys.sql similarity index 100% rename from synapse/storage/databases/main/schema/delta/17/server_keys.sql rename to synapse/storage/schema/main/delta/17/server_keys.sql diff --git a/synapse/storage/databases/main/schema/delta/17/user_threepids.sql b/synapse/storage/schema/main/delta/17/user_threepids.sql similarity index 100% rename from synapse/storage/databases/main/schema/delta/17/user_threepids.sql rename to synapse/storage/schema/main/delta/17/user_threepids.sql diff --git a/synapse/storage/databases/main/schema/delta/18/server_keys_bigger_ints.sql b/synapse/storage/schema/main/delta/18/server_keys_bigger_ints.sql similarity index 100% rename from synapse/storage/databases/main/schema/delta/18/server_keys_bigger_ints.sql rename to synapse/storage/schema/main/delta/18/server_keys_bigger_ints.sql diff --git a/synapse/storage/databases/main/schema/delta/19/event_index.sql b/synapse/storage/schema/main/delta/19/event_index.sql similarity index 100% rename from synapse/storage/databases/main/schema/delta/19/event_index.sql rename to synapse/storage/schema/main/delta/19/event_index.sql diff --git a/synapse/storage/databases/main/schema/delta/20/dummy.sql b/synapse/storage/schema/main/delta/20/dummy.sql similarity index 100% rename from synapse/storage/databases/main/schema/delta/20/dummy.sql rename to synapse/storage/schema/main/delta/20/dummy.sql diff --git a/synapse/storage/databases/main/schema/delta/20/pushers.py b/synapse/storage/schema/main/delta/20/pushers.py similarity index 100% rename from synapse/storage/databases/main/schema/delta/20/pushers.py rename to synapse/storage/schema/main/delta/20/pushers.py diff --git a/synapse/storage/databases/main/schema/delta/21/end_to_end_keys.sql b/synapse/storage/schema/main/delta/21/end_to_end_keys.sql similarity index 100% rename from synapse/storage/databases/main/schema/delta/21/end_to_end_keys.sql rename to synapse/storage/schema/main/delta/21/end_to_end_keys.sql diff --git a/synapse/storage/databases/main/schema/delta/21/receipts.sql b/synapse/storage/schema/main/delta/21/receipts.sql similarity index 100% rename from synapse/storage/databases/main/schema/delta/21/receipts.sql rename to synapse/storage/schema/main/delta/21/receipts.sql diff --git a/synapse/storage/databases/main/schema/delta/22/receipts_index.sql b/synapse/storage/schema/main/delta/22/receipts_index.sql similarity index 100% rename from synapse/storage/databases/main/schema/delta/22/receipts_index.sql rename to synapse/storage/schema/main/delta/22/receipts_index.sql diff --git a/synapse/storage/databases/main/schema/delta/22/user_threepids_unique.sql b/synapse/storage/schema/main/delta/22/user_threepids_unique.sql similarity index 100% rename from synapse/storage/databases/main/schema/delta/22/user_threepids_unique.sql rename to synapse/storage/schema/main/delta/22/user_threepids_unique.sql diff --git a/synapse/storage/databases/main/schema/delta/24/stats_reporting.sql b/synapse/storage/schema/main/delta/24/stats_reporting.sql similarity index 100% rename from synapse/storage/databases/main/schema/delta/24/stats_reporting.sql rename to synapse/storage/schema/main/delta/24/stats_reporting.sql diff --git a/synapse/storage/databases/main/schema/delta/25/fts.py b/synapse/storage/schema/main/delta/25/fts.py similarity index 100% rename from synapse/storage/databases/main/schema/delta/25/fts.py rename to synapse/storage/schema/main/delta/25/fts.py diff --git a/synapse/storage/databases/main/schema/delta/25/guest_access.sql b/synapse/storage/schema/main/delta/25/guest_access.sql similarity index 100% rename from synapse/storage/databases/main/schema/delta/25/guest_access.sql rename to synapse/storage/schema/main/delta/25/guest_access.sql diff --git a/synapse/storage/databases/main/schema/delta/25/history_visibility.sql b/synapse/storage/schema/main/delta/25/history_visibility.sql similarity index 100% rename from synapse/storage/databases/main/schema/delta/25/history_visibility.sql rename to synapse/storage/schema/main/delta/25/history_visibility.sql diff --git a/synapse/storage/databases/main/schema/delta/25/tags.sql b/synapse/storage/schema/main/delta/25/tags.sql similarity index 100% rename from synapse/storage/databases/main/schema/delta/25/tags.sql rename to synapse/storage/schema/main/delta/25/tags.sql diff --git a/synapse/storage/databases/main/schema/delta/26/account_data.sql b/synapse/storage/schema/main/delta/26/account_data.sql similarity index 100% rename from synapse/storage/databases/main/schema/delta/26/account_data.sql rename to synapse/storage/schema/main/delta/26/account_data.sql diff --git a/synapse/storage/databases/main/schema/delta/27/account_data.sql b/synapse/storage/schema/main/delta/27/account_data.sql similarity index 100% rename from synapse/storage/databases/main/schema/delta/27/account_data.sql rename to synapse/storage/schema/main/delta/27/account_data.sql diff --git a/synapse/storage/databases/main/schema/delta/27/forgotten_memberships.sql b/synapse/storage/schema/main/delta/27/forgotten_memberships.sql similarity index 100% rename from synapse/storage/databases/main/schema/delta/27/forgotten_memberships.sql rename to synapse/storage/schema/main/delta/27/forgotten_memberships.sql diff --git a/synapse/storage/databases/main/schema/delta/27/ts.py b/synapse/storage/schema/main/delta/27/ts.py similarity index 100% rename from synapse/storage/databases/main/schema/delta/27/ts.py rename to synapse/storage/schema/main/delta/27/ts.py diff --git a/synapse/storage/databases/main/schema/delta/28/event_push_actions.sql b/synapse/storage/schema/main/delta/28/event_push_actions.sql similarity index 100% rename from synapse/storage/databases/main/schema/delta/28/event_push_actions.sql rename to synapse/storage/schema/main/delta/28/event_push_actions.sql diff --git a/synapse/storage/databases/main/schema/delta/28/events_room_stream.sql b/synapse/storage/schema/main/delta/28/events_room_stream.sql similarity index 100% rename from synapse/storage/databases/main/schema/delta/28/events_room_stream.sql rename to synapse/storage/schema/main/delta/28/events_room_stream.sql diff --git a/synapse/storage/databases/main/schema/delta/28/public_roms_index.sql b/synapse/storage/schema/main/delta/28/public_roms_index.sql similarity index 100% rename from synapse/storage/databases/main/schema/delta/28/public_roms_index.sql rename to synapse/storage/schema/main/delta/28/public_roms_index.sql diff --git a/synapse/storage/databases/main/schema/delta/28/receipts_user_id_index.sql b/synapse/storage/schema/main/delta/28/receipts_user_id_index.sql similarity index 100% rename from synapse/storage/databases/main/schema/delta/28/receipts_user_id_index.sql rename to synapse/storage/schema/main/delta/28/receipts_user_id_index.sql diff --git a/synapse/storage/databases/main/schema/delta/28/upgrade_times.sql b/synapse/storage/schema/main/delta/28/upgrade_times.sql similarity index 100% rename from synapse/storage/databases/main/schema/delta/28/upgrade_times.sql rename to synapse/storage/schema/main/delta/28/upgrade_times.sql diff --git a/synapse/storage/databases/main/schema/delta/28/users_is_guest.sql b/synapse/storage/schema/main/delta/28/users_is_guest.sql similarity index 100% rename from synapse/storage/databases/main/schema/delta/28/users_is_guest.sql rename to synapse/storage/schema/main/delta/28/users_is_guest.sql diff --git a/synapse/storage/databases/main/schema/delta/29/push_actions.sql b/synapse/storage/schema/main/delta/29/push_actions.sql similarity index 100% rename from synapse/storage/databases/main/schema/delta/29/push_actions.sql rename to synapse/storage/schema/main/delta/29/push_actions.sql diff --git a/synapse/storage/databases/main/schema/delta/30/alias_creator.sql b/synapse/storage/schema/main/delta/30/alias_creator.sql similarity index 100% rename from synapse/storage/databases/main/schema/delta/30/alias_creator.sql rename to synapse/storage/schema/main/delta/30/alias_creator.sql diff --git a/synapse/storage/databases/main/schema/delta/30/as_users.py b/synapse/storage/schema/main/delta/30/as_users.py similarity index 100% rename from synapse/storage/databases/main/schema/delta/30/as_users.py rename to synapse/storage/schema/main/delta/30/as_users.py diff --git a/synapse/storage/databases/main/schema/delta/30/deleted_pushers.sql b/synapse/storage/schema/main/delta/30/deleted_pushers.sql similarity index 100% rename from synapse/storage/databases/main/schema/delta/30/deleted_pushers.sql rename to synapse/storage/schema/main/delta/30/deleted_pushers.sql diff --git a/synapse/storage/databases/main/schema/delta/30/presence_stream.sql b/synapse/storage/schema/main/delta/30/presence_stream.sql similarity index 100% rename from synapse/storage/databases/main/schema/delta/30/presence_stream.sql rename to synapse/storage/schema/main/delta/30/presence_stream.sql diff --git a/synapse/storage/databases/main/schema/delta/30/public_rooms.sql b/synapse/storage/schema/main/delta/30/public_rooms.sql similarity index 100% rename from synapse/storage/databases/main/schema/delta/30/public_rooms.sql rename to synapse/storage/schema/main/delta/30/public_rooms.sql diff --git a/synapse/storage/databases/main/schema/delta/30/push_rule_stream.sql b/synapse/storage/schema/main/delta/30/push_rule_stream.sql similarity index 100% rename from synapse/storage/databases/main/schema/delta/30/push_rule_stream.sql rename to synapse/storage/schema/main/delta/30/push_rule_stream.sql diff --git a/synapse/storage/databases/main/schema/delta/30/threepid_guest_access_tokens.sql b/synapse/storage/schema/main/delta/30/threepid_guest_access_tokens.sql similarity index 100% rename from synapse/storage/databases/main/schema/delta/30/threepid_guest_access_tokens.sql rename to synapse/storage/schema/main/delta/30/threepid_guest_access_tokens.sql diff --git a/synapse/storage/databases/main/schema/delta/31/invites.sql b/synapse/storage/schema/main/delta/31/invites.sql similarity index 100% rename from synapse/storage/databases/main/schema/delta/31/invites.sql rename to synapse/storage/schema/main/delta/31/invites.sql diff --git a/synapse/storage/databases/main/schema/delta/31/local_media_repository_url_cache.sql b/synapse/storage/schema/main/delta/31/local_media_repository_url_cache.sql similarity index 100% rename from synapse/storage/databases/main/schema/delta/31/local_media_repository_url_cache.sql rename to synapse/storage/schema/main/delta/31/local_media_repository_url_cache.sql diff --git a/synapse/storage/databases/main/schema/delta/31/pushers.py b/synapse/storage/schema/main/delta/31/pushers.py similarity index 100% rename from synapse/storage/databases/main/schema/delta/31/pushers.py rename to synapse/storage/schema/main/delta/31/pushers.py diff --git a/synapse/storage/databases/main/schema/delta/31/pushers_index.sql b/synapse/storage/schema/main/delta/31/pushers_index.sql similarity index 100% rename from synapse/storage/databases/main/schema/delta/31/pushers_index.sql rename to synapse/storage/schema/main/delta/31/pushers_index.sql diff --git a/synapse/storage/databases/main/schema/delta/31/search_update.py b/synapse/storage/schema/main/delta/31/search_update.py similarity index 100% rename from synapse/storage/databases/main/schema/delta/31/search_update.py rename to synapse/storage/schema/main/delta/31/search_update.py diff --git a/synapse/storage/databases/main/schema/delta/32/events.sql b/synapse/storage/schema/main/delta/32/events.sql similarity index 100% rename from synapse/storage/databases/main/schema/delta/32/events.sql rename to synapse/storage/schema/main/delta/32/events.sql diff --git a/synapse/storage/databases/main/schema/delta/32/openid.sql b/synapse/storage/schema/main/delta/32/openid.sql similarity index 100% rename from synapse/storage/databases/main/schema/delta/32/openid.sql rename to synapse/storage/schema/main/delta/32/openid.sql diff --git a/synapse/storage/databases/main/schema/delta/32/pusher_throttle.sql b/synapse/storage/schema/main/delta/32/pusher_throttle.sql similarity index 100% rename from synapse/storage/databases/main/schema/delta/32/pusher_throttle.sql rename to synapse/storage/schema/main/delta/32/pusher_throttle.sql diff --git a/synapse/storage/databases/main/schema/delta/32/remove_indices.sql b/synapse/storage/schema/main/delta/32/remove_indices.sql similarity index 100% rename from synapse/storage/databases/main/schema/delta/32/remove_indices.sql rename to synapse/storage/schema/main/delta/32/remove_indices.sql diff --git a/synapse/storage/databases/main/schema/delta/32/reports.sql b/synapse/storage/schema/main/delta/32/reports.sql similarity index 100% rename from synapse/storage/databases/main/schema/delta/32/reports.sql rename to synapse/storage/schema/main/delta/32/reports.sql diff --git a/synapse/storage/databases/main/schema/delta/33/access_tokens_device_index.sql b/synapse/storage/schema/main/delta/33/access_tokens_device_index.sql similarity index 100% rename from synapse/storage/databases/main/schema/delta/33/access_tokens_device_index.sql rename to synapse/storage/schema/main/delta/33/access_tokens_device_index.sql diff --git a/synapse/storage/databases/main/schema/delta/33/devices.sql b/synapse/storage/schema/main/delta/33/devices.sql similarity index 100% rename from synapse/storage/databases/main/schema/delta/33/devices.sql rename to synapse/storage/schema/main/delta/33/devices.sql diff --git a/synapse/storage/databases/main/schema/delta/33/devices_for_e2e_keys.sql b/synapse/storage/schema/main/delta/33/devices_for_e2e_keys.sql similarity index 100% rename from synapse/storage/databases/main/schema/delta/33/devices_for_e2e_keys.sql rename to synapse/storage/schema/main/delta/33/devices_for_e2e_keys.sql diff --git a/synapse/storage/databases/main/schema/delta/33/devices_for_e2e_keys_clear_unknown_device.sql b/synapse/storage/schema/main/delta/33/devices_for_e2e_keys_clear_unknown_device.sql similarity index 100% rename from synapse/storage/databases/main/schema/delta/33/devices_for_e2e_keys_clear_unknown_device.sql rename to synapse/storage/schema/main/delta/33/devices_for_e2e_keys_clear_unknown_device.sql diff --git a/synapse/storage/databases/main/schema/delta/33/event_fields.py b/synapse/storage/schema/main/delta/33/event_fields.py similarity index 100% rename from synapse/storage/databases/main/schema/delta/33/event_fields.py rename to synapse/storage/schema/main/delta/33/event_fields.py diff --git a/synapse/storage/databases/main/schema/delta/33/remote_media_ts.py b/synapse/storage/schema/main/delta/33/remote_media_ts.py similarity index 100% rename from synapse/storage/databases/main/schema/delta/33/remote_media_ts.py rename to synapse/storage/schema/main/delta/33/remote_media_ts.py diff --git a/synapse/storage/databases/main/schema/delta/33/user_ips_index.sql b/synapse/storage/schema/main/delta/33/user_ips_index.sql similarity index 100% rename from synapse/storage/databases/main/schema/delta/33/user_ips_index.sql rename to synapse/storage/schema/main/delta/33/user_ips_index.sql diff --git a/synapse/storage/databases/main/schema/delta/34/appservice_stream.sql b/synapse/storage/schema/main/delta/34/appservice_stream.sql similarity index 100% rename from synapse/storage/databases/main/schema/delta/34/appservice_stream.sql rename to synapse/storage/schema/main/delta/34/appservice_stream.sql diff --git a/synapse/storage/databases/main/schema/delta/34/cache_stream.py b/synapse/storage/schema/main/delta/34/cache_stream.py similarity index 100% rename from synapse/storage/databases/main/schema/delta/34/cache_stream.py rename to synapse/storage/schema/main/delta/34/cache_stream.py diff --git a/synapse/storage/databases/main/schema/delta/34/device_inbox.sql b/synapse/storage/schema/main/delta/34/device_inbox.sql similarity index 100% rename from synapse/storage/databases/main/schema/delta/34/device_inbox.sql rename to synapse/storage/schema/main/delta/34/device_inbox.sql diff --git a/synapse/storage/databases/main/schema/delta/34/push_display_name_rename.sql b/synapse/storage/schema/main/delta/34/push_display_name_rename.sql similarity index 100% rename from synapse/storage/databases/main/schema/delta/34/push_display_name_rename.sql rename to synapse/storage/schema/main/delta/34/push_display_name_rename.sql diff --git a/synapse/storage/databases/main/schema/delta/34/received_txn_purge.py b/synapse/storage/schema/main/delta/34/received_txn_purge.py similarity index 100% rename from synapse/storage/databases/main/schema/delta/34/received_txn_purge.py rename to synapse/storage/schema/main/delta/34/received_txn_purge.py diff --git a/synapse/storage/databases/main/schema/delta/35/contains_url.sql b/synapse/storage/schema/main/delta/35/contains_url.sql similarity index 100% rename from synapse/storage/databases/main/schema/delta/35/contains_url.sql rename to synapse/storage/schema/main/delta/35/contains_url.sql diff --git a/synapse/storage/databases/main/schema/delta/35/device_outbox.sql b/synapse/storage/schema/main/delta/35/device_outbox.sql similarity index 100% rename from synapse/storage/databases/main/schema/delta/35/device_outbox.sql rename to synapse/storage/schema/main/delta/35/device_outbox.sql diff --git a/synapse/storage/databases/main/schema/delta/35/device_stream_id.sql b/synapse/storage/schema/main/delta/35/device_stream_id.sql similarity index 100% rename from synapse/storage/databases/main/schema/delta/35/device_stream_id.sql rename to synapse/storage/schema/main/delta/35/device_stream_id.sql diff --git a/synapse/storage/databases/main/schema/delta/35/event_push_actions_index.sql b/synapse/storage/schema/main/delta/35/event_push_actions_index.sql similarity index 100% rename from synapse/storage/databases/main/schema/delta/35/event_push_actions_index.sql rename to synapse/storage/schema/main/delta/35/event_push_actions_index.sql diff --git a/synapse/storage/databases/main/schema/delta/35/public_room_list_change_stream.sql b/synapse/storage/schema/main/delta/35/public_room_list_change_stream.sql similarity index 100% rename from synapse/storage/databases/main/schema/delta/35/public_room_list_change_stream.sql rename to synapse/storage/schema/main/delta/35/public_room_list_change_stream.sql diff --git a/synapse/storage/databases/main/schema/delta/35/stream_order_to_extrem.sql b/synapse/storage/schema/main/delta/35/stream_order_to_extrem.sql similarity index 100% rename from synapse/storage/databases/main/schema/delta/35/stream_order_to_extrem.sql rename to synapse/storage/schema/main/delta/35/stream_order_to_extrem.sql diff --git a/synapse/storage/databases/main/schema/delta/36/readd_public_rooms.sql b/synapse/storage/schema/main/delta/36/readd_public_rooms.sql similarity index 100% rename from synapse/storage/databases/main/schema/delta/36/readd_public_rooms.sql rename to synapse/storage/schema/main/delta/36/readd_public_rooms.sql diff --git a/synapse/storage/databases/main/schema/delta/37/remove_auth_idx.py b/synapse/storage/schema/main/delta/37/remove_auth_idx.py similarity index 100% rename from synapse/storage/databases/main/schema/delta/37/remove_auth_idx.py rename to synapse/storage/schema/main/delta/37/remove_auth_idx.py diff --git a/synapse/storage/databases/main/schema/delta/37/user_threepids.sql b/synapse/storage/schema/main/delta/37/user_threepids.sql similarity index 100% rename from synapse/storage/databases/main/schema/delta/37/user_threepids.sql rename to synapse/storage/schema/main/delta/37/user_threepids.sql diff --git a/synapse/storage/databases/main/schema/delta/38/postgres_fts_gist.sql b/synapse/storage/schema/main/delta/38/postgres_fts_gist.sql similarity index 100% rename from synapse/storage/databases/main/schema/delta/38/postgres_fts_gist.sql rename to synapse/storage/schema/main/delta/38/postgres_fts_gist.sql diff --git a/synapse/storage/databases/main/schema/delta/39/appservice_room_list.sql b/synapse/storage/schema/main/delta/39/appservice_room_list.sql similarity index 100% rename from synapse/storage/databases/main/schema/delta/39/appservice_room_list.sql rename to synapse/storage/schema/main/delta/39/appservice_room_list.sql diff --git a/synapse/storage/databases/main/schema/delta/39/device_federation_stream_idx.sql b/synapse/storage/schema/main/delta/39/device_federation_stream_idx.sql similarity index 100% rename from synapse/storage/databases/main/schema/delta/39/device_federation_stream_idx.sql rename to synapse/storage/schema/main/delta/39/device_federation_stream_idx.sql diff --git a/synapse/storage/databases/main/schema/delta/39/event_push_index.sql b/synapse/storage/schema/main/delta/39/event_push_index.sql similarity index 100% rename from synapse/storage/databases/main/schema/delta/39/event_push_index.sql rename to synapse/storage/schema/main/delta/39/event_push_index.sql diff --git a/synapse/storage/databases/main/schema/delta/39/federation_out_position.sql b/synapse/storage/schema/main/delta/39/federation_out_position.sql similarity index 100% rename from synapse/storage/databases/main/schema/delta/39/federation_out_position.sql rename to synapse/storage/schema/main/delta/39/federation_out_position.sql diff --git a/synapse/storage/databases/main/schema/delta/39/membership_profile.sql b/synapse/storage/schema/main/delta/39/membership_profile.sql similarity index 100% rename from synapse/storage/databases/main/schema/delta/39/membership_profile.sql rename to synapse/storage/schema/main/delta/39/membership_profile.sql diff --git a/synapse/storage/databases/main/schema/delta/40/current_state_idx.sql b/synapse/storage/schema/main/delta/40/current_state_idx.sql similarity index 100% rename from synapse/storage/databases/main/schema/delta/40/current_state_idx.sql rename to synapse/storage/schema/main/delta/40/current_state_idx.sql diff --git a/synapse/storage/databases/main/schema/delta/40/device_inbox.sql b/synapse/storage/schema/main/delta/40/device_inbox.sql similarity index 100% rename from synapse/storage/databases/main/schema/delta/40/device_inbox.sql rename to synapse/storage/schema/main/delta/40/device_inbox.sql diff --git a/synapse/storage/databases/main/schema/delta/40/device_list_streams.sql b/synapse/storage/schema/main/delta/40/device_list_streams.sql similarity index 100% rename from synapse/storage/databases/main/schema/delta/40/device_list_streams.sql rename to synapse/storage/schema/main/delta/40/device_list_streams.sql diff --git a/synapse/storage/databases/main/schema/delta/40/event_push_summary.sql b/synapse/storage/schema/main/delta/40/event_push_summary.sql similarity index 100% rename from synapse/storage/databases/main/schema/delta/40/event_push_summary.sql rename to synapse/storage/schema/main/delta/40/event_push_summary.sql diff --git a/synapse/storage/databases/main/schema/delta/40/pushers.sql b/synapse/storage/schema/main/delta/40/pushers.sql similarity index 100% rename from synapse/storage/databases/main/schema/delta/40/pushers.sql rename to synapse/storage/schema/main/delta/40/pushers.sql diff --git a/synapse/storage/databases/main/schema/delta/41/device_list_stream_idx.sql b/synapse/storage/schema/main/delta/41/device_list_stream_idx.sql similarity index 100% rename from synapse/storage/databases/main/schema/delta/41/device_list_stream_idx.sql rename to synapse/storage/schema/main/delta/41/device_list_stream_idx.sql diff --git a/synapse/storage/databases/main/schema/delta/41/device_outbound_index.sql b/synapse/storage/schema/main/delta/41/device_outbound_index.sql similarity index 100% rename from synapse/storage/databases/main/schema/delta/41/device_outbound_index.sql rename to synapse/storage/schema/main/delta/41/device_outbound_index.sql diff --git a/synapse/storage/databases/main/schema/delta/41/event_search_event_id_idx.sql b/synapse/storage/schema/main/delta/41/event_search_event_id_idx.sql similarity index 100% rename from synapse/storage/databases/main/schema/delta/41/event_search_event_id_idx.sql rename to synapse/storage/schema/main/delta/41/event_search_event_id_idx.sql diff --git a/synapse/storage/databases/main/schema/delta/41/ratelimit.sql b/synapse/storage/schema/main/delta/41/ratelimit.sql similarity index 100% rename from synapse/storage/databases/main/schema/delta/41/ratelimit.sql rename to synapse/storage/schema/main/delta/41/ratelimit.sql diff --git a/synapse/storage/databases/main/schema/delta/42/current_state_delta.sql b/synapse/storage/schema/main/delta/42/current_state_delta.sql similarity index 100% rename from synapse/storage/databases/main/schema/delta/42/current_state_delta.sql rename to synapse/storage/schema/main/delta/42/current_state_delta.sql diff --git a/synapse/storage/databases/main/schema/delta/42/device_list_last_id.sql b/synapse/storage/schema/main/delta/42/device_list_last_id.sql similarity index 100% rename from synapse/storage/databases/main/schema/delta/42/device_list_last_id.sql rename to synapse/storage/schema/main/delta/42/device_list_last_id.sql diff --git a/synapse/storage/databases/main/schema/delta/42/event_auth_state_only.sql b/synapse/storage/schema/main/delta/42/event_auth_state_only.sql similarity index 100% rename from synapse/storage/databases/main/schema/delta/42/event_auth_state_only.sql rename to synapse/storage/schema/main/delta/42/event_auth_state_only.sql diff --git a/synapse/storage/databases/main/schema/delta/42/user_dir.py b/synapse/storage/schema/main/delta/42/user_dir.py similarity index 100% rename from synapse/storage/databases/main/schema/delta/42/user_dir.py rename to synapse/storage/schema/main/delta/42/user_dir.py diff --git a/synapse/storage/databases/main/schema/delta/43/blocked_rooms.sql b/synapse/storage/schema/main/delta/43/blocked_rooms.sql similarity index 100% rename from synapse/storage/databases/main/schema/delta/43/blocked_rooms.sql rename to synapse/storage/schema/main/delta/43/blocked_rooms.sql diff --git a/synapse/storage/databases/main/schema/delta/43/quarantine_media.sql b/synapse/storage/schema/main/delta/43/quarantine_media.sql similarity index 100% rename from synapse/storage/databases/main/schema/delta/43/quarantine_media.sql rename to synapse/storage/schema/main/delta/43/quarantine_media.sql diff --git a/synapse/storage/databases/main/schema/delta/43/url_cache.sql b/synapse/storage/schema/main/delta/43/url_cache.sql similarity index 100% rename from synapse/storage/databases/main/schema/delta/43/url_cache.sql rename to synapse/storage/schema/main/delta/43/url_cache.sql diff --git a/synapse/storage/databases/main/schema/delta/43/user_share.sql b/synapse/storage/schema/main/delta/43/user_share.sql similarity index 100% rename from synapse/storage/databases/main/schema/delta/43/user_share.sql rename to synapse/storage/schema/main/delta/43/user_share.sql diff --git a/synapse/storage/databases/main/schema/delta/44/expire_url_cache.sql b/synapse/storage/schema/main/delta/44/expire_url_cache.sql similarity index 100% rename from synapse/storage/databases/main/schema/delta/44/expire_url_cache.sql rename to synapse/storage/schema/main/delta/44/expire_url_cache.sql diff --git a/synapse/storage/databases/main/schema/delta/45/group_server.sql b/synapse/storage/schema/main/delta/45/group_server.sql similarity index 100% rename from synapse/storage/databases/main/schema/delta/45/group_server.sql rename to synapse/storage/schema/main/delta/45/group_server.sql diff --git a/synapse/storage/databases/main/schema/delta/45/profile_cache.sql b/synapse/storage/schema/main/delta/45/profile_cache.sql similarity index 100% rename from synapse/storage/databases/main/schema/delta/45/profile_cache.sql rename to synapse/storage/schema/main/delta/45/profile_cache.sql diff --git a/synapse/storage/databases/main/schema/delta/46/drop_refresh_tokens.sql b/synapse/storage/schema/main/delta/46/drop_refresh_tokens.sql similarity index 100% rename from synapse/storage/databases/main/schema/delta/46/drop_refresh_tokens.sql rename to synapse/storage/schema/main/delta/46/drop_refresh_tokens.sql diff --git a/synapse/storage/databases/main/schema/delta/46/drop_unique_deleted_pushers.sql b/synapse/storage/schema/main/delta/46/drop_unique_deleted_pushers.sql similarity index 100% rename from synapse/storage/databases/main/schema/delta/46/drop_unique_deleted_pushers.sql rename to synapse/storage/schema/main/delta/46/drop_unique_deleted_pushers.sql diff --git a/synapse/storage/databases/main/schema/delta/46/group_server.sql b/synapse/storage/schema/main/delta/46/group_server.sql similarity index 100% rename from synapse/storage/databases/main/schema/delta/46/group_server.sql rename to synapse/storage/schema/main/delta/46/group_server.sql diff --git a/synapse/storage/databases/main/schema/delta/46/local_media_repository_url_idx.sql b/synapse/storage/schema/main/delta/46/local_media_repository_url_idx.sql similarity index 100% rename from synapse/storage/databases/main/schema/delta/46/local_media_repository_url_idx.sql rename to synapse/storage/schema/main/delta/46/local_media_repository_url_idx.sql diff --git a/synapse/storage/databases/main/schema/delta/46/user_dir_null_room_ids.sql b/synapse/storage/schema/main/delta/46/user_dir_null_room_ids.sql similarity index 100% rename from synapse/storage/databases/main/schema/delta/46/user_dir_null_room_ids.sql rename to synapse/storage/schema/main/delta/46/user_dir_null_room_ids.sql diff --git a/synapse/storage/databases/main/schema/delta/46/user_dir_typos.sql b/synapse/storage/schema/main/delta/46/user_dir_typos.sql similarity index 100% rename from synapse/storage/databases/main/schema/delta/46/user_dir_typos.sql rename to synapse/storage/schema/main/delta/46/user_dir_typos.sql diff --git a/synapse/storage/databases/main/schema/delta/47/last_access_media.sql b/synapse/storage/schema/main/delta/47/last_access_media.sql similarity index 100% rename from synapse/storage/databases/main/schema/delta/47/last_access_media.sql rename to synapse/storage/schema/main/delta/47/last_access_media.sql diff --git a/synapse/storage/databases/main/schema/delta/47/postgres_fts_gin.sql b/synapse/storage/schema/main/delta/47/postgres_fts_gin.sql similarity index 100% rename from synapse/storage/databases/main/schema/delta/47/postgres_fts_gin.sql rename to synapse/storage/schema/main/delta/47/postgres_fts_gin.sql diff --git a/synapse/storage/databases/main/schema/delta/47/push_actions_staging.sql b/synapse/storage/schema/main/delta/47/push_actions_staging.sql similarity index 100% rename from synapse/storage/databases/main/schema/delta/47/push_actions_staging.sql rename to synapse/storage/schema/main/delta/47/push_actions_staging.sql diff --git a/synapse/storage/databases/main/schema/delta/48/add_user_consent.sql b/synapse/storage/schema/main/delta/48/add_user_consent.sql similarity index 100% rename from synapse/storage/databases/main/schema/delta/48/add_user_consent.sql rename to synapse/storage/schema/main/delta/48/add_user_consent.sql diff --git a/synapse/storage/databases/main/schema/delta/48/add_user_ips_last_seen_index.sql b/synapse/storage/schema/main/delta/48/add_user_ips_last_seen_index.sql similarity index 100% rename from synapse/storage/databases/main/schema/delta/48/add_user_ips_last_seen_index.sql rename to synapse/storage/schema/main/delta/48/add_user_ips_last_seen_index.sql diff --git a/synapse/storage/databases/main/schema/delta/48/deactivated_users.sql b/synapse/storage/schema/main/delta/48/deactivated_users.sql similarity index 100% rename from synapse/storage/databases/main/schema/delta/48/deactivated_users.sql rename to synapse/storage/schema/main/delta/48/deactivated_users.sql diff --git a/synapse/storage/databases/main/schema/delta/48/group_unique_indexes.py b/synapse/storage/schema/main/delta/48/group_unique_indexes.py similarity index 100% rename from synapse/storage/databases/main/schema/delta/48/group_unique_indexes.py rename to synapse/storage/schema/main/delta/48/group_unique_indexes.py diff --git a/synapse/storage/databases/main/schema/delta/48/groups_joinable.sql b/synapse/storage/schema/main/delta/48/groups_joinable.sql similarity index 100% rename from synapse/storage/databases/main/schema/delta/48/groups_joinable.sql rename to synapse/storage/schema/main/delta/48/groups_joinable.sql diff --git a/synapse/storage/databases/main/schema/delta/49/add_user_consent_server_notice_sent.sql b/synapse/storage/schema/main/delta/49/add_user_consent_server_notice_sent.sql similarity index 100% rename from synapse/storage/databases/main/schema/delta/49/add_user_consent_server_notice_sent.sql rename to synapse/storage/schema/main/delta/49/add_user_consent_server_notice_sent.sql diff --git a/synapse/storage/databases/main/schema/delta/49/add_user_daily_visits.sql b/synapse/storage/schema/main/delta/49/add_user_daily_visits.sql similarity index 100% rename from synapse/storage/databases/main/schema/delta/49/add_user_daily_visits.sql rename to synapse/storage/schema/main/delta/49/add_user_daily_visits.sql diff --git a/synapse/storage/databases/main/schema/delta/49/add_user_ips_last_seen_only_index.sql b/synapse/storage/schema/main/delta/49/add_user_ips_last_seen_only_index.sql similarity index 100% rename from synapse/storage/databases/main/schema/delta/49/add_user_ips_last_seen_only_index.sql rename to synapse/storage/schema/main/delta/49/add_user_ips_last_seen_only_index.sql diff --git a/synapse/storage/databases/main/schema/delta/50/add_creation_ts_users_index.sql b/synapse/storage/schema/main/delta/50/add_creation_ts_users_index.sql similarity index 100% rename from synapse/storage/databases/main/schema/delta/50/add_creation_ts_users_index.sql rename to synapse/storage/schema/main/delta/50/add_creation_ts_users_index.sql diff --git a/synapse/storage/databases/main/schema/delta/50/erasure_store.sql b/synapse/storage/schema/main/delta/50/erasure_store.sql similarity index 100% rename from synapse/storage/databases/main/schema/delta/50/erasure_store.sql rename to synapse/storage/schema/main/delta/50/erasure_store.sql diff --git a/synapse/storage/databases/main/schema/delta/50/make_event_content_nullable.py b/synapse/storage/schema/main/delta/50/make_event_content_nullable.py similarity index 100% rename from synapse/storage/databases/main/schema/delta/50/make_event_content_nullable.py rename to synapse/storage/schema/main/delta/50/make_event_content_nullable.py diff --git a/synapse/storage/databases/main/schema/delta/51/e2e_room_keys.sql b/synapse/storage/schema/main/delta/51/e2e_room_keys.sql similarity index 100% rename from synapse/storage/databases/main/schema/delta/51/e2e_room_keys.sql rename to synapse/storage/schema/main/delta/51/e2e_room_keys.sql diff --git a/synapse/storage/databases/main/schema/delta/51/monthly_active_users.sql b/synapse/storage/schema/main/delta/51/monthly_active_users.sql similarity index 100% rename from synapse/storage/databases/main/schema/delta/51/monthly_active_users.sql rename to synapse/storage/schema/main/delta/51/monthly_active_users.sql diff --git a/synapse/storage/databases/main/schema/delta/52/add_event_to_state_group_index.sql b/synapse/storage/schema/main/delta/52/add_event_to_state_group_index.sql similarity index 100% rename from synapse/storage/databases/main/schema/delta/52/add_event_to_state_group_index.sql rename to synapse/storage/schema/main/delta/52/add_event_to_state_group_index.sql diff --git a/synapse/storage/databases/main/schema/delta/52/device_list_streams_unique_idx.sql b/synapse/storage/schema/main/delta/52/device_list_streams_unique_idx.sql similarity index 100% rename from synapse/storage/databases/main/schema/delta/52/device_list_streams_unique_idx.sql rename to synapse/storage/schema/main/delta/52/device_list_streams_unique_idx.sql diff --git a/synapse/storage/databases/main/schema/delta/52/e2e_room_keys.sql b/synapse/storage/schema/main/delta/52/e2e_room_keys.sql similarity index 100% rename from synapse/storage/databases/main/schema/delta/52/e2e_room_keys.sql rename to synapse/storage/schema/main/delta/52/e2e_room_keys.sql diff --git a/synapse/storage/databases/main/schema/delta/53/add_user_type_to_users.sql b/synapse/storage/schema/main/delta/53/add_user_type_to_users.sql similarity index 100% rename from synapse/storage/databases/main/schema/delta/53/add_user_type_to_users.sql rename to synapse/storage/schema/main/delta/53/add_user_type_to_users.sql diff --git a/synapse/storage/databases/main/schema/delta/53/drop_sent_transactions.sql b/synapse/storage/schema/main/delta/53/drop_sent_transactions.sql similarity index 100% rename from synapse/storage/databases/main/schema/delta/53/drop_sent_transactions.sql rename to synapse/storage/schema/main/delta/53/drop_sent_transactions.sql diff --git a/synapse/storage/databases/main/schema/delta/53/event_format_version.sql b/synapse/storage/schema/main/delta/53/event_format_version.sql similarity index 100% rename from synapse/storage/databases/main/schema/delta/53/event_format_version.sql rename to synapse/storage/schema/main/delta/53/event_format_version.sql diff --git a/synapse/storage/databases/main/schema/delta/53/user_dir_populate.sql b/synapse/storage/schema/main/delta/53/user_dir_populate.sql similarity index 100% rename from synapse/storage/databases/main/schema/delta/53/user_dir_populate.sql rename to synapse/storage/schema/main/delta/53/user_dir_populate.sql diff --git a/synapse/storage/databases/main/schema/delta/53/user_ips_index.sql b/synapse/storage/schema/main/delta/53/user_ips_index.sql similarity index 100% rename from synapse/storage/databases/main/schema/delta/53/user_ips_index.sql rename to synapse/storage/schema/main/delta/53/user_ips_index.sql diff --git a/synapse/storage/databases/main/schema/delta/53/user_share.sql b/synapse/storage/schema/main/delta/53/user_share.sql similarity index 100% rename from synapse/storage/databases/main/schema/delta/53/user_share.sql rename to synapse/storage/schema/main/delta/53/user_share.sql diff --git a/synapse/storage/databases/main/schema/delta/53/user_threepid_id.sql b/synapse/storage/schema/main/delta/53/user_threepid_id.sql similarity index 100% rename from synapse/storage/databases/main/schema/delta/53/user_threepid_id.sql rename to synapse/storage/schema/main/delta/53/user_threepid_id.sql diff --git a/synapse/storage/databases/main/schema/delta/53/users_in_public_rooms.sql b/synapse/storage/schema/main/delta/53/users_in_public_rooms.sql similarity index 100% rename from synapse/storage/databases/main/schema/delta/53/users_in_public_rooms.sql rename to synapse/storage/schema/main/delta/53/users_in_public_rooms.sql diff --git a/synapse/storage/databases/main/schema/delta/54/account_validity_with_renewal.sql b/synapse/storage/schema/main/delta/54/account_validity_with_renewal.sql similarity index 100% rename from synapse/storage/databases/main/schema/delta/54/account_validity_with_renewal.sql rename to synapse/storage/schema/main/delta/54/account_validity_with_renewal.sql diff --git a/synapse/storage/databases/main/schema/delta/54/add_validity_to_server_keys.sql b/synapse/storage/schema/main/delta/54/add_validity_to_server_keys.sql similarity index 100% rename from synapse/storage/databases/main/schema/delta/54/add_validity_to_server_keys.sql rename to synapse/storage/schema/main/delta/54/add_validity_to_server_keys.sql diff --git a/synapse/storage/databases/main/schema/delta/54/delete_forward_extremities.sql b/synapse/storage/schema/main/delta/54/delete_forward_extremities.sql similarity index 100% rename from synapse/storage/databases/main/schema/delta/54/delete_forward_extremities.sql rename to synapse/storage/schema/main/delta/54/delete_forward_extremities.sql diff --git a/synapse/storage/databases/main/schema/delta/54/drop_legacy_tables.sql b/synapse/storage/schema/main/delta/54/drop_legacy_tables.sql similarity index 100% rename from synapse/storage/databases/main/schema/delta/54/drop_legacy_tables.sql rename to synapse/storage/schema/main/delta/54/drop_legacy_tables.sql diff --git a/synapse/storage/databases/main/schema/delta/54/drop_presence_list.sql b/synapse/storage/schema/main/delta/54/drop_presence_list.sql similarity index 100% rename from synapse/storage/databases/main/schema/delta/54/drop_presence_list.sql rename to synapse/storage/schema/main/delta/54/drop_presence_list.sql diff --git a/synapse/storage/databases/main/schema/delta/54/relations.sql b/synapse/storage/schema/main/delta/54/relations.sql similarity index 100% rename from synapse/storage/databases/main/schema/delta/54/relations.sql rename to synapse/storage/schema/main/delta/54/relations.sql diff --git a/synapse/storage/databases/main/schema/delta/54/stats.sql b/synapse/storage/schema/main/delta/54/stats.sql similarity index 100% rename from synapse/storage/databases/main/schema/delta/54/stats.sql rename to synapse/storage/schema/main/delta/54/stats.sql diff --git a/synapse/storage/databases/main/schema/delta/54/stats2.sql b/synapse/storage/schema/main/delta/54/stats2.sql similarity index 100% rename from synapse/storage/databases/main/schema/delta/54/stats2.sql rename to synapse/storage/schema/main/delta/54/stats2.sql diff --git a/synapse/storage/databases/main/schema/delta/55/access_token_expiry.sql b/synapse/storage/schema/main/delta/55/access_token_expiry.sql similarity index 100% rename from synapse/storage/databases/main/schema/delta/55/access_token_expiry.sql rename to synapse/storage/schema/main/delta/55/access_token_expiry.sql diff --git a/synapse/storage/databases/main/schema/delta/55/track_threepid_validations.sql b/synapse/storage/schema/main/delta/55/track_threepid_validations.sql similarity index 100% rename from synapse/storage/databases/main/schema/delta/55/track_threepid_validations.sql rename to synapse/storage/schema/main/delta/55/track_threepid_validations.sql diff --git a/synapse/storage/databases/main/schema/delta/55/users_alter_deactivated.sql b/synapse/storage/schema/main/delta/55/users_alter_deactivated.sql similarity index 100% rename from synapse/storage/databases/main/schema/delta/55/users_alter_deactivated.sql rename to synapse/storage/schema/main/delta/55/users_alter_deactivated.sql diff --git a/synapse/storage/databases/main/schema/delta/56/add_spans_to_device_lists.sql b/synapse/storage/schema/main/delta/56/add_spans_to_device_lists.sql similarity index 100% rename from synapse/storage/databases/main/schema/delta/56/add_spans_to_device_lists.sql rename to synapse/storage/schema/main/delta/56/add_spans_to_device_lists.sql diff --git a/synapse/storage/databases/main/schema/delta/56/current_state_events_membership.sql b/synapse/storage/schema/main/delta/56/current_state_events_membership.sql similarity index 100% rename from synapse/storage/databases/main/schema/delta/56/current_state_events_membership.sql rename to synapse/storage/schema/main/delta/56/current_state_events_membership.sql diff --git a/synapse/storage/databases/main/schema/delta/56/current_state_events_membership_mk2.sql b/synapse/storage/schema/main/delta/56/current_state_events_membership_mk2.sql similarity index 100% rename from synapse/storage/databases/main/schema/delta/56/current_state_events_membership_mk2.sql rename to synapse/storage/schema/main/delta/56/current_state_events_membership_mk2.sql diff --git a/synapse/storage/databases/main/schema/delta/56/delete_keys_from_deleted_backups.sql b/synapse/storage/schema/main/delta/56/delete_keys_from_deleted_backups.sql similarity index 100% rename from synapse/storage/databases/main/schema/delta/56/delete_keys_from_deleted_backups.sql rename to synapse/storage/schema/main/delta/56/delete_keys_from_deleted_backups.sql diff --git a/synapse/storage/databases/main/schema/delta/56/destinations_failure_ts.sql b/synapse/storage/schema/main/delta/56/destinations_failure_ts.sql similarity index 100% rename from synapse/storage/databases/main/schema/delta/56/destinations_failure_ts.sql rename to synapse/storage/schema/main/delta/56/destinations_failure_ts.sql diff --git a/synapse/storage/databases/main/schema/delta/56/destinations_retry_interval_type.sql.postgres b/synapse/storage/schema/main/delta/56/destinations_retry_interval_type.sql.postgres similarity index 100% rename from synapse/storage/databases/main/schema/delta/56/destinations_retry_interval_type.sql.postgres rename to synapse/storage/schema/main/delta/56/destinations_retry_interval_type.sql.postgres diff --git a/synapse/storage/databases/main/schema/delta/56/device_stream_id_insert.sql b/synapse/storage/schema/main/delta/56/device_stream_id_insert.sql similarity index 100% rename from synapse/storage/databases/main/schema/delta/56/device_stream_id_insert.sql rename to synapse/storage/schema/main/delta/56/device_stream_id_insert.sql diff --git a/synapse/storage/databases/main/schema/delta/56/devices_last_seen.sql b/synapse/storage/schema/main/delta/56/devices_last_seen.sql similarity index 100% rename from synapse/storage/databases/main/schema/delta/56/devices_last_seen.sql rename to synapse/storage/schema/main/delta/56/devices_last_seen.sql diff --git a/synapse/storage/databases/main/schema/delta/56/drop_unused_event_tables.sql b/synapse/storage/schema/main/delta/56/drop_unused_event_tables.sql similarity index 100% rename from synapse/storage/databases/main/schema/delta/56/drop_unused_event_tables.sql rename to synapse/storage/schema/main/delta/56/drop_unused_event_tables.sql diff --git a/synapse/storage/databases/main/schema/delta/56/event_expiry.sql b/synapse/storage/schema/main/delta/56/event_expiry.sql similarity index 100% rename from synapse/storage/databases/main/schema/delta/56/event_expiry.sql rename to synapse/storage/schema/main/delta/56/event_expiry.sql diff --git a/synapse/storage/databases/main/schema/delta/56/event_labels.sql b/synapse/storage/schema/main/delta/56/event_labels.sql similarity index 100% rename from synapse/storage/databases/main/schema/delta/56/event_labels.sql rename to synapse/storage/schema/main/delta/56/event_labels.sql diff --git a/synapse/storage/databases/main/schema/delta/56/event_labels_background_update.sql b/synapse/storage/schema/main/delta/56/event_labels_background_update.sql similarity index 100% rename from synapse/storage/databases/main/schema/delta/56/event_labels_background_update.sql rename to synapse/storage/schema/main/delta/56/event_labels_background_update.sql diff --git a/synapse/storage/databases/main/schema/delta/56/fix_room_keys_index.sql b/synapse/storage/schema/main/delta/56/fix_room_keys_index.sql similarity index 100% rename from synapse/storage/databases/main/schema/delta/56/fix_room_keys_index.sql rename to synapse/storage/schema/main/delta/56/fix_room_keys_index.sql diff --git a/synapse/storage/databases/main/schema/delta/56/hidden_devices.sql b/synapse/storage/schema/main/delta/56/hidden_devices.sql similarity index 100% rename from synapse/storage/databases/main/schema/delta/56/hidden_devices.sql rename to synapse/storage/schema/main/delta/56/hidden_devices.sql diff --git a/synapse/storage/databases/main/schema/delta/56/hidden_devices_fix.sql.sqlite b/synapse/storage/schema/main/delta/56/hidden_devices_fix.sql.sqlite similarity index 100% rename from synapse/storage/databases/main/schema/delta/56/hidden_devices_fix.sql.sqlite rename to synapse/storage/schema/main/delta/56/hidden_devices_fix.sql.sqlite diff --git a/synapse/storage/databases/main/schema/delta/56/nuke_empty_communities_from_db.sql b/synapse/storage/schema/main/delta/56/nuke_empty_communities_from_db.sql similarity index 100% rename from synapse/storage/databases/main/schema/delta/56/nuke_empty_communities_from_db.sql rename to synapse/storage/schema/main/delta/56/nuke_empty_communities_from_db.sql diff --git a/synapse/storage/databases/main/schema/delta/56/public_room_list_idx.sql b/synapse/storage/schema/main/delta/56/public_room_list_idx.sql similarity index 100% rename from synapse/storage/databases/main/schema/delta/56/public_room_list_idx.sql rename to synapse/storage/schema/main/delta/56/public_room_list_idx.sql diff --git a/synapse/storage/databases/main/schema/delta/56/redaction_censor.sql b/synapse/storage/schema/main/delta/56/redaction_censor.sql similarity index 100% rename from synapse/storage/databases/main/schema/delta/56/redaction_censor.sql rename to synapse/storage/schema/main/delta/56/redaction_censor.sql diff --git a/synapse/storage/databases/main/schema/delta/56/redaction_censor2.sql b/synapse/storage/schema/main/delta/56/redaction_censor2.sql similarity index 100% rename from synapse/storage/databases/main/schema/delta/56/redaction_censor2.sql rename to synapse/storage/schema/main/delta/56/redaction_censor2.sql diff --git a/synapse/storage/databases/main/schema/delta/56/redaction_censor3_fix_update.sql.postgres b/synapse/storage/schema/main/delta/56/redaction_censor3_fix_update.sql.postgres similarity index 100% rename from synapse/storage/databases/main/schema/delta/56/redaction_censor3_fix_update.sql.postgres rename to synapse/storage/schema/main/delta/56/redaction_censor3_fix_update.sql.postgres diff --git a/synapse/storage/databases/main/schema/delta/56/redaction_censor4.sql b/synapse/storage/schema/main/delta/56/redaction_censor4.sql similarity index 100% rename from synapse/storage/databases/main/schema/delta/56/redaction_censor4.sql rename to synapse/storage/schema/main/delta/56/redaction_censor4.sql diff --git a/synapse/storage/databases/main/schema/delta/56/remove_tombstoned_rooms_from_directory.sql b/synapse/storage/schema/main/delta/56/remove_tombstoned_rooms_from_directory.sql similarity index 100% rename from synapse/storage/databases/main/schema/delta/56/remove_tombstoned_rooms_from_directory.sql rename to synapse/storage/schema/main/delta/56/remove_tombstoned_rooms_from_directory.sql diff --git a/synapse/storage/databases/main/schema/delta/56/room_key_etag.sql b/synapse/storage/schema/main/delta/56/room_key_etag.sql similarity index 100% rename from synapse/storage/databases/main/schema/delta/56/room_key_etag.sql rename to synapse/storage/schema/main/delta/56/room_key_etag.sql diff --git a/synapse/storage/databases/main/schema/delta/56/room_membership_idx.sql b/synapse/storage/schema/main/delta/56/room_membership_idx.sql similarity index 100% rename from synapse/storage/databases/main/schema/delta/56/room_membership_idx.sql rename to synapse/storage/schema/main/delta/56/room_membership_idx.sql diff --git a/synapse/storage/databases/main/schema/delta/56/room_retention.sql b/synapse/storage/schema/main/delta/56/room_retention.sql similarity index 100% rename from synapse/storage/databases/main/schema/delta/56/room_retention.sql rename to synapse/storage/schema/main/delta/56/room_retention.sql diff --git a/synapse/storage/databases/main/schema/delta/56/signing_keys.sql b/synapse/storage/schema/main/delta/56/signing_keys.sql similarity index 100% rename from synapse/storage/databases/main/schema/delta/56/signing_keys.sql rename to synapse/storage/schema/main/delta/56/signing_keys.sql diff --git a/synapse/storage/databases/main/schema/delta/56/signing_keys_nonunique_signatures.sql b/synapse/storage/schema/main/delta/56/signing_keys_nonunique_signatures.sql similarity index 100% rename from synapse/storage/databases/main/schema/delta/56/signing_keys_nonunique_signatures.sql rename to synapse/storage/schema/main/delta/56/signing_keys_nonunique_signatures.sql diff --git a/synapse/storage/databases/main/schema/delta/56/stats_separated.sql b/synapse/storage/schema/main/delta/56/stats_separated.sql similarity index 100% rename from synapse/storage/databases/main/schema/delta/56/stats_separated.sql rename to synapse/storage/schema/main/delta/56/stats_separated.sql diff --git a/synapse/storage/databases/main/schema/delta/56/unique_user_filter_index.py b/synapse/storage/schema/main/delta/56/unique_user_filter_index.py similarity index 100% rename from synapse/storage/databases/main/schema/delta/56/unique_user_filter_index.py rename to synapse/storage/schema/main/delta/56/unique_user_filter_index.py diff --git a/synapse/storage/databases/main/schema/delta/56/user_external_ids.sql b/synapse/storage/schema/main/delta/56/user_external_ids.sql similarity index 100% rename from synapse/storage/databases/main/schema/delta/56/user_external_ids.sql rename to synapse/storage/schema/main/delta/56/user_external_ids.sql diff --git a/synapse/storage/databases/main/schema/delta/56/users_in_public_rooms_idx.sql b/synapse/storage/schema/main/delta/56/users_in_public_rooms_idx.sql similarity index 100% rename from synapse/storage/databases/main/schema/delta/56/users_in_public_rooms_idx.sql rename to synapse/storage/schema/main/delta/56/users_in_public_rooms_idx.sql diff --git a/synapse/storage/databases/main/schema/delta/57/delete_old_current_state_events.sql b/synapse/storage/schema/main/delta/57/delete_old_current_state_events.sql similarity index 100% rename from synapse/storage/databases/main/schema/delta/57/delete_old_current_state_events.sql rename to synapse/storage/schema/main/delta/57/delete_old_current_state_events.sql diff --git a/synapse/storage/databases/main/schema/delta/57/device_list_remote_cache_stale.sql b/synapse/storage/schema/main/delta/57/device_list_remote_cache_stale.sql similarity index 100% rename from synapse/storage/databases/main/schema/delta/57/device_list_remote_cache_stale.sql rename to synapse/storage/schema/main/delta/57/device_list_remote_cache_stale.sql diff --git a/synapse/storage/databases/main/schema/delta/57/local_current_membership.py b/synapse/storage/schema/main/delta/57/local_current_membership.py similarity index 100% rename from synapse/storage/databases/main/schema/delta/57/local_current_membership.py rename to synapse/storage/schema/main/delta/57/local_current_membership.py diff --git a/synapse/storage/databases/main/schema/delta/57/remove_sent_outbound_pokes.sql b/synapse/storage/schema/main/delta/57/remove_sent_outbound_pokes.sql similarity index 100% rename from synapse/storage/databases/main/schema/delta/57/remove_sent_outbound_pokes.sql rename to synapse/storage/schema/main/delta/57/remove_sent_outbound_pokes.sql diff --git a/synapse/storage/databases/main/schema/delta/57/rooms_version_column.sql b/synapse/storage/schema/main/delta/57/rooms_version_column.sql similarity index 100% rename from synapse/storage/databases/main/schema/delta/57/rooms_version_column.sql rename to synapse/storage/schema/main/delta/57/rooms_version_column.sql diff --git a/synapse/storage/databases/main/schema/delta/57/rooms_version_column_2.sql.postgres b/synapse/storage/schema/main/delta/57/rooms_version_column_2.sql.postgres similarity index 100% rename from synapse/storage/databases/main/schema/delta/57/rooms_version_column_2.sql.postgres rename to synapse/storage/schema/main/delta/57/rooms_version_column_2.sql.postgres diff --git a/synapse/storage/databases/main/schema/delta/57/rooms_version_column_2.sql.sqlite b/synapse/storage/schema/main/delta/57/rooms_version_column_2.sql.sqlite similarity index 100% rename from synapse/storage/databases/main/schema/delta/57/rooms_version_column_2.sql.sqlite rename to synapse/storage/schema/main/delta/57/rooms_version_column_2.sql.sqlite diff --git a/synapse/storage/databases/main/schema/delta/57/rooms_version_column_3.sql.postgres b/synapse/storage/schema/main/delta/57/rooms_version_column_3.sql.postgres similarity index 100% rename from synapse/storage/databases/main/schema/delta/57/rooms_version_column_3.sql.postgres rename to synapse/storage/schema/main/delta/57/rooms_version_column_3.sql.postgres diff --git a/synapse/storage/databases/main/schema/delta/57/rooms_version_column_3.sql.sqlite b/synapse/storage/schema/main/delta/57/rooms_version_column_3.sql.sqlite similarity index 100% rename from synapse/storage/databases/main/schema/delta/57/rooms_version_column_3.sql.sqlite rename to synapse/storage/schema/main/delta/57/rooms_version_column_3.sql.sqlite diff --git a/synapse/storage/databases/main/schema/delta/58/02remove_dup_outbound_pokes.sql b/synapse/storage/schema/main/delta/58/02remove_dup_outbound_pokes.sql similarity index 100% rename from synapse/storage/databases/main/schema/delta/58/02remove_dup_outbound_pokes.sql rename to synapse/storage/schema/main/delta/58/02remove_dup_outbound_pokes.sql diff --git a/synapse/storage/databases/main/schema/delta/58/03persist_ui_auth.sql b/synapse/storage/schema/main/delta/58/03persist_ui_auth.sql similarity index 100% rename from synapse/storage/databases/main/schema/delta/58/03persist_ui_auth.sql rename to synapse/storage/schema/main/delta/58/03persist_ui_auth.sql diff --git a/synapse/storage/databases/main/schema/delta/58/05cache_instance.sql.postgres b/synapse/storage/schema/main/delta/58/05cache_instance.sql.postgres similarity index 100% rename from synapse/storage/databases/main/schema/delta/58/05cache_instance.sql.postgres rename to synapse/storage/schema/main/delta/58/05cache_instance.sql.postgres diff --git a/synapse/storage/databases/main/schema/delta/58/06dlols_unique_idx.py b/synapse/storage/schema/main/delta/58/06dlols_unique_idx.py similarity index 100% rename from synapse/storage/databases/main/schema/delta/58/06dlols_unique_idx.py rename to synapse/storage/schema/main/delta/58/06dlols_unique_idx.py diff --git a/synapse/storage/databases/main/schema/delta/58/07add_method_to_thumbnail_constraint.sql.postgres b/synapse/storage/schema/main/delta/58/07add_method_to_thumbnail_constraint.sql.postgres similarity index 100% rename from synapse/storage/databases/main/schema/delta/58/07add_method_to_thumbnail_constraint.sql.postgres rename to synapse/storage/schema/main/delta/58/07add_method_to_thumbnail_constraint.sql.postgres diff --git a/synapse/storage/databases/main/schema/delta/58/07add_method_to_thumbnail_constraint.sql.sqlite b/synapse/storage/schema/main/delta/58/07add_method_to_thumbnail_constraint.sql.sqlite similarity index 100% rename from synapse/storage/databases/main/schema/delta/58/07add_method_to_thumbnail_constraint.sql.sqlite rename to synapse/storage/schema/main/delta/58/07add_method_to_thumbnail_constraint.sql.sqlite diff --git a/synapse/storage/databases/main/schema/delta/58/07persist_ui_auth_ips.sql b/synapse/storage/schema/main/delta/58/07persist_ui_auth_ips.sql similarity index 100% rename from synapse/storage/databases/main/schema/delta/58/07persist_ui_auth_ips.sql rename to synapse/storage/schema/main/delta/58/07persist_ui_auth_ips.sql diff --git a/synapse/storage/databases/main/schema/delta/58/08_media_safe_from_quarantine.sql.postgres b/synapse/storage/schema/main/delta/58/08_media_safe_from_quarantine.sql.postgres similarity index 100% rename from synapse/storage/databases/main/schema/delta/58/08_media_safe_from_quarantine.sql.postgres rename to synapse/storage/schema/main/delta/58/08_media_safe_from_quarantine.sql.postgres diff --git a/synapse/storage/databases/main/schema/delta/58/08_media_safe_from_quarantine.sql.sqlite b/synapse/storage/schema/main/delta/58/08_media_safe_from_quarantine.sql.sqlite similarity index 100% rename from synapse/storage/databases/main/schema/delta/58/08_media_safe_from_quarantine.sql.sqlite rename to synapse/storage/schema/main/delta/58/08_media_safe_from_quarantine.sql.sqlite diff --git a/synapse/storage/databases/main/schema/delta/58/09shadow_ban.sql b/synapse/storage/schema/main/delta/58/09shadow_ban.sql similarity index 100% rename from synapse/storage/databases/main/schema/delta/58/09shadow_ban.sql rename to synapse/storage/schema/main/delta/58/09shadow_ban.sql diff --git a/synapse/storage/databases/main/schema/delta/58/10_pushrules_enabled_delete_obsolete.sql b/synapse/storage/schema/main/delta/58/10_pushrules_enabled_delete_obsolete.sql similarity index 100% rename from synapse/storage/databases/main/schema/delta/58/10_pushrules_enabled_delete_obsolete.sql rename to synapse/storage/schema/main/delta/58/10_pushrules_enabled_delete_obsolete.sql diff --git a/synapse/storage/databases/main/schema/delta/58/10drop_local_rejections_stream.sql b/synapse/storage/schema/main/delta/58/10drop_local_rejections_stream.sql similarity index 100% rename from synapse/storage/databases/main/schema/delta/58/10drop_local_rejections_stream.sql rename to synapse/storage/schema/main/delta/58/10drop_local_rejections_stream.sql diff --git a/synapse/storage/databases/main/schema/delta/58/10federation_pos_instance_name.sql b/synapse/storage/schema/main/delta/58/10federation_pos_instance_name.sql similarity index 100% rename from synapse/storage/databases/main/schema/delta/58/10federation_pos_instance_name.sql rename to synapse/storage/schema/main/delta/58/10federation_pos_instance_name.sql diff --git a/synapse/storage/databases/main/schema/delta/58/11dehydration.sql b/synapse/storage/schema/main/delta/58/11dehydration.sql similarity index 100% rename from synapse/storage/databases/main/schema/delta/58/11dehydration.sql rename to synapse/storage/schema/main/delta/58/11dehydration.sql diff --git a/synapse/storage/databases/main/schema/delta/58/11fallback.sql b/synapse/storage/schema/main/delta/58/11fallback.sql similarity index 100% rename from synapse/storage/databases/main/schema/delta/58/11fallback.sql rename to synapse/storage/schema/main/delta/58/11fallback.sql diff --git a/synapse/storage/databases/main/schema/delta/58/11user_id_seq.py b/synapse/storage/schema/main/delta/58/11user_id_seq.py similarity index 100% rename from synapse/storage/databases/main/schema/delta/58/11user_id_seq.py rename to synapse/storage/schema/main/delta/58/11user_id_seq.py diff --git a/synapse/storage/databases/main/schema/delta/58/12room_stats.sql b/synapse/storage/schema/main/delta/58/12room_stats.sql similarity index 100% rename from synapse/storage/databases/main/schema/delta/58/12room_stats.sql rename to synapse/storage/schema/main/delta/58/12room_stats.sql diff --git a/synapse/storage/databases/main/schema/delta/58/13remove_presence_allow_inbound.sql b/synapse/storage/schema/main/delta/58/13remove_presence_allow_inbound.sql similarity index 100% rename from synapse/storage/databases/main/schema/delta/58/13remove_presence_allow_inbound.sql rename to synapse/storage/schema/main/delta/58/13remove_presence_allow_inbound.sql diff --git a/synapse/storage/databases/main/schema/delta/58/14events_instance_name.sql b/synapse/storage/schema/main/delta/58/14events_instance_name.sql similarity index 100% rename from synapse/storage/databases/main/schema/delta/58/14events_instance_name.sql rename to synapse/storage/schema/main/delta/58/14events_instance_name.sql diff --git a/synapse/storage/databases/main/schema/delta/58/14events_instance_name.sql.postgres b/synapse/storage/schema/main/delta/58/14events_instance_name.sql.postgres similarity index 100% rename from synapse/storage/databases/main/schema/delta/58/14events_instance_name.sql.postgres rename to synapse/storage/schema/main/delta/58/14events_instance_name.sql.postgres diff --git a/synapse/storage/databases/main/schema/delta/58/15_catchup_destination_rooms.sql b/synapse/storage/schema/main/delta/58/15_catchup_destination_rooms.sql similarity index 100% rename from synapse/storage/databases/main/schema/delta/58/15_catchup_destination_rooms.sql rename to synapse/storage/schema/main/delta/58/15_catchup_destination_rooms.sql diff --git a/synapse/storage/databases/main/schema/delta/58/15unread_count.sql b/synapse/storage/schema/main/delta/58/15unread_count.sql similarity index 100% rename from synapse/storage/databases/main/schema/delta/58/15unread_count.sql rename to synapse/storage/schema/main/delta/58/15unread_count.sql diff --git a/synapse/storage/databases/main/schema/delta/58/16populate_stats_process_rooms_fix.sql b/synapse/storage/schema/main/delta/58/16populate_stats_process_rooms_fix.sql similarity index 100% rename from synapse/storage/databases/main/schema/delta/58/16populate_stats_process_rooms_fix.sql rename to synapse/storage/schema/main/delta/58/16populate_stats_process_rooms_fix.sql diff --git a/synapse/storage/databases/main/schema/delta/58/17_catchup_last_successful.sql b/synapse/storage/schema/main/delta/58/17_catchup_last_successful.sql similarity index 100% rename from synapse/storage/databases/main/schema/delta/58/17_catchup_last_successful.sql rename to synapse/storage/schema/main/delta/58/17_catchup_last_successful.sql diff --git a/synapse/storage/databases/main/schema/delta/58/18stream_positions.sql b/synapse/storage/schema/main/delta/58/18stream_positions.sql similarity index 100% rename from synapse/storage/databases/main/schema/delta/58/18stream_positions.sql rename to synapse/storage/schema/main/delta/58/18stream_positions.sql diff --git a/synapse/storage/databases/main/schema/delta/58/19instance_map.sql.postgres b/synapse/storage/schema/main/delta/58/19instance_map.sql.postgres similarity index 100% rename from synapse/storage/databases/main/schema/delta/58/19instance_map.sql.postgres rename to synapse/storage/schema/main/delta/58/19instance_map.sql.postgres diff --git a/synapse/storage/databases/main/schema/delta/58/19txn_id.sql b/synapse/storage/schema/main/delta/58/19txn_id.sql similarity index 100% rename from synapse/storage/databases/main/schema/delta/58/19txn_id.sql rename to synapse/storage/schema/main/delta/58/19txn_id.sql diff --git a/synapse/storage/databases/main/schema/delta/58/20instance_name_event_tables.sql b/synapse/storage/schema/main/delta/58/20instance_name_event_tables.sql similarity index 100% rename from synapse/storage/databases/main/schema/delta/58/20instance_name_event_tables.sql rename to synapse/storage/schema/main/delta/58/20instance_name_event_tables.sql diff --git a/synapse/storage/databases/main/schema/delta/58/20user_daily_visits.sql b/synapse/storage/schema/main/delta/58/20user_daily_visits.sql similarity index 100% rename from synapse/storage/databases/main/schema/delta/58/20user_daily_visits.sql rename to synapse/storage/schema/main/delta/58/20user_daily_visits.sql diff --git a/synapse/storage/databases/main/schema/delta/58/21as_device_stream.sql b/synapse/storage/schema/main/delta/58/21as_device_stream.sql similarity index 100% rename from synapse/storage/databases/main/schema/delta/58/21as_device_stream.sql rename to synapse/storage/schema/main/delta/58/21as_device_stream.sql diff --git a/synapse/storage/databases/main/schema/delta/58/21drop_device_max_stream_id.sql b/synapse/storage/schema/main/delta/58/21drop_device_max_stream_id.sql similarity index 100% rename from synapse/storage/databases/main/schema/delta/58/21drop_device_max_stream_id.sql rename to synapse/storage/schema/main/delta/58/21drop_device_max_stream_id.sql diff --git a/synapse/storage/databases/main/schema/delta/58/22puppet_token.sql b/synapse/storage/schema/main/delta/58/22puppet_token.sql similarity index 100% rename from synapse/storage/databases/main/schema/delta/58/22puppet_token.sql rename to synapse/storage/schema/main/delta/58/22puppet_token.sql diff --git a/synapse/storage/databases/main/schema/delta/58/22users_have_local_media.sql b/synapse/storage/schema/main/delta/58/22users_have_local_media.sql similarity index 100% rename from synapse/storage/databases/main/schema/delta/58/22users_have_local_media.sql rename to synapse/storage/schema/main/delta/58/22users_have_local_media.sql diff --git a/synapse/storage/databases/main/schema/delta/58/23e2e_cross_signing_keys_idx.sql b/synapse/storage/schema/main/delta/58/23e2e_cross_signing_keys_idx.sql similarity index 100% rename from synapse/storage/databases/main/schema/delta/58/23e2e_cross_signing_keys_idx.sql rename to synapse/storage/schema/main/delta/58/23e2e_cross_signing_keys_idx.sql diff --git a/synapse/storage/databases/main/schema/delta/58/24drop_event_json_index.sql b/synapse/storage/schema/main/delta/58/24drop_event_json_index.sql similarity index 100% rename from synapse/storage/databases/main/schema/delta/58/24drop_event_json_index.sql rename to synapse/storage/schema/main/delta/58/24drop_event_json_index.sql diff --git a/synapse/storage/databases/main/schema/delta/58/25user_external_ids_user_id_idx.sql b/synapse/storage/schema/main/delta/58/25user_external_ids_user_id_idx.sql similarity index 100% rename from synapse/storage/databases/main/schema/delta/58/25user_external_ids_user_id_idx.sql rename to synapse/storage/schema/main/delta/58/25user_external_ids_user_id_idx.sql diff --git a/synapse/storage/databases/main/schema/delta/58/26access_token_last_validated.sql b/synapse/storage/schema/main/delta/58/26access_token_last_validated.sql similarity index 100% rename from synapse/storage/databases/main/schema/delta/58/26access_token_last_validated.sql rename to synapse/storage/schema/main/delta/58/26access_token_last_validated.sql diff --git a/synapse/storage/databases/main/schema/delta/58/27local_invites.sql b/synapse/storage/schema/main/delta/58/27local_invites.sql similarity index 100% rename from synapse/storage/databases/main/schema/delta/58/27local_invites.sql rename to synapse/storage/schema/main/delta/58/27local_invites.sql diff --git a/synapse/storage/databases/main/schema/delta/58/28drop_last_used_column.sql.postgres b/synapse/storage/schema/main/delta/58/28drop_last_used_column.sql.postgres similarity index 100% rename from synapse/storage/databases/main/schema/delta/58/28drop_last_used_column.sql.postgres rename to synapse/storage/schema/main/delta/58/28drop_last_used_column.sql.postgres diff --git a/synapse/storage/databases/main/schema/delta/58/28drop_last_used_column.sql.sqlite b/synapse/storage/schema/main/delta/58/28drop_last_used_column.sql.sqlite similarity index 100% rename from synapse/storage/databases/main/schema/delta/58/28drop_last_used_column.sql.sqlite rename to synapse/storage/schema/main/delta/58/28drop_last_used_column.sql.sqlite diff --git a/synapse/storage/databases/main/schema/delta/59/01ignored_user.py b/synapse/storage/schema/main/delta/59/01ignored_user.py similarity index 100% rename from synapse/storage/databases/main/schema/delta/59/01ignored_user.py rename to synapse/storage/schema/main/delta/59/01ignored_user.py diff --git a/synapse/storage/databases/main/schema/delta/59/02shard_send_to_device.sql b/synapse/storage/schema/main/delta/59/02shard_send_to_device.sql similarity index 100% rename from synapse/storage/databases/main/schema/delta/59/02shard_send_to_device.sql rename to synapse/storage/schema/main/delta/59/02shard_send_to_device.sql diff --git a/synapse/storage/databases/main/schema/delta/59/03shard_send_to_device_sequence.sql.postgres b/synapse/storage/schema/main/delta/59/03shard_send_to_device_sequence.sql.postgres similarity index 100% rename from synapse/storage/databases/main/schema/delta/59/03shard_send_to_device_sequence.sql.postgres rename to synapse/storage/schema/main/delta/59/03shard_send_to_device_sequence.sql.postgres diff --git a/synapse/storage/databases/main/schema/delta/59/04_event_auth_chains.sql b/synapse/storage/schema/main/delta/59/04_event_auth_chains.sql similarity index 100% rename from synapse/storage/databases/main/schema/delta/59/04_event_auth_chains.sql rename to synapse/storage/schema/main/delta/59/04_event_auth_chains.sql diff --git a/synapse/storage/databases/main/schema/delta/59/04_event_auth_chains.sql.postgres b/synapse/storage/schema/main/delta/59/04_event_auth_chains.sql.postgres similarity index 100% rename from synapse/storage/databases/main/schema/delta/59/04_event_auth_chains.sql.postgres rename to synapse/storage/schema/main/delta/59/04_event_auth_chains.sql.postgres diff --git a/synapse/storage/databases/main/schema/delta/59/04drop_account_data.sql b/synapse/storage/schema/main/delta/59/04drop_account_data.sql similarity index 100% rename from synapse/storage/databases/main/schema/delta/59/04drop_account_data.sql rename to synapse/storage/schema/main/delta/59/04drop_account_data.sql diff --git a/synapse/storage/databases/main/schema/delta/59/05cache_invalidation.sql b/synapse/storage/schema/main/delta/59/05cache_invalidation.sql similarity index 100% rename from synapse/storage/databases/main/schema/delta/59/05cache_invalidation.sql rename to synapse/storage/schema/main/delta/59/05cache_invalidation.sql diff --git a/synapse/storage/databases/main/schema/delta/59/06chain_cover_index.sql b/synapse/storage/schema/main/delta/59/06chain_cover_index.sql similarity index 100% rename from synapse/storage/databases/main/schema/delta/59/06chain_cover_index.sql rename to synapse/storage/schema/main/delta/59/06chain_cover_index.sql diff --git a/synapse/storage/databases/main/schema/delta/59/06shard_account_data.sql b/synapse/storage/schema/main/delta/59/06shard_account_data.sql similarity index 100% rename from synapse/storage/databases/main/schema/delta/59/06shard_account_data.sql rename to synapse/storage/schema/main/delta/59/06shard_account_data.sql diff --git a/synapse/storage/databases/main/schema/delta/59/06shard_account_data.sql.postgres b/synapse/storage/schema/main/delta/59/06shard_account_data.sql.postgres similarity index 100% rename from synapse/storage/databases/main/schema/delta/59/06shard_account_data.sql.postgres rename to synapse/storage/schema/main/delta/59/06shard_account_data.sql.postgres diff --git a/synapse/storage/databases/main/schema/delta/59/07shard_account_data_fix.sql b/synapse/storage/schema/main/delta/59/07shard_account_data_fix.sql similarity index 100% rename from synapse/storage/databases/main/schema/delta/59/07shard_account_data_fix.sql rename to synapse/storage/schema/main/delta/59/07shard_account_data_fix.sql diff --git a/synapse/storage/databases/main/schema/delta/59/08delete_pushers_for_deactivated_accounts.sql b/synapse/storage/schema/main/delta/59/08delete_pushers_for_deactivated_accounts.sql similarity index 100% rename from synapse/storage/databases/main/schema/delta/59/08delete_pushers_for_deactivated_accounts.sql rename to synapse/storage/schema/main/delta/59/08delete_pushers_for_deactivated_accounts.sql diff --git a/synapse/storage/databases/main/schema/delta/59/08delete_stale_pushers.sql b/synapse/storage/schema/main/delta/59/08delete_stale_pushers.sql similarity index 100% rename from synapse/storage/databases/main/schema/delta/59/08delete_stale_pushers.sql rename to synapse/storage/schema/main/delta/59/08delete_stale_pushers.sql diff --git a/synapse/storage/databases/main/schema/delta/59/09rejected_events_metadata.sql b/synapse/storage/schema/main/delta/59/09rejected_events_metadata.sql similarity index 100% rename from synapse/storage/databases/main/schema/delta/59/09rejected_events_metadata.sql rename to synapse/storage/schema/main/delta/59/09rejected_events_metadata.sql diff --git a/synapse/storage/databases/main/schema/delta/59/10delete_purged_chain_cover.sql b/synapse/storage/schema/main/delta/59/10delete_purged_chain_cover.sql similarity index 100% rename from synapse/storage/databases/main/schema/delta/59/10delete_purged_chain_cover.sql rename to synapse/storage/schema/main/delta/59/10delete_purged_chain_cover.sql diff --git a/synapse/storage/databases/main/schema/delta/59/11drop_thumbnail_constraint.sql.postgres b/synapse/storage/schema/main/delta/59/11drop_thumbnail_constraint.sql.postgres similarity index 100% rename from synapse/storage/databases/main/schema/delta/59/11drop_thumbnail_constraint.sql.postgres rename to synapse/storage/schema/main/delta/59/11drop_thumbnail_constraint.sql.postgres diff --git a/synapse/storage/databases/main/schema/delta/59/12account_validity_token_used_ts_ms.sql b/synapse/storage/schema/main/delta/59/12account_validity_token_used_ts_ms.sql similarity index 100% rename from synapse/storage/databases/main/schema/delta/59/12account_validity_token_used_ts_ms.sql rename to synapse/storage/schema/main/delta/59/12account_validity_token_used_ts_ms.sql diff --git a/synapse/storage/databases/main/schema/delta/59/12presence_stream_instance.sql b/synapse/storage/schema/main/delta/59/12presence_stream_instance.sql similarity index 100% rename from synapse/storage/databases/main/schema/delta/59/12presence_stream_instance.sql rename to synapse/storage/schema/main/delta/59/12presence_stream_instance.sql diff --git a/synapse/storage/databases/main/schema/delta/59/12presence_stream_instance_seq.sql.postgres b/synapse/storage/schema/main/delta/59/12presence_stream_instance_seq.sql.postgres similarity index 100% rename from synapse/storage/databases/main/schema/delta/59/12presence_stream_instance_seq.sql.postgres rename to synapse/storage/schema/main/delta/59/12presence_stream_instance_seq.sql.postgres diff --git a/synapse/storage/databases/main/schema/full_schemas/16/application_services.sql b/synapse/storage/schema/main/full_schemas/16/application_services.sql similarity index 100% rename from synapse/storage/databases/main/schema/full_schemas/16/application_services.sql rename to synapse/storage/schema/main/full_schemas/16/application_services.sql diff --git a/synapse/storage/databases/main/schema/full_schemas/16/event_edges.sql b/synapse/storage/schema/main/full_schemas/16/event_edges.sql similarity index 100% rename from synapse/storage/databases/main/schema/full_schemas/16/event_edges.sql rename to synapse/storage/schema/main/full_schemas/16/event_edges.sql diff --git a/synapse/storage/databases/main/schema/full_schemas/16/event_signatures.sql b/synapse/storage/schema/main/full_schemas/16/event_signatures.sql similarity index 100% rename from synapse/storage/databases/main/schema/full_schemas/16/event_signatures.sql rename to synapse/storage/schema/main/full_schemas/16/event_signatures.sql diff --git a/synapse/storage/databases/main/schema/full_schemas/16/im.sql b/synapse/storage/schema/main/full_schemas/16/im.sql similarity index 100% rename from synapse/storage/databases/main/schema/full_schemas/16/im.sql rename to synapse/storage/schema/main/full_schemas/16/im.sql diff --git a/synapse/storage/databases/main/schema/full_schemas/16/keys.sql b/synapse/storage/schema/main/full_schemas/16/keys.sql similarity index 100% rename from synapse/storage/databases/main/schema/full_schemas/16/keys.sql rename to synapse/storage/schema/main/full_schemas/16/keys.sql diff --git a/synapse/storage/databases/main/schema/full_schemas/16/media_repository.sql b/synapse/storage/schema/main/full_schemas/16/media_repository.sql similarity index 100% rename from synapse/storage/databases/main/schema/full_schemas/16/media_repository.sql rename to synapse/storage/schema/main/full_schemas/16/media_repository.sql diff --git a/synapse/storage/databases/main/schema/full_schemas/16/presence.sql b/synapse/storage/schema/main/full_schemas/16/presence.sql similarity index 100% rename from synapse/storage/databases/main/schema/full_schemas/16/presence.sql rename to synapse/storage/schema/main/full_schemas/16/presence.sql diff --git a/synapse/storage/databases/main/schema/full_schemas/16/profiles.sql b/synapse/storage/schema/main/full_schemas/16/profiles.sql similarity index 100% rename from synapse/storage/databases/main/schema/full_schemas/16/profiles.sql rename to synapse/storage/schema/main/full_schemas/16/profiles.sql diff --git a/synapse/storage/databases/main/schema/full_schemas/16/push.sql b/synapse/storage/schema/main/full_schemas/16/push.sql similarity index 100% rename from synapse/storage/databases/main/schema/full_schemas/16/push.sql rename to synapse/storage/schema/main/full_schemas/16/push.sql diff --git a/synapse/storage/databases/main/schema/full_schemas/16/redactions.sql b/synapse/storage/schema/main/full_schemas/16/redactions.sql similarity index 100% rename from synapse/storage/databases/main/schema/full_schemas/16/redactions.sql rename to synapse/storage/schema/main/full_schemas/16/redactions.sql diff --git a/synapse/storage/databases/main/schema/full_schemas/16/room_aliases.sql b/synapse/storage/schema/main/full_schemas/16/room_aliases.sql similarity index 100% rename from synapse/storage/databases/main/schema/full_schemas/16/room_aliases.sql rename to synapse/storage/schema/main/full_schemas/16/room_aliases.sql diff --git a/synapse/storage/databases/main/schema/full_schemas/16/state.sql b/synapse/storage/schema/main/full_schemas/16/state.sql similarity index 100% rename from synapse/storage/databases/main/schema/full_schemas/16/state.sql rename to synapse/storage/schema/main/full_schemas/16/state.sql diff --git a/synapse/storage/databases/main/schema/full_schemas/16/transactions.sql b/synapse/storage/schema/main/full_schemas/16/transactions.sql similarity index 100% rename from synapse/storage/databases/main/schema/full_schemas/16/transactions.sql rename to synapse/storage/schema/main/full_schemas/16/transactions.sql diff --git a/synapse/storage/databases/main/schema/full_schemas/16/users.sql b/synapse/storage/schema/main/full_schemas/16/users.sql similarity index 100% rename from synapse/storage/databases/main/schema/full_schemas/16/users.sql rename to synapse/storage/schema/main/full_schemas/16/users.sql diff --git a/synapse/storage/databases/main/schema/full_schemas/54/full.sql.postgres b/synapse/storage/schema/main/full_schemas/54/full.sql.postgres similarity index 100% rename from synapse/storage/databases/main/schema/full_schemas/54/full.sql.postgres rename to synapse/storage/schema/main/full_schemas/54/full.sql.postgres diff --git a/synapse/storage/databases/main/schema/full_schemas/54/full.sql.sqlite b/synapse/storage/schema/main/full_schemas/54/full.sql.sqlite similarity index 100% rename from synapse/storage/databases/main/schema/full_schemas/54/full.sql.sqlite rename to synapse/storage/schema/main/full_schemas/54/full.sql.sqlite diff --git a/synapse/storage/databases/main/schema/full_schemas/54/stream_positions.sql b/synapse/storage/schema/main/full_schemas/54/stream_positions.sql similarity index 100% rename from synapse/storage/databases/main/schema/full_schemas/54/stream_positions.sql rename to synapse/storage/schema/main/full_schemas/54/stream_positions.sql diff --git a/synapse/storage/databases/state/schema/delta/23/drop_state_index.sql b/synapse/storage/schema/state/delta/23/drop_state_index.sql similarity index 100% rename from synapse/storage/databases/state/schema/delta/23/drop_state_index.sql rename to synapse/storage/schema/state/delta/23/drop_state_index.sql diff --git a/synapse/storage/databases/state/schema/delta/30/state_stream.sql b/synapse/storage/schema/state/delta/30/state_stream.sql similarity index 100% rename from synapse/storage/databases/state/schema/delta/30/state_stream.sql rename to synapse/storage/schema/state/delta/30/state_stream.sql diff --git a/synapse/storage/databases/state/schema/delta/32/remove_state_indices.sql b/synapse/storage/schema/state/delta/32/remove_state_indices.sql similarity index 100% rename from synapse/storage/databases/state/schema/delta/32/remove_state_indices.sql rename to synapse/storage/schema/state/delta/32/remove_state_indices.sql diff --git a/synapse/storage/databases/state/schema/delta/35/add_state_index.sql b/synapse/storage/schema/state/delta/35/add_state_index.sql similarity index 100% rename from synapse/storage/databases/state/schema/delta/35/add_state_index.sql rename to synapse/storage/schema/state/delta/35/add_state_index.sql diff --git a/synapse/storage/databases/state/schema/delta/35/state.sql b/synapse/storage/schema/state/delta/35/state.sql similarity index 100% rename from synapse/storage/databases/state/schema/delta/35/state.sql rename to synapse/storage/schema/state/delta/35/state.sql diff --git a/synapse/storage/databases/state/schema/delta/35/state_dedupe.sql b/synapse/storage/schema/state/delta/35/state_dedupe.sql similarity index 100% rename from synapse/storage/databases/state/schema/delta/35/state_dedupe.sql rename to synapse/storage/schema/state/delta/35/state_dedupe.sql diff --git a/synapse/storage/databases/state/schema/delta/47/state_group_seq.py b/synapse/storage/schema/state/delta/47/state_group_seq.py similarity index 100% rename from synapse/storage/databases/state/schema/delta/47/state_group_seq.py rename to synapse/storage/schema/state/delta/47/state_group_seq.py diff --git a/synapse/storage/databases/state/schema/delta/56/state_group_room_idx.sql b/synapse/storage/schema/state/delta/56/state_group_room_idx.sql similarity index 100% rename from synapse/storage/databases/state/schema/delta/56/state_group_room_idx.sql rename to synapse/storage/schema/state/delta/56/state_group_room_idx.sql diff --git a/synapse/storage/databases/state/schema/full_schemas/54/full.sql b/synapse/storage/schema/state/full_schemas/54/full.sql similarity index 100% rename from synapse/storage/databases/state/schema/full_schemas/54/full.sql rename to synapse/storage/schema/state/full_schemas/54/full.sql diff --git a/synapse/storage/databases/state/schema/full_schemas/54/sequence.sql.postgres b/synapse/storage/schema/state/full_schemas/54/sequence.sql.postgres similarity index 100% rename from synapse/storage/databases/state/schema/full_schemas/54/sequence.sql.postgres rename to synapse/storage/schema/state/full_schemas/54/sequence.sql.postgres diff --git a/tests/storage/test_cleanup_extrems.py b/tests/storage/test_cleanup_extrems.py index aa20588bbe..77c4fe721c 100644 --- a/tests/storage/test_cleanup_extrems.py +++ b/tests/storage/test_cleanup_extrems.py @@ -47,10 +47,8 @@ class CleanupExtremBackgroundUpdateStoreTestCase(HomeserverTestCase): ) schema_path = os.path.join( - prepare_database.dir_path, - "databases", + prepare_database.schema_path, "main", - "schema", "delta", "54", "delete_forward_extremities.sql", From 4df26abf28a13f005dc4b220a5f7eb96bc4df116 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 7 May 2021 12:57:21 +0100 Subject: [PATCH 20/34] Unpin attrs dep after new version has been released (#9946) c.f. #9936 --- changelog.d/9946.misc | 1 + synapse/python_dependencies.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) create mode 100644 changelog.d/9946.misc diff --git a/changelog.d/9946.misc b/changelog.d/9946.misc new file mode 100644 index 0000000000..142ec5496f --- /dev/null +++ b/changelog.d/9946.misc @@ -0,0 +1 @@ +Unpin attrs dependency. diff --git a/synapse/python_dependencies.py b/synapse/python_dependencies.py index 7709361f16..45a6b82834 100644 --- a/synapse/python_dependencies.py +++ b/synapse/python_dependencies.py @@ -79,7 +79,7 @@ REQUIREMENTS = [ # Fedora 31 only has 19.1, so if we want to upgrade we should wait until 33 # is out in November.) # Note: 21.1.0 broke `/sync`, see #9936 - "attrs>=19.1.0,<21.1.0", + "attrs>=19.1.0,!=21.1.0", "netaddr>=0.7.18", "Jinja2>=2.9", "bleach>=1.4.3", From 765473567cd7e9520bdb9f85491bb5fe719c360b Mon Sep 17 00:00:00 2001 From: Richard van der Hoff <1389908+richvdh@users.noreply.github.com> Date: Fri, 7 May 2021 14:01:57 +0100 Subject: [PATCH 21/34] Fix make_full_schema to create the db with the right options and user (#9931) --- changelog.d/9931.misc | 1 + scripts-dev/make_full_schema.sh | 19 ++++++++++--------- 2 files changed, 11 insertions(+), 9 deletions(-) create mode 100644 changelog.d/9931.misc diff --git a/changelog.d/9931.misc b/changelog.d/9931.misc new file mode 100644 index 0000000000..326adc7f3c --- /dev/null +++ b/changelog.d/9931.misc @@ -0,0 +1 @@ +Minor fixes to the `make_full_schema.sh` script. diff --git a/scripts-dev/make_full_schema.sh b/scripts-dev/make_full_schema.sh index bc8f978660..39bf30d258 100755 --- a/scripts-dev/make_full_schema.sh +++ b/scripts-dev/make_full_schema.sh @@ -6,7 +6,7 @@ # It does so by having Synapse generate an up-to-date SQLite DB, then running # synapse_port_db to convert it to Postgres. It then dumps the contents of both. -POSTGRES_HOST="localhost" +export PGHOST="localhost" POSTGRES_DB_NAME="synapse_full_schema.$$" SQLITE_FULL_SCHEMA_OUTPUT_FILE="full.sql.sqlite" @@ -32,7 +32,7 @@ usage() { while getopts "p:co:h" opt; do case $opt in p) - POSTGRES_USERNAME=$OPTARG + export PGUSER=$OPTARG ;; c) # Print all commands that are being executed @@ -69,7 +69,7 @@ if [ ${#unsatisfied_requirements} -ne 0 ]; then exit 1 fi -if [ -z "$POSTGRES_USERNAME" ]; then +if [ -z "$PGUSER" ]; then echo "No postgres username supplied" usage exit 1 @@ -84,8 +84,9 @@ fi # Create the output directory if it doesn't exist mkdir -p "$OUTPUT_DIR" -read -rsp "Postgres password for '$POSTGRES_USERNAME': " POSTGRES_PASSWORD +read -rsp "Postgres password for '$PGUSER': " PGPASSWORD echo "" +export PGPASSWORD # Exit immediately if a command fails set -e @@ -131,9 +132,9 @@ report_stats: false database: name: "psycopg2" args: - user: "$POSTGRES_USERNAME" - host: "$POSTGRES_HOST" - password: "$POSTGRES_PASSWORD" + user: "$PGUSER" + host: "$PGHOST" + password: "$PGPASSWORD" database: "$POSTGRES_DB_NAME" # Suppress the key server warning. @@ -150,7 +151,7 @@ scripts-dev/update_database --database-config "$SQLITE_CONFIG" # Create the PostgreSQL database. echo "Creating postgres database..." -createdb $POSTGRES_DB_NAME +createdb --lc-collate=C --lc-ctype=C --template=template0 "$POSTGRES_DB_NAME" echo "Copying data from SQLite3 to Postgres with synapse_port_db..." if [ -z "$COVERAGE" ]; then @@ -181,7 +182,7 @@ DROP TABLE user_directory_search_docsize; DROP TABLE user_directory_search_stat; " sqlite3 "$SQLITE_DB" <<< "$SQL" -psql $POSTGRES_DB_NAME -U "$POSTGRES_USERNAME" -w <<< "$SQL" +psql "$POSTGRES_DB_NAME" -w <<< "$SQL" echo "Dumping SQLite3 schema to '$OUTPUT_DIR/$SQLITE_FULL_SCHEMA_OUTPUT_FILE'..." sqlite3 "$SQLITE_DB" ".dump" > "$OUTPUT_DIR/$SQLITE_FULL_SCHEMA_OUTPUT_FILE" From 6c847785494473c0a430f368a0d79c9202b8ddd0 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 7 May 2021 14:54:09 +0100 Subject: [PATCH 22/34] Always cache 'event_to_prev_state_group' (#9950) Fixes regression in send PDU times introduced in #9905. --- changelog.d/9950.feature | 1 + synapse/handlers/message.py | 13 +++++++------ 2 files changed, 8 insertions(+), 6 deletions(-) create mode 100644 changelog.d/9950.feature diff --git a/changelog.d/9950.feature b/changelog.d/9950.feature new file mode 100644 index 0000000000..96a0e7f09f --- /dev/null +++ b/changelog.d/9950.feature @@ -0,0 +1 @@ +Improve performance of sending events for worker-based deployments using Redis. diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py index 8729332d4b..5afb7fc261 100644 --- a/synapse/handlers/message.py +++ b/synapse/handlers/message.py @@ -1050,6 +1050,13 @@ class EventCreationHandler: ) if state_entry.state_group: + await self._external_cache.set( + "event_to_prev_state_group", + event.event_id, + state_entry.state_group, + expiry_ms=60 * 60 * 1000, + ) + if state_entry.state_group in self._external_cache_joined_hosts_updates: return @@ -1057,12 +1064,6 @@ class EventCreationHandler: # Note that the expiry times must be larger than the expiry time in # _external_cache_joined_hosts_updates. - await self._external_cache.set( - "event_to_prev_state_group", - event.event_id, - state_entry.state_group, - expiry_ms=60 * 60 * 1000, - ) await self._external_cache.set( "get_joined_hosts", str(state_entry.state_group), From 51065c44bb0875373fada2838e69e4bc5005a95d Mon Sep 17 00:00:00 2001 From: Richard van der Hoff <1389908+richvdh@users.noreply.github.com> Date: Mon, 10 May 2021 13:02:55 +0100 Subject: [PATCH 23/34] Fix port_db on empty db (#9930) ... and test it. --- .buildkite/scripts/create_postgres_db.py | 36 --------------------- .buildkite/scripts/postgres_exec.py | 31 ++++++++++++++++++ .buildkite/scripts/test_synapse_port_db.sh | 37 ++++++++++++++++------ .github/workflows/tests.yml | 2 +- changelog.d/9930.bugfix | 1 + scripts/synapse_port_db | 18 ++++++----- 6 files changed, 70 insertions(+), 55 deletions(-) delete mode 100755 .buildkite/scripts/create_postgres_db.py create mode 100755 .buildkite/scripts/postgres_exec.py create mode 100644 changelog.d/9930.bugfix diff --git a/.buildkite/scripts/create_postgres_db.py b/.buildkite/scripts/create_postgres_db.py deleted file mode 100755 index cc829db216..0000000000 --- a/.buildkite/scripts/create_postgres_db.py +++ /dev/null @@ -1,36 +0,0 @@ -#!/usr/bin/env python -# Copyright 2019 The Matrix.org Foundation C.I.C. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import logging - -from synapse.storage.engines import create_engine - -logger = logging.getLogger("create_postgres_db") - -if __name__ == "__main__": - # Create a PostgresEngine. - db_engine = create_engine({"name": "psycopg2", "args": {}}) - - # Connect to postgres to create the base database. - # We use "postgres" as a database because it's bound to exist and the "synapse" one - # doesn't exist yet. - db_conn = db_engine.module.connect( - user="postgres", host="postgres", password="postgres", dbname="postgres" - ) - db_conn.autocommit = True - cur = db_conn.cursor() - cur.execute("CREATE DATABASE synapse;") - cur.close() - db_conn.close() diff --git a/.buildkite/scripts/postgres_exec.py b/.buildkite/scripts/postgres_exec.py new file mode 100755 index 0000000000..086b391724 --- /dev/null +++ b/.buildkite/scripts/postgres_exec.py @@ -0,0 +1,31 @@ +#!/usr/bin/env python +# Copyright 2019 The Matrix.org Foundation C.I.C. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import sys + +import psycopg2 + +# a very simple replacment for `psql`, to make up for the lack of the postgres client +# libraries in the synapse docker image. + +# We use "postgres" as a database because it's bound to exist and the "synapse" one +# doesn't exist yet. +db_conn = psycopg2.connect( + user="postgres", host="postgres", password="postgres", dbname="postgres" +) +db_conn.autocommit = True +cur = db_conn.cursor() +for c in sys.argv[1:]: + cur.execute(c) diff --git a/.buildkite/scripts/test_synapse_port_db.sh b/.buildkite/scripts/test_synapse_port_db.sh index 8914319e38..a7e2454769 100755 --- a/.buildkite/scripts/test_synapse_port_db.sh +++ b/.buildkite/scripts/test_synapse_port_db.sh @@ -1,10 +1,10 @@ #!/usr/bin/env bash # -# Test script for 'synapse_port_db', which creates a virtualenv, installs Synapse along -# with additional dependencies needed for the test (such as coverage or the PostgreSQL -# driver), update the schema of the test SQLite database and run background updates on it, -# create an empty test database in PostgreSQL, then run the 'synapse_port_db' script to -# test porting the SQLite database to the PostgreSQL database (with coverage). +# Test script for 'synapse_port_db'. +# - sets up synapse and deps +# - runs the port script on a prepopulated test sqlite db +# - also runs it against an new sqlite db + set -xe cd `dirname $0`/../.. @@ -22,15 +22,32 @@ echo "--- Generate the signing key" # Generate the server's signing key. python -m synapse.app.homeserver --generate-keys -c .buildkite/sqlite-config.yaml -echo "--- Prepare the databases" +echo "--- Prepare test database" # Make sure the SQLite3 database is using the latest schema and has no pending background update. scripts-dev/update_database --database-config .buildkite/sqlite-config.yaml # Create the PostgreSQL database. -./.buildkite/scripts/create_postgres_db.py +./.buildkite/scripts/postgres_exec.py "CREATE DATABASE synapse" -echo "+++ Run synapse_port_db" - -# Run the script +echo "+++ Run synapse_port_db against test database" +coverage run scripts/synapse_port_db --sqlite-database .buildkite/test_db.db --postgres-config .buildkite/postgres-config.yaml + +##### + +# Now do the same again, on an empty database. + +echo "--- Prepare empty SQLite database" + +# we do this by deleting the sqlite db, and then doing the same again. +rm .buildkite/test_db.db + +scripts-dev/update_database --database-config .buildkite/sqlite-config.yaml + +# re-create the PostgreSQL database. +./.buildkite/scripts/postgres_exec.py \ + "DROP DATABASE synapse" \ + "CREATE DATABASE synapse" + +echo "+++ Run synapse_port_db against empty database" coverage run scripts/synapse_port_db --sqlite-database .buildkite/test_db.db --postgres-config .buildkite/postgres-config.yaml diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 12c82ac620..e7f3be1b4e 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -273,7 +273,7 @@ jobs: python-version: ${{ matrix.python-version }} - name: Patch Buildkite-specific test scripts run: | - sed -i -e 's/host="postgres"/host="localhost"/' .buildkite/scripts/create_postgres_db.py + sed -i -e 's/host="postgres"/host="localhost"/' .buildkite/scripts/postgres_exec.py sed -i -e 's/host: postgres/host: localhost/' .buildkite/postgres-config.yaml sed -i -e 's|/src/||' .buildkite/{sqlite,postgres}-config.yaml sed -i -e 's/\$TOP/\$GITHUB_WORKSPACE/' .coveragerc diff --git a/changelog.d/9930.bugfix b/changelog.d/9930.bugfix new file mode 100644 index 0000000000..9b22ed4458 --- /dev/null +++ b/changelog.d/9930.bugfix @@ -0,0 +1 @@ +Fix bugs introduced in v1.23.0 which made the PostgreSQL port script fail when run with a newly-created SQLite database. diff --git a/scripts/synapse_port_db b/scripts/synapse_port_db index f0c93d5226..5fb5bb35f7 100755 --- a/scripts/synapse_port_db +++ b/scripts/synapse_port_db @@ -913,10 +913,11 @@ class Porter(object): (curr_forward_id + 1,), ) - txn.execute( - "ALTER SEQUENCE events_backfill_stream_seq RESTART WITH %s", - (curr_backward_id + 1,), - ) + if curr_backward_id: + txn.execute( + "ALTER SEQUENCE events_backfill_stream_seq RESTART WITH %s", + (curr_backward_id + 1,), + ) await self.postgres_store.db_pool.runInteraction( "_setup_events_stream_seqs", _setup_events_stream_seqs_set_pos, @@ -954,10 +955,11 @@ class Porter(object): (curr_chain_id,), ) - await self.postgres_store.db_pool.runInteraction( - "_setup_event_auth_chain_id", r, - ) - + if curr_chain_id is not None: + await self.postgres_store.db_pool.runInteraction( + "_setup_event_auth_chain_id", + r, + ) ############################################## From 2b2985b5cfc4267dfb0f6b900e1a0f69f3f2cdc5 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 10 May 2021 13:29:02 +0100 Subject: [PATCH 24/34] Improve performance of backfilling in large rooms. (#9935) We were pulling the full auth chain for the room out of the DB each time we backfilled, which can be *huge* for large rooms and is totally unnecessary. --- changelog.d/9935.feature | 1 + synapse/handlers/federation.py | 131 +++++++++++++++------------------ 2 files changed, 59 insertions(+), 73 deletions(-) create mode 100644 changelog.d/9935.feature diff --git a/changelog.d/9935.feature b/changelog.d/9935.feature new file mode 100644 index 0000000000..eeda5bf50e --- /dev/null +++ b/changelog.d/9935.feature @@ -0,0 +1 @@ +Improve performance of backfilling in large rooms. diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index e8330a2b50..798ed75b30 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -552,8 +552,12 @@ class FederationHandler(BaseHandler): destination: str, room_id: str, event_id: str, - ) -> Tuple[List[EventBase], List[EventBase]]: - """Requests all of the room state at a given event from a remote homeserver. + ) -> List[EventBase]: + """Requests all of the room state at a given event from a remote + homeserver. + + Will also fetch any missing events reported in the `auth_chain_ids` + section of `/state_ids`. Args: destination: The remote homeserver to query for the state. @@ -561,8 +565,7 @@ class FederationHandler(BaseHandler): event_id: The id of the event we want the state at. Returns: - A list of events in the state, not including the event itself, and - a list of events in the auth chain for the given event. + A list of events in the state, not including the event itself. """ ( state_event_ids, @@ -571,68 +574,53 @@ class FederationHandler(BaseHandler): destination, room_id, event_id=event_id ) - desired_events = set(state_event_ids + auth_event_ids) - - event_map = await self._get_events_from_store_or_dest( - destination, room_id, desired_events - ) - - failed_to_fetch = desired_events - event_map.keys() - if failed_to_fetch: - logger.warning( - "Failed to fetch missing state/auth events for %s %s", - event_id, - failed_to_fetch, - ) - - remote_state = [ - event_map[e_id] for e_id in state_event_ids if e_id in event_map - ] - - auth_chain = [event_map[e_id] for e_id in auth_event_ids if e_id in event_map] - auth_chain.sort(key=lambda e: e.depth) - - return remote_state, auth_chain - - async def _get_events_from_store_or_dest( - self, destination: str, room_id: str, event_ids: Iterable[str] - ) -> Dict[str, EventBase]: - """Fetch events from a remote destination, checking if we already have them. - - Persists any events we don't already have as outliers. - - If we fail to fetch any of the events, a warning will be logged, and the event - will be omitted from the result. Likewise, any events which turn out not to - be in the given room. - - This function *does not* automatically get missing auth events of the - newly fetched events. Callers must include the full auth chain of - of the missing events in the `event_ids` argument, to ensure that any - missing auth events are correctly fetched. - - Returns: - map from event_id to event - """ - fetched_events = await self.store.get_events(event_ids, allow_rejected=True) - - missing_events = set(event_ids) - fetched_events.keys() - - if missing_events: - logger.debug( - "Fetching unknown state/auth events %s for room %s", - missing_events, - room_id, - ) + # Fetch the state events from the DB, and check we have the auth events. + event_map = await self.store.get_events(state_event_ids, allow_rejected=True) + auth_events_in_store = await self.store.have_seen_events(auth_event_ids) + # Check for missing events. We handle state and auth event seperately, + # as we want to pull the state from the DB, but we don't for the auth + # events. (Note: we likely won't use the majority of the auth chain, and + # it can be *huge* for large rooms, so it's worth ensuring that we don't + # unnecessarily pull it from the DB). + missing_state_events = set(state_event_ids) - set(event_map) + missing_auth_events = set(auth_event_ids) - set(auth_events_in_store) + if missing_state_events or missing_auth_events: await self._get_events_and_persist( - destination=destination, room_id=room_id, events=missing_events + destination=destination, + room_id=room_id, + events=missing_state_events | missing_auth_events, ) - # we need to make sure we re-load from the database to get the rejected - # state correct. - fetched_events.update( - (await self.store.get_events(missing_events, allow_rejected=True)) - ) + if missing_state_events: + new_events = await self.store.get_events( + missing_state_events, allow_rejected=True + ) + event_map.update(new_events) + + missing_state_events.difference_update(new_events) + + if missing_state_events: + logger.warning( + "Failed to fetch missing state events for %s %s", + event_id, + missing_state_events, + ) + + if missing_auth_events: + auth_events_in_store = await self.store.have_seen_events( + missing_auth_events + ) + missing_auth_events.difference_update(auth_events_in_store) + + if missing_auth_events: + logger.warning( + "Failed to fetch missing auth events for %s %s", + event_id, + missing_auth_events, + ) + + remote_state = list(event_map.values()) # check for events which were in the wrong room. # @@ -640,8 +628,8 @@ class FederationHandler(BaseHandler): # auth_events at an event in room A are actually events in room B bad_events = [ - (event_id, event.room_id) - for event_id, event in fetched_events.items() + (event.event_id, event.room_id) + for event in remote_state if event.room_id != room_id ] @@ -658,9 +646,10 @@ class FederationHandler(BaseHandler): room_id, ) - del fetched_events[bad_event_id] + if bad_events: + remote_state = [e for e in remote_state if e.room_id == room_id] - return fetched_events + return remote_state async def _get_state_after_missing_prev_event( self, @@ -963,27 +952,23 @@ class FederationHandler(BaseHandler): # For each edge get the current state. - auth_events = {} state_events = {} events_to_state = {} for e_id in edges: - state, auth = await self._get_state_for_room( + state = await self._get_state_for_room( destination=dest, room_id=room_id, event_id=e_id, ) - auth_events.update({a.event_id: a for a in auth}) - auth_events.update({s.event_id: s for s in state}) state_events.update({s.event_id: s for s in state}) events_to_state[e_id] = state required_auth = { a_id - for event in events - + list(state_events.values()) - + list(auth_events.values()) + for event in events + list(state_events.values()) for a_id in event.auth_event_ids() } + auth_events = await self.store.get_events(required_auth, allow_rejected=True) auth_events.update( {e_id: event_map[e_id] for e_id in required_auth if e_id in event_map} ) From 03318a766cac9f8b053db2214d9c332a977d226c Mon Sep 17 00:00:00 2001 From: Richard van der Hoff <1389908+richvdh@users.noreply.github.com> Date: Tue, 11 May 2021 10:47:23 +0100 Subject: [PATCH 25/34] Merge pull request from GHSA-x345-32rc-8h85 * tests for push rule pattern matching * tests for acl pattern matching * factor out common `re.escape` * Factor out common re.compile * Factor out common anchoring code * add word_boundary support to `glob_to_regex` * Use `glob_to_regex` in push rule evaluator NB that this drops support for character classes. I don't think anyone ever used them. * Improve efficiency of globs with multiple wildcards The idea here is that we compress multiple `*` globs into a single `.*`. We also need to consider `?`, since `*?*` is as hard to implement efficiently as `**`. * add assertion on regex pattern * Fix mypy * Simplify glob_to_regex * Inline the glob_to_regex helper function Signed-off-by: Dan Callahan * Moar comments Signed-off-by: Dan Callahan Co-authored-by: Dan Callahan --- synapse/config/tls.py | 4 +- synapse/push/push_rule_evaluator.py | 55 +------ synapse/util/__init__.py | 63 ++++++-- tests/federation/test_federation_server.py | 19 +++ tests/push/test_push_rule_evaluator.py | 166 +++++++++++++++++++++ tests/util/test_glob_to_regex.py | 59 ++++++++ 6 files changed, 297 insertions(+), 69 deletions(-) create mode 100644 tests/util/test_glob_to_regex.py diff --git a/synapse/config/tls.py b/synapse/config/tls.py index b041869758..7df4e4c3e6 100644 --- a/synapse/config/tls.py +++ b/synapse/config/tls.py @@ -17,7 +17,7 @@ import os import warnings from datetime import datetime from hashlib import sha256 -from typing import List, Optional +from typing import List, Optional, Pattern from unpaddedbase64 import encode_base64 @@ -124,7 +124,7 @@ class TlsConfig(Config): fed_whitelist_entries = [] # Support globs (*) in whitelist values - self.federation_certificate_verification_whitelist = [] # type: List[str] + self.federation_certificate_verification_whitelist = [] # type: List[Pattern] for entry in fed_whitelist_entries: try: entry_regex = glob_to_regex(entry.encode("ascii").decode("ascii")) diff --git a/synapse/push/push_rule_evaluator.py b/synapse/push/push_rule_evaluator.py index 49ecb38522..98b90a4f51 100644 --- a/synapse/push/push_rule_evaluator.py +++ b/synapse/push/push_rule_evaluator.py @@ -19,6 +19,7 @@ from typing import Any, Dict, List, Optional, Pattern, Tuple, Union from synapse.events import EventBase from synapse.types import UserID +from synapse.util import glob_to_regex, re_word_boundary from synapse.util.caches.lrucache import LruCache logger = logging.getLogger(__name__) @@ -183,7 +184,7 @@ class PushRuleEvaluatorForEvent: r = regex_cache.get((display_name, False, True), None) if not r: r1 = re.escape(display_name) - r1 = _re_word_boundary(r1) + r1 = re_word_boundary(r1) r = re.compile(r1, flags=re.IGNORECASE) regex_cache[(display_name, False, True)] = r @@ -212,7 +213,7 @@ def _glob_matches(glob: str, value: str, word_boundary: bool = False) -> bool: try: r = regex_cache.get((glob, True, word_boundary), None) if not r: - r = _glob_to_re(glob, word_boundary) + r = glob_to_regex(glob, word_boundary) regex_cache[(glob, True, word_boundary)] = r return bool(r.search(value)) except re.error: @@ -220,56 +221,6 @@ def _glob_matches(glob: str, value: str, word_boundary: bool = False) -> bool: return False -def _glob_to_re(glob: str, word_boundary: bool) -> Pattern: - """Generates regex for a given glob. - - Args: - glob - word_boundary: Whether to match against word boundaries or entire string. - """ - if IS_GLOB.search(glob): - r = re.escape(glob) - - r = r.replace(r"\*", ".*?") - r = r.replace(r"\?", ".") - - # handle [abc], [a-z] and [!a-z] style ranges. - r = GLOB_REGEX.sub( - lambda x: ( - "[%s%s]" % (x.group(1) and "^" or "", x.group(2).replace(r"\\\-", "-")) - ), - r, - ) - if word_boundary: - r = _re_word_boundary(r) - - return re.compile(r, flags=re.IGNORECASE) - else: - r = "^" + r + "$" - - return re.compile(r, flags=re.IGNORECASE) - elif word_boundary: - r = re.escape(glob) - r = _re_word_boundary(r) - - return re.compile(r, flags=re.IGNORECASE) - else: - r = "^" + re.escape(glob) + "$" - return re.compile(r, flags=re.IGNORECASE) - - -def _re_word_boundary(r: str) -> str: - """ - Adds word boundary characters to the start and end of an - expression to require that the match occur as a whole word, - but do so respecting the fact that strings starting or ending - with non-word characters will change word boundaries. - """ - # we can't use \b as it chokes on unicode. however \W seems to be okay - # as shorthand for [^0-9A-Za-z_]. - return r"(^|\W)%s(\W|$)" % (r,) - - def _flatten_dict( d: Union[EventBase, dict], prefix: Optional[List[str]] = None, diff --git a/synapse/util/__init__.py b/synapse/util/__init__.py index 0f84fa3f4e..b69f562ca5 100644 --- a/synapse/util/__init__.py +++ b/synapse/util/__init__.py @@ -15,6 +15,7 @@ import json import logging import re +from typing import Pattern import attr from frozendict import frozendict @@ -26,6 +27,9 @@ from synapse.logging import context logger = logging.getLogger(__name__) +_WILDCARD_RUN = re.compile(r"([\?\*]+)") + + def _reject_invalid_json(val): """Do not allow Infinity, -Infinity, or NaN values in JSON.""" raise ValueError("Invalid JSON value: '%s'" % val) @@ -158,25 +162,54 @@ def log_failure(failure, msg, consumeErrors=True): return failure -def glob_to_regex(glob): +def glob_to_regex(glob: str, word_boundary: bool = False) -> Pattern: """Converts a glob to a compiled regex object. - The regex is anchored at the beginning and end of the string. - Args: - glob (str) + glob: pattern to match + word_boundary: If True, the pattern will be allowed to match at word boundaries + anywhere in the string. Otherwise, the pattern is anchored at the start and + end of the string. Returns: - re.RegexObject + compiled regex pattern """ - res = "" - for c in glob: - if c == "*": - res = res + ".*" - elif c == "?": - res = res + "." - else: - res = res + re.escape(c) - # \A anchors at start of string, \Z at end of string - return re.compile(r"\A" + res + r"\Z", re.IGNORECASE) + # Patterns with wildcards must be simplified to avoid performance cliffs + # - The glob `?**?**?` is equivalent to the glob `???*` + # - The glob `???*` is equivalent to the regex `.{3,}` + chunks = [] + for chunk in _WILDCARD_RUN.split(glob): + # No wildcards? re.escape() + if not _WILDCARD_RUN.match(chunk): + chunks.append(re.escape(chunk)) + continue + + # Wildcards? Simplify. + qmarks = chunk.count("?") + if "*" in chunk: + chunks.append(".{%d,}" % qmarks) + else: + chunks.append(".{%d}" % qmarks) + + res = "".join(chunks) + + if word_boundary: + res = re_word_boundary(res) + else: + # \A anchors at start of string, \Z at end of string + res = r"\A" + res + r"\Z" + + return re.compile(res, re.IGNORECASE) + + +def re_word_boundary(r: str) -> str: + """ + Adds word boundary characters to the start and end of an + expression to require that the match occur as a whole word, + but do so respecting the fact that strings starting or ending + with non-word characters will change word boundaries. + """ + # we can't use \b as it chokes on unicode. however \W seems to be okay + # as shorthand for [^0-9A-Za-z_]. + return r"(^|\W)%s(\W|$)" % (r,) diff --git a/tests/federation/test_federation_server.py b/tests/federation/test_federation_server.py index 8508b6bd3b..1737891564 100644 --- a/tests/federation/test_federation_server.py +++ b/tests/federation/test_federation_server.py @@ -74,6 +74,25 @@ class ServerACLsTestCase(unittest.TestCase): self.assertFalse(server_matches_acl_event("[1:2::]", e)) self.assertTrue(server_matches_acl_event("1:2:3:4", e)) + def test_wildcard_matching(self): + e = _create_acl_event({"allow": ["good*.com"]}) + self.assertTrue( + server_matches_acl_event("good.com", e), + "* matches 0 characters", + ) + self.assertTrue( + server_matches_acl_event("GOOD.COM", e), + "pattern is case-insensitive", + ) + self.assertTrue( + server_matches_acl_event("good.aa.com", e), + "* matches several characters, including '.'", + ) + self.assertFalse( + server_matches_acl_event("ishgood.com", e), + "pattern does not allow prefixes", + ) + class StateQueryTests(unittest.FederatingHomeserverTestCase): diff --git a/tests/push/test_push_rule_evaluator.py b/tests/push/test_push_rule_evaluator.py index 45906ce720..a52e89e407 100644 --- a/tests/push/test_push_rule_evaluator.py +++ b/tests/push/test_push_rule_evaluator.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from typing import Any, Dict + from synapse.api.room_versions import RoomVersions from synapse.events import FrozenEvent from synapse.push import push_rule_evaluator @@ -66,6 +68,170 @@ class PushRuleEvaluatorTestCase(unittest.TestCase): # A display name with spaces should work fine. self.assertTrue(evaluator.matches(condition, "@user:test", "foo bar")) + def _assert_matches( + self, condition: Dict[str, Any], content: Dict[str, Any], msg=None + ) -> None: + evaluator = self._get_evaluator(content) + self.assertTrue(evaluator.matches(condition, "@user:test", "display_name"), msg) + + def _assert_not_matches( + self, condition: Dict[str, Any], content: Dict[str, Any], msg=None + ) -> None: + evaluator = self._get_evaluator(content) + self.assertFalse( + evaluator.matches(condition, "@user:test", "display_name"), msg + ) + + def test_event_match_body(self): + """Check that event_match conditions on content.body work as expected""" + + # if the key is `content.body`, the pattern matches substrings. + + # non-wildcards should match + condition = { + "kind": "event_match", + "key": "content.body", + "pattern": "foobaz", + } + self._assert_matches( + condition, + {"body": "aaa FoobaZ zzz"}, + "patterns should match and be case-insensitive", + ) + self._assert_not_matches( + condition, + {"body": "aa xFoobaZ yy"}, + "pattern should only match at word boundaries", + ) + self._assert_not_matches( + condition, + {"body": "aa foobazx yy"}, + "pattern should only match at word boundaries", + ) + + # wildcards should match + condition = { + "kind": "event_match", + "key": "content.body", + "pattern": "f?o*baz", + } + + self._assert_matches( + condition, + {"body": "aaa FoobarbaZ zzz"}, + "* should match string and pattern should be case-insensitive", + ) + self._assert_matches( + condition, {"body": "aa foobaz yy"}, "* should match 0 characters" + ) + self._assert_not_matches( + condition, {"body": "aa fobbaz yy"}, "? should not match 0 characters" + ) + self._assert_not_matches( + condition, {"body": "aa fiiobaz yy"}, "? should not match 2 characters" + ) + self._assert_not_matches( + condition, + {"body": "aa xfooxbaz yy"}, + "pattern should only match at word boundaries", + ) + self._assert_not_matches( + condition, + {"body": "aa fooxbazx yy"}, + "pattern should only match at word boundaries", + ) + + # test backslashes + condition = { + "kind": "event_match", + "key": "content.body", + "pattern": r"f\oobaz", + } + self._assert_matches( + condition, + {"body": r"F\oobaz"}, + "backslash should match itself", + ) + condition = { + "kind": "event_match", + "key": "content.body", + "pattern": r"f\?obaz", + } + self._assert_matches( + condition, + {"body": r"F\oobaz"}, + r"? after \ should match any character", + ) + + def test_event_match_non_body(self): + """Check that event_match conditions on other keys work as expected""" + + # if the key is anything other than 'content.body', the pattern must match the + # whole value. + + # non-wildcards should match + condition = { + "kind": "event_match", + "key": "content.value", + "pattern": "foobaz", + } + self._assert_matches( + condition, + {"value": "FoobaZ"}, + "patterns should match and be case-insensitive", + ) + self._assert_not_matches( + condition, + {"value": "xFoobaZ"}, + "pattern should only match at the start/end of the value", + ) + self._assert_not_matches( + condition, + {"value": "FoobaZz"}, + "pattern should only match at the start/end of the value", + ) + + # wildcards should match + condition = { + "kind": "event_match", + "key": "content.value", + "pattern": "f?o*baz", + } + self._assert_matches( + condition, + {"value": "FoobarbaZ"}, + "* should match string and pattern should be case-insensitive", + ) + self._assert_matches( + condition, {"value": "foobaz"}, "* should match 0 characters" + ) + self._assert_not_matches( + condition, {"value": "fobbaz"}, "? should not match 0 characters" + ) + self._assert_not_matches( + condition, {"value": "fiiobaz"}, "? should not match 2 characters" + ) + self._assert_not_matches( + condition, + {"value": "xfooxbaz"}, + "pattern should only match at the start/end of the value", + ) + self._assert_not_matches( + condition, + {"value": "fooxbazx"}, + "pattern should only match at the start/end of the value", + ) + self._assert_not_matches( + condition, + {"value": "x\nfooxbaz"}, + "pattern should not match after a newline", + ) + self._assert_not_matches( + condition, + {"value": "fooxbaz\nx"}, + "pattern should not match before a newline", + ) + def test_no_body(self): """Not having a body shouldn't break the evaluator.""" evaluator = self._get_evaluator({}) diff --git a/tests/util/test_glob_to_regex.py b/tests/util/test_glob_to_regex.py new file mode 100644 index 0000000000..220accb92b --- /dev/null +++ b/tests/util/test_glob_to_regex.py @@ -0,0 +1,59 @@ +# Copyright 2021 The Matrix.org Foundation C.I.C. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from synapse.util import glob_to_regex + +from tests.unittest import TestCase + + +class GlobToRegexTestCase(TestCase): + def test_literal_match(self): + """patterns without wildcards should match""" + pat = glob_to_regex("foobaz") + self.assertTrue( + pat.match("FoobaZ"), "patterns should match and be case-insensitive" + ) + self.assertFalse( + pat.match("x foobaz"), "pattern should not match at word boundaries" + ) + + def test_wildcard_match(self): + pat = glob_to_regex("f?o*baz") + + self.assertTrue( + pat.match("FoobarbaZ"), + "* should match string and pattern should be case-insensitive", + ) + self.assertTrue(pat.match("foobaz"), "* should match 0 characters") + self.assertFalse(pat.match("fooxaz"), "the character after * must match") + self.assertFalse(pat.match("fobbaz"), "? should not match 0 characters") + self.assertFalse(pat.match("fiiobaz"), "? should not match 2 characters") + + def test_multi_wildcard(self): + """patterns with multiple wildcards in a row should match""" + pat = glob_to_regex("**baz") + self.assertTrue(pat.match("agsgsbaz"), "** should match any string") + self.assertTrue(pat.match("baz"), "** should match the empty string") + self.assertEqual(pat.pattern, r"\A.{0,}baz\Z") + + pat = glob_to_regex("*?baz") + self.assertTrue(pat.match("agsgsbaz"), "*? should match any string") + self.assertTrue(pat.match("abaz"), "*? should match a single char") + self.assertFalse(pat.match("baz"), "*? should not match the empty string") + self.assertEqual(pat.pattern, r"\A.{1,}baz\Z") + + pat = glob_to_regex("a?*?*?baz") + self.assertTrue(pat.match("a g baz"), "?*?*? should match 3 chars") + self.assertFalse(pat.match("a..baz"), "?*?*? should not match 2 chars") + self.assertTrue(pat.match("a.gg.baz"), "?*?*? should match 4 chars") + self.assertEqual(pat.pattern, r"\Aa.{3,}baz\Z") From 7967b36efe6a033f46cd882d0b31a8c3eb18631c Mon Sep 17 00:00:00 2001 From: Richard van der Hoff <1389908+richvdh@users.noreply.github.com> Date: Tue, 11 May 2021 11:02:56 +0100 Subject: [PATCH 26/34] Fix `m.room_key_request` to-device messages (#9961) fixes #9960 --- changelog.d/9961.bugfix | 1 + synapse/api/constants.py | 5 +++- synapse/federation/federation_server.py | 19 -------------- synapse/handlers/devicemessage.py | 33 ++++++++++++++++++++----- 4 files changed, 32 insertions(+), 26 deletions(-) create mode 100644 changelog.d/9961.bugfix diff --git a/changelog.d/9961.bugfix b/changelog.d/9961.bugfix new file mode 100644 index 0000000000..e26d141a53 --- /dev/null +++ b/changelog.d/9961.bugfix @@ -0,0 +1 @@ +Fix a bug introduced in Synapse 1.29.0 which caused `m.room_key_request` to-device messages sent from one user to another to be dropped. diff --git a/synapse/api/constants.py b/synapse/api/constants.py index ab628b2be7..3940da5c88 100644 --- a/synapse/api/constants.py +++ b/synapse/api/constants.py @@ -116,9 +116,12 @@ class EventTypes: MSC1772_SPACE_PARENT = "org.matrix.msc1772.space.parent" +class ToDeviceEventTypes: + RoomKeyRequest = "m.room_key_request" + + class EduTypes: Presence = "m.presence" - RoomKeyRequest = "m.room_key_request" class RejectedReason: diff --git a/synapse/federation/federation_server.py b/synapse/federation/federation_server.py index b729a69203..ace30aa450 100644 --- a/synapse/federation/federation_server.py +++ b/synapse/federation/federation_server.py @@ -44,7 +44,6 @@ from synapse.api.errors import ( SynapseError, UnsupportedRoomVersionError, ) -from synapse.api.ratelimiting import Ratelimiter from synapse.api.room_versions import KNOWN_ROOM_VERSIONS from synapse.events import EventBase from synapse.federation.federation_base import FederationBase, event_from_pdu_json @@ -865,14 +864,6 @@ class FederationHandlerRegistry: # EDU received. self._edu_type_to_instance = {} # type: Dict[str, List[str]] - # A rate limiter for incoming room key requests per origin. - self._room_key_request_rate_limiter = Ratelimiter( - store=hs.get_datastore(), - clock=self.clock, - rate_hz=self.config.rc_key_requests.per_second, - burst_count=self.config.rc_key_requests.burst_count, - ) - def register_edu_handler( self, edu_type: str, handler: Callable[[str, JsonDict], Awaitable[None]] ) -> None: @@ -926,16 +917,6 @@ class FederationHandlerRegistry: if not self.config.use_presence and edu_type == EduTypes.Presence: return - # If the incoming room key requests from a particular origin are over - # the limit, drop them. - if ( - edu_type == EduTypes.RoomKeyRequest - and not await self._room_key_request_rate_limiter.can_do_action( - None, origin - ) - ): - return - # Check if we have a handler on this instance handler = self.edu_handlers.get(edu_type) if handler: diff --git a/synapse/handlers/devicemessage.py b/synapse/handlers/devicemessage.py index c5d631de07..580b941595 100644 --- a/synapse/handlers/devicemessage.py +++ b/synapse/handlers/devicemessage.py @@ -15,7 +15,7 @@ import logging from typing import TYPE_CHECKING, Any, Dict -from synapse.api.constants import EduTypes +from synapse.api.constants import ToDeviceEventTypes from synapse.api.errors import SynapseError from synapse.api.ratelimiting import Ratelimiter from synapse.logging.context import run_in_background @@ -79,6 +79,8 @@ class DeviceMessageHandler: ReplicationUserDevicesResyncRestServlet.make_client(hs) ) + # a rate limiter for room key requests. The keys are + # (sending_user_id, sending_device_id). self._ratelimiter = Ratelimiter( store=self.store, clock=hs.get_clock(), @@ -100,12 +102,25 @@ class DeviceMessageHandler: for user_id, by_device in content["messages"].items(): # we use UserID.from_string to catch invalid user ids if not self.is_mine(UserID.from_string(user_id)): - logger.warning("Request for keys for non-local user %s", user_id) + logger.warning("To-device message to non-local user %s", user_id) raise SynapseError(400, "Not a user here") if not by_device: continue + # Ratelimit key requests by the sending user. + if message_type == ToDeviceEventTypes.RoomKeyRequest: + allowed, _ = await self._ratelimiter.can_do_action( + None, (sender_user_id, None) + ) + if not allowed: + logger.info( + "Dropping room_key_request from %s to %s due to rate limit", + sender_user_id, + user_id, + ) + continue + messages_by_device = { device_id: { "content": message_content, @@ -192,13 +207,19 @@ class DeviceMessageHandler: for user_id, by_device in messages.items(): # Ratelimit local cross-user key requests by the sending device. if ( - message_type == EduTypes.RoomKeyRequest + message_type == ToDeviceEventTypes.RoomKeyRequest and user_id != sender_user_id - and await self._ratelimiter.can_do_action( + ): + allowed, _ = await self._ratelimiter.can_do_action( requester, (sender_user_id, requester.device_id) ) - ): - continue + if not allowed: + logger.info( + "Dropping room_key_request from %s to %s due to rate limit", + sender_user_id, + user_id, + ) + continue # we use UserID.from_string to catch invalid user ids if self.is_mine(UserID.from_string(user_id)): From b378d98c8f93412bc51f0d4b9c4aa2d1701cf523 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff <1389908+richvdh@users.noreply.github.com> Date: Tue, 11 May 2021 11:04:03 +0100 Subject: [PATCH 27/34] Add debug logging for issue #9533 (#9959) Hopefully this will help us track down where to-device messages are getting lost/delayed. --- changelog.d/9959.misc | 1 + .../federation/sender/per_destination_queue.py | 9 +++++++++ synapse/logging/__init__.py | 7 ++++++- synapse/notifier.py | 8 ++++++++ synapse/replication/tcp/client.py | 1 - synapse/storage/databases/main/deviceinbox.py | 18 ++++++++++++++++++ 6 files changed, 42 insertions(+), 2 deletions(-) create mode 100644 changelog.d/9959.misc diff --git a/changelog.d/9959.misc b/changelog.d/9959.misc new file mode 100644 index 0000000000..7231f29d79 --- /dev/null +++ b/changelog.d/9959.misc @@ -0,0 +1 @@ +Add debug logging for lost/delayed to-device messages. diff --git a/synapse/federation/sender/per_destination_queue.py b/synapse/federation/sender/per_destination_queue.py index 3b053ebcfb..3a2efd56ee 100644 --- a/synapse/federation/sender/per_destination_queue.py +++ b/synapse/federation/sender/per_destination_queue.py @@ -28,6 +28,7 @@ from synapse.api.presence import UserPresenceState from synapse.events import EventBase from synapse.federation.units import Edu from synapse.handlers.presence import format_user_presence_state +from synapse.logging import issue9533_logger from synapse.logging.opentracing import SynapseTags, set_tag from synapse.metrics import sent_transactions_counter from synapse.metrics.background_process_metrics import run_as_background_process @@ -574,6 +575,14 @@ class PerDestinationQueue: for content in contents ] + if edus: + issue9533_logger.debug( + "Sending %i to-device messages to %s, up to stream id %i", + len(edus), + self._destination, + stream_id, + ) + return (edus, stream_id) def _start_catching_up(self) -> None: diff --git a/synapse/logging/__init__.py b/synapse/logging/__init__.py index e00969f8b1..b50a4f95eb 100644 --- a/synapse/logging/__init__.py +++ b/synapse/logging/__init__.py @@ -12,8 +12,13 @@ # See the License for the specific language governing permissions and # limitations under the License. -# These are imported to allow for nicer logging configuration files. +import logging + from synapse.logging._remote import RemoteHandler from synapse.logging._terse_json import JsonFormatter, TerseJsonFormatter +# These are imported to allow for nicer logging configuration files. __all__ = ["RemoteHandler", "JsonFormatter", "TerseJsonFormatter"] + +# Debug logger for https://github.com/matrix-org/synapse/issues/9533 etc +issue9533_logger = logging.getLogger("synapse.9533_debug") diff --git a/synapse/notifier.py b/synapse/notifier.py index b9531007e2..24b4e6649f 100644 --- a/synapse/notifier.py +++ b/synapse/notifier.py @@ -38,6 +38,7 @@ from synapse.api.constants import EventTypes, HistoryVisibility, Membership from synapse.api.errors import AuthError from synapse.events import EventBase from synapse.handlers.presence import format_user_presence_state +from synapse.logging import issue9533_logger from synapse.logging.context import PreserveLoggingContext from synapse.logging.opentracing import log_kv, start_active_span from synapse.logging.utils import log_function @@ -426,6 +427,13 @@ class Notifier: for room in rooms: user_streams |= self.room_to_user_streams.get(room, set()) + if stream_key == "to_device_key": + issue9533_logger.debug( + "to-device messages stream id %s, awaking streams for %s", + new_token, + users, + ) + time_now_ms = self.clock.time_msec() for user_stream in user_streams: try: diff --git a/synapse/replication/tcp/client.py b/synapse/replication/tcp/client.py index 4f3c6a18b6..62d7809175 100644 --- a/synapse/replication/tcp/client.py +++ b/synapse/replication/tcp/client.py @@ -51,7 +51,6 @@ if TYPE_CHECKING: logger = logging.getLogger(__name__) - # How long we allow callers to wait for replication updates before timing out. _WAIT_FOR_REPLICATION_TIMEOUT_SECONDS = 30 diff --git a/synapse/storage/databases/main/deviceinbox.py b/synapse/storage/databases/main/deviceinbox.py index 7c9d1f744e..50e7ddd735 100644 --- a/synapse/storage/databases/main/deviceinbox.py +++ b/synapse/storage/databases/main/deviceinbox.py @@ -15,6 +15,7 @@ import logging from typing import List, Optional, Tuple +from synapse.logging import issue9533_logger from synapse.logging.opentracing import log_kv, set_tag, trace from synapse.replication.tcp.streams import ToDeviceStream from synapse.storage._base import SQLBaseStore, db_to_json @@ -404,6 +405,13 @@ class DeviceInboxWorkerStore(SQLBaseStore): ], ) + if remote_messages_by_destination: + issue9533_logger.debug( + "Queued outgoing to-device messages with stream_id %i for %s", + stream_id, + list(remote_messages_by_destination.keys()), + ) + async with self._device_inbox_id_gen.get_next() as stream_id: now_ms = self.clock.time_msec() await self.db_pool.runInteraction( @@ -533,6 +541,16 @@ class DeviceInboxWorkerStore(SQLBaseStore): ], ) + issue9533_logger.debug( + "Stored to-device messages with stream_id %i for %s", + stream_id, + [ + (user_id, device_id) + for (user_id, messages_by_device) in local_by_user_then_device.items() + for device_id in messages_by_device.keys() + ], + ) + class DeviceInboxBackgroundUpdateStore(SQLBaseStore): DEVICE_INBOX_STREAM_ID = "device_inbox_stream_drop" From 86fb71431ca562ba536c50620d1e367a9c6d4ac9 Mon Sep 17 00:00:00 2001 From: Brendan Abolivier Date: Tue, 11 May 2021 13:53:49 +0100 Subject: [PATCH 28/34] 1.33.2 --- CHANGES.md | 16 ++++++++++++++++ changelog.d/9946.misc | 1 - debian/changelog | 6 ++++++ synapse/__init__.py | 2 +- 4 files changed, 23 insertions(+), 2 deletions(-) delete mode 100644 changelog.d/9946.misc diff --git a/CHANGES.md b/CHANGES.md index a41abbefba..7ae0e7b3c1 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,3 +1,19 @@ +Synapse 1.33.2 (2021-05-11) +=========================== + +Due to the security issue highlighted below, server administrators are encouraged to update Synapse. We are not aware of these vulnerabilities being exploited in the wild. + +Security advisory +----------------- + +This release fixes a denial of service attack ([CVE-2021-29471](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-29471)) against Synapse's push rules implementation. Server admins are encouraged to upgrade. + +Internal Changes +---------------- + +- Unpin attrs dependency. ([\#9946](https://github.com/matrix-org/synapse/issues/9946)) + + Synapse 1.33.1 (2021-05-06) =========================== diff --git a/changelog.d/9946.misc b/changelog.d/9946.misc deleted file mode 100644 index 142ec5496f..0000000000 --- a/changelog.d/9946.misc +++ /dev/null @@ -1 +0,0 @@ -Unpin attrs dependency. diff --git a/debian/changelog b/debian/changelog index de50dd14ea..76b82c172e 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,9 @@ +matrix-synapse-py3 (1.33.2) stable; urgency=medium + + * New synapse release 1.33.2. + + -- Synapse Packaging team Tue, 11 May 2021 11:17:59 +0100 + matrix-synapse-py3 (1.33.1) stable; urgency=medium * New synapse release 1.33.1. diff --git a/synapse/__init__.py b/synapse/__init__.py index 441cd8b339..ce822ccb04 100644 --- a/synapse/__init__.py +++ b/synapse/__init__.py @@ -47,7 +47,7 @@ try: except ImportError: pass -__version__ = "1.33.1" +__version__ = "1.33.2" if bool(os.environ.get("SYNAPSE_TEST_PATCH_LOG_CONTEXTS", False)): # We import here so that we don't have to install a bunch of deps when From dc6366a9bd370a0f772f376a2053c0ce48cb6607 Mon Sep 17 00:00:00 2001 From: Aaron Raimist Date: Tue, 11 May 2021 08:03:23 -0500 Subject: [PATCH 29/34] Add config option to hide device names over federation (#9945) Now that cross signing exists there is much less of a need for other people to look at devices and verify them individually. This PR adds a config option to allow you to prevent device display names from being shared with other servers. Signed-off-by: Aaron Raimist --- changelog.d/9945.feature | 1 + docs/sample_config.yaml | 6 ++++++ synapse/config/federation.py | 10 ++++++++++ synapse/storage/databases/main/end_to_end_keys.py | 4 +++- 4 files changed, 20 insertions(+), 1 deletion(-) create mode 100644 changelog.d/9945.feature diff --git a/changelog.d/9945.feature b/changelog.d/9945.feature new file mode 100644 index 0000000000..84308e8cce --- /dev/null +++ b/changelog.d/9945.feature @@ -0,0 +1 @@ +Add a config option to allow you to prevent device display names from being shared over federation. Contributed by @aaronraimist. diff --git a/docs/sample_config.yaml b/docs/sample_config.yaml index f469d6e54f..7cf222d356 100644 --- a/docs/sample_config.yaml +++ b/docs/sample_config.yaml @@ -741,6 +741,12 @@ acme: # #allow_profile_lookup_over_federation: false +# Uncomment to disable device display name lookup over federation. By default, the +# Federation API allows other homeservers to obtain device display names of any user +# on this homeserver. Defaults to 'true'. +# +#allow_device_name_lookup_over_federation: false + ## Caching ## diff --git a/synapse/config/federation.py b/synapse/config/federation.py index 090ba047fa..cdd7a1ef05 100644 --- a/synapse/config/federation.py +++ b/synapse/config/federation.py @@ -44,6 +44,10 @@ class FederationConfig(Config): "allow_profile_lookup_over_federation", True ) + self.allow_device_name_lookup_over_federation = config.get( + "allow_device_name_lookup_over_federation", True + ) + def generate_config_section(self, config_dir_path, server_name, **kwargs): return """\ ## Federation ## @@ -75,6 +79,12 @@ class FederationConfig(Config): # on this homeserver. Defaults to 'true'. # #allow_profile_lookup_over_federation: false + + # Uncomment to disable device display name lookup over federation. By default, the + # Federation API allows other homeservers to obtain device display names of any user + # on this homeserver. Defaults to 'true'. + # + #allow_device_name_lookup_over_federation: false """ diff --git a/synapse/storage/databases/main/end_to_end_keys.py b/synapse/storage/databases/main/end_to_end_keys.py index 88afe97c41..398d6b6acb 100644 --- a/synapse/storage/databases/main/end_to_end_keys.py +++ b/synapse/storage/databases/main/end_to_end_keys.py @@ -84,7 +84,9 @@ class EndToEndKeyWorkerStore(EndToEndKeyBackgroundStore): if keys: result["keys"] = keys - device_display_name = device.display_name + device_display_name = None + if self.hs.config.allow_device_name_lookup_over_federation: + device_display_name = device.display_name if device_display_name: result["device_display_name"] = device_display_name From d1473f7362e9b146dbd256076c8e3c7d163e7d94 Mon Sep 17 00:00:00 2001 From: Brendan Abolivier Date: Tue, 11 May 2021 14:09:46 +0100 Subject: [PATCH 30/34] Use link to advisory rather than to the CVE repo --- CHANGES.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index 7ae0e7b3c1..93efa3ce56 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -6,7 +6,7 @@ Due to the security issue highlighted below, server administrators are encourage Security advisory ----------------- -This release fixes a denial of service attack ([CVE-2021-29471](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-29471)) against Synapse's push rules implementation. Server admins are encouraged to upgrade. +This release fixes a denial of service attack ([CVE-2021-29471](https://github.com/matrix-org/synapse/security/advisories/GHSA-x345-32rc-8h85)) against Synapse's push rules implementation. Server admins are encouraged to upgrade. Internal Changes ---------------- From 28c68411028e15858819f2a5896313ce0e71c25b Mon Sep 17 00:00:00 2001 From: Patrick Cloke Date: Tue, 11 May 2021 10:58:58 -0400 Subject: [PATCH 31/34] Send the `m.room.create` stripped event with invites (support MSC1772). (#9966) MSC1772 specifies the m.room.create event should be sent as part of the invite_state. This was done optionally behind an experimental flag, but is now done by default due to MSC1772 being approved. --- UPGRADE.rst | 29 +++++++++++++++++++++++++++++ changelog.d/9915.feature | 2 +- changelog.d/9966.feature | 1 + docs/sample_config.yaml | 1 + synapse/config/api.py | 6 ++---- 5 files changed, 34 insertions(+), 5 deletions(-) create mode 100644 changelog.d/9966.feature diff --git a/UPGRADE.rst b/UPGRADE.rst index e921e0c08a..606e357b6e 100644 --- a/UPGRADE.rst +++ b/UPGRADE.rst @@ -85,6 +85,35 @@ for example: wget https://packages.matrix.org/debian/pool/main/m/matrix-synapse-py3/matrix-synapse-py3_1.3.0+stretch1_amd64.deb dpkg -i matrix-synapse-py3_1.3.0+stretch1_amd64.deb +Upgrading to v1.34.0 +==================== + +`room_invite_state_types` configuration setting +----------------------------------------------- + +The ``room_invite_state_types`` configuration setting has been deprecated and +replaced with ``room_prejoin_state``. See the `sample configuration file `_. + +If you have set ``room_invite_state_types`` to the default value you should simply +remove it from your configuration file. The default value used to be: + +.. code:: yaml + + room_invite_state_types: + - "m.room.join_rules" + - "m.room.canonical_alias" + - "m.room.avatar" + - "m.room.encryption" + - "m.room.name" + +If you have customised this value by adding addition state types, you should +remove ``room_invite_state_types`` and configure ``additional_event_types`` with +your customisations. + +If you have customised this value by removing state types, you should rename +``room_invite_state_types`` to ``additional_event_types``, and set +``disable_default_event_types`` to ``true``. + Upgrading to v1.33.0 ==================== diff --git a/changelog.d/9915.feature b/changelog.d/9915.feature index 832916cb01..7b81faabea 100644 --- a/changelog.d/9915.feature +++ b/changelog.d/9915.feature @@ -1 +1 @@ -Support stable identifiers from [MSC1772](https://github.com/matrix-org/matrix-doc/pull/1772). +Support stable identifiers for [MSC1772](https://github.com/matrix-org/matrix-doc/pull/1772) Spaces. `m.space.child` events will now be taken into account when populating the experimental spaces summary response. Please see `UPGRADE.rst` if you have customised `room_invite_state_types` in your configuration. \ No newline at end of file diff --git a/changelog.d/9966.feature b/changelog.d/9966.feature new file mode 100644 index 0000000000..7b81faabea --- /dev/null +++ b/changelog.d/9966.feature @@ -0,0 +1 @@ +Support stable identifiers for [MSC1772](https://github.com/matrix-org/matrix-doc/pull/1772) Spaces. `m.space.child` events will now be taken into account when populating the experimental spaces summary response. Please see `UPGRADE.rst` if you have customised `room_invite_state_types` in your configuration. \ No newline at end of file diff --git a/docs/sample_config.yaml b/docs/sample_config.yaml index 7cf222d356..67ad57b1aa 100644 --- a/docs/sample_config.yaml +++ b/docs/sample_config.yaml @@ -1521,6 +1521,7 @@ room_prejoin_state: # - m.room.avatar # - m.room.encryption # - m.room.name + # - m.room.create # # Uncomment the following to disable these defaults (so that only the event # types listed in 'additional_event_types' are shared). Defaults to 'false'. diff --git a/synapse/config/api.py b/synapse/config/api.py index 55c038c0c4..b18044f982 100644 --- a/synapse/config/api.py +++ b/synapse/config/api.py @@ -88,10 +88,6 @@ class ApiConfig(Config): if not room_prejoin_state_config.get("disable_default_event_types"): yield from _DEFAULT_PREJOIN_STATE_TYPES - if self.spaces_enabled: - # MSC1772 suggests adding m.room.create to the prejoin state - yield EventTypes.Create - yield from room_prejoin_state_config.get("additional_event_types", []) @@ -109,6 +105,8 @@ _DEFAULT_PREJOIN_STATE_TYPES = [ EventTypes.RoomAvatar, EventTypes.RoomEncryption, EventTypes.Name, + # Per MSC1772. + EventTypes.Create, ] From f4833e0c062e189fcfd8186fc197d1fc1052814a Mon Sep 17 00:00:00 2001 From: Patrick Cloke Date: Tue, 11 May 2021 12:21:43 -0400 Subject: [PATCH 32/34] Support fetching the spaces summary via GET over federation. (#9947) Per changes in MSC2946, the C-S and S-S APIs for spaces summary should use GET requests. Until this is stable, the POST endpoints still exist. This does not switch federation requests to use the GET version yet since it is newly added and already deployed servers might not support it. When switching to the stable endpoint we should switch to GET requests. --- changelog.d/9947.feature | 1 + synapse/federation/transport/client.py | 1 + synapse/federation/transport/server.py | 26 ++++++++++++++++++++++++++ synapse/rest/client/v1/room.py | 1 + 4 files changed, 29 insertions(+) create mode 100644 changelog.d/9947.feature diff --git a/changelog.d/9947.feature b/changelog.d/9947.feature new file mode 100644 index 0000000000..ce8874f810 --- /dev/null +++ b/changelog.d/9947.feature @@ -0,0 +1 @@ +Update support for [MSC2946](https://github.com/matrix-org/matrix-doc/pull/2946): Spaces Summary. diff --git a/synapse/federation/transport/client.py b/synapse/federation/transport/client.py index ada322a81e..497848a2b7 100644 --- a/synapse/federation/transport/client.py +++ b/synapse/federation/transport/client.py @@ -995,6 +995,7 @@ class TransportLayerClient: returned per space exclude_rooms: a list of any rooms we can skip """ + # TODO When switching to the stable endpoint, use GET instead of POST. path = _create_path( FEDERATION_UNSTABLE_PREFIX, "/org.matrix.msc2946/spaces/%s", room_id ) diff --git a/synapse/federation/transport/server.py b/synapse/federation/transport/server.py index a3759bdda1..e1b7462474 100644 --- a/synapse/federation/transport/server.py +++ b/synapse/federation/transport/server.py @@ -1376,6 +1376,32 @@ class FederationSpaceSummaryServlet(BaseFederationServlet): PREFIX = FEDERATION_UNSTABLE_PREFIX + "/org.matrix.msc2946" PATH = "/spaces/(?P[^/]*)" + async def on_GET( + self, + origin: str, + content: JsonDict, + query: Mapping[bytes, Sequence[bytes]], + room_id: str, + ) -> Tuple[int, JsonDict]: + suggested_only = parse_boolean_from_args(query, "suggested_only", default=False) + max_rooms_per_space = parse_integer_from_args(query, "max_rooms_per_space") + + exclude_rooms = [] + if b"exclude_rooms" in query: + try: + exclude_rooms = [ + room_id.decode("ascii") for room_id in query[b"exclude_rooms"] + ] + except Exception: + raise SynapseError( + 400, "Bad query parameter for exclude_rooms", Codes.INVALID_PARAM + ) + + return 200, await self.handler.federation_space_summary( + room_id, suggested_only, max_rooms_per_space, exclude_rooms + ) + + # TODO When switching to the stable endpoint, remove the POST handler. async def on_POST( self, origin: str, diff --git a/synapse/rest/client/v1/room.py b/synapse/rest/client/v1/room.py index 5cab4d3c7b..51813cccbe 100644 --- a/synapse/rest/client/v1/room.py +++ b/synapse/rest/client/v1/room.py @@ -1020,6 +1020,7 @@ class RoomSpaceSummaryRestServlet(RestServlet): max_rooms_per_space=parse_integer(request, "max_rooms_per_space"), ) + # TODO When switching to the stable endpoint, remove the POST handler. async def on_POST( self, request: SynapseRequest, room_id: str ) -> Tuple[int, JsonDict]: From 27c375f812725bead5fa41a55aacf316f6e2376c Mon Sep 17 00:00:00 2001 From: Patrick Cloke Date: Tue, 11 May 2021 12:57:39 -0400 Subject: [PATCH 33/34] Sort child events according to MSC1772 for the spaces summary API. (#9954) This should help ensure that equivalent results are achieved between homeservers querying for the summary of a space. This implements modified MSC1772 rules, according to MSC2946. The different is that the origin_server_ts of the m.room.create event is not used as a tie-breaker since this might not be known if the homeserver is not part of the room. --- changelog.d/9954.feature | 1 + synapse/handlers/space_summary.py | 71 +++++++++++++++++++++++- tests/handlers/test_space_summary.py | 81 ++++++++++++++++++++++++++++ 3 files changed, 151 insertions(+), 2 deletions(-) create mode 100644 changelog.d/9954.feature create mode 100644 tests/handlers/test_space_summary.py diff --git a/changelog.d/9954.feature b/changelog.d/9954.feature new file mode 100644 index 0000000000..ce8874f810 --- /dev/null +++ b/changelog.d/9954.feature @@ -0,0 +1 @@ +Update support for [MSC2946](https://github.com/matrix-org/matrix-doc/pull/2946): Spaces Summary. diff --git a/synapse/handlers/space_summary.py b/synapse/handlers/space_summary.py index 2e997841f1..e35d91832b 100644 --- a/synapse/handlers/space_summary.py +++ b/synapse/handlers/space_summary.py @@ -14,6 +14,7 @@ import itertools import logging +import re from collections import deque from typing import TYPE_CHECKING, Iterable, List, Optional, Sequence, Set, Tuple, cast @@ -226,6 +227,23 @@ class SpaceSummaryHandler: suggested_only: bool, max_children: Optional[int], ) -> Tuple[Sequence[JsonDict], Sequence[JsonDict]]: + """ + Generate a room entry and a list of event entries for a given room. + + Args: + requester: The requesting user, or None if this is over federation. + room_id: The room ID to summarize. + suggested_only: True if only suggested children should be returned. + Otherwise, all children are returned. + max_children: The maximum number of children to return for this node. + + Returns: + A tuple of: + An iterable of a single value of the room. + + An iterable of the sorted children events. This may be limited + to a maximum size or may include all children. + """ if not await self._is_room_accessible(room_id, requester): return (), () @@ -357,6 +375,18 @@ class SpaceSummaryHandler: return room_entry async def _get_child_events(self, room_id: str) -> Iterable[EventBase]: + """ + Get the child events for a given room. + + The returned results are sorted for stability. + + Args: + room_id: The room id to get the children of. + + Returns: + An iterable of sorted child events. + """ + # look for child rooms/spaces. current_state_ids = await self._store.get_current_state_ids(room_id) @@ -370,8 +400,9 @@ class SpaceSummaryHandler: ] ) - # filter out any events without a "via" (which implies it has been redacted) - return (e for e in events if _has_valid_via(e)) + # filter out any events without a "via" (which implies it has been redacted), + # and order to ensure we return stable results. + return sorted(filter(_has_valid_via, events), key=_child_events_comparison_key) @attr.s(frozen=True, slots=True) @@ -397,3 +428,39 @@ def _is_suggested_child_event(edge_event: EventBase) -> bool: return True logger.debug("Ignorning not-suggested child %s", edge_event.state_key) return False + + +# Order may only contain characters in the range of \x20 (space) to \x7F (~). +_INVALID_ORDER_CHARS_RE = re.compile(r"[^\x20-\x7F]") + + +def _child_events_comparison_key(child: EventBase) -> Tuple[bool, Optional[str], str]: + """ + Generate a value for comparing two child events for ordering. + + The rules for ordering are supposed to be: + + 1. The 'order' key, if it is valid. + 2. The 'origin_server_ts' of the 'm.room.create' event. + 3. The 'room_id'. + + But we skip step 2 since we may not have any state from the room. + + Args: + child: The event for generating a comparison key. + + Returns: + The comparison key as a tuple of: + False if the ordering is valid. + The ordering field. + The room ID. + """ + order = child.content.get("order") + # If order is not a string or doesn't meet the requirements, ignore it. + if not isinstance(order, str): + order = None + elif len(order) > 50 or _INVALID_ORDER_CHARS_RE.search(order): + order = None + + # Items without an order come last. + return (order is None, order, child.room_id) diff --git a/tests/handlers/test_space_summary.py b/tests/handlers/test_space_summary.py new file mode 100644 index 0000000000..2c5e81531b --- /dev/null +++ b/tests/handlers/test_space_summary.py @@ -0,0 +1,81 @@ +# Copyright 2021 The Matrix.org Foundation C.I.C. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from typing import Any, Optional +from unittest import mock + +from synapse.handlers.space_summary import _child_events_comparison_key + +from tests import unittest + + +def _create_event(room_id: str, order: Optional[Any] = None): + result = mock.Mock() + result.room_id = room_id + result.content = {} + if order is not None: + result.content["order"] = order + return result + + +def _order(*events): + return sorted(events, key=_child_events_comparison_key) + + +class TestSpaceSummarySort(unittest.TestCase): + def test_no_order_last(self): + """An event with no ordering is placed behind those with an ordering.""" + ev1 = _create_event("!abc:test") + ev2 = _create_event("!xyz:test", "xyz") + + self.assertEqual([ev2, ev1], _order(ev1, ev2)) + + def test_order(self): + """The ordering should be used.""" + ev1 = _create_event("!abc:test", "xyz") + ev2 = _create_event("!xyz:test", "abc") + + self.assertEqual([ev2, ev1], _order(ev1, ev2)) + + def test_order_room_id(self): + """Room ID is a tie-breaker for ordering.""" + ev1 = _create_event("!abc:test", "abc") + ev2 = _create_event("!xyz:test", "abc") + + self.assertEqual([ev1, ev2], _order(ev1, ev2)) + + def test_invalid_ordering_type(self): + """Invalid orderings are considered the same as missing.""" + ev1 = _create_event("!abc:test", 1) + ev2 = _create_event("!xyz:test", "xyz") + + self.assertEqual([ev2, ev1], _order(ev1, ev2)) + + ev1 = _create_event("!abc:test", {}) + self.assertEqual([ev2, ev1], _order(ev1, ev2)) + + ev1 = _create_event("!abc:test", []) + self.assertEqual([ev2, ev1], _order(ev1, ev2)) + + ev1 = _create_event("!abc:test", True) + self.assertEqual([ev2, ev1], _order(ev1, ev2)) + + def test_invalid_ordering_value(self): + """Invalid orderings are considered the same as missing.""" + ev1 = _create_event("!abc:test", "foo\n") + ev2 = _create_event("!xyz:test", "xyz") + + self.assertEqual([ev2, ev1], _order(ev1, ev2)) + + ev1 = _create_event("!abc:test", "a" * 51) + self.assertEqual([ev2, ev1], _order(ev1, ev2)) From 63fb220e5f90b63acfa4bdfb61788f2c2d867c86 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff <1389908+richvdh@users.noreply.github.com> Date: Tue, 11 May 2021 18:01:11 +0100 Subject: [PATCH 34/34] Tests for to-device messages (#9965) --- changelog.d/9965.bugfix | 1 + .../rest/client/v2_alpha/test_sendtodevice.py | 201 ++++++++++++++++++ 2 files changed, 202 insertions(+) create mode 100644 changelog.d/9965.bugfix create mode 100644 tests/rest/client/v2_alpha/test_sendtodevice.py diff --git a/changelog.d/9965.bugfix b/changelog.d/9965.bugfix new file mode 100644 index 0000000000..e26d141a53 --- /dev/null +++ b/changelog.d/9965.bugfix @@ -0,0 +1 @@ +Fix a bug introduced in Synapse 1.29.0 which caused `m.room_key_request` to-device messages sent from one user to another to be dropped. diff --git a/tests/rest/client/v2_alpha/test_sendtodevice.py b/tests/rest/client/v2_alpha/test_sendtodevice.py new file mode 100644 index 0000000000..c9c99cc5d7 --- /dev/null +++ b/tests/rest/client/v2_alpha/test_sendtodevice.py @@ -0,0 +1,201 @@ +# Copyright 2021 The Matrix.org Foundation C.I.C. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from synapse.rest import admin +from synapse.rest.client.v1 import login +from synapse.rest.client.v2_alpha import sendtodevice, sync + +from tests.unittest import HomeserverTestCase, override_config + + +class SendToDeviceTestCase(HomeserverTestCase): + servlets = [ + admin.register_servlets, + login.register_servlets, + sendtodevice.register_servlets, + sync.register_servlets, + ] + + def test_user_to_user(self): + """A to-device message from one user to another should get delivered""" + + user1 = self.register_user("u1", "pass") + user1_tok = self.login("u1", "pass", "d1") + + user2 = self.register_user("u2", "pass") + user2_tok = self.login("u2", "pass", "d2") + + # send the message + test_msg = {"foo": "bar"} + chan = self.make_request( + "PUT", + "/_matrix/client/r0/sendToDevice/m.test/1234", + content={"messages": {user2: {"d2": test_msg}}}, + access_token=user1_tok, + ) + self.assertEqual(chan.code, 200, chan.result) + + # check it appears + channel = self.make_request("GET", "/sync", access_token=user2_tok) + self.assertEqual(channel.code, 200, channel.result) + expected_result = { + "events": [ + { + "sender": user1, + "type": "m.test", + "content": test_msg, + } + ] + } + self.assertEqual(channel.json_body["to_device"], expected_result) + + # it should re-appear if we do another sync + channel = self.make_request("GET", "/sync", access_token=user2_tok) + self.assertEqual(channel.code, 200, channel.result) + self.assertEqual(channel.json_body["to_device"], expected_result) + + # it should *not* appear if we do an incremental sync + sync_token = channel.json_body["next_batch"] + channel = self.make_request( + "GET", f"/sync?since={sync_token}", access_token=user2_tok + ) + self.assertEqual(channel.code, 200, channel.result) + self.assertEqual(channel.json_body.get("to_device", {}).get("events", []), []) + + @override_config({"rc_key_requests": {"per_second": 10, "burst_count": 2}}) + def test_local_room_key_request(self): + """m.room_key_request has special-casing; test from local user""" + user1 = self.register_user("u1", "pass") + user1_tok = self.login("u1", "pass", "d1") + + user2 = self.register_user("u2", "pass") + user2_tok = self.login("u2", "pass", "d2") + + # send three messages + for i in range(3): + chan = self.make_request( + "PUT", + f"/_matrix/client/r0/sendToDevice/m.room_key_request/{i}", + content={"messages": {user2: {"d2": {"idx": i}}}}, + access_token=user1_tok, + ) + self.assertEqual(chan.code, 200, chan.result) + + # now sync: we should get two of the three + channel = self.make_request("GET", "/sync", access_token=user2_tok) + self.assertEqual(channel.code, 200, channel.result) + msgs = channel.json_body["to_device"]["events"] + self.assertEqual(len(msgs), 2) + for i in range(2): + self.assertEqual( + msgs[i], + {"sender": user1, "type": "m.room_key_request", "content": {"idx": i}}, + ) + sync_token = channel.json_body["next_batch"] + + # ... time passes + self.reactor.advance(1) + + # and we can send more messages + chan = self.make_request( + "PUT", + "/_matrix/client/r0/sendToDevice/m.room_key_request/3", + content={"messages": {user2: {"d2": {"idx": 3}}}}, + access_token=user1_tok, + ) + self.assertEqual(chan.code, 200, chan.result) + + # ... which should arrive + channel = self.make_request( + "GET", f"/sync?since={sync_token}", access_token=user2_tok + ) + self.assertEqual(channel.code, 200, channel.result) + msgs = channel.json_body["to_device"]["events"] + self.assertEqual(len(msgs), 1) + self.assertEqual( + msgs[0], + {"sender": user1, "type": "m.room_key_request", "content": {"idx": 3}}, + ) + + @override_config({"rc_key_requests": {"per_second": 10, "burst_count": 2}}) + def test_remote_room_key_request(self): + """m.room_key_request has special-casing; test from remote user""" + user2 = self.register_user("u2", "pass") + user2_tok = self.login("u2", "pass", "d2") + + federation_registry = self.hs.get_federation_registry() + + # send three messages + for i in range(3): + self.get_success( + federation_registry.on_edu( + "m.direct_to_device", + "remote_server", + { + "sender": "@user:remote_server", + "type": "m.room_key_request", + "messages": {user2: {"d2": {"idx": i}}}, + "message_id": f"{i}", + }, + ) + ) + + # now sync: we should get two of the three + channel = self.make_request("GET", "/sync", access_token=user2_tok) + self.assertEqual(channel.code, 200, channel.result) + msgs = channel.json_body["to_device"]["events"] + self.assertEqual(len(msgs), 2) + for i in range(2): + self.assertEqual( + msgs[i], + { + "sender": "@user:remote_server", + "type": "m.room_key_request", + "content": {"idx": i}, + }, + ) + sync_token = channel.json_body["next_batch"] + + # ... time passes + self.reactor.advance(1) + + # and we can send more messages + self.get_success( + federation_registry.on_edu( + "m.direct_to_device", + "remote_server", + { + "sender": "@user:remote_server", + "type": "m.room_key_request", + "messages": {user2: {"d2": {"idx": 3}}}, + "message_id": "3", + }, + ) + ) + + # ... which should arrive + channel = self.make_request( + "GET", f"/sync?since={sync_token}", access_token=user2_tok + ) + self.assertEqual(channel.code, 200, channel.result) + msgs = channel.json_body["to_device"]["events"] + self.assertEqual(len(msgs), 1) + self.assertEqual( + msgs[0], + { + "sender": "@user:remote_server", + "type": "m.room_key_request", + "content": {"idx": 3}, + }, + )