From 43f5cc7adc02a05ba4075b8aab3b479bda67f441 Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Thu, 16 Dec 2021 11:25:37 -0600 Subject: [PATCH 01/76] Add MSC2716 and MSC3030 to `/versions` -> `unstable_features` (#11582) As suggested in https://github.com/matrix-org/matrix-react-sdk/pull/7372#discussion_r769523369 --- changelog.d/11582.misc | 1 + synapse/config/experimental.py | 2 +- synapse/rest/client/versions.py | 4 ++++ 3 files changed, 6 insertions(+), 1 deletion(-) create mode 100644 changelog.d/11582.misc diff --git a/changelog.d/11582.misc b/changelog.d/11582.misc new file mode 100644 index 0000000000..a0291f64e2 --- /dev/null +++ b/changelog.d/11582.misc @@ -0,0 +1 @@ +Add [MSC2716](https://github.com/matrix-org/matrix-doc/pull/2716) and [MSC3030](https://github.com/matrix-org/matrix-doc/pull/3030) to `/versions` -> `unstable_features` to detect server support. diff --git a/synapse/config/experimental.py b/synapse/config/experimental.py index 678c78d565..dbaeb10918 100644 --- a/synapse/config/experimental.py +++ b/synapse/config/experimental.py @@ -32,7 +32,7 @@ class ExperimentalConfig(Config): # MSC3026 (busy presence state) self.msc3026_enabled: bool = experimental.get("msc3026_enabled", False) - # MSC2716 (backfill existing history) + # MSC2716 (importing historical messages) self.msc2716_enabled: bool = experimental.get("msc2716_enabled", False) # MSC2285 (hidden read receipts) diff --git a/synapse/rest/client/versions.py b/synapse/rest/client/versions.py index 8d888f4565..2290c57c12 100644 --- a/synapse/rest/client/versions.py +++ b/synapse/rest/client/versions.py @@ -93,6 +93,10 @@ class VersionsRestServlet(RestServlet): "org.matrix.msc3026.busy_presence": self.config.experimental.msc3026_enabled, # Supports receiving hidden read receipts as per MSC2285 "org.matrix.msc2285": self.config.experimental.msc2285_enabled, + # Adds support for importing historical messages as per MSC2716 + "org.matrix.msc2716": self.config.experimental.msc2716_enabled, + # Adds support for jump to date endpoints (/timestamp_to_event) as per MSC3030 + "org.matrix.msc3030": self.config.experimental.msc3030_enabled, }, }, ) From 1847d027e67e41f4cba132b854fb865055998d16 Mon Sep 17 00:00:00 2001 From: Dirk Klimpel <5740567+dklimpel@users.noreply.github.com> Date: Thu, 16 Dec 2021 20:59:35 +0100 Subject: [PATCH 02/76] Add type hints to `synapse/storage/databases/main/transactions.py` (#11589) --- changelog.d/11589.misc | 1 + mypy.ini | 4 +- .../storage/databases/main/transactions.py | 49 ++++++++++--------- 3 files changed, 29 insertions(+), 25 deletions(-) create mode 100644 changelog.d/11589.misc diff --git a/changelog.d/11589.misc b/changelog.d/11589.misc new file mode 100644 index 0000000000..8e405b9226 --- /dev/null +++ b/changelog.d/11589.misc @@ -0,0 +1 @@ +Add missing type hints to storage classes. \ No newline at end of file diff --git a/mypy.ini b/mypy.ini index cbe1e8302c..c546487bdb 100644 --- a/mypy.ini +++ b/mypy.ini @@ -41,7 +41,6 @@ exclude = (?x) |synapse/storage/databases/main/search.py |synapse/storage/databases/main/state.py |synapse/storage/databases/main/stats.py - |synapse/storage/databases/main/transactions.py |synapse/storage/databases/main/user_directory.py |synapse/storage/schema/ @@ -216,6 +215,9 @@ disallow_untyped_defs = True [mypy-synapse.storage.databases.main.state_deltas] disallow_untyped_defs = True +[mypy-synapse.storage.databases.main.transactions] +disallow_untyped_defs = True + [mypy-synapse.storage.databases.main.user_erasure_store] disallow_untyped_defs = True diff --git a/synapse/storage/databases/main/transactions.py b/synapse/storage/databases/main/transactions.py index 54b41513ee..6c299cafa5 100644 --- a/synapse/storage/databases/main/transactions.py +++ b/synapse/storage/databases/main/transactions.py @@ -13,9 +13,8 @@ # limitations under the License. import logging -from collections import namedtuple from enum import Enum -from typing import TYPE_CHECKING, Iterable, List, Optional, Tuple +from typing import TYPE_CHECKING, Iterable, List, Optional, Tuple, cast import attr from canonicaljson import encode_canonical_json @@ -39,16 +38,6 @@ db_binary_type = memoryview logger = logging.getLogger(__name__) -_TransactionRow = namedtuple( - "_TransactionRow", - ("id", "transaction_id", "destination", "ts", "response_code", "response_json"), -) - -_UpdateTransactionRow = namedtuple( - "_TransactionRow", ("response_code", "response_json") -) - - class DestinationSortOrder(Enum): """Enum to define the sorting method used when returning destinations.""" @@ -91,7 +80,7 @@ class TransactionWorkerStore(CacheInvalidationWorkerStore): now = self._clock.time_msec() month_ago = now - 30 * 24 * 60 * 60 * 1000 - def _cleanup_transactions_txn(txn): + def _cleanup_transactions_txn(txn: LoggingTransaction) -> None: txn.execute("DELETE FROM received_transactions WHERE ts < ?", (month_ago,)) await self.db_pool.runInteraction( @@ -121,7 +110,9 @@ class TransactionWorkerStore(CacheInvalidationWorkerStore): origin, ) - def _get_received_txn_response(self, txn, transaction_id, origin): + def _get_received_txn_response( + self, txn: LoggingTransaction, transaction_id: str, origin: str + ) -> Optional[Tuple[int, JsonDict]]: result = self.db_pool.simple_select_one_txn( txn, table="received_transactions", @@ -196,7 +187,7 @@ class TransactionWorkerStore(CacheInvalidationWorkerStore): return result def _get_destination_retry_timings( - self, txn, destination: str + self, txn: LoggingTransaction, destination: str ) -> Optional[DestinationRetryTimings]: result = self.db_pool.simple_select_one_txn( txn, @@ -231,7 +222,7 @@ class TransactionWorkerStore(CacheInvalidationWorkerStore): """ if self.database_engine.can_native_upsert: - return await self.db_pool.runInteraction( + await self.db_pool.runInteraction( "set_destination_retry_timings", self._set_destination_retry_timings_native, destination, @@ -241,7 +232,7 @@ class TransactionWorkerStore(CacheInvalidationWorkerStore): db_autocommit=True, # Safe as its a single upsert ) else: - return await self.db_pool.runInteraction( + await self.db_pool.runInteraction( "set_destination_retry_timings", self._set_destination_retry_timings_emulated, destination, @@ -251,8 +242,13 @@ class TransactionWorkerStore(CacheInvalidationWorkerStore): ) def _set_destination_retry_timings_native( - self, txn, destination, failure_ts, retry_last_ts, retry_interval - ): + self, + txn: LoggingTransaction, + destination: str, + failure_ts: Optional[int], + retry_last_ts: int, + retry_interval: int, + ) -> None: assert self.database_engine.can_native_upsert # Upsert retry time interval if retry_interval is zero (i.e. we're @@ -282,8 +278,13 @@ class TransactionWorkerStore(CacheInvalidationWorkerStore): ) def _set_destination_retry_timings_emulated( - self, txn, destination, failure_ts, retry_last_ts, retry_interval - ): + self, + txn: LoggingTransaction, + destination: str, + failure_ts: Optional[int], + retry_last_ts: int, + retry_interval: int, + ) -> None: self.database_engine.lock_table(txn, "destinations") # We need to be careful here as the data may have changed from under us @@ -393,7 +394,7 @@ class TransactionWorkerStore(CacheInvalidationWorkerStore): last_successful_stream_ordering: the stream_ordering of the most recent successfully-sent PDU """ - return await self.db_pool.simple_upsert( + await self.db_pool.simple_upsert( "destinations", keyvalues={"destination": destination}, values={"last_successful_stream_ordering": last_successful_stream_ordering}, @@ -534,7 +535,7 @@ class TransactionWorkerStore(CacheInvalidationWorkerStore): else: order = "ASC" - args = [] + args: List[object] = [] where_statement = "" if destination: args.extend(["%" + destination.lower() + "%"]) @@ -543,7 +544,7 @@ class TransactionWorkerStore(CacheInvalidationWorkerStore): sql_base = f"FROM destinations {where_statement} " sql = f"SELECT COUNT(*) as total_destinations {sql_base}" txn.execute(sql, args) - count = txn.fetchone()[0] + count = cast(Tuple[int], txn.fetchone())[0] sql = f""" SELECT destination, retry_last_ts, retry_interval, failure_ts, From 8428ef66c73efcc01cdbe05ac8bc1c99c4c20a20 Mon Sep 17 00:00:00 2001 From: Dirk Klimpel <5740567+dklimpel@users.noreply.github.com> Date: Thu, 16 Dec 2021 20:59:56 +0100 Subject: [PATCH 03/76] Add type hints to `synapse/tests/rest/admin` (#11590) --- changelog.d/11590.misc | 1 + mypy.ini | 3 + tests/rest/admin/test_background_updates.py | 3 +- tests/rest/admin/test_federation.py | 33 ++++++----- tests/rest/admin/test_media.py | 4 +- tests/rest/admin/test_registration_tokens.py | 25 +++++--- tests/rest/admin/test_room.py | 62 ++++++++++---------- 7 files changed, 74 insertions(+), 57 deletions(-) create mode 100644 changelog.d/11590.misc diff --git a/changelog.d/11590.misc b/changelog.d/11590.misc new file mode 100644 index 0000000000..40e01194df --- /dev/null +++ b/changelog.d/11590.misc @@ -0,0 +1 @@ +Add type hints to `synapse/tests/rest/admin`. \ No newline at end of file diff --git a/mypy.ini b/mypy.ini index c546487bdb..c2cb71e0c0 100644 --- a/mypy.ini +++ b/mypy.ini @@ -242,6 +242,9 @@ disallow_untyped_defs = True [mypy-tests.storage.test_user_directory] disallow_untyped_defs = True +[mypy-tests.rest.admin.*] +disallow_untyped_defs = True + [mypy-tests.rest.client.test_directory] disallow_untyped_defs = True diff --git a/tests/rest/admin/test_background_updates.py b/tests/rest/admin/test_background_updates.py index 4d152c0d66..1e3fe9c62c 100644 --- a/tests/rest/admin/test_background_updates.py +++ b/tests/rest/admin/test_background_updates.py @@ -23,6 +23,7 @@ from synapse.api.errors import Codes from synapse.rest.client import login from synapse.server import HomeServer from synapse.storage.background_updates import BackgroundUpdater +from synapse.types import JsonDict from synapse.util import Clock from tests import unittest @@ -96,7 +97,7 @@ class BackgroundUpdatesTestCase(unittest.HomeserverTestCase): def _register_bg_update(self) -> None: "Adds a bg update but doesn't start it" - async def _fake_update(progress, batch_size) -> int: + async def _fake_update(progress: JsonDict, batch_size: int) -> int: await self.clock.sleep(0.2) return batch_size diff --git a/tests/rest/admin/test_federation.py b/tests/rest/admin/test_federation.py index d1cd5b0751..742f194257 100644 --- a/tests/rest/admin/test_federation.py +++ b/tests/rest/admin/test_federation.py @@ -16,11 +16,14 @@ from typing import List, Optional from parameterized import parameterized +from twisted.test.proto_helpers import MemoryReactor + import synapse.rest.admin from synapse.api.errors import Codes from synapse.rest.client import login from synapse.server import HomeServer from synapse.types import JsonDict +from synapse.util import Clock from tests import unittest @@ -31,7 +34,7 @@ class FederationTestCase(unittest.HomeserverTestCase): login.register_servlets, ] - def prepare(self, reactor, clock, hs: HomeServer): + def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None: self.store = hs.get_datastore() self.register_user("admin", "pass", admin=True) self.admin_user_tok = self.login("admin", "pass") @@ -44,7 +47,7 @@ class FederationTestCase(unittest.HomeserverTestCase): ("/_synapse/admin/v1/federation/destinations/dummy",), ] ) - def test_requester_is_no_admin(self, url: str): + def test_requester_is_no_admin(self, url: str) -> None: """ If the user is not a server admin, an error 403 is returned. """ @@ -62,7 +65,7 @@ class FederationTestCase(unittest.HomeserverTestCase): self.assertEqual(HTTPStatus.FORBIDDEN, channel.code, msg=channel.json_body) self.assertEqual(Codes.FORBIDDEN, channel.json_body["errcode"]) - def test_invalid_parameter(self): + def test_invalid_parameter(self) -> None: """ If parameters are invalid, an error is returned. """ @@ -117,7 +120,7 @@ class FederationTestCase(unittest.HomeserverTestCase): self.assertEqual(HTTPStatus.NOT_FOUND, channel.code, msg=channel.json_body) self.assertEqual(Codes.NOT_FOUND, channel.json_body["errcode"]) - def test_limit(self): + def test_limit(self) -> None: """ Testing list of destinations with limit """ @@ -137,7 +140,7 @@ class FederationTestCase(unittest.HomeserverTestCase): self.assertEqual(channel.json_body["next_token"], "5") self._check_fields(channel.json_body["destinations"]) - def test_from(self): + def test_from(self) -> None: """ Testing list of destinations with a defined starting point (from) """ @@ -157,7 +160,7 @@ class FederationTestCase(unittest.HomeserverTestCase): self.assertNotIn("next_token", channel.json_body) self._check_fields(channel.json_body["destinations"]) - def test_limit_and_from(self): + def test_limit_and_from(self) -> None: """ Testing list of destinations with a defined starting point and limit """ @@ -177,7 +180,7 @@ class FederationTestCase(unittest.HomeserverTestCase): self.assertEqual(len(channel.json_body["destinations"]), 10) self._check_fields(channel.json_body["destinations"]) - def test_next_token(self): + def test_next_token(self) -> None: """ Testing that `next_token` appears at the right place """ @@ -238,7 +241,7 @@ class FederationTestCase(unittest.HomeserverTestCase): self.assertEqual(len(channel.json_body["destinations"]), 1) self.assertNotIn("next_token", channel.json_body) - def test_list_all_destinations(self): + def test_list_all_destinations(self) -> None: """ List all destinations. """ @@ -259,7 +262,7 @@ class FederationTestCase(unittest.HomeserverTestCase): # Check that all fields are available self._check_fields(channel.json_body["destinations"]) - def test_order_by(self): + def test_order_by(self) -> None: """ Testing order list with parameter `order_by` """ @@ -268,7 +271,7 @@ class FederationTestCase(unittest.HomeserverTestCase): expected_destination_list: List[str], order_by: Optional[str], dir: Optional[str] = None, - ): + ) -> None: """Request the list of destinations in a certain order. Assert that order is what we expect @@ -358,13 +361,13 @@ class FederationTestCase(unittest.HomeserverTestCase): [dest[0][0], dest[2][0], dest[1][0]], "last_successful_stream_ordering", "b" ) - def test_search_term(self): + def test_search_term(self) -> None: """Test that searching for a destination works correctly""" def _search_test( expected_destination: Optional[str], search_term: str, - ): + ) -> None: """Search for a destination and check that the returned destinationis a match Args: @@ -410,7 +413,7 @@ class FederationTestCase(unittest.HomeserverTestCase): _search_test(None, "foo") _search_test(None, "bar") - def test_get_single_destination(self): + def test_get_single_destination(self) -> None: """ Get one specific destinations. """ @@ -429,7 +432,7 @@ class FederationTestCase(unittest.HomeserverTestCase): # convert channel.json_body into a List self._check_fields([channel.json_body]) - def _create_destinations(self, number_destinations: int): + def _create_destinations(self, number_destinations: int) -> None: """Create a number of destinations Args: @@ -442,7 +445,7 @@ class FederationTestCase(unittest.HomeserverTestCase): self.store.set_destination_last_successful_stream_ordering(dest, 100) ) - def _check_fields(self, content: List[JsonDict]): + def _check_fields(self, content: List[JsonDict]) -> None: """Checks that the expected destination attributes are present in content Args: diff --git a/tests/rest/admin/test_media.py b/tests/rest/admin/test_media.py index 3f727788ce..86aff7575c 100644 --- a/tests/rest/admin/test_media.py +++ b/tests/rest/admin/test_media.py @@ -580,7 +580,9 @@ class DeleteMediaByDateSizeTestCase(unittest.HomeserverTestCase): return server_and_media_id - def _access_media(self, server_and_media_id, expect_success=True) -> None: + def _access_media( + self, server_and_media_id: str, expect_success: bool = True + ) -> None: """ Try to access a media and check the result """ diff --git a/tests/rest/admin/test_registration_tokens.py b/tests/rest/admin/test_registration_tokens.py index 350a62dda6..81f3ac7f04 100644 --- a/tests/rest/admin/test_registration_tokens.py +++ b/tests/rest/admin/test_registration_tokens.py @@ -14,6 +14,7 @@ import random import string from http import HTTPStatus +from typing import Optional from twisted.test.proto_helpers import MemoryReactor @@ -42,21 +43,27 @@ class ManageRegistrationTokensTestCase(unittest.HomeserverTestCase): self.url = "/_synapse/admin/v1/registration_tokens" - def _new_token(self, **kwargs) -> str: + def _new_token( + self, + token: Optional[str] = None, + uses_allowed: Optional[int] = None, + pending: int = 0, + completed: int = 0, + expiry_time: Optional[int] = None, + ) -> str: """Helper function to create a token.""" - token = kwargs.get( - "token", - "".join(random.choices(string.ascii_letters, k=8)), - ) + if token is None: + token = "".join(random.choices(string.ascii_letters, k=8)) + self.get_success( self.store.db_pool.simple_insert( "registration_tokens", { "token": token, - "uses_allowed": kwargs.get("uses_allowed", None), - "pending": kwargs.get("pending", 0), - "completed": kwargs.get("completed", 0), - "expiry_time": kwargs.get("expiry_time", None), + "uses_allowed": uses_allowed, + "pending": pending, + "completed": completed, + "expiry_time": expiry_time, }, ) ) diff --git a/tests/rest/admin/test_room.py b/tests/rest/admin/test_room.py index 22f9aa6234..d2c8781cd4 100644 --- a/tests/rest/admin/test_room.py +++ b/tests/rest/admin/test_room.py @@ -66,7 +66,7 @@ class DeleteRoomTestCase(unittest.HomeserverTestCase): ) self.url = "/_synapse/admin/v1/rooms/%s" % self.room_id - def test_requester_is_no_admin(self): + def test_requester_is_no_admin(self) -> None: """ If the user is not a server admin, an error HTTPStatus.FORBIDDEN is returned. """ @@ -81,7 +81,7 @@ class DeleteRoomTestCase(unittest.HomeserverTestCase): self.assertEqual(HTTPStatus.FORBIDDEN, channel.code, msg=channel.json_body) self.assertEqual(Codes.FORBIDDEN, channel.json_body["errcode"]) - def test_room_does_not_exist(self): + def test_room_does_not_exist(self) -> None: """ Check that unknown rooms/server return 200 """ @@ -96,7 +96,7 @@ class DeleteRoomTestCase(unittest.HomeserverTestCase): self.assertEqual(HTTPStatus.OK, channel.code, msg=channel.json_body) - def test_room_is_not_valid(self): + def test_room_is_not_valid(self) -> None: """ Check that invalid room names, return an error HTTPStatus.BAD_REQUEST. """ @@ -115,7 +115,7 @@ class DeleteRoomTestCase(unittest.HomeserverTestCase): channel.json_body["error"], ) - def test_new_room_user_does_not_exist(self): + def test_new_room_user_does_not_exist(self) -> None: """ Tests that the user ID must be from local server but it does not have to exist. """ @@ -133,7 +133,7 @@ class DeleteRoomTestCase(unittest.HomeserverTestCase): self.assertIn("failed_to_kick_users", channel.json_body) self.assertIn("local_aliases", channel.json_body) - def test_new_room_user_is_not_local(self): + def test_new_room_user_is_not_local(self) -> None: """ Check that only local users can create new room to move members. """ @@ -151,7 +151,7 @@ class DeleteRoomTestCase(unittest.HomeserverTestCase): channel.json_body["error"], ) - def test_block_is_not_bool(self): + def test_block_is_not_bool(self) -> None: """ If parameter `block` is not boolean, return an error """ @@ -166,7 +166,7 @@ class DeleteRoomTestCase(unittest.HomeserverTestCase): self.assertEqual(HTTPStatus.BAD_REQUEST, channel.code, msg=channel.json_body) self.assertEqual(Codes.BAD_JSON, channel.json_body["errcode"]) - def test_purge_is_not_bool(self): + def test_purge_is_not_bool(self) -> None: """ If parameter `purge` is not boolean, return an error """ @@ -181,7 +181,7 @@ class DeleteRoomTestCase(unittest.HomeserverTestCase): self.assertEqual(HTTPStatus.BAD_REQUEST, channel.code, msg=channel.json_body) self.assertEqual(Codes.BAD_JSON, channel.json_body["errcode"]) - def test_purge_room_and_block(self): + def test_purge_room_and_block(self) -> None: """Test to purge a room and block it. Members will not be moved to a new room and will not receive a message. """ @@ -212,7 +212,7 @@ class DeleteRoomTestCase(unittest.HomeserverTestCase): self._is_blocked(self.room_id, expect=True) self._has_no_members(self.room_id) - def test_purge_room_and_not_block(self): + def test_purge_room_and_not_block(self) -> None: """Test to purge a room and do not block it. Members will not be moved to a new room and will not receive a message. """ @@ -243,7 +243,7 @@ class DeleteRoomTestCase(unittest.HomeserverTestCase): self._is_blocked(self.room_id, expect=False) self._has_no_members(self.room_id) - def test_block_room_and_not_purge(self): + def test_block_room_and_not_purge(self) -> None: """Test to block a room without purging it. Members will not be moved to a new room and will not receive a message. The room will not be purged. @@ -299,7 +299,7 @@ class DeleteRoomTestCase(unittest.HomeserverTestCase): self.assertEqual(HTTPStatus.OK, channel.code, msg=channel.json_body) self._is_blocked(room_id) - def test_shutdown_room_consent(self): + def test_shutdown_room_consent(self) -> None: """Test that we can shutdown rooms with local users who have not yet accepted the privacy policy. This used to fail when we tried to force part the user from the old room. @@ -351,7 +351,7 @@ class DeleteRoomTestCase(unittest.HomeserverTestCase): self._is_purged(self.room_id) self._has_no_members(self.room_id) - def test_shutdown_room_block_peek(self): + def test_shutdown_room_block_peek(self) -> None: """Test that a world_readable room can no longer be peeked into after it has been shut down. Members will be moved to a new room and will receive a message. @@ -400,7 +400,7 @@ class DeleteRoomTestCase(unittest.HomeserverTestCase): # Assert we can no longer peek into the room self._assert_peek(self.room_id, expect_code=HTTPStatus.FORBIDDEN) - def _is_blocked(self, room_id, expect=True): + def _is_blocked(self, room_id: str, expect: bool = True) -> None: """Assert that the room is blocked or not""" d = self.store.is_room_blocked(room_id) if expect: @@ -408,17 +408,17 @@ class DeleteRoomTestCase(unittest.HomeserverTestCase): else: self.assertIsNone(self.get_success(d)) - def _has_no_members(self, room_id): + def _has_no_members(self, room_id: str) -> None: """Assert there is now no longer anyone in the room""" users_in_room = self.get_success(self.store.get_users_in_room(room_id)) self.assertEqual([], users_in_room) - def _is_member(self, room_id, user_id): + def _is_member(self, room_id: str, user_id: str) -> None: """Test that user is member of the room""" users_in_room = self.get_success(self.store.get_users_in_room(room_id)) self.assertIn(user_id, users_in_room) - def _is_purged(self, room_id): + def _is_purged(self, room_id: str) -> None: """Test that the following tables have been purged of all rows related to the room.""" for table in PURGE_TABLES: count = self.get_success( @@ -432,7 +432,7 @@ class DeleteRoomTestCase(unittest.HomeserverTestCase): self.assertEqual(count, 0, msg=f"Rows not purged in {table}") - def _assert_peek(self, room_id, expect_code): + def _assert_peek(self, room_id: str, expect_code: int) -> None: """Assert that the admin user can (or cannot) peek into the room.""" url = "rooms/%s/initialSync" % (room_id,) @@ -492,7 +492,7 @@ class DeleteRoomV2TestCase(unittest.HomeserverTestCase): ("GET", "/_synapse/admin/v2/rooms/delete_status/%s"), ] ) - def test_requester_is_no_admin(self, method: str, url: str): + def test_requester_is_no_admin(self, method: str, url: str) -> None: """ If the user is not a server admin, an error HTTPStatus.FORBIDDEN is returned. """ @@ -507,7 +507,7 @@ class DeleteRoomV2TestCase(unittest.HomeserverTestCase): self.assertEqual(HTTPStatus.FORBIDDEN, channel.code, msg=channel.json_body) self.assertEqual(Codes.FORBIDDEN, channel.json_body["errcode"]) - def test_room_does_not_exist(self): + def test_room_does_not_exist(self) -> None: """ Check that unknown rooms/server return 200 @@ -544,7 +544,7 @@ class DeleteRoomV2TestCase(unittest.HomeserverTestCase): ("GET", "/_synapse/admin/v2/rooms/%s/delete_status"), ] ) - def test_room_is_not_valid(self, method: str, url: str): + def test_room_is_not_valid(self, method: str, url: str) -> None: """ Check that invalid room names, return an error HTTPStatus.BAD_REQUEST. """ @@ -562,7 +562,7 @@ class DeleteRoomV2TestCase(unittest.HomeserverTestCase): channel.json_body["error"], ) - def test_new_room_user_does_not_exist(self): + def test_new_room_user_does_not_exist(self) -> None: """ Tests that the user ID must be from local server but it does not have to exist. """ @@ -580,7 +580,7 @@ class DeleteRoomV2TestCase(unittest.HomeserverTestCase): self._test_result(delete_id, self.other_user, expect_new_room=True) - def test_new_room_user_is_not_local(self): + def test_new_room_user_is_not_local(self) -> None: """ Check that only local users can create new room to move members. """ @@ -598,7 +598,7 @@ class DeleteRoomV2TestCase(unittest.HomeserverTestCase): channel.json_body["error"], ) - def test_block_is_not_bool(self): + def test_block_is_not_bool(self) -> None: """ If parameter `block` is not boolean, return an error """ @@ -613,7 +613,7 @@ class DeleteRoomV2TestCase(unittest.HomeserverTestCase): self.assertEqual(HTTPStatus.BAD_REQUEST, channel.code, msg=channel.json_body) self.assertEqual(Codes.BAD_JSON, channel.json_body["errcode"]) - def test_purge_is_not_bool(self): + def test_purge_is_not_bool(self) -> None: """ If parameter `purge` is not boolean, return an error """ @@ -628,7 +628,7 @@ class DeleteRoomV2TestCase(unittest.HomeserverTestCase): self.assertEqual(HTTPStatus.BAD_REQUEST, channel.code, msg=channel.json_body) self.assertEqual(Codes.BAD_JSON, channel.json_body["errcode"]) - def test_delete_expired_status(self): + def test_delete_expired_status(self) -> None: """Test that the task status is removed after expiration.""" # first task, do not purge, that we can create a second task @@ -699,7 +699,7 @@ class DeleteRoomV2TestCase(unittest.HomeserverTestCase): self.assertEqual(HTTPStatus.NOT_FOUND, channel.code, msg=channel.json_body) self.assertEqual(Codes.NOT_FOUND, channel.json_body["errcode"]) - def test_delete_same_room_twice(self): + def test_delete_same_room_twice(self) -> None: """Test that the call for delete a room at second time gives an exception.""" body = {"new_room_user_id": self.admin_user} @@ -743,7 +743,7 @@ class DeleteRoomV2TestCase(unittest.HomeserverTestCase): expect_new_room=True, ) - def test_purge_room_and_block(self): + def test_purge_room_and_block(self) -> None: """Test to purge a room and block it. Members will not be moved to a new room and will not receive a message. """ @@ -774,7 +774,7 @@ class DeleteRoomV2TestCase(unittest.HomeserverTestCase): self._is_blocked(self.room_id, expect=True) self._has_no_members(self.room_id) - def test_purge_room_and_not_block(self): + def test_purge_room_and_not_block(self) -> None: """Test to purge a room and do not block it. Members will not be moved to a new room and will not receive a message. """ @@ -805,7 +805,7 @@ class DeleteRoomV2TestCase(unittest.HomeserverTestCase): self._is_blocked(self.room_id, expect=False) self._has_no_members(self.room_id) - def test_block_room_and_not_purge(self): + def test_block_room_and_not_purge(self) -> None: """Test to block a room without purging it. Members will not be moved to a new room and will not receive a message. The room will not be purged. @@ -838,7 +838,7 @@ class DeleteRoomV2TestCase(unittest.HomeserverTestCase): self._is_blocked(self.room_id, expect=True) self._has_no_members(self.room_id) - def test_shutdown_room_consent(self): + def test_shutdown_room_consent(self) -> None: """Test that we can shutdown rooms with local users who have not yet accepted the privacy policy. This used to fail when we tried to force part the user from the old room. @@ -899,7 +899,7 @@ class DeleteRoomV2TestCase(unittest.HomeserverTestCase): self._is_purged(self.room_id) self._has_no_members(self.room_id) - def test_shutdown_room_block_peek(self): + def test_shutdown_room_block_peek(self) -> None: """Test that a world_readable room can no longer be peeked into after it has been shut down. Members will be moved to a new room and will receive a message. From 8ad39438fafef653d019e3214037c96257507e55 Mon Sep 17 00:00:00 2001 From: Shay Date: Mon, 20 Dec 2021 04:18:09 -0800 Subject: [PATCH 04/76] Add opentracing types (#11603) --- changelog.d/11603.misc | 1 + mypy.ini | 3 --- setup.py | 1 + synapse/logging/opentracing.py | 24 +++++++++++++++--------- synapse/logging/scopecontextmanager.py | 2 +- 5 files changed, 18 insertions(+), 13 deletions(-) create mode 100644 changelog.d/11603.misc diff --git a/changelog.d/11603.misc b/changelog.d/11603.misc new file mode 100644 index 0000000000..def24afb8d --- /dev/null +++ b/changelog.d/11603.misc @@ -0,0 +1 @@ +Add opentracing type stubs and fix associated mypy errors. \ No newline at end of file diff --git a/mypy.ini b/mypy.ini index c2cb71e0c0..3279c9bb21 100644 --- a/mypy.ini +++ b/mypy.ini @@ -308,9 +308,6 @@ ignore_missing_imports = True [mypy-netaddr] ignore_missing_imports = True -[mypy-opentracing] -ignore_missing_imports = True - [mypy-parameterized.*] ignore_missing_imports = True diff --git a/setup.py b/setup.py index 2c6fb9aacb..812459074a 100755 --- a/setup.py +++ b/setup.py @@ -107,6 +107,7 @@ CONDITIONAL_REQUIREMENTS["mypy"] = [ "mypy-zope==0.3.2", "types-bleach>=4.1.0", "types-jsonschema>=3.2.0", + "types-opentracing>=2.4.2", "types-Pillow>=8.3.4", "types-pyOpenSSL>=20.0.7", "types-PyYAML>=5.4.10", diff --git a/synapse/logging/opentracing.py b/synapse/logging/opentracing.py index 20d23a4260..5d93ab07f1 100644 --- a/synapse/logging/opentracing.py +++ b/synapse/logging/opentracing.py @@ -222,8 +222,8 @@ try: tags = opentracing.tags except ImportError: - opentracing = None - tags = _DummyTagNames + opentracing = None # type: ignore[assignment] + tags = _DummyTagNames # type: ignore[assignment] try: from jaeger_client import Config as JaegerConfig @@ -366,7 +366,7 @@ def init_tracer(hs: "HomeServer"): global opentracing if not hs.config.tracing.opentracer_enabled: # We don't have a tracer - opentracing = None + opentracing = None # type: ignore[assignment] return if not opentracing or not JaegerConfig: @@ -452,7 +452,7 @@ def start_active_span( """ if opentracing is None: - return noop_context_manager() + return noop_context_manager() # type: ignore[unreachable] return opentracing.tracer.start_active_span( operation_name, @@ -477,7 +477,7 @@ def start_active_span_follows_from( forced, the new span will also have tracing forced. """ if opentracing is None: - return noop_context_manager() + return noop_context_manager() # type: ignore[unreachable] references = [opentracing.follows_from(context) for context in contexts] scope = start_active_span(operation_name, references=references) @@ -514,7 +514,7 @@ def start_active_span_from_request( # Also, twisted uses byte arrays while opentracing expects strings. if opentracing is None: - return noop_context_manager() + return noop_context_manager() # type: ignore[unreachable] header_dict = { k.decode(): v[0].decode() for k, v in request.requestHeaders.getAllRawHeaders() @@ -553,7 +553,7 @@ def start_active_span_from_edu( references = references or [] if opentracing is None: - return noop_context_manager() + return noop_context_manager() # type: ignore[unreachable] carrier = json_decoder.decode(edu_content.get("context", "{}")).get( "opentracing", {} @@ -594,18 +594,21 @@ def active_span(): @ensure_active_span("set a tag") def set_tag(key, value): """Sets a tag on the active span""" + assert opentracing.tracer.active_span is not None opentracing.tracer.active_span.set_tag(key, value) @ensure_active_span("log") def log_kv(key_values, timestamp=None): """Log to the active span""" + assert opentracing.tracer.active_span is not None opentracing.tracer.active_span.log_kv(key_values, timestamp) @ensure_active_span("set the traces operation name") def set_operation_name(operation_name): """Sets the operation name of the active span""" + assert opentracing.tracer.active_span is not None opentracing.tracer.active_span.set_operation_name(operation_name) @@ -674,6 +677,7 @@ def inject_header_dict( span = opentracing.tracer.active_span carrier: Dict[str, str] = {} + assert span is not None opentracing.tracer.inject(span.context, opentracing.Format.HTTP_HEADERS, carrier) for key, value in carrier.items(): @@ -716,6 +720,7 @@ def get_active_span_text_map(destination=None): return {} carrier: Dict[str, str] = {} + assert opentracing.tracer.active_span is not None opentracing.tracer.inject( opentracing.tracer.active_span.context, opentracing.Format.TEXT_MAP, carrier ) @@ -731,6 +736,7 @@ def active_span_context_as_string(): """ carrier: Dict[str, str] = {} if opentracing: + assert opentracing.tracer.active_span is not None opentracing.tracer.inject( opentracing.tracer.active_span.context, opentracing.Format.TEXT_MAP, carrier ) @@ -773,7 +779,7 @@ def trace(func=None, opname=None): def decorator(func): if opentracing is None: - return func + return func # type: ignore[unreachable] _opname = opname if opname else func.__name__ @@ -864,7 +870,7 @@ def trace_servlet(request: "SynapseRequest", extract_context: bool = False): """ if opentracing is None: - yield + yield # type: ignore[unreachable] return request_tags = { diff --git a/synapse/logging/scopecontextmanager.py b/synapse/logging/scopecontextmanager.py index b1e8e08fe9..db8ca2c049 100644 --- a/synapse/logging/scopecontextmanager.py +++ b/synapse/logging/scopecontextmanager.py @@ -71,7 +71,7 @@ class LogContextScopeManager(ScopeManager): if not ctx: # We don't want this scope to affect. logger.error("Tried to activate scope outside of loggingcontext") - return Scope(None, span) + return Scope(None, span) # type: ignore[arg-type] elif ctx.scope is not None: # We want the logging scope to look exactly the same so we give it # a blank suffix From 7a7ca8f2263f8750b8ff2c18b9aefaf4a3626235 Mon Sep 17 00:00:00 2001 From: V02460 Date: Mon, 20 Dec 2021 16:34:46 +0100 Subject: [PATCH 05/76] Use mock from standard library (#11588) Instead of the backported version. --- changelog.d/11588.removal | 1 + setup.py | 4 +--- tests/storage/test_background_update.py | 17 ++++++++--------- 3 files changed, 10 insertions(+), 12 deletions(-) create mode 100644 changelog.d/11588.removal diff --git a/changelog.d/11588.removal b/changelog.d/11588.removal new file mode 100644 index 0000000000..f781021e11 --- /dev/null +++ b/changelog.d/11588.removal @@ -0,0 +1 @@ +Replace `mock` package by its standard library version. diff --git a/setup.py b/setup.py index 812459074a..e113da6782 100755 --- a/setup.py +++ b/setup.py @@ -120,9 +120,7 @@ CONDITIONAL_REQUIREMENTS["mypy"] = [ # Tests assume that all optional dependencies are installed. # # parameterized_class decorator was introduced in parameterized 0.7.0 -# -# We use `mock` library as that backports `AsyncMock` to Python 3.6 -CONDITIONAL_REQUIREMENTS["test"] = ["parameterized>=0.7.0", "mock>=4.0.0"] +CONDITIONAL_REQUIREMENTS["test"] = ["parameterized>=0.7.0"] CONDITIONAL_REQUIREMENTS["dev"] = ( CONDITIONAL_REQUIREMENTS["lint"] diff --git a/tests/storage/test_background_update.py b/tests/storage/test_background_update.py index d77c001506..6156dfac4e 100644 --- a/tests/storage/test_background_update.py +++ b/tests/storage/test_background_update.py @@ -12,15 +12,14 @@ # See the License for the specific language governing permissions and # limitations under the License. -# Use backported mock for AsyncMock support on Python 3.6. -from mock import Mock +from unittest.mock import Mock from twisted.internet.defer import Deferred, ensureDeferred from synapse.storage.background_updates import BackgroundUpdater from tests import unittest -from tests.test_utils import make_awaitable +from tests.test_utils import make_awaitable, simple_async_mock class BackgroundUpdateTestCase(unittest.HomeserverTestCase): @@ -116,14 +115,14 @@ class BackgroundUpdateControllerTestCase(unittest.HomeserverTestCase): ) # Mock out the AsyncContextManager - self._update_ctx_manager = Mock(spec=["__aenter__", "__aexit__"]) - self._update_ctx_manager.__aenter__ = Mock( - return_value=make_awaitable(None), - ) - self._update_ctx_manager.__aexit__ = Mock(return_value=make_awaitable(None)) + class MockCM: + __aenter__ = simple_async_mock(return_value=None) + __aexit__ = simple_async_mock(return_value=None) + + self._update_ctx_manager = MockCM # Mock out the `update_handler` callback - self._on_update = Mock(return_value=self._update_ctx_manager) + self._on_update = Mock(return_value=self._update_ctx_manager()) # Define a default batch size value that's not the same as the internal default # value (100). From 3e0cfd447e17658a937fe62555db9e968f00b15b Mon Sep 17 00:00:00 2001 From: Patrick Cloke Date: Mon, 20 Dec 2021 11:00:13 -0500 Subject: [PATCH 06/76] Return JSON errors for unknown resources under /matrix/client. (#11602) Instead of returning 404 errors with HTML bodies when an unknown prefix was requested (e.g. /matrix/client/v1 before Synapse v1.49.0). --- changelog.d/11602.bugfix | 1 + synapse/app/homeserver.py | 9 ++------- synapse/http/server.py | 6 +++--- 3 files changed, 6 insertions(+), 10 deletions(-) create mode 100644 changelog.d/11602.bugfix diff --git a/changelog.d/11602.bugfix b/changelog.d/11602.bugfix new file mode 100644 index 0000000000..e0dfbf1a15 --- /dev/null +++ b/changelog.d/11602.bugfix @@ -0,0 +1 @@ +Fix a long-standing bug that some unknown endpoints would return HTML error pages instead of JSON `M_UNRECOGNIZED` errors. diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index dd76e07321..177ce040e8 100644 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -27,6 +27,7 @@ import synapse import synapse.config.logger from synapse import events from synapse.api.urls import ( + CLIENT_API_PREFIX, FEDERATION_PREFIX, LEGACY_MEDIA_PREFIX, MEDIA_R0_PREFIX, @@ -192,13 +193,7 @@ class SynapseHomeServer(HomeServer): resources.update( { - "/_matrix/client/api/v1": client_resource, - "/_matrix/client/r0": client_resource, - "/_matrix/client/v1": client_resource, - "/_matrix/client/v3": client_resource, - "/_matrix/client/unstable": client_resource, - "/_matrix/client/v2_alpha": client_resource, - "/_matrix/client/versions": client_resource, + CLIENT_API_PREFIX: client_resource, "/.well-known": well_known_resource(self), "/_synapse/admin": AdminRestResource(self), **build_synapse_client_resource_tree(self), diff --git a/synapse/http/server.py b/synapse/http/server.py index 4fd5660a08..7bbbe7648b 100644 --- a/synapse/http/server.py +++ b/synapse/http/server.py @@ -530,7 +530,7 @@ class RootRedirect(resource.Resource): """Redirects the root '/' path to another path.""" def __init__(self, path: str): - resource.Resource.__init__(self) + super().__init__() self.url = path def render_GET(self, request: Request) -> bytes: @@ -539,7 +539,7 @@ class RootRedirect(resource.Resource): def getChild(self, name: str, request: Request) -> resource.Resource: if len(name) == 0: return self # select ourselves as the child to render - return resource.Resource.getChild(self, name, request) + return super().getChild(name, request) class OptionsResource(resource.Resource): @@ -556,7 +556,7 @@ class OptionsResource(resource.Resource): def getChildWithDefault(self, path: str, request: Request) -> resource.Resource: if request.method == b"OPTIONS": return self # select ourselves as the child to render - return resource.Resource.getChildWithDefault(self, path, request) + return super().getChildWithDefault(path, request) class RootOptionsRedirectResource(OptionsResource, RootRedirect): From 76aa5537ad4f41cad130862a230c1f4cc4bfcbcf Mon Sep 17 00:00:00 2001 From: Richard van der Hoff <1389908+richvdh@users.noreply.github.com> Date: Mon, 20 Dec 2021 16:33:35 +0000 Subject: [PATCH 07/76] Disable aggregation bundling on `/sync` responses (#11583) * Disable aggregation bundling on `/sync` responses A partial revert of #11478. This turns out to have had a significant CPU impact on initial-sync handling. For now, let's disable it, until we find a more efficient way of achieving this. * Fix tests. Co-authored-by: Patrick Cloke --- changelog.d/11583.bugfix | 1 + synapse/rest/client/sync.py | 10 +++++++++- tests/rest/client/test_relations.py | 10 +++++----- 3 files changed, 15 insertions(+), 6 deletions(-) create mode 100644 changelog.d/11583.bugfix diff --git a/changelog.d/11583.bugfix b/changelog.d/11583.bugfix new file mode 100644 index 0000000000..d2ed113e21 --- /dev/null +++ b/changelog.d/11583.bugfix @@ -0,0 +1 @@ +Fix a performance regression in `/sync` handling, introduced in 1.49.0. diff --git a/synapse/rest/client/sync.py b/synapse/rest/client/sync.py index 88e4f5e063..7f5846d389 100644 --- a/synapse/rest/client/sync.py +++ b/synapse/rest/client/sync.py @@ -522,7 +522,15 @@ class SyncRestServlet(RestServlet): time_now=time_now, # Don't bother to bundle aggregations if the timeline is unlimited, # as clients will have all the necessary information. - bundle_aggregations=room.timeline.limited, + # bundle_aggregations=room.timeline.limited, + # + # richvdh 2021-12-15: disable this temporarily as it has too high an + # overhead for initialsyncs. We need to figure out a way that the + # bundling can be done *before* the events are stored in the + # SyncResponseCache so that this part can be synchronous. + # + # Ensure to re-enable the test at tests/rest/client/test_relations.py::RelationsTestCase.test_bundled_aggregations. + bundle_aggregations=False, token_id=token_id, event_format=event_formatter, only_event_fields=only_fields, diff --git a/tests/rest/client/test_relations.py b/tests/rest/client/test_relations.py index 397c12c2a6..1b58b73136 100644 --- a/tests/rest/client/test_relations.py +++ b/tests/rest/client/test_relations.py @@ -574,11 +574,11 @@ class RelationsTestCase(unittest.HomeserverTestCase): assert_bundle(channel.json_body["event"]["unsigned"].get("m.relations")) # Request sync. - channel = self.make_request("GET", "/sync", access_token=self.user_token) - self.assertEquals(200, channel.code, channel.json_body) - room_timeline = channel.json_body["rooms"]["join"][self.room]["timeline"] - self.assertTrue(room_timeline["limited"]) - _find_and_assert_event(room_timeline["events"]) + # channel = self.make_request("GET", "/sync", access_token=self.user_token) + # self.assertEquals(200, channel.code, channel.json_body) + # room_timeline = channel.json_body["rooms"]["join"][self.room]["timeline"] + # self.assertTrue(room_timeline["limited"]) + # _find_and_assert_event(room_timeline["events"]) # Note that /relations is tested separately in test_aggregation_get_event_for_thread # since it needs different data configured. From 60fa4935b5d3ee26f9ebb4b25ec74bed26d3c98d Mon Sep 17 00:00:00 2001 From: Richard van der Hoff <1389908+richvdh@users.noreply.github.com> Date: Mon, 20 Dec 2021 17:45:03 +0000 Subject: [PATCH 08/76] Improve opentracing for incoming HTTP requests (#11618) * remove `start_active_span_from_request` Instead, pull out a separate function, `span_context_from_request`, to extract the parent span, which we can then pass into `start_active_span` as normal. This seems to be clearer all round. * Remove redundant tags from `incoming-federation-request` These are all wrapped up inside a parent span generated in AsyncResource, so there's no point duplicating all the tags that are set there. * Leave request spans open until the request completes It may take some time for the response to be encoded into JSON, and that JSON to be streamed back to the client, and really we want that inside the top-level span, so let's hand responsibility for closure to the SynapseRequest. * opentracing logs for HTTP request events * changelog --- changelog.d/11618.misc | 1 + synapse/federation/transport/server/_base.py | 37 ++++------- synapse/http/site.py | 30 ++++++++- synapse/logging/opentracing.py | 68 +++++++------------- 4 files changed, 64 insertions(+), 72 deletions(-) create mode 100644 changelog.d/11618.misc diff --git a/changelog.d/11618.misc b/changelog.d/11618.misc new file mode 100644 index 0000000000..4076b30bf7 --- /dev/null +++ b/changelog.d/11618.misc @@ -0,0 +1 @@ +Improve opentracing support for incoming HTTP requests. diff --git a/synapse/federation/transport/server/_base.py b/synapse/federation/transport/server/_base.py index dc39e3537b..da1fbf8b63 100644 --- a/synapse/federation/transport/server/_base.py +++ b/synapse/federation/transport/server/_base.py @@ -22,13 +22,11 @@ from synapse.api.urls import FEDERATION_V1_PREFIX from synapse.http.server import HttpServer, ServletCallback from synapse.http.servlet import parse_json_object_from_request from synapse.http.site import SynapseRequest -from synapse.logging import opentracing from synapse.logging.context import run_in_background from synapse.logging.opentracing import ( - SynapseTags, - start_active_span, - start_active_span_from_request, - tags, + set_tag, + span_context_from_request, + start_active_span_follows_from, whitelisted_homeserver, ) from synapse.server import HomeServer @@ -279,30 +277,19 @@ class BaseFederationServlet: logger.warning("authenticate_request failed: %s", e) raise - request_tags = { - SynapseTags.REQUEST_ID: request.get_request_id(), - tags.SPAN_KIND: tags.SPAN_KIND_RPC_SERVER, - tags.HTTP_METHOD: request.get_method(), - tags.HTTP_URL: request.get_redacted_uri(), - tags.PEER_HOST_IPV6: request.getClientIP(), - "authenticated_entity": origin, - "servlet_name": request.request_metrics.name, - } + # update the active opentracing span with the authenticated entity + set_tag("authenticated_entity", origin) - # Only accept the span context if the origin is authenticated - # and whitelisted + # if the origin is authenticated and whitelisted, link to its span context + context = None if origin and whitelisted_homeserver(origin): - scope = start_active_span_from_request( - request, "incoming-federation-request", tags=request_tags - ) - else: - scope = start_active_span( - "incoming-federation-request", tags=request_tags - ) + context = span_context_from_request(request) + + scope = start_active_span_follows_from( + "incoming-federation-request", contexts=(context,) if context else () + ) with scope: - opentracing.inject_response_headers(request.responseHeaders) - if origin and self.RATELIMIT: with ratelimiter.ratelimit(origin) as d: await d diff --git a/synapse/http/site.py b/synapse/http/site.py index 9f68d7e191..80f7a2ff58 100644 --- a/synapse/http/site.py +++ b/synapse/http/site.py @@ -14,7 +14,7 @@ import contextlib import logging import time -from typing import Any, Generator, Optional, Tuple, Union +from typing import TYPE_CHECKING, Any, Generator, Optional, Tuple, Union import attr from zope.interface import implementer @@ -35,6 +35,9 @@ from synapse.logging.context import ( ) from synapse.types import Requester +if TYPE_CHECKING: + import opentracing + logger = logging.getLogger(__name__) _next_request_seq = 0 @@ -81,6 +84,10 @@ class SynapseRequest(Request): # server name, for client requests this is the Requester object. self._requester: Optional[Union[Requester, str]] = None + # An opentracing span for this request. Will be closed when the request is + # completely processed. + self._opentracing_span: "Optional[opentracing.Span]" = None + # we can't yet create the logcontext, as we don't know the method. self.logcontext: Optional[LoggingContext] = None @@ -148,6 +155,13 @@ class SynapseRequest(Request): # If there's no authenticated entity, it was the requester. self.logcontext.request.authenticated_entity = authenticated_entity or requester + def set_opentracing_span(self, span: "opentracing.Span") -> None: + """attach an opentracing span to this request + + Doing so will cause the span to be closed when we finish processing the request + """ + self._opentracing_span = span + def get_request_id(self) -> str: return "%s-%i" % (self.get_method(), self.request_seq) @@ -286,6 +300,9 @@ class SynapseRequest(Request): self._processing_finished_time = time.time() self._is_processing = False + if self._opentracing_span: + self._opentracing_span.log_kv({"event": "finished processing"}) + # if we've already sent the response, log it now; otherwise, we wait for the # response to be sent. if self.finish_time is not None: @@ -299,6 +316,8 @@ class SynapseRequest(Request): """ self.finish_time = time.time() Request.finish(self) + if self._opentracing_span: + self._opentracing_span.log_kv({"event": "response sent"}) if not self._is_processing: assert self.logcontext is not None with PreserveLoggingContext(self.logcontext): @@ -333,6 +352,11 @@ class SynapseRequest(Request): with PreserveLoggingContext(self.logcontext): logger.info("Connection from client lost before response was sent") + if self._opentracing_span: + self._opentracing_span.log_kv( + {"event": "client connection lost", "reason": str(reason.value)} + ) + if not self._is_processing: self._finished_processing() @@ -421,6 +445,10 @@ class SynapseRequest(Request): usage.evt_db_fetch_count, ) + # complete the opentracing span, if any. + if self._opentracing_span: + self._opentracing_span.finish() + try: self.request_metrics.stop(self.finish_time, self.code, self.sentLength) except Exception as e: diff --git a/synapse/logging/opentracing.py b/synapse/logging/opentracing.py index 5d93ab07f1..6364290615 100644 --- a/synapse/logging/opentracing.py +++ b/synapse/logging/opentracing.py @@ -173,6 +173,7 @@ from typing import TYPE_CHECKING, Collection, Dict, List, Optional, Pattern, Typ import attr from twisted.internet import defer +from twisted.web.http import Request from twisted.web.http_headers import Headers from synapse.config import ConfigError @@ -490,48 +491,6 @@ def start_active_span_follows_from( return scope -def start_active_span_from_request( - request, - operation_name, - references=None, - tags=None, - start_time=None, - ignore_active_span=False, - finish_on_close=True, -): - """ - Extracts a span context from a Twisted Request. - args: - headers (twisted.web.http.Request) - - For the other args see opentracing.tracer - - returns: - span_context (opentracing.span.SpanContext) - """ - # Twisted encodes the values as lists whereas opentracing doesn't. - # So, we take the first item in the list. - # Also, twisted uses byte arrays while opentracing expects strings. - - if opentracing is None: - return noop_context_manager() # type: ignore[unreachable] - - header_dict = { - k.decode(): v[0].decode() for k, v in request.requestHeaders.getAllRawHeaders() - } - context = opentracing.tracer.extract(opentracing.Format.HTTP_HEADERS, header_dict) - - return opentracing.tracer.start_active_span( - operation_name, - child_of=context, - references=references, - tags=tags, - start_time=start_time, - ignore_active_span=ignore_active_span, - finish_on_close=finish_on_close, - ) - - def start_active_span_from_edu( edu_content, operation_name, @@ -743,6 +702,20 @@ def active_span_context_as_string(): return json_encoder.encode(carrier) +def span_context_from_request(request: Request) -> "Optional[opentracing.SpanContext]": + """Extract an opentracing context from the headers on an HTTP request + + This is useful when we have received an HTTP request from another part of our + system, and want to link our spans to those of the remote system. + """ + if not opentracing: + return None + header_dict = { + k.decode(): v[0].decode() for k, v in request.requestHeaders.getAllRawHeaders() + } + return opentracing.tracer.extract(opentracing.Format.HTTP_HEADERS, header_dict) + + @only_if_tracing def span_context_from_string(carrier): """ @@ -882,10 +855,13 @@ def trace_servlet(request: "SynapseRequest", extract_context: bool = False): } request_name = request.request_metrics.name - if extract_context: - scope = start_active_span_from_request(request, request_name) - else: - scope = start_active_span(request_name) + context = span_context_from_request(request) if extract_context else None + + # we configure the scope not to finish the span immediately on exit, and instead + # pass the span into the SynapseRequest, which will finish it once we've finished + # sending the response to the client. + scope = start_active_span(request_name, child_of=context, finish_on_close=False) + request.set_opentracing_span(scope.span) with scope: inject_response_headers(request.responseHeaders) From c3e38b88f221971fdafbca1283cc628bd439efd3 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff <1389908+richvdh@users.noreply.github.com> Date: Mon, 20 Dec 2021 18:12:08 +0000 Subject: [PATCH 09/76] Improve opentracing support for `ResponseCache` (#11607) This adds some opentracing annotations to ResponseCache, to make it easier to see what's going on; in particular, it adds a link back to the initial trace which is actually doing the work of generating the response. --- changelog.d/11607.misc | 1 + synapse/handlers/room.py | 2 +- synapse/util/async_helpers.py | 22 +++- synapse/util/caches/response_cache.py | 129 ++++++++++++++++------- tests/util/caches/test_response_cache.py | 45 ++++++-- 5 files changed, 150 insertions(+), 49 deletions(-) create mode 100644 changelog.d/11607.misc diff --git a/changelog.d/11607.misc b/changelog.d/11607.misc new file mode 100644 index 0000000000..e82f467763 --- /dev/null +++ b/changelog.d/11607.misc @@ -0,0 +1 @@ +Improve opentracing support for requests which use a `ResponseCache`. diff --git a/synapse/handlers/room.py b/synapse/handlers/room.py index ead2198e14..b9c1cbffa5 100644 --- a/synapse/handlers/room.py +++ b/synapse/handlers/room.py @@ -172,7 +172,7 @@ class RoomCreationHandler: user_id = requester.user.to_string() # Check if this room is already being upgraded by another person - for key in self._upgrade_response_cache.pending_result_cache: + for key in self._upgrade_response_cache.keys(): if key[0] == old_room_id and key[1] != user_id: # Two different people are trying to upgrade the same room. # Send the second an error. diff --git a/synapse/util/async_helpers.py b/synapse/util/async_helpers.py index bde99ea878..150a04b53e 100644 --- a/synapse/util/async_helpers.py +++ b/synapse/util/async_helpers.py @@ -13,6 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +import abc import collections import inspect import itertools @@ -57,7 +58,26 @@ logger = logging.getLogger(__name__) _T = TypeVar("_T") -class ObservableDeferred(Generic[_T]): +class AbstractObservableDeferred(Generic[_T], metaclass=abc.ABCMeta): + """Abstract base class defining the consumer interface of ObservableDeferred""" + + __slots__ = () + + @abc.abstractmethod + def observe(self) -> "defer.Deferred[_T]": + """Add a new observer for this ObservableDeferred + + This returns a brand new deferred that is resolved when the underlying + deferred is resolved. Interacting with the returned deferred does not + effect the underlying deferred. + + Note that the returned Deferred doesn't follow the Synapse logcontext rules - + you will probably want to `make_deferred_yieldable` it. + """ + ... + + +class ObservableDeferred(Generic[_T], AbstractObservableDeferred[_T]): """Wraps a deferred object so that we can add observer deferreds. These observer deferreds do not affect the callback chain of the original deferred. diff --git a/synapse/util/caches/response_cache.py b/synapse/util/caches/response_cache.py index 88ccf44337..a3eb5f741b 100644 --- a/synapse/util/caches/response_cache.py +++ b/synapse/util/caches/response_cache.py @@ -12,19 +12,37 @@ # See the License for the specific language governing permissions and # limitations under the License. import logging -from typing import Any, Awaitable, Callable, Dict, Generic, Optional, TypeVar +from typing import ( + TYPE_CHECKING, + Any, + Awaitable, + Callable, + Dict, + Generic, + Iterable, + Optional, + TypeVar, +) import attr from twisted.internet import defer from synapse.logging.context import make_deferred_yieldable, run_in_background +from synapse.logging.opentracing import ( + active_span, + start_active_span, + start_active_span_follows_from, +) from synapse.util import Clock -from synapse.util.async_helpers import ObservableDeferred +from synapse.util.async_helpers import AbstractObservableDeferred, ObservableDeferred from synapse.util.caches import register_cache logger = logging.getLogger(__name__) +if TYPE_CHECKING: + import opentracing + # the type of the key in the cache KV = TypeVar("KV") @@ -54,6 +72,20 @@ class ResponseCacheContext(Generic[KV]): """ +@attr.s(auto_attribs=True) +class ResponseCacheEntry: + result: AbstractObservableDeferred + """The (possibly incomplete) result of the operation. + + Note that we continue to store an ObservableDeferred even after the operation + completes (rather than switching to an immediate value), since that makes it + easier to cache Failure results. + """ + + opentracing_span_context: "Optional[opentracing.SpanContext]" + """The opentracing span which generated/is generating the result""" + + class ResponseCache(Generic[KV]): """ This caches a deferred response. Until the deferred completes it will be @@ -63,10 +95,7 @@ class ResponseCache(Generic[KV]): """ def __init__(self, clock: Clock, name: str, timeout_ms: float = 0): - # This is poorly-named: it includes both complete and incomplete results. - # We keep complete results rather than switching to absolute values because - # that makes it easier to cache Failure results. - self.pending_result_cache: Dict[KV, ObservableDeferred] = {} + self._result_cache: Dict[KV, ResponseCacheEntry] = {} self.clock = clock self.timeout_sec = timeout_ms / 1000.0 @@ -75,56 +104,63 @@ class ResponseCache(Generic[KV]): self._metrics = register_cache("response_cache", name, self, resizable=False) def size(self) -> int: - return len(self.pending_result_cache) + return len(self._result_cache) def __len__(self) -> int: return self.size() - def get(self, key: KV) -> Optional[defer.Deferred]: + def keys(self) -> Iterable[KV]: + """Get the keys currently in the result cache + + Returns both incomplete entries, and (if the timeout on this cache is non-zero), + complete entries which are still in the cache. + + Note that the returned iterator is not safe in the face of concurrent execution: + behaviour is undefined if `wrap` is called during iteration. + """ + return self._result_cache.keys() + + def _get(self, key: KV) -> Optional[ResponseCacheEntry]: """Look up the given key. - Returns a new Deferred (which also doesn't follow the synapse - logcontext rules). You will probably want to make_deferred_yieldable the result. - - If there is no entry for the key, returns None. - Args: - key: key to get/set in the cache + key: key to get in the cache Returns: - None if there is no entry for this key; otherwise a deferred which - resolves to the result. + The entry for this key, if any; else None. """ - result = self.pending_result_cache.get(key) - if result is not None: + entry = self._result_cache.get(key) + if entry is not None: self._metrics.inc_hits() - return result.observe() + return entry else: self._metrics.inc_misses() return None def _set( - self, context: ResponseCacheContext[KV], deferred: "defer.Deferred[RV]" - ) -> "defer.Deferred[RV]": + self, + context: ResponseCacheContext[KV], + deferred: "defer.Deferred[RV]", + opentracing_span_context: "Optional[opentracing.SpanContext]", + ) -> ResponseCacheEntry: """Set the entry for the given key to the given deferred. *deferred* should run its callbacks in the sentinel logcontext (ie, you should wrap normal synapse deferreds with synapse.logging.context.run_in_background). - Returns a new Deferred (which also doesn't follow the synapse logcontext rules). - You will probably want to make_deferred_yieldable the result. - Args: context: Information about the cache miss deferred: The deferred which resolves to the result. + opentracing_span_context: An opentracing span wrapping the calculation Returns: - A new deferred which resolves to the actual result. + The cache entry object. """ result = ObservableDeferred(deferred, consumeErrors=True) key = context.cache_key - self.pending_result_cache[key] = result + entry = ResponseCacheEntry(result, opentracing_span_context) + self._result_cache[key] = entry def on_complete(r: RV) -> RV: # if this cache has a non-zero timeout, and the callback has not cleared @@ -132,18 +168,18 @@ class ResponseCache(Generic[KV]): # its removal later. if self.timeout_sec and context.should_cache: self.clock.call_later( - self.timeout_sec, self.pending_result_cache.pop, key, None + self.timeout_sec, self._result_cache.pop, key, None ) else: # otherwise, remove the result immediately. - self.pending_result_cache.pop(key, None) + self._result_cache.pop(key, None) return r - # make sure we do this *after* adding the entry to pending_result_cache, + # make sure we do this *after* adding the entry to result_cache, # in case the result is already complete (in which case flipping the order would # leave us with a stuck entry in the cache). result.addBoth(on_complete) - return result.observe() + return entry async def wrap( self, @@ -189,20 +225,41 @@ class ResponseCache(Generic[KV]): Returns: The result of the callback (from the cache, or otherwise) """ - result = self.get(key) - if not result: + entry = self._get(key) + if not entry: logger.debug( "[%s]: no cached result for [%s], calculating new one", self._name, key ) context = ResponseCacheContext(cache_key=key) if cache_context: kwargs["cache_context"] = context - d = run_in_background(callback, *args, **kwargs) - result = self._set(context, d) - elif not isinstance(result, defer.Deferred) or result.called: + + span_context: Optional[opentracing.SpanContext] = None + + async def cb() -> RV: + # NB it is important that we do not `await` before setting span_context! + nonlocal span_context + with start_active_span(f"ResponseCache[{self._name}].calculate"): + span = active_span() + if span: + span_context = span.context + return await callback(*args, **kwargs) + + d = run_in_background(cb) + entry = self._set(context, d, span_context) + return await make_deferred_yieldable(entry.result.observe()) + + result = entry.result.observe() + if result.called: logger.info("[%s]: using completed cached result for [%s]", self._name, key) else: logger.info( "[%s]: using incomplete cached result for [%s]", self._name, key ) - return await make_deferred_yieldable(result) + + span_context = entry.opentracing_span_context + with start_active_span_follows_from( + f"ResponseCache[{self._name}].wait", + contexts=(span_context,) if span_context else (), + ): + return await make_deferred_yieldable(result) diff --git a/tests/util/caches/test_response_cache.py b/tests/util/caches/test_response_cache.py index 1e83ef2f33..025b73e32f 100644 --- a/tests/util/caches/test_response_cache.py +++ b/tests/util/caches/test_response_cache.py @@ -11,6 +11,9 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. + +from unittest.mock import Mock + from parameterized import parameterized from twisted.internet import defer @@ -60,10 +63,15 @@ class ResponseCacheTestCase(TestCase): self.successResultOf(wrap_d), "initial wrap result should be the same", ) + + # a second call should return the result without a call to the wrapped function + unexpected = Mock(spec=()) + wrap2_d = defer.ensureDeferred(cache.wrap(0, unexpected)) + unexpected.assert_not_called() self.assertEqual( expected_result, - self.successResultOf(cache.get(0)), - "cache should have the result", + self.successResultOf(wrap2_d), + "cache should still have the result", ) def test_cache_miss(self): @@ -80,7 +88,7 @@ class ResponseCacheTestCase(TestCase): self.successResultOf(wrap_d), "initial wrap result should be the same", ) - self.assertIsNone(cache.get(0), "cache should not have the result now") + self.assertCountEqual([], cache.keys(), "cache should not have the result now") def test_cache_expire(self): cache = self.with_cache("short_cache", ms=1000) @@ -92,16 +100,20 @@ class ResponseCacheTestCase(TestCase): ) self.assertEqual(expected_result, self.successResultOf(wrap_d)) + + # a second call should return the result without a call to the wrapped function + unexpected = Mock(spec=()) + wrap2_d = defer.ensureDeferred(cache.wrap(0, unexpected)) + unexpected.assert_not_called() self.assertEqual( expected_result, - self.successResultOf(cache.get(0)), + self.successResultOf(wrap2_d), "cache should still have the result", ) # cache eviction timer is handled self.reactor.pump((2,)) - - self.assertIsNone(cache.get(0), "cache should not have the result now") + self.assertCountEqual([], cache.keys(), "cache should not have the result now") def test_cache_wait_hit(self): cache = self.with_cache("neutral_cache") @@ -133,16 +145,21 @@ class ResponseCacheTestCase(TestCase): self.reactor.pump((1, 1)) self.assertEqual(expected_result, self.successResultOf(wrap_d)) + + # a second call should immediately return the result without a call to the + # wrapped function + unexpected = Mock(spec=()) + wrap2_d = defer.ensureDeferred(cache.wrap(0, unexpected)) + unexpected.assert_not_called() self.assertEqual( expected_result, - self.successResultOf(cache.get(0)), + self.successResultOf(wrap2_d), "cache should still have the result", ) # (1 + 1 + 2) > 3.0, cache eviction timer is handled self.reactor.pump((2,)) - - self.assertIsNone(cache.get(0), "cache should not have the result now") + self.assertCountEqual([], cache.keys(), "cache should not have the result now") @parameterized.expand([(True,), (False,)]) def test_cache_context_nocache(self, should_cache: bool): @@ -183,10 +200,16 @@ class ResponseCacheTestCase(TestCase): self.assertEqual(expected_result, self.successResultOf(wrap2_d)) if should_cache: + unexpected = Mock(spec=()) + wrap3_d = defer.ensureDeferred(cache.wrap(0, unexpected)) + unexpected.assert_not_called() self.assertEqual( expected_result, - self.successResultOf(cache.get(0)), + self.successResultOf(wrap3_d), "cache should still have the result", ) + else: - self.assertIsNone(cache.get(0), "cache should not have the result") + self.assertCountEqual( + [], cache.keys(), "cache should not have the result now" + ) From dd4778875213d9cb8be7ee71d32751fbd6cdaba2 Mon Sep 17 00:00:00 2001 From: Patrick Cloke Date: Mon, 20 Dec 2021 14:14:38 -0500 Subject: [PATCH 10/76] Do not bundle aggregations for APIs which shouldn't include them. (#11592) And make bundling aggregations opt-in, instead of opt-out to avoid having APIs to include extraneous data (and being much heavier than necessary). --- changelog.d/11592.bugfix | 1 + synapse/events/utils.py | 2 +- synapse/handlers/events.py | 2 -- synapse/handlers/initial_sync.py | 18 ++++-------------- synapse/handlers/message.py | 4 +++- synapse/handlers/pagination.py | 5 ++++- synapse/rest/admin/rooms.py | 12 +++++++++--- synapse/rest/client/relations.py | 4 +++- synapse/rest/client/room.py | 10 ++++++---- 9 files changed, 31 insertions(+), 27 deletions(-) create mode 100644 changelog.d/11592.bugfix diff --git a/changelog.d/11592.bugfix b/changelog.d/11592.bugfix new file mode 100644 index 0000000000..4116e5dd7c --- /dev/null +++ b/changelog.d/11592.bugfix @@ -0,0 +1 @@ +Fix a long-standing bug where responses included bundled aggregations when they should not, per [MSC2675](https://github.com/matrix-org/matrix-doc/pull/2675). diff --git a/synapse/events/utils.py b/synapse/events/utils.py index 3da432c1df..2038e72924 100644 --- a/synapse/events/utils.py +++ b/synapse/events/utils.py @@ -395,7 +395,7 @@ class EventClientSerializer: event: Union[JsonDict, EventBase], time_now: int, *, - bundle_aggregations: bool = True, + bundle_aggregations: bool = False, **kwargs: Any, ) -> JsonDict: """Serializes a single event. diff --git a/synapse/handlers/events.py b/synapse/handlers/events.py index afed80ba14..1b996c420d 100644 --- a/synapse/handlers/events.py +++ b/synapse/handlers/events.py @@ -123,8 +123,6 @@ class EventStreamHandler: events, time_now, as_client_event=as_client_event, - # Don't bundle aggregations as this is a deprecated API. - bundle_aggregations=False, ) chunk = { diff --git a/synapse/handlers/initial_sync.py b/synapse/handlers/initial_sync.py index 9ab723ff97..601bab67f9 100644 --- a/synapse/handlers/initial_sync.py +++ b/synapse/handlers/initial_sync.py @@ -173,8 +173,6 @@ class InitialSyncHandler: d["invite"] = await self._event_serializer.serialize_event( invite_event, time_now, - # Don't bundle aggregations as this is a deprecated API. - bundle_aggregations=False, as_client_event=as_client_event, ) @@ -227,8 +225,6 @@ class InitialSyncHandler: await self._event_serializer.serialize_events( messages, time_now=time_now, - # Don't bundle aggregations as this is a deprecated API. - bundle_aggregations=False, as_client_event=as_client_event, ) ), @@ -239,8 +235,6 @@ class InitialSyncHandler: d["state"] = await self._event_serializer.serialize_events( current_state.values(), time_now=time_now, - # Don't bundle aggregations as this is a deprecated API. - bundle_aggregations=False, as_client_event=as_client_event, ) @@ -382,9 +376,7 @@ class InitialSyncHandler: "messages": { "chunk": ( # Don't bundle aggregations as this is a deprecated API. - await self._event_serializer.serialize_events( - messages, time_now, bundle_aggregations=False - ) + await self._event_serializer.serialize_events(messages, time_now) ), "start": await start_token.to_string(self.store), "end": await end_token.to_string(self.store), @@ -392,7 +384,7 @@ class InitialSyncHandler: "state": ( # Don't bundle aggregations as this is a deprecated API. await self._event_serializer.serialize_events( - room_state.values(), time_now, bundle_aggregations=False + room_state.values(), time_now ) ), "presence": [], @@ -413,7 +405,7 @@ class InitialSyncHandler: time_now = self.clock.time_msec() # Don't bundle aggregations as this is a deprecated API. state = await self._event_serializer.serialize_events( - current_state.values(), time_now, bundle_aggregations=False + current_state.values(), time_now ) now_token = self.hs.get_event_sources().get_current_token() @@ -488,9 +480,7 @@ class InitialSyncHandler: "messages": { "chunk": ( # Don't bundle aggregations as this is a deprecated API. - await self._event_serializer.serialize_events( - messages, time_now, bundle_aggregations=False - ) + await self._event_serializer.serialize_events(messages, time_now) ), "start": await start_token.to_string(self.store), "end": await end_token.to_string(self.store), diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py index 5e3d3886eb..1a7190085a 100644 --- a/synapse/handlers/message.py +++ b/synapse/handlers/message.py @@ -246,7 +246,9 @@ class MessageHandler: room_state = room_state_events[membership_event_id] now = self.clock.time_msec() - events = await self._event_serializer.serialize_events(room_state.values(), now) + events = await self._event_serializer.serialize_events( + room_state.values(), now, bundle_aggregations=True + ) return events async def get_joined_members(self, requester: Requester, room_id: str) -> dict: diff --git a/synapse/handlers/pagination.py b/synapse/handlers/pagination.py index 4f42438053..7469cc55a2 100644 --- a/synapse/handlers/pagination.py +++ b/synapse/handlers/pagination.py @@ -542,7 +542,10 @@ class PaginationHandler: chunk = { "chunk": ( await self._event_serializer.serialize_events( - events, time_now, as_client_event=as_client_event + events, + time_now, + bundle_aggregations=True, + as_client_event=as_client_event, ) ), "start": await from_token.to_string(self.store), diff --git a/synapse/rest/admin/rooms.py b/synapse/rest/admin/rooms.py index 17c6df1cc8..6030373ebc 100644 --- a/synapse/rest/admin/rooms.py +++ b/synapse/rest/admin/rooms.py @@ -745,13 +745,19 @@ class RoomEventContextServlet(RestServlet): time_now = self.clock.time_msec() results["events_before"] = await self._event_serializer.serialize_events( - results["events_before"], time_now + results["events_before"], + time_now, + bundle_aggregations=True, ) results["event"] = await self._event_serializer.serialize_event( - results["event"], time_now + results["event"], + time_now, + bundle_aggregations=True, ) results["events_after"] = await self._event_serializer.serialize_events( - results["events_after"], time_now + results["events_after"], + time_now, + bundle_aggregations=True, ) results["state"] = await self._event_serializer.serialize_events( results["state"], time_now diff --git a/synapse/rest/client/relations.py b/synapse/rest/client/relations.py index ffa37ef06c..5815650ee6 100644 --- a/synapse/rest/client/relations.py +++ b/synapse/rest/client/relations.py @@ -232,7 +232,9 @@ class RelationPaginationServlet(RestServlet): ) # The relations returned for the requested event do include their # bundled aggregations. - serialized_events = await self._event_serializer.serialize_events(events, now) + serialized_events = await self._event_serializer.serialize_events( + events, now, bundle_aggregations=True + ) return_value = pagination_chunk.to_dict() return_value["chunk"] = serialized_events diff --git a/synapse/rest/client/room.py b/synapse/rest/client/room.py index 60719331b6..40330749e5 100644 --- a/synapse/rest/client/room.py +++ b/synapse/rest/client/room.py @@ -662,7 +662,9 @@ class RoomEventServlet(RestServlet): time_now = self.clock.time_msec() if event: - event_dict = await self._event_serializer.serialize_event(event, time_now) + event_dict = await self._event_serializer.serialize_event( + event, time_now, bundle_aggregations=True + ) return 200, event_dict raise SynapseError(404, "Event not found.", errcode=Codes.NOT_FOUND) @@ -707,13 +709,13 @@ class RoomEventContextServlet(RestServlet): time_now = self.clock.time_msec() results["events_before"] = await self._event_serializer.serialize_events( - results["events_before"], time_now + results["events_before"], time_now, bundle_aggregations=True ) results["event"] = await self._event_serializer.serialize_event( - results["event"], time_now + results["event"], time_now, bundle_aggregations=True ) results["events_after"] = await self._event_serializer.serialize_events( - results["events_after"], time_now + results["events_after"], time_now, bundle_aggregations=True ) results["state"] = await self._event_serializer.serialize_events( results["state"], time_now From 8c36d332d598f55f471196fb9415bf27db3e6fcd Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Tue, 21 Dec 2021 11:07:41 +0000 Subject: [PATCH 11/76] 1.49.1 --- CHANGES.md | 9 +++++++++ changelog.d/11583.bugfix | 1 - debian/changelog | 6 ++++++ synapse/__init__.py | 2 +- 4 files changed, 16 insertions(+), 2 deletions(-) delete mode 100644 changelog.d/11583.bugfix diff --git a/CHANGES.md b/CHANGES.md index 58217e655e..bdfa40686f 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,3 +1,12 @@ +Synapse 1.49.1 (2021-12-21) +=========================== + +Bugfixes +-------- + +- Fix a performance regression in `/sync` handling, introduced in 1.49.0. ([\#11583](https://github.com/matrix-org/synapse/issues/11583)) + + Synapse 1.49.0 (2021-12-14) =========================== diff --git a/changelog.d/11583.bugfix b/changelog.d/11583.bugfix deleted file mode 100644 index d2ed113e21..0000000000 --- a/changelog.d/11583.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix a performance regression in `/sync` handling, introduced in 1.49.0. diff --git a/debian/changelog b/debian/changelog index 794d8efa40..61a1d003b2 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,9 @@ +matrix-synapse-py3 (1.49.1) stable; urgency=medium + + * New synapse release 1.49.1. + + -- Synapse Packaging team Tue, 21 Dec 2021 11:07:30 +0000 + matrix-synapse-py3 (1.49.0) stable; urgency=medium * New synapse release 1.49.0. diff --git a/synapse/__init__.py b/synapse/__init__.py index d44a03a2cb..d45e0f1883 100644 --- a/synapse/__init__.py +++ b/synapse/__init__.py @@ -47,7 +47,7 @@ try: except ImportError: pass -__version__ = "1.49.0" +__version__ = "1.49.1" if bool(os.environ.get("SYNAPSE_TEST_PATCH_LOG_CONTEXTS", False)): # We import here so that we don't have to install a bunch of deps when From 221595414751f7b8fd0c79772c5ac4ffefeca10a Mon Sep 17 00:00:00 2001 From: Richard van der Hoff <1389908+richvdh@users.noreply.github.com> Date: Tue, 21 Dec 2021 11:10:36 +0000 Subject: [PATCH 12/76] Various opentracing enhancements (#11619) * Wrap `auth.get_user_by_req` in an opentracing span give `get_user_by_req` its own opentracing span, since it can result in a non-trivial number of sub-spans which it is useful to group together. This requires a bit of reorganisation because it also sets some tags (and may force tracing) on the servlet span. * Emit opentracing span for encoding json responses This can be a significant time sink. * Rename all sync spans with a prefix * Write an opentracing span for encoding sync response * opentracing span to group generate_room_entries * opentracing spans within sync.encode_response * changelog * Use the `trace` decorator instead of context managers --- changelog.d/11619.misc | 1 + synapse/api/auth.py | 53 ++++++++++++++++++++++++++----------- synapse/handlers/sync.py | 7 ++--- synapse/http/server.py | 19 +++++++++++-- synapse/rest/client/sync.py | 6 +++++ 5 files changed, 65 insertions(+), 21 deletions(-) create mode 100644 changelog.d/11619.misc diff --git a/changelog.d/11619.misc b/changelog.d/11619.misc new file mode 100644 index 0000000000..2125cbddd2 --- /dev/null +++ b/changelog.d/11619.misc @@ -0,0 +1 @@ +A number of improvements to opentracing support. diff --git a/synapse/api/auth.py b/synapse/api/auth.py index 0bf58dff08..4a32d430bd 100644 --- a/synapse/api/auth.py +++ b/synapse/api/auth.py @@ -32,7 +32,7 @@ from synapse.appservice import ApplicationService from synapse.events import EventBase from synapse.http import get_request_user_agent from synapse.http.site import SynapseRequest -from synapse.logging import opentracing as opentracing +from synapse.logging.opentracing import active_span, force_tracing, start_active_span from synapse.storage.databases.main.registration import TokenLookupResult from synapse.types import Requester, StateMap, UserID, create_requester from synapse.util.caches.lrucache import LruCache @@ -149,6 +149,42 @@ class Auth: is invalid. AuthError if access is denied for the user in the access token """ + parent_span = active_span() + with start_active_span("get_user_by_req"): + requester = await self._wrapped_get_user_by_req( + request, allow_guest, rights, allow_expired + ) + + if parent_span: + if requester.authenticated_entity in self._force_tracing_for_users: + # request tracing is enabled for this user, so we need to force it + # tracing on for the parent span (which will be the servlet span). + # + # It's too late for the get_user_by_req span to inherit the setting, + # so we also force it on for that. + force_tracing() + force_tracing(parent_span) + parent_span.set_tag( + "authenticated_entity", requester.authenticated_entity + ) + parent_span.set_tag("user_id", requester.user.to_string()) + if requester.device_id is not None: + parent_span.set_tag("device_id", requester.device_id) + if requester.app_service is not None: + parent_span.set_tag("appservice_id", requester.app_service.id) + return requester + + async def _wrapped_get_user_by_req( + self, + request: SynapseRequest, + allow_guest: bool, + rights: str, + allow_expired: bool, + ) -> Requester: + """Helper for get_user_by_req + + Once get_user_by_req has set up the opentracing span, this does the actual work. + """ try: ip_addr = request.getClientIP() user_agent = get_request_user_agent(request) @@ -177,14 +213,6 @@ class Auth: ) request.requester = user_id - if user_id in self._force_tracing_for_users: - opentracing.force_tracing() - opentracing.set_tag("authenticated_entity", user_id) - opentracing.set_tag("user_id", user_id) - if device_id is not None: - opentracing.set_tag("device_id", device_id) - opentracing.set_tag("appservice_id", app_service.id) - return requester user_info = await self.get_user_by_access_token( @@ -242,13 +270,6 @@ class Auth: ) request.requester = requester - if user_info.token_owner in self._force_tracing_for_users: - opentracing.force_tracing() - opentracing.set_tag("authenticated_entity", user_info.token_owner) - opentracing.set_tag("user_id", user_info.user_id) - if device_id: - opentracing.set_tag("device_id", device_id) - return requester except KeyError: raise MissingClientTokenError() diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py index bcd10cbb30..d24124d6ac 100644 --- a/synapse/handlers/sync.py +++ b/synapse/handlers/sync.py @@ -421,7 +421,7 @@ class SyncHandler: span to track the sync. See `generate_sync_result` for the next part of your indoctrination. """ - with start_active_span("current_sync_for_user"): + with start_active_span("sync.current_sync_for_user"): log_kv({"since_token": since_token}) sync_result = await self.generate_sync_result( sync_config, since_token, full_state @@ -1585,7 +1585,8 @@ class SyncHandler: ) logger.debug("Generated room entry for %s", room_entry.room_id) - await concurrently_execute(handle_room_entries, room_entries, 10) + with start_active_span("sync.generate_room_entries"): + await concurrently_execute(handle_room_entries, room_entries, 10) sync_result_builder.invited.extend(invited) sync_result_builder.knocked.extend(knocked) @@ -2045,7 +2046,7 @@ class SyncHandler: since_token = room_builder.since_token upto_token = room_builder.upto_token - with start_active_span("generate_room_entry"): + with start_active_span("sync.generate_room_entry"): set_tag("room_id", room_id) log_kv({"events": len(events or ())}) diff --git a/synapse/http/server.py b/synapse/http/server.py index 7bbbe7648b..e302946591 100644 --- a/synapse/http/server.py +++ b/synapse/http/server.py @@ -58,12 +58,14 @@ from synapse.api.errors import ( ) from synapse.http.site import SynapseRequest from synapse.logging.context import defer_to_thread, preserve_fn, run_in_background -from synapse.logging.opentracing import trace_servlet +from synapse.logging.opentracing import active_span, start_active_span, trace_servlet from synapse.util import json_encoder from synapse.util.caches import intern_dict from synapse.util.iterutils import chunk_seq if TYPE_CHECKING: + import opentracing + from synapse.server import HomeServer logger = logging.getLogger(__name__) @@ -759,7 +761,20 @@ async def _async_write_json_to_request_in_thread( expensive. """ - json_str = await defer_to_thread(request.reactor, json_encoder, json_object) + def encode(opentracing_span: "Optional[opentracing.Span]") -> bytes: + # it might take a while for the threadpool to schedule us, so we write + # opentracing logs once we actually get scheduled, so that we can see how + # much that contributed. + if opentracing_span: + opentracing_span.log_kv({"event": "scheduled"}) + res = json_encoder(json_object) + if opentracing_span: + opentracing_span.log_kv({"event": "encoded"}) + return res + + with start_active_span("encode_json_response"): + span = active_span() + json_str = await defer_to_thread(request.reactor, encode, span) _write_bytes_to_request(request, json_str) diff --git a/synapse/rest/client/sync.py b/synapse/rest/client/sync.py index 8c4b0f6e5d..e99a943d0d 100644 --- a/synapse/rest/client/sync.py +++ b/synapse/rest/client/sync.py @@ -48,6 +48,7 @@ from synapse.handlers.sync import ( from synapse.http.server import HttpServer from synapse.http.servlet import RestServlet, parse_boolean, parse_integer, parse_string from synapse.http.site import SynapseRequest +from synapse.logging.opentracing import trace from synapse.types import JsonDict, StreamToken from synapse.util import json_decoder @@ -222,6 +223,7 @@ class SyncRestServlet(RestServlet): logger.debug("Event formatting complete") return 200, response_content + @trace(opname="sync.encode_response") async def encode_response( self, time_now: int, @@ -332,6 +334,7 @@ class SyncRestServlet(RestServlet): ] } + @trace(opname="sync.encode_joined") async def encode_joined( self, rooms: List[JoinedSyncResult], @@ -368,6 +371,7 @@ class SyncRestServlet(RestServlet): return joined + @trace(opname="sync.encode_invited") async def encode_invited( self, rooms: List[InvitedSyncResult], @@ -406,6 +410,7 @@ class SyncRestServlet(RestServlet): return invited + @trace(opname="sync.encode_knocked") async def encode_knocked( self, rooms: List[KnockedSyncResult], @@ -460,6 +465,7 @@ class SyncRestServlet(RestServlet): return knocked + @trace(opname="sync.encode_archived") async def encode_archived( self, rooms: List[ArchivedSyncResult], From b9f2f6d3c414a5b0d2bbce15ef12d669dbcea82a Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Tue, 21 Dec 2021 11:23:35 +0000 Subject: [PATCH 13/76] more words --- CHANGES.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGES.md b/CHANGES.md index bdfa40686f..0fcfb3c782 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,6 +1,8 @@ Synapse 1.49.1 (2021-12-21) =========================== +This release fixes a regression introduced in Synapse 1.49.0 which could cause `/sync` requests to take significantly longer. This would particularly affect "initial" syncs for users participating in a large number of rooms, and in extreme cases, could make it imposssible for such users to log in on a new client. + Bugfixes -------- From aa58e8a28a05c5a71003570f8bdcd1256276f243 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Tue, 21 Dec 2021 11:24:24 +0000 Subject: [PATCH 14/76] typopo --- CHANGES.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index 0fcfb3c782..6932045848 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,7 +1,7 @@ Synapse 1.49.1 (2021-12-21) =========================== -This release fixes a regression introduced in Synapse 1.49.0 which could cause `/sync` requests to take significantly longer. This would particularly affect "initial" syncs for users participating in a large number of rooms, and in extreme cases, could make it imposssible for such users to log in on a new client. +This release fixes a regression introduced in Synapse 1.49.0 which could cause `/sync` requests to take significantly longer. This would particularly affect "initial" syncs for users participating in a large number of rooms, and in extreme cases, could make it impossible for such users to log in on a new client. Bugfixes -------- From 57ca8ab10f74e9f36b3a93ff31438cabb146fb5f Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Tue, 21 Dec 2021 12:06:31 +0000 Subject: [PATCH 15/76] Add notes about dropping support for Python 3.6 and Postgres 9.6. --- CHANGES.md | 4 ++++ docs/deprecation_policy.md | 4 ++-- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 6932045848..dc6fa17794 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -3,6 +3,10 @@ Synapse 1.49.1 (2021-12-21) This release fixes a regression introduced in Synapse 1.49.0 which could cause `/sync` requests to take significantly longer. This would particularly affect "initial" syncs for users participating in a large number of rooms, and in extreme cases, could make it impossible for such users to log in on a new client. +**Note:** in line with our [deprecation policy](https://matrix-org.github.io/synapse/latest/deprecation_policy.html) for platform dependencies, this will be the last release to support Python 3.6 and PostgreSQL 9.6, both of which have now reached upstream end-of-life. Synapse will require Python 3.7+ and PostgreSQL 10. + +**Note:** We will also stop producing packages for Ubuntu 18.04 (Bionic Beaver) after this release, as it uses Python 3.6. + Bugfixes -------- diff --git a/docs/deprecation_policy.md b/docs/deprecation_policy.md index 06ea340559..359dac07c3 100644 --- a/docs/deprecation_policy.md +++ b/docs/deprecation_policy.md @@ -14,8 +14,8 @@ i.e. when a version reaches End of Life Synapse will withdraw support for that version in future releases. Details on the upstream support life cycles for Python and PostgreSQL are -documented at https://endoflife.date/python and -https://endoflife.date/postgresql. +documented at [https://endoflife.date/python](https://endoflife.date/python) and +[https://endoflife.date/postgresql](https://endoflife.date/postgresql). Context From b6102230a7391d1acaa50cc6c389813f7e0fab84 Mon Sep 17 00:00:00 2001 From: Patrick Cloke Date: Tue, 21 Dec 2021 08:25:34 -0500 Subject: [PATCH 16/76] Add type hints to event_push_actions. (#11594) --- changelog.d/11594.misc | 1 + mypy.ini | 4 +- synapse/handlers/sync.py | 12 +- synapse/push/emailpusher.py | 18 +- synapse/push/httppusher.py | 12 +- synapse/push/mailer.py | 40 ++- synapse/push/push_tools.py | 4 +- synapse/rest/client/notifications.py | 20 +- .../databases/main/event_push_actions.py | 251 +++++++++++------- .../replication/slave/storage/test_events.py | 7 +- tests/storage/test_event_push_actions.py | 12 +- 11 files changed, 226 insertions(+), 155 deletions(-) create mode 100644 changelog.d/11594.misc diff --git a/changelog.d/11594.misc b/changelog.d/11594.misc new file mode 100644 index 0000000000..d451940bf2 --- /dev/null +++ b/changelog.d/11594.misc @@ -0,0 +1 @@ +Add missing type hints to storage classes. diff --git a/mypy.ini b/mypy.ini index 3279c9bb21..57e1a5df43 100644 --- a/mypy.ini +++ b/mypy.ini @@ -28,7 +28,6 @@ exclude = (?x) |synapse/storage/databases/main/cache.py |synapse/storage/databases/main/devices.py |synapse/storage/databases/main/event_federation.py - |synapse/storage/databases/main/event_push_actions.py |synapse/storage/databases/main/events_bg_updates.py |synapse/storage/databases/main/group_server.py |synapse/storage/databases/main/metrics.py @@ -200,6 +199,9 @@ disallow_untyped_defs = True [mypy-synapse.storage.databases.main.end_to_end_keys] disallow_untyped_defs = True +[mypy-synapse.storage.databases.main.event_push_actions] +disallow_untyped_defs = True + [mypy-synapse.storage.databases.main.events_worker] disallow_untyped_defs = True diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py index d24124d6ac..7baf3f199c 100644 --- a/synapse/handlers/sync.py +++ b/synapse/handlers/sync.py @@ -36,6 +36,7 @@ from synapse.events import EventBase from synapse.logging.context import current_context from synapse.logging.opentracing import SynapseTags, log_kv, set_tag, start_active_span from synapse.push.clientformat import format_push_rules_for_user +from synapse.storage.databases.main.event_push_actions import NotifCounts from synapse.storage.roommember import MemberSummary from synapse.storage.state import StateFilter from synapse.types import ( @@ -1041,7 +1042,7 @@ class SyncHandler: async def unread_notifs_for_room_id( self, room_id: str, sync_config: SyncConfig - ) -> Dict[str, int]: + ) -> NotifCounts: with Measure(self.clock, "unread_notifs_for_room_id"): last_unread_event_id = await self.store.get_last_receipt_event_id_for_user( user_id=sync_config.user.to_string(), @@ -1049,10 +1050,9 @@ class SyncHandler: receipt_type=ReceiptTypes.READ, ) - notifs = await self.store.get_unread_event_push_actions_by_room_for_user( + return await self.store.get_unread_event_push_actions_by_room_for_user( room_id, sync_config.user.to_string(), last_unread_event_id ) - return notifs async def generate_sync_result( self, @@ -2174,10 +2174,10 @@ class SyncHandler: if room_sync or always_include: notifs = await self.unread_notifs_for_room_id(room_id, sync_config) - unread_notifications["notification_count"] = notifs["notify_count"] - unread_notifications["highlight_count"] = notifs["highlight_count"] + unread_notifications["notification_count"] = notifs.notify_count + unread_notifications["highlight_count"] = notifs.highlight_count - room_sync.unread_count = notifs["unread_count"] + room_sync.unread_count = notifs.unread_count sync_result_builder.joined.append(room_sync) diff --git a/synapse/push/emailpusher.py b/synapse/push/emailpusher.py index 4f13c0418a..39bb2acae4 100644 --- a/synapse/push/emailpusher.py +++ b/synapse/push/emailpusher.py @@ -177,12 +177,12 @@ class EmailPusher(Pusher): return for push_action in unprocessed: - received_at = push_action["received_ts"] + received_at = push_action.received_ts if received_at is None: received_at = 0 notif_ready_at = received_at + DELAY_BEFORE_MAIL_MS - room_ready_at = self.room_ready_to_notify_at(push_action["room_id"]) + room_ready_at = self.room_ready_to_notify_at(push_action.room_id) should_notify_at = max(notif_ready_at, room_ready_at) @@ -193,23 +193,23 @@ class EmailPusher(Pusher): # to be delivered. reason: EmailReason = { - "room_id": push_action["room_id"], + "room_id": push_action.room_id, "now": self.clock.time_msec(), "received_at": received_at, "delay_before_mail_ms": DELAY_BEFORE_MAIL_MS, - "last_sent_ts": self.get_room_last_sent_ts(push_action["room_id"]), - "throttle_ms": self.get_room_throttle_ms(push_action["room_id"]), + "last_sent_ts": self.get_room_last_sent_ts(push_action.room_id), + "throttle_ms": self.get_room_throttle_ms(push_action.room_id), } await self.send_notification(unprocessed, reason) await self.save_last_stream_ordering_and_success( - max(ea["stream_ordering"] for ea in unprocessed) + max(ea.stream_ordering for ea in unprocessed) ) # we update the throttle on all the possible unprocessed push actions for ea in unprocessed: - await self.sent_notif_update_throttle(ea["room_id"], ea) + await self.sent_notif_update_throttle(ea.room_id, ea) break else: if soonest_due_at is None or should_notify_at < soonest_due_at: @@ -284,10 +284,10 @@ class EmailPusher(Pusher): # THROTTLE_RESET_AFTER_MS after the previous one that triggered a # notif, we release the throttle. Otherwise, the throttle is increased. time_of_previous_notifs = await self.store.get_time_of_last_push_action_before( - notified_push_action["stream_ordering"] + notified_push_action.stream_ordering ) - time_of_this_notifs = notified_push_action["received_ts"] + time_of_this_notifs = notified_push_action.received_ts if time_of_previous_notifs is not None and time_of_this_notifs is not None: gap = time_of_this_notifs - time_of_previous_notifs diff --git a/synapse/push/httppusher.py b/synapse/push/httppusher.py index 3fa603ccb7..96559081d0 100644 --- a/synapse/push/httppusher.py +++ b/synapse/push/httppusher.py @@ -199,7 +199,7 @@ class HttpPusher(Pusher): "http-push", tags={ "authenticated_entity": self.user_id, - "event_id": push_action["event_id"], + "event_id": push_action.event_id, "app_id": self.app_id, "app_display_name": self.app_display_name, }, @@ -209,7 +209,7 @@ class HttpPusher(Pusher): if processed: http_push_processed_counter.inc() self.backoff_delay = HttpPusher.INITIAL_BACKOFF_SEC - self.last_stream_ordering = push_action["stream_ordering"] + self.last_stream_ordering = push_action.stream_ordering pusher_still_exists = ( await self.store.update_pusher_last_stream_ordering_and_success( self.app_id, @@ -252,7 +252,7 @@ class HttpPusher(Pusher): self.pushkey, ) self.backoff_delay = HttpPusher.INITIAL_BACKOFF_SEC - self.last_stream_ordering = push_action["stream_ordering"] + self.last_stream_ordering = push_action.stream_ordering await self.store.update_pusher_last_stream_ordering( self.app_id, self.pushkey, @@ -275,17 +275,17 @@ class HttpPusher(Pusher): break async def _process_one(self, push_action: HttpPushAction) -> bool: - if "notify" not in push_action["actions"]: + if "notify" not in push_action.actions: return True - tweaks = push_rule_evaluator.tweaks_for_actions(push_action["actions"]) + tweaks = push_rule_evaluator.tweaks_for_actions(push_action.actions) badge = await push_tools.get_badge_count( self.hs.get_datastore(), self.user_id, group_by_room=self._group_unread_count_by_room, ) - event = await self.store.get_event(push_action["event_id"], allow_none=True) + event = await self.store.get_event(push_action.event_id, allow_none=True) if event is None: return True # It's been redacted rejected = await self.dispatch_push(event, tweaks, badge) diff --git a/synapse/push/mailer.py b/synapse/push/mailer.py index ba4f866487..ff904c2b4a 100644 --- a/synapse/push/mailer.py +++ b/synapse/push/mailer.py @@ -232,15 +232,13 @@ class Mailer: reason: The notification that was ready and is the cause of an email being sent. """ - rooms_in_order = deduped_ordered_list([pa["room_id"] for pa in push_actions]) + rooms_in_order = deduped_ordered_list([pa.room_id for pa in push_actions]) - notif_events = await self.store.get_events( - [pa["event_id"] for pa in push_actions] - ) + notif_events = await self.store.get_events([pa.event_id for pa in push_actions]) notifs_by_room: Dict[str, List[EmailPushAction]] = {} for pa in push_actions: - notifs_by_room.setdefault(pa["room_id"], []).append(pa) + notifs_by_room.setdefault(pa.room_id, []).append(pa) # collect the current state for all the rooms in which we have # notifications @@ -264,7 +262,7 @@ class Mailer: await concurrently_execute(_fetch_room_state, rooms_in_order, 3) # actually sort our so-called rooms_in_order list, most recent room first - rooms_in_order.sort(key=lambda r: -(notifs_by_room[r][-1]["received_ts"] or 0)) + rooms_in_order.sort(key=lambda r: -(notifs_by_room[r][-1].received_ts or 0)) rooms: List[RoomVars] = [] @@ -356,7 +354,7 @@ class Mailer: # Check if one of the notifs is an invite event for the user. is_invite = False for n in notifs: - ev = notif_events[n["event_id"]] + ev = notif_events[n.event_id] if ev.type == EventTypes.Member and ev.state_key == user_id: if ev.content.get("membership") == Membership.INVITE: is_invite = True @@ -376,7 +374,7 @@ class Mailer: if not is_invite: for n in notifs: notifvars = await self._get_notif_vars( - n, user_id, notif_events[n["event_id"]], room_state_ids + n, user_id, notif_events[n.event_id], room_state_ids ) # merge overlapping notifs together. @@ -444,15 +442,15 @@ class Mailer: """ results = await self.store.get_events_around( - notif["room_id"], - notif["event_id"], + notif.room_id, + notif.event_id, before_limit=CONTEXT_BEFORE, after_limit=CONTEXT_AFTER, ) ret: NotifVars = { "link": self._make_notif_link(notif), - "ts": notif["received_ts"], + "ts": notif.received_ts, "messages": [], } @@ -516,7 +514,7 @@ class Mailer: ret: MessageVars = { "event_type": event.type, - "is_historical": event.event_id != notif["event_id"], + "is_historical": event.event_id != notif.event_id, "id": event.event_id, "ts": event.origin_server_ts, "sender_name": sender_name, @@ -610,7 +608,7 @@ class Mailer: # See if one of the notifs is an invite event for the user invite_event = None for n in notifs: - ev = notif_events[n["event_id"]] + ev = notif_events[n.event_id] if ev.type == EventTypes.Member and ev.state_key == user_id: if ev.content.get("membership") == Membership.INVITE: invite_event = ev @@ -659,7 +657,7 @@ class Mailer: if len(notifs) == 1: # There is just the one notification, so give some detail sender_name = None - event = notif_events[notifs[0]["event_id"]] + event = notif_events[notifs[0].event_id] if ("m.room.member", event.sender) in room_state_ids: state_event_id = room_state_ids[("m.room.member", event.sender)] state_event = await self.store.get_event(state_event_id) @@ -753,9 +751,9 @@ class Mailer: # are already in descending received_ts. sender_ids = {} for n in notifs: - sender = notif_events[n["event_id"]].sender + sender = notif_events[n.event_id].sender if sender not in sender_ids: - sender_ids[sender] = n["event_id"] + sender_ids[sender] = n.event_id # Get the actual member events (in order to calculate a pretty name for # the room). @@ -830,17 +828,17 @@ class Mailer: if self.hs.config.email.email_riot_base_url: return "%s/#/room/%s/%s" % ( self.hs.config.email.email_riot_base_url, - notif["room_id"], - notif["event_id"], + notif.room_id, + notif.event_id, ) elif self.app_name == "Vector": # need /beta for Universal Links to work on iOS return "https://vector.im/beta/#/room/%s/%s" % ( - notif["room_id"], - notif["event_id"], + notif.room_id, + notif.event_id, ) else: - return "https://matrix.to/#/%s/%s" % (notif["room_id"], notif["event_id"]) + return "https://matrix.to/#/%s/%s" % (notif.room_id, notif.event_id) def _make_unsubscribe_link( self, user_id: str, app_id: str, email_address: str diff --git a/synapse/push/push_tools.py b/synapse/push/push_tools.py index da641aca47..957c9b780b 100644 --- a/synapse/push/push_tools.py +++ b/synapse/push/push_tools.py @@ -37,7 +37,7 @@ async def get_badge_count(store: DataStore, user_id: str, group_by_room: bool) - room_id, user_id, last_unread_event_id ) ) - if notifs["notify_count"] == 0: + if notifs.notify_count == 0: continue if group_by_room: @@ -45,7 +45,7 @@ async def get_badge_count(store: DataStore, user_id: str, group_by_room: bool) - badge += 1 else: # increment the badge count by the number of unread messages in the room - badge += notifs["notify_count"] + badge += notifs.notify_count return badge diff --git a/synapse/rest/client/notifications.py b/synapse/rest/client/notifications.py index b12a332776..acd0c9e135 100644 --- a/synapse/rest/client/notifications.py +++ b/synapse/rest/client/notifications.py @@ -58,7 +58,7 @@ class NotificationsServlet(RestServlet): user_id, ReceiptTypes.READ ) - notif_event_ids = [pa["event_id"] for pa in push_actions] + notif_event_ids = [pa.event_id for pa in push_actions] notif_events = await self.store.get_events(notif_event_ids) returned_push_actions = [] @@ -67,30 +67,30 @@ class NotificationsServlet(RestServlet): for pa in push_actions: returned_pa = { - "room_id": pa["room_id"], - "profile_tag": pa["profile_tag"], - "actions": pa["actions"], - "ts": pa["received_ts"], + "room_id": pa.room_id, + "profile_tag": pa.profile_tag, + "actions": pa.actions, + "ts": pa.received_ts, "event": ( await self._event_serializer.serialize_event( - notif_events[pa["event_id"]], + notif_events[pa.event_id], self.clock.time_msec(), event_format=format_event_for_client_v2_without_room_id, ) ), } - if pa["room_id"] not in receipts_by_room: + if pa.room_id not in receipts_by_room: returned_pa["read"] = False else: - receipt = receipts_by_room[pa["room_id"]] + receipt = receipts_by_room[pa.room_id] returned_pa["read"] = ( receipt["topological_ordering"], receipt["stream_ordering"], - ) >= (pa["topological_ordering"], pa["stream_ordering"]) + ) >= (pa.topological_ordering, pa.stream_ordering) returned_push_actions.append(returned_pa) - next_token = str(pa["stream_ordering"]) + next_token = str(pa.stream_ordering) return 200, {"notifications": returned_push_actions, "next_token": next_token} diff --git a/synapse/storage/databases/main/event_push_actions.py b/synapse/storage/databases/main/event_push_actions.py index eacff3e432..98ea0e884c 100644 --- a/synapse/storage/databases/main/event_push_actions.py +++ b/synapse/storage/databases/main/event_push_actions.py @@ -16,7 +16,6 @@ import logging from typing import TYPE_CHECKING, Dict, List, Optional, Tuple, Union import attr -from typing_extensions import TypedDict from synapse.metrics.background_process_metrics import wrap_as_background_process from synapse.storage._base import SQLBaseStore, db_to_json @@ -34,29 +33,64 @@ if TYPE_CHECKING: logger = logging.getLogger(__name__) -DEFAULT_NOTIF_ACTION = ["notify", {"set_tweak": "highlight", "value": False}] -DEFAULT_HIGHLIGHT_ACTION = [ +DEFAULT_NOTIF_ACTION: List[Union[dict, str]] = [ + "notify", + {"set_tweak": "highlight", "value": False}, +] +DEFAULT_HIGHLIGHT_ACTION: List[Union[dict, str]] = [ "notify", {"set_tweak": "sound", "value": "default"}, {"set_tweak": "highlight"}, ] -class BasePushAction(TypedDict): +@attr.s(slots=True, frozen=True, auto_attribs=True) +class HttpPushAction: + """ + HttpPushAction instances include the information used to generate HTTP + requests to a push gateway. + """ + event_id: str + room_id: str + stream_ordering: int actions: List[Union[dict, str]] -class HttpPushAction(BasePushAction): - room_id: str - stream_ordering: int - - +@attr.s(slots=True, frozen=True, auto_attribs=True) class EmailPushAction(HttpPushAction): + """ + EmailPushAction instances include the information used to render an email + push notification. + """ + received_ts: Optional[int] -def _serialize_action(actions, is_highlight): +@attr.s(slots=True, frozen=True, auto_attribs=True) +class UserPushAction(EmailPushAction): + """ + UserPushAction instances include the necessary information to respond to + /notifications requests. + """ + + topological_ordering: int + highlight: bool + profile_tag: str + + +@attr.s(slots=True, frozen=True, auto_attribs=True) +class NotifCounts: + """ + The per-user, per-room count of notifications. Used by sync and push. + """ + + notify_count: int + unread_count: int + highlight_count: int + + +def _serialize_action(actions: List[Union[dict, str]], is_highlight: bool) -> str: """Custom serializer for actions. This allows us to "compress" common actions. We use the fact that most users have the same actions for notifs (and for @@ -74,7 +108,7 @@ def _serialize_action(actions, is_highlight): return json_encoder.encode(actions) -def _deserialize_action(actions, is_highlight): +def _deserialize_action(actions: str, is_highlight: bool) -> List[Union[dict, str]]: """Custom deserializer for actions. This allows us to "compress" common actions""" if actions: return db_to_json(actions) @@ -95,8 +129,8 @@ class EventPushActionsWorkerStore(SQLBaseStore): super().__init__(database, db_conn, hs) # These get correctly set by _find_stream_orderings_for_times_txn - self.stream_ordering_month_ago = None - self.stream_ordering_day_ago = None + self.stream_ordering_month_ago: Optional[int] = None + self.stream_ordering_day_ago: Optional[int] = None cur = db_conn.cursor(txn_name="_find_stream_orderings_for_times_txn") self._find_stream_orderings_for_times_txn(cur) @@ -120,7 +154,7 @@ class EventPushActionsWorkerStore(SQLBaseStore): room_id: str, user_id: str, last_read_event_id: Optional[str], - ) -> Dict[str, int]: + ) -> NotifCounts: """Get the notification count, the highlight count and the unread message count for a given user in a given room after the given read receipt. @@ -149,15 +183,15 @@ class EventPushActionsWorkerStore(SQLBaseStore): def _get_unread_counts_by_receipt_txn( self, - txn, - room_id, - user_id, - last_read_event_id, - ): + txn: LoggingTransaction, + room_id: str, + user_id: str, + last_read_event_id: Optional[str], + ) -> NotifCounts: stream_ordering = None if last_read_event_id is not None: - stream_ordering = self.get_stream_id_for_event_txn( + stream_ordering = self.get_stream_id_for_event_txn( # type: ignore[attr-defined] txn, last_read_event_id, allow_none=True, @@ -175,13 +209,15 @@ class EventPushActionsWorkerStore(SQLBaseStore): retcol="event_id", ) - stream_ordering = self.get_stream_id_for_event_txn(txn, event_id) + stream_ordering = self.get_stream_id_for_event_txn(txn, event_id) # type: ignore[attr-defined] return self._get_unread_counts_by_pos_txn( txn, room_id, user_id, stream_ordering ) - def _get_unread_counts_by_pos_txn(self, txn, room_id, user_id, stream_ordering): + def _get_unread_counts_by_pos_txn( + self, txn: LoggingTransaction, room_id: str, user_id: str, stream_ordering: int + ) -> NotifCounts: sql = ( "SELECT" " COUNT(CASE WHEN notif = 1 THEN 1 END)," @@ -219,16 +255,16 @@ class EventPushActionsWorkerStore(SQLBaseStore): # for this row. unread_count += row[1] - return { - "notify_count": notif_count, - "unread_count": unread_count, - "highlight_count": highlight_count, - } + return NotifCounts( + notify_count=notif_count, + unread_count=unread_count, + highlight_count=highlight_count, + ) async def get_push_action_users_in_range( - self, min_stream_ordering, max_stream_ordering - ): - def f(txn): + self, min_stream_ordering: int, max_stream_ordering: int + ) -> List[str]: + def f(txn: LoggingTransaction) -> List[str]: sql = ( "SELECT DISTINCT(user_id) FROM event_push_actions WHERE" " stream_ordering >= ? AND stream_ordering <= ? AND notif = 1" @@ -236,8 +272,7 @@ class EventPushActionsWorkerStore(SQLBaseStore): txn.execute(sql, (min_stream_ordering, max_stream_ordering)) return [r[0] for r in txn] - ret = await self.db_pool.runInteraction("get_push_action_users_in_range", f) - return ret + return await self.db_pool.runInteraction("get_push_action_users_in_range", f) async def get_unread_push_actions_for_user_in_range_for_http( self, @@ -263,7 +298,9 @@ class EventPushActionsWorkerStore(SQLBaseStore): """ # find rooms that have a read receipt in them and return the next # push actions - def get_after_receipt(txn): + def get_after_receipt( + txn: LoggingTransaction, + ) -> List[Tuple[str, str, int, str, bool]]: # find rooms that have a read receipt in them and return the next # push actions sql = ( @@ -289,7 +326,7 @@ class EventPushActionsWorkerStore(SQLBaseStore): ) args = [user_id, user_id, min_stream_ordering, max_stream_ordering, limit] txn.execute(sql, args) - return txn.fetchall() + return txn.fetchall() # type: ignore[return-value] after_read_receipt = await self.db_pool.runInteraction( "get_unread_push_actions_for_user_in_range_http_arr", get_after_receipt @@ -298,7 +335,9 @@ class EventPushActionsWorkerStore(SQLBaseStore): # There are rooms with push actions in them but you don't have a read receipt in # them e.g. rooms you've been invited to, so get push actions for rooms which do # not have read receipts in them too. - def get_no_receipt(txn): + def get_no_receipt( + txn: LoggingTransaction, + ) -> List[Tuple[str, str, int, str, bool]]: sql = ( "SELECT ep.event_id, ep.room_id, ep.stream_ordering, ep.actions," " ep.highlight " @@ -318,19 +357,19 @@ class EventPushActionsWorkerStore(SQLBaseStore): ) args = [user_id, user_id, min_stream_ordering, max_stream_ordering, limit] txn.execute(sql, args) - return txn.fetchall() + return txn.fetchall() # type: ignore[return-value] no_read_receipt = await self.db_pool.runInteraction( "get_unread_push_actions_for_user_in_range_http_nrr", get_no_receipt ) notifs = [ - { - "event_id": row[0], - "room_id": row[1], - "stream_ordering": row[2], - "actions": _deserialize_action(row[3], row[4]), - } + HttpPushAction( + event_id=row[0], + room_id=row[1], + stream_ordering=row[2], + actions=_deserialize_action(row[3], row[4]), + ) for row in after_read_receipt + no_read_receipt ] @@ -338,7 +377,7 @@ class EventPushActionsWorkerStore(SQLBaseStore): # contain results from the first query, correctly ordered, followed # by results from the second query, but we want them all ordered # by stream_ordering, oldest first. - notifs.sort(key=lambda r: r["stream_ordering"]) + notifs.sort(key=lambda r: r.stream_ordering) # Take only up to the limit. We have to stop at the limit because # one of the subqueries may have hit the limit. @@ -368,7 +407,9 @@ class EventPushActionsWorkerStore(SQLBaseStore): """ # find rooms that have a read receipt in them and return the most recent # push actions - def get_after_receipt(txn): + def get_after_receipt( + txn: LoggingTransaction, + ) -> List[Tuple[str, str, int, str, bool, int]]: sql = ( "SELECT ep.event_id, ep.room_id, ep.stream_ordering, ep.actions," " ep.highlight, e.received_ts" @@ -393,7 +434,7 @@ class EventPushActionsWorkerStore(SQLBaseStore): ) args = [user_id, user_id, min_stream_ordering, max_stream_ordering, limit] txn.execute(sql, args) - return txn.fetchall() + return txn.fetchall() # type: ignore[return-value] after_read_receipt = await self.db_pool.runInteraction( "get_unread_push_actions_for_user_in_range_email_arr", get_after_receipt @@ -402,7 +443,9 @@ class EventPushActionsWorkerStore(SQLBaseStore): # There are rooms with push actions in them but you don't have a read receipt in # them e.g. rooms you've been invited to, so get push actions for rooms which do # not have read receipts in them too. - def get_no_receipt(txn): + def get_no_receipt( + txn: LoggingTransaction, + ) -> List[Tuple[str, str, int, str, bool, int]]: sql = ( "SELECT ep.event_id, ep.room_id, ep.stream_ordering, ep.actions," " ep.highlight, e.received_ts" @@ -422,7 +465,7 @@ class EventPushActionsWorkerStore(SQLBaseStore): ) args = [user_id, user_id, min_stream_ordering, max_stream_ordering, limit] txn.execute(sql, args) - return txn.fetchall() + return txn.fetchall() # type: ignore[return-value] no_read_receipt = await self.db_pool.runInteraction( "get_unread_push_actions_for_user_in_range_email_nrr", get_no_receipt @@ -430,13 +473,13 @@ class EventPushActionsWorkerStore(SQLBaseStore): # Make a list of dicts from the two sets of results. notifs = [ - { - "event_id": row[0], - "room_id": row[1], - "stream_ordering": row[2], - "actions": _deserialize_action(row[3], row[4]), - "received_ts": row[5], - } + EmailPushAction( + event_id=row[0], + room_id=row[1], + stream_ordering=row[2], + actions=_deserialize_action(row[3], row[4]), + received_ts=row[5], + ) for row in after_read_receipt + no_read_receipt ] @@ -444,7 +487,7 @@ class EventPushActionsWorkerStore(SQLBaseStore): # contain results from the first query, correctly ordered, followed # by results from the second query, but we want them all ordered # by received_ts (most recent first) - notifs.sort(key=lambda r: -(r["received_ts"] or 0)) + notifs.sort(key=lambda r: -(r.received_ts or 0)) # Now return the first `limit` return notifs[:limit] @@ -465,7 +508,7 @@ class EventPushActionsWorkerStore(SQLBaseStore): True if there may be push to process, False if there definitely isn't. """ - def _get_if_maybe_push_in_range_for_user_txn(txn): + def _get_if_maybe_push_in_range_for_user_txn(txn: LoggingTransaction) -> bool: sql = """ SELECT 1 FROM event_push_actions WHERE user_id = ? AND stream_ordering > ? AND notif = 1 @@ -499,19 +542,21 @@ class EventPushActionsWorkerStore(SQLBaseStore): # This is a helper function for generating the necessary tuple that # can be used to insert into the `event_push_actions_staging` table. - def _gen_entry(user_id, actions): + def _gen_entry( + user_id: str, actions: List[Union[dict, str]] + ) -> Tuple[str, str, str, int, int, int]: is_highlight = 1 if _action_has_highlight(actions) else 0 notif = 1 if "notify" in actions else 0 return ( event_id, # event_id column user_id, # user_id column - _serialize_action(actions, is_highlight), # actions column + _serialize_action(actions, bool(is_highlight)), # actions column notif, # notif column is_highlight, # highlight column int(count_as_unread), # unread column ) - def _add_push_actions_to_staging_txn(txn): + def _add_push_actions_to_staging_txn(txn: LoggingTransaction) -> None: # We don't use simple_insert_many here to avoid the overhead # of generating lists of dicts. @@ -539,12 +584,11 @@ class EventPushActionsWorkerStore(SQLBaseStore): """ try: - res = await self.db_pool.simple_delete( + await self.db_pool.simple_delete( table="event_push_actions_staging", keyvalues={"event_id": event_id}, desc="remove_push_actions_from_staging", ) - return res except Exception: # this method is called from an exception handler, so propagating # another exception here really isn't helpful - there's nothing @@ -597,7 +641,9 @@ class EventPushActionsWorkerStore(SQLBaseStore): ) @staticmethod - def _find_first_stream_ordering_after_ts_txn(txn, ts): + def _find_first_stream_ordering_after_ts_txn( + txn: LoggingTransaction, ts: int + ) -> int: """ Find the stream_ordering of the first event that was received on or after a given timestamp. This is relatively slow as there is no index @@ -609,14 +655,14 @@ class EventPushActionsWorkerStore(SQLBaseStore): stream_ordering Args: - txn (twisted.enterprise.adbapi.Transaction): - ts (int): timestamp to search for + txn: + ts: timestamp to search for Returns: - int: stream ordering + The stream ordering """ txn.execute("SELECT MAX(stream_ordering) FROM events") - max_stream_ordering = txn.fetchone()[0] + max_stream_ordering = txn.fetchone()[0] # type: ignore[index] if max_stream_ordering is None: return 0 @@ -672,8 +718,10 @@ class EventPushActionsWorkerStore(SQLBaseStore): return range_end - async def get_time_of_last_push_action_before(self, stream_ordering): - def f(txn): + async def get_time_of_last_push_action_before( + self, stream_ordering: int + ) -> Optional[int]: + def f(txn: LoggingTransaction) -> Optional[Tuple[int]]: sql = ( "SELECT e.received_ts" " FROM event_push_actions AS ep" @@ -683,7 +731,7 @@ class EventPushActionsWorkerStore(SQLBaseStore): " LIMIT 1" ) txn.execute(sql, (stream_ordering,)) - return txn.fetchone() + return txn.fetchone() # type: ignore[return-value] result = await self.db_pool.runInteraction( "get_time_of_last_push_action_before", f @@ -691,7 +739,7 @@ class EventPushActionsWorkerStore(SQLBaseStore): return result[0] if result else None @wrap_as_background_process("rotate_notifs") - async def _rotate_notifs(self): + async def _rotate_notifs(self) -> None: if self._doing_notif_rotation or self.stream_ordering_day_ago is None: return self._doing_notif_rotation = True @@ -709,7 +757,7 @@ class EventPushActionsWorkerStore(SQLBaseStore): finally: self._doing_notif_rotation = False - def _rotate_notifs_txn(self, txn): + def _rotate_notifs_txn(self, txn: LoggingTransaction) -> bool: """Archives older notifications into event_push_summary. Returns whether the archiving process has caught up or not. """ @@ -734,6 +782,7 @@ class EventPushActionsWorkerStore(SQLBaseStore): stream_row = txn.fetchone() if stream_row: (offset_stream_ordering,) = stream_row + assert self.stream_ordering_day_ago is not None rotate_to_stream_ordering = min( self.stream_ordering_day_ago, offset_stream_ordering ) @@ -749,7 +798,9 @@ class EventPushActionsWorkerStore(SQLBaseStore): # We have caught up iff we were limited by `stream_ordering_day_ago` return caught_up - def _rotate_notifs_before_txn(self, txn, rotate_to_stream_ordering): + def _rotate_notifs_before_txn( + self, txn: LoggingTransaction, rotate_to_stream_ordering: int + ) -> None: old_rotate_stream_ordering = self.db_pool.simple_select_one_onecol_txn( txn, table="event_push_summary_stream_ordering", @@ -870,8 +921,8 @@ class EventPushActionsWorkerStore(SQLBaseStore): ) def _remove_old_push_actions_before_txn( - self, txn, room_id, user_id, stream_ordering - ): + self, txn: LoggingTransaction, room_id: str, user_id: str, stream_ordering: int + ) -> None: """ Purges old push actions for a user and room before a given stream_ordering. @@ -943,9 +994,15 @@ class EventPushActionsStore(EventPushActionsWorkerStore): ) async def get_push_actions_for_user( - self, user_id, before=None, limit=50, only_highlight=False - ): - def f(txn): + self, + user_id: str, + before: Optional[str] = None, + limit: int = 50, + only_highlight: bool = False, + ) -> List[UserPushAction]: + def f( + txn: LoggingTransaction, + ) -> List[Tuple[str, str, int, int, str, bool, str, int]]: before_clause = "" if before: before_clause = "AND epa.stream_ordering < ?" @@ -972,32 +1029,42 @@ class EventPushActionsStore(EventPushActionsWorkerStore): " LIMIT ?" % (before_clause,) ) txn.execute(sql, args) - return self.db_pool.cursor_to_dict(txn) + return txn.fetchall() # type: ignore[return-value] push_actions = await self.db_pool.runInteraction("get_push_actions_for_user", f) - for pa in push_actions: - pa["actions"] = _deserialize_action(pa["actions"], pa["highlight"]) - return push_actions + return [ + UserPushAction( + event_id=row[0], + room_id=row[1], + stream_ordering=row[2], + actions=_deserialize_action(row[4], row[5]), + received_ts=row[7], + topological_ordering=row[3], + highlight=row[5], + profile_tag=row[6], + ) + for row in push_actions + ] -def _action_has_highlight(actions): +def _action_has_highlight(actions: List[Union[dict, str]]) -> bool: for action in actions: - try: - if action.get("set_tweak", None) == "highlight": - return action.get("value", True) - except AttributeError: - pass + if not isinstance(action, dict): + continue + + if action.get("set_tweak", None) == "highlight": + return action.get("value", True) return False -@attr.s(slots=True) +@attr.s(slots=True, auto_attribs=True) class _EventPushSummary: """Summary of pending event push actions for a given user in a given room. Used in _rotate_notifs_before_txn to manipulate results from event_push_actions. """ - unread_count = attr.ib(type=int) - stream_ordering = attr.ib(type=int) - old_user_id = attr.ib(type=str) - notif_count = attr.ib(type=int) + unread_count: int + stream_ordering: int + old_user_id: str + notif_count: int diff --git a/tests/replication/slave/storage/test_events.py b/tests/replication/slave/storage/test_events.py index b25a06b427..eca6a443af 100644 --- a/tests/replication/slave/storage/test_events.py +++ b/tests/replication/slave/storage/test_events.py @@ -20,6 +20,7 @@ from synapse.api.room_versions import RoomVersions from synapse.events import FrozenEvent, _EventInternalMetadata, make_event_from_dict from synapse.handlers.room import RoomEventSource from synapse.replication.slave.storage.events import SlavedEventStore +from synapse.storage.databases.main.event_push_actions import NotifCounts from synapse.storage.roommember import GetRoomsForUserWithStreamOrdering, RoomsForUser from synapse.types import PersistedEventPosition @@ -166,7 +167,7 @@ class SlavedEventStoreTestCase(BaseSlavedStoreTestCase): self.check( "get_unread_event_push_actions_by_room_for_user", [ROOM_ID, USER_ID_2, event1.event_id], - {"highlight_count": 0, "unread_count": 0, "notify_count": 0}, + NotifCounts(highlight_count=0, unread_count=0, notify_count=0), ) self.persist( @@ -179,7 +180,7 @@ class SlavedEventStoreTestCase(BaseSlavedStoreTestCase): self.check( "get_unread_event_push_actions_by_room_for_user", [ROOM_ID, USER_ID_2, event1.event_id], - {"highlight_count": 0, "unread_count": 0, "notify_count": 1}, + NotifCounts(highlight_count=0, unread_count=0, notify_count=1), ) self.persist( @@ -194,7 +195,7 @@ class SlavedEventStoreTestCase(BaseSlavedStoreTestCase): self.check( "get_unread_event_push_actions_by_room_for_user", [ROOM_ID, USER_ID_2, event1.event_id], - {"highlight_count": 1, "unread_count": 0, "notify_count": 2}, + NotifCounts(highlight_count=1, unread_count=0, notify_count=2), ) def test_get_rooms_for_user_with_stream_ordering(self): diff --git a/tests/storage/test_event_push_actions.py b/tests/storage/test_event_push_actions.py index bb5939ba4a..738f3ad1dc 100644 --- a/tests/storage/test_event_push_actions.py +++ b/tests/storage/test_event_push_actions.py @@ -14,6 +14,8 @@ from unittest.mock import Mock +from synapse.storage.databases.main.event_push_actions import NotifCounts + from tests.unittest import HomeserverTestCase USER_ID = "@user:example.com" @@ -57,11 +59,11 @@ class EventPushActionsStoreTestCase(HomeserverTestCase): ) self.assertEquals( counts, - { - "notify_count": noitf_count, - "unread_count": 0, # Unread counts are tested in the sync tests. - "highlight_count": highlight_count, - }, + NotifCounts( + notify_count=noitf_count, + unread_count=0, # Unread counts are tested in the sync tests. + highlight_count=highlight_count, + ), ) def _inject_actions(stream, action): From e6897e7383178db0bd6243e68346a420bf90b982 Mon Sep 17 00:00:00 2001 From: reivilibre Date: Tue, 21 Dec 2021 16:12:05 +0000 Subject: [PATCH 17/76] Refactor `tests.util.setup_test_homeserver` and `tests.server.setup_test_homeserver`. (#11503) --- changelog.d/11503.misc | 1 + tests/server.py | 199 +++++++++++++++++++++++++++++-- tests/storage/test_base.py | 3 +- tests/storage/test_roommember.py | 2 +- tests/utils.py | 175 +-------------------------- 5 files changed, 195 insertions(+), 185 deletions(-) create mode 100644 changelog.d/11503.misc diff --git a/changelog.d/11503.misc b/changelog.d/11503.misc new file mode 100644 index 0000000000..03a24a9224 --- /dev/null +++ b/changelog.d/11503.misc @@ -0,0 +1 @@ +Refactor `tests.util.setup_test_homeserver` and `tests.server.setup_test_homeserver`. \ No newline at end of file diff --git a/tests/server.py b/tests/server.py index 40cf5b12c3..ca2b7a5b97 100644 --- a/tests/server.py +++ b/tests/server.py @@ -11,9 +11,12 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - +import hashlib import json import logging +import time +import uuid +import warnings from collections import deque from io import SEEK_END, BytesIO from typing import ( @@ -27,6 +30,7 @@ from typing import ( Type, Union, ) +from unittest.mock import Mock import attr from typing_extensions import Deque @@ -53,11 +57,24 @@ from twisted.web.http_headers import Headers from twisted.web.resource import IResource from twisted.web.server import Request, Site +from synapse.config.database import DatabaseConnectionConfig from synapse.http.site import SynapseRequest +from synapse.server import HomeServer +from synapse.storage import DataStore +from synapse.storage.engines import PostgresEngine, create_engine from synapse.types import JsonDict from synapse.util import Clock -from tests.utils import setup_test_homeserver as _sth +from tests.utils import ( + LEAVE_DB, + POSTGRES_BASE_DB, + POSTGRES_HOST, + POSTGRES_PASSWORD, + POSTGRES_USER, + USE_POSTGRES_FOR_TESTS, + MockClock, + default_config, +) logger = logging.getLogger(__name__) @@ -450,14 +467,11 @@ class ThreadPool: return d -def setup_test_homeserver(cleanup_func, *args, **kwargs): +def _make_test_homeserver_synchronous(server: HomeServer) -> None: """ - Set up a synchronous test server, driven by the reactor used by - the homeserver. + Make the given test homeserver's database interactions synchronous. """ - server = _sth(cleanup_func, *args, **kwargs) - # Make the thread pool synchronous. clock = server.get_clock() for database in server.get_datastores().databases: @@ -485,6 +499,7 @@ def setup_test_homeserver(cleanup_func, *args, **kwargs): pool.runWithConnection = runWithConnection pool.runInteraction = runInteraction + # Replace the thread pool with a threadless 'thread' pool pool.threadpool = ThreadPool(clock._reactor) pool.running = True @@ -492,8 +507,6 @@ def setup_test_homeserver(cleanup_func, *args, **kwargs): # thread, so we need to disable the dedicated thread behaviour. server.get_datastores().main.USE_DEDICATED_DB_THREADS_FOR_EVENT_FETCHING = False - return server - def get_clock() -> Tuple[ThreadedMemoryReactorClock, Clock]: clock = ThreadedMemoryReactorClock() @@ -673,3 +686,171 @@ def connect_client( client.makeConnection(FakeTransport(server, reactor)) return client, server + + +class TestHomeServer(HomeServer): + DATASTORE_CLASS = DataStore + + +def setup_test_homeserver( + cleanup_func, + name="test", + config=None, + reactor=None, + homeserver_to_use: Type[HomeServer] = TestHomeServer, + **kwargs, +): + """ + Setup a homeserver suitable for running tests against. Keyword arguments + are passed to the Homeserver constructor. + + If no datastore is supplied, one is created and given to the homeserver. + + Args: + cleanup_func : The function used to register a cleanup routine for + after the test. + + Calling this method directly is deprecated: you should instead derive from + HomeserverTestCase. + """ + if reactor is None: + from twisted.internet import reactor + + if config is None: + config = default_config(name, parse=True) + + config.ldap_enabled = False + + if "clock" not in kwargs: + kwargs["clock"] = MockClock() + + if USE_POSTGRES_FOR_TESTS: + test_db = "synapse_test_%s" % uuid.uuid4().hex + + database_config = { + "name": "psycopg2", + "args": { + "database": test_db, + "host": POSTGRES_HOST, + "password": POSTGRES_PASSWORD, + "user": POSTGRES_USER, + "cp_min": 1, + "cp_max": 5, + }, + } + else: + database_config = { + "name": "sqlite3", + "args": {"database": ":memory:", "cp_min": 1, "cp_max": 1}, + } + + if "db_txn_limit" in kwargs: + database_config["txn_limit"] = kwargs["db_txn_limit"] + + database = DatabaseConnectionConfig("master", database_config) + config.database.databases = [database] + + db_engine = create_engine(database.config) + + # Create the database before we actually try and connect to it, based off + # the template database we generate in setupdb() + if isinstance(db_engine, PostgresEngine): + db_conn = db_engine.module.connect( + database=POSTGRES_BASE_DB, + user=POSTGRES_USER, + host=POSTGRES_HOST, + password=POSTGRES_PASSWORD, + ) + db_conn.autocommit = True + cur = db_conn.cursor() + cur.execute("DROP DATABASE IF EXISTS %s;" % (test_db,)) + cur.execute( + "CREATE DATABASE %s WITH TEMPLATE %s;" % (test_db, POSTGRES_BASE_DB) + ) + cur.close() + db_conn.close() + + hs = homeserver_to_use( + name, + config=config, + version_string="Synapse/tests", + reactor=reactor, + ) + + # Install @cache_in_self attributes + for key, val in kwargs.items(): + setattr(hs, "_" + key, val) + + # Mock TLS + hs.tls_server_context_factory = Mock() + hs.tls_client_options_factory = Mock() + + hs.setup() + if homeserver_to_use == TestHomeServer: + hs.setup_background_tasks() + + if isinstance(db_engine, PostgresEngine): + database = hs.get_datastores().databases[0] + + # We need to do cleanup on PostgreSQL + def cleanup(): + import psycopg2 + + # Close all the db pools + database._db_pool.close() + + dropped = False + + # Drop the test database + db_conn = db_engine.module.connect( + database=POSTGRES_BASE_DB, + user=POSTGRES_USER, + host=POSTGRES_HOST, + password=POSTGRES_PASSWORD, + ) + db_conn.autocommit = True + cur = db_conn.cursor() + + # Try a few times to drop the DB. Some things may hold on to the + # database for a few more seconds due to flakiness, preventing + # us from dropping it when the test is over. If we can't drop + # it, warn and move on. + for _ in range(5): + try: + cur.execute("DROP DATABASE IF EXISTS %s;" % (test_db,)) + db_conn.commit() + dropped = True + except psycopg2.OperationalError as e: + warnings.warn( + "Couldn't drop old db: " + str(e), category=UserWarning + ) + time.sleep(0.5) + + cur.close() + db_conn.close() + + if not dropped: + warnings.warn("Failed to drop old DB.", category=UserWarning) + + if not LEAVE_DB: + # Register the cleanup hook + cleanup_func(cleanup) + + # bcrypt is far too slow to be doing in unit tests + # Need to let the HS build an auth handler and then mess with it + # because AuthHandler's constructor requires the HS, so we can't make one + # beforehand and pass it in to the HS's constructor (chicken / egg) + async def hash(p): + return hashlib.md5(p.encode("utf8")).hexdigest() + + hs.get_auth_handler().hash = hash + + async def validate_hash(p, h): + return hashlib.md5(p.encode("utf8")).hexdigest() == h + + hs.get_auth_handler().validate_hash = validate_hash + + # Make the threadpool and database transactions synchronous for testing. + _make_test_homeserver_synchronous(hs) + + return hs diff --git a/tests/storage/test_base.py b/tests/storage/test_base.py index ddad44bd6c..3e4f0579c9 100644 --- a/tests/storage/test_base.py +++ b/tests/storage/test_base.py @@ -23,7 +23,8 @@ from synapse.storage.database import DatabasePool from synapse.storage.engines import create_engine from tests import unittest -from tests.utils import TestHomeServer, default_config +from tests.server import TestHomeServer +from tests.utils import default_config class SQLBaseStoreTestCase(unittest.TestCase): diff --git a/tests/storage/test_roommember.py b/tests/storage/test_roommember.py index fccab733c0..5cfdfe9b85 100644 --- a/tests/storage/test_roommember.py +++ b/tests/storage/test_roommember.py @@ -19,8 +19,8 @@ from synapse.rest.client import login, room from synapse.types import UserID, create_requester from tests import unittest +from tests.server import TestHomeServer from tests.test_utils import event_injection -from tests.utils import TestHomeServer class RoomMemberStoreTestCase(unittest.HomeserverTestCase): diff --git a/tests/utils.py b/tests/utils.py index 983859120f..6d013e8518 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -14,12 +14,7 @@ # limitations under the License. import atexit -import hashlib import os -import time -import uuid -import warnings -from typing import Type from unittest.mock import Mock, patch from urllib import parse as urlparse @@ -28,14 +23,11 @@ from twisted.internet import defer from synapse.api.constants import EventTypes from synapse.api.errors import CodeMessageException, cs_error from synapse.api.room_versions import RoomVersions -from synapse.config.database import DatabaseConnectionConfig from synapse.config.homeserver import HomeServerConfig from synapse.config.server import DEFAULT_ROOM_VERSION from synapse.logging.context import current_context, set_current_context -from synapse.server import HomeServer -from synapse.storage import DataStore from synapse.storage.database import LoggingDatabaseConnection -from synapse.storage.engines import PostgresEngine, create_engine +from synapse.storage.engines import create_engine from synapse.storage.prepare_database import prepare_database # set this to True to run the tests against postgres instead of sqlite. @@ -182,171 +174,6 @@ def default_config(name, parse=False): return config_dict -class TestHomeServer(HomeServer): - DATASTORE_CLASS = DataStore - - -def setup_test_homeserver( - cleanup_func, - name="test", - config=None, - reactor=None, - homeserver_to_use: Type[HomeServer] = TestHomeServer, - **kwargs, -): - """ - Setup a homeserver suitable for running tests against. Keyword arguments - are passed to the Homeserver constructor. - - If no datastore is supplied, one is created and given to the homeserver. - - Args: - cleanup_func : The function used to register a cleanup routine for - after the test. - - Calling this method directly is deprecated: you should instead derive from - HomeserverTestCase. - """ - if reactor is None: - from twisted.internet import reactor - - if config is None: - config = default_config(name, parse=True) - - config.ldap_enabled = False - - if "clock" not in kwargs: - kwargs["clock"] = MockClock() - - if USE_POSTGRES_FOR_TESTS: - test_db = "synapse_test_%s" % uuid.uuid4().hex - - database_config = { - "name": "psycopg2", - "args": { - "database": test_db, - "host": POSTGRES_HOST, - "password": POSTGRES_PASSWORD, - "user": POSTGRES_USER, - "cp_min": 1, - "cp_max": 5, - }, - } - else: - database_config = { - "name": "sqlite3", - "args": {"database": ":memory:", "cp_min": 1, "cp_max": 1}, - } - - if "db_txn_limit" in kwargs: - database_config["txn_limit"] = kwargs["db_txn_limit"] - - database = DatabaseConnectionConfig("master", database_config) - config.database.databases = [database] - - db_engine = create_engine(database.config) - - # Create the database before we actually try and connect to it, based off - # the template database we generate in setupdb() - if isinstance(db_engine, PostgresEngine): - db_conn = db_engine.module.connect( - database=POSTGRES_BASE_DB, - user=POSTGRES_USER, - host=POSTGRES_HOST, - password=POSTGRES_PASSWORD, - ) - db_conn.autocommit = True - cur = db_conn.cursor() - cur.execute("DROP DATABASE IF EXISTS %s;" % (test_db,)) - cur.execute( - "CREATE DATABASE %s WITH TEMPLATE %s;" % (test_db, POSTGRES_BASE_DB) - ) - cur.close() - db_conn.close() - - hs = homeserver_to_use( - name, - config=config, - version_string="Synapse/tests", - reactor=reactor, - ) - - # Install @cache_in_self attributes - for key, val in kwargs.items(): - setattr(hs, "_" + key, val) - - # Mock TLS - hs.tls_server_context_factory = Mock() - hs.tls_client_options_factory = Mock() - - hs.setup() - if homeserver_to_use == TestHomeServer: - hs.setup_background_tasks() - - if isinstance(db_engine, PostgresEngine): - database = hs.get_datastores().databases[0] - - # We need to do cleanup on PostgreSQL - def cleanup(): - import psycopg2 - - # Close all the db pools - database._db_pool.close() - - dropped = False - - # Drop the test database - db_conn = db_engine.module.connect( - database=POSTGRES_BASE_DB, - user=POSTGRES_USER, - host=POSTGRES_HOST, - password=POSTGRES_PASSWORD, - ) - db_conn.autocommit = True - cur = db_conn.cursor() - - # Try a few times to drop the DB. Some things may hold on to the - # database for a few more seconds due to flakiness, preventing - # us from dropping it when the test is over. If we can't drop - # it, warn and move on. - for _ in range(5): - try: - cur.execute("DROP DATABASE IF EXISTS %s;" % (test_db,)) - db_conn.commit() - dropped = True - except psycopg2.OperationalError as e: - warnings.warn( - "Couldn't drop old db: " + str(e), category=UserWarning - ) - time.sleep(0.5) - - cur.close() - db_conn.close() - - if not dropped: - warnings.warn("Failed to drop old DB.", category=UserWarning) - - if not LEAVE_DB: - # Register the cleanup hook - cleanup_func(cleanup) - - # bcrypt is far too slow to be doing in unit tests - # Need to let the HS build an auth handler and then mess with it - # because AuthHandler's constructor requires the HS, so we can't make one - # beforehand and pass it in to the HS's constructor (chicken / egg) - async def hash(p): - return hashlib.md5(p.encode("utf8")).hexdigest() - - hs.get_auth_handler().hash = hash - - async def validate_hash(p, h): - return hashlib.md5(p.encode("utf8")).hexdigest() == h - - hs.get_auth_handler().validate_hash = validate_hash - - return hs - - def mock_getRawHeaders(headers=None): headers = headers if headers is not None else {} From 2bf31f7807c7a0c229170803c97090d612dc16f9 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff <1389908+richvdh@users.noreply.github.com> Date: Tue, 21 Dec 2021 17:26:04 +0000 Subject: [PATCH 18/76] Pin to frozendict<2.1.2 (#11625) ... to work around breakage on buster (https://github.com/Marco-Sulla/python-frozendict/issues/41) --- changelog.d/11625.misc | 1 + synapse/python_dependencies.py | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) create mode 100644 changelog.d/11625.misc diff --git a/changelog.d/11625.misc b/changelog.d/11625.misc new file mode 100644 index 0000000000..82ee78e724 --- /dev/null +++ b/changelog.d/11625.misc @@ -0,0 +1 @@ +Work around a build problem on Debian Buster. diff --git a/synapse/python_dependencies.py b/synapse/python_dependencies.py index 7d26954244..13fb69460e 100644 --- a/synapse/python_dependencies.py +++ b/synapse/python_dependencies.py @@ -50,7 +50,8 @@ logger = logging.getLogger(__name__) REQUIREMENTS = [ # we use the TYPE_CHECKER.redefine method added in jsonschema 3.0.0 "jsonschema>=3.0.0", - "frozendict>=1", + # frozendict 2.1.2 is broken on Debian 10: https://github.com/Marco-Sulla/python-frozendict/issues/41 + "frozendict>=1,<2.1.2", "unpaddedbase64>=1.1.0", "canonicaljson>=1.4.0", # we use the type definitions added in signedjson 1.1. From aa874a13902fcf7f3e4024cd95a8491f308367bb Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Tue, 21 Dec 2021 17:32:16 +0000 Subject: [PATCH 19/76] 1.49.2 --- CHANGES.md | 13 ++++++++++++- changelog.d/11625.misc | 1 - debian/changelog | 6 ++++++ synapse/__init__.py | 2 +- 4 files changed, 19 insertions(+), 3 deletions(-) delete mode 100644 changelog.d/11625.misc diff --git a/CHANGES.md b/CHANGES.md index dc6fa17794..9301ec8804 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,4 +1,4 @@ -Synapse 1.49.1 (2021-12-21) +Synapse 1.49.2 (2021-12-21) =========================== This release fixes a regression introduced in Synapse 1.49.0 which could cause `/sync` requests to take significantly longer. This would particularly affect "initial" syncs for users participating in a large number of rooms, and in extreme cases, could make it impossible for such users to log in on a new client. @@ -12,6 +12,17 @@ Bugfixes - Fix a performance regression in `/sync` handling, introduced in 1.49.0. ([\#11583](https://github.com/matrix-org/synapse/issues/11583)) +Internal Changes +---------------- + +- Work around a build problem on Debian Buster. ([\#11625](https://github.com/matrix-org/synapse/issues/11625)) + + +Synapse 1.49.1 (2021-12-21) +=========================== + +Not released due to problems building the debian packages. + Synapse 1.49.0 (2021-12-14) =========================== diff --git a/changelog.d/11625.misc b/changelog.d/11625.misc deleted file mode 100644 index 82ee78e724..0000000000 --- a/changelog.d/11625.misc +++ /dev/null @@ -1 +0,0 @@ -Work around a build problem on Debian Buster. diff --git a/debian/changelog b/debian/changelog index 61a1d003b2..ebe3e0cbf9 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,9 @@ +matrix-synapse-py3 (1.49.2) stable; urgency=medium + + * New synapse release 1.49.2. + + -- Synapse Packaging team Tue, 21 Dec 2021 17:31:03 +0000 + matrix-synapse-py3 (1.49.1) stable; urgency=medium * New synapse release 1.49.1. diff --git a/synapse/__init__.py b/synapse/__init__.py index d45e0f1883..95a49c20be 100644 --- a/synapse/__init__.py +++ b/synapse/__init__.py @@ -47,7 +47,7 @@ try: except ImportError: pass -__version__ = "1.49.1" +__version__ = "1.49.2" if bool(os.environ.get("SYNAPSE_TEST_PATCH_LOG_CONTEXTS", False)): # We import here so that we don't have to install a bunch of deps when From 6b6dcdc338a3fcfa1da6c3e2a38f62c268df1cbc Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Tue, 21 Dec 2021 17:41:34 +0000 Subject: [PATCH 20/76] update changelog postgres 10 _+_ --- CHANGES.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index 9301ec8804..9f6e29631d 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -3,7 +3,7 @@ Synapse 1.49.2 (2021-12-21) This release fixes a regression introduced in Synapse 1.49.0 which could cause `/sync` requests to take significantly longer. This would particularly affect "initial" syncs for users participating in a large number of rooms, and in extreme cases, could make it impossible for such users to log in on a new client. -**Note:** in line with our [deprecation policy](https://matrix-org.github.io/synapse/latest/deprecation_policy.html) for platform dependencies, this will be the last release to support Python 3.6 and PostgreSQL 9.6, both of which have now reached upstream end-of-life. Synapse will require Python 3.7+ and PostgreSQL 10. +**Note:** in line with our [deprecation policy](https://matrix-org.github.io/synapse/latest/deprecation_policy.html) for platform dependencies, this will be the last release to support Python 3.6 and PostgreSQL 9.6, both of which have now reached upstream end-of-life. Synapse will require Python 3.7+ and PostgreSQL 10+. **Note:** We will also stop producing packages for Ubuntu 18.04 (Bionic Beaver) after this release, as it uses Python 3.6. From 87da37374a68b935fe8a9378a7d369448461c128 Mon Sep 17 00:00:00 2001 From: Shay Date: Tue, 21 Dec 2021 11:37:04 -0800 Subject: [PATCH 21/76] Drop EOL python 3.6 from CI (#11595) * remove python 3.6 and postgres 9.6 from github workflow * remove python 3.6 env from tox * newsfragment * correct postgres version * add py310 to tox env list --- .github/workflows/tests.yml | 12 ++++++------ changelog.d/11595.misc | 1 + tox.ini | 2 +- 3 files changed, 8 insertions(+), 7 deletions(-) create mode 100644 changelog.d/11595.misc diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 21c9ee7823..cb72e1a233 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -76,7 +76,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: ["3.6", "3.7", "3.8", "3.9", "3.10"] + python-version: ["3.7", "3.8", "3.9", "3.10"] database: ["sqlite"] toxenv: ["py"] include: @@ -85,9 +85,9 @@ jobs: toxenv: "py-noextras" # Oldest Python with PostgreSQL - - python-version: "3.6" + - python-version: "3.7" database: "postgres" - postgres-version: "9.6" + postgres-version: "10" toxenv: "py" # Newest Python with newest PostgreSQL @@ -167,7 +167,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: ["pypy-3.6"] + python-version: ["pypy-3.7"] steps: - uses: actions/checkout@v2 @@ -291,8 +291,8 @@ jobs: strategy: matrix: include: - - python-version: "3.6" - postgres-version: "9.6" + - python-version: "3.7" + postgres-version: "10" - python-version: "3.10" postgres-version: "14" diff --git a/changelog.d/11595.misc b/changelog.d/11595.misc new file mode 100644 index 0000000000..57e54db5a9 --- /dev/null +++ b/changelog.d/11595.misc @@ -0,0 +1 @@ +Drop EOL python 3.6 and postgres 9.6 from CI. \ No newline at end of file diff --git a/tox.ini b/tox.ini index cfe6a06942..2ffca14b22 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = packaging, py36, py37, py38, py39, check_codestyle, check_isort +envlist = packaging, py37, py38, py39, py310, check_codestyle, check_isort # we require tox>=2.3.2 for the fix to https://github.com/tox-dev/tox/issues/208 minversion = 2.3.2 From c500bf37d660b08efb48501b7690dc4448b39eca Mon Sep 17 00:00:00 2001 From: AndrewFerr Date: Thu, 23 Dec 2021 05:42:50 -0500 Subject: [PATCH 22/76] Add details for how to set up TURN behind NAT (#11553) Signed-off-by: Andrew Ferrazzutti --- changelog.d/11553.doc | 1 + docs/turn-howto.md | 69 +++++++++++++++++++++++++++++++++---------- 2 files changed, 54 insertions(+), 16 deletions(-) create mode 100644 changelog.d/11553.doc diff --git a/changelog.d/11553.doc b/changelog.d/11553.doc new file mode 100644 index 0000000000..810ba16928 --- /dev/null +++ b/changelog.d/11553.doc @@ -0,0 +1 @@ +Add details for how to configure a TURN server when behind a NAT. Contibuted by @AndrewFerr. diff --git a/docs/turn-howto.md b/docs/turn-howto.md index e6812de69e..e32aaa1850 100644 --- a/docs/turn-howto.md +++ b/docs/turn-howto.md @@ -15,8 +15,8 @@ The following sections describe how to install [coturn](TURN->TURN->client flows work + # this should be one of the turn server's listening IPs allowed-peer-ip=10.0.0.1 # consider whether you want to limit the quota of relayed streams per user (or total) to avoid risk of DoS. @@ -123,7 +139,7 @@ This will install and start a systemd service called `coturn`. pkey=/path/to/privkey.pem ``` - In this case, replace the `turn:` schemes in the `turn_uri` settings below + In this case, replace the `turn:` schemes in the `turn_uris` settings below with `turns:`. We recommend that you only try to set up TLS/DTLS once you have set up a @@ -134,21 +150,33 @@ This will install and start a systemd service called `coturn`. traffic (remember to allow both TCP and UDP traffic), and ports 49152-65535 for the UDP relay.) -1. We do not recommend running a TURN server behind NAT, and are not aware of - anyone doing so successfully. - - If you want to try it anyway, you will at least need to tell coturn its - external IP address: +1. If your TURN server is behind NAT, the NAT gateway must have an external, + publicly-reachable IP address. You must configure coturn to advertise that + address to connecting clients: ``` - external-ip=192.88.99.1 + external-ip=EXTERNAL_NAT_IPv4_ADDRESS ``` - ... and your NAT gateway must forward all of the relayed ports directly - (eg, port 56789 on the external IP must be always be forwarded to port - 56789 on the internal IP). + You may optionally limit the TURN server to listen only on the local + address that is mapped by NAT to the external address: - If you get this working, let us know! + ``` + listening-ip=INTERNAL_TURNSERVER_IPv4_ADDRESS + ``` + + If your NAT gateway is reachable over both IPv4 and IPv6, you may + configure coturn to advertise each available address: + + ``` + external-ip=EXTERNAL_NAT_IPv4_ADDRESS + external-ip=EXTERNAL_NAT_IPv6_ADDRESS + ``` + + When advertising an external IPv6 address, ensure that the firewall and + network settings of the system running your TURN server are configured to + accept IPv6 traffic, and that the TURN server is listening on the local + IPv6 address that is mapped by NAT to the external IPv6 address. 1. (Re)start the turn server: @@ -216,9 +244,6 @@ connecting". Unfortunately, troubleshooting this can be tricky. Here are a few things to try: - * Check that your TURN server is not behind NAT. As above, we're not aware of - anyone who has successfully set this up. - * Check that you have opened your firewall to allow TCP and UDP traffic to the TURN ports (normally 3478 and 5349). @@ -234,6 +259,18 @@ Here are a few things to try: Try removing any AAAA records for your TURN server, so that it is only reachable over IPv4. + * If your TURN server is behind NAT: + + * double-check that your NAT gateway is correctly forwarding all TURN + ports (normally 3478 & 5349 for TCP & UDP TURN traffic, and 49152-65535 for the UDP + relay) to the NAT-internal address of your TURN server. If advertising + both IPv4 and IPv6 external addresses via the `external-ip` option, ensure + that the NAT is forwarding both IPv4 and IPv6 traffic to the IPv4 and IPv6 + internal addresses of your TURN server. When in doubt, remove AAAA records + for your TURN server and specify only an IPv4 address as your `external-ip`. + + * ensure that your TURN server uses the NAT gateway as its default route. + * Enable more verbose logging in coturn via the `verbose` setting: ``` From 66d7aa783a8ad1b3eb6d17ba086e2dda8a912ab6 Mon Sep 17 00:00:00 2001 From: Patrick Cloke Date: Thu, 23 Dec 2021 06:47:24 -0500 Subject: [PATCH 23/76] Fix mypy error with opentracing.tags. (#11622) --- changelog.d/11622.misc | 1 + synapse/logging/opentracing.py | 1 + 2 files changed, 2 insertions(+) create mode 100644 changelog.d/11622.misc diff --git a/changelog.d/11622.misc b/changelog.d/11622.misc new file mode 100644 index 0000000000..def24afb8d --- /dev/null +++ b/changelog.d/11622.misc @@ -0,0 +1 @@ +Add opentracing type stubs and fix associated mypy errors. \ No newline at end of file diff --git a/synapse/logging/opentracing.py b/synapse/logging/opentracing.py index 6364290615..622445e9f4 100644 --- a/synapse/logging/opentracing.py +++ b/synapse/logging/opentracing.py @@ -220,6 +220,7 @@ class _DummyTagNames: try: import opentracing + import opentracing.tags tags = opentracing.tags except ImportError: From fcfe67578f70bb4f150b83cfad708f5bc0474c1e Mon Sep 17 00:00:00 2001 From: Shay Date: Thu, 23 Dec 2021 20:22:15 -0800 Subject: [PATCH 24/76] Update to the current version of Black and run it on Synapse codebase (#11596) * update black version * run updated version of black on code * newsfragment * enumerate python versions --- changelog.d/11596.misc | 1 + pyproject.toml | 2 +- setup.py | 2 +- synapse/logging/context.py | 1 - synapse/util/caches/lrucache.py | 1 - 5 files changed, 3 insertions(+), 4 deletions(-) create mode 100644 changelog.d/11596.misc diff --git a/changelog.d/11596.misc b/changelog.d/11596.misc new file mode 100644 index 0000000000..3064bc632d --- /dev/null +++ b/changelog.d/11596.misc @@ -0,0 +1 @@ +Update black version and run it on all the files. \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml index 8bca1fa4ef..963f149c6a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -35,7 +35,7 @@ showcontent = true [tool.black] -target-version = ['py36'] +target-version = ['py37', 'py38', 'py39', 'py310'] exclude = ''' ( diff --git a/setup.py b/setup.py index e113da6782..fbb0133016 100755 --- a/setup.py +++ b/setup.py @@ -96,7 +96,7 @@ CONDITIONAL_REQUIREMENTS["all"] = list(ALL_OPTIONAL_REQUIREMENTS) # We pin black so that our tests don't start failing on new releases. CONDITIONAL_REQUIREMENTS["lint"] = [ "isort==5.7.0", - "black==21.6b0", + "black==21.12b0", "flake8-comprehensions", "flake8-bugbear==21.3.2", "flake8", diff --git a/synapse/logging/context.py b/synapse/logging/context.py index 25e78cc82f..d4ee893376 100644 --- a/synapse/logging/context.py +++ b/synapse/logging/context.py @@ -68,7 +68,6 @@ try: def get_thread_resource_usage() -> "Optional[resource.struct_rusage]": return resource.getrusage(RUSAGE_THREAD) - except Exception: # If the system doesn't support resource.getrusage(RUSAGE_THREAD) then we # won't track resource usage. diff --git a/synapse/util/caches/lrucache.py b/synapse/util/caches/lrucache.py index eb96f7e665..3f11a2f9dd 100644 --- a/synapse/util/caches/lrucache.py +++ b/synapse/util/caches/lrucache.py @@ -69,7 +69,6 @@ try: sizer.exclude_refs((), None, "") return sizer.asizeof(val, limit=100 if recurse else 0) - except ImportError: def _get_size_of(val: Any, *, recurse: bool = True) -> int: From 15bb1c8511c13197a75df93f6a8021ec5f9586e6 Mon Sep 17 00:00:00 2001 From: Dirk Klimpel <5740567+dklimpel@users.noreply.github.com> Date: Wed, 29 Dec 2021 14:01:13 +0100 Subject: [PATCH 25/76] Add type hints to `synapse/storage/databases/main/stats.py` (#11653) --- changelog.d/11653.misc | 1 + mypy.ini | 4 +- synapse/storage/databases/main/stats.py | 94 ++++++++++++++----------- 3 files changed, 57 insertions(+), 42 deletions(-) create mode 100644 changelog.d/11653.misc diff --git a/changelog.d/11653.misc b/changelog.d/11653.misc new file mode 100644 index 0000000000..8e405b9226 --- /dev/null +++ b/changelog.d/11653.misc @@ -0,0 +1 @@ +Add missing type hints to storage classes. \ No newline at end of file diff --git a/mypy.ini b/mypy.ini index 57e1a5df43..724c7e2ae4 100644 --- a/mypy.ini +++ b/mypy.ini @@ -39,7 +39,6 @@ exclude = (?x) |synapse/storage/databases/main/roommember.py |synapse/storage/databases/main/search.py |synapse/storage/databases/main/state.py - |synapse/storage/databases/main/stats.py |synapse/storage/databases/main/user_directory.py |synapse/storage/schema/ @@ -214,6 +213,9 @@ disallow_untyped_defs = True [mypy-synapse.storage.databases.main.profile] disallow_untyped_defs = True +[mypy-synapse.storage.databases.main.stats] +disallow_untyped_defs = True + [mypy-synapse.storage.databases.main.state_deltas] disallow_untyped_defs = True diff --git a/synapse/storage/databases/main/stats.py b/synapse/storage/databases/main/stats.py index a0472e37f5..427ae1f649 100644 --- a/synapse/storage/databases/main/stats.py +++ b/synapse/storage/databases/main/stats.py @@ -16,7 +16,7 @@ import logging from enum import Enum from itertools import chain -from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple +from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, cast from typing_extensions import Counter @@ -24,7 +24,11 @@ from twisted.internet.defer import DeferredLock from synapse.api.constants import EventContentFields, EventTypes, Membership from synapse.api.errors import StoreError -from synapse.storage.database import DatabasePool, LoggingDatabaseConnection +from synapse.storage.database import ( + DatabasePool, + LoggingDatabaseConnection, + LoggingTransaction, +) from synapse.storage.databases.main.state_deltas import StateDeltasStore from synapse.types import JsonDict from synapse.util.caches.descriptors import cached @@ -122,7 +126,9 @@ class StatsStore(StateDeltasStore): self.db_pool.updates.register_noop_background_update("populate_stats_cleanup") self.db_pool.updates.register_noop_background_update("populate_stats_prepare") - async def _populate_stats_process_users(self, progress, batch_size): + async def _populate_stats_process_users( + self, progress: JsonDict, batch_size: int + ) -> int: """ This is a background update which regenerates statistics for users. """ @@ -134,7 +140,7 @@ class StatsStore(StateDeltasStore): last_user_id = progress.get("last_user_id", "") - def _get_next_batch(txn): + def _get_next_batch(txn: LoggingTransaction) -> List[str]: sql = """ SELECT DISTINCT name FROM users WHERE name > ? @@ -168,7 +174,9 @@ class StatsStore(StateDeltasStore): return len(users_to_work_on) - async def _populate_stats_process_rooms(self, progress, batch_size): + async def _populate_stats_process_rooms( + self, progress: JsonDict, batch_size: int + ) -> int: """This is a background update which regenerates statistics for rooms.""" if not self.stats_enabled: await self.db_pool.updates._end_background_update( @@ -178,7 +186,7 @@ class StatsStore(StateDeltasStore): last_room_id = progress.get("last_room_id", "") - def _get_next_batch(txn): + def _get_next_batch(txn: LoggingTransaction) -> List[str]: sql = """ SELECT DISTINCT room_id FROM current_state_events WHERE room_id > ? @@ -307,7 +315,7 @@ class StatsStore(StateDeltasStore): stream_id: Current position. """ - def _bulk_update_stats_delta_txn(txn): + def _bulk_update_stats_delta_txn(txn: LoggingTransaction) -> None: for stats_type, stats_updates in updates.items(): for stats_id, fields in stats_updates.items(): logger.debug( @@ -339,7 +347,7 @@ class StatsStore(StateDeltasStore): stats_type: str, stats_id: str, fields: Dict[str, int], - complete_with_stream_id: Optional[int], + complete_with_stream_id: int, absolute_field_overrides: Optional[Dict[str, int]] = None, ) -> None: """ @@ -372,14 +380,14 @@ class StatsStore(StateDeltasStore): def _update_stats_delta_txn( self, - txn, - ts, - stats_type, - stats_id, - fields, - complete_with_stream_id, - absolute_field_overrides=None, - ): + txn: LoggingTransaction, + ts: int, + stats_type: str, + stats_id: str, + fields: Dict[str, int], + complete_with_stream_id: int, + absolute_field_overrides: Optional[Dict[str, int]] = None, + ) -> None: if absolute_field_overrides is None: absolute_field_overrides = {} @@ -422,20 +430,23 @@ class StatsStore(StateDeltasStore): ) def _upsert_with_additive_relatives_txn( - self, txn, table, keyvalues, absolutes, additive_relatives - ): + self, + txn: LoggingTransaction, + table: str, + keyvalues: Dict[str, Any], + absolutes: Dict[str, Any], + additive_relatives: Dict[str, int], + ) -> None: """Used to update values in the stats tables. This is basically a slightly convoluted upsert that *adds* to any existing rows. Args: - txn - table (str): Table name - keyvalues (dict[str, any]): Row-identifying key values - absolutes (dict[str, any]): Absolute (set) fields - additive_relatives (dict[str, int]): Fields that will be added onto - if existing row present. + table: Table name + keyvalues: Row-identifying key values + absolutes: Absolute (set) fields + additive_relatives: Fields that will be added onto if existing row present. """ if self.database_engine.can_native_upsert: absolute_updates = [ @@ -491,20 +502,17 @@ class StatsStore(StateDeltasStore): current_row.update(absolutes) self.db_pool.simple_update_one_txn(txn, table, keyvalues, current_row) - async def _calculate_and_set_initial_state_for_room( - self, room_id: str - ) -> Tuple[dict, dict, int]: + async def _calculate_and_set_initial_state_for_room(self, room_id: str) -> None: """Calculate and insert an entry into room_stats_current. Args: room_id: The room ID under calculation. - - Returns: - A tuple of room state, membership counts and stream position. """ - def _fetch_current_state_stats(txn): - pos = self.get_room_max_stream_ordering() + def _fetch_current_state_stats( + txn: LoggingTransaction, + ) -> Tuple[List[str], Dict[str, int], int, List[str], int]: + pos = self.get_room_max_stream_ordering() # type: ignore[attr-defined] rows = self.db_pool.simple_select_many_txn( txn, @@ -524,7 +532,7 @@ class StatsStore(StateDeltasStore): retcols=["event_id"], ) - event_ids = [row["event_id"] for row in rows] + event_ids = cast(List[str], [row["event_id"] for row in rows]) txn.execute( """ @@ -544,9 +552,9 @@ class StatsStore(StateDeltasStore): (room_id,), ) - (current_state_events_count,) = txn.fetchone() + current_state_events_count = cast(Tuple[int], txn.fetchone())[0] - users_in_room = self.get_users_in_room_txn(txn, room_id) + users_in_room = self.get_users_in_room_txn(txn, room_id) # type: ignore[attr-defined] return ( event_ids, @@ -566,7 +574,7 @@ class StatsStore(StateDeltasStore): "get_initial_state_for_room", _fetch_current_state_stats ) - state_event_map = await self.get_events(event_ids, get_prev_content=False) + state_event_map = await self.get_events(event_ids, get_prev_content=False) # type: ignore[attr-defined] room_state = { "join_rules": None, @@ -622,8 +630,10 @@ class StatsStore(StateDeltasStore): }, ) - async def _calculate_and_set_initial_state_for_user(self, user_id): - def _calculate_and_set_initial_state_for_user_txn(txn): + async def _calculate_and_set_initial_state_for_user(self, user_id: str) -> None: + def _calculate_and_set_initial_state_for_user_txn( + txn: LoggingTransaction, + ) -> Tuple[int, int]: pos = self._get_max_stream_id_in_current_state_deltas_txn(txn) txn.execute( @@ -634,7 +644,7 @@ class StatsStore(StateDeltasStore): """, (user_id,), ) - (count,) = txn.fetchone() + count = cast(Tuple[int], txn.fetchone())[0] return count, pos joined_rooms, pos = await self.db_pool.runInteraction( @@ -678,7 +688,9 @@ class StatsStore(StateDeltasStore): users that exist given this query """ - def get_users_media_usage_paginate_txn(txn): + def get_users_media_usage_paginate_txn( + txn: LoggingTransaction, + ) -> Tuple[List[JsonDict], int]: filters = [] args = [self.hs.config.server.server_name] @@ -733,7 +745,7 @@ class StatsStore(StateDeltasStore): sql_base=sql_base, ) txn.execute(sql, args) - count = txn.fetchone()[0] + count = cast(Tuple[int], txn.fetchone())[0] sql = """ SELECT From f58b300d271ddaad7ce81687547f3c6ed7e7187a Mon Sep 17 00:00:00 2001 From: Patrick Cloke Date: Wed, 29 Dec 2021 08:02:03 -0500 Subject: [PATCH 26/76] Do not attempt to bundled aggregations for /members and /state. (#11623) Both of those APIs return state events, which will not have bundled aggregations added anyway. --- changelog.d/11623.bugfix | 1 + synapse/handlers/message.py | 4 +--- 2 files changed, 2 insertions(+), 3 deletions(-) create mode 100644 changelog.d/11623.bugfix diff --git a/changelog.d/11623.bugfix b/changelog.d/11623.bugfix new file mode 100644 index 0000000000..4116e5dd7c --- /dev/null +++ b/changelog.d/11623.bugfix @@ -0,0 +1 @@ +Fix a long-standing bug where responses included bundled aggregations when they should not, per [MSC2675](https://github.com/matrix-org/matrix-doc/pull/2675). diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py index 1a7190085a..5e3d3886eb 100644 --- a/synapse/handlers/message.py +++ b/synapse/handlers/message.py @@ -246,9 +246,7 @@ class MessageHandler: room_state = room_state_events[membership_event_id] now = self.clock.time_msec() - events = await self._event_serializer.serialize_events( - room_state.values(), now, bundle_aggregations=True - ) + events = await self._event_serializer.serialize_events(room_state.values(), now) return events async def get_joined_members(self, requester: Requester, room_id: str) -> dict: From f82d38ed2e8e07c81a0d60f31401e3edecd5e57f Mon Sep 17 00:00:00 2001 From: Dirk Klimpel <5740567+dklimpel@users.noreply.github.com> Date: Wed, 29 Dec 2021 14:04:28 +0100 Subject: [PATCH 27/76] Improve type hints in storage classes. (#11652) By using cast and making ignores more specific. --- changelog.d/11652.misc | 1 + synapse/storage/databases/main/deviceinbox.py | 4 ++-- .../storage/databases/main/event_federation.py | 6 +++--- .../databases/main/event_push_actions.py | 18 ++++++++++-------- synapse/storage/databases/main/filtering.py | 4 ++-- .../storage/databases/main/media_repository.py | 3 ++- synapse/storage/databases/main/pusher.py | 6 +++--- synapse/storage/databases/main/registration.py | 17 ++++++++++------- synapse/storage/databases/main/relations.py | 4 ++-- synapse/storage/databases/main/ui_auth.py | 15 +++++++++------ 10 files changed, 44 insertions(+), 34 deletions(-) create mode 100644 changelog.d/11652.misc diff --git a/changelog.d/11652.misc b/changelog.d/11652.misc new file mode 100644 index 0000000000..8e405b9226 --- /dev/null +++ b/changelog.d/11652.misc @@ -0,0 +1 @@ +Add missing type hints to storage classes. \ No newline at end of file diff --git a/synapse/storage/databases/main/deviceinbox.py b/synapse/storage/databases/main/deviceinbox.py index b410eefdc7..3682cb6a81 100644 --- a/synapse/storage/databases/main/deviceinbox.py +++ b/synapse/storage/databases/main/deviceinbox.py @@ -14,7 +14,7 @@ # limitations under the License. import logging -from typing import TYPE_CHECKING, List, Optional, Tuple +from typing import TYPE_CHECKING, List, Optional, Tuple, cast from synapse.logging import issue9533_logger from synapse.logging.opentracing import log_kv, set_tag, trace @@ -673,7 +673,7 @@ class DeviceInboxBackgroundUpdateStore(SQLBaseStore): # There's a type mismatch here between how we want to type the row and # what fetchone says it returns, but we silence it because we know that # res can't be None. - res: Tuple[Optional[int]] = txn.fetchone() # type: ignore[assignment] + res = cast(Tuple[Optional[int]], txn.fetchone()) if res[0] is None: # this can only happen if the `device_inbox` table is empty, in which # case we have no work to do. diff --git a/synapse/storage/databases/main/event_federation.py b/synapse/storage/databases/main/event_federation.py index bc5ff25d08..270b30800b 100644 --- a/synapse/storage/databases/main/event_federation.py +++ b/synapse/storage/databases/main/event_federation.py @@ -288,7 +288,7 @@ class EventFederationWorkerStore(EventsWorkerStore, SignatureWorkerStore, SQLBas new_front = set() for chunk in batch_iter(front, 100): # Pull the auth events either from the cache or DB. - to_fetch = [] # Event IDs to fetch from DB # type: List[str] + to_fetch: List[str] = [] # Event IDs to fetch from DB for event_id in chunk: res = self._event_auth_cache.get(event_id) if res is None: @@ -615,8 +615,8 @@ class EventFederationWorkerStore(EventsWorkerStore, SignatureWorkerStore, SQLBas # currently walking, either from cache or DB. search, chunk = search[:-100], search[-100:] - found = [] # Results found # type: List[Tuple[str, str, int]] - to_fetch = [] # Event IDs to fetch from DB # type: List[str] + found: List[Tuple[str, str, int]] = [] # Results found + to_fetch: List[str] = [] # Event IDs to fetch from DB for _, event_id in chunk: res = self._event_auth_cache.get(event_id) if res is None: diff --git a/synapse/storage/databases/main/event_push_actions.py b/synapse/storage/databases/main/event_push_actions.py index 98ea0e884c..a98e6b2593 100644 --- a/synapse/storage/databases/main/event_push_actions.py +++ b/synapse/storage/databases/main/event_push_actions.py @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. import logging -from typing import TYPE_CHECKING, Dict, List, Optional, Tuple, Union +from typing import TYPE_CHECKING, Dict, List, Optional, Tuple, Union, cast import attr @@ -326,7 +326,7 @@ class EventPushActionsWorkerStore(SQLBaseStore): ) args = [user_id, user_id, min_stream_ordering, max_stream_ordering, limit] txn.execute(sql, args) - return txn.fetchall() # type: ignore[return-value] + return cast(List[Tuple[str, str, int, str, bool]], txn.fetchall()) after_read_receipt = await self.db_pool.runInteraction( "get_unread_push_actions_for_user_in_range_http_arr", get_after_receipt @@ -357,7 +357,7 @@ class EventPushActionsWorkerStore(SQLBaseStore): ) args = [user_id, user_id, min_stream_ordering, max_stream_ordering, limit] txn.execute(sql, args) - return txn.fetchall() # type: ignore[return-value] + return cast(List[Tuple[str, str, int, str, bool]], txn.fetchall()) no_read_receipt = await self.db_pool.runInteraction( "get_unread_push_actions_for_user_in_range_http_nrr", get_no_receipt @@ -434,7 +434,7 @@ class EventPushActionsWorkerStore(SQLBaseStore): ) args = [user_id, user_id, min_stream_ordering, max_stream_ordering, limit] txn.execute(sql, args) - return txn.fetchall() # type: ignore[return-value] + return cast(List[Tuple[str, str, int, str, bool, int]], txn.fetchall()) after_read_receipt = await self.db_pool.runInteraction( "get_unread_push_actions_for_user_in_range_email_arr", get_after_receipt @@ -465,7 +465,7 @@ class EventPushActionsWorkerStore(SQLBaseStore): ) args = [user_id, user_id, min_stream_ordering, max_stream_ordering, limit] txn.execute(sql, args) - return txn.fetchall() # type: ignore[return-value] + return cast(List[Tuple[str, str, int, str, bool, int]], txn.fetchall()) no_read_receipt = await self.db_pool.runInteraction( "get_unread_push_actions_for_user_in_range_email_nrr", get_no_receipt @@ -662,7 +662,7 @@ class EventPushActionsWorkerStore(SQLBaseStore): The stream ordering """ txn.execute("SELECT MAX(stream_ordering) FROM events") - max_stream_ordering = txn.fetchone()[0] # type: ignore[index] + max_stream_ordering = cast(Tuple[Optional[int]], txn.fetchone())[0] if max_stream_ordering is None: return 0 @@ -731,7 +731,7 @@ class EventPushActionsWorkerStore(SQLBaseStore): " LIMIT 1" ) txn.execute(sql, (stream_ordering,)) - return txn.fetchone() # type: ignore[return-value] + return cast(Optional[Tuple[int]], txn.fetchone()) result = await self.db_pool.runInteraction( "get_time_of_last_push_action_before", f @@ -1029,7 +1029,9 @@ class EventPushActionsStore(EventPushActionsWorkerStore): " LIMIT ?" % (before_clause,) ) txn.execute(sql, args) - return txn.fetchall() # type: ignore[return-value] + return cast( + List[Tuple[str, str, int, int, str, bool, str, int]], txn.fetchall() + ) push_actions = await self.db_pool.runInteraction("get_push_actions_for_user", f) return [ diff --git a/synapse/storage/databases/main/filtering.py b/synapse/storage/databases/main/filtering.py index cf842803bc..cb9ee08fa8 100644 --- a/synapse/storage/databases/main/filtering.py +++ b/synapse/storage/databases/main/filtering.py @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -from typing import Union +from typing import Optional, Tuple, Union, cast from canonicaljson import encode_canonical_json @@ -63,7 +63,7 @@ class FilteringStore(SQLBaseStore): sql = "SELECT MAX(filter_id) FROM user_filters WHERE user_id = ?" txn.execute(sql, (user_localpart,)) - max_id = txn.fetchone()[0] # type: ignore[index] + max_id = cast(Tuple[Optional[int]], txn.fetchone())[0] if max_id is None: filter_id = 0 else: diff --git a/synapse/storage/databases/main/media_repository.py b/synapse/storage/databases/main/media_repository.py index 1b076683f7..cbba356b4a 100644 --- a/synapse/storage/databases/main/media_repository.py +++ b/synapse/storage/databases/main/media_repository.py @@ -23,6 +23,7 @@ from typing import ( Optional, Tuple, Union, + cast, ) from synapse.storage._base import SQLBaseStore @@ -220,7 +221,7 @@ class MediaRepositoryStore(MediaRepositoryBackgroundUpdateStore): WHERE user_id = ? """ txn.execute(sql, args) - count = txn.fetchone()[0] # type: ignore[index] + count = cast(Tuple[int], txn.fetchone())[0] sql = """ SELECT diff --git a/synapse/storage/databases/main/pusher.py b/synapse/storage/databases/main/pusher.py index 7ab681ed6f..747b4f31df 100644 --- a/synapse/storage/databases/main/pusher.py +++ b/synapse/storage/databases/main/pusher.py @@ -494,7 +494,7 @@ class PusherStore(PusherWorkerStore): # invalidate, since we the user might not have had a pusher before await self.db_pool.runInteraction( "add_pusher", - self._invalidate_cache_and_stream, # type: ignore + self._invalidate_cache_and_stream, # type: ignore[attr-defined] self.get_if_user_has_pusher, (user_id,), ) @@ -503,7 +503,7 @@ class PusherStore(PusherWorkerStore): self, app_id: str, pushkey: str, user_id: str ) -> None: def delete_pusher_txn(txn, stream_id): - self._invalidate_cache_and_stream( # type: ignore + self._invalidate_cache_and_stream( # type: ignore[attr-defined] txn, self.get_if_user_has_pusher, (user_id,) ) @@ -548,7 +548,7 @@ class PusherStore(PusherWorkerStore): pushers = list(await self.get_pushers_by_user_id(user_id)) def delete_pushers_txn(txn, stream_ids): - self._invalidate_cache_and_stream( # type: ignore + self._invalidate_cache_and_stream( # type: ignore[attr-defined] txn, self.get_if_user_has_pusher, (user_id,) ) diff --git a/synapse/storage/databases/main/registration.py b/synapse/storage/databases/main/registration.py index 29d9d4de96..4175c82a25 100644 --- a/synapse/storage/databases/main/registration.py +++ b/synapse/storage/databases/main/registration.py @@ -16,7 +16,7 @@ import logging import random import re -from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union +from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union, cast import attr @@ -1357,12 +1357,15 @@ class RegistrationWorkerStore(CacheInvalidationWorkerStore): # Override type because the return type is only optional if # allow_none is True, and we don't want mypy throwing errors # about None not being indexable. - res: Dict[str, Any] = self.db_pool.simple_select_one_txn( - txn, - "registration_tokens", - keyvalues={"token": token}, - retcols=["pending", "completed"], - ) # type: ignore + res = cast( + Dict[str, Any], + self.db_pool.simple_select_one_txn( + txn, + "registration_tokens", + keyvalues={"token": token}, + retcols=["pending", "completed"], + ), + ) # Decrement pending and increment completed self.db_pool.simple_update_one_txn( diff --git a/synapse/storage/databases/main/relations.py b/synapse/storage/databases/main/relations.py index 729ff17e2e..4ff6aed253 100644 --- a/synapse/storage/databases/main/relations.py +++ b/synapse/storage/databases/main/relations.py @@ -13,7 +13,7 @@ # limitations under the License. import logging -from typing import List, Optional, Tuple, Union +from typing import List, Optional, Tuple, Union, cast import attr @@ -399,7 +399,7 @@ class RelationsWorkerStore(SQLBaseStore): AND relation_type = ? """ txn.execute(sql, (event_id, room_id, RelationTypes.THREAD)) - count = txn.fetchone()[0] # type: ignore[index] + count = cast(Tuple[int], txn.fetchone())[0] return count, latest_event_id diff --git a/synapse/storage/databases/main/ui_auth.py b/synapse/storage/databases/main/ui_auth.py index 340ca9e47d..a1a1a6a14a 100644 --- a/synapse/storage/databases/main/ui_auth.py +++ b/synapse/storage/databases/main/ui_auth.py @@ -11,7 +11,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from typing import Any, Dict, List, Optional, Tuple, Union +from typing import Any, Dict, List, Optional, Tuple, Union, cast import attr @@ -225,11 +225,14 @@ class UIAuthWorkerStore(SQLBaseStore): self, txn: LoggingTransaction, session_id: str, key: str, value: Any ): # Get the current value. - result: Dict[str, Any] = self.db_pool.simple_select_one_txn( # type: ignore - txn, - table="ui_auth_sessions", - keyvalues={"session_id": session_id}, - retcols=("serverdict",), + result = cast( + Dict[str, Any], + self.db_pool.simple_select_one_txn( + txn, + table="ui_auth_sessions", + keyvalues={"session_id": session_id}, + retcols=("serverdict",), + ), ) # Update it and add it back to the database. From 2c7f5e74e5ae5eb7e5d0b99001610a829b0b35da Mon Sep 17 00:00:00 2001 From: reivilibre Date: Wed, 29 Dec 2021 15:12:30 +0000 Subject: [PATCH 28/76] Fix a type annotation in `test_account_data.py` and remove it from the Mypy exclusion list. (#11657) Co-authored-by: Patrick Cloke --- changelog.d/11657.misc | 1 + mypy.ini | 1 - tests/storage/test_account_data.py | 4 ++-- 3 files changed, 3 insertions(+), 3 deletions(-) create mode 100644 changelog.d/11657.misc diff --git a/changelog.d/11657.misc b/changelog.d/11657.misc new file mode 100644 index 0000000000..8e405b9226 --- /dev/null +++ b/changelog.d/11657.misc @@ -0,0 +1 @@ +Add missing type hints to storage classes. \ No newline at end of file diff --git a/mypy.ini b/mypy.ini index 724c7e2ae4..e0204a3c04 100644 --- a/mypy.ini +++ b/mypy.ini @@ -100,7 +100,6 @@ exclude = (?x) |tests/server.py |tests/server_notices/test_resource_limits_server_notices.py |tests/state/test_v2.py - |tests/storage/test_account_data.py |tests/storage/test_background_update.py |tests/storage/test_base.py |tests/storage/test_client_ips.py diff --git a/tests/storage/test_account_data.py b/tests/storage/test_account_data.py index 01af49a16b..d697d2bc1e 100644 --- a/tests/storage/test_account_data.py +++ b/tests/storage/test_account_data.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -from typing import Iterable, Set +from typing import Iterable, Optional, Set from synapse.api.constants import AccountDataTypes @@ -25,7 +25,7 @@ class IgnoredUsersTestCase(unittest.HomeserverTestCase): self.user = "@user:test" def _update_ignore_list( - self, *ignored_user_ids: Iterable[str], ignorer_user_id: str = None + self, *ignored_user_ids: Iterable[str], ignorer_user_id: Optional[str] = None ) -> None: """Update the account data to block the given users.""" if ignorer_user_id is None: From 07a3b5dabad98678bae52275b327c302adbf67af Mon Sep 17 00:00:00 2001 From: Dirk Klimpel <5740567+dklimpel@users.noreply.github.com> Date: Thu, 30 Dec 2021 13:22:31 +0100 Subject: [PATCH 29/76] Add type hints to `synapse/storage/databases/main/events_bg_updates.py` (#11654) --- changelog.d/11654.misc | 1 + mypy.ini | 4 +- .../databases/main/events_bg_updates.py | 69 +++++++++++-------- 3 files changed, 44 insertions(+), 30 deletions(-) create mode 100644 changelog.d/11654.misc diff --git a/changelog.d/11654.misc b/changelog.d/11654.misc new file mode 100644 index 0000000000..8e405b9226 --- /dev/null +++ b/changelog.d/11654.misc @@ -0,0 +1 @@ +Add missing type hints to storage classes. \ No newline at end of file diff --git a/mypy.ini b/mypy.ini index e0204a3c04..85fa22d28f 100644 --- a/mypy.ini +++ b/mypy.ini @@ -28,7 +28,6 @@ exclude = (?x) |synapse/storage/databases/main/cache.py |synapse/storage/databases/main/devices.py |synapse/storage/databases/main/event_federation.py - |synapse/storage/databases/main/events_bg_updates.py |synapse/storage/databases/main/group_server.py |synapse/storage/databases/main/metrics.py |synapse/storage/databases/main/monthly_active_users.py @@ -200,6 +199,9 @@ disallow_untyped_defs = True [mypy-synapse.storage.databases.main.event_push_actions] disallow_untyped_defs = True +[mypy-synapse.storage.databases.main.events_bg_updates] +disallow_untyped_defs = True + [mypy-synapse.storage.databases.main.events_worker] disallow_untyped_defs = True diff --git a/synapse/storage/databases/main/events_bg_updates.py b/synapse/storage/databases/main/events_bg_updates.py index 9b36941fec..a68f14ba48 100644 --- a/synapse/storage/databases/main/events_bg_updates.py +++ b/synapse/storage/databases/main/events_bg_updates.py @@ -13,7 +13,7 @@ # limitations under the License. import logging -from typing import TYPE_CHECKING, Dict, List, Optional, Tuple +from typing import TYPE_CHECKING, Dict, List, Optional, Sequence, Set, Tuple, cast import attr @@ -240,12 +240,14 @@ class EventsBackgroundUpdatesStore(SQLBaseStore): ################################################################################ - async def _background_reindex_fields_sender(self, progress, batch_size): + async def _background_reindex_fields_sender( + self, progress: JsonDict, batch_size: int + ) -> int: target_min_stream_id = progress["target_min_stream_id_inclusive"] max_stream_id = progress["max_stream_id_exclusive"] rows_inserted = progress.get("rows_inserted", 0) - def reindex_txn(txn): + def reindex_txn(txn: LoggingTransaction) -> int: sql = ( "SELECT stream_ordering, event_id, json FROM events" " INNER JOIN event_json USING (event_id)" @@ -307,12 +309,14 @@ class EventsBackgroundUpdatesStore(SQLBaseStore): return result - async def _background_reindex_origin_server_ts(self, progress, batch_size): + async def _background_reindex_origin_server_ts( + self, progress: JsonDict, batch_size: int + ) -> int: target_min_stream_id = progress["target_min_stream_id_inclusive"] max_stream_id = progress["max_stream_id_exclusive"] rows_inserted = progress.get("rows_inserted", 0) - def reindex_search_txn(txn): + def reindex_search_txn(txn: LoggingTransaction) -> int: sql = ( "SELECT stream_ordering, event_id FROM events" " WHERE ? <= stream_ordering AND stream_ordering < ?" @@ -381,7 +385,9 @@ class EventsBackgroundUpdatesStore(SQLBaseStore): return result - async def _cleanup_extremities_bg_update(self, progress, batch_size): + async def _cleanup_extremities_bg_update( + self, progress: JsonDict, batch_size: int + ) -> int: """Background update to clean out extremities that should have been deleted previously. @@ -402,12 +408,12 @@ class EventsBackgroundUpdatesStore(SQLBaseStore): # have any descendants, but if they do then we should delete those # extremities. - def _cleanup_extremities_bg_update_txn(txn): + def _cleanup_extremities_bg_update_txn(txn: LoggingTransaction) -> int: # The set of extremity event IDs that we're checking this round original_set = set() - # A dict[str, set[str]] of event ID to their prev events. - graph = {} + # A dict[str, Set[str]] of event ID to their prev events. + graph: Dict[str, Set[str]] = {} # The set of descendants of the original set that are not rejected # nor soft-failed. Ancestors of these events should be removed @@ -536,7 +542,7 @@ class EventsBackgroundUpdatesStore(SQLBaseStore): room_ids = {row["room_id"] for row in rows} for room_id in room_ids: txn.call_after( - self.get_latest_event_ids_in_room.invalidate, (room_id,) + self.get_latest_event_ids_in_room.invalidate, (room_id,) # type: ignore[attr-defined] ) self.db_pool.simple_delete_many_txn( @@ -558,7 +564,7 @@ class EventsBackgroundUpdatesStore(SQLBaseStore): _BackgroundUpdates.DELETE_SOFT_FAILED_EXTREMITIES ) - def _drop_table_txn(txn): + def _drop_table_txn(txn: LoggingTransaction) -> None: txn.execute("DROP TABLE _extremities_to_check") await self.db_pool.runInteraction( @@ -567,11 +573,11 @@ class EventsBackgroundUpdatesStore(SQLBaseStore): return num_handled - async def _redactions_received_ts(self, progress, batch_size): + async def _redactions_received_ts(self, progress: JsonDict, batch_size: int) -> int: """Handles filling out the `received_ts` column in redactions.""" last_event_id = progress.get("last_event_id", "") - def _redactions_received_ts_txn(txn): + def _redactions_received_ts_txn(txn: LoggingTransaction) -> int: # Fetch the set of event IDs that we want to update sql = """ SELECT event_id FROM redactions @@ -622,10 +628,12 @@ class EventsBackgroundUpdatesStore(SQLBaseStore): return count - async def _event_fix_redactions_bytes(self, progress, batch_size): + async def _event_fix_redactions_bytes( + self, progress: JsonDict, batch_size: int + ) -> int: """Undoes hex encoded censored redacted event JSON.""" - def _event_fix_redactions_bytes_txn(txn): + def _event_fix_redactions_bytes_txn(txn: LoggingTransaction) -> None: # This update is quite fast due to new index. txn.execute( """ @@ -650,11 +658,11 @@ class EventsBackgroundUpdatesStore(SQLBaseStore): return 1 - async def _event_store_labels(self, progress, batch_size): + async def _event_store_labels(self, progress: JsonDict, batch_size: int) -> int: """Background update handler which will store labels for existing events.""" last_event_id = progress.get("last_event_id", "") - def _event_store_labels_txn(txn): + def _event_store_labels_txn(txn: LoggingTransaction) -> int: txn.execute( """ SELECT event_id, json FROM event_json @@ -754,7 +762,10 @@ class EventsBackgroundUpdatesStore(SQLBaseStore): ), ) - return [(row[0], row[1], db_to_json(row[2]), row[3], row[4]) for row in txn] # type: ignore + return cast( + List[Tuple[str, str, JsonDict, bool, bool]], + [(row[0], row[1], db_to_json(row[2]), row[3], row[4]) for row in txn], + ) results = await self.db_pool.runInteraction( desc="_rejected_events_metadata_get", func=get_rejected_events @@ -912,7 +923,7 @@ class EventsBackgroundUpdatesStore(SQLBaseStore): def _calculate_chain_cover_txn( self, - txn: Cursor, + txn: LoggingTransaction, last_room_id: str, last_depth: int, last_stream: int, @@ -1023,10 +1034,10 @@ class EventsBackgroundUpdatesStore(SQLBaseStore): PersistEventsStore._add_chain_cover_index( txn, self.db_pool, - self.event_chain_id_gen, + self.event_chain_id_gen, # type: ignore[attr-defined] event_to_room_id, event_to_types, - event_to_auth_chain, + cast(Dict[str, Sequence[str]], event_to_auth_chain), ) return _CalculateChainCover( @@ -1046,7 +1057,7 @@ class EventsBackgroundUpdatesStore(SQLBaseStore): """ current_event_id = progress.get("current_event_id", "") - def purged_chain_cover_txn(txn) -> int: + def purged_chain_cover_txn(txn: LoggingTransaction) -> int: # The event ID from events will be null if the chain ID / sequence # number points to a purged event. sql = """ @@ -1181,14 +1192,14 @@ class EventsBackgroundUpdatesStore(SQLBaseStore): # Iterate the parent IDs and invalidate caches. for parent_id in {r[1] for r in relations_to_insert}: cache_tuple = (parent_id,) - self._invalidate_cache_and_stream( - txn, self.get_relations_for_event, cache_tuple + self._invalidate_cache_and_stream( # type: ignore[attr-defined] + txn, self.get_relations_for_event, cache_tuple # type: ignore[attr-defined] ) - self._invalidate_cache_and_stream( - txn, self.get_aggregation_groups_for_event, cache_tuple + self._invalidate_cache_and_stream( # type: ignore[attr-defined] + txn, self.get_aggregation_groups_for_event, cache_tuple # type: ignore[attr-defined] ) - self._invalidate_cache_and_stream( - txn, self.get_thread_summary, cache_tuple + self._invalidate_cache_and_stream( # type: ignore[attr-defined] + txn, self.get_thread_summary, cache_tuple # type: ignore[attr-defined] ) if results: @@ -1220,7 +1231,7 @@ class EventsBackgroundUpdatesStore(SQLBaseStore): """ batch_size = max(batch_size, 1) - def process(txn: Cursor) -> int: + def process(txn: LoggingTransaction) -> int: last_stream = progress.get("last_stream", -(1 << 31)) txn.execute( """ From cbd82d0b2db069400b5d43373838817d8a0209e7 Mon Sep 17 00:00:00 2001 From: Patrick Cloke Date: Thu, 30 Dec 2021 13:47:12 -0500 Subject: [PATCH 30/76] Convert all namedtuples to attrs. (#11665) To improve type hints throughout the code. --- changelog.d/11665.misc | 1 + synapse/api/filtering.py | 3 +- synapse/config/repository.py | 34 ++--- synapse/federation/federation_base.py | 5 - synapse/federation/send_queue.py | 47 ++++--- synapse/handlers/appservice.py | 4 +- synapse/handlers/directory.py | 10 +- synapse/handlers/room_list.py | 22 ++- synapse/handlers/typing.py | 14 +- synapse/http/server.py | 10 +- synapse/replication/tcp/streams/_base.py | 129 +++++++++--------- synapse/replication/tcp/streams/federation.py | 15 +- synapse/rest/media/v1/media_repository.py | 19 ++- synapse/state/__init__.py | 5 +- synapse/storage/databases/main/directory.py | 10 +- synapse/storage/databases/main/events.py | 13 +- synapse/storage/databases/main/room.py | 26 +++- synapse/storage/databases/main/search.py | 16 ++- synapse/storage/databases/main/state.py | 14 -- synapse/storage/databases/main/stream.py | 12 +- synapse/types.py | 22 ++- tests/replication/test_federation_ack.py | 6 +- 22 files changed, 231 insertions(+), 206 deletions(-) create mode 100644 changelog.d/11665.misc diff --git a/changelog.d/11665.misc b/changelog.d/11665.misc new file mode 100644 index 0000000000..e7cc8ff23f --- /dev/null +++ b/changelog.d/11665.misc @@ -0,0 +1 @@ +Convert `namedtuples` to `attrs`. diff --git a/synapse/api/filtering.py b/synapse/api/filtering.py index 13dd6ce248..d087c816db 100644 --- a/synapse/api/filtering.py +++ b/synapse/api/filtering.py @@ -351,8 +351,7 @@ class Filter: True if the event matches the filter. """ # We usually get the full "events" as dictionaries coming through, - # except for presence which actually gets passed around as its own - # namedtuple type. + # except for presence which actually gets passed around as its own type. if isinstance(event, UserPresenceState): user_id = event.user_id field_matchers = { diff --git a/synapse/config/repository.py b/synapse/config/repository.py index b129b9dd68..1980351e77 100644 --- a/synapse/config/repository.py +++ b/synapse/config/repository.py @@ -14,10 +14,11 @@ import logging import os -from collections import namedtuple from typing import Dict, List, Tuple from urllib.request import getproxies_environment # type: ignore +import attr + from synapse.config.server import DEFAULT_IP_RANGE_BLACKLIST, generate_ip_set from synapse.python_dependencies import DependencyException, check_requirements from synapse.types import JsonDict @@ -44,18 +45,20 @@ THUMBNAIL_SIZE_YAML = """\ HTTP_PROXY_SET_WARNING = """\ The Synapse config url_preview_ip_range_blacklist will be ignored as an HTTP(s) proxy is configured.""" -ThumbnailRequirement = namedtuple( - "ThumbnailRequirement", ["width", "height", "method", "media_type"] -) -MediaStorageProviderConfig = namedtuple( - "MediaStorageProviderConfig", - ( - "store_local", # Whether to store newly uploaded local files - "store_remote", # Whether to store newly downloaded remote files - "store_synchronous", # Whether to wait for successful storage for local uploads - ), -) +@attr.s(frozen=True, slots=True, auto_attribs=True) +class ThumbnailRequirement: + width: int + height: int + method: str + media_type: str + + +@attr.s(frozen=True, slots=True, auto_attribs=True) +class MediaStorageProviderConfig: + store_local: bool # Whether to store newly uploaded local files + store_remote: bool # Whether to store newly downloaded remote files + store_synchronous: bool # Whether to wait for successful storage for local uploads def parse_thumbnail_requirements( @@ -66,11 +69,10 @@ def parse_thumbnail_requirements( method, and thumbnail media type to precalculate Args: - thumbnail_sizes(list): List of dicts with "width", "height", and - "method" keys + thumbnail_sizes: List of dicts with "width", "height", and "method" keys + Returns: - Dictionary mapping from media type string to list of - ThumbnailRequirement tuples. + Dictionary mapping from media type string to list of ThumbnailRequirement. """ requirements: Dict[str, List[ThumbnailRequirement]] = {} for size in thumbnail_sizes: diff --git a/synapse/federation/federation_base.py b/synapse/federation/federation_base.py index f56344a3b9..4df90e02d7 100644 --- a/synapse/federation/federation_base.py +++ b/synapse/federation/federation_base.py @@ -13,7 +13,6 @@ # See the License for the specific language governing permissions and # limitations under the License. import logging -from collections import namedtuple from typing import TYPE_CHECKING from synapse.api.constants import MAX_DEPTH, EventContentFields, EventTypes, Membership @@ -104,10 +103,6 @@ class FederationBase: return pdu -class PduToCheckSig(namedtuple("PduToCheckSig", ["pdu", "sender_domain", "deferreds"])): - pass - - async def _check_sigs_on_pdu( keyring: Keyring, room_version: RoomVersion, pdu: EventBase ) -> None: diff --git a/synapse/federation/send_queue.py b/synapse/federation/send_queue.py index 63289a5a33..0d7c4f5067 100644 --- a/synapse/federation/send_queue.py +++ b/synapse/federation/send_queue.py @@ -30,7 +30,6 @@ Events are replicated via a separate events stream. """ import logging -from collections import namedtuple from typing import ( TYPE_CHECKING, Dict, @@ -43,6 +42,7 @@ from typing import ( Type, ) +import attr from sortedcontainers import SortedDict from synapse.api.presence import UserPresenceState @@ -382,13 +382,11 @@ class BaseFederationRow: raise NotImplementedError() -class PresenceDestinationsRow( - BaseFederationRow, - namedtuple( - "PresenceDestinationsRow", - ("state", "destinations"), # UserPresenceState # list[str] - ), -): +@attr.s(slots=True, frozen=True, auto_attribs=True) +class PresenceDestinationsRow(BaseFederationRow): + state: UserPresenceState + destinations: List[str] + TypeId = "pd" @staticmethod @@ -404,17 +402,15 @@ class PresenceDestinationsRow( buff.presence_destinations.append((self.state, self.destinations)) -class KeyedEduRow( - BaseFederationRow, - namedtuple( - "KeyedEduRow", - ("key", "edu"), # tuple(str) - the edu key passed to send_edu # Edu - ), -): +@attr.s(slots=True, frozen=True, auto_attribs=True) +class KeyedEduRow(BaseFederationRow): """Streams EDUs that have an associated key that is ued to clobber. For example, typing EDUs clobber based on room_id. """ + key: Tuple[str, ...] # the edu key passed to send_edu + edu: Edu + TypeId = "k" @staticmethod @@ -428,9 +424,12 @@ class KeyedEduRow( buff.keyed_edus.setdefault(self.edu.destination, {})[self.key] = self.edu -class EduRow(BaseFederationRow, namedtuple("EduRow", ("edu",))): # Edu +@attr.s(slots=True, frozen=True, auto_attribs=True) +class EduRow(BaseFederationRow): """Streams EDUs that don't have keys. See KeyedEduRow""" + edu: Edu + TypeId = "e" @staticmethod @@ -453,14 +452,14 @@ _rowtypes: Tuple[Type[BaseFederationRow], ...] = ( TypeToRow = {Row.TypeId: Row for Row in _rowtypes} -ParsedFederationStreamData = namedtuple( - "ParsedFederationStreamData", - ( - "presence_destinations", # list of tuples of UserPresenceState and destinations - "keyed_edus", # dict of destination -> { key -> Edu } - "edus", # dict of destination -> [Edu] - ), -) +@attr.s(slots=True, frozen=True, auto_attribs=True) +class ParsedFederationStreamData: + # list of tuples of UserPresenceState and destinations + presence_destinations: List[Tuple[UserPresenceState, List[str]]] + # dict of destination -> { key -> Edu } + keyed_edus: Dict[str, Dict[Tuple[str, ...], Edu]] + # dict of destination -> [Edu] + edus: Dict[str, List[Edu]] def process_rows_for_federation( diff --git a/synapse/handlers/appservice.py b/synapse/handlers/appservice.py index 9abdad262b..7833e77e2b 100644 --- a/synapse/handlers/appservice.py +++ b/synapse/handlers/appservice.py @@ -462,9 +462,9 @@ class ApplicationServicesHandler: Args: room_alias: The room alias to query. + Returns: - namedtuple: with keys "room_id" and "servers" or None if no - association can be found. + RoomAliasMapping or None if no association can be found. """ room_alias_str = room_alias.to_string() services = self.store.get_app_services() diff --git a/synapse/handlers/directory.py b/synapse/handlers/directory.py index 7ee5c47fd9..082f521791 100644 --- a/synapse/handlers/directory.py +++ b/synapse/handlers/directory.py @@ -278,13 +278,15 @@ class DirectoryHandler: users = await self.store.get_users_in_room(room_id) extra_servers = {get_domain_from_id(u) for u in users} - servers = set(extra_servers) | set(servers) + servers_set = set(extra_servers) | set(servers) # If this server is in the list of servers, return it first. - if self.server_name in servers: - servers = [self.server_name] + [s for s in servers if s != self.server_name] + if self.server_name in servers_set: + servers = [self.server_name] + [ + s for s in servers_set if s != self.server_name + ] else: - servers = list(servers) + servers = list(servers_set) return {"room_id": room_id, "servers": servers} diff --git a/synapse/handlers/room_list.py b/synapse/handlers/room_list.py index ba7a14d651..1a33211a1f 100644 --- a/synapse/handlers/room_list.py +++ b/synapse/handlers/room_list.py @@ -13,9 +13,9 @@ # limitations under the License. import logging -from collections import namedtuple from typing import TYPE_CHECKING, Any, Optional, Tuple +import attr import msgpack from unpaddedbase64 import decode_base64, encode_base64 @@ -474,16 +474,12 @@ class RoomListHandler: ) -class RoomListNextBatch( - namedtuple( - "RoomListNextBatch", - ( - "last_joined_members", # The count to get rooms after/before - "last_room_id", # The room_id to get rooms after/before - "direction_is_forward", # Bool if this is a next_batch, false if prev_batch - ), - ) -): +@attr.s(slots=True, frozen=True, auto_attribs=True) +class RoomListNextBatch: + last_joined_members: int # The count to get rooms after/before + last_room_id: str # The room_id to get rooms after/before + direction_is_forward: bool # True if this is a next_batch, false if prev_batch + KEY_DICT = { "last_joined_members": "m", "last_room_id": "r", @@ -502,12 +498,12 @@ class RoomListNextBatch( def to_token(self) -> str: return encode_base64( msgpack.dumps( - {self.KEY_DICT[key]: val for key, val in self._asdict().items()} + {self.KEY_DICT[key]: val for key, val in attr.asdict(self).items()} ) ) def copy_and_replace(self, **kwds: Any) -> "RoomListNextBatch": - return self._replace(**kwds) + return attr.evolve(self, **kwds) def _matches_room_entry(room_entry: JsonDict, search_filter: dict) -> bool: diff --git a/synapse/handlers/typing.py b/synapse/handlers/typing.py index 1676ebd057..e43c22832d 100644 --- a/synapse/handlers/typing.py +++ b/synapse/handlers/typing.py @@ -13,9 +13,10 @@ # limitations under the License. import logging import random -from collections import namedtuple from typing import TYPE_CHECKING, Dict, Iterable, List, Optional, Set, Tuple +import attr + from synapse.api.errors import AuthError, ShadowBanError, SynapseError from synapse.appservice import ApplicationService from synapse.metrics.background_process_metrics import ( @@ -37,7 +38,10 @@ logger = logging.getLogger(__name__) # A tiny object useful for storing a user's membership in a room, as a mapping # key -RoomMember = namedtuple("RoomMember", ("room_id", "user_id")) +@attr.s(slots=True, frozen=True, auto_attribs=True) +class RoomMember: + room_id: str + user_id: str # How often we expect remote servers to resend us presence. @@ -119,7 +123,7 @@ class FollowerTypingHandler: self.wheel_timer.insert(now=now, obj=member, then=now + 60 * 1000) def is_typing(self, member: RoomMember) -> bool: - return member.user_id in self._room_typing.get(member.room_id, []) + return member.user_id in self._room_typing.get(member.room_id, set()) async def _push_remote(self, member: RoomMember, typing: bool) -> None: if not self.federation: @@ -166,9 +170,9 @@ class FollowerTypingHandler: for row in rows: self._room_serials[row.room_id] = token - prev_typing = set(self._room_typing.get(row.room_id, [])) + prev_typing = self._room_typing.get(row.room_id, set()) now_typing = set(row.user_ids) - self._room_typing[row.room_id] = row.user_ids + self._room_typing[row.room_id] = now_typing if self.federation: run_as_background_process( diff --git a/synapse/http/server.py b/synapse/http/server.py index e302946591..09b4125489 100644 --- a/synapse/http/server.py +++ b/synapse/http/server.py @@ -14,7 +14,6 @@ # limitations under the License. import abc -import collections import html import logging import types @@ -37,6 +36,7 @@ from typing import ( Union, ) +import attr import jinja2 from canonicaljson import encode_canonical_json from typing_extensions import Protocol @@ -354,9 +354,11 @@ class DirectServeJsonResource(_AsyncResource): return_json_error(f, request) -_PathEntry = collections.namedtuple( - "_PathEntry", ["pattern", "callback", "servlet_classname"] -) +@attr.s(slots=True, frozen=True, auto_attribs=True) +class _PathEntry: + pattern: Pattern + callback: ServletCallback + servlet_classname: str class JsonResource(DirectServeJsonResource): diff --git a/synapse/replication/tcp/streams/_base.py b/synapse/replication/tcp/streams/_base.py index 743a01da08..5a2d90c530 100644 --- a/synapse/replication/tcp/streams/_base.py +++ b/synapse/replication/tcp/streams/_base.py @@ -15,7 +15,6 @@ import heapq import logging -from collections import namedtuple from typing import ( TYPE_CHECKING, Any, @@ -30,6 +29,7 @@ from typing import ( import attr from synapse.replication.http.streams import ReplicationGetStreamUpdates +from synapse.types import JsonDict if TYPE_CHECKING: from synapse.server import HomeServer @@ -226,17 +226,14 @@ class BackfillStream(Stream): or it went from being an outlier to not. """ - BackfillStreamRow = namedtuple( - "BackfillStreamRow", - ( - "event_id", # str - "room_id", # str - "type", # str - "state_key", # str, optional - "redacts", # str, optional - "relates_to", # str, optional - ), - ) + @attr.s(slots=True, frozen=True, auto_attribs=True) + class BackfillStreamRow: + event_id: str + room_id: str + type: str + state_key: Optional[str] + redacts: Optional[str] + relates_to: Optional[str] NAME = "backfill" ROW_TYPE = BackfillStreamRow @@ -256,18 +253,15 @@ class BackfillStream(Stream): class PresenceStream(Stream): - PresenceStreamRow = namedtuple( - "PresenceStreamRow", - ( - "user_id", # str - "state", # str - "last_active_ts", # int - "last_federation_update_ts", # int - "last_user_sync_ts", # int - "status_msg", # str - "currently_active", # bool - ), - ) + @attr.s(slots=True, frozen=True, auto_attribs=True) + class PresenceStreamRow: + user_id: str + state: str + last_active_ts: int + last_federation_update_ts: int + last_user_sync_ts: int + status_msg: str + currently_active: bool NAME = "presence" ROW_TYPE = PresenceStreamRow @@ -302,7 +296,7 @@ class PresenceFederationStream(Stream): send. """ - @attr.s(slots=True, auto_attribs=True) + @attr.s(slots=True, frozen=True, auto_attribs=True) class PresenceFederationStreamRow: destination: str user_id: str @@ -320,9 +314,10 @@ class PresenceFederationStream(Stream): class TypingStream(Stream): - TypingStreamRow = namedtuple( - "TypingStreamRow", ("room_id", "user_ids") # str # list(str) - ) + @attr.s(slots=True, frozen=True, auto_attribs=True) + class TypingStreamRow: + room_id: str + user_ids: List[str] NAME = "typing" ROW_TYPE = TypingStreamRow @@ -348,16 +343,13 @@ class TypingStream(Stream): class ReceiptsStream(Stream): - ReceiptsStreamRow = namedtuple( - "ReceiptsStreamRow", - ( - "room_id", # str - "receipt_type", # str - "user_id", # str - "event_id", # str - "data", # dict - ), - ) + @attr.s(slots=True, frozen=True, auto_attribs=True) + class ReceiptsStreamRow: + room_id: str + receipt_type: str + user_id: str + event_id: str + data: dict NAME = "receipts" ROW_TYPE = ReceiptsStreamRow @@ -374,7 +366,9 @@ class ReceiptsStream(Stream): class PushRulesStream(Stream): """A user has changed their push rules""" - PushRulesStreamRow = namedtuple("PushRulesStreamRow", ("user_id",)) # str + @attr.s(slots=True, frozen=True, auto_attribs=True) + class PushRulesStreamRow: + user_id: str NAME = "push_rules" ROW_TYPE = PushRulesStreamRow @@ -396,10 +390,12 @@ class PushRulesStream(Stream): class PushersStream(Stream): """A user has added/changed/removed a pusher""" - PushersStreamRow = namedtuple( - "PushersStreamRow", - ("user_id", "app_id", "pushkey", "deleted"), # str # str # str # bool - ) + @attr.s(slots=True, frozen=True, auto_attribs=True) + class PushersStreamRow: + user_id: str + app_id: str + pushkey: str + deleted: bool NAME = "pushers" ROW_TYPE = PushersStreamRow @@ -419,7 +415,7 @@ class CachesStream(Stream): the cache on the workers """ - @attr.s(slots=True) + @attr.s(slots=True, frozen=True, auto_attribs=True) class CachesStreamRow: """Stream to inform workers they should invalidate their cache. @@ -430,9 +426,9 @@ class CachesStream(Stream): invalidation_ts: Timestamp of when the invalidation took place. """ - cache_func = attr.ib(type=str) - keys = attr.ib(type=Optional[List[Any]]) - invalidation_ts = attr.ib(type=int) + cache_func: str + keys: Optional[List[Any]] + invalidation_ts: int NAME = "caches" ROW_TYPE = CachesStreamRow @@ -451,9 +447,9 @@ class DeviceListsStream(Stream): told about a device update. """ - @attr.s(slots=True) + @attr.s(slots=True, frozen=True, auto_attribs=True) class DeviceListsStreamRow: - entity = attr.ib(type=str) + entity: str NAME = "device_lists" ROW_TYPE = DeviceListsStreamRow @@ -470,7 +466,9 @@ class DeviceListsStream(Stream): class ToDeviceStream(Stream): """New to_device messages for a client""" - ToDeviceStreamRow = namedtuple("ToDeviceStreamRow", ("entity",)) # str + @attr.s(slots=True, frozen=True, auto_attribs=True) + class ToDeviceStreamRow: + entity: str NAME = "to_device" ROW_TYPE = ToDeviceStreamRow @@ -487,9 +485,11 @@ class ToDeviceStream(Stream): class TagAccountDataStream(Stream): """Someone added/removed a tag for a room""" - TagAccountDataStreamRow = namedtuple( - "TagAccountDataStreamRow", ("user_id", "room_id", "data") # str # str # dict - ) + @attr.s(slots=True, frozen=True, auto_attribs=True) + class TagAccountDataStreamRow: + user_id: str + room_id: str + data: JsonDict NAME = "tag_account_data" ROW_TYPE = TagAccountDataStreamRow @@ -506,10 +506,11 @@ class TagAccountDataStream(Stream): class AccountDataStream(Stream): """Global or per room account data was changed""" - AccountDataStreamRow = namedtuple( - "AccountDataStreamRow", - ("user_id", "room_id", "data_type"), # str # Optional[str] # str - ) + @attr.s(slots=True, frozen=True, auto_attribs=True) + class AccountDataStreamRow: + user_id: str + room_id: Optional[str] + data_type: str NAME = "account_data" ROW_TYPE = AccountDataStreamRow @@ -573,10 +574,12 @@ class AccountDataStream(Stream): class GroupServerStream(Stream): - GroupsStreamRow = namedtuple( - "GroupsStreamRow", - ("group_id", "user_id", "type", "content"), # str # str # str # dict - ) + @attr.s(slots=True, frozen=True, auto_attribs=True) + class GroupsStreamRow: + group_id: str + user_id: str + type: str + content: JsonDict NAME = "groups" ROW_TYPE = GroupsStreamRow @@ -593,7 +596,9 @@ class GroupServerStream(Stream): class UserSignatureStream(Stream): """A user has signed their own device with their user-signing key""" - UserSignatureStreamRow = namedtuple("UserSignatureStreamRow", ("user_id")) # str + @attr.s(slots=True, frozen=True, auto_attribs=True) + class UserSignatureStreamRow: + user_id: str NAME = "user_signature" ROW_TYPE = UserSignatureStreamRow diff --git a/synapse/replication/tcp/streams/federation.py b/synapse/replication/tcp/streams/federation.py index 0600cdbf36..4046bdec69 100644 --- a/synapse/replication/tcp/streams/federation.py +++ b/synapse/replication/tcp/streams/federation.py @@ -12,14 +12,16 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from collections import namedtuple from typing import TYPE_CHECKING, Any, Awaitable, Callable, List, Tuple +import attr + from synapse.replication.tcp.streams._base import ( Stream, current_token_without_instance, make_http_update_function, ) +from synapse.types import JsonDict if TYPE_CHECKING: from synapse.server import HomeServer @@ -30,13 +32,10 @@ class FederationStream(Stream): sending disabled. """ - FederationStreamRow = namedtuple( - "FederationStreamRow", - ( - "type", # str, the type of data as defined in the BaseFederationRows - "data", # dict, serialization of a federation.send_queue.BaseFederationRow - ), - ) + @attr.s(slots=True, frozen=True, auto_attribs=True) + class FederationStreamRow: + type: str # the type of data as defined in the BaseFederationRows + data: JsonDict # serialization of a federation.send_queue.BaseFederationRow NAME = "federation" ROW_TYPE = FederationStreamRow diff --git a/synapse/rest/media/v1/media_repository.py b/synapse/rest/media/v1/media_repository.py index 244ba261bb..71b9a34b14 100644 --- a/synapse/rest/media/v1/media_repository.py +++ b/synapse/rest/media/v1/media_repository.py @@ -739,14 +739,21 @@ class MediaRepository: # We deduplicate the thumbnail sizes by ignoring the cropped versions if # they have the same dimensions of a scaled one. thumbnails: Dict[Tuple[int, int, str], str] = {} - for r_width, r_height, r_method, r_type in requirements: - if r_method == "crop": - thumbnails.setdefault((r_width, r_height, r_type), r_method) - elif r_method == "scale": - t_width, t_height = thumbnailer.aspect(r_width, r_height) + for requirement in requirements: + if requirement.method == "crop": + thumbnails.setdefault( + (requirement.width, requirement.height, requirement.media_type), + requirement.method, + ) + elif requirement.method == "scale": + t_width, t_height = thumbnailer.aspect( + requirement.width, requirement.height + ) t_width = min(m_width, t_width) t_height = min(m_height, t_height) - thumbnails[(t_width, t_height, r_type)] = r_method + thumbnails[ + (t_width, t_height, requirement.media_type) + ] = requirement.method # Now we generate the thumbnails for each dimension, store it for (t_width, t_height, t_type), t_method in thumbnails.items(): diff --git a/synapse/state/__init__.py b/synapse/state/__init__.py index 446204dbe5..69ac8c3423 100644 --- a/synapse/state/__init__.py +++ b/synapse/state/__init__.py @@ -14,7 +14,7 @@ # limitations under the License. import heapq import logging -from collections import defaultdict, namedtuple +from collections import defaultdict from typing import ( TYPE_CHECKING, Any, @@ -69,9 +69,6 @@ state_groups_histogram = Histogram( ) -KeyStateTuple = namedtuple("KeyStateTuple", ("context", "type", "state_key")) - - EVICTION_TIMEOUT_SECONDS = 60 * 60 diff --git a/synapse/storage/databases/main/directory.py b/synapse/storage/databases/main/directory.py index a3442814d7..f76c6121e8 100644 --- a/synapse/storage/databases/main/directory.py +++ b/synapse/storage/databases/main/directory.py @@ -12,16 +12,22 @@ # See the License for the specific language governing permissions and # limitations under the License. -from collections import namedtuple from typing import Iterable, List, Optional, Tuple +import attr + from synapse.api.errors import SynapseError from synapse.storage.database import LoggingTransaction from synapse.storage.databases.main import CacheInvalidationWorkerStore from synapse.types import RoomAlias from synapse.util.caches.descriptors import cached -RoomAliasMapping = namedtuple("RoomAliasMapping", ("room_id", "room_alias", "servers")) + +@attr.s(slots=True, frozen=True, auto_attribs=True) +class RoomAliasMapping: + room_id: str + room_alias: str + servers: List[str] class DirectoryWorkerStore(CacheInvalidationWorkerStore): diff --git a/synapse/storage/databases/main/events.py b/synapse/storage/databases/main/events.py index 81e67ece55..dd255aefb9 100644 --- a/synapse/storage/databases/main/events.py +++ b/synapse/storage/databases/main/events.py @@ -1976,14 +1976,17 @@ class PersistEventsStore: txn, self.store.get_retention_policy_for_room, (event.room_id,) ) - def store_event_search_txn(self, txn, event, key, value): + def store_event_search_txn( + self, txn: LoggingTransaction, event: EventBase, key: str, value: str + ) -> None: """Add event to the search table Args: - txn (cursor): - event (EventBase): - key (str): - value (str): + txn: The database transaction. + event: The event being added to the search table. + key: A key describing the search value (one of "content.name", + "content.topic", or "content.body") + value: The value from the event's content. """ self.store.store_search_entries_txn( txn, diff --git a/synapse/storage/databases/main/room.py b/synapse/storage/databases/main/room.py index 4472335af9..c0e837854a 100644 --- a/synapse/storage/databases/main/room.py +++ b/synapse/storage/databases/main/room.py @@ -13,11 +13,22 @@ # See the License for the specific language governing permissions and # limitations under the License. -import collections import logging from abc import abstractmethod from enum import Enum -from typing import TYPE_CHECKING, Any, Awaitable, Dict, List, Optional, Tuple, cast +from typing import ( + TYPE_CHECKING, + Any, + Awaitable, + Dict, + List, + Optional, + Tuple, + Union, + cast, +) + +import attr from synapse.api.constants import EventContentFields, EventTypes, JoinRules from synapse.api.errors import StoreError @@ -43,9 +54,10 @@ if TYPE_CHECKING: logger = logging.getLogger(__name__) -RatelimitOverride = collections.namedtuple( - "RatelimitOverride", ("messages_per_second", "burst_count") -) +@attr.s(slots=True, frozen=True, auto_attribs=True) +class RatelimitOverride: + messages_per_second: int + burst_count: int class RoomSortOrder(Enum): @@ -207,6 +219,7 @@ class RoomWorkerStore(CacheInvalidationWorkerStore): WHERE appservice_id = ? AND network_id = ? """ query_args.append(network_tuple.appservice_id) + assert network_tuple.network_id is not None query_args.append(network_tuple.network_id) else: published_sql = """ @@ -284,7 +297,7 @@ class RoomWorkerStore(CacheInvalidationWorkerStore): """ where_clauses = [] - query_args = [] + query_args: List[Union[str, int]] = [] if network_tuple: if network_tuple.appservice_id: @@ -293,6 +306,7 @@ class RoomWorkerStore(CacheInvalidationWorkerStore): WHERE appservice_id = ? AND network_id = ? """ query_args.append(network_tuple.appservice_id) + assert network_tuple.network_id is not None query_args.append(network_tuple.network_id) else: published_sql = """ diff --git a/synapse/storage/databases/main/search.py b/synapse/storage/databases/main/search.py index f87acfb866..2d085a5764 100644 --- a/synapse/storage/databases/main/search.py +++ b/synapse/storage/databases/main/search.py @@ -14,9 +14,10 @@ import logging import re -from collections import namedtuple from typing import TYPE_CHECKING, Collection, Iterable, List, Optional, Set +import attr + from synapse.api.errors import SynapseError from synapse.events import EventBase from synapse.storage._base import SQLBaseStore, db_to_json, make_in_list_sql_clause @@ -33,10 +34,15 @@ if TYPE_CHECKING: logger = logging.getLogger(__name__) -SearchEntry = namedtuple( - "SearchEntry", - ["key", "value", "event_id", "room_id", "stream_ordering", "origin_server_ts"], -) + +@attr.s(slots=True, frozen=True, auto_attribs=True) +class SearchEntry: + key: str + value: str + event_id: str + room_id: str + stream_ordering: Optional[int] + origin_server_ts: int def _clean_value_for_search(value: str) -> str: diff --git a/synapse/storage/databases/main/state.py b/synapse/storage/databases/main/state.py index 4bc044fb16..7e5a6aae18 100644 --- a/synapse/storage/databases/main/state.py +++ b/synapse/storage/databases/main/state.py @@ -14,7 +14,6 @@ # limitations under the License. import collections.abc import logging -from collections import namedtuple from typing import TYPE_CHECKING, Iterable, Optional, Set from synapse.api.constants import EventTypes, Membership @@ -43,19 +42,6 @@ logger = logging.getLogger(__name__) MAX_STATE_DELTA_HOPS = 100 -class _GetStateGroupDelta( - namedtuple("_GetStateGroupDelta", ("prev_group", "delta_ids")) -): - """Return type of get_state_group_delta that implements __len__, which lets - us use the itrable flag when caching - """ - - __slots__ = [] - - def __len__(self): - return len(self.delta_ids) if self.delta_ids else 0 - - # this inherits from EventsWorkerStore because it calls self.get_events class StateGroupWorkerStore(EventsWorkerStore, SQLBaseStore): """The parts of StateGroupStore that can be called from workers.""" diff --git a/synapse/storage/databases/main/stream.py b/synapse/storage/databases/main/stream.py index 9488fd5094..b0642ca69f 100644 --- a/synapse/storage/databases/main/stream.py +++ b/synapse/storage/databases/main/stream.py @@ -36,9 +36,9 @@ what sort order was used: """ import abc import logging -from collections import namedtuple from typing import TYPE_CHECKING, Collection, Dict, List, Optional, Set, Tuple +import attr from frozendict import frozendict from twisted.internet import defer @@ -74,9 +74,11 @@ _TOPOLOGICAL_TOKEN = "topological" # Used as return values for pagination APIs -_EventDictReturn = namedtuple( - "_EventDictReturn", ("event_id", "topological_ordering", "stream_ordering") -) +@attr.s(slots=True, frozen=True, auto_attribs=True) +class _EventDictReturn: + event_id: str + topological_ordering: Optional[int] + stream_ordering: int def generate_pagination_where_clause( @@ -825,7 +827,7 @@ class StreamWorkerStore(EventsWorkerStore, SQLBaseStore, metaclass=abc.ABCMeta): for event, row in zip(events, rows): stream = row.stream_ordering if topo_order and row.topological_ordering: - topo = row.topological_ordering + topo: Optional[int] = row.topological_ordering else: topo = None internal = event.internal_metadata diff --git a/synapse/types.py b/synapse/types.py index b06979e8e8..42aeaf6270 100644 --- a/synapse/types.py +++ b/synapse/types.py @@ -15,7 +15,6 @@ import abc import re import string -from collections import namedtuple from typing import ( TYPE_CHECKING, Any, @@ -227,8 +226,7 @@ class DomainSpecificString(metaclass=abc.ABCMeta): localpart = attr.ib(type=str) domain = attr.ib(type=str) - # Because this class is a namedtuple of strings and booleans, it is deeply - # immutable. + # Because this is a frozen class, it is deeply immutable. def __copy__(self): return self @@ -708,16 +706,18 @@ class PersistedEventPosition: return RoomStreamToken(None, self.stream) -class ThirdPartyInstanceID( - namedtuple("ThirdPartyInstanceID", ("appservice_id", "network_id")) -): +@attr.s(slots=True, frozen=True, auto_attribs=True) +class ThirdPartyInstanceID: + appservice_id: Optional[str] + network_id: Optional[str] + # Deny iteration because it will bite you if you try to create a singleton # set by: # users = set(user) def __iter__(self): raise ValueError("Attempted to iterate a %s" % (type(self).__name__,)) - # Because this class is a namedtuple of strings, it is deeply immutable. + # Because this class is a frozen class, it is deeply immutable. def __copy__(self): return self @@ -725,22 +725,18 @@ class ThirdPartyInstanceID( return self @classmethod - def from_string(cls, s): + def from_string(cls, s: str) -> "ThirdPartyInstanceID": bits = s.split("|", 2) if len(bits) != 2: raise SynapseError(400, "Invalid ID %r" % (s,)) return cls(appservice_id=bits[0], network_id=bits[1]) - def to_string(self): + def to_string(self) -> str: return "%s|%s" % (self.appservice_id, self.network_id) __str__ = to_string - @classmethod - def create(cls, appservice_id, network_id): - return cls(appservice_id=appservice_id, network_id=network_id) - @attr.s(slots=True) class ReadReceipt: diff --git a/tests/replication/test_federation_ack.py b/tests/replication/test_federation_ack.py index 04a869e295..1b6a4bf4b0 100644 --- a/tests/replication/test_federation_ack.py +++ b/tests/replication/test_federation_ack.py @@ -62,7 +62,11 @@ class FederationAckTestCase(HomeserverTestCase): "federation", "master", token=10, - rows=[FederationStream.FederationStreamRow(type="x", data=[1, 2, 3])], + rows=[ + FederationStream.FederationStreamRow( + type="x", data={"test": [1, 2, 3]} + ) + ], ) ) From 13c974ed358c28940588f447b363063c76063fc2 Mon Sep 17 00:00:00 2001 From: Shay Date: Mon, 3 Jan 2022 11:17:16 -0800 Subject: [PATCH 31/76] Drop Bionic from Debian builds (#11633) * update Trove classifiers to remove py36 * stop building bionic * update dh-virtualenv * newsfragment * fix newsfragment * update version refs * another try at correct tag * Update changelog --- changelog.d/11633.misc | 1 + docker/Dockerfile-dhvirtualenv | 9 ++++----- scripts-dev/build_debian_packages | 1 - setup.py | 1 - 4 files changed, 5 insertions(+), 7 deletions(-) create mode 100644 changelog.d/11633.misc diff --git a/changelog.d/11633.misc b/changelog.d/11633.misc new file mode 100644 index 0000000000..73e814e58e --- /dev/null +++ b/changelog.d/11633.misc @@ -0,0 +1 @@ +Drop support for Python 3.6 and Ubuntu 18.04. \ No newline at end of file diff --git a/docker/Dockerfile-dhvirtualenv b/docker/Dockerfile-dhvirtualenv index 1dd88140c7..fbc1d2346f 100644 --- a/docker/Dockerfile-dhvirtualenv +++ b/docker/Dockerfile-dhvirtualenv @@ -16,7 +16,7 @@ ARG distro="" ### Stage 0: build a dh-virtualenv ### -# This is only really needed on bionic and focal, since other distributions we +# This is only really needed on focal, since other distributions we # care about have a recent version of dh-virtualenv by default. Unfortunately, # it looks like focal is going to be with us for a while. # @@ -36,9 +36,8 @@ RUN env DEBIAN_FRONTEND=noninteractive apt-get install \ wget # fetch and unpack the package -# TODO: Upgrade to 1.2.2 once bionic is dropped (1.2.2 requires debhelper 12; bionic has only 11) RUN mkdir /dh-virtualenv -RUN wget -q -O /dh-virtualenv.tar.gz https://github.com/spotify/dh-virtualenv/archive/ac6e1b1.tar.gz +RUN wget -q -O /dh-virtualenv.tar.gz https://github.com/spotify/dh-virtualenv/archive/refs/tags/1.2.2.tar.gz RUN tar -xv --strip-components=1 -C /dh-virtualenv -f /dh-virtualenv.tar.gz # install its build deps. We do another apt-cache-update here, because we might @@ -86,12 +85,12 @@ RUN apt-get update -qq -o Acquire::Languages=none \ libpq-dev \ xmlsec1 -COPY --from=builder /dh-virtualenv_1.2~dev-1_all.deb / +COPY --from=builder /dh-virtualenv_1.2.2-1_all.deb / # install dhvirtualenv. Update the apt cache again first, in case we got a # cached cache from docker the first time. RUN apt-get update -qq -o Acquire::Languages=none \ - && apt-get install -yq /dh-virtualenv_1.2~dev-1_all.deb + && apt-get install -yq /dh-virtualenv_1.2.2-1_all.deb WORKDIR /synapse/source ENTRYPOINT ["bash","/synapse/source/docker/build_debian.sh"] diff --git a/scripts-dev/build_debian_packages b/scripts-dev/build_debian_packages index 3a9a2d257c..4d34e90703 100755 --- a/scripts-dev/build_debian_packages +++ b/scripts-dev/build_debian_packages @@ -24,7 +24,6 @@ DISTS = ( "debian:bullseye", "debian:bookworm", "debian:sid", - "ubuntu:bionic", # 18.04 LTS (our EOL forced by Py36 on 2021-12-23) "ubuntu:focal", # 20.04 LTS (our EOL forced by Py38 on 2024-10-14) "ubuntu:hirsute", # 21.04 (EOL 2022-01-05) "ubuntu:impish", # 21.10 (EOL 2022-07) diff --git a/setup.py b/setup.py index fbb0133016..e618ff898b 100755 --- a/setup.py +++ b/setup.py @@ -162,7 +162,6 @@ setup( "Topic :: Communications :: Chat", "License :: OSI Approved :: Apache Software License", "Programming Language :: Python :: 3 :: Only", - "Programming Language :: Python :: 3.6", "Programming Language :: Python :: 3.7", "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", From 8422a7f7f6154bcbadc52f0d0d27b8e6b989cb4c Mon Sep 17 00:00:00 2001 From: Patrick Cloke Date: Tue, 4 Jan 2022 11:08:08 -0500 Subject: [PATCH 32/76] Include the topic event in the prejoin state, per MSC3173. (#11666) Invites and knocks will now include the topic in the stripped state send to clients before joining the room. --- changelog.d/11666.feature | 1 + docs/sample_config.yaml | 1 + synapse/config/api.py | 2 ++ tests/federation/transport/test_knocking.py | 9 +++++++++ 4 files changed, 13 insertions(+) create mode 100644 changelog.d/11666.feature diff --git a/changelog.d/11666.feature b/changelog.d/11666.feature new file mode 100644 index 0000000000..6f6b127e22 --- /dev/null +++ b/changelog.d/11666.feature @@ -0,0 +1 @@ +Include the room topic in the stripped state included with invites and knocking. diff --git a/docs/sample_config.yaml b/docs/sample_config.yaml index 6696ed5d1e..00dfd2c013 100644 --- a/docs/sample_config.yaml +++ b/docs/sample_config.yaml @@ -1488,6 +1488,7 @@ room_prejoin_state: # - m.room.encryption # - m.room.name # - m.room.create + # - m.room.topic # # Uncomment the following to disable these defaults (so that only the event # types listed in 'additional_event_types' are shared). Defaults to 'false'. diff --git a/synapse/config/api.py b/synapse/config/api.py index b18044f982..25538b82d5 100644 --- a/synapse/config/api.py +++ b/synapse/config/api.py @@ -107,6 +107,8 @@ _DEFAULT_PREJOIN_STATE_TYPES = [ EventTypes.Name, # Per MSC1772. EventTypes.Create, + # Per MSC3173. + EventTypes.Topic, ] diff --git a/tests/federation/transport/test_knocking.py b/tests/federation/transport/test_knocking.py index 663960ff53..bfa156eebb 100644 --- a/tests/federation/transport/test_knocking.py +++ b/tests/federation/transport/test_knocking.py @@ -108,6 +108,15 @@ class KnockingStrippedStateEventHelperMixin(TestCase): "state_key": "", }, ), + ( + EventTypes.Topic, + { + "content": { + "topic": "A really cool room", + }, + "state_key": "", + }, + ), ] ) From bd9821f7f17916deec9266483e26b69cd5ce2b3b Mon Sep 17 00:00:00 2001 From: Richard van der Hoff <1389908+richvdh@users.noreply.github.com> Date: Tue, 4 Jan 2022 16:10:05 +0000 Subject: [PATCH 33/76] Better error messages from `get_create_event_for_room` (#11638) "Unknown room" can mean a multitude of things here. To help with debugging, add some more words to the exception text. --- changelog.d/11638.misc | 1 + synapse/storage/databases/main/state.py | 6 +++++- 2 files changed, 6 insertions(+), 1 deletion(-) create mode 100644 changelog.d/11638.misc diff --git a/changelog.d/11638.misc b/changelog.d/11638.misc new file mode 100644 index 0000000000..76dfb56bda --- /dev/null +++ b/changelog.d/11638.misc @@ -0,0 +1 @@ +Improve the error messages from `get_create_event_for_room`. diff --git a/synapse/storage/databases/main/state.py b/synapse/storage/databases/main/state.py index 7e5a6aae18..2fb3e65192 100644 --- a/synapse/storage/databases/main/state.py +++ b/synapse/storage/databases/main/state.py @@ -177,11 +177,15 @@ class StateGroupWorkerStore(EventsWorkerStore, SQLBaseStore): NotFoundError if the room is unknown """ state_ids = await self.get_current_state_ids(room_id) + + if not state_ids: + raise NotFoundError(f"Current state for room {room_id} is empty") + create_id = state_ids.get((EventTypes.Create, "")) # If we can't find the create event, assume we've hit a dead end if not create_id: - raise NotFoundError("Unknown room %s" % (room_id,)) + raise NotFoundError(f"No create event in current state for room {room_id}") # Retrieve the room's create event and return create_event = await self.get_event(create_id) From 2359ee3864a065229c80e3ff58faa981edd24558 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff <1389908+richvdh@users.noreply.github.com> Date: Tue, 4 Jan 2022 16:10:27 +0000 Subject: [PATCH 34/76] Remove redundant `get_current_events_token` (#11643) * Push `get_room_{min,max_stream_ordering}` into StreamStore Both implementations of this are identical, so we may as well push it down and get rid of the abstract base class nonsense. * Remove redundant `StreamStore` class This is empty now * Remove redundant `get_current_events_token` This was an exact duplicate of `get_room_max_stream_ordering`, so let's get rid of it. * newsfile --- changelog.d/11643.misc | 1 + synapse/handlers/federation_event.py | 2 +- synapse/handlers/presence.py | 2 +- synapse/replication/slave/storage/events.py | 9 ----- synapse/storage/databases/main/__init__.py | 4 +-- .../storage/databases/main/events_worker.py | 4 --- synapse/storage/databases/main/stream.py | 34 ++++++++----------- 7 files changed, 20 insertions(+), 36 deletions(-) create mode 100644 changelog.d/11643.misc diff --git a/changelog.d/11643.misc b/changelog.d/11643.misc new file mode 100644 index 0000000000..1c3b3071f6 --- /dev/null +++ b/changelog.d/11643.misc @@ -0,0 +1 @@ +Remove redundant `get_current_events_token` method. diff --git a/synapse/handlers/federation_event.py b/synapse/handlers/federation_event.py index 9917613298..d08e48da58 100644 --- a/synapse/handlers/federation_event.py +++ b/synapse/handlers/federation_event.py @@ -1838,7 +1838,7 @@ class FederationEventHandler: The stream ID after which all events have been persisted. """ if not event_and_contexts: - return self._store.get_current_events_token() + return self._store.get_room_max_stream_ordering() instance = self._config.worker.events_shard_config.get_instance(room_id) if instance != self._instance_name: diff --git a/synapse/handlers/presence.py b/synapse/handlers/presence.py index 454d06c973..c781fefb1b 100644 --- a/synapse/handlers/presence.py +++ b/synapse/handlers/presence.py @@ -729,7 +729,7 @@ class PresenceHandler(BasePresenceHandler): # Presence is best effort and quickly heals itself, so lets just always # stream from the current state when we restart. - self._event_pos = self.store.get_current_events_token() + self._event_pos = self.store.get_room_max_stream_ordering() self._event_processing = False async def _on_shutdown(self) -> None: diff --git a/synapse/replication/slave/storage/events.py b/synapse/replication/slave/storage/events.py index 50e7379e83..0f08372694 100644 --- a/synapse/replication/slave/storage/events.py +++ b/synapse/replication/slave/storage/events.py @@ -80,12 +80,3 @@ class SlavedEventStore( min_curr_state_delta_id, prefilled_cache=curr_state_delta_prefill, ) - - # Cached functions can't be accessed through a class instance so we need - # to reach inside the __dict__ to extract them. - - def get_room_max_stream_ordering(self): - return self._stream_id_gen.get_current_token() - - def get_room_min_stream_ordering(self): - return self._backfill_id_gen.get_current_token() diff --git a/synapse/storage/databases/main/__init__.py b/synapse/storage/databases/main/__init__.py index a594223fc6..f024761ba7 100644 --- a/synapse/storage/databases/main/__init__.py +++ b/synapse/storage/databases/main/__init__.py @@ -68,7 +68,7 @@ from .session import SessionStore from .signatures import SignatureStore from .state import StateStore from .stats import StatsStore -from .stream import StreamStore +from .stream import StreamWorkerStore from .tags import TagsStore from .transactions import TransactionWorkerStore from .ui_auth import UIAuthStore @@ -87,7 +87,7 @@ class DataStore( RoomStore, RoomBatchStore, RegistrationStore, - StreamStore, + StreamWorkerStore, ProfileStore, PresenceStore, TransactionWorkerStore, diff --git a/synapse/storage/databases/main/events_worker.py b/synapse/storage/databases/main/events_worker.py index c7b660ac5a..8d4287045a 100644 --- a/synapse/storage/databases/main/events_worker.py +++ b/synapse/storage/databases/main/events_worker.py @@ -1383,10 +1383,6 @@ class EventsWorkerStore(SQLBaseStore): return {"v1": complexity_v1} - def get_current_events_token(self) -> int: - """The current maximum token that events have reached""" - return self._stream_id_gen.get_current_token() - async def get_all_new_forward_event_rows( self, instance_name: str, last_id: int, current_id: int, limit: int ) -> List[Tuple[int, str, str, str, str, str, str, str, str]]: diff --git a/synapse/storage/databases/main/stream.py b/synapse/storage/databases/main/stream.py index b0642ca69f..319464b1fa 100644 --- a/synapse/storage/databases/main/stream.py +++ b/synapse/storage/databases/main/stream.py @@ -34,7 +34,7 @@ what sort order was used: - topological tokems: "t%d-%d", where the integers map to the topological and stream ordering columns respectively. """ -import abc + import logging from typing import TYPE_CHECKING, Collection, Dict, List, Optional, Set, Tuple @@ -336,12 +336,7 @@ def filter_to_clause(event_filter: Optional[Filter]) -> Tuple[str, List[str]]: return " AND ".join(clauses), args -class StreamWorkerStore(EventsWorkerStore, SQLBaseStore, metaclass=abc.ABCMeta): - """This is an abstract base class where subclasses must implement - `get_room_max_stream_ordering` and `get_room_min_stream_ordering` - which can be called in the initializer. - """ - +class StreamWorkerStore(EventsWorkerStore, SQLBaseStore): def __init__( self, database: DatabasePool, @@ -379,13 +374,22 @@ class StreamWorkerStore(EventsWorkerStore, SQLBaseStore, metaclass=abc.ABCMeta): self._stream_order_on_start = self.get_room_max_stream_ordering() - @abc.abstractmethod def get_room_max_stream_ordering(self) -> int: - raise NotImplementedError() + """Get the stream_ordering of regular events that we have committed up to + + Returns the maximum stream id such that all stream ids less than or + equal to it have been successfully persisted. + """ + return self._stream_id_gen.get_current_token() - @abc.abstractmethod def get_room_min_stream_ordering(self) -> int: - raise NotImplementedError() + """Get the stream_ordering of backfilled events that we have committed up to + + Backfilled events use *negative* stream orderings, so this returns the + minimum negative stream id such that all stream ids greater than or + equal to it have been successfully persisted. + """ + return self._backfill_id_gen.get_current_token() def get_room_max_token(self) -> RoomStreamToken: """Get a `RoomStreamToken` that marks the current maximum persisted @@ -1351,11 +1355,3 @@ class StreamWorkerStore(EventsWorkerStore, SQLBaseStore, metaclass=abc.ABCMeta): retcol="instance_name", desc="get_name_from_instance_id", ) - - -class StreamStore(StreamWorkerStore): - def get_room_max_stream_ordering(self) -> int: - return self._stream_id_gen.get_current_token() - - def get_room_min_stream_ordering(self) -> int: - return self._backfill_id_gen.get_current_token() From 878aa5529399ece8fbb68674a6de86d6958bacae Mon Sep 17 00:00:00 2001 From: Richard van der Hoff <1389908+richvdh@users.noreply.github.com> Date: Tue, 4 Jan 2022 16:31:32 +0000 Subject: [PATCH 35/76] `FederationClient.backfill`: stop flagging events as outliers (#11632) Events returned by `backfill` should not be flagged as outliers. Fixes: ``` AssertionError: null File "synapse/handlers/federation.py", line 313, in try_backfill dom, room_id, limit=100, extremities=extremities File "synapse/handlers/federation_event.py", line 517, in backfill await self._process_pulled_events(dest, events, backfilled=True) File "synapse/handlers/federation_event.py", line 642, in _process_pulled_events await self._process_pulled_event(origin, ev, backfilled=backfilled) File "synapse/handlers/federation_event.py", line 669, in _process_pulled_event assert not event.internal_metadata.is_outlier() ``` See https://sentry.matrix.org/sentry/synapse-matrixorg/issues/231992 Fixes #8894. --- changelog.d/11632.bugfix | 1 + synapse/federation/federation_client.py | 2 +- synapse/handlers/federation_event.py | 4 +++- 3 files changed, 5 insertions(+), 2 deletions(-) create mode 100644 changelog.d/11632.bugfix diff --git a/changelog.d/11632.bugfix b/changelog.d/11632.bugfix new file mode 100644 index 0000000000..c73d41652a --- /dev/null +++ b/changelog.d/11632.bugfix @@ -0,0 +1 @@ +Fix a bug introduced in Synapse 1.19.3 which could sometimes cause `AssertionError`s when backfilling rooms over federation. diff --git a/synapse/federation/federation_client.py b/synapse/federation/federation_client.py index fee1477ab6..7353c2b6b1 100644 --- a/synapse/federation/federation_client.py +++ b/synapse/federation/federation_client.py @@ -272,7 +272,7 @@ class FederationClient(FederationBase): # Check signatures and hash of pdus, removing any from the list that fail checks pdus[:] = await self._check_sigs_and_hash_and_fetch( - dest, pdus, outlier=True, room_version=room_version + dest, pdus, room_version=room_version ) return pdus diff --git a/synapse/handlers/federation_event.py b/synapse/handlers/federation_event.py index d08e48da58..7c81e3651f 100644 --- a/synapse/handlers/federation_event.py +++ b/synapse/handlers/federation_event.py @@ -666,7 +666,9 @@ class FederationEventHandler: logger.info("Processing pulled event %s", event) # these should not be outliers. - assert not event.internal_metadata.is_outlier() + assert ( + not event.internal_metadata.is_outlier() + ), "pulled event unexpectedly flagged as outlier" event_id = event.event_id From b38bdae3a2e5b7cfe862580368b996b8d7dfa50f Mon Sep 17 00:00:00 2001 From: Richard van der Hoff <1389908+richvdh@users.noreply.github.com> Date: Tue, 4 Jan 2022 16:36:33 +0000 Subject: [PATCH 36/76] Fix AssertionErrors after purging events (#11642) * Fix AssertionErrors after purging events If you purged a bunch of events from your database, and then restarted synapse without receiving more events, then you would get a bunch of AssertionErrors on restart. This fixes the situation by rewinding the stream processors. * `check-newsfragment`: ignore deleted newsfiles --- changelog.d/11536.bugfix | 1 + changelog.d/11536.misc | 1 - changelog.d/11642.bugfix | 1 + scripts-dev/check-newsfragment | 4 ++-- synapse/handlers/stats.py | 11 +++++++++++ synapse/handlers/user_directory.py | 18 +++++++++++++++--- 6 files changed, 30 insertions(+), 6 deletions(-) create mode 100644 changelog.d/11536.bugfix delete mode 100644 changelog.d/11536.misc create mode 100644 changelog.d/11642.bugfix diff --git a/changelog.d/11536.bugfix b/changelog.d/11536.bugfix new file mode 100644 index 0000000000..4a1b007254 --- /dev/null +++ b/changelog.d/11536.bugfix @@ -0,0 +1 @@ +Fix a long-standing bug which could cause `AssertionError`s to be written to the log when Synapse was restarted after purging events from the database. diff --git a/changelog.d/11536.misc b/changelog.d/11536.misc deleted file mode 100644 index b9191c111b..0000000000 --- a/changelog.d/11536.misc +++ /dev/null @@ -1 +0,0 @@ -Improvements to log messages around handling stream ids. diff --git a/changelog.d/11642.bugfix b/changelog.d/11642.bugfix new file mode 100644 index 0000000000..4a1b007254 --- /dev/null +++ b/changelog.d/11642.bugfix @@ -0,0 +1 @@ +Fix a long-standing bug which could cause `AssertionError`s to be written to the log when Synapse was restarted after purging events from the database. diff --git a/scripts-dev/check-newsfragment b/scripts-dev/check-newsfragment index af4de345df..c764011d6a 100755 --- a/scripts-dev/check-newsfragment +++ b/scripts-dev/check-newsfragment @@ -42,8 +42,8 @@ echo "--------------------------" echo matched=0 -for f in $(git diff --name-only FETCH_HEAD... -- changelog.d); do - # check that any modified newsfiles on this branch end with a full stop. +for f in $(git diff --diff-filter=d --name-only FETCH_HEAD... -- changelog.d); do + # check that any added newsfiles on this branch end with a full stop. lastchar=$(tr -d '\n' < "$f" | tail -c 1) if [ "$lastchar" != '.' ] && [ "$lastchar" != '!' ]; then echo -e "\e[31mERROR: newsfragment $f does not end with a '.' or '!'\e[39m" >&2 diff --git a/synapse/handlers/stats.py b/synapse/handlers/stats.py index bd3e6f2ec7..29e41a4c79 100644 --- a/synapse/handlers/stats.py +++ b/synapse/handlers/stats.py @@ -80,6 +80,17 @@ class StatsHandler: # If self.pos is None then means we haven't fetched it from DB if self.pos is None: self.pos = await self.store.get_stats_positions() + room_max_stream_ordering = self.store.get_room_max_stream_ordering() + if self.pos > room_max_stream_ordering: + # apparently, we've processed more events than exist in the database! + # this can happen if events are removed with history purge or similar. + logger.warning( + "Event stream ordering appears to have gone backwards (%i -> %i): " + "rewinding stats processor", + self.pos, + room_max_stream_ordering, + ) + self.pos = room_max_stream_ordering # Loop round handling deltas until we're up to date diff --git a/synapse/handlers/user_directory.py b/synapse/handlers/user_directory.py index a0eb45446f..1565e034cb 100644 --- a/synapse/handlers/user_directory.py +++ b/synapse/handlers/user_directory.py @@ -148,9 +148,21 @@ class UserDirectoryHandler(StateDeltasHandler): if self.pos is None: self.pos = await self.store.get_user_directory_stream_pos() - # If still None then the initial background update hasn't happened yet. - if self.pos is None: - return None + # If still None then the initial background update hasn't happened yet. + if self.pos is None: + return None + + room_max_stream_ordering = self.store.get_room_max_stream_ordering() + if self.pos > room_max_stream_ordering: + # apparently, we've processed more events than exist in the database! + # this can happen if events are removed with history purge or similar. + logger.warning( + "Event stream ordering appears to have gone backwards (%i -> %i): " + "rewinding user directory processor", + self.pos, + room_max_stream_ordering, + ) + self.pos = room_max_stream_ordering # Loop round handling deltas until we're up to date while True: From 79f6d3550a14cacadacbb47d1d1ccf9d06f73158 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff <1389908+richvdh@users.noreply.github.com> Date: Tue, 4 Jan 2022 22:31:45 +0000 Subject: [PATCH 37/76] update ngnix reverse-proxy example (#11680) this should not be a case-insensitive match. --- changelog.d/11680.doc | 1 + docs/reverse_proxy.md | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) create mode 100644 changelog.d/11680.doc diff --git a/changelog.d/11680.doc b/changelog.d/11680.doc new file mode 100644 index 0000000000..09399ad9d0 --- /dev/null +++ b/changelog.d/11680.doc @@ -0,0 +1 @@ +Correct the documentation for `nginx` to use a case-sensitive url pattern. Fixes an error introduced in v1.21.0. diff --git a/docs/reverse_proxy.md b/docs/reverse_proxy.md index f3b3aea732..1a89da50fd 100644 --- a/docs/reverse_proxy.md +++ b/docs/reverse_proxy.md @@ -63,7 +63,7 @@ server { server_name matrix.example.com; - location ~* ^(\/_matrix|\/_synapse\/client) { + location ~ ^(/_matrix|/_synapse/client) { # note: do not add a path (even a single /) after the port in `proxy_pass`, # otherwise nginx will canonicalise the URI and cause signature verification # errors. From 0715e77b06ce4b73c3061562081b979b68440209 Mon Sep 17 00:00:00 2001 From: Donny Johnson Date: Wed, 5 Jan 2022 04:38:51 -0600 Subject: [PATCH 38/76] Correct Synapse install command for FreeBSD. (#11267) Co-authored-by: reivilibre --- changelog.d/11267.doc | 1 + docs/setup/installation.md | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) create mode 100644 changelog.d/11267.doc diff --git a/changelog.d/11267.doc b/changelog.d/11267.doc new file mode 100644 index 0000000000..3a720158de --- /dev/null +++ b/changelog.d/11267.doc @@ -0,0 +1 @@ +Update Synapse install command for FreeBSD as the package is now prefixed with `py38`. Contributed by @itchychips. diff --git a/docs/setup/installation.md b/docs/setup/installation.md index 16562be953..210c80dace 100644 --- a/docs/setup/installation.md +++ b/docs/setup/installation.md @@ -164,7 +164,7 @@ xbps-install -S synapse Synapse can be installed via FreeBSD Ports or Packages contributed by Brendan Molloy from: - Ports: `cd /usr/ports/net-im/py-matrix-synapse && make install clean` -- Packages: `pkg install py37-matrix-synapse` +- Packages: `pkg install py38-matrix-synapse` #### OpenBSD From 7013e06e2f60e1401349fd054372808376facc87 Mon Sep 17 00:00:00 2001 From: Callum Macdonald Date: Wed, 5 Jan 2022 11:50:28 +0100 Subject: [PATCH 39/76] Improve Docker docs for use with Postgres (#11640) --- changelog.d/11640.doc | 1 + contrib/docker/docker-compose.yml | 1 + docker/README.md | 4 ++++ 3 files changed, 6 insertions(+) create mode 100644 changelog.d/11640.doc diff --git a/changelog.d/11640.doc b/changelog.d/11640.doc new file mode 100644 index 0000000000..c4773e4f3a --- /dev/null +++ b/changelog.d/11640.doc @@ -0,0 +1 @@ +Add references for using Postgres to the Docker documentation. diff --git a/contrib/docker/docker-compose.yml b/contrib/docker/docker-compose.yml index 26d640c448..5ac41139e3 100644 --- a/contrib/docker/docker-compose.yml +++ b/contrib/docker/docker-compose.yml @@ -14,6 +14,7 @@ services: # failure restart: unless-stopped # See the readme for a full documentation of the environment settings + # NOTE: You must edit homeserver.yaml to use postgres, it defaults to sqlite environment: - SYNAPSE_CONFIG_PATH=/data/homeserver.yaml volumes: diff --git a/docker/README.md b/docker/README.md index 4349e71f87..67c3bc65f0 100644 --- a/docker/README.md +++ b/docker/README.md @@ -68,6 +68,10 @@ The following environment variables are supported in `generate` mode: directories. If unset, and no user is set via `docker run --user`, defaults to `991`, `991`. +## Postgres + +By default the config will use SQLite. See the [docs on using Postgres](https://github.com/matrix-org/synapse/blob/develop/docs/postgres.md) for more info on how to use Postgres. Until this section is improved [this issue](https://github.com/matrix-org/synapse/issues/8304) may provide useful information. + ## Running synapse Once you have a valid configuration file, you can start synapse as follows: From 84d790a32ec4641365adf494a21b28bd680faf38 Mon Sep 17 00:00:00 2001 From: reivilibre Date: Wed, 5 Jan 2022 11:25:32 +0000 Subject: [PATCH 40/76] Clarify SSO mapping provider documentation by writing `def` or `async def` before the names of methods, as appropriate. (#11681) --- changelog.d/11681.doc | 1 + docs/sso_mapping_providers.md | 24 ++++++++++++------------ 2 files changed, 13 insertions(+), 12 deletions(-) create mode 100644 changelog.d/11681.doc diff --git a/changelog.d/11681.doc b/changelog.d/11681.doc new file mode 100644 index 0000000000..fef70211cd --- /dev/null +++ b/changelog.d/11681.doc @@ -0,0 +1 @@ +Clarify SSO mapping provider documentation by writing `def` or `async def` before the names of methods, as appropriate. \ No newline at end of file diff --git a/docs/sso_mapping_providers.md b/docs/sso_mapping_providers.md index 7a407012e0..7b4ddc5b74 100644 --- a/docs/sso_mapping_providers.md +++ b/docs/sso_mapping_providers.md @@ -49,12 +49,12 @@ comment these options out and use those specified by the module instead. A custom mapping provider must specify the following methods: -* `__init__(self, parsed_config)` +* `def __init__(self, parsed_config)` - Arguments: - `parsed_config` - A configuration object that is the return value of the `parse_config` method. You should set any configuration options needed by the module here. -* `parse_config(config)` +* `def parse_config(config)` - This method should have the `@staticmethod` decoration. - Arguments: - `config` - A `dict` representing the parsed content of the @@ -63,13 +63,13 @@ A custom mapping provider must specify the following methods: any option values they need here. - Whatever is returned will be passed back to the user mapping provider module's `__init__` method during construction. -* `get_remote_user_id(self, userinfo)` +* `def get_remote_user_id(self, userinfo)` - Arguments: - `userinfo` - A `authlib.oidc.core.claims.UserInfo` object to extract user information from. - This method must return a string, which is the unique, immutable identifier for the user. Commonly the `sub` claim of the response. -* `map_user_attributes(self, userinfo, token, failures)` +* `async def map_user_attributes(self, userinfo, token, failures)` - This method must be async. - Arguments: - `userinfo` - A `authlib.oidc.core.claims.UserInfo` object to extract user @@ -91,7 +91,7 @@ A custom mapping provider must specify the following methods: during a user's first login. Once a localpart has been associated with a remote user ID (see `get_remote_user_id`) it cannot be updated. - `displayname`: An optional string, the display name for the user. -* `get_extra_attributes(self, userinfo, token)` +* `async def get_extra_attributes(self, userinfo, token)` - This method must be async. - Arguments: - `userinfo` - A `authlib.oidc.core.claims.UserInfo` object to extract user @@ -125,15 +125,15 @@ comment these options out and use those specified by the module instead. A custom mapping provider must specify the following methods: -* `__init__(self, parsed_config, module_api)` +* `def __init__(self, parsed_config, module_api)` - Arguments: - `parsed_config` - A configuration object that is the return value of the `parse_config` method. You should set any configuration options needed by the module here. - `module_api` - a `synapse.module_api.ModuleApi` object which provides the stable API available for extension modules. -* `parse_config(config)` - - This method should have the `@staticmethod` decoration. +* `def parse_config(config)` + - **This method should have the `@staticmethod` decoration.** - Arguments: - `config` - A `dict` representing the parsed content of the `saml_config.user_mapping_provider.config` homeserver config option. @@ -141,15 +141,15 @@ A custom mapping provider must specify the following methods: any option values they need here. - Whatever is returned will be passed back to the user mapping provider module's `__init__` method during construction. -* `get_saml_attributes(config)` - - This method should have the `@staticmethod` decoration. +* `def get_saml_attributes(config)` + - **This method should have the `@staticmethod` decoration.** - Arguments: - `config` - A object resulting from a call to `parse_config`. - Returns a tuple of two sets. The first set equates to the SAML auth response attributes that are required for the module to function, whereas the second set consists of those attributes which can be used if available, but are not necessary. -* `get_remote_user_id(self, saml_response, client_redirect_url)` +* `def get_remote_user_id(self, saml_response, client_redirect_url)` - Arguments: - `saml_response` - A `saml2.response.AuthnResponse` object to extract user information from. @@ -157,7 +157,7 @@ A custom mapping provider must specify the following methods: redirected to. - This method must return a string, which is the unique, immutable identifier for the user. Commonly the `uid` claim of the response. -* `saml_response_to_user_attributes(self, saml_response, failures, client_redirect_url)` +* `def saml_response_to_user_attributes(self, saml_response, failures, client_redirect_url)` - Arguments: - `saml_response` - A `saml2.response.AuthnResponse` object to extract user information from. From 84bfe47b01402cf037417f0026ed4077223ed259 Mon Sep 17 00:00:00 2001 From: reivilibre Date: Wed, 5 Jan 2022 11:41:49 +0000 Subject: [PATCH 41/76] Re-apply: Move glob_to_regex and re_word_boundary to matrix-python-common #11505 (#11687) Co-authored-by: Sean Quah --- changelog.d/11505.misc | 1 + changelog.d/11687.misc | 1 + synapse/config/room_directory.py | 3 +- synapse/config/tls.py | 3 +- synapse/federation/federation_server.py | 3 +- synapse/push/push_rule_evaluator.py | 7 +-- synapse/python_dependencies.py | 1 + synapse/util/__init__.py | 59 +------------------------ tests/util/test_glob_to_regex.py | 59 ------------------------- 9 files changed, 14 insertions(+), 123 deletions(-) create mode 100644 changelog.d/11505.misc create mode 100644 changelog.d/11687.misc delete mode 100644 tests/util/test_glob_to_regex.py diff --git a/changelog.d/11505.misc b/changelog.d/11505.misc new file mode 100644 index 0000000000..926b562fad --- /dev/null +++ b/changelog.d/11505.misc @@ -0,0 +1 @@ +Move `glob_to_regex` and `re_word_boundary` to `matrix-python-common`. diff --git a/changelog.d/11687.misc b/changelog.d/11687.misc new file mode 100644 index 0000000000..926b562fad --- /dev/null +++ b/changelog.d/11687.misc @@ -0,0 +1 @@ +Move `glob_to_regex` and `re_word_boundary` to `matrix-python-common`. diff --git a/synapse/config/room_directory.py b/synapse/config/room_directory.py index 57316c59b6..3c5e0f7ce7 100644 --- a/synapse/config/room_directory.py +++ b/synapse/config/room_directory.py @@ -15,8 +15,9 @@ from typing import List +from matrix_common.regex import glob_to_regex + from synapse.types import JsonDict -from synapse.util import glob_to_regex from ._base import Config, ConfigError diff --git a/synapse/config/tls.py b/synapse/config/tls.py index ffb316e4c0..6e673d65a7 100644 --- a/synapse/config/tls.py +++ b/synapse/config/tls.py @@ -16,11 +16,12 @@ import logging import os from typing import List, Optional, Pattern +from matrix_common.regex import glob_to_regex + from OpenSSL import SSL, crypto from twisted.internet._sslverify import Certificate, trustRootFromCertificates from synapse.config._base import Config, ConfigError -from synapse.util import glob_to_regex logger = logging.getLogger(__name__) diff --git a/synapse/federation/federation_server.py b/synapse/federation/federation_server.py index cf067b56c6..ee71f289c8 100644 --- a/synapse/federation/federation_server.py +++ b/synapse/federation/federation_server.py @@ -28,6 +28,7 @@ from typing import ( Union, ) +from matrix_common.regex import glob_to_regex from prometheus_client import Counter, Gauge, Histogram from twisted.internet.abstract import isIPAddress @@ -65,7 +66,7 @@ from synapse.replication.http.federation import ( ) from synapse.storage.databases.main.lock import Lock from synapse.types import JsonDict, get_domain_from_id -from synapse.util import glob_to_regex, json_decoder, unwrapFirstError +from synapse.util import json_decoder, unwrapFirstError from synapse.util.async_helpers import Linearizer, concurrently_execute, gather_results from synapse.util.caches.response_cache import ResponseCache from synapse.util.stringutils import parse_server_name diff --git a/synapse/push/push_rule_evaluator.py b/synapse/push/push_rule_evaluator.py index 7f68092ec5..659a53805d 100644 --- a/synapse/push/push_rule_evaluator.py +++ b/synapse/push/push_rule_evaluator.py @@ -17,9 +17,10 @@ import logging import re from typing import Any, Dict, List, Optional, Pattern, Tuple, Union +from matrix_common.regex import glob_to_regex, to_word_pattern + from synapse.events import EventBase from synapse.types import JsonDict, UserID -from synapse.util import glob_to_regex, re_word_boundary from synapse.util.caches.lrucache import LruCache logger = logging.getLogger(__name__) @@ -184,7 +185,7 @@ class PushRuleEvaluatorForEvent: r = regex_cache.get((display_name, False, True), None) if not r: r1 = re.escape(display_name) - r1 = re_word_boundary(r1) + r1 = to_word_pattern(r1) r = re.compile(r1, flags=re.IGNORECASE) regex_cache[(display_name, False, True)] = r @@ -213,7 +214,7 @@ def _glob_matches(glob: str, value: str, word_boundary: bool = False) -> bool: try: r = regex_cache.get((glob, True, word_boundary), None) if not r: - r = glob_to_regex(glob, word_boundary) + r = glob_to_regex(glob, word_boundary=word_boundary) regex_cache[(glob, True, word_boundary)] = r return bool(r.search(value)) except re.error: diff --git a/synapse/python_dependencies.py b/synapse/python_dependencies.py index 13fb69460e..d844fbb3b3 100644 --- a/synapse/python_dependencies.py +++ b/synapse/python_dependencies.py @@ -88,6 +88,7 @@ REQUIREMENTS = [ # with the latest security patches. "cryptography>=3.4.7", "ijson>=3.1", + "matrix-common==1.0.0", ] CONDITIONAL_REQUIREMENTS = { diff --git a/synapse/util/__init__.py b/synapse/util/__init__.py index 95f23e27b6..f157132210 100644 --- a/synapse/util/__init__.py +++ b/synapse/util/__init__.py @@ -14,9 +14,8 @@ import json import logging -import re import typing -from typing import Any, Callable, Dict, Generator, Optional, Pattern +from typing import Any, Callable, Dict, Generator, Optional import attr from frozendict import frozendict @@ -35,9 +34,6 @@ if typing.TYPE_CHECKING: logger = logging.getLogger(__name__) -_WILDCARD_RUN = re.compile(r"([\?\*]+)") - - def _reject_invalid_json(val: Any) -> None: """Do not allow Infinity, -Infinity, or NaN values in JSON.""" raise ValueError("Invalid JSON value: '%s'" % val) @@ -185,56 +181,3 @@ def log_failure( if not consumeErrors: return failure return None - - -def glob_to_regex(glob: str, word_boundary: bool = False) -> Pattern: - """Converts a glob to a compiled regex object. - - Args: - glob: pattern to match - word_boundary: If True, the pattern will be allowed to match at word boundaries - anywhere in the string. Otherwise, the pattern is anchored at the start and - end of the string. - - Returns: - compiled regex pattern - """ - - # Patterns with wildcards must be simplified to avoid performance cliffs - # - The glob `?**?**?` is equivalent to the glob `???*` - # - The glob `???*` is equivalent to the regex `.{3,}` - chunks = [] - for chunk in _WILDCARD_RUN.split(glob): - # No wildcards? re.escape() - if not _WILDCARD_RUN.match(chunk): - chunks.append(re.escape(chunk)) - continue - - # Wildcards? Simplify. - qmarks = chunk.count("?") - if "*" in chunk: - chunks.append(".{%d,}" % qmarks) - else: - chunks.append(".{%d}" % qmarks) - - res = "".join(chunks) - - if word_boundary: - res = re_word_boundary(res) - else: - # \A anchors at start of string, \Z at end of string - res = r"\A" + res + r"\Z" - - return re.compile(res, re.IGNORECASE) - - -def re_word_boundary(r: str) -> str: - """ - Adds word boundary characters to the start and end of an - expression to require that the match occur as a whole word, - but do so respecting the fact that strings starting or ending - with non-word characters will change word boundaries. - """ - # we can't use \b as it chokes on unicode. however \W seems to be okay - # as shorthand for [^0-9A-Za-z_]. - return r"(^|\W)%s(\W|$)" % (r,) diff --git a/tests/util/test_glob_to_regex.py b/tests/util/test_glob_to_regex.py deleted file mode 100644 index 220accb92b..0000000000 --- a/tests/util/test_glob_to_regex.py +++ /dev/null @@ -1,59 +0,0 @@ -# Copyright 2021 The Matrix.org Foundation C.I.C. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -from synapse.util import glob_to_regex - -from tests.unittest import TestCase - - -class GlobToRegexTestCase(TestCase): - def test_literal_match(self): - """patterns without wildcards should match""" - pat = glob_to_regex("foobaz") - self.assertTrue( - pat.match("FoobaZ"), "patterns should match and be case-insensitive" - ) - self.assertFalse( - pat.match("x foobaz"), "pattern should not match at word boundaries" - ) - - def test_wildcard_match(self): - pat = glob_to_regex("f?o*baz") - - self.assertTrue( - pat.match("FoobarbaZ"), - "* should match string and pattern should be case-insensitive", - ) - self.assertTrue(pat.match("foobaz"), "* should match 0 characters") - self.assertFalse(pat.match("fooxaz"), "the character after * must match") - self.assertFalse(pat.match("fobbaz"), "? should not match 0 characters") - self.assertFalse(pat.match("fiiobaz"), "? should not match 2 characters") - - def test_multi_wildcard(self): - """patterns with multiple wildcards in a row should match""" - pat = glob_to_regex("**baz") - self.assertTrue(pat.match("agsgsbaz"), "** should match any string") - self.assertTrue(pat.match("baz"), "** should match the empty string") - self.assertEqual(pat.pattern, r"\A.{0,}baz\Z") - - pat = glob_to_regex("*?baz") - self.assertTrue(pat.match("agsgsbaz"), "*? should match any string") - self.assertTrue(pat.match("abaz"), "*? should match a single char") - self.assertFalse(pat.match("baz"), "*? should not match the empty string") - self.assertEqual(pat.pattern, r"\A.{1,}baz\Z") - - pat = glob_to_regex("a?*?*?baz") - self.assertTrue(pat.match("a g baz"), "?*?*? should match 3 chars") - self.assertFalse(pat.match("a..baz"), "?*?*? should not match 2 chars") - self.assertTrue(pat.match("a.gg.baz"), "?*?*? should match 4 chars") - self.assertEqual(pat.pattern, r"\Aa.{3,}baz\Z") From 7a1cefc6e37aa583647f2804c9d9c9765712c59a Mon Sep 17 00:00:00 2001 From: Dirk Klimpel <5740567+dklimpel@users.noreply.github.com> Date: Wed, 5 Jan 2022 12:49:06 +0100 Subject: [PATCH 42/76] Add admin API to get users' account data (#11664) Co-authored-by: reivilibre --- changelog.d/11664.feature | 1 + docs/admin_api/user_admin_api.md | 75 ++++++++++++++++++++++++++ synapse/rest/admin/__init__.py | 2 + synapse/rest/admin/users.py | 30 +++++++++++ tests/rest/admin/test_user.py | 90 ++++++++++++++++++++++++++++++++ 5 files changed, 198 insertions(+) create mode 100644 changelog.d/11664.feature diff --git a/changelog.d/11664.feature b/changelog.d/11664.feature new file mode 100644 index 0000000000..df81783c66 --- /dev/null +++ b/changelog.d/11664.feature @@ -0,0 +1 @@ +Add admin API to get users' account data. \ No newline at end of file diff --git a/docs/admin_api/user_admin_api.md b/docs/admin_api/user_admin_api.md index ba574d795f..74933d2fcf 100644 --- a/docs/admin_api/user_admin_api.md +++ b/docs/admin_api/user_admin_api.md @@ -480,6 +480,81 @@ The following fields are returned in the JSON response body: - `joined_rooms` - An array of `room_id`. - `total` - Number of rooms. +## Account Data +Gets information about account data for a specific `user_id`. + +The API is: + +``` +GET /_synapse/admin/v1/users//accountdata +``` + +A response body like the following is returned: + +```json +{ + "account_data": { + "global": { + "m.secret_storage.key.LmIGHTg5W": { + "algorithm": "m.secret_storage.v1.aes-hmac-sha2", + "iv": "fwjNZatxg==", + "mac": "eWh9kNnLWZUNOgnc=" + }, + "im.vector.hide_profile": { + "hide_profile": true + }, + "org.matrix.preview_urls": { + "disable": false + }, + "im.vector.riot.breadcrumb_rooms": { + "rooms": [ + "!LxcBDAsDUVAfJDEo:matrix.org", + "!MAhRxqasbItjOqxu:matrix.org" + ] + }, + "m.accepted_terms": { + "accepted": [ + "https://example.org/somewhere/privacy-1.2-en.html", + "https://example.org/somewhere/terms-2.0-en.html" + ] + }, + "im.vector.setting.breadcrumbs": { + "recent_rooms": [ + "!MAhRxqasbItqxuEt:matrix.org", + "!ZtSaPCawyWtxiImy:matrix.org" + ] + } + }, + "rooms": { + "!GUdfZSHUJibpiVqHYd:matrix.org": { + "m.fully_read": { + "event_id": "$156334540fYIhZ:matrix.org" + } + }, + "!tOZwOOiqwCYQkLhV:matrix.org": { + "m.fully_read": { + "event_id": "$xjsIyp4_NaVl2yPvIZs_k1Jl8tsC_Sp23wjqXPno" + } + } + } + } +} +``` + +**Parameters** + +The following parameters should be set in the URL: + +- `user_id` - fully qualified: for example, `@user:server.com`. + +**Response** + +The following fields are returned in the JSON response body: + +- `account_data` - A map containing the account data for the user + - `global` - A map containing the global account data for the user + - `rooms` - A map containing the account data per room for the user + ## User media ### List media uploaded by a user diff --git a/synapse/rest/admin/__init__.py b/synapse/rest/admin/__init__.py index 701c609c12..465e06772b 100644 --- a/synapse/rest/admin/__init__.py +++ b/synapse/rest/admin/__init__.py @@ -69,6 +69,7 @@ from synapse.rest.admin.server_notice_servlet import SendServerNoticeServlet from synapse.rest.admin.statistics import UserMediaStatisticsRestServlet from synapse.rest.admin.username_available import UsernameAvailableRestServlet from synapse.rest.admin.users import ( + AccountDataRestServlet, AccountValidityRenewServlet, DeactivateAccountRestServlet, PushersRestServlet, @@ -255,6 +256,7 @@ def register_servlets(hs: "HomeServer", http_server: HttpServer) -> None: UserMediaStatisticsRestServlet(hs).register(http_server) EventReportDetailRestServlet(hs).register(http_server) EventReportsRestServlet(hs).register(http_server) + AccountDataRestServlet(hs).register(http_server) PushersRestServlet(hs).register(http_server) MakeRoomAdminRestServlet(hs).register(http_server) ShadowBanRestServlet(hs).register(http_server) diff --git a/synapse/rest/admin/users.py b/synapse/rest/admin/users.py index db678da4cf..78e795c347 100644 --- a/synapse/rest/admin/users.py +++ b/synapse/rest/admin/users.py @@ -1121,3 +1121,33 @@ class RateLimitRestServlet(RestServlet): await self.store.delete_ratelimit_for_user(user_id) return HTTPStatus.OK, {} + + +class AccountDataRestServlet(RestServlet): + """Retrieve the given user's account data""" + + PATTERNS = admin_patterns("/users/(?P[^/]*)/accountdata") + + def __init__(self, hs: "HomeServer"): + self._auth = hs.get_auth() + self._store = hs.get_datastore() + self._is_mine_id = hs.is_mine_id + + async def on_GET( + self, request: SynapseRequest, user_id: str + ) -> Tuple[int, JsonDict]: + await assert_requester_is_admin(self._auth, request) + + if not self._is_mine_id(user_id): + raise SynapseError(HTTPStatus.BAD_REQUEST, "Can only look up local users") + + if not await self._store.get_user_by_id(user_id): + raise NotFoundError("User not found") + + global_data, by_room_data = await self._store.get_account_data_for_user(user_id) + return HTTPStatus.OK, { + "account_data": { + "global": global_data, + "rooms": by_room_data, + }, + } diff --git a/tests/rest/admin/test_user.py b/tests/rest/admin/test_user.py index eea675991c..e0b9fe8e91 100644 --- a/tests/rest/admin/test_user.py +++ b/tests/rest/admin/test_user.py @@ -3883,3 +3883,93 @@ class RateLimitTestCase(unittest.HomeserverTestCase): self.assertEqual(HTTPStatus.OK, channel.code, msg=channel.json_body) self.assertNotIn("messages_per_second", channel.json_body) self.assertNotIn("burst_count", channel.json_body) + + +class AccountDataTestCase(unittest.HomeserverTestCase): + + servlets = [ + synapse.rest.admin.register_servlets, + login.register_servlets, + ] + + def prepare(self, reactor, clock, hs) -> None: + self.store = hs.get_datastore() + + self.admin_user = self.register_user("admin", "pass", admin=True) + self.admin_user_tok = self.login("admin", "pass") + + self.other_user = self.register_user("user", "pass") + self.url = f"/_synapse/admin/v1/users/{self.other_user}/accountdata" + + def test_no_auth(self) -> None: + """Try to get information of a user without authentication.""" + channel = self.make_request("GET", self.url, {}) + + self.assertEqual(HTTPStatus.UNAUTHORIZED, channel.code, msg=channel.json_body) + self.assertEqual(Codes.MISSING_TOKEN, channel.json_body["errcode"]) + + def test_requester_is_no_admin(self) -> None: + """If the user is not a server admin, an error is returned.""" + other_user_token = self.login("user", "pass") + + channel = self.make_request( + "GET", + self.url, + access_token=other_user_token, + ) + + self.assertEqual(HTTPStatus.FORBIDDEN, channel.code, msg=channel.json_body) + self.assertEqual(Codes.FORBIDDEN, channel.json_body["errcode"]) + + def test_user_does_not_exist(self) -> None: + """Tests that a lookup for a user that does not exist returns a 404""" + url = "/_synapse/admin/v1/users/@unknown_person:test/override_ratelimit" + + channel = self.make_request( + "GET", + url, + access_token=self.admin_user_tok, + ) + + self.assertEqual(HTTPStatus.NOT_FOUND, channel.code, msg=channel.json_body) + self.assertEqual(Codes.NOT_FOUND, channel.json_body["errcode"]) + + def test_user_is_not_local(self) -> None: + """Tests that a lookup for a user that is not a local returns a 400""" + url = "/_synapse/admin/v1/users/@unknown_person:unknown_domain/accountdata" + + channel = self.make_request( + "GET", + url, + access_token=self.admin_user_tok, + ) + + self.assertEqual(HTTPStatus.BAD_REQUEST, channel.code, msg=channel.json_body) + self.assertEqual("Can only look up local users", channel.json_body["error"]) + + def test_success(self) -> None: + """Request account data should succeed for an admin.""" + + # add account data + self.get_success( + self.store.add_account_data_for_user(self.other_user, "m.global", {"a": 1}) + ) + self.get_success( + self.store.add_account_data_to_room( + self.other_user, "test_room", "m.per_room", {"b": 2} + ) + ) + + channel = self.make_request( + "GET", + self.url, + access_token=self.admin_user_tok, + ) + self.assertEqual(HTTPStatus.OK, channel.code, msg=channel.json_body) + self.assertEqual( + {"a": 1}, channel.json_body["account_data"]["global"]["m.global"] + ) + self.assertEqual( + {"b": 2}, + channel.json_body["account_data"]["rooms"]["test_room"]["m.per_room"], + ) From 0201c6371cdfa0e8245c59686c131e40384bbac2 Mon Sep 17 00:00:00 2001 From: Fr3shTea <31766876+Fr3shTea@users.noreply.github.com> Date: Wed, 5 Jan 2022 11:59:29 +0000 Subject: [PATCH 43/76] Fix SimpleHttpClient not sending Accept header in `get_json` (#11677) Co-authored-by: reivilibre --- changelog.d/11677.bugfix | 1 + synapse/http/client.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) create mode 100644 changelog.d/11677.bugfix diff --git a/changelog.d/11677.bugfix b/changelog.d/11677.bugfix new file mode 100644 index 0000000000..5691064a30 --- /dev/null +++ b/changelog.d/11677.bugfix @@ -0,0 +1 @@ +Fix wrong variable reference in `SimpleHttpClient.get_json` that results in the absence of the `Accept` header in the request. diff --git a/synapse/http/client.py b/synapse/http/client.py index fbbeceabeb..ca33b45cb2 100644 --- a/synapse/http/client.py +++ b/synapse/http/client.py @@ -588,7 +588,7 @@ class SimpleHttpClient: if headers: actual_headers.update(headers) # type: ignore - body = await self.get_raw(uri, args, headers=headers) + body = await self.get_raw(uri, args, headers=actual_headers) return json_decoder.decode(body.decode("utf-8")) async def put_json( From eedb4527f1524fc3b83cd4838774b04d6f1e3911 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Philipp=20Matthias=20Sch=C3=A4fer?= Date: Wed, 5 Jan 2022 13:16:52 +0100 Subject: [PATCH 44/76] Fix link from generated configuration file to documentation (#11678) Co-authored-by: reivilibre Co-authored-by: reivilibre --- changelog.d/11678.doc | 1 + docs/sample_config.yaml | 2 +- synapse/config/modules.py | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) create mode 100644 changelog.d/11678.doc diff --git a/changelog.d/11678.doc b/changelog.d/11678.doc new file mode 100644 index 0000000000..dff663e782 --- /dev/null +++ b/changelog.d/11678.doc @@ -0,0 +1 @@ +Fix the documentation link in newly-generated configuration files. diff --git a/docs/sample_config.yaml b/docs/sample_config.yaml index 00dfd2c013..810a14b077 100644 --- a/docs/sample_config.yaml +++ b/docs/sample_config.yaml @@ -37,7 +37,7 @@ # Server admins can expand Synapse's functionality with external modules. # -# See https://matrix-org.github.io/synapse/latest/modules.html for more +# See https://matrix-org.github.io/synapse/latest/modules/index.html for more # documentation on how to configure or create custom modules for Synapse. # modules: diff --git a/synapse/config/modules.py b/synapse/config/modules.py index ae0821e5a5..85fb05890d 100644 --- a/synapse/config/modules.py +++ b/synapse/config/modules.py @@ -37,7 +37,7 @@ class ModulesConfig(Config): # Server admins can expand Synapse's functionality with external modules. # - # See https://matrix-org.github.io/synapse/latest/modules.html for more + # See https://matrix-org.github.io/synapse/latest/modules/index.html for more # documentation on how to configure or create custom modules for Synapse. # modules: From 0fb3dd0830e476c0e0b89c3bf6c7855a4129ff11 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff <1389908+richvdh@users.noreply.github.com> Date: Wed, 5 Jan 2022 12:26:11 +0000 Subject: [PATCH 45/76] Refactor the way we set `outlier` (#11634) * `_auth_and_persist_outliers`: mark persisted events as outliers Mark any events that get persisted via `_auth_and_persist_outliers` as, well, outliers. Currently this will be a no-op as everything will already be flagged as an outlier, but I'm going to change that. * `process_remote_join`: stop flagging as outlier The events are now flagged as outliers later on, by `_auth_and_persist_outliers`. * `send_join`: remove `outlier=True` The events created here are returned in the result of `send_join` to `FederationHandler.do_invite_join`. From there they are passed into `FederationEventHandler.process_remote_join`, which passes them to `_auth_and_persist_outliers`... which sets the `outlier` flag. * `get_event_auth`: remove `outlier=True` stop flagging the events returned by `get_event_auth` as outliers. This method is only called by `_get_remote_auth_chain_for_event`, which passes the results into `_auth_and_persist_outliers`, which will flag them as outliers. * `_get_remote_auth_chain_for_event`: remove `outlier=True` we pass all the events into `_auth_and_persist_outliers`, which will now flag the events as outliers. * `_check_sigs_and_hash_and_fetch`: remove unused `outlier` parameter This param is now never set to True, so we can remove it. * `_check_sigs_and_hash_and_fetch_one`: remove unused `outlier` param This is no longer set anywhere, so we can remove it. * `get_pdu`: remove unused `outlier` parameter ... and chase it down into `get_pdu_from_destination_raw`. * `event_from_pdu_json`: remove redundant `outlier` param This is never set to `True`, so can be removed. * changelog * update docstring --- changelog.d/11634.misc | 1 + synapse/federation/federation_base.py | 7 +---- synapse/federation/federation_client.py | 36 ++++--------------------- synapse/handlers/federation_event.py | 15 ++++++----- tests/handlers/test_federation.py | 4 +-- 5 files changed, 16 insertions(+), 47 deletions(-) create mode 100644 changelog.d/11634.misc diff --git a/changelog.d/11634.misc b/changelog.d/11634.misc new file mode 100644 index 0000000000..4069cbc2f4 --- /dev/null +++ b/changelog.d/11634.misc @@ -0,0 +1 @@ +Refactor the way that the `outlier` flag is set on events received over federation. diff --git a/synapse/federation/federation_base.py b/synapse/federation/federation_base.py index 4df90e02d7..addc0bf000 100644 --- a/synapse/federation/federation_base.py +++ b/synapse/federation/federation_base.py @@ -215,15 +215,12 @@ def _is_invite_via_3pid(event: EventBase) -> bool: ) -def event_from_pdu_json( - pdu_json: JsonDict, room_version: RoomVersion, outlier: bool = False -) -> EventBase: +def event_from_pdu_json(pdu_json: JsonDict, room_version: RoomVersion) -> EventBase: """Construct an EventBase from an event json received over federation Args: pdu_json: pdu as received over federation room_version: The version of the room this event belongs to - outlier: True to mark this event as an outlier Raises: SynapseError: if the pdu is missing required fields or is otherwise @@ -247,6 +244,4 @@ def event_from_pdu_json( validate_canonicaljson(pdu_json) event = make_event_from_dict(pdu_json, room_version) - event.internal_metadata.outlier = outlier - return event diff --git a/synapse/federation/federation_client.py b/synapse/federation/federation_client.py index 7353c2b6b1..6ea4edfc71 100644 --- a/synapse/federation/federation_client.py +++ b/synapse/federation/federation_client.py @@ -265,10 +265,7 @@ class FederationClient(FederationBase): room_version = await self.store.get_room_version(room_id) - pdus = [ - event_from_pdu_json(p, room_version, outlier=False) - for p in transaction_data_pdus - ] + pdus = [event_from_pdu_json(p, room_version) for p in transaction_data_pdus] # Check signatures and hash of pdus, removing any from the list that fail checks pdus[:] = await self._check_sigs_and_hash_and_fetch( @@ -282,7 +279,6 @@ class FederationClient(FederationBase): destination: str, event_id: str, room_version: RoomVersion, - outlier: bool = False, timeout: Optional[int] = None, ) -> Optional[EventBase]: """Requests the PDU with given origin and ID from the remote home @@ -292,9 +288,6 @@ class FederationClient(FederationBase): destination: Which homeserver to query event_id: event to fetch room_version: version of the room - outlier: Indicates whether the PDU is an `outlier`, i.e. if - it's from an arbitrary point in the context as opposed to part - of the current block of PDUs. Defaults to `False` timeout: How long to try (in ms) each destination for before moving to the next destination. None indicates no timeout. @@ -316,8 +309,7 @@ class FederationClient(FederationBase): ) pdu_list: List[EventBase] = [ - event_from_pdu_json(p, room_version, outlier=outlier) - for p in transaction_data["pdus"] + event_from_pdu_json(p, room_version) for p in transaction_data["pdus"] ] if pdu_list and pdu_list[0]: @@ -334,7 +326,6 @@ class FederationClient(FederationBase): destinations: Iterable[str], event_id: str, room_version: RoomVersion, - outlier: bool = False, timeout: Optional[int] = None, ) -> Optional[EventBase]: """Requests the PDU with given origin and ID from the remote home @@ -347,9 +338,6 @@ class FederationClient(FederationBase): destinations: Which homeservers to query event_id: event to fetch room_version: version of the room - outlier: Indicates whether the PDU is an `outlier`, i.e. if - it's from an arbitrary point in the context as opposed to part - of the current block of PDUs. Defaults to `False` timeout: How long to try (in ms) each destination for before moving to the next destination. None indicates no timeout. @@ -377,7 +365,6 @@ class FederationClient(FederationBase): destination=destination, event_id=event_id, room_version=room_version, - outlier=outlier, timeout=timeout, ) @@ -435,7 +422,6 @@ class FederationClient(FederationBase): origin: str, pdus: Collection[EventBase], room_version: RoomVersion, - outlier: bool = False, ) -> List[EventBase]: """Takes a list of PDUs and checks the signatures and hashes of each one. If a PDU fails its signature check then we check if we have it in @@ -451,7 +437,6 @@ class FederationClient(FederationBase): origin pdu room_version - outlier: Whether the events are outliers or not Returns: A list of PDUs that have valid signatures and hashes. @@ -466,7 +451,6 @@ class FederationClient(FederationBase): valid_pdu = await self._check_sigs_and_hash_and_fetch_one( pdu=pdu, origin=origin, - outlier=outlier, room_version=room_version, ) @@ -482,7 +466,6 @@ class FederationClient(FederationBase): pdu: EventBase, origin: str, room_version: RoomVersion, - outlier: bool = False, ) -> Optional[EventBase]: """Takes a PDU and checks its signatures and hashes. If the PDU fails its signature check then we check if we have it in the database and if @@ -494,9 +477,6 @@ class FederationClient(FederationBase): origin pdu room_version - outlier: Whether the events are outliers or not - include_none: Whether to include None in the returned list - for events that have failed their checks Returns: The PDU (possibly redacted) if it has valid signatures and hashes. @@ -521,7 +501,6 @@ class FederationClient(FederationBase): destinations=[pdu_origin], event_id=pdu.event_id, room_version=room_version, - outlier=outlier, timeout=10000, ) except SynapseError: @@ -541,13 +520,10 @@ class FederationClient(FederationBase): room_version = await self.store.get_room_version(room_id) - auth_chain = [ - event_from_pdu_json(p, room_version, outlier=True) - for p in res["auth_chain"] - ] + auth_chain = [event_from_pdu_json(p, room_version) for p in res["auth_chain"]] signed_auth = await self._check_sigs_and_hash_and_fetch( - destination, auth_chain, outlier=True, room_version=room_version + destination, auth_chain, room_version=room_version ) return signed_auth @@ -816,7 +792,6 @@ class FederationClient(FederationBase): valid_pdu = await self._check_sigs_and_hash_and_fetch_one( pdu=event, origin=destination, - outlier=True, room_version=room_version, ) @@ -864,7 +839,6 @@ class FederationClient(FederationBase): valid_pdu = await self._check_sigs_and_hash_and_fetch_one( pdu=pdu, origin=destination, - outlier=True, room_version=room_version, ) @@ -1235,7 +1209,7 @@ class FederationClient(FederationBase): ] signed_events = await self._check_sigs_and_hash_and_fetch( - destination, events, outlier=False, room_version=room_version + destination, events, room_version=room_version ) except HttpResponseException as e: if not e.code == 400: diff --git a/synapse/handlers/federation_event.py b/synapse/handlers/federation_event.py index 7c81e3651f..11771f3c9c 100644 --- a/synapse/handlers/federation_event.py +++ b/synapse/handlers/federation_event.py @@ -421,9 +421,6 @@ class FederationEventHandler: Raises: SynapseError if the response is in some way invalid. """ - for e in itertools.chain(auth_events, state): - e.internal_metadata.outlier = True - event_map = {e.event_id: e for e in itertools.chain(auth_events, state)} create_event = None @@ -1194,7 +1191,6 @@ class FederationEventHandler: [destination], event_id, room_version, - outlier=True, ) if event is None: logger.warning( @@ -1223,9 +1219,10 @@ class FederationEventHandler: """Persist a batch of outlier events fetched from remote servers. We first sort the events to make sure that we process each event's auth_events - before the event itself, and then auth and persist them. + before the event itself. - Notifies about the events where appropriate. + We then mark the events as outliers, persist them to the database, and, where + appropriate (eg, an invite), awake the notifier. Params: room_id: the room that the events are meant to be in (though this has @@ -1276,7 +1273,8 @@ class FederationEventHandler: Persists a batch of events where we have (theoretically) already persisted all of their auth events. - Notifies about the events where appropriate. + Marks the events as outliers, auths them, persists them to the database, and, + where appropriate (eg, an invite), awakes the notifier. Params: origin: where the events came from @@ -1314,6 +1312,9 @@ class FederationEventHandler: return None auth.append(ae) + # we're not bothering about room state, so flag the event as an outlier. + event.internal_metadata.outlier = True + context = EventContext.for_outlier() try: validate_event_for_room_version(room_version_obj, event) diff --git a/tests/handlers/test_federation.py b/tests/handlers/test_federation.py index e1557566e4..496b581726 100644 --- a/tests/handlers/test_federation.py +++ b/tests/handlers/test_federation.py @@ -373,9 +373,7 @@ class FederationTestCase(unittest.HomeserverTestCase): destination: str, room_id: str, event_id: str ) -> List[EventBase]: return [ - event_from_pdu_json( - ae.get_pdu_json(), room_version=room_version, outlier=True - ) + event_from_pdu_json(ae.get_pdu_json(), room_version=room_version) for ae in auth_events ] From 9be5aacc2d901f553ec972d0d62738bd42c87037 Mon Sep 17 00:00:00 2001 From: "Olivier Wilkinson (reivilibre)" Date: Wed, 5 Jan 2022 12:39:48 +0000 Subject: [PATCH 46/76] 1.50.0rc1 --- CHANGES.md | 85 +++++++++++++++++++++++++++++++++++++++ changelog.d/10520.misc | 1 - changelog.d/11243.misc | 1 - changelog.d/11267.doc | 1 - changelog.d/11331.misc | 1 - changelog.d/11360.misc | 1 - changelog.d/11378.feature | 1 - changelog.d/11427.doc | 1 - changelog.d/11480.misc | 1 - changelog.d/11487.misc | 1 - changelog.d/11503.misc | 1 - changelog.d/11505.misc | 1 - changelog.d/11516.bugfix | 1 - changelog.d/11520.misc | 1 - changelog.d/11531.misc | 1 - changelog.d/11535.misc | 1 - changelog.d/11536.bugfix | 1 - changelog.d/11538.feature | 1 - changelog.d/11541.misc | 1 - changelog.d/11542.misc | 1 - changelog.d/11543.misc | 1 - changelog.d/11546.misc | 1 - changelog.d/11547.bugfix | 1 - changelog.d/11549.misc | 1 - changelog.d/11550.misc | 1 - changelog.d/11551.misc | 1 - changelog.d/11553.doc | 1 - changelog.d/11555.misc | 1 - changelog.d/11556.misc | 1 - changelog.d/11558.misc | 1 - changelog.d/11560.misc | 1 - changelog.d/11564.misc | 1 - changelog.d/11565.misc | 1 - changelog.d/11566.misc | 1 - changelog.d/11570.misc | 1 - changelog.d/11571.misc | 1 - changelog.d/11574.misc | 1 - changelog.d/11575.misc | 1 - changelog.d/11580.misc | 1 - changelog.d/11582.misc | 1 - changelog.d/11588.removal | 1 - changelog.d/11589.misc | 1 - changelog.d/11590.misc | 1 - changelog.d/11592.bugfix | 1 - changelog.d/11594.misc | 1 - changelog.d/11595.misc | 1 - changelog.d/11596.misc | 1 - changelog.d/11602.bugfix | 1 - changelog.d/11603.misc | 1 - changelog.d/11607.misc | 1 - changelog.d/11618.misc | 1 - changelog.d/11619.misc | 1 - changelog.d/11622.misc | 1 - changelog.d/11623.bugfix | 1 - changelog.d/11632.bugfix | 1 - changelog.d/11633.misc | 1 - changelog.d/11634.misc | 1 - changelog.d/11638.misc | 1 - changelog.d/11640.doc | 1 - changelog.d/11642.bugfix | 1 - changelog.d/11643.misc | 1 - changelog.d/11652.misc | 1 - changelog.d/11653.misc | 1 - changelog.d/11654.misc | 1 - changelog.d/11657.misc | 1 - changelog.d/11664.feature | 1 - changelog.d/11665.misc | 1 - changelog.d/11666.feature | 1 - changelog.d/11677.bugfix | 1 - changelog.d/11678.doc | 1 - changelog.d/11680.doc | 1 - changelog.d/11681.doc | 1 - changelog.d/11687.misc | 1 - debian/changelog | 6 +++ synapse/__init__.py | 2 +- 75 files changed, 92 insertions(+), 73 deletions(-) delete mode 100644 changelog.d/10520.misc delete mode 100644 changelog.d/11243.misc delete mode 100644 changelog.d/11267.doc delete mode 100644 changelog.d/11331.misc delete mode 100644 changelog.d/11360.misc delete mode 100644 changelog.d/11378.feature delete mode 100644 changelog.d/11427.doc delete mode 100644 changelog.d/11480.misc delete mode 100644 changelog.d/11487.misc delete mode 100644 changelog.d/11503.misc delete mode 100644 changelog.d/11505.misc delete mode 100644 changelog.d/11516.bugfix delete mode 100644 changelog.d/11520.misc delete mode 100644 changelog.d/11531.misc delete mode 100644 changelog.d/11535.misc delete mode 100644 changelog.d/11536.bugfix delete mode 100644 changelog.d/11538.feature delete mode 100644 changelog.d/11541.misc delete mode 100644 changelog.d/11542.misc delete mode 100644 changelog.d/11543.misc delete mode 100644 changelog.d/11546.misc delete mode 100644 changelog.d/11547.bugfix delete mode 100644 changelog.d/11549.misc delete mode 100644 changelog.d/11550.misc delete mode 100644 changelog.d/11551.misc delete mode 100644 changelog.d/11553.doc delete mode 100644 changelog.d/11555.misc delete mode 100644 changelog.d/11556.misc delete mode 100644 changelog.d/11558.misc delete mode 100644 changelog.d/11560.misc delete mode 100644 changelog.d/11564.misc delete mode 100644 changelog.d/11565.misc delete mode 100644 changelog.d/11566.misc delete mode 100644 changelog.d/11570.misc delete mode 100644 changelog.d/11571.misc delete mode 100644 changelog.d/11574.misc delete mode 100644 changelog.d/11575.misc delete mode 100644 changelog.d/11580.misc delete mode 100644 changelog.d/11582.misc delete mode 100644 changelog.d/11588.removal delete mode 100644 changelog.d/11589.misc delete mode 100644 changelog.d/11590.misc delete mode 100644 changelog.d/11592.bugfix delete mode 100644 changelog.d/11594.misc delete mode 100644 changelog.d/11595.misc delete mode 100644 changelog.d/11596.misc delete mode 100644 changelog.d/11602.bugfix delete mode 100644 changelog.d/11603.misc delete mode 100644 changelog.d/11607.misc delete mode 100644 changelog.d/11618.misc delete mode 100644 changelog.d/11619.misc delete mode 100644 changelog.d/11622.misc delete mode 100644 changelog.d/11623.bugfix delete mode 100644 changelog.d/11632.bugfix delete mode 100644 changelog.d/11633.misc delete mode 100644 changelog.d/11634.misc delete mode 100644 changelog.d/11638.misc delete mode 100644 changelog.d/11640.doc delete mode 100644 changelog.d/11642.bugfix delete mode 100644 changelog.d/11643.misc delete mode 100644 changelog.d/11652.misc delete mode 100644 changelog.d/11653.misc delete mode 100644 changelog.d/11654.misc delete mode 100644 changelog.d/11657.misc delete mode 100644 changelog.d/11664.feature delete mode 100644 changelog.d/11665.misc delete mode 100644 changelog.d/11666.feature delete mode 100644 changelog.d/11677.bugfix delete mode 100644 changelog.d/11678.doc delete mode 100644 changelog.d/11680.doc delete mode 100644 changelog.d/11681.doc delete mode 100644 changelog.d/11687.misc diff --git a/CHANGES.md b/CHANGES.md index 9f6e29631d..142cf5381e 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,3 +1,88 @@ +Synapse 1.50.0rc1 (2022-01-05) +============================== + +Features +-------- + +- Allow guests to send state events per [MSC3419](https://github.com/matrix-org/matrix-doc/pull/3419). ([\#11378](https://github.com/matrix-org/synapse/issues/11378)) +- Add experimental support for part of MSC3202: allowing application services to masquerade as specific devices. ([\#11538](https://github.com/matrix-org/synapse/issues/11538)) +- Add admin API to get users' account data. ([\#11664](https://github.com/matrix-org/synapse/issues/11664)) +- Include the room topic in the stripped state included with invites and knocking. ([\#11666](https://github.com/matrix-org/synapse/issues/11666)) + + +Bugfixes +-------- + +- Fix a long-standing bug where relations from other rooms could be included in the bundled aggregations of an event. ([\#11516](https://github.com/matrix-org/synapse/issues/11516)) +- Fix a long-standing bug which could cause `AssertionError`s to be written to the log when Synapse was restarted after purging events from the database. ([\#11536](https://github.com/matrix-org/synapse/issues/11536), [\#11642](https://github.com/matrix-org/synapse/issues/11642)) +- Fix a bug introduced in Synapse 1.17.0 where a pusher created for an email with capital letters would fail to be created. ([\#11547](https://github.com/matrix-org/synapse/issues/11547)) +- Fix a long-standing bug where responses included bundled aggregations when they should not, per [MSC2675](https://github.com/matrix-org/matrix-doc/pull/2675). ([\#11592](https://github.com/matrix-org/synapse/issues/11592), [\#11623](https://github.com/matrix-org/synapse/issues/11623)) +- Fix a long-standing bug that some unknown endpoints would return HTML error pages instead of JSON `M_UNRECOGNIZED` errors. ([\#11602](https://github.com/matrix-org/synapse/issues/11602)) +- Fix a bug introduced in Synapse 1.19.3 which could sometimes cause `AssertionError`s when backfilling rooms over federation. ([\#11632](https://github.com/matrix-org/synapse/issues/11632)) +- Fix a bug in `SimpleHttpClient.get_json` that results in the `Accept` request header being absent. ([\#11677](https://github.com/matrix-org/synapse/issues/11677)) + + +Improved Documentation +---------------------- + +- Update Synapse install command for FreeBSD as the package is now prefixed with `py38`. Contributed by @itchychips. ([\#11267](https://github.com/matrix-org/synapse/issues/11267)) +- Document the usage of refresh tokens. ([\#11427](https://github.com/matrix-org/synapse/issues/11427)) +- Add details for how to configure a TURN server when behind a NAT. Contibuted by @AndrewFerr. ([\#11553](https://github.com/matrix-org/synapse/issues/11553)) +- Add references for using Postgres to the Docker documentation. ([\#11640](https://github.com/matrix-org/synapse/issues/11640)) +- Fix the documentation link in newly-generated configuration files. ([\#11678](https://github.com/matrix-org/synapse/issues/11678)) +- Correct the documentation for `nginx` to use a case-sensitive url pattern. Fixes an error introduced in v1.21.0. ([\#11680](https://github.com/matrix-org/synapse/issues/11680)) +- Clarify SSO mapping provider documentation by writing `def` or `async def` before the names of methods, as appropriate. ([\#11681](https://github.com/matrix-org/synapse/issues/11681)) + + +Deprecations and Removals +------------------------- + +- Replace `mock` package by its standard library version. ([\#11588](https://github.com/matrix-org/synapse/issues/11588)) + + +Internal Changes +---------------- + +- Send and handle cross-signing messages using the stable prefix. ([\#10520](https://github.com/matrix-org/synapse/issues/10520)) +- Allow specific, experimental events to be created without `prev_events`. Used by [MSC2716](https://github.com/matrix-org/matrix-doc/pull/2716). ([\#11243](https://github.com/matrix-org/synapse/issues/11243)) +- A test helper (`wait_for_background_updates`) no longer depends on classes defining a `store` property. ([\#11331](https://github.com/matrix-org/synapse/issues/11331)) +- Add type hints to `synapse.appservice`. ([\#11360](https://github.com/matrix-org/synapse/issues/11360)) +- Add missing type hints to `synapse.config` module. ([\#11480](https://github.com/matrix-org/synapse/issues/11480)) +- Add test to ensure we share the same `state_group` across the whole historical batch when using the [MSC2716](https://github.com/matrix-org/matrix-doc/pull/2716) `/batch_send` endpoint. ([\#11487](https://github.com/matrix-org/synapse/issues/11487)) +- Refactor `tests.util.setup_test_homeserver` and `tests.server.setup_test_homeserver`. ([\#11503](https://github.com/matrix-org/synapse/issues/11503)) +- Move `glob_to_regex` and `re_word_boundary` to `matrix-python-common`. ([\#11505](https://github.com/matrix-org/synapse/issues/11505), [\#11687](https://github.com/matrix-org/synapse/issues/11687)) +- Use HTTPStatus constants in place of literals in `tests.rest.client.test_auth`. ([\#11520](https://github.com/matrix-org/synapse/issues/11520)) +- Add a receipt types constant for `m.read`. ([\#11531](https://github.com/matrix-org/synapse/issues/11531)) +- Clean up `synapse.rest.admin`. ([\#11535](https://github.com/matrix-org/synapse/issues/11535)) +- Support unprefixed versions of fallback key property names. ([\#11541](https://github.com/matrix-org/synapse/issues/11541)) +- Add missing `errcode` to `parse_string` and `parse_boolean`. ([\#11542](https://github.com/matrix-org/synapse/issues/11542)) +- Use HTTPStatus constants in place of literals in `synapse.http`. ([\#11543](https://github.com/matrix-org/synapse/issues/11543)) +- Add missing type hints to storage classes. ([\#11546](https://github.com/matrix-org/synapse/issues/11546), [\#11549](https://github.com/matrix-org/synapse/issues/11549), [\#11551](https://github.com/matrix-org/synapse/issues/11551), [\#11555](https://github.com/matrix-org/synapse/issues/11555), [\#11575](https://github.com/matrix-org/synapse/issues/11575), [\#11589](https://github.com/matrix-org/synapse/issues/11589), [\#11594](https://github.com/matrix-org/synapse/issues/11594), [\#11652](https://github.com/matrix-org/synapse/issues/11652), [\#11653](https://github.com/matrix-org/synapse/issues/11653), [\#11654](https://github.com/matrix-org/synapse/issues/11654), [\#11657](https://github.com/matrix-org/synapse/issues/11657)) +- Fix an inaccurate and misleading comment in the `/sync` code. ([\#11550](https://github.com/matrix-org/synapse/issues/11550)) +- Add missing type hints to `synapse.logging.context`. ([\#11556](https://github.com/matrix-org/synapse/issues/11556)) +- Stop populating unused database column `state_events.prev_state`. ([\#11558](https://github.com/matrix-org/synapse/issues/11558)) +- Minor efficiency improvements in event persistence. ([\#11560](https://github.com/matrix-org/synapse/issues/11560)) +- Add some safety checks that storage functions are used correctly. ([\#11564](https://github.com/matrix-org/synapse/issues/11564), [\#11580](https://github.com/matrix-org/synapse/issues/11580)) +- Make `get_device` return `None` if the device doesn't exist rather than raising an exception. ([\#11565](https://github.com/matrix-org/synapse/issues/11565)) +- Split the HTML parsing code from the URL preview resource code. ([\#11566](https://github.com/matrix-org/synapse/issues/11566)) +- Remove redundant `COALESCE()`s around `COUNT()`s in database queries. ([\#11570](https://github.com/matrix-org/synapse/issues/11570)) +- Add missing type hints to `synapse.http`. ([\#11571](https://github.com/matrix-org/synapse/issues/11571)) +- Convert `EventStreamResult` from a `namedtuple` to `attrs` to improve type hints. ([\#11574](https://github.com/matrix-org/synapse/issues/11574)) +- Add [MSC2716](https://github.com/matrix-org/matrix-doc/pull/2716) and [MSC3030](https://github.com/matrix-org/matrix-doc/pull/3030) to `/versions` -> `unstable_features` to detect server support. ([\#11582](https://github.com/matrix-org/synapse/issues/11582)) +- Add type hints to `synapse/tests/rest/admin`. ([\#11590](https://github.com/matrix-org/synapse/issues/11590)) +- Drop end-of-life Python 3.6 and Postgres 9.6 from CI. ([\#11595](https://github.com/matrix-org/synapse/issues/11595)) +- Update black version and run it on all the files. ([\#11596](https://github.com/matrix-org/synapse/issues/11596)) +- Add opentracing type stubs and fix associated mypy errors. ([\#11603](https://github.com/matrix-org/synapse/issues/11603), [\#11622](https://github.com/matrix-org/synapse/issues/11622)) +- Improve opentracing support for requests which use a `ResponseCache`. ([\#11607](https://github.com/matrix-org/synapse/issues/11607)) +- Improve opentracing support for incoming HTTP requests. ([\#11618](https://github.com/matrix-org/synapse/issues/11618)) +- A number of improvements to opentracing support. ([\#11619](https://github.com/matrix-org/synapse/issues/11619)) +- Drop support for Python 3.6 and Ubuntu 18.04. ([\#11633](https://github.com/matrix-org/synapse/issues/11633)) +- Refactor the way that the `outlier` flag is set on events received over federation. ([\#11634](https://github.com/matrix-org/synapse/issues/11634)) +- Improve the error messages from `get_create_event_for_room`. ([\#11638](https://github.com/matrix-org/synapse/issues/11638)) +- Remove redundant `get_current_events_token` method. ([\#11643](https://github.com/matrix-org/synapse/issues/11643)) +- Convert `namedtuples` to `attrs`. ([\#11665](https://github.com/matrix-org/synapse/issues/11665)) + + Synapse 1.49.2 (2021-12-21) =========================== diff --git a/changelog.d/10520.misc b/changelog.d/10520.misc deleted file mode 100644 index a911e165da..0000000000 --- a/changelog.d/10520.misc +++ /dev/null @@ -1 +0,0 @@ -Send and handle cross-signing messages using the stable prefix. diff --git a/changelog.d/11243.misc b/changelog.d/11243.misc deleted file mode 100644 index 5ef7fe16d4..0000000000 --- a/changelog.d/11243.misc +++ /dev/null @@ -1 +0,0 @@ -Allow specific, experimental events to be created without `prev_events`. Used by [MSC2716](https://github.com/matrix-org/matrix-doc/pull/2716). diff --git a/changelog.d/11267.doc b/changelog.d/11267.doc deleted file mode 100644 index 3a720158de..0000000000 --- a/changelog.d/11267.doc +++ /dev/null @@ -1 +0,0 @@ -Update Synapse install command for FreeBSD as the package is now prefixed with `py38`. Contributed by @itchychips. diff --git a/changelog.d/11331.misc b/changelog.d/11331.misc deleted file mode 100644 index 1ab3a6a975..0000000000 --- a/changelog.d/11331.misc +++ /dev/null @@ -1 +0,0 @@ -A test helper (`wait_for_background_updates`) no longer depends on classes defining a `store` property. diff --git a/changelog.d/11360.misc b/changelog.d/11360.misc deleted file mode 100644 index 43e25720c5..0000000000 --- a/changelog.d/11360.misc +++ /dev/null @@ -1 +0,0 @@ -Add type hints to `synapse.appservice`. diff --git a/changelog.d/11378.feature b/changelog.d/11378.feature deleted file mode 100644 index 524bf84f32..0000000000 --- a/changelog.d/11378.feature +++ /dev/null @@ -1 +0,0 @@ -Allow guests to send state events per [MSC3419](https://github.com/matrix-org/matrix-doc/pull/3419). \ No newline at end of file diff --git a/changelog.d/11427.doc b/changelog.d/11427.doc deleted file mode 100644 index 01cdfcf2b7..0000000000 --- a/changelog.d/11427.doc +++ /dev/null @@ -1 +0,0 @@ -Document the usage of refresh tokens. \ No newline at end of file diff --git a/changelog.d/11480.misc b/changelog.d/11480.misc deleted file mode 100644 index aadc938b2b..0000000000 --- a/changelog.d/11480.misc +++ /dev/null @@ -1 +0,0 @@ -Add missing type hints to `synapse.config` module. diff --git a/changelog.d/11487.misc b/changelog.d/11487.misc deleted file mode 100644 index 376b9078be..0000000000 --- a/changelog.d/11487.misc +++ /dev/null @@ -1 +0,0 @@ -Add test to ensure we share the same `state_group` across the whole historical batch when using the [MSC2716](https://github.com/matrix-org/matrix-doc/pull/2716) `/batch_send` endpoint. diff --git a/changelog.d/11503.misc b/changelog.d/11503.misc deleted file mode 100644 index 03a24a9224..0000000000 --- a/changelog.d/11503.misc +++ /dev/null @@ -1 +0,0 @@ -Refactor `tests.util.setup_test_homeserver` and `tests.server.setup_test_homeserver`. \ No newline at end of file diff --git a/changelog.d/11505.misc b/changelog.d/11505.misc deleted file mode 100644 index 926b562fad..0000000000 --- a/changelog.d/11505.misc +++ /dev/null @@ -1 +0,0 @@ -Move `glob_to_regex` and `re_word_boundary` to `matrix-python-common`. diff --git a/changelog.d/11516.bugfix b/changelog.d/11516.bugfix deleted file mode 100644 index 22bba93671..0000000000 --- a/changelog.d/11516.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix a long-standing bug where relations from other rooms could be included in the bundled aggregations of an event. diff --git a/changelog.d/11520.misc b/changelog.d/11520.misc deleted file mode 100644 index 2d84120e19..0000000000 --- a/changelog.d/11520.misc +++ /dev/null @@ -1 +0,0 @@ -Use HTTPStatus constants in place of literals in `tests.rest.client.test_auth`. \ No newline at end of file diff --git a/changelog.d/11531.misc b/changelog.d/11531.misc deleted file mode 100644 index ed6ef3bb3e..0000000000 --- a/changelog.d/11531.misc +++ /dev/null @@ -1 +0,0 @@ -Add a receipt types constant for `m.read`. diff --git a/changelog.d/11535.misc b/changelog.d/11535.misc deleted file mode 100644 index 580ac354ab..0000000000 --- a/changelog.d/11535.misc +++ /dev/null @@ -1 +0,0 @@ -Clean up `synapse.rest.admin`. \ No newline at end of file diff --git a/changelog.d/11536.bugfix b/changelog.d/11536.bugfix deleted file mode 100644 index 4a1b007254..0000000000 --- a/changelog.d/11536.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix a long-standing bug which could cause `AssertionError`s to be written to the log when Synapse was restarted after purging events from the database. diff --git a/changelog.d/11538.feature b/changelog.d/11538.feature deleted file mode 100644 index b6229e2b45..0000000000 --- a/changelog.d/11538.feature +++ /dev/null @@ -1 +0,0 @@ -Add experimental support for MSC3202: allowing application services to masquerade as specific devices. \ No newline at end of file diff --git a/changelog.d/11541.misc b/changelog.d/11541.misc deleted file mode 100644 index 31c72c2a20..0000000000 --- a/changelog.d/11541.misc +++ /dev/null @@ -1 +0,0 @@ -Support unprefixed versions of fallback key property names. diff --git a/changelog.d/11542.misc b/changelog.d/11542.misc deleted file mode 100644 index f614165037..0000000000 --- a/changelog.d/11542.misc +++ /dev/null @@ -1 +0,0 @@ -Add missing `errcode` to `parse_string` and `parse_boolean`. \ No newline at end of file diff --git a/changelog.d/11543.misc b/changelog.d/11543.misc deleted file mode 100644 index 99817d71a4..0000000000 --- a/changelog.d/11543.misc +++ /dev/null @@ -1 +0,0 @@ -Use HTTPStatus constants in place of literals in `synapse.http`. \ No newline at end of file diff --git a/changelog.d/11546.misc b/changelog.d/11546.misc deleted file mode 100644 index d451940bf2..0000000000 --- a/changelog.d/11546.misc +++ /dev/null @@ -1 +0,0 @@ -Add missing type hints to storage classes. diff --git a/changelog.d/11547.bugfix b/changelog.d/11547.bugfix deleted file mode 100644 index 3950c4c8d3..0000000000 --- a/changelog.d/11547.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix a bug introduced in Synapse 1.17.0 where a pusher created for an email with capital letters would fail to be created. diff --git a/changelog.d/11549.misc b/changelog.d/11549.misc deleted file mode 100644 index d451940bf2..0000000000 --- a/changelog.d/11549.misc +++ /dev/null @@ -1 +0,0 @@ -Add missing type hints to storage classes. diff --git a/changelog.d/11550.misc b/changelog.d/11550.misc deleted file mode 100644 index d5577e0b63..0000000000 --- a/changelog.d/11550.misc +++ /dev/null @@ -1 +0,0 @@ -Fix an inaccurate and misleading comment in the `/sync` code. \ No newline at end of file diff --git a/changelog.d/11551.misc b/changelog.d/11551.misc deleted file mode 100644 index d451940bf2..0000000000 --- a/changelog.d/11551.misc +++ /dev/null @@ -1 +0,0 @@ -Add missing type hints to storage classes. diff --git a/changelog.d/11553.doc b/changelog.d/11553.doc deleted file mode 100644 index 810ba16928..0000000000 --- a/changelog.d/11553.doc +++ /dev/null @@ -1 +0,0 @@ -Add details for how to configure a TURN server when behind a NAT. Contibuted by @AndrewFerr. diff --git a/changelog.d/11555.misc b/changelog.d/11555.misc deleted file mode 100644 index d451940bf2..0000000000 --- a/changelog.d/11555.misc +++ /dev/null @@ -1 +0,0 @@ -Add missing type hints to storage classes. diff --git a/changelog.d/11556.misc b/changelog.d/11556.misc deleted file mode 100644 index 53b26aa676..0000000000 --- a/changelog.d/11556.misc +++ /dev/null @@ -1 +0,0 @@ -Add missing type hints to `synapse.logging.context`. diff --git a/changelog.d/11558.misc b/changelog.d/11558.misc deleted file mode 100644 index 7c334f17e0..0000000000 --- a/changelog.d/11558.misc +++ /dev/null @@ -1 +0,0 @@ -Stop populating unused database column `state_events.prev_state`. diff --git a/changelog.d/11560.misc b/changelog.d/11560.misc deleted file mode 100644 index eb968167f5..0000000000 --- a/changelog.d/11560.misc +++ /dev/null @@ -1 +0,0 @@ -Minor efficiency improvements in event persistence. diff --git a/changelog.d/11564.misc b/changelog.d/11564.misc deleted file mode 100644 index 2c48e22de0..0000000000 --- a/changelog.d/11564.misc +++ /dev/null @@ -1 +0,0 @@ -Add some safety checks that storage functions are used correctly. diff --git a/changelog.d/11565.misc b/changelog.d/11565.misc deleted file mode 100644 index ddcafd32cb..0000000000 --- a/changelog.d/11565.misc +++ /dev/null @@ -1 +0,0 @@ -Make `get_device` return `None` if the device doesn't exist rather than raising an exception. diff --git a/changelog.d/11566.misc b/changelog.d/11566.misc deleted file mode 100644 index c48e73cd48..0000000000 --- a/changelog.d/11566.misc +++ /dev/null @@ -1 +0,0 @@ -Split the HTML parsing code from the URL preview resource code. diff --git a/changelog.d/11570.misc b/changelog.d/11570.misc deleted file mode 100644 index d9af8bdb05..0000000000 --- a/changelog.d/11570.misc +++ /dev/null @@ -1 +0,0 @@ -Remove redundant `COALESCE()`s around `COUNT()`s in database queries. diff --git a/changelog.d/11571.misc b/changelog.d/11571.misc deleted file mode 100644 index 4e396b271e..0000000000 --- a/changelog.d/11571.misc +++ /dev/null @@ -1 +0,0 @@ -Add missing type hints to `synapse.http`. diff --git a/changelog.d/11574.misc b/changelog.d/11574.misc deleted file mode 100644 index 2b090a3780..0000000000 --- a/changelog.d/11574.misc +++ /dev/null @@ -1 +0,0 @@ -Convert `EventStreamResult` from a `namedtuple` to `attrs` to improve type hints. diff --git a/changelog.d/11575.misc b/changelog.d/11575.misc deleted file mode 100644 index d451940bf2..0000000000 --- a/changelog.d/11575.misc +++ /dev/null @@ -1 +0,0 @@ -Add missing type hints to storage classes. diff --git a/changelog.d/11580.misc b/changelog.d/11580.misc deleted file mode 100644 index 2c48e22de0..0000000000 --- a/changelog.d/11580.misc +++ /dev/null @@ -1 +0,0 @@ -Add some safety checks that storage functions are used correctly. diff --git a/changelog.d/11582.misc b/changelog.d/11582.misc deleted file mode 100644 index a0291f64e2..0000000000 --- a/changelog.d/11582.misc +++ /dev/null @@ -1 +0,0 @@ -Add [MSC2716](https://github.com/matrix-org/matrix-doc/pull/2716) and [MSC3030](https://github.com/matrix-org/matrix-doc/pull/3030) to `/versions` -> `unstable_features` to detect server support. diff --git a/changelog.d/11588.removal b/changelog.d/11588.removal deleted file mode 100644 index f781021e11..0000000000 --- a/changelog.d/11588.removal +++ /dev/null @@ -1 +0,0 @@ -Replace `mock` package by its standard library version. diff --git a/changelog.d/11589.misc b/changelog.d/11589.misc deleted file mode 100644 index 8e405b9226..0000000000 --- a/changelog.d/11589.misc +++ /dev/null @@ -1 +0,0 @@ -Add missing type hints to storage classes. \ No newline at end of file diff --git a/changelog.d/11590.misc b/changelog.d/11590.misc deleted file mode 100644 index 40e01194df..0000000000 --- a/changelog.d/11590.misc +++ /dev/null @@ -1 +0,0 @@ -Add type hints to `synapse/tests/rest/admin`. \ No newline at end of file diff --git a/changelog.d/11592.bugfix b/changelog.d/11592.bugfix deleted file mode 100644 index 4116e5dd7c..0000000000 --- a/changelog.d/11592.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix a long-standing bug where responses included bundled aggregations when they should not, per [MSC2675](https://github.com/matrix-org/matrix-doc/pull/2675). diff --git a/changelog.d/11594.misc b/changelog.d/11594.misc deleted file mode 100644 index d451940bf2..0000000000 --- a/changelog.d/11594.misc +++ /dev/null @@ -1 +0,0 @@ -Add missing type hints to storage classes. diff --git a/changelog.d/11595.misc b/changelog.d/11595.misc deleted file mode 100644 index 57e54db5a9..0000000000 --- a/changelog.d/11595.misc +++ /dev/null @@ -1 +0,0 @@ -Drop EOL python 3.6 and postgres 9.6 from CI. \ No newline at end of file diff --git a/changelog.d/11596.misc b/changelog.d/11596.misc deleted file mode 100644 index 3064bc632d..0000000000 --- a/changelog.d/11596.misc +++ /dev/null @@ -1 +0,0 @@ -Update black version and run it on all the files. \ No newline at end of file diff --git a/changelog.d/11602.bugfix b/changelog.d/11602.bugfix deleted file mode 100644 index e0dfbf1a15..0000000000 --- a/changelog.d/11602.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix a long-standing bug that some unknown endpoints would return HTML error pages instead of JSON `M_UNRECOGNIZED` errors. diff --git a/changelog.d/11603.misc b/changelog.d/11603.misc deleted file mode 100644 index def24afb8d..0000000000 --- a/changelog.d/11603.misc +++ /dev/null @@ -1 +0,0 @@ -Add opentracing type stubs and fix associated mypy errors. \ No newline at end of file diff --git a/changelog.d/11607.misc b/changelog.d/11607.misc deleted file mode 100644 index e82f467763..0000000000 --- a/changelog.d/11607.misc +++ /dev/null @@ -1 +0,0 @@ -Improve opentracing support for requests which use a `ResponseCache`. diff --git a/changelog.d/11618.misc b/changelog.d/11618.misc deleted file mode 100644 index 4076b30bf7..0000000000 --- a/changelog.d/11618.misc +++ /dev/null @@ -1 +0,0 @@ -Improve opentracing support for incoming HTTP requests. diff --git a/changelog.d/11619.misc b/changelog.d/11619.misc deleted file mode 100644 index 2125cbddd2..0000000000 --- a/changelog.d/11619.misc +++ /dev/null @@ -1 +0,0 @@ -A number of improvements to opentracing support. diff --git a/changelog.d/11622.misc b/changelog.d/11622.misc deleted file mode 100644 index def24afb8d..0000000000 --- a/changelog.d/11622.misc +++ /dev/null @@ -1 +0,0 @@ -Add opentracing type stubs and fix associated mypy errors. \ No newline at end of file diff --git a/changelog.d/11623.bugfix b/changelog.d/11623.bugfix deleted file mode 100644 index 4116e5dd7c..0000000000 --- a/changelog.d/11623.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix a long-standing bug where responses included bundled aggregations when they should not, per [MSC2675](https://github.com/matrix-org/matrix-doc/pull/2675). diff --git a/changelog.d/11632.bugfix b/changelog.d/11632.bugfix deleted file mode 100644 index c73d41652a..0000000000 --- a/changelog.d/11632.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix a bug introduced in Synapse 1.19.3 which could sometimes cause `AssertionError`s when backfilling rooms over federation. diff --git a/changelog.d/11633.misc b/changelog.d/11633.misc deleted file mode 100644 index 73e814e58e..0000000000 --- a/changelog.d/11633.misc +++ /dev/null @@ -1 +0,0 @@ -Drop support for Python 3.6 and Ubuntu 18.04. \ No newline at end of file diff --git a/changelog.d/11634.misc b/changelog.d/11634.misc deleted file mode 100644 index 4069cbc2f4..0000000000 --- a/changelog.d/11634.misc +++ /dev/null @@ -1 +0,0 @@ -Refactor the way that the `outlier` flag is set on events received over federation. diff --git a/changelog.d/11638.misc b/changelog.d/11638.misc deleted file mode 100644 index 76dfb56bda..0000000000 --- a/changelog.d/11638.misc +++ /dev/null @@ -1 +0,0 @@ -Improve the error messages from `get_create_event_for_room`. diff --git a/changelog.d/11640.doc b/changelog.d/11640.doc deleted file mode 100644 index c4773e4f3a..0000000000 --- a/changelog.d/11640.doc +++ /dev/null @@ -1 +0,0 @@ -Add references for using Postgres to the Docker documentation. diff --git a/changelog.d/11642.bugfix b/changelog.d/11642.bugfix deleted file mode 100644 index 4a1b007254..0000000000 --- a/changelog.d/11642.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix a long-standing bug which could cause `AssertionError`s to be written to the log when Synapse was restarted after purging events from the database. diff --git a/changelog.d/11643.misc b/changelog.d/11643.misc deleted file mode 100644 index 1c3b3071f6..0000000000 --- a/changelog.d/11643.misc +++ /dev/null @@ -1 +0,0 @@ -Remove redundant `get_current_events_token` method. diff --git a/changelog.d/11652.misc b/changelog.d/11652.misc deleted file mode 100644 index 8e405b9226..0000000000 --- a/changelog.d/11652.misc +++ /dev/null @@ -1 +0,0 @@ -Add missing type hints to storage classes. \ No newline at end of file diff --git a/changelog.d/11653.misc b/changelog.d/11653.misc deleted file mode 100644 index 8e405b9226..0000000000 --- a/changelog.d/11653.misc +++ /dev/null @@ -1 +0,0 @@ -Add missing type hints to storage classes. \ No newline at end of file diff --git a/changelog.d/11654.misc b/changelog.d/11654.misc deleted file mode 100644 index 8e405b9226..0000000000 --- a/changelog.d/11654.misc +++ /dev/null @@ -1 +0,0 @@ -Add missing type hints to storage classes. \ No newline at end of file diff --git a/changelog.d/11657.misc b/changelog.d/11657.misc deleted file mode 100644 index 8e405b9226..0000000000 --- a/changelog.d/11657.misc +++ /dev/null @@ -1 +0,0 @@ -Add missing type hints to storage classes. \ No newline at end of file diff --git a/changelog.d/11664.feature b/changelog.d/11664.feature deleted file mode 100644 index df81783c66..0000000000 --- a/changelog.d/11664.feature +++ /dev/null @@ -1 +0,0 @@ -Add admin API to get users' account data. \ No newline at end of file diff --git a/changelog.d/11665.misc b/changelog.d/11665.misc deleted file mode 100644 index e7cc8ff23f..0000000000 --- a/changelog.d/11665.misc +++ /dev/null @@ -1 +0,0 @@ -Convert `namedtuples` to `attrs`. diff --git a/changelog.d/11666.feature b/changelog.d/11666.feature deleted file mode 100644 index 6f6b127e22..0000000000 --- a/changelog.d/11666.feature +++ /dev/null @@ -1 +0,0 @@ -Include the room topic in the stripped state included with invites and knocking. diff --git a/changelog.d/11677.bugfix b/changelog.d/11677.bugfix deleted file mode 100644 index 5691064a30..0000000000 --- a/changelog.d/11677.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix wrong variable reference in `SimpleHttpClient.get_json` that results in the absence of the `Accept` header in the request. diff --git a/changelog.d/11678.doc b/changelog.d/11678.doc deleted file mode 100644 index dff663e782..0000000000 --- a/changelog.d/11678.doc +++ /dev/null @@ -1 +0,0 @@ -Fix the documentation link in newly-generated configuration files. diff --git a/changelog.d/11680.doc b/changelog.d/11680.doc deleted file mode 100644 index 09399ad9d0..0000000000 --- a/changelog.d/11680.doc +++ /dev/null @@ -1 +0,0 @@ -Correct the documentation for `nginx` to use a case-sensitive url pattern. Fixes an error introduced in v1.21.0. diff --git a/changelog.d/11681.doc b/changelog.d/11681.doc deleted file mode 100644 index fef70211cd..0000000000 --- a/changelog.d/11681.doc +++ /dev/null @@ -1 +0,0 @@ -Clarify SSO mapping provider documentation by writing `def` or `async def` before the names of methods, as appropriate. \ No newline at end of file diff --git a/changelog.d/11687.misc b/changelog.d/11687.misc deleted file mode 100644 index 926b562fad..0000000000 --- a/changelog.d/11687.misc +++ /dev/null @@ -1 +0,0 @@ -Move `glob_to_regex` and `re_word_boundary` to `matrix-python-common`. diff --git a/debian/changelog b/debian/changelog index ebe3e0cbf9..b54c0ff348 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,9 @@ +matrix-synapse-py3 (1.50.0~rc1) stable; urgency=medium + + * New synapse release 1.50.0~rc1. + + -- Synapse Packaging team Wed, 05 Jan 2022 12:36:17 +0000 + matrix-synapse-py3 (1.49.2) stable; urgency=medium * New synapse release 1.49.2. diff --git a/synapse/__init__.py b/synapse/__init__.py index 95a49c20be..92aec334e6 100644 --- a/synapse/__init__.py +++ b/synapse/__init__.py @@ -47,7 +47,7 @@ try: except ImportError: pass -__version__ = "1.49.2" +__version__ = "1.50.0rc1" if bool(os.environ.get("SYNAPSE_TEST_PATCH_LOG_CONTEXTS", False)): # We import here so that we don't have to install a bunch of deps when From 88a78c6577086527e4569541b09e437a1ca0d1a9 Mon Sep 17 00:00:00 2001 From: David Robertson Date: Wed, 5 Jan 2022 13:33:28 +0000 Subject: [PATCH 47/76] Cache empty responses from `/user/devices` (#11587) If we've never made a request to a remote homeserver, we should cache the response---even if the response is "this user has no devices". --- changelog.d/11587.bugfix | 1 + synapse/handlers/device.py | 10 ++- synapse/storage/databases/main/devices.py | 8 +- tests/handlers/test_e2e_keys.py | 96 +++++++++++++++++++++++ tests/test_utils/__init__.py | 4 +- 5 files changed, 114 insertions(+), 5 deletions(-) create mode 100644 changelog.d/11587.bugfix diff --git a/changelog.d/11587.bugfix b/changelog.d/11587.bugfix new file mode 100644 index 0000000000..ad2b83edf7 --- /dev/null +++ b/changelog.d/11587.bugfix @@ -0,0 +1 @@ +Fix a long-standing bug where Synapse wouldn't cache a response indicating that a remote user has no devices. \ No newline at end of file diff --git a/synapse/handlers/device.py b/synapse/handlers/device.py index 7665425232..b184a48cb1 100644 --- a/synapse/handlers/device.py +++ b/synapse/handlers/device.py @@ -948,8 +948,16 @@ class DeviceListUpdater: devices = [] ignore_devices = True else: + prev_stream_id = await self.store.get_device_list_last_stream_id_for_remote( + user_id + ) cached_devices = await self.store.get_cached_devices_for_user(user_id) - if cached_devices == {d["device_id"]: d for d in devices}: + + # To ensure that a user with no devices is cached, we skip the resync only + # if we have a stream_id from previously writing a cache entry. + if prev_stream_id is not None and cached_devices == { + d["device_id"]: d for d in devices + }: logging.info( "Skipping device list resync for %s, as our cache matches already", user_id, diff --git a/synapse/storage/databases/main/devices.py b/synapse/storage/databases/main/devices.py index 273adb61fd..52fbf50db6 100644 --- a/synapse/storage/databases/main/devices.py +++ b/synapse/storage/databases/main/devices.py @@ -713,7 +713,7 @@ class DeviceWorkerStore(SQLBaseStore): @cached(max_entries=10000) async def get_device_list_last_stream_id_for_remote( self, user_id: str - ) -> Optional[Any]: + ) -> Optional[str]: """Get the last stream_id we got for a user. May be None if we haven't got any information for them. """ @@ -729,7 +729,9 @@ class DeviceWorkerStore(SQLBaseStore): cached_method_name="get_device_list_last_stream_id_for_remote", list_name="user_ids", ) - async def get_device_list_last_stream_id_for_remotes(self, user_ids: Iterable[str]): + async def get_device_list_last_stream_id_for_remotes( + self, user_ids: Iterable[str] + ) -> Dict[str, Optional[str]]: rows = await self.db_pool.simple_select_many_batch( table="device_lists_remote_extremeties", column="user_id", @@ -1316,6 +1318,7 @@ class DeviceStore(DeviceWorkerStore, DeviceBackgroundUpdateStore): content: JsonDict, stream_id: str, ) -> None: + """Delete, update or insert a cache entry for this (user, device) pair.""" if content.get("deleted"): self.db_pool.simple_delete_txn( txn, @@ -1375,6 +1378,7 @@ class DeviceStore(DeviceWorkerStore, DeviceBackgroundUpdateStore): def _update_remote_device_list_cache_txn( self, txn: LoggingTransaction, user_id: str, devices: List[dict], stream_id: int ) -> None: + """Replace the list of cached devices for this user with the given list.""" self.db_pool.simple_delete_txn( txn, table="device_lists_remote_cache", keyvalues={"user_id": user_id} ) diff --git a/tests/handlers/test_e2e_keys.py b/tests/handlers/test_e2e_keys.py index ddcf3ee348..734ed84d78 100644 --- a/tests/handlers/test_e2e_keys.py +++ b/tests/handlers/test_e2e_keys.py @@ -13,8 +13,10 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +from typing import Iterable from unittest import mock +from parameterized import parameterized from signedjson import key as key, sign as sign from twisted.internet import defer @@ -23,6 +25,7 @@ from synapse.api.constants import RoomEncryptionAlgorithms from synapse.api.errors import Codes, SynapseError from tests import unittest +from tests.test_utils import make_awaitable class E2eKeysHandlerTestCase(unittest.HomeserverTestCase): @@ -765,6 +768,8 @@ class E2eKeysHandlerTestCase(unittest.HomeserverTestCase): remote_user_id = "@test:other" local_user_id = "@test:test" + # Pretend we're sharing a room with the user we're querying. If not, + # `_query_devices_for_destination` will return early. self.store.get_rooms_for_user = mock.Mock( return_value=defer.succeed({"some_room_id"}) ) @@ -831,3 +836,94 @@ class E2eKeysHandlerTestCase(unittest.HomeserverTestCase): } }, ) + + @parameterized.expand( + [ + # The remote homeserver's response indicates that this user has 0/1/2 devices. + ([],), + (["device_1"],), + (["device_1", "device_2"],), + ] + ) + def test_query_all_devices_caches_result(self, device_ids: Iterable[str]): + """Test that requests for all of a remote user's devices are cached. + + We do this by asserting that only one call over federation was made, and that + the two queries to the local homeserver produce the same response. + """ + local_user_id = "@test:test" + remote_user_id = "@test:other" + request_body = {"device_keys": {remote_user_id: []}} + + response_devices = [ + { + "device_id": device_id, + "keys": { + "algorithms": ["dummy"], + "device_id": device_id, + "keys": {f"dummy:{device_id}": "dummy"}, + "signatures": {device_id: {f"dummy:{device_id}": "dummy"}}, + "unsigned": {}, + "user_id": "@test:other", + }, + } + for device_id in device_ids + ] + + response_body = { + "devices": response_devices, + "user_id": remote_user_id, + "stream_id": 12345, # an integer, according to the spec + } + + e2e_handler = self.hs.get_e2e_keys_handler() + + # Pretend we're sharing a room with the user we're querying. If not, + # `_query_devices_for_destination` will return early. + mock_get_rooms = mock.patch.object( + self.store, + "get_rooms_for_user", + new_callable=mock.MagicMock, + return_value=make_awaitable(["some_room_id"]), + ) + mock_request = mock.patch.object( + self.hs.get_federation_client(), + "query_user_devices", + new_callable=mock.MagicMock, + return_value=make_awaitable(response_body), + ) + + with mock_get_rooms, mock_request as mocked_federation_request: + # Make the first query and sanity check it succeeds. + response_1 = self.get_success( + e2e_handler.query_devices( + request_body, + timeout=10, + from_user_id=local_user_id, + from_device_id="some_device_id", + ) + ) + self.assertEqual(response_1["failures"], {}) + + # We should have made a federation request to do so. + mocked_federation_request.assert_called_once() + + # Reset the mock so we can prove we don't make a second federation request. + mocked_federation_request.reset_mock() + + # Repeat the query. + response_2 = self.get_success( + e2e_handler.query_devices( + request_body, + timeout=10, + from_user_id=local_user_id, + from_device_id="some_device_id", + ) + ) + self.assertEqual(response_2["failures"], {}) + + # We should not have made a second federation request. + mocked_federation_request.assert_not_called() + + # The two requests to the local homeserver should be identical. + self.assertEqual(response_1, response_2) diff --git a/tests/test_utils/__init__.py b/tests/test_utils/__init__.py index 15ac2bfeba..f05a373aa0 100644 --- a/tests/test_utils/__init__.py +++ b/tests/test_utils/__init__.py @@ -19,7 +19,7 @@ import sys import warnings from asyncio import Future from binascii import unhexlify -from typing import Any, Awaitable, Callable, TypeVar +from typing import Awaitable, Callable, TypeVar from unittest.mock import Mock import attr @@ -46,7 +46,7 @@ def get_awaitable_result(awaitable: Awaitable[TV]) -> TV: raise Exception("awaitable has not yet completed") -def make_awaitable(result: Any) -> Awaitable[Any]: +def make_awaitable(result: TV) -> Awaitable[TV]: """ Makes an awaitable, suitable for mocking an `async` function. This uses Futures as they can be awaited multiple times so can be returned From 5f98d8e6fd437d172cc0e784e1f33503ee7ead8b Mon Sep 17 00:00:00 2001 From: "Olivier Wilkinson (reivilibre)" Date: Wed, 5 Jan 2022 14:05:15 +0000 Subject: [PATCH 48/76] Tweak changelog --- CHANGES.md | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 142cf5381e..856164df4f 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -5,9 +5,11 @@ Features -------- - Allow guests to send state events per [MSC3419](https://github.com/matrix-org/matrix-doc/pull/3419). ([\#11378](https://github.com/matrix-org/synapse/issues/11378)) -- Add experimental support for part of MSC3202: allowing application services to masquerade as specific devices. ([\#11538](https://github.com/matrix-org/synapse/issues/11538)) +- Add experimental support for part of [MSC3202](https://github.com/matrix-org/matrix-doc/pull/3202): allowing application services to masquerade as specific devices. ([\#11538](https://github.com/matrix-org/synapse/issues/11538)) - Add admin API to get users' account data. ([\#11664](https://github.com/matrix-org/synapse/issues/11664)) - Include the room topic in the stripped state included with invites and knocking. ([\#11666](https://github.com/matrix-org/synapse/issues/11666)) +- Send and handle cross-signing messages using the stable prefix. ([\#10520](https://github.com/matrix-org/synapse/issues/10520)) +- Support unprefixed versions of fallback key property names. ([\#11541](https://github.com/matrix-org/synapse/issues/11541)) Bugfixes @@ -43,7 +45,6 @@ Deprecations and Removals Internal Changes ---------------- -- Send and handle cross-signing messages using the stable prefix. ([\#10520](https://github.com/matrix-org/synapse/issues/10520)) - Allow specific, experimental events to be created without `prev_events`. Used by [MSC2716](https://github.com/matrix-org/matrix-doc/pull/2716). ([\#11243](https://github.com/matrix-org/synapse/issues/11243)) - A test helper (`wait_for_background_updates`) no longer depends on classes defining a `store` property. ([\#11331](https://github.com/matrix-org/synapse/issues/11331)) - Add type hints to `synapse.appservice`. ([\#11360](https://github.com/matrix-org/synapse/issues/11360)) @@ -51,12 +52,11 @@ Internal Changes - Add test to ensure we share the same `state_group` across the whole historical batch when using the [MSC2716](https://github.com/matrix-org/matrix-doc/pull/2716) `/batch_send` endpoint. ([\#11487](https://github.com/matrix-org/synapse/issues/11487)) - Refactor `tests.util.setup_test_homeserver` and `tests.server.setup_test_homeserver`. ([\#11503](https://github.com/matrix-org/synapse/issues/11503)) - Move `glob_to_regex` and `re_word_boundary` to `matrix-python-common`. ([\#11505](https://github.com/matrix-org/synapse/issues/11505), [\#11687](https://github.com/matrix-org/synapse/issues/11687)) -- Use HTTPStatus constants in place of literals in `tests.rest.client.test_auth`. ([\#11520](https://github.com/matrix-org/synapse/issues/11520)) +- Use `HTTPStatus` constants in place of literals in `tests.rest.client.test_auth`. ([\#11520](https://github.com/matrix-org/synapse/issues/11520)) - Add a receipt types constant for `m.read`. ([\#11531](https://github.com/matrix-org/synapse/issues/11531)) - Clean up `synapse.rest.admin`. ([\#11535](https://github.com/matrix-org/synapse/issues/11535)) -- Support unprefixed versions of fallback key property names. ([\#11541](https://github.com/matrix-org/synapse/issues/11541)) - Add missing `errcode` to `parse_string` and `parse_boolean`. ([\#11542](https://github.com/matrix-org/synapse/issues/11542)) -- Use HTTPStatus constants in place of literals in `synapse.http`. ([\#11543](https://github.com/matrix-org/synapse/issues/11543)) +- Use `HTTPStatus` constants in place of literals in `synapse.http`. ([\#11543](https://github.com/matrix-org/synapse/issues/11543)) - Add missing type hints to storage classes. ([\#11546](https://github.com/matrix-org/synapse/issues/11546), [\#11549](https://github.com/matrix-org/synapse/issues/11549), [\#11551](https://github.com/matrix-org/synapse/issues/11551), [\#11555](https://github.com/matrix-org/synapse/issues/11555), [\#11575](https://github.com/matrix-org/synapse/issues/11575), [\#11589](https://github.com/matrix-org/synapse/issues/11589), [\#11594](https://github.com/matrix-org/synapse/issues/11594), [\#11652](https://github.com/matrix-org/synapse/issues/11652), [\#11653](https://github.com/matrix-org/synapse/issues/11653), [\#11654](https://github.com/matrix-org/synapse/issues/11654), [\#11657](https://github.com/matrix-org/synapse/issues/11657)) - Fix an inaccurate and misleading comment in the `/sync` code. ([\#11550](https://github.com/matrix-org/synapse/issues/11550)) - Add missing type hints to `synapse.logging.context`. ([\#11556](https://github.com/matrix-org/synapse/issues/11556)) @@ -67,20 +67,19 @@ Internal Changes - Split the HTML parsing code from the URL preview resource code. ([\#11566](https://github.com/matrix-org/synapse/issues/11566)) - Remove redundant `COALESCE()`s around `COUNT()`s in database queries. ([\#11570](https://github.com/matrix-org/synapse/issues/11570)) - Add missing type hints to `synapse.http`. ([\#11571](https://github.com/matrix-org/synapse/issues/11571)) -- Convert `EventStreamResult` from a `namedtuple` to `attrs` to improve type hints. ([\#11574](https://github.com/matrix-org/synapse/issues/11574)) - Add [MSC2716](https://github.com/matrix-org/matrix-doc/pull/2716) and [MSC3030](https://github.com/matrix-org/matrix-doc/pull/3030) to `/versions` -> `unstable_features` to detect server support. ([\#11582](https://github.com/matrix-org/synapse/issues/11582)) - Add type hints to `synapse/tests/rest/admin`. ([\#11590](https://github.com/matrix-org/synapse/issues/11590)) - Drop end-of-life Python 3.6 and Postgres 9.6 from CI. ([\#11595](https://github.com/matrix-org/synapse/issues/11595)) - Update black version and run it on all the files. ([\#11596](https://github.com/matrix-org/synapse/issues/11596)) - Add opentracing type stubs and fix associated mypy errors. ([\#11603](https://github.com/matrix-org/synapse/issues/11603), [\#11622](https://github.com/matrix-org/synapse/issues/11622)) -- Improve opentracing support for requests which use a `ResponseCache`. ([\#11607](https://github.com/matrix-org/synapse/issues/11607)) -- Improve opentracing support for incoming HTTP requests. ([\#11618](https://github.com/matrix-org/synapse/issues/11618)) +- Improve OpenTracing support for requests which use a `ResponseCache`. ([\#11607](https://github.com/matrix-org/synapse/issues/11607)) +- Improve OpenTracing support for incoming HTTP requests. ([\#11618](https://github.com/matrix-org/synapse/issues/11618)) - A number of improvements to opentracing support. ([\#11619](https://github.com/matrix-org/synapse/issues/11619)) - Drop support for Python 3.6 and Ubuntu 18.04. ([\#11633](https://github.com/matrix-org/synapse/issues/11633)) - Refactor the way that the `outlier` flag is set on events received over federation. ([\#11634](https://github.com/matrix-org/synapse/issues/11634)) - Improve the error messages from `get_create_event_for_room`. ([\#11638](https://github.com/matrix-org/synapse/issues/11638)) - Remove redundant `get_current_events_token` method. ([\#11643](https://github.com/matrix-org/synapse/issues/11643)) -- Convert `namedtuples` to `attrs`. ([\#11665](https://github.com/matrix-org/synapse/issues/11665)) +- Convert `namedtuples` to `attrs`. ([\#11665](https://github.com/matrix-org/synapse/issues/11665), [\#11574](https://github.com/matrix-org/synapse/issues/11574)) Synapse 1.49.2 (2021-12-21) From ffd71029ab8dfef76184ffedd54af6de1c083b81 Mon Sep 17 00:00:00 2001 From: "Olivier Wilkinson (reivilibre)" Date: Wed, 5 Jan 2022 14:10:00 +0000 Subject: [PATCH 49/76] Add support removal notice to the upgrade notes --- docs/upgrade.md | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/docs/upgrade.md b/docs/upgrade.md index 136c806c41..30bb0dcd9c 100644 --- a/docs/upgrade.md +++ b/docs/upgrade.md @@ -85,6 +85,17 @@ process, for example: dpkg -i matrix-synapse-py3_1.3.0+stretch1_amd64.deb ``` +# Upgrading to v1.50.0 + +## Dropping support for old Python and Postgres versions + +In line with our [deprecation policy](deprecation_policy.md), +we've dropped support for Python 3.6 and PostgreSQL 9.6, as they are no +longer supported upstream. + +This release of Synapse requires Python 3.7+ and PostgreSQL 10+. + + # Upgrading to v1.47.0 ## Removal of old Room Admin API From 102f4d3598a387f90ff6a9c5c86a3f4de360931e Mon Sep 17 00:00:00 2001 From: "Olivier Wilkinson (reivilibre)" Date: Wed, 5 Jan 2022 14:14:57 +0000 Subject: [PATCH 50/76] Mention drop of support in changelog --- CHANGES.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGES.md b/CHANGES.md index 856164df4f..e8cd60e9e5 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,6 +1,9 @@ Synapse 1.50.0rc1 (2022-01-05) ============================== +Please note that we now only support Python 3.7+ and PostgreSQL 10+ (if applicable), because Python 3.6 and PostgreSQL 9.6 have reached end-of-life. + + Features -------- From 877b45e8120c5f80ed0965b7c072fd0b8d7ce36a Mon Sep 17 00:00:00 2001 From: Patrick Cloke Date: Wed, 5 Jan 2022 12:08:03 -0500 Subject: [PATCH 51/76] Include `io.element.thread` capability for MSC3440. (#11690) --- changelog.d/11690.misc | 1 + synapse/rest/client/capabilities.py | 3 +++ 2 files changed, 4 insertions(+) create mode 100644 changelog.d/11690.misc diff --git a/changelog.d/11690.misc b/changelog.d/11690.misc new file mode 100644 index 0000000000..76cd286862 --- /dev/null +++ b/changelog.d/11690.misc @@ -0,0 +1 @@ +Update the `/capabilities` response to include whether support for [MSC3440](https://github.com/matrix-org/matrix-doc/pull/3440) is available. diff --git a/synapse/rest/client/capabilities.py b/synapse/rest/client/capabilities.py index 2a3e24ae7e..5c0e3a5680 100644 --- a/synapse/rest/client/capabilities.py +++ b/synapse/rest/client/capabilities.py @@ -73,6 +73,9 @@ class CapabilitiesRestServlet(RestServlet): "enabled": self.config.registration.enable_3pid_changes } + if self.config.experimental.msc3440_enabled: + response["capabilities"]["io.element.thread"] = {"enabled": True} + return 200, response From 83acdb23fe318f8a41d2a6b800d994d46754a903 Mon Sep 17 00:00:00 2001 From: Patrick Cloke Date: Wed, 5 Jan 2022 12:09:15 -0500 Subject: [PATCH 52/76] Re-run towncrier. --- CHANGES.md | 1 + changelog.d/11690.misc | 1 - 2 files changed, 1 insertion(+), 1 deletion(-) delete mode 100644 changelog.d/11690.misc diff --git a/CHANGES.md b/CHANGES.md index e8cd60e9e5..486cfae86f 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -83,6 +83,7 @@ Internal Changes - Improve the error messages from `get_create_event_for_room`. ([\#11638](https://github.com/matrix-org/synapse/issues/11638)) - Remove redundant `get_current_events_token` method. ([\#11643](https://github.com/matrix-org/synapse/issues/11643)) - Convert `namedtuples` to `attrs`. ([\#11665](https://github.com/matrix-org/synapse/issues/11665), [\#11574](https://github.com/matrix-org/synapse/issues/11574)) +- Update the `/capabilities` response to include whether support for [MSC3440](https://github.com/matrix-org/matrix-doc/pull/3440) is available. ([\#11690](https://github.com/matrix-org/synapse/issues/11690)) Synapse 1.49.2 (2021-12-21) diff --git a/changelog.d/11690.misc b/changelog.d/11690.misc deleted file mode 100644 index 76cd286862..0000000000 --- a/changelog.d/11690.misc +++ /dev/null @@ -1 +0,0 @@ -Update the `/capabilities` response to include whether support for [MSC3440](https://github.com/matrix-org/matrix-doc/pull/3440) is available. From d8f94eeec23eba6274896e1aa2f92aa97ff10bee Mon Sep 17 00:00:00 2001 From: Shay Date: Wed, 5 Jan 2022 09:53:05 -0800 Subject: [PATCH 53/76] Run `pyupgrade --py37-plus --keep-percent-format` on Synapse (#11685) * newsfragment * fix newsfragment number * update changelog * remove extra space --- changelog.d/11685.misc | 1 + synapse/rest/admin/media.py | 2 +- synapse/storage/databases/main/session.py | 1 - 3 files changed, 2 insertions(+), 2 deletions(-) create mode 100644 changelog.d/11685.misc diff --git a/changelog.d/11685.misc b/changelog.d/11685.misc new file mode 100644 index 0000000000..c4566b2012 --- /dev/null +++ b/changelog.d/11685.misc @@ -0,0 +1 @@ +Run `pyupgrade --py37-plus --keep-percent-format` on Synapse. diff --git a/synapse/rest/admin/media.py b/synapse/rest/admin/media.py index 7236e4027f..299f5c9eb0 100644 --- a/synapse/rest/admin/media.py +++ b/synapse/rest/admin/media.py @@ -466,7 +466,7 @@ class UserMediaRestServlet(RestServlet): ) deleted_media, total = await self.media_repository.delete_local_media_ids( - ([row["media_id"] for row in media]) + [row["media_id"] for row in media] ) return HTTPStatus.OK, {"deleted_media": deleted_media, "total": total} diff --git a/synapse/storage/databases/main/session.py b/synapse/storage/databases/main/session.py index 5a97120437..e8c776b97a 100644 --- a/synapse/storage/databases/main/session.py +++ b/synapse/storage/databases/main/session.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # Copyright 2021 The Matrix.org Foundation C.I.C. # # Licensed under the Apache License, Version 2.0 (the "License"); From daea7bcc34a0b5c4834fd06d204c620a23eab52c Mon Sep 17 00:00:00 2001 From: "Olivier Wilkinson (reivilibre)" Date: Wed, 5 Jan 2022 18:16:10 +0000 Subject: [PATCH 54/76] Tweak changelog for #11677 --- CHANGES.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index 486cfae86f..77a56dd481 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -24,7 +24,6 @@ Bugfixes - Fix a long-standing bug where responses included bundled aggregations when they should not, per [MSC2675](https://github.com/matrix-org/matrix-doc/pull/2675). ([\#11592](https://github.com/matrix-org/synapse/issues/11592), [\#11623](https://github.com/matrix-org/synapse/issues/11623)) - Fix a long-standing bug that some unknown endpoints would return HTML error pages instead of JSON `M_UNRECOGNIZED` errors. ([\#11602](https://github.com/matrix-org/synapse/issues/11602)) - Fix a bug introduced in Synapse 1.19.3 which could sometimes cause `AssertionError`s when backfilling rooms over federation. ([\#11632](https://github.com/matrix-org/synapse/issues/11632)) -- Fix a bug in `SimpleHttpClient.get_json` that results in the `Accept` request header being absent. ([\#11677](https://github.com/matrix-org/synapse/issues/11677)) Improved Documentation @@ -84,6 +83,7 @@ Internal Changes - Remove redundant `get_current_events_token` method. ([\#11643](https://github.com/matrix-org/synapse/issues/11643)) - Convert `namedtuples` to `attrs`. ([\#11665](https://github.com/matrix-org/synapse/issues/11665), [\#11574](https://github.com/matrix-org/synapse/issues/11574)) - Update the `/capabilities` response to include whether support for [MSC3440](https://github.com/matrix-org/matrix-doc/pull/3440) is available. ([\#11690](https://github.com/matrix-org/synapse/issues/11690)) +- Send the `Accept` header in HTTP requests made using `SimpleHttpClient.get_json`. ([\#11677](https://github.com/matrix-org/synapse/issues/11677)) Synapse 1.49.2 (2021-12-21) From 3b51c763ba5601e155e3e27a46cddf0370da83eb Mon Sep 17 00:00:00 2001 From: Dirk Klimpel <5740567+dklimpel@users.noreply.github.com> Date: Wed, 5 Jan 2022 20:46:50 +0100 Subject: [PATCH 55/76] Fix get federation status of destination if no error occured (#11593) --- changelog.d/11593.bugfix | 1 + synapse/rest/admin/federation.py | 26 +++++-- .../storage/databases/main/transactions.py | 11 +++ tests/rest/admin/test_federation.py | 75 ++++++++++++++----- 4 files changed, 88 insertions(+), 25 deletions(-) create mode 100644 changelog.d/11593.bugfix diff --git a/changelog.d/11593.bugfix b/changelog.d/11593.bugfix new file mode 100644 index 0000000000..963fd0e58e --- /dev/null +++ b/changelog.d/11593.bugfix @@ -0,0 +1 @@ +Fix an error in to get federation status of a destination server even if no error has occurred. This admin API was new introduced in Synapse 1.49.0. diff --git a/synapse/rest/admin/federation.py b/synapse/rest/admin/federation.py index 50d88c9109..8cd3fa189e 100644 --- a/synapse/rest/admin/federation.py +++ b/synapse/rest/admin/federation.py @@ -111,25 +111,37 @@ class DestinationsRestServlet(RestServlet): ) -> Tuple[int, JsonDict]: await assert_requester_is_admin(self._auth, request) + if not await self._store.is_destination_known(destination): + raise NotFoundError("Unknown destination") + destination_retry_timings = await self._store.get_destination_retry_timings( destination ) - if not destination_retry_timings: - raise NotFoundError("Unknown destination") - last_successful_stream_ordering = ( await self._store.get_destination_last_successful_stream_ordering( destination ) ) - response = { + response: JsonDict = { "destination": destination, - "failure_ts": destination_retry_timings.failure_ts, - "retry_last_ts": destination_retry_timings.retry_last_ts, - "retry_interval": destination_retry_timings.retry_interval, "last_successful_stream_ordering": last_successful_stream_ordering, } + if destination_retry_timings: + response = { + **response, + "failure_ts": destination_retry_timings.failure_ts, + "retry_last_ts": destination_retry_timings.retry_last_ts, + "retry_interval": destination_retry_timings.retry_interval, + } + else: + response = { + **response, + "failure_ts": None, + "retry_last_ts": 0, + "retry_interval": 0, + } + return HTTPStatus.OK, response diff --git a/synapse/storage/databases/main/transactions.py b/synapse/storage/databases/main/transactions.py index 6c299cafa5..4b78b4d098 100644 --- a/synapse/storage/databases/main/transactions.py +++ b/synapse/storage/databases/main/transactions.py @@ -560,3 +560,14 @@ class TransactionWorkerStore(CacheInvalidationWorkerStore): return await self.db_pool.runInteraction( "get_destinations_paginate_txn", get_destinations_paginate_txn ) + + async def is_destination_known(self, destination: str) -> bool: + """Check if a destination is known to the server.""" + result = await self.db_pool.simple_select_one_onecol( + table="destinations", + keyvalues={"destination": destination}, + retcol="1", + allow_none=True, + desc="is_destination_known", + ) + return bool(result) diff --git a/tests/rest/admin/test_federation.py b/tests/rest/admin/test_federation.py index 742f194257..b70350b6f1 100644 --- a/tests/rest/admin/test_federation.py +++ b/tests/rest/admin/test_federation.py @@ -314,15 +314,12 @@ class FederationTestCase(unittest.HomeserverTestCase): retry_interval, last_successful_stream_ordering, ) in dest: - self.get_success( - self.store.set_destination_retry_timings( - destination, failure_ts, retry_last_ts, retry_interval - ) - ) - self.get_success( - self.store.set_destination_last_successful_stream_ordering( - destination, last_successful_stream_ordering - ) + self._create_destination( + destination, + failure_ts, + retry_last_ts, + retry_interval, + last_successful_stream_ordering, ) # order by default (destination) @@ -413,11 +410,9 @@ class FederationTestCase(unittest.HomeserverTestCase): _search_test(None, "foo") _search_test(None, "bar") - def test_get_single_destination(self) -> None: - """ - Get one specific destinations. - """ - self._create_destinations(5) + def test_get_single_destination_with_retry_timings(self) -> None: + """Get one specific destination which has retry timings.""" + self._create_destinations(1) channel = self.make_request( "GET", @@ -432,6 +427,53 @@ class FederationTestCase(unittest.HomeserverTestCase): # convert channel.json_body into a List self._check_fields([channel.json_body]) + def test_get_single_destination_no_retry_timings(self) -> None: + """Get one specific destination which has no retry timings.""" + self._create_destination("sub0.example.com") + + channel = self.make_request( + "GET", + self.url + "/sub0.example.com", + access_token=self.admin_user_tok, + ) + + self.assertEqual(HTTPStatus.OK, channel.code, msg=channel.json_body) + self.assertEqual("sub0.example.com", channel.json_body["destination"]) + self.assertEqual(0, channel.json_body["retry_last_ts"]) + self.assertEqual(0, channel.json_body["retry_interval"]) + self.assertIsNone(channel.json_body["failure_ts"]) + self.assertIsNone(channel.json_body["last_successful_stream_ordering"]) + + def _create_destination( + self, + destination: str, + failure_ts: Optional[int] = None, + retry_last_ts: int = 0, + retry_interval: int = 0, + last_successful_stream_ordering: Optional[int] = None, + ) -> None: + """Create one specific destination + + Args: + destination: the destination we have successfully sent to + failure_ts: when the server started failing (ms since epoch) + retry_last_ts: time of last retry attempt in unix epoch ms + retry_interval: how long until next retry in ms + last_successful_stream_ordering: the stream_ordering of the most + recent successfully-sent PDU + """ + self.get_success( + self.store.set_destination_retry_timings( + destination, failure_ts, retry_last_ts, retry_interval + ) + ) + if last_successful_stream_ordering is not None: + self.get_success( + self.store.set_destination_last_successful_stream_ordering( + destination, last_successful_stream_ordering + ) + ) + def _create_destinations(self, number_destinations: int) -> None: """Create a number of destinations @@ -440,10 +482,7 @@ class FederationTestCase(unittest.HomeserverTestCase): """ for i in range(0, number_destinations): dest = f"sub{i}.example.com" - self.get_success(self.store.set_destination_retry_timings(dest, 50, 50, 50)) - self.get_success( - self.store.set_destination_last_successful_stream_ordering(dest, 100) - ) + self._create_destination(dest, 50, 50, 50, 100) def _check_fields(self, content: List[JsonDict]) -> None: """Checks that the expected destination attributes are present in content From feb3e006d7b75d523018980b03a8a827a8440a86 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Wed, 5 Jan 2022 13:33:43 -0700 Subject: [PATCH 56/76] Fix space hierarchy endpoint to match MSC2946 (#11667) Fixes minor discrepancies between the /hierarchy endpoint described in MSC2946 and the implementation. Note that the changes impact the stable and unstable /hierarchy and unstable /spaces endpoints for both client and federation APIs. --- changelog.d/11667.bugfix | 1 + synapse/handlers/room_summary.py | 6 ++++-- 2 files changed, 5 insertions(+), 2 deletions(-) create mode 100644 changelog.d/11667.bugfix diff --git a/changelog.d/11667.bugfix b/changelog.d/11667.bugfix new file mode 100644 index 0000000000..bf65fd4c8b --- /dev/null +++ b/changelog.d/11667.bugfix @@ -0,0 +1 @@ +Fix `/_matrix/client/v1/room/{roomId}/hierarchy` endpoint returning incorrect fields which have been present since Synapse 1.49.0. diff --git a/synapse/handlers/room_summary.py b/synapse/handlers/room_summary.py index b2cfe537df..9ef88feb8a 100644 --- a/synapse/handlers/room_summary.py +++ b/synapse/handlers/room_summary.py @@ -209,7 +209,7 @@ class RoomSummaryHandler: # Before returning to the client, remove the allowed_room_ids # and allowed_spaces keys. room.pop("allowed_room_ids", None) - room.pop("allowed_spaces", None) + room.pop("allowed_spaces", None) # historical rooms_result.append(room) events.extend(room_entry.children_state_events) @@ -988,12 +988,14 @@ class RoomSummaryHandler: "canonical_alias": stats["canonical_alias"], "num_joined_members": stats["joined_members"], "avatar_url": stats["avatar"], + # plural join_rules is a documentation error but kept for historical + # purposes. Should match /publicRooms. "join_rules": stats["join_rules"], + "join_rule": stats["join_rules"], "world_readable": ( stats["history_visibility"] == HistoryVisibility.WORLD_READABLE ), "guest_can_join": stats["guest_access"] == "can_join", - "creation_ts": create_event.origin_server_ts, "room_type": create_event.content.get(EventContentFields.ROOM_TYPE), } From c9eb678b73838c2c90db1232c18b5cf658839609 Mon Sep 17 00:00:00 2001 From: David Robertson Date: Thu, 6 Jan 2022 12:44:36 +0000 Subject: [PATCH 57/76] Remove debug logging for #4422 (#11693) as per https://github.com/matrix-org/synapse/pull/11532#discussion_r769123269 --- changelog.d/11693.misc | 1 + synapse/handlers/sync.py | 38 +------------------------------------- 2 files changed, 2 insertions(+), 37 deletions(-) create mode 100644 changelog.d/11693.misc diff --git a/changelog.d/11693.misc b/changelog.d/11693.misc new file mode 100644 index 0000000000..521a1796b8 --- /dev/null +++ b/changelog.d/11693.misc @@ -0,0 +1 @@ +Remove debug logging for #4422, which has been closed since Synapse 0.99. \ No newline at end of file diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py index 7baf3f199c..4b3f1ea059 100644 --- a/synapse/handlers/sync.py +++ b/synapse/handlers/sync.py @@ -60,10 +60,6 @@ if TYPE_CHECKING: logger = logging.getLogger(__name__) -# Debug logger for https://github.com/matrix-org/synapse/issues/4422 -issue4422_logger = logging.getLogger("synapse.handler.sync.4422_debug") - - # Counts the number of times we returned a non-empty sync. `type` is one of # "initial_sync", "full_state_sync" or "incremental_sync", `lazy_loaded` is # "true" or "false" depending on if the request asked for lazy loaded members or @@ -1161,13 +1157,8 @@ class SyncHandler: num_events = 0 - # debug for https://github.com/matrix-org/synapse/issues/4422 + # debug for https://github.com/matrix-org/synapse/issues/9424 for joined_room in sync_result_builder.joined: - room_id = joined_room.room_id - if room_id in newly_joined_rooms: - issue4422_logger.debug( - "Sync result for newly joined room %s: %r", room_id, joined_room - ) num_events += len(joined_room.timeline.events) log_kv( @@ -1740,18 +1731,6 @@ class SyncHandler: old_mem_ev_id, allow_none=True ) - # debug for #4422 - if has_join: - prev_membership = None - if old_mem_ev: - prev_membership = old_mem_ev.membership - issue4422_logger.debug( - "Previous membership for room %s with join: %s (event %s)", - room_id, - prev_membership, - old_mem_ev_id, - ) - if not old_mem_ev or old_mem_ev.membership != Membership.JOIN: newly_joined_rooms.append(room_id) @@ -1893,13 +1872,6 @@ class SyncHandler: upto_token=since_token, ) - if newly_joined: - # debugging for https://github.com/matrix-org/synapse/issues/4422 - issue4422_logger.debug( - "RoomSyncResultBuilder events for newly joined room %s: %r", - room_id, - entry.events, - ) room_entries.append(entry) return _RoomChanges( @@ -2077,14 +2049,6 @@ class SyncHandler: # `_load_filtered_recents` can't find any events the user should see # (e.g. due to having ignored the sender of the last 50 events). - if newly_joined: - # debug for https://github.com/matrix-org/synapse/issues/4422 - issue4422_logger.debug( - "Timeline events after filtering in newly-joined room %s: %r", - room_id, - batch, - ) - # When we join the room (or the client requests full_state), we should # send down any existing tags. Usually the user won't have tags in a # newly joined room, unless either a) they've joined before or b) the From 2ef1fea8d2021e7a42259d21b57b32e9a7f04f1a Mon Sep 17 00:00:00 2001 From: lukasdenk <63459921+lukasdenk@users.noreply.github.com> Date: Thu, 6 Jan 2022 14:16:42 +0100 Subject: [PATCH 58/76] Make room creations denied by `user_may_create_room` cause an `M_FORBIDDEN` error to be returned, not `M_UNKNOWN` (#11672) Co-authored-by: reivilibre --- changelog.d/11672.feature | 1 + synapse/handlers/room.py | 8 ++++++-- 2 files changed, 7 insertions(+), 2 deletions(-) create mode 100644 changelog.d/11672.feature diff --git a/changelog.d/11672.feature b/changelog.d/11672.feature new file mode 100644 index 0000000000..ce8b3e9547 --- /dev/null +++ b/changelog.d/11672.feature @@ -0,0 +1 @@ +Return an `M_FORBIDDEN` error code instead of `M_UNKNOWN` when a spam checker module prevents a user from creating a room. diff --git a/synapse/handlers/room.py b/synapse/handlers/room.py index b9c1cbffa5..3d3a0f6ac3 100644 --- a/synapse/handlers/room.py +++ b/synapse/handlers/room.py @@ -393,7 +393,9 @@ class RoomCreationHandler: user_id = requester.user.to_string() if not await self.spam_checker.user_may_create_room(user_id): - raise SynapseError(403, "You are not permitted to create rooms") + raise SynapseError( + 403, "You are not permitted to create rooms", Codes.FORBIDDEN + ) creation_content: JsonDict = { "room_version": new_room_version.identifier, @@ -685,7 +687,9 @@ class RoomCreationHandler: invite_3pid_list, ) ): - raise SynapseError(403, "You are not permitted to create rooms") + raise SynapseError( + 403, "You are not permitted to create rooms", Codes.FORBIDDEN + ) if ratelimit: await self.request_ratelimiter.ratelimit(requester) From eec34b1f2a53af45807cb718a26861be3f2fd43c Mon Sep 17 00:00:00 2001 From: reivilibre Date: Thu, 6 Jan 2022 16:36:26 +0000 Subject: [PATCH 59/76] Work around Mjolnir compatibility issue by adding an import for `glob_to_regex` in `synapse.util`, where it moved from. (#11696) --- changelog.d/11696.misc | 1 + synapse/util/__init__.py | 7 +++++++ 2 files changed, 8 insertions(+) create mode 100644 changelog.d/11696.misc diff --git a/changelog.d/11696.misc b/changelog.d/11696.misc new file mode 100644 index 0000000000..e8f39dde18 --- /dev/null +++ b/changelog.d/11696.misc @@ -0,0 +1 @@ +Work around Mjolnir compatibility issue by adding an import for `glob_to_regex` in `synapse.util`, where it moved from. \ No newline at end of file diff --git a/synapse/util/__init__.py b/synapse/util/__init__.py index f157132210..511f52534b 100644 --- a/synapse/util/__init__.py +++ b/synapse/util/__init__.py @@ -31,6 +31,13 @@ from synapse.logging import context if typing.TYPE_CHECKING: pass +# FIXME Mjolnir imports glob_to_regex from this file, but it was moved to +# matrix_common. +# As a temporary workaround, we import glob_to_regex here for +# compatibility with current versions of Mjolnir. +# See https://github.com/matrix-org/mjolnir/pull/174 +from matrix_common.regex import glob_to_regex # noqa + logger = logging.getLogger(__name__) From 70ce9aea7183890cca435e2c4b462e24221d1d2e Mon Sep 17 00:00:00 2001 From: Shay Date: Thu, 6 Jan 2022 09:09:30 -0800 Subject: [PATCH 60/76] Strip unauthorized fields from `unsigned` object in events received over federation (#11530) * add some tests to verify we are stripping unauthorized fields out of unsigned * add function to strip unauthorized fields from the unsigned object of event * newsfragment * update newsfragment number * add check to on_send_membership_event * refactor tests * fix lint error * slightly refactor tests and add some comments * slight refactor * refactor tests * fix import error * slight refactor * remove unsigned filtration code from synapse/handlers/federation_event.py * lint * move unsigned filtering code to event base * refactor tests * update newsfragment * requested changes * remove unused retun values --- changelog.d/11530.bugfix | 2 + synapse/federation/federation_base.py | 25 ++++++++++ tests/test_federation.py | 72 +++++++++++++++++++++++++++ 3 files changed, 99 insertions(+) create mode 100644 changelog.d/11530.bugfix diff --git a/changelog.d/11530.bugfix b/changelog.d/11530.bugfix new file mode 100644 index 0000000000..7ea9ba4e49 --- /dev/null +++ b/changelog.d/11530.bugfix @@ -0,0 +1,2 @@ +Fix a long-standing issue which could cause Synapse to incorrectly accept data in the unsigned field of events +received over federation. \ No newline at end of file diff --git a/synapse/federation/federation_base.py b/synapse/federation/federation_base.py index addc0bf000..896168c05c 100644 --- a/synapse/federation/federation_base.py +++ b/synapse/federation/federation_base.py @@ -230,6 +230,10 @@ def event_from_pdu_json(pdu_json: JsonDict, room_version: RoomVersion) -> EventB # origin, etc etc) assert_params_in_dict(pdu_json, ("type", "depth")) + # Strip any unauthorized values from "unsigned" if they exist + if "unsigned" in pdu_json: + _strip_unsigned_values(pdu_json) + depth = pdu_json["depth"] if not isinstance(depth, int): raise SynapseError(400, "Depth %r not an intger" % (depth,), Codes.BAD_JSON) @@ -245,3 +249,24 @@ def event_from_pdu_json(pdu_json: JsonDict, room_version: RoomVersion) -> EventB event = make_event_from_dict(pdu_json, room_version) return event + + +def _strip_unsigned_values(pdu_dict: JsonDict) -> None: + """ + Strip any unsigned values unless specifically allowed, as defined by the whitelist. + + pdu: the json dict to strip values from. Note that the dict is mutated by this + function + """ + unsigned = pdu_dict["unsigned"] + + if not isinstance(unsigned, dict): + pdu_dict["unsigned"] = {} + + if pdu_dict["type"] == "m.room.member": + whitelist = ["knock_room_state", "invite_room_state", "age"] + else: + whitelist = ["age"] + + filtered_unsigned = {k: v for k, v in unsigned.items() if k in whitelist} + pdu_dict["unsigned"] = filtered_unsigned diff --git a/tests/test_federation.py b/tests/test_federation.py index 3eef1c4c05..2b9804aba0 100644 --- a/tests/test_federation.py +++ b/tests/test_federation.py @@ -17,7 +17,9 @@ from unittest.mock import Mock from twisted.internet.defer import succeed from synapse.api.errors import FederationError +from synapse.api.room_versions import RoomVersions from synapse.events import make_event_from_dict +from synapse.federation.federation_base import event_from_pdu_json from synapse.logging.context import LoggingContext from synapse.types import UserID, create_requester from synapse.util import Clock @@ -276,3 +278,73 @@ class MessageAcceptTests(unittest.HomeserverTestCase): "ed25519:" + remote_self_signing_key in self_signing_key["keys"].keys(), ) self.assertTrue(remote_self_signing_key in self_signing_key["keys"].values()) + + +class StripUnsignedFromEventsTestCase(unittest.TestCase): + def test_strip_unauthorized_unsigned_values(self): + event1 = { + "sender": "@baduser:test.serv", + "state_key": "@baduser:test.serv", + "event_id": "$event1:test.serv", + "depth": 1000, + "origin_server_ts": 1, + "type": "m.room.member", + "origin": "test.servx", + "content": {"membership": "join"}, + "auth_events": [], + "unsigned": {"malicious garbage": "hackz", "more warez": "more hackz"}, + } + filtered_event = event_from_pdu_json(event1, RoomVersions.V1) + # Make sure unauthorized fields are stripped from unsigned + self.assertNotIn("more warez", filtered_event.unsigned) + + def test_strip_event_maintains_allowed_fields(self): + event2 = { + "sender": "@baduser:test.serv", + "state_key": "@baduser:test.serv", + "event_id": "$event2:test.serv", + "depth": 1000, + "origin_server_ts": 1, + "type": "m.room.member", + "origin": "test.servx", + "auth_events": [], + "content": {"membership": "join"}, + "unsigned": { + "malicious garbage": "hackz", + "more warez": "more hackz", + "age": 14, + "invite_room_state": [], + }, + } + + filtered_event2 = event_from_pdu_json(event2, RoomVersions.V1) + self.assertIn("age", filtered_event2.unsigned) + self.assertEqual(14, filtered_event2.unsigned["age"]) + self.assertNotIn("more warez", filtered_event2.unsigned) + # Invite_room_state is allowed in events of type m.room.member + self.assertIn("invite_room_state", filtered_event2.unsigned) + self.assertEqual([], filtered_event2.unsigned["invite_room_state"]) + + def test_strip_event_removes_fields_based_on_event_type(self): + event3 = { + "sender": "@baduser:test.serv", + "state_key": "@baduser:test.serv", + "event_id": "$event3:test.serv", + "depth": 1000, + "origin_server_ts": 1, + "type": "m.room.power_levels", + "origin": "test.servx", + "content": {}, + "auth_events": [], + "unsigned": { + "malicious garbage": "hackz", + "more warez": "more hackz", + "age": 14, + "invite_room_state": [], + }, + } + filtered_event3 = event_from_pdu_json(event3, RoomVersions.V1) + self.assertIn("age", filtered_event3.unsigned) + # Invite_room_state field is only permitted in event type m.room.member + self.assertNotIn("invite_room_state", filtered_event3.unsigned) + self.assertNotIn("more warez", filtered_event3.unsigned) From e87540abb1647b513f6385edccc5164b20f2a6ab Mon Sep 17 00:00:00 2001 From: "Olivier Wilkinson (reivilibre)" Date: Thu, 6 Jan 2022 17:54:21 +0000 Subject: [PATCH 61/76] Re-run Towncrier to add in the changelog entry for the Mjolnir workaround --- CHANGES.md | 1 + changelog.d/11696.misc | 1 - 2 files changed, 1 insertion(+), 1 deletion(-) delete mode 100644 changelog.d/11696.misc diff --git a/CHANGES.md b/CHANGES.md index 77a56dd481..0ba0cb4eca 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -84,6 +84,7 @@ Internal Changes - Convert `namedtuples` to `attrs`. ([\#11665](https://github.com/matrix-org/synapse/issues/11665), [\#11574](https://github.com/matrix-org/synapse/issues/11574)) - Update the `/capabilities` response to include whether support for [MSC3440](https://github.com/matrix-org/matrix-doc/pull/3440) is available. ([\#11690](https://github.com/matrix-org/synapse/issues/11690)) - Send the `Accept` header in HTTP requests made using `SimpleHttpClient.get_json`. ([\#11677](https://github.com/matrix-org/synapse/issues/11677)) +- Work around Mjolnir compatibility issue by adding an import for `glob_to_regex` in `synapse.util`, where it moved from. ([\#11696](https://github.com/matrix-org/synapse/issues/11696)) Synapse 1.49.2 (2021-12-21) diff --git a/changelog.d/11696.misc b/changelog.d/11696.misc deleted file mode 100644 index e8f39dde18..0000000000 --- a/changelog.d/11696.misc +++ /dev/null @@ -1 +0,0 @@ -Work around Mjolnir compatibility issue by adding an import for `glob_to_regex` in `synapse.util`, where it moved from. \ No newline at end of file From 201c48c8de35547c5e13b28a2616a8b7f880bad6 Mon Sep 17 00:00:00 2001 From: Patrick Cloke Date: Thu, 6 Jan 2022 13:08:48 -0500 Subject: [PATCH 62/76] Remove a Python 2-ism and improve type hints. (#11699) On Python 2, indexing a byte-string gives back a byte-string, while on Python 3 it gives back the ASCII equivalent as an int. --- changelog.d/11699.misc | 1 + synapse/types.py | 19 +++++++------------ 2 files changed, 8 insertions(+), 12 deletions(-) create mode 100644 changelog.d/11699.misc diff --git a/changelog.d/11699.misc b/changelog.d/11699.misc new file mode 100644 index 0000000000..ffae5f2960 --- /dev/null +++ b/changelog.d/11699.misc @@ -0,0 +1 @@ +Remove fallback code for Python 2. diff --git a/synapse/types.py b/synapse/types.py index 42aeaf6270..74a2c51857 100644 --- a/synapse/types.py +++ b/synapse/types.py @@ -21,6 +21,7 @@ from typing import ( ClassVar, Dict, Mapping, + Match, MutableMapping, Optional, Tuple, @@ -380,7 +381,7 @@ def map_username_to_mxid_localpart( onto different mxids Returns: - unicode: string suitable for a mxid localpart + string suitable for a mxid localpart """ if not isinstance(username, bytes): username = username.encode("utf-8") @@ -388,29 +389,23 @@ def map_username_to_mxid_localpart( # first we sort out upper-case characters if case_sensitive: - def f1(m): + def f1(m: Match[bytes]) -> bytes: return b"_" + m.group().lower() username = UPPER_CASE_PATTERN.sub(f1, username) else: username = username.lower() - # then we sort out non-ascii characters - def f2(m): - g = m.group()[0] - if isinstance(g, str): - # on python 2, we need to do a ord(). On python 3, the - # byte itself will do. - g = ord(g) - return b"=%02x" % (g,) + # then we sort out non-ascii characters by converting to the hex equivalent. + def f2(m: Match[bytes]) -> bytes: + return b"=%02x" % (m.group()[0],) username = NON_MXID_CHARACTER_PATTERN.sub(f2, username) # we also do the =-escaping to mxids starting with an underscore. username = re.sub(b"^_", b"=5f", username) - # we should now only have ascii bytes left, so can decode back to a - # unicode. + # we should now only have ascii bytes left, so can decode back to a string. return username.decode("ascii") From 6c68e874b1eafe75db51e46064c0d3af702b4358 Mon Sep 17 00:00:00 2001 From: Patrick Cloke Date: Thu, 6 Jan 2022 14:00:34 -0500 Subject: [PATCH 63/76] Remove the /send_relation endpoint. (#11682) This was removed from MSC2674 before that was approved and is not used by any known clients. --- changelog.d/11682.removal | 1 + synapse/rest/client/relations.py | 125 ++-------------------------- tests/rest/client/test_relations.py | 26 +++--- 3 files changed, 19 insertions(+), 133 deletions(-) create mode 100644 changelog.d/11682.removal diff --git a/changelog.d/11682.removal b/changelog.d/11682.removal new file mode 100644 index 0000000000..50bdf35b20 --- /dev/null +++ b/changelog.d/11682.removal @@ -0,0 +1 @@ +Remove the unstable `/send_relation` endpoint. diff --git a/synapse/rest/client/relations.py b/synapse/rest/client/relations.py index 5815650ee6..3823498012 100644 --- a/synapse/rest/client/relations.py +++ b/synapse/rest/client/relations.py @@ -19,28 +19,20 @@ any time to reflect changes in the MSC. """ import logging -from typing import TYPE_CHECKING, Awaitable, Optional, Tuple +from typing import TYPE_CHECKING, Optional, Tuple -from synapse.api.constants import EventTypes, RelationTypes -from synapse.api.errors import ShadowBanError, SynapseError +from synapse.api.constants import RelationTypes +from synapse.api.errors import SynapseError from synapse.http.server import HttpServer -from synapse.http.servlet import ( - RestServlet, - parse_integer, - parse_json_object_from_request, - parse_string, -) +from synapse.http.servlet import RestServlet, parse_integer, parse_string from synapse.http.site import SynapseRequest -from synapse.rest.client.transactions import HttpTransactionCache +from synapse.rest.client._base import client_patterns from synapse.storage.relations import ( AggregationPaginationToken, PaginationChunk, RelationPaginationToken, ) from synapse.types import JsonDict -from synapse.util.stringutils import random_string - -from ._base import client_patterns if TYPE_CHECKING: from synapse.server import HomeServer @@ -48,112 +40,6 @@ if TYPE_CHECKING: logger = logging.getLogger(__name__) -class RelationSendServlet(RestServlet): - """Helper API for sending events that have relation data. - - Example API shape to send a 👍 reaction to a room: - - POST /rooms/!foo/send_relation/$bar/m.annotation/m.reaction?key=%F0%9F%91%8D - {} - - { - "event_id": "$foobar" - } - """ - - PATTERN = ( - "/rooms/(?P[^/]*)/send_relation" - "/(?P[^/]*)/(?P[^/]*)/(?P[^/]*)" - ) - - def __init__(self, hs: "HomeServer"): - super().__init__() - self.auth = hs.get_auth() - self.event_creation_handler = hs.get_event_creation_handler() - self.txns = HttpTransactionCache(hs) - - def register(self, http_server: HttpServer) -> None: - http_server.register_paths( - "POST", - client_patterns(self.PATTERN + "$", releases=()), - self.on_PUT_or_POST, - self.__class__.__name__, - ) - http_server.register_paths( - "PUT", - client_patterns(self.PATTERN + "/(?P[^/]*)$", releases=()), - self.on_PUT, - self.__class__.__name__, - ) - - def on_PUT( - self, - request: SynapseRequest, - room_id: str, - parent_id: str, - relation_type: str, - event_type: str, - txn_id: Optional[str] = None, - ) -> Awaitable[Tuple[int, JsonDict]]: - return self.txns.fetch_or_execute_request( - request, - self.on_PUT_or_POST, - request, - room_id, - parent_id, - relation_type, - event_type, - txn_id, - ) - - async def on_PUT_or_POST( - self, - request: SynapseRequest, - room_id: str, - parent_id: str, - relation_type: str, - event_type: str, - txn_id: Optional[str] = None, - ) -> Tuple[int, JsonDict]: - requester = await self.auth.get_user_by_req(request, allow_guest=True) - - if event_type == EventTypes.Member: - # Add relations to a membership is meaningless, so we just deny it - # at the CS API rather than trying to handle it correctly. - raise SynapseError(400, "Cannot send member events with relations") - - content = parse_json_object_from_request(request) - - aggregation_key = parse_string(request, "key", encoding="utf-8") - - content["m.relates_to"] = { - "event_id": parent_id, - "rel_type": relation_type, - } - if aggregation_key is not None: - content["m.relates_to"]["key"] = aggregation_key - - event_dict = { - "type": event_type, - "content": content, - "room_id": room_id, - "sender": requester.user.to_string(), - } - - try: - ( - event, - _, - ) = await self.event_creation_handler.create_and_send_nonmember_event( - requester, event_dict=event_dict, txn_id=txn_id - ) - event_id = event.event_id - except ShadowBanError: - event_id = "$" + random_string(43) - - return 200, {"event_id": event_id} - - class RelationPaginationServlet(RestServlet): """API to paginate relations on an event by topological ordering, optionally filtered by relation type and event type. @@ -431,7 +317,6 @@ class RelationAggregationGroupPaginationServlet(RestServlet): def register_servlets(hs: "HomeServer", http_server: HttpServer) -> None: - RelationSendServlet(hs).register(http_server) RelationPaginationServlet(hs).register(http_server) RelationAggregationPaginationServlet(hs).register(http_server) RelationAggregationGroupPaginationServlet(hs).register(http_server) diff --git a/tests/rest/client/test_relations.py b/tests/rest/client/test_relations.py index c026d526ef..ff4e81d069 100644 --- a/tests/rest/client/test_relations.py +++ b/tests/rest/client/test_relations.py @@ -93,11 +93,6 @@ class RelationsTestCase(unittest.HomeserverTestCase): channel.json_body, ) - def test_deny_membership(self): - """Test that we deny relations on membership events""" - channel = self._send_relation(RelationTypes.ANNOTATION, EventTypes.Member) - self.assertEquals(400, channel.code, channel.json_body) - def test_deny_invalid_event(self): """Test that we deny relations on non-existant events""" channel = self._send_relation( @@ -1119,7 +1114,8 @@ class RelationsTestCase(unittest.HomeserverTestCase): relation_type: One of `RelationTypes` event_type: The type of the event to create key: The aggregation key used for m.annotation relation type. - content: The content of the created event. + content: The content of the created event. Will be modified to configure + the m.relates_to key based on the other provided parameters. access_token: The access token used to send the relation, defaults to `self.user_token` parent_id: The event_id this relation relates to. If None, then self.parent_id @@ -1130,17 +1126,21 @@ class RelationsTestCase(unittest.HomeserverTestCase): if not access_token: access_token = self.user_token - query = "" - if key: - query = "?key=" + urllib.parse.quote_plus(key.encode("utf-8")) - original_id = parent_id if parent_id else self.parent_id + if content is None: + content = {} + content["m.relates_to"] = { + "event_id": original_id, + "rel_type": relation_type, + } + if key is not None: + content["m.relates_to"]["key"] = key + channel = self.make_request( "POST", - "/_matrix/client/unstable/rooms/%s/send_relation/%s/%s/%s%s" - % (self.room, original_id, relation_type, event_type, query), - content or {}, + f"/_matrix/client/v3/rooms/{self.room}/send/{event_type}", + content, access_token=access_token, ) return channel From 7fe7c454380a39a295aa77e75ab460975e9eb6a9 Mon Sep 17 00:00:00 2001 From: "Olivier Wilkinson (reivilibre)" Date: Fri, 7 Jan 2022 12:51:20 +0000 Subject: [PATCH 64/76] Move changelog entry for drop of support to 'Deprecations and Removals' --- CHANGES.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index 0ba0cb4eca..f91109f885 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -42,6 +42,7 @@ Deprecations and Removals ------------------------- - Replace `mock` package by its standard library version. ([\#11588](https://github.com/matrix-org/synapse/issues/11588)) +- Drop support for Python 3.6 and Ubuntu 18.04. ([\#11633](https://github.com/matrix-org/synapse/issues/11633)) Internal Changes @@ -77,7 +78,6 @@ Internal Changes - Improve OpenTracing support for requests which use a `ResponseCache`. ([\#11607](https://github.com/matrix-org/synapse/issues/11607)) - Improve OpenTracing support for incoming HTTP requests. ([\#11618](https://github.com/matrix-org/synapse/issues/11618)) - A number of improvements to opentracing support. ([\#11619](https://github.com/matrix-org/synapse/issues/11619)) -- Drop support for Python 3.6 and Ubuntu 18.04. ([\#11633](https://github.com/matrix-org/synapse/issues/11633)) - Refactor the way that the `outlier` flag is set on events received over federation. ([\#11634](https://github.com/matrix-org/synapse/issues/11634)) - Improve the error messages from `get_create_event_for_room`. ([\#11638](https://github.com/matrix-org/synapse/issues/11638)) - Remove redundant `get_current_events_token` method. ([\#11643](https://github.com/matrix-org/synapse/issues/11643)) From 6bf81a7a61d8d5248be5def955104c44fcb78dae Mon Sep 17 00:00:00 2001 From: Patrick Cloke Date: Fri, 7 Jan 2022 09:10:46 -0500 Subject: [PATCH 65/76] Bundle aggregations outside of the serialization method. (#11612) This makes the serialization of events synchronous (and it no longer access the database), but we must manually calculate and provide the bundled aggregations. Overall this should cause no change in behavior, but is prep work for other improvements. --- changelog.d/11612.misc | 1 + synapse/events/utils.py | 124 ++++++------------- synapse/handlers/events.py | 2 +- synapse/handlers/initial_sync.py | 16 ++- synapse/handlers/message.py | 2 +- synapse/handlers/pagination.py | 8 +- synapse/handlers/room.py | 10 ++ synapse/handlers/search.py | 10 +- synapse/rest/admin/rooms.py | 16 +-- synapse/rest/client/events.py | 2 +- synapse/rest/client/notifications.py | 2 +- synapse/rest/client/relations.py | 11 +- synapse/rest/client/room.py | 28 +++-- synapse/rest/client/sync.py | 39 +++--- synapse/server.py | 2 +- synapse/storage/databases/main/relations.py | 128 +++++++++++++++++++- tests/rest/client/test_retention.py | 2 +- 17 files changed, 248 insertions(+), 155 deletions(-) create mode 100644 changelog.d/11612.misc diff --git a/changelog.d/11612.misc b/changelog.d/11612.misc new file mode 100644 index 0000000000..2d886169c5 --- /dev/null +++ b/changelog.d/11612.misc @@ -0,0 +1 @@ +Avoid database access in the JSON serialization process. diff --git a/synapse/events/utils.py b/synapse/events/utils.py index 2038e72924..de0e0c1731 100644 --- a/synapse/events/utils.py +++ b/synapse/events/utils.py @@ -14,17 +14,7 @@ # limitations under the License. import collections.abc import re -from typing import ( - TYPE_CHECKING, - Any, - Callable, - Dict, - Iterable, - List, - Mapping, - Optional, - Union, -) +from typing import Any, Callable, Dict, Iterable, List, Mapping, Optional, Union from frozendict import frozendict @@ -32,14 +22,10 @@ from synapse.api.constants import EventContentFields, EventTypes, RelationTypes from synapse.api.errors import Codes, SynapseError from synapse.api.room_versions import RoomVersion from synapse.types import JsonDict -from synapse.util.async_helpers import yieldable_gather_results from synapse.util.frozenutils import unfreeze from . import EventBase -if TYPE_CHECKING: - from synapse.server import HomeServer - # Split strings on "." but not "\." This uses a negative lookbehind assertion for '\' # (? JsonDict: """Serializes a single event. @@ -418,66 +399,41 @@ class EventClientSerializer: serialized_event = serialize_event(event, time_now, **kwargs) # Check if there are any bundled aggregations to include with the event. - # - # Do not bundle aggregations if any of the following at true: - # - # * Support is disabled via the configuration or the caller. - # * The event is a state event. - # * The event has been redacted. - if ( - self._msc1849_enabled - and bundle_aggregations - and not event.is_state() - and not event.internal_metadata.is_redacted() - ): - await self._injected_bundled_aggregations(event, time_now, serialized_event) + if bundle_aggregations: + event_aggregations = bundle_aggregations.get(event.event_id) + if event_aggregations: + self._injected_bundled_aggregations( + event, + time_now, + bundle_aggregations[event.event_id], + serialized_event, + ) return serialized_event - async def _injected_bundled_aggregations( - self, event: EventBase, time_now: int, serialized_event: JsonDict + def _injected_bundled_aggregations( + self, + event: EventBase, + time_now: int, + aggregations: JsonDict, + serialized_event: JsonDict, ) -> None: """Potentially injects bundled aggregations into the unsigned portion of the serialized event. Args: event: The event being serialized. time_now: The current time in milliseconds + aggregations: The bundled aggregation to serialize. serialized_event: The serialized event which may be modified. """ - # Do not bundle aggregations for an event which represents an edit or an - # annotation. It does not make sense for them to have related events. - relates_to = event.content.get("m.relates_to") - if isinstance(relates_to, (dict, frozendict)): - relation_type = relates_to.get("rel_type") - if relation_type in (RelationTypes.ANNOTATION, RelationTypes.REPLACE): - return + # Make a copy in-case the object is cached. + aggregations = aggregations.copy() - event_id = event.event_id - room_id = event.room_id - - # The bundled aggregations to include. - aggregations = {} - - annotations = await self.store.get_aggregation_groups_for_event( - event_id, room_id - ) - if annotations.chunk: - aggregations[RelationTypes.ANNOTATION] = annotations.to_dict() - - references = await self.store.get_relations_for_event( - event_id, room_id, RelationTypes.REFERENCE, direction="f" - ) - if references.chunk: - aggregations[RelationTypes.REFERENCE] = references.to_dict() - - edit = None - if event.type == EventTypes.Message: - edit = await self.store.get_applicable_edit(event_id, room_id) - - if edit: + if RelationTypes.REPLACE in aggregations: # If there is an edit replace the content, preserving existing # relations. + edit = aggregations[RelationTypes.REPLACE] # Ensure we take copies of the edit content, otherwise we risk modifying # the original event. @@ -502,27 +458,19 @@ class EventClientSerializer: } # If this event is the start of a thread, include a summary of the replies. - if self._msc3440_enabled: - ( - thread_count, - latest_thread_event, - ) = await self.store.get_thread_summary(event_id, room_id) - if latest_thread_event: - aggregations[RelationTypes.THREAD] = { - # Don't bundle aggregations as this could recurse forever. - "latest_event": await self.serialize_event( - latest_thread_event, time_now, bundle_aggregations=False - ), - "count": thread_count, - } + if RelationTypes.THREAD in aggregations: + # Serialize the latest thread event. + latest_thread_event = aggregations[RelationTypes.THREAD]["latest_event"] - # If any bundled aggregations were found, include them. - if aggregations: - serialized_event["unsigned"].setdefault("m.relations", {}).update( - aggregations + # Don't bundle aggregations as this could recurse forever. + aggregations[RelationTypes.THREAD]["latest_event"] = self.serialize_event( + latest_thread_event, time_now, bundle_aggregations=None ) - async def serialize_events( + # Include the bundled aggregations in the event. + serialized_event["unsigned"].setdefault("m.relations", {}).update(aggregations) + + def serialize_events( self, events: Iterable[Union[JsonDict, EventBase]], time_now: int, **kwargs: Any ) -> List[JsonDict]: """Serializes multiple events. @@ -535,9 +483,9 @@ class EventClientSerializer: Returns: The list of serialized events """ - return await yieldable_gather_results( - self.serialize_event, events, time_now=time_now, **kwargs - ) + return [ + self.serialize_event(event, time_now=time_now, **kwargs) for event in events + ] def copy_power_levels_contents( diff --git a/synapse/handlers/events.py b/synapse/handlers/events.py index 1b996c420d..a3add8a586 100644 --- a/synapse/handlers/events.py +++ b/synapse/handlers/events.py @@ -119,7 +119,7 @@ class EventStreamHandler: events.extend(to_add) - chunks = await self._event_serializer.serialize_events( + chunks = self._event_serializer.serialize_events( events, time_now, as_client_event=as_client_event, diff --git a/synapse/handlers/initial_sync.py b/synapse/handlers/initial_sync.py index 601bab67f9..346a06ff49 100644 --- a/synapse/handlers/initial_sync.py +++ b/synapse/handlers/initial_sync.py @@ -170,7 +170,7 @@ class InitialSyncHandler: d["inviter"] = event.sender invite_event = await self.store.get_event(event.event_id) - d["invite"] = await self._event_serializer.serialize_event( + d["invite"] = self._event_serializer.serialize_event( invite_event, time_now, as_client_event=as_client_event, @@ -222,7 +222,7 @@ class InitialSyncHandler: d["messages"] = { "chunk": ( - await self._event_serializer.serialize_events( + self._event_serializer.serialize_events( messages, time_now=time_now, as_client_event=as_client_event, @@ -232,7 +232,7 @@ class InitialSyncHandler: "end": await end_token.to_string(self.store), } - d["state"] = await self._event_serializer.serialize_events( + d["state"] = self._event_serializer.serialize_events( current_state.values(), time_now=time_now, as_client_event=as_client_event, @@ -376,16 +376,14 @@ class InitialSyncHandler: "messages": { "chunk": ( # Don't bundle aggregations as this is a deprecated API. - await self._event_serializer.serialize_events(messages, time_now) + self._event_serializer.serialize_events(messages, time_now) ), "start": await start_token.to_string(self.store), "end": await end_token.to_string(self.store), }, "state": ( # Don't bundle aggregations as this is a deprecated API. - await self._event_serializer.serialize_events( - room_state.values(), time_now - ) + self._event_serializer.serialize_events(room_state.values(), time_now) ), "presence": [], "receipts": [], @@ -404,7 +402,7 @@ class InitialSyncHandler: # TODO: These concurrently time_now = self.clock.time_msec() # Don't bundle aggregations as this is a deprecated API. - state = await self._event_serializer.serialize_events( + state = self._event_serializer.serialize_events( current_state.values(), time_now ) @@ -480,7 +478,7 @@ class InitialSyncHandler: "messages": { "chunk": ( # Don't bundle aggregations as this is a deprecated API. - await self._event_serializer.serialize_events(messages, time_now) + self._event_serializer.serialize_events(messages, time_now) ), "start": await start_token.to_string(self.store), "end": await end_token.to_string(self.store), diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py index 5e3d3886eb..b37250aa38 100644 --- a/synapse/handlers/message.py +++ b/synapse/handlers/message.py @@ -246,7 +246,7 @@ class MessageHandler: room_state = room_state_events[membership_event_id] now = self.clock.time_msec() - events = await self._event_serializer.serialize_events(room_state.values(), now) + events = self._event_serializer.serialize_events(room_state.values(), now) return events async def get_joined_members(self, requester: Requester, room_id: str) -> dict: diff --git a/synapse/handlers/pagination.py b/synapse/handlers/pagination.py index 7469cc55a2..472688f045 100644 --- a/synapse/handlers/pagination.py +++ b/synapse/handlers/pagination.py @@ -537,14 +537,16 @@ class PaginationHandler: state_dict = await self.store.get_events(list(state_ids.values())) state = state_dict.values() + aggregations = await self.store.get_bundled_aggregations(events) + time_now = self.clock.time_msec() chunk = { "chunk": ( - await self._event_serializer.serialize_events( + self._event_serializer.serialize_events( events, time_now, - bundle_aggregations=True, + bundle_aggregations=aggregations, as_client_event=as_client_event, ) ), @@ -553,7 +555,7 @@ class PaginationHandler: } if state: - chunk["state"] = await self._event_serializer.serialize_events( + chunk["state"] = self._event_serializer.serialize_events( state, time_now, as_client_event=as_client_event ) diff --git a/synapse/handlers/room.py b/synapse/handlers/room.py index 3d3a0f6ac3..3d47163f25 100644 --- a/synapse/handlers/room.py +++ b/synapse/handlers/room.py @@ -1181,6 +1181,16 @@ class RoomContextHandler: # `filtered` rather than the event we retrieved from the datastore. results["event"] = filtered[0] + # Fetch the aggregations. + aggregations = await self.store.get_bundled_aggregations([results["event"]]) + aggregations.update( + await self.store.get_bundled_aggregations(results["events_before"]) + ) + aggregations.update( + await self.store.get_bundled_aggregations(results["events_after"]) + ) + results["aggregations"] = aggregations + if results["events_after"]: last_event_id = results["events_after"][-1].event_id else: diff --git a/synapse/handlers/search.py b/synapse/handlers/search.py index ab7eaab2fb..0b153a6822 100644 --- a/synapse/handlers/search.py +++ b/synapse/handlers/search.py @@ -420,10 +420,10 @@ class SearchHandler: time_now = self.clock.time_msec() for context in contexts.values(): - context["events_before"] = await self._event_serializer.serialize_events( + context["events_before"] = self._event_serializer.serialize_events( context["events_before"], time_now ) - context["events_after"] = await self._event_serializer.serialize_events( + context["events_after"] = self._event_serializer.serialize_events( context["events_after"], time_now ) @@ -441,9 +441,7 @@ class SearchHandler: results.append( { "rank": rank_map[e.event_id], - "result": ( - await self._event_serializer.serialize_event(e, time_now) - ), + "result": self._event_serializer.serialize_event(e, time_now), "context": contexts.get(e.event_id, {}), } ) @@ -457,7 +455,7 @@ class SearchHandler: if state_results: s = {} for room_id, state_events in state_results.items(): - s[room_id] = await self._event_serializer.serialize_events( + s[room_id] = self._event_serializer.serialize_events( state_events, time_now ) diff --git a/synapse/rest/admin/rooms.py b/synapse/rest/admin/rooms.py index 6030373ebc..2e714ac87b 100644 --- a/synapse/rest/admin/rooms.py +++ b/synapse/rest/admin/rooms.py @@ -424,7 +424,7 @@ class RoomStateRestServlet(RestServlet): event_ids = await self.store.get_current_state_ids(room_id) events = await self.store.get_events(event_ids.values()) now = self.clock.time_msec() - room_state = await self._event_serializer.serialize_events(events.values(), now) + room_state = self._event_serializer.serialize_events(events.values(), now) ret = {"state": room_state} return HTTPStatus.OK, ret @@ -744,22 +744,22 @@ class RoomEventContextServlet(RestServlet): ) time_now = self.clock.time_msec() - results["events_before"] = await self._event_serializer.serialize_events( + results["events_before"] = self._event_serializer.serialize_events( results["events_before"], time_now, - bundle_aggregations=True, + bundle_aggregations=results["aggregations"], ) - results["event"] = await self._event_serializer.serialize_event( + results["event"] = self._event_serializer.serialize_event( results["event"], time_now, - bundle_aggregations=True, + bundle_aggregations=results["aggregations"], ) - results["events_after"] = await self._event_serializer.serialize_events( + results["events_after"] = self._event_serializer.serialize_events( results["events_after"], time_now, - bundle_aggregations=True, + bundle_aggregations=results["aggregations"], ) - results["state"] = await self._event_serializer.serialize_events( + results["state"] = self._event_serializer.serialize_events( results["state"], time_now ) diff --git a/synapse/rest/client/events.py b/synapse/rest/client/events.py index 13b72a045a..672c821061 100644 --- a/synapse/rest/client/events.py +++ b/synapse/rest/client/events.py @@ -91,7 +91,7 @@ class EventRestServlet(RestServlet): time_now = self.clock.time_msec() if event: - result = await self._event_serializer.serialize_event(event, time_now) + result = self._event_serializer.serialize_event(event, time_now) return 200, result else: return 404, "Event not found." diff --git a/synapse/rest/client/notifications.py b/synapse/rest/client/notifications.py index acd0c9e135..8e427a96a3 100644 --- a/synapse/rest/client/notifications.py +++ b/synapse/rest/client/notifications.py @@ -72,7 +72,7 @@ class NotificationsServlet(RestServlet): "actions": pa.actions, "ts": pa.received_ts, "event": ( - await self._event_serializer.serialize_event( + self._event_serializer.serialize_event( notif_events[pa.event_id], self.clock.time_msec(), event_format=format_event_for_client_v2_without_room_id, diff --git a/synapse/rest/client/relations.py b/synapse/rest/client/relations.py index 3823498012..37d949a71e 100644 --- a/synapse/rest/client/relations.py +++ b/synapse/rest/client/relations.py @@ -113,13 +113,14 @@ class RelationPaginationServlet(RestServlet): now = self.clock.time_msec() # Do not bundle aggregations when retrieving the original event because # we want the content before relations are applied to it. - original_event = await self._event_serializer.serialize_event( - event, now, bundle_aggregations=False + original_event = self._event_serializer.serialize_event( + event, now, bundle_aggregations=None ) # The relations returned for the requested event do include their # bundled aggregations. - serialized_events = await self._event_serializer.serialize_events( - events, now, bundle_aggregations=True + aggregations = await self.store.get_bundled_aggregations(events) + serialized_events = self._event_serializer.serialize_events( + events, now, bundle_aggregations=aggregations ) return_value = pagination_chunk.to_dict() @@ -308,7 +309,7 @@ class RelationAggregationGroupPaginationServlet(RestServlet): ) now = self.clock.time_msec() - serialized_events = await self._event_serializer.serialize_events(events, now) + serialized_events = self._event_serializer.serialize_events(events, now) return_value = result.to_dict() return_value["chunk"] = serialized_events diff --git a/synapse/rest/client/room.py b/synapse/rest/client/room.py index 40330749e5..da6014900a 100644 --- a/synapse/rest/client/room.py +++ b/synapse/rest/client/room.py @@ -642,6 +642,7 @@ class RoomEventServlet(RestServlet): def __init__(self, hs: "HomeServer"): super().__init__() self.clock = hs.get_clock() + self._store = hs.get_datastore() self.event_handler = hs.get_event_handler() self._event_serializer = hs.get_event_client_serializer() self.auth = hs.get_auth() @@ -660,10 +661,13 @@ class RoomEventServlet(RestServlet): # https://matrix.org/docs/spec/client_server/r0.5.0#get-matrix-client-r0-rooms-roomid-event-eventid raise SynapseError(404, "Event not found.", errcode=Codes.NOT_FOUND) - time_now = self.clock.time_msec() if event: - event_dict = await self._event_serializer.serialize_event( - event, time_now, bundle_aggregations=True + # Ensure there are bundled aggregations available. + aggregations = await self._store.get_bundled_aggregations([event]) + + time_now = self.clock.time_msec() + event_dict = self._event_serializer.serialize_event( + event, time_now, bundle_aggregations=aggregations ) return 200, event_dict @@ -708,16 +712,20 @@ class RoomEventContextServlet(RestServlet): raise SynapseError(404, "Event not found.", errcode=Codes.NOT_FOUND) time_now = self.clock.time_msec() - results["events_before"] = await self._event_serializer.serialize_events( - results["events_before"], time_now, bundle_aggregations=True + results["events_before"] = self._event_serializer.serialize_events( + results["events_before"], + time_now, + bundle_aggregations=results["aggregations"], ) - results["event"] = await self._event_serializer.serialize_event( - results["event"], time_now, bundle_aggregations=True + results["event"] = self._event_serializer.serialize_event( + results["event"], time_now, bundle_aggregations=results["aggregations"] ) - results["events_after"] = await self._event_serializer.serialize_events( - results["events_after"], time_now, bundle_aggregations=True + results["events_after"] = self._event_serializer.serialize_events( + results["events_after"], + time_now, + bundle_aggregations=results["aggregations"], ) - results["state"] = await self._event_serializer.serialize_events( + results["state"] = self._event_serializer.serialize_events( results["state"], time_now ) diff --git a/synapse/rest/client/sync.py b/synapse/rest/client/sync.py index e99a943d0d..a3e57e4b20 100644 --- a/synapse/rest/client/sync.py +++ b/synapse/rest/client/sync.py @@ -17,7 +17,6 @@ from collections import defaultdict from typing import ( TYPE_CHECKING, Any, - Awaitable, Callable, Dict, Iterable, @@ -395,7 +394,7 @@ class SyncRestServlet(RestServlet): """ invited = {} for room in rooms: - invite = await self._event_serializer.serialize_event( + invite = self._event_serializer.serialize_event( room.invite, time_now, token_id=token_id, @@ -432,7 +431,7 @@ class SyncRestServlet(RestServlet): """ knocked = {} for room in rooms: - knock = await self._event_serializer.serialize_event( + knock = self._event_serializer.serialize_event( room.knock, time_now, token_id=token_id, @@ -525,21 +524,14 @@ class SyncRestServlet(RestServlet): The room, encoded in our response format """ - def serialize(events: Iterable[EventBase]) -> Awaitable[List[JsonDict]]: + def serialize( + events: Iterable[EventBase], + aggregations: Optional[Dict[str, Dict[str, Any]]] = None, + ) -> List[JsonDict]: return self._event_serializer.serialize_events( events, time_now=time_now, - # Don't bother to bundle aggregations if the timeline is unlimited, - # as clients will have all the necessary information. - # bundle_aggregations=room.timeline.limited, - # - # richvdh 2021-12-15: disable this temporarily as it has too high an - # overhead for initialsyncs. We need to figure out a way that the - # bundling can be done *before* the events are stored in the - # SyncResponseCache so that this part can be synchronous. - # - # Ensure to re-enable the test at tests/rest/client/test_relations.py::RelationsTestCase.test_bundled_aggregations. - bundle_aggregations=False, + bundle_aggregations=aggregations, token_id=token_id, event_format=event_formatter, only_event_fields=only_fields, @@ -561,8 +553,21 @@ class SyncRestServlet(RestServlet): event.room_id, ) - serialized_state = await serialize(state_events) - serialized_timeline = await serialize(timeline_events) + serialized_state = serialize(state_events) + # Don't bother to bundle aggregations if the timeline is unlimited, + # as clients will have all the necessary information. + # bundle_aggregations=room.timeline.limited, + # + # richvdh 2021-12-15: disable this temporarily as it has too high an + # overhead for initialsyncs. We need to figure out a way that the + # bundling can be done *before* the events are stored in the + # SyncResponseCache so that this part can be synchronous. + # + # Ensure to re-enable the test at tests/rest/client/test_relations.py::RelationsTestCase.test_bundled_aggregations. + # if room.timeline.limited: + # aggregations = await self.store.get_bundled_aggregations(timeline_events) + aggregations = None + serialized_timeline = serialize(timeline_events, aggregations) account_data = room.account_data diff --git a/synapse/server.py b/synapse/server.py index 185e40e4da..3032f0b738 100644 --- a/synapse/server.py +++ b/synapse/server.py @@ -759,7 +759,7 @@ class HomeServer(metaclass=abc.ABCMeta): @cache_in_self def get_event_client_serializer(self) -> EventClientSerializer: - return EventClientSerializer(self) + return EventClientSerializer() @cache_in_self def get_password_policy_handler(self) -> PasswordPolicyHandler: diff --git a/synapse/storage/databases/main/relations.py b/synapse/storage/databases/main/relations.py index 4ff6aed253..c6c4bd18da 100644 --- a/synapse/storage/databases/main/relations.py +++ b/synapse/storage/databases/main/relations.py @@ -13,14 +13,30 @@ # limitations under the License. import logging -from typing import List, Optional, Tuple, Union, cast +from typing import ( + TYPE_CHECKING, + Any, + Dict, + Iterable, + List, + Optional, + Tuple, + Union, + cast, +) import attr +from frozendict import frozendict -from synapse.api.constants import RelationTypes +from synapse.api.constants import EventTypes, RelationTypes from synapse.events import EventBase from synapse.storage._base import SQLBaseStore -from synapse.storage.database import LoggingTransaction, make_in_list_sql_clause +from synapse.storage.database import ( + DatabasePool, + LoggingDatabaseConnection, + LoggingTransaction, + make_in_list_sql_clause, +) from synapse.storage.databases.main.stream import generate_pagination_where_clause from synapse.storage.relations import ( AggregationPaginationToken, @@ -29,10 +45,24 @@ from synapse.storage.relations import ( ) from synapse.util.caches.descriptors import cached +if TYPE_CHECKING: + from synapse.server import HomeServer + logger = logging.getLogger(__name__) class RelationsWorkerStore(SQLBaseStore): + def __init__( + self, + database: DatabasePool, + db_conn: LoggingDatabaseConnection, + hs: "HomeServer", + ): + super().__init__(database, db_conn, hs) + + self._msc1849_enabled = hs.config.experimental.msc1849_enabled + self._msc3440_enabled = hs.config.experimental.msc3440_enabled + @cached(tree=True) async def get_relations_for_event( self, @@ -515,6 +545,98 @@ class RelationsWorkerStore(SQLBaseStore): "get_if_user_has_annotated_event", _get_if_user_has_annotated_event ) + async def _get_bundled_aggregation_for_event( + self, event: EventBase + ) -> Optional[Dict[str, Any]]: + """Generate bundled aggregations for an event. + + Note that this does not use a cache, but depends on cached methods. + + Args: + event: The event to calculate bundled aggregations for. + + Returns: + The bundled aggregations for an event, if bundled aggregations are + enabled and the event can have bundled aggregations. + """ + # State events and redacted events do not get bundled aggregations. + if event.is_state() or event.internal_metadata.is_redacted(): + return None + + # Do not bundle aggregations for an event which represents an edit or an + # annotation. It does not make sense for them to have related events. + relates_to = event.content.get("m.relates_to") + if isinstance(relates_to, (dict, frozendict)): + relation_type = relates_to.get("rel_type") + if relation_type in (RelationTypes.ANNOTATION, RelationTypes.REPLACE): + return None + + event_id = event.event_id + room_id = event.room_id + + # The bundled aggregations to include, a mapping of relation type to a + # type-specific value. Some types include the direct return type here + # while others need more processing during serialization. + aggregations: Dict[str, Any] = {} + + annotations = await self.get_aggregation_groups_for_event(event_id, room_id) + if annotations.chunk: + aggregations[RelationTypes.ANNOTATION] = annotations.to_dict() + + references = await self.get_relations_for_event( + event_id, room_id, RelationTypes.REFERENCE, direction="f" + ) + if references.chunk: + aggregations[RelationTypes.REFERENCE] = references.to_dict() + + edit = None + if event.type == EventTypes.Message: + edit = await self.get_applicable_edit(event_id, room_id) + + if edit: + aggregations[RelationTypes.REPLACE] = edit + + # If this event is the start of a thread, include a summary of the replies. + if self._msc3440_enabled: + ( + thread_count, + latest_thread_event, + ) = await self.get_thread_summary(event_id, room_id) + if latest_thread_event: + aggregations[RelationTypes.THREAD] = { + # Don't bundle aggregations as this could recurse forever. + "latest_event": latest_thread_event, + "count": thread_count, + } + + # Store the bundled aggregations in the event metadata for later use. + return aggregations + + async def get_bundled_aggregations( + self, events: Iterable[EventBase] + ) -> Dict[str, Dict[str, Any]]: + """Generate bundled aggregations for events. + + Args: + events: The iterable of events to calculate bundled aggregations for. + + Returns: + A map of event ID to the bundled aggregation for the event. Not all + events may have bundled aggregations in the results. + """ + # If bundled aggregations are disabled, nothing to do. + if not self._msc1849_enabled: + return {} + + # TODO Parallelize. + results = {} + for event in events: + event_result = await self._get_bundled_aggregation_for_event(event) + if event_result is not None: + results[event.event_id] = event_result + + return results + class RelationsStore(RelationsWorkerStore): pass diff --git a/tests/rest/client/test_retention.py b/tests/rest/client/test_retention.py index b58452195a..fe5b536d97 100644 --- a/tests/rest/client/test_retention.py +++ b/tests/rest/client/test_retention.py @@ -228,7 +228,7 @@ class RetentionTestCase(unittest.HomeserverTestCase): self.assertIsNotNone(event) time_now = self.clock.time_msec() - serialized = self.get_success(self.serializer.serialize_event(event, time_now)) + serialized = self.serializer.serialize_event(event, time_now) return serialized From 2bb4bd126946df46aacf6849f0acb01e78f7d807 Mon Sep 17 00:00:00 2001 From: David Robertson Date: Fri, 7 Jan 2022 16:43:21 +0000 Subject: [PATCH 66/76] Test that bans win a join against a race when computing `/sync` response (#11701) --- changelog.d/11701.misc | 1 + tests/handlers/test_sync.py | 97 +++++++++++++++++++++++++++++++++++-- tests/rest/client/utils.py | 10 ++++ 3 files changed, 105 insertions(+), 3 deletions(-) create mode 100644 changelog.d/11701.misc diff --git a/changelog.d/11701.misc b/changelog.d/11701.misc new file mode 100644 index 0000000000..68905e0412 --- /dev/null +++ b/changelog.d/11701.misc @@ -0,0 +1 @@ +Add a test for [an edge case](https://github.com/matrix-org/synapse/pull/11532#discussion_r769104461) in the `/sync` logic. \ No newline at end of file diff --git a/tests/handlers/test_sync.py b/tests/handlers/test_sync.py index 638186f173..07a760e91a 100644 --- a/tests/handlers/test_sync.py +++ b/tests/handlers/test_sync.py @@ -11,15 +11,14 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - from typing import Optional -from unittest.mock import Mock +from unittest.mock import MagicMock, Mock, patch from synapse.api.constants import EventTypes, JoinRules from synapse.api.errors import Codes, ResourceLimitError from synapse.api.filtering import Filtering from synapse.api.room_versions import RoomVersions -from synapse.handlers.sync import SyncConfig +from synapse.handlers.sync import SyncConfig, SyncResult from synapse.rest import admin from synapse.rest.client import knock, login, room from synapse.server import HomeServer @@ -27,6 +26,7 @@ from synapse.types import UserID, create_requester import tests.unittest import tests.utils +from tests.test_utils import make_awaitable class SyncTestCase(tests.unittest.HomeserverTestCase): @@ -186,6 +186,97 @@ class SyncTestCase(tests.unittest.HomeserverTestCase): self.assertNotIn(invite_room, [r.room_id for r in result.invited]) self.assertNotIn(knock_room, [r.room_id for r in result.knocked]) + def test_ban_wins_race_with_join(self): + """Rooms shouldn't appear under "joined" if a join loses a race to a ban. + + A complicated edge case. Imagine the following scenario: + + * you attempt to join a room + * racing with that is a ban which comes in over federation, which ends up with + an earlier stream_ordering than the join. + * you get a sync response with a sync token which is _after_ the ban, but before + the join + * now your join lands; it is a valid event because its `prev_event`s predate the + ban, but will not make it into current_state_events (because bans win over + joins in state res, essentially). + * When we do a sync from the incremental sync, the only event in the timeline + is your join ... and yet you aren't joined. + + The ban coming in over federation isn't crucial for this behaviour; the key + requirements are: + 1. the homeserver generates a join event with prev_events that precede the ban + (so that it passes the "are you banned" test) + 2. the join event has a stream_ordering after that of the ban. + + We use monkeypatching to artificially trigger condition (1). + """ + # A local user Alice creates a room. + owner = self.register_user("alice", "password") + owner_tok = self.login(owner, "password") + room_id = self.helper.create_room_as(owner, is_public=True, tok=owner_tok) + + # Do a sync as Alice to get the latest event in the room. + alice_sync_result: SyncResult = self.get_success( + self.sync_handler.wait_for_sync_for_user( + create_requester(owner), generate_sync_config(owner) + ) + ) + self.assertEqual(len(alice_sync_result.joined), 1) + self.assertEqual(alice_sync_result.joined[0].room_id, room_id) + last_room_creation_event_id = ( + alice_sync_result.joined[0].timeline.events[-1].event_id + ) + + # Eve, a ne'er-do-well, registers. + eve = self.register_user("eve", "password") + eve_token = self.login(eve, "password") + + # Alice preemptively bans Eve. + self.helper.ban(room_id, owner, eve, tok=owner_tok) + + # Eve syncs. + eve_requester = create_requester(eve) + eve_sync_config = generate_sync_config(eve) + eve_sync_after_ban: SyncResult = self.get_success( + self.sync_handler.wait_for_sync_for_user(eve_requester, eve_sync_config) + ) + + # Sanity check this sync result. We shouldn't be joined to the room. + self.assertEqual(eve_sync_after_ban.joined, []) + + # Eve tries to join the room. We monkey patch the internal logic which selects + # the prev_events used when creating the join event, such that the ban does not + # precede the join. + mocked_get_prev_events = patch.object( + self.hs.get_datastore(), + "get_prev_events_for_room", + new_callable=MagicMock, + return_value=make_awaitable([last_room_creation_event_id]), + ) + with mocked_get_prev_events: + self.helper.join(room_id, eve, tok=eve_token) + + # Eve makes a second, incremental sync. + eve_incremental_sync_after_join: SyncResult = self.get_success( + self.sync_handler.wait_for_sync_for_user( + eve_requester, + eve_sync_config, + since_token=eve_sync_after_ban.next_batch, + ) + ) + # Eve should not see herself as joined to the room. + self.assertEqual(eve_incremental_sync_after_join.joined, []) + + # If we did a third initial sync, we should _still_ see eve is not joined to the room. + eve_initial_sync_after_join: SyncResult = self.get_success( + self.sync_handler.wait_for_sync_for_user( + eve_requester, + eve_sync_config, + since_token=None, + ) + ) + self.assertEqual(eve_initial_sync_after_join.joined, []) + _request_key = 0 diff --git a/tests/rest/client/utils.py b/tests/rest/client/utils.py index 1af5e5cee5..8424383580 100644 --- a/tests/rest/client/utils.py +++ b/tests/rest/client/utils.py @@ -196,6 +196,16 @@ class RestHelper: expect_code=expect_code, ) + def ban(self, room: str, src: str, targ: str, **kwargs: object): + """A convenience helper: `change_membership` with `membership` preset to "ban".""" + self.change_membership( + room=room, + src=src, + targ=targ, + membership=Membership.BAN, + **kwargs, + ) + def change_membership( self, room: str, From d3cf0730f8bf3de58d37060eff0dca3f69b4a0e1 Mon Sep 17 00:00:00 2001 From: David Robertson Date: Fri, 7 Jan 2022 19:13:41 +0000 Subject: [PATCH 67/76] Optionally use an on-disk sqlite db in tests (#11702) * Optionally use an on-disk sqlite db in tests When debugging a test it is sometimes useful to inspect the state of the DB. This is not easy when the db is in-memory: one cannot attach the sqlite CLI to another process's DB. With this change, if SYNAPSE_TEST_PERSIST_SQLITE_DB is set, we use `_trial_temp/test.db` as our sqlite database. One can then use `sqlite3 _trial_temp/test.db` and query to your heart's content. The DB is destroyed and recreated between different test cases. Co-authored-by: Patrick Cloke --- changelog.d/11702.misc | 1 + tests/server.py | 19 ++++++++++++++++++- tests/utils.py | 4 ++++ 3 files changed, 23 insertions(+), 1 deletion(-) create mode 100644 changelog.d/11702.misc diff --git a/changelog.d/11702.misc b/changelog.d/11702.misc new file mode 100644 index 0000000000..fc1069cae0 --- /dev/null +++ b/changelog.d/11702.misc @@ -0,0 +1 @@ +Add the option to write sqlite test dbs to disk when running tests. \ No newline at end of file diff --git a/tests/server.py b/tests/server.py index ca2b7a5b97..a0cd14ea45 100644 --- a/tests/server.py +++ b/tests/server.py @@ -14,6 +14,8 @@ import hashlib import json import logging +import os +import os.path import time import uuid import warnings @@ -71,6 +73,7 @@ from tests.utils import ( POSTGRES_HOST, POSTGRES_PASSWORD, POSTGRES_USER, + SQLITE_PERSIST_DB, USE_POSTGRES_FOR_TESTS, MockClock, default_config, @@ -739,9 +742,23 @@ def setup_test_homeserver( }, } else: + if SQLITE_PERSIST_DB: + # The current working directory is in _trial_temp, so this gets created within that directory. + test_db_location = os.path.abspath("test.db") + logger.debug("Will persist db to %s", test_db_location) + # Ensure each test gets a clean database. + try: + os.remove(test_db_location) + except FileNotFoundError: + pass + else: + logger.debug("Removed existing DB at %s", test_db_location) + else: + test_db_location = ":memory:" + database_config = { "name": "sqlite3", - "args": {"database": ":memory:", "cp_min": 1, "cp_max": 1}, + "args": {"database": test_db_location, "cp_min": 1, "cp_max": 1}, } if "db_txn_limit" in kwargs: diff --git a/tests/utils.py b/tests/utils.py index 6d013e8518..c06fc320f3 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -42,6 +42,10 @@ POSTGRES_HOST = os.environ.get("SYNAPSE_POSTGRES_HOST", None) POSTGRES_PASSWORD = os.environ.get("SYNAPSE_POSTGRES_PASSWORD", None) POSTGRES_BASE_DB = "_synapse_unit_tests_base_%s" % (os.getpid(),) +# When debugging a specific test, it's occasionally useful to write the +# DB to disk and query it with the sqlite CLI. +SQLITE_PERSIST_DB = os.environ.get("SYNAPSE_TEST_PERSIST_SQLITE_DB") is not None + # the dbname we will connect to in order to create the base database. POSTGRES_DBNAME_FOR_INITIAL_CREATE = "postgres" From 8e57584a5859a9002759963eb546d523d2498a01 Mon Sep 17 00:00:00 2001 From: Patrick Cloke Date: Fri, 7 Jan 2022 19:27:58 -0500 Subject: [PATCH 68/76] Support spaces with > 50 rooms in the /hierarchy endpoint. (#11695) By returning all of the m.space.child state of the space, not just the first 50. The number of rooms returned is still capped at 50. For the federation API this implies that the requesting server will need to individually query for any other rooms it is not joined to. --- changelog.d/11695.bugfix | 1 + synapse/handlers/room_summary.py | 30 +++++++++++++++++++-------- tests/handlers/test_room_summary.py | 32 +++++++++++++++++++++++++++++ 3 files changed, 55 insertions(+), 8 deletions(-) create mode 100644 changelog.d/11695.bugfix diff --git a/changelog.d/11695.bugfix b/changelog.d/11695.bugfix new file mode 100644 index 0000000000..7799aefb82 --- /dev/null +++ b/changelog.d/11695.bugfix @@ -0,0 +1 @@ +Fix a bug where the only the first 50 rooms from a space were returned from the `/hierarchy` API. This has existed since the introduction of the API in Synapse v1.41.0. diff --git a/synapse/handlers/room_summary.py b/synapse/handlers/room_summary.py index 9ef88feb8a..7c60cb0bdd 100644 --- a/synapse/handlers/room_summary.py +++ b/synapse/handlers/room_summary.py @@ -153,6 +153,9 @@ class RoomSummaryHandler: rooms_result: List[JsonDict] = [] events_result: List[JsonDict] = [] + if max_rooms_per_space is None or max_rooms_per_space > MAX_ROOMS_PER_SPACE: + max_rooms_per_space = MAX_ROOMS_PER_SPACE + while room_queue and len(rooms_result) < MAX_ROOMS: queue_entry = room_queue.popleft() room_id = queue_entry.room_id @@ -167,7 +170,7 @@ class RoomSummaryHandler: # The client-specified max_rooms_per_space limit doesn't apply to the # room_id specified in the request, so we ignore it if this is the # first room we are processing. - max_children = max_rooms_per_space if processed_rooms else None + max_children = max_rooms_per_space if processed_rooms else MAX_ROOMS if is_in_room: room_entry = await self._summarize_local_room( @@ -395,7 +398,7 @@ class RoomSummaryHandler: None, room_id, suggested_only, - # TODO Handle max children. + # Do not limit the maximum children. max_children=None, ) @@ -525,6 +528,10 @@ class RoomSummaryHandler: rooms_result: List[JsonDict] = [] events_result: List[JsonDict] = [] + # Set a limit on the number of rooms to return. + if max_rooms_per_space is None or max_rooms_per_space > MAX_ROOMS_PER_SPACE: + max_rooms_per_space = MAX_ROOMS_PER_SPACE + while room_queue and len(rooms_result) < MAX_ROOMS: room_id = room_queue.popleft() if room_id in processed_rooms: @@ -583,7 +590,9 @@ class RoomSummaryHandler: # Iterate through each child and potentially add it, but not its children, # to the response. - for child_room in root_room_entry.children_state_events: + for child_room in itertools.islice( + root_room_entry.children_state_events, MAX_ROOMS_PER_SPACE + ): room_id = child_room.get("state_key") assert isinstance(room_id, str) # If the room is unknown, skip it. @@ -633,8 +642,8 @@ class RoomSummaryHandler: suggested_only: True if only suggested children should be returned. Otherwise, all children are returned. max_children: - The maximum number of children rooms to include. This is capped - to a server-set limit. + The maximum number of children rooms to include. A value of None + means no limit. Returns: A room entry if the room should be returned. None, otherwise. @@ -656,8 +665,13 @@ class RoomSummaryHandler: # we only care about suggested children child_events = filter(_is_suggested_child_event, child_events) - if max_children is None or max_children > MAX_ROOMS_PER_SPACE: - max_children = MAX_ROOMS_PER_SPACE + # TODO max_children is legacy code for the /spaces endpoint. + if max_children is not None: + child_iter: Iterable[EventBase] = itertools.islice( + child_events, max_children + ) + else: + child_iter = child_events stripped_events: List[JsonDict] = [ { @@ -668,7 +682,7 @@ class RoomSummaryHandler: "sender": e.sender, "origin_server_ts": e.origin_server_ts, } - for e in itertools.islice(child_events, max_children) + for e in child_iter ] return _RoomEntry(room_id, room_entry, stripped_events) diff --git a/tests/handlers/test_room_summary.py b/tests/handlers/test_room_summary.py index e5a6a6c747..ce3ebcf2f2 100644 --- a/tests/handlers/test_room_summary.py +++ b/tests/handlers/test_room_summary.py @@ -253,6 +253,38 @@ class SpaceSummaryTestCase(unittest.HomeserverTestCase): ) self._assert_hierarchy(result, expected) + def test_large_space(self): + """Test a space with a large number of rooms.""" + rooms = [self.room] + # Make at least 51 rooms that are part of the space. + for _ in range(55): + room = self.helper.create_room_as(self.user, tok=self.token) + self._add_child(self.space, room, self.token) + rooms.append(room) + + result = self.get_success(self.handler.get_space_summary(self.user, self.space)) + # The spaces result should have the space and the first 50 rooms in it, + # along with the links from space -> room for those 50 rooms. + expected = [(self.space, rooms[:50])] + [(room, []) for room in rooms[:49]] + self._assert_rooms(result, expected) + + # The result should have the space and the rooms in it, along with the links + # from space -> room. + expected = [(self.space, rooms)] + [(room, []) for room in rooms] + + # Make two requests to fully paginate the results. + result = self.get_success( + self.handler.get_room_hierarchy(create_requester(self.user), self.space) + ) + result2 = self.get_success( + self.handler.get_room_hierarchy( + create_requester(self.user), self.space, from_token=result["next_batch"] + ) + ) + # Combine the results. + result["rooms"] += result2["rooms"] + self._assert_hierarchy(result, expected) + def test_visibility(self): """A user not in a space cannot inspect it.""" user2 = self.register_user("user2", "pass") From 3be63654e40a137c469318e67048211800b9f985 Mon Sep 17 00:00:00 2001 From: kegsay Date: Mon, 10 Jan 2022 11:46:40 +0000 Subject: [PATCH 69/76] Prettier complement logs (#11707) * Prettier complement logs * Changelog --- .github/workflows/tests.yml | 3 ++- changelog.d/11707.misc | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) create mode 100644 changelog.d/11707.misc diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index cb72e1a233..6fa61ab559 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -374,7 +374,8 @@ jobs: working-directory: complement/dockerfiles # Run Complement - - run: go test -v -tags synapse_blacklist,msc2403 ./tests/... + - run: set -o pipefail && go test -v -json -tags synapse_blacklist,msc2403 ./tests/... 2>&1 | gotestfmt + shell: bash env: COMPLEMENT_BASE_IMAGE: complement-synapse:latest working-directory: complement diff --git a/changelog.d/11707.misc b/changelog.d/11707.misc new file mode 100644 index 0000000000..ef1e01cac8 --- /dev/null +++ b/changelog.d/11707.misc @@ -0,0 +1 @@ +Improve Complement test output for Gitub Actions. From c43dd4d01b2344014004dff3834a86cd59e85adc Mon Sep 17 00:00:00 2001 From: David Robertson Date: Mon, 10 Jan 2022 13:40:46 +0000 Subject: [PATCH 70/76] Deal with mypy errors w/ type-hinted pynacl 1.5.0 (#11714) * Deal with mypy errors w/ type-hinted pynacl 1.5.0 Fixes #11644. I really don't like that we're monkey patching pynacl SignedKey instances with alg and version objects. But I'm too scared to make the changes necessary right now. (Ideally I would replace `signedjson.types.SingingKey` with a runtime class which wraps or inherits from `nacl.signing.SigningKey`.) C.f. https://github.com/matrix-org/python-signedjson/issues/16 --- changelog.d/11714.misc | 1 + tests/crypto/test_event_signing.py | 8 +++++++- 2 files changed, 8 insertions(+), 1 deletion(-) create mode 100644 changelog.d/11714.misc diff --git a/changelog.d/11714.misc b/changelog.d/11714.misc new file mode 100644 index 0000000000..7f39bf0e3d --- /dev/null +++ b/changelog.d/11714.misc @@ -0,0 +1 @@ +Fix a typechecker problem related to our (ab)use of `nacl.signing.SigningKey`s. \ No newline at end of file diff --git a/tests/crypto/test_event_signing.py b/tests/crypto/test_event_signing.py index 1c920157f5..a72a0103d3 100644 --- a/tests/crypto/test_event_signing.py +++ b/tests/crypto/test_event_signing.py @@ -14,6 +14,7 @@ import nacl.signing +import signedjson.types from unpaddedbase64 import decode_base64 from synapse.api.room_versions import RoomVersions @@ -35,7 +36,12 @@ HOSTNAME = "domain" class EventSigningTestCase(unittest.TestCase): def setUp(self): - self.signing_key = nacl.signing.SigningKey(SIGNING_KEY_SEED) + # NB: `signedjson` expects `nacl.signing.SigningKey` instances which have been + # monkeypatched to include new `alg` and `version` attributes. This is captured + # by the `signedjson.types.SigningKey` protocol. + self.signing_key: signedjson.types.SigningKey = nacl.signing.SigningKey( + SIGNING_KEY_SEED + ) self.signing_key.alg = KEY_ALG self.signing_key.version = KEY_VER From ffd227c3822b6e20b9dc203b3eae253adc0cf663 Mon Sep 17 00:00:00 2001 From: reivilibre Date: Mon, 10 Jan 2022 15:38:22 +0000 Subject: [PATCH 71/76] Fix docstring on `add_account_data_for_user`. (#11716) --- changelog.d/11716.misc | 1 + synapse/handlers/account_data.py | 2 +- synapse/storage/databases/main/account_data.py | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) create mode 100644 changelog.d/11716.misc diff --git a/changelog.d/11716.misc b/changelog.d/11716.misc new file mode 100644 index 0000000000..08f7310498 --- /dev/null +++ b/changelog.d/11716.misc @@ -0,0 +1 @@ +Fix docstring on `add_account_data_for_user`. \ No newline at end of file diff --git a/synapse/handlers/account_data.py b/synapse/handlers/account_data.py index 96273e2f81..bad48713bc 100644 --- a/synapse/handlers/account_data.py +++ b/synapse/handlers/account_data.py @@ -77,7 +77,7 @@ class AccountDataHandler: async def add_account_data_for_user( self, user_id: str, account_data_type: str, content: JsonDict ) -> int: - """Add some account_data to a room for a user. + """Add some global account_data for a user. Args: user_id: The user to add a tag for. diff --git a/synapse/storage/databases/main/account_data.py b/synapse/storage/databases/main/account_data.py index 32a553fdd7..93db71d1b4 100644 --- a/synapse/storage/databases/main/account_data.py +++ b/synapse/storage/databases/main/account_data.py @@ -450,7 +450,7 @@ class AccountDataWorkerStore(CacheInvalidationWorkerStore): async def add_account_data_for_user( self, user_id: str, account_data_type: str, content: JsonDict ) -> int: - """Add some account_data to a room for a user. + """Add some global account_data for a user. Args: user_id: The user to add a tag for. From 7c3408d1a88a24c2db917ab48cb15d13ac683427 Mon Sep 17 00:00:00 2001 From: Andrew Morgan <1342360+anoadragon453@users.noreply.github.com> Date: Mon, 10 Jan 2022 17:06:42 +0000 Subject: [PATCH 72/76] Document the `SYNAPSE_TEST_PERSIST_SQLITE_DB` unit test env var (#11715) --- changelog.d/11715.doc | 1 + docs/development/contributing_guide.md | 21 +++++++++++++++++++++ 2 files changed, 22 insertions(+) create mode 100644 changelog.d/11715.doc diff --git a/changelog.d/11715.doc b/changelog.d/11715.doc new file mode 100644 index 0000000000..32b7c10b0b --- /dev/null +++ b/changelog.d/11715.doc @@ -0,0 +1 @@ +Document the new `SYNAPSE_TEST_PERSIST_SQLITE_DB` environment variable in the contributing guide. diff --git a/docs/development/contributing_guide.md b/docs/development/contributing_guide.md index abdb808438..121f7c0687 100644 --- a/docs/development/contributing_guide.md +++ b/docs/development/contributing_guide.md @@ -169,6 +169,27 @@ To increase the log level for the tests, set `SYNAPSE_TEST_LOG_LEVEL`: SYNAPSE_TEST_LOG_LEVEL=DEBUG trial tests ``` +By default, tests will use an in-memory SQLite database for test data. For additional +help with debugging, one can use an on-disk SQLite database file instead, in order to +review database state during and after running tests. This can be done by setting +the `SYNAPSE_TEST_PERSIST_SQLITE_DB` environment variable. Doing so will cause the +database state to be stored in a file named `test.db` under the trial process' +working directory. Typically, this ends up being `_trial_temp/test.db`. For example: + +```sh +SYNAPSE_TEST_PERSIST_SQLITE_DB=1 trial tests +``` + +The database file can then be inspected with: + +```sh +sqlite3 _trial_temp/test.db +``` + +Note that the database file is cleared at the beginning of each test run. Thus it +will always only contain the data generated by the *last run test*. Though generally +when debugging, one is only running a single test anyway. + ### Running tests under PostgreSQL Invoking `trial` as above will use an in-memory SQLite database. This is great for From 338e70c6170fe6f964f2080b2585eb2add9f5b77 Mon Sep 17 00:00:00 2001 From: Michael Kaye <1917473+michaelkaye@users.noreply.github.com> Date: Mon, 10 Jan 2022 23:18:56 +0000 Subject: [PATCH 73/76] Complement environment variable name change and update .gitignore. (#11718) --- .gitignore | 4 ++++ changelog.d/11718.misc | 1 + scripts-dev/complement.sh | 2 +- 3 files changed, 6 insertions(+), 1 deletion(-) create mode 100644 changelog.d/11718.misc diff --git a/.gitignore b/.gitignore index fe137f3370..8eb4eda73d 100644 --- a/.gitignore +++ b/.gitignore @@ -50,3 +50,7 @@ __pycache__/ # docs book/ + +# complement +/complement-master +/master.tar.gz diff --git a/changelog.d/11718.misc b/changelog.d/11718.misc new file mode 100644 index 0000000000..91dc5b5874 --- /dev/null +++ b/changelog.d/11718.misc @@ -0,0 +1 @@ +Complement environment variable name change and update `.gitignore`. diff --git a/scripts-dev/complement.sh b/scripts-dev/complement.sh index 53295b58fc..820427be9e 100755 --- a/scripts-dev/complement.sh +++ b/scripts-dev/complement.sh @@ -47,7 +47,7 @@ if [[ -n "$WORKERS" ]]; then COMPLEMENT_DOCKERFILE=SynapseWorkers.Dockerfile # And provide some more configuration to complement. export COMPLEMENT_CA=true - export COMPLEMENT_VERSION_CHECK_ITERATIONS=500 + export COMPLEMENT_SPAWN_HS_TIMEOUT_SECS=25 else export COMPLEMENT_BASE_IMAGE=complement-synapse COMPLEMENT_DOCKERFILE=Synapse.Dockerfile From d41c4654db03dc76812e98222feb62668a1bf94c Mon Sep 17 00:00:00 2001 From: Richard van der Hoff <1389908+richvdh@users.noreply.github.com> Date: Wed, 12 Jan 2022 10:37:57 +0000 Subject: [PATCH 74/76] Use buildkit's cache feature to speed up docker builds (#11691) Having spent much of the last week attempting to run complement tests from somewhere with damp string instead of internet... something had to be done. --- .github/workflows/tests.yml | 2 ++ changelog.d/11691.misc | 1 + docker/Dockerfile | 26 +++++++++++++++++++++----- scripts-dev/complement.sh | 4 ++++ 4 files changed, 28 insertions(+), 5 deletions(-) create mode 100644 changelog.d/11691.misc diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 6fa61ab559..4f58069702 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -366,6 +366,8 @@ jobs: # Build initial Synapse image - run: docker build -t matrixdotorg/synapse:latest -f docker/Dockerfile . working-directory: synapse + env: + DOCKER_BUILDKIT: 1 # Build a ready-to-run Synapse image based on the initial image above. # This new image includes a config file, keys for signing and TLS, and diff --git a/changelog.d/11691.misc b/changelog.d/11691.misc new file mode 100644 index 0000000000..383d0b3064 --- /dev/null +++ b/changelog.d/11691.misc @@ -0,0 +1 @@ +Use buildkit's cache feature to speed up docker builds. diff --git a/docker/Dockerfile b/docker/Dockerfile index 2bdc607e66..306f75ae56 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,14 +1,17 @@ # Dockerfile to build the matrixdotorg/synapse docker images. # +# Note that it uses features which are only available in BuildKit - see +# https://docs.docker.com/go/buildkit/ for more information. +# # To build the image, run `docker build` command from the root of the # synapse repository: # -# docker build -f docker/Dockerfile . +# DOCKER_BUILDKIT=1 docker build -f docker/Dockerfile . # # There is an optional PYTHON_VERSION build argument which sets the # version of python to build against: for example: # -# docker build -f docker/Dockerfile --build-arg PYTHON_VERSION=3.6 . +# DOCKER_BUILDKIT=1 docker build -f docker/Dockerfile --build-arg PYTHON_VERSION=3.9 . # ARG PYTHON_VERSION=3.8 @@ -19,7 +22,16 @@ ARG PYTHON_VERSION=3.8 FROM docker.io/python:${PYTHON_VERSION}-slim as builder # install the OS build deps -RUN apt-get update && apt-get install -y \ +# +# RUN --mount is specific to buildkit and is documented at +# https://github.com/moby/buildkit/blob/master/frontend/dockerfile/docs/syntax.md#build-mounts-run---mount. +# Here we use it to set up a cache for apt, to improve rebuild speeds on +# slow connections. +# +RUN \ + --mount=type=cache,target=/var/cache/apt,sharing=locked \ + --mount=type=cache,target=/var/lib/apt,sharing=locked \ + apt-get update && apt-get install -y \ build-essential \ libffi-dev \ libjpeg-dev \ @@ -44,7 +56,8 @@ COPY synapse/python_dependencies.py /synapse/synapse/python_dependencies.py # used while you develop on the source # # This is aiming at installing the `install_requires` and `extras_require` from `setup.py` -RUN pip install --prefix="/install" --no-warn-script-location \ +RUN --mount=type=cache,target=/root/.cache/pip \ + pip install --prefix="/install" --no-warn-script-location \ /synapse[all] # Copy over the rest of the project @@ -66,7 +79,10 @@ LABEL org.opencontainers.image.documentation='https://github.com/matrix-org/syna LABEL org.opencontainers.image.source='https://github.com/matrix-org/synapse.git' LABEL org.opencontainers.image.licenses='Apache-2.0' -RUN apt-get update && apt-get install -y \ +RUN \ + --mount=type=cache,target=/var/cache/apt,sharing=locked \ + --mount=type=cache,target=/var/lib/apt,sharing=locked \ + apt-get update && apt-get install -y \ curl \ gosu \ libjpeg62-turbo \ diff --git a/scripts-dev/complement.sh b/scripts-dev/complement.sh index 820427be9e..67a22d3ed3 100755 --- a/scripts-dev/complement.sh +++ b/scripts-dev/complement.sh @@ -23,6 +23,9 @@ # Exit if a line returns a non-zero exit code set -e +# enable buildkit for the docker builds +export DOCKER_BUILDKIT=1 + # Change to the repository root cd "$(dirname $0)/.." @@ -65,4 +68,5 @@ if [[ -n "$1" ]]; then fi # Run the tests! +echo "Images built; running complement" go test -v -tags synapse_blacklist,msc2403 -count=1 $EXTRA_COMPLEMENT_ARGS ./tests/... From 99ba5ae7b73b74e27c17f68f5998fee985a969b2 Mon Sep 17 00:00:00 2001 From: haslersn Date: Wed, 12 Jan 2022 15:43:48 +0100 Subject: [PATCH 75/76] Fix documentation of supported PostgreSQL version (#11725) Signed-off-by: Sebastian Hasler --- changelog.d/11725.doc | 1 + docs/postgres.md | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) create mode 100644 changelog.d/11725.doc diff --git a/changelog.d/11725.doc b/changelog.d/11725.doc new file mode 100644 index 0000000000..46eb9b814f --- /dev/null +++ b/changelog.d/11725.doc @@ -0,0 +1 @@ +Document that now the minimum supported PostgreSQL version is 10. diff --git a/docs/postgres.md b/docs/postgres.md index e4861c1f12..0562021da5 100644 --- a/docs/postgres.md +++ b/docs/postgres.md @@ -1,6 +1,6 @@ # Using Postgres -Synapse supports PostgreSQL versions 9.6 or later. +Synapse supports PostgreSQL versions 10 or later. ## Install postgres client libraries From 2560b1b6b2f74b5724253396c0e3665fa1f7968c Mon Sep 17 00:00:00 2001 From: Jason Robinson Date: Wed, 12 Jan 2022 18:09:36 +0200 Subject: [PATCH 76/76] Allow tracking puppeted users for MAU (#11561) Currently when puppeting another user, the user doing the puppeting is tracked for client IPs and MAU (if configured). When tracking MAU is important, it becomes necessary to be possible to also track the client IPs and MAU of puppeted users. As an example a client that manages user creation and creation of tokens via the Synapse admin API, passing those tokens for the client to use. This PR adds optional configuration to enable tracking of puppeted users into monthly active users. The default behaviour stays the same. Signed-off-by: Jason Robinson --- changelog.d/11561.feature | 1 + docs/sample_config.yaml | 6 ++++++ synapse/api/auth.py | 13 +++++++++++++ synapse/config/api.py | 10 ++++++++++ tests/api/test_auth.py | 33 +++++++++++++++++++++++++++++++++ 5 files changed, 63 insertions(+) create mode 100644 changelog.d/11561.feature diff --git a/changelog.d/11561.feature b/changelog.d/11561.feature new file mode 100644 index 0000000000..19dada883b --- /dev/null +++ b/changelog.d/11561.feature @@ -0,0 +1 @@ +Add `track_puppeted_user_ips` config flag to track puppeted user IP addresses. This also includes them in monthly active user counts. diff --git a/docs/sample_config.yaml b/docs/sample_config.yaml index 810a14b077..26894fae34 100644 --- a/docs/sample_config.yaml +++ b/docs/sample_config.yaml @@ -1503,6 +1503,12 @@ room_prejoin_state: #additional_event_types: # - org.example.custom.event.type +# If enabled, puppeted user IP's can also be tracked. By default when +# puppeting another user, the user who has created the access token +# for puppeting is tracked. If this is enabled, both requests are tracked. +# Implicitly enables MAU tracking for puppeted users. +#track_puppeted_user_ips: false + # A list of application service config files to use # diff --git a/synapse/api/auth.py b/synapse/api/auth.py index 4a32d430bd..683241201c 100644 --- a/synapse/api/auth.py +++ b/synapse/api/auth.py @@ -71,6 +71,7 @@ class Auth: self._auth_blocking = AuthBlocking(self.hs) self._track_appservice_user_ips = hs.config.appservice.track_appservice_user_ips + self._track_puppeted_user_ips = hs.config.api.track_puppeted_user_ips self._macaroon_secret_key = hs.config.key.macaroon_secret_key self._force_tracing_for_users = hs.config.tracing.force_tracing_for_users @@ -246,6 +247,18 @@ class Auth: user_agent=user_agent, device_id=device_id, ) + # Track also the puppeted user client IP if enabled and the user is puppeting + if ( + user_info.user_id != user_info.token_owner + and self._track_puppeted_user_ips + ): + await self.store.insert_client_ip( + user_id=user_info.user_id, + access_token=access_token, + ip=ip_addr, + user_agent=user_agent, + device_id=device_id, + ) if is_guest and not allow_guest: raise AuthError( diff --git a/synapse/config/api.py b/synapse/config/api.py index 25538b82d5..bdbe9f0fa2 100644 --- a/synapse/config/api.py +++ b/synapse/config/api.py @@ -29,6 +29,7 @@ class ApiConfig(Config): def read_config(self, config: JsonDict, **kwargs): validate_config(_MAIN_SCHEMA, config, ()) self.room_prejoin_state = list(self._get_prejoin_state_types(config)) + self.track_puppeted_user_ips = config.get("track_puppeted_user_ips", False) def generate_config_section(cls, **kwargs) -> str: formatted_default_state_types = "\n".join( @@ -59,6 +60,12 @@ class ApiConfig(Config): # #additional_event_types: # - org.example.custom.event.type + + # If enabled, puppeted user IP's can also be tracked. By default when + # puppeting another user, the user who has created the access token + # for puppeting is tracked. If this is enabled, both requests are tracked. + # Implicitly enables MAU tracking for puppeted users. + #track_puppeted_user_ips: false """ % { "formatted_default_state_types": formatted_default_state_types } @@ -138,5 +145,8 @@ _MAIN_SCHEMA = { "properties": { "room_prejoin_state": _ROOM_PREJOIN_STATE_CONFIG_SCHEMA, "room_invite_state_types": _ROOM_INVITE_STATE_TYPES_SCHEMA, + "track_puppeted_user_ips": { + "type": "boolean", + }, }, } diff --git a/tests/api/test_auth.py b/tests/api/test_auth.py index a2dfa1ed05..4b53b6d40b 100644 --- a/tests/api/test_auth.py +++ b/tests/api/test_auth.py @@ -274,6 +274,39 @@ class AuthTestCase(unittest.HomeserverTestCase): self.assertEquals(failure.value.code, 400) self.assertEquals(failure.value.errcode, Codes.EXCLUSIVE) + def test_get_user_by_req__puppeted_token__not_tracking_puppeted_mau(self): + self.store.get_user_by_access_token = simple_async_mock( + TokenLookupResult( + user_id="@baldrick:matrix.org", + device_id="device", + token_owner="@admin:matrix.org", + ) + ) + self.store.insert_client_ip = simple_async_mock(None) + request = Mock(args={}) + request.getClientIP.return_value = "127.0.0.1" + request.args[b"access_token"] = [self.test_token] + request.requestHeaders.getRawHeaders = mock_getRawHeaders() + self.get_success(self.auth.get_user_by_req(request)) + self.store.insert_client_ip.assert_called_once() + + def test_get_user_by_req__puppeted_token__tracking_puppeted_mau(self): + self.auth._track_puppeted_user_ips = True + self.store.get_user_by_access_token = simple_async_mock( + TokenLookupResult( + user_id="@baldrick:matrix.org", + device_id="device", + token_owner="@admin:matrix.org", + ) + ) + self.store.insert_client_ip = simple_async_mock(None) + request = Mock(args={}) + request.getClientIP.return_value = "127.0.0.1" + request.args[b"access_token"] = [self.test_token] + request.requestHeaders.getRawHeaders = mock_getRawHeaders() + self.get_success(self.auth.get_user_by_req(request)) + self.assertEquals(self.store.insert_client_ip.call_count, 2) + def test_get_user_from_macaroon(self): self.store.get_user_by_access_token = simple_async_mock( TokenLookupResult(user_id="@baldrick:matrix.org", device_id="device")