From a40802bcbc2838033ecfe468d7845adbb1f3cbb7 Mon Sep 17 00:00:00 2001 From: Luca Weiss Date: Fri, 31 Aug 2018 14:40:17 +0200 Subject: [PATCH 01/84] Fix build of Docker image with docker-compose ... and fix a typo --- contrib/docker/docker-compose.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/contrib/docker/docker-compose.yml b/contrib/docker/docker-compose.yml index 3a8dfbae34..61ec700afd 100644 --- a/contrib/docker/docker-compose.yml +++ b/contrib/docker/docker-compose.yml @@ -6,9 +6,11 @@ version: '3' services: synapse: - build: ../.. + build: + context: ../.. + dockerfile: docker/Dockerfile image: docker.io/matrixdotorg/synapse:latest - # Since snyapse does not retry to connect to the database, restart upon + # Since synapse does not retry to connect to the database, restart upon # failure restart: unless-stopped # See the readme for a full documentation of the environment settings From f8825748dd76f3d806b0db14b8e61d267f2ea121 Mon Sep 17 00:00:00 2001 From: Luca Weiss Date: Tue, 11 Sep 2018 12:56:31 +0200 Subject: [PATCH 02/84] changelog.d entry somehow got lost --- changelog.d/3778.misc | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog.d/3778.misc diff --git a/changelog.d/3778.misc b/changelog.d/3778.misc new file mode 100644 index 0000000000..b78a2c9f42 --- /dev/null +++ b/changelog.d/3778.misc @@ -0,0 +1 @@ +Fix build of Docker image with docker-compose From fd99787162113857119c033355548c5b3769a309 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Thu, 27 Sep 2018 14:53:58 -0600 Subject: [PATCH 03/84] Incorporate Dave's work for GDPR login flows As per https://github.com/vector-im/riot-web/issues/7168#issuecomment-419996117 --- synapse/api/constants.py | 1 + synapse/handlers/auth.py | 4 ++++ synapse/rest/client/v2_alpha/auth.py | 20 ++++++++++++++++++++ synapse/rest/client/v2_alpha/register.py | 15 +++++++++++++++ 4 files changed, 40 insertions(+) diff --git a/synapse/api/constants.py b/synapse/api/constants.py index c2630c4c64..b2815da0ab 100644 --- a/synapse/api/constants.py +++ b/synapse/api/constants.py @@ -51,6 +51,7 @@ class LoginType(object): EMAIL_IDENTITY = u"m.login.email.identity" MSISDN = u"m.login.msisdn" RECAPTCHA = u"m.login.recaptcha" + TERMS = u"m.login.terms" DUMMY = u"m.login.dummy" # Only for C/S API v1 diff --git a/synapse/handlers/auth.py b/synapse/handlers/auth.py index 2a5eab124f..f08a2cdd7e 100644 --- a/synapse/handlers/auth.py +++ b/synapse/handlers/auth.py @@ -59,6 +59,7 @@ class AuthHandler(BaseHandler): LoginType.EMAIL_IDENTITY: self._check_email_identity, LoginType.MSISDN: self._check_msisdn, LoginType.DUMMY: self._check_dummy_auth, + LoginType.TERMS: self._check_terms_auth, } self.bcrypt_rounds = hs.config.bcrypt_rounds @@ -431,6 +432,9 @@ class AuthHandler(BaseHandler): def _check_dummy_auth(self, authdict, _): return defer.succeed(True) + def _check_terms_auth(self, authdict, _): + return defer.succeed(True) + @defer.inlineCallbacks def _check_threepid(self, medium, authdict): if 'threepid_creds' not in authdict: diff --git a/synapse/rest/client/v2_alpha/auth.py b/synapse/rest/client/v2_alpha/auth.py index bd8b5f4afa..bc3bfee4a0 100644 --- a/synapse/rest/client/v2_alpha/auth.py +++ b/synapse/rest/client/v2_alpha/auth.py @@ -130,6 +130,26 @@ class AuthRestServlet(RestServlet): request.setHeader(b"Content-Type", b"text/html; charset=utf-8") request.setHeader(b"Content-Length", b"%d" % (len(html_bytes),)) + request.write(html_bytes) + finish_request(request) + defer.returnValue(None) + elif stagetype == LoginType.TERMS: + session = request.args['session'][0] + authdict = { + 'session': session, + } + success = yield self.auth_handler.add_oob_auth( + LoginType.TERMS, + authdict, + self.hs.get_ip_from_request(request) + ) + + html = "hai" + html_bytes = html.encode("utf8") + request.setResponseCode(200) + request.setHeader(b"Content-Type", b"text/html; charset=utf-8") + request.setHeader(b"Content-Length", b"%d" % (len(html_bytes),)) + request.write(html_bytes) finish_request(request) defer.returnValue(None) diff --git a/synapse/rest/client/v2_alpha/register.py b/synapse/rest/client/v2_alpha/register.py index 192f52e462..dedf5269ed 100644 --- a/synapse/rest/client/v2_alpha/register.py +++ b/synapse/rest/client/v2_alpha/register.py @@ -359,6 +359,21 @@ class RegisterRestServlet(RestServlet): [LoginType.MSISDN, LoginType.EMAIL_IDENTITY] ]) + if self.hs.config.block_events_without_consent_error is not None: + new_flows = [] + for flow in flows: + # To only allow registration if completing GDPR auth, + # making clients that don't support it use fallback auth. + #flow.append(LoginType.TERMS) + + # or to duplicate all the flows above with the GDPR flow on the + # end so clients that support it can use it but clients that don't + # continue to consent via the DM from server notices bot. + new_flows.extend([ + flow + [LoginType.TERMS] + ]) + flows.extend(new_flows) + auth_result, params, session_id = yield self.auth_handler.check_auth( flows, body, self.hs.get_ip_from_request(request) ) From 149c4f176563bd8c976d9c4601825753f7292b12 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Wed, 3 Oct 2018 15:25:53 -0600 Subject: [PATCH 04/84] Supply params for terms auth stage As per https://github.com/matrix-org/matrix-doc/pull/1692 --- synapse/handlers/auth.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/synapse/handlers/auth.py b/synapse/handlers/auth.py index f08a2cdd7e..d6a19b74e9 100644 --- a/synapse/handlers/auth.py +++ b/synapse/handlers/auth.py @@ -466,6 +466,15 @@ class AuthHandler(BaseHandler): def _get_params_recaptcha(self): return {"public_key": self.hs.config.recaptcha_public_key} + def _get_params_terms(self): + return { + "policies": [{ + "name": "Privacy Policy", + "version": self.hs.config.user_consent_version, + "url": "%s/_matrix/consent/public" % (self.hs.config.public_baseurl,), + }], + } + def _auth_dict_for_flows(self, flows, session): public_flows = [] for f in flows: From 3099d96dba1c5a24cdd81575f6b8b8e07a9e8c94 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Wed, 3 Oct 2018 15:54:19 -0600 Subject: [PATCH 05/84] Flesh out the fallback auth for terms --- synapse/rest/client/v2_alpha/auth.py | 74 +++++++++++++++++++++++++--- 1 file changed, 67 insertions(+), 7 deletions(-) diff --git a/synapse/rest/client/v2_alpha/auth.py b/synapse/rest/client/v2_alpha/auth.py index bc3bfee4a0..f86f09adcf 100644 --- a/synapse/rest/client/v2_alpha/auth.py +++ b/synapse/rest/client/v2_alpha/auth.py @@ -68,6 +68,29 @@ function captchaDone() { """ +TERMS_TEMPLATE = """ + + +Authentication + + + + +
+
+

+ Please click the button below if you agree to the + privacy policy of this homeserver. +

+ + +
+
+ + +""" + SUCCESS_TEMPLATE = """ @@ -138,13 +161,16 @@ class AuthRestServlet(RestServlet): authdict = { 'session': session, } - success = yield self.auth_handler.add_oob_auth( - LoginType.TERMS, - authdict, - self.hs.get_ip_from_request(request) - ) - html = "hai" + html = TERMS_TEMPLATE % { + 'session': session, + 'terms_url': "%s/_matrix/consent/public" % ( + self.hs.config.public_baseurl, + ), + 'myurl': "%s/auth/%s/fallback/web" % ( + CLIENT_V2_ALPHA_PREFIX, LoginType.TERMS + ), + } html_bytes = html.encode("utf8") request.setResponseCode(200) request.setHeader(b"Content-Type", b"text/html; charset=utf-8") @@ -159,7 +185,7 @@ class AuthRestServlet(RestServlet): @defer.inlineCallbacks def on_POST(self, request, stagetype): yield - if stagetype == "m.login.recaptcha": + if stagetype == LoginType.RECAPTCHA: if ('g-recaptcha-response' not in request.args or len(request.args['g-recaptcha-response'])) == 0: raise SynapseError(400, "No captcha response supplied") @@ -198,6 +224,40 @@ class AuthRestServlet(RestServlet): request.write(html_bytes) finish_request(request) + defer.returnValue(None) + elif stagetype == LoginType.TERMS: + if ('session' not in request.args or + len(request.args['session'])) == 0: + raise SynapseError(400, "No session supplied") + + session = request.args['session'][0] + authdict = {'session': session} + + success = yield self.auth_handler.add_oob_auth( + LoginType.TERMS, + authdict, + self.hs.get_ip_from_request(request) + ) + + if success: + html = SUCCESS_TEMPLATE + else: + html = TERMS_TEMPLATE % { + 'session': session, + 'terms_url': "%s/_matrix/consent/public" % ( + self.hs.config.public_baseurl, + ), + 'myurl': "%s/auth/%s/fallback/web" % ( + CLIENT_V2_ALPHA_PREFIX, LoginType.TERMS + ), + } + html_bytes = html.encode("utf8") + request.setResponseCode(200) + request.setHeader(b"Content-Type", b"text/html; charset=utf-8") + request.setHeader(b"Content-Length", b"%d" % (len(html_bytes),)) + + request.write(html_bytes) + finish_request(request) defer.returnValue(None) else: raise SynapseError(404, "Unknown auth stage type") From dfcad5fad5fbfac0a9182853d1acfe410e7cd888 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Wed, 3 Oct 2018 15:54:32 -0600 Subject: [PATCH 06/84] Make the terms flow requried --- synapse/rest/client/v2_alpha/register.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/synapse/rest/client/v2_alpha/register.py b/synapse/rest/client/v2_alpha/register.py index dedf5269ed..78e63447a7 100644 --- a/synapse/rest/client/v2_alpha/register.py +++ b/synapse/rest/client/v2_alpha/register.py @@ -364,14 +364,14 @@ class RegisterRestServlet(RestServlet): for flow in flows: # To only allow registration if completing GDPR auth, # making clients that don't support it use fallback auth. - #flow.append(LoginType.TERMS) + flow.append(LoginType.TERMS) # or to duplicate all the flows above with the GDPR flow on the # end so clients that support it can use it but clients that don't # continue to consent via the DM from server notices bot. - new_flows.extend([ - flow + [LoginType.TERMS] - ]) + #new_flows.extend([ + # flow + [LoginType.TERMS] + #]) flows.extend(new_flows) auth_result, params, session_id = yield self.auth_handler.check_auth( From f9d34a763c90811cd53965825799767569ba0e68 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Wed, 3 Oct 2018 15:54:54 -0600 Subject: [PATCH 07/84] Auto-consent to the privacy policy if the user registered with terms --- synapse/rest/client/v2_alpha/register.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/synapse/rest/client/v2_alpha/register.py b/synapse/rest/client/v2_alpha/register.py index 78e63447a7..851ce6e9a4 100644 --- a/synapse/rest/client/v2_alpha/register.py +++ b/synapse/rest/client/v2_alpha/register.py @@ -460,6 +460,12 @@ class RegisterRestServlet(RestServlet): params.get("bind_msisdn") ) + if auth_result and LoginType.TERMS in auth_result: + logger.info("User %s has consented to the privacy policy" % registered_user_id) + yield self.store.user_set_consent_version( + registered_user_id, self.hs.config.user_consent_version, + ) + defer.returnValue((200, return_dict)) def on_OPTIONS(self, _): From 537d0b7b3632789e40cec13f3120151098f11d75 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Wed, 3 Oct 2018 17:50:11 -0600 Subject: [PATCH 08/84] Use a flag rather than a new route for the public policy This also means that the template now has optional parameters, which will need to be documented somehow. --- synapse/handlers/auth.py | 2 +- synapse/rest/client/v2_alpha/auth.py | 4 +-- synapse/rest/consent/consent_resource.py | 36 +++++++++++++++--------- 3 files changed, 25 insertions(+), 17 deletions(-) diff --git a/synapse/handlers/auth.py b/synapse/handlers/auth.py index d6a19b74e9..42d1336d6e 100644 --- a/synapse/handlers/auth.py +++ b/synapse/handlers/auth.py @@ -471,7 +471,7 @@ class AuthHandler(BaseHandler): "policies": [{ "name": "Privacy Policy", "version": self.hs.config.user_consent_version, - "url": "%s/_matrix/consent/public" % (self.hs.config.public_baseurl,), + "url": "%s/_matrix/consent?public=true" % (self.hs.config.public_baseurl,), }], } diff --git a/synapse/rest/client/v2_alpha/auth.py b/synapse/rest/client/v2_alpha/auth.py index f86f09adcf..77a5ea66f3 100644 --- a/synapse/rest/client/v2_alpha/auth.py +++ b/synapse/rest/client/v2_alpha/auth.py @@ -164,7 +164,7 @@ class AuthRestServlet(RestServlet): html = TERMS_TEMPLATE % { 'session': session, - 'terms_url': "%s/_matrix/consent/public" % ( + 'terms_url': "%s/_matrix/consent?public=true" % ( self.hs.config.public_baseurl, ), 'myurl': "%s/auth/%s/fallback/web" % ( @@ -244,7 +244,7 @@ class AuthRestServlet(RestServlet): else: html = TERMS_TEMPLATE % { 'session': session, - 'terms_url': "%s/_matrix/consent/public" % ( + 'terms_url': "%s/_matrix/consent?public=true" % ( self.hs.config.public_baseurl, ), 'myurl': "%s/auth/%s/fallback/web" % ( diff --git a/synapse/rest/consent/consent_resource.py b/synapse/rest/consent/consent_resource.py index 7362e1858d..7a5786f164 100644 --- a/synapse/rest/consent/consent_resource.py +++ b/synapse/rest/consent/consent_resource.py @@ -30,7 +30,7 @@ from twisted.web.server import NOT_DONE_YET from synapse.api.errors import NotFoundError, StoreError, SynapseError from synapse.config import ConfigError from synapse.http.server import finish_request, wrap_html_request_handler -from synapse.http.servlet import parse_string +from synapse.http.servlet import parse_string, parse_boolean from synapse.types import UserID # language to use for the templates. TODO: figure this out from Accept-Language @@ -137,27 +137,35 @@ class ConsentResource(Resource): request (twisted.web.http.Request): """ - version = parse_string(request, "v", - default=self._default_consent_version) - username = parse_string(request, "u", required=True) - userhmac = parse_string(request, "h", required=True, encoding=None) + public_version = parse_boolean(request, "public", default=False) - self._check_hash(username, userhmac) + version = self._default_consent_version + username = None + userhmac = None + has_consented = False + if not public_version: + version = parse_string(request, "v", + default=self._default_consent_version) + username = parse_string(request, "u", required=True) + userhmac = parse_string(request, "h", required=True, encoding=None) - if username.startswith('@'): - qualified_user_id = username - else: - qualified_user_id = UserID(username, self.hs.hostname).to_string() + self._check_hash(username, userhmac) - u = yield self.store.get_user_by_id(qualified_user_id) - if u is None: - raise NotFoundError("Unknown user") + if username.startswith('@'): + qualified_user_id = username + else: + qualified_user_id = UserID(username, self.hs.hostname).to_string() + + u = yield self.store.get_user_by_id(qualified_user_id) + if u is None: + raise NotFoundError("Unknown user") + has_consented = u["consent_version"] == version try: self._render_template( request, "%s.html" % (version,), user=username, userhmac=userhmac, version=version, - has_consented=(u["consent_version"] == version), + has_consented=has_consented, public_version=public_version, ) except TemplateNotFound: raise NotFoundError("Unknown policy version") From 158d6c75b6a9d62c710ea612e1961fbb5772fd34 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Wed, 3 Oct 2018 17:54:08 -0600 Subject: [PATCH 09/84] Changelog --- changelog.d/4004.feature | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog.d/4004.feature diff --git a/changelog.d/4004.feature b/changelog.d/4004.feature new file mode 100644 index 0000000000..ef5cdaf5ec --- /dev/null +++ b/changelog.d/4004.feature @@ -0,0 +1 @@ +Add `m.login.terms` to the registration flow when consent tracking is enabled. **This makes the template arguments conditionally optional on a new `public_version` variable - update your privacy templates to support this.** From 17d585753f1df2b2c2b13ddb8171e174cef97aac Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 4 Oct 2018 15:18:52 +0100 Subject: [PATCH 10/84] Delete unreferened state groups during purge --- synapse/storage/events.py | 33 +++++++++++++++++++++----- synapse/storage/state.py | 50 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 77 insertions(+), 6 deletions(-) diff --git a/synapse/storage/events.py b/synapse/storage/events.py index e7487311ce..0fb190530a 100644 --- a/synapse/storage/events.py +++ b/synapse/storage/events.py @@ -2025,6 +2025,7 @@ class EventsStore(EventFederationStore, EventsWorkerStore, BackgroundUpdateStore logger.info("[purge] finding state groups which depend on redundant" " state groups") remaining_state_groups = [] + unreferenced_state_groups = 0 for i in range(0, len(state_rows), 100): chunk = [sg for sg, in state_rows[i:i + 100]] # look for state groups whose prev_state_group is one we are about @@ -2037,13 +2038,33 @@ class EventsStore(EventFederationStore, EventsWorkerStore, BackgroundUpdateStore retcols=["state_group"], keyvalues={}, ) - remaining_state_groups.extend( - row["state_group"] for row in rows - # exclude state groups we are about to delete: no point in - # updating them - if row["state_group"] not in state_groups_to_delete - ) + for row in rows: + sg = row["state_group"] + + if sg in state_groups_to_delete: + # exclude state groups we are about to delete: no point in + # updating them + continue + + if not self._is_state_group_referenced(txn, sg): + # Let's also delete unreferenced state groups while we're + # here, since otherwise we'd need to de-delta them + state_groups_to_delete.add(sg) + unreferenced_state_groups += 1 + continue + + remaining_state_groups.append(sg) + + logger.info( + "[purge] found %i extra unreferenced state groups to delete", + unreferenced_state_groups, + ) + + logger.info( + "[purge] de-delta-ing %i remaining state groups", + len(remaining_state_groups), + ) # Now we turn the state groups that reference to-be-deleted state # groups to non delta versions. diff --git a/synapse/storage/state.py b/synapse/storage/state.py index 3f4cbd61c4..b88c7dc091 100644 --- a/synapse/storage/state.py +++ b/synapse/storage/state.py @@ -1041,6 +1041,56 @@ class StateGroupWorkerStore(EventsWorkerStore, SQLBaseStore): return count + def _is_state_group_referenced(self, txn, state_group): + """Checks if a given state group is referenced, or is safe to delete. + + A state groups is referenced if it or any of its descendants are + pointed at by an event. (A descendant is a group which has the given + state_group as a prev group) + """ + + # We check this by doing a depth first search to look for any + # descendant referenced by `event_to_state_groups`. + + # State groups we need to check, contains state groups that are + # descendants of `state_group` + state_groups_to_search = [state_group] + + # Set of state groups we've already checked + state_groups_searched = set() + + while state_groups_to_search: + state_group = state_groups_to_search.pop() # Next state group to check + + is_referenced = self._simple_select_one_onecol_txn( + txn, + table="event_to_state_groups", + keyvalues={"state_group": state_group}, + retcol="event_id", + allow_none=True, + ) + if is_referenced: + # A descendant is referenced by event_to_state_groups, so + # original state group is referenced. + return True + + state_groups_searched.add(state_group) + + # Find all children of current state group and add to search + references = self._simple_select_onecol_txn( + txn, + table="state_group_edges", + keyvalues={"prev_state_group": state_group}, + retcol="state_group", + ) + state_groups_to_search.extend(references) + + # Lets be paranoid and check for cycles + if state_groups_searched.intersection(references): + raise Exception("State group %s has cyclic dependency", state_group) + + return False + class StateStore(StateGroupWorkerStore, BackgroundUpdateStore): """ Keeps track of the state at a given event. From 4917ff55234718c0e650c6dc2a1117304465b9be Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 4 Oct 2018 15:24:01 +0100 Subject: [PATCH 11/84] Add state_group index to event_to_state_groups This is needed to efficiently check for unreferenced state groups during purge. --- synapse/storage/prepare_database.py | 2 +- .../52/add_event_to_state_group_index.sql | 19 +++++++++++++++++++ synapse/storage/state.py | 7 +++++++ 3 files changed, 27 insertions(+), 1 deletion(-) create mode 100644 synapse/storage/schema/delta/52/add_event_to_state_group_index.sql diff --git a/synapse/storage/prepare_database.py b/synapse/storage/prepare_database.py index b364719312..bd740e1e45 100644 --- a/synapse/storage/prepare_database.py +++ b/synapse/storage/prepare_database.py @@ -25,7 +25,7 @@ logger = logging.getLogger(__name__) # Remember to update this number every time a change is made to database # schema files, so the users will be informed on server restarts. -SCHEMA_VERSION = 51 +SCHEMA_VERSION = 52 dir_path = os.path.abspath(os.path.dirname(__file__)) diff --git a/synapse/storage/schema/delta/52/add_event_to_state_group_index.sql b/synapse/storage/schema/delta/52/add_event_to_state_group_index.sql new file mode 100644 index 0000000000..91e03d13e1 --- /dev/null +++ b/synapse/storage/schema/delta/52/add_event_to_state_group_index.sql @@ -0,0 +1,19 @@ +/* Copyright 2018 New Vector Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +-- This is needed to efficiently check for unreferenced state groups during +-- purge. Added events_to_state_group(state_group) index +INSERT into background_updates (update_name, progress_json) + VALUES ('event_to_state_groups_sg_index', '{}'); diff --git a/synapse/storage/state.py b/synapse/storage/state.py index b88c7dc091..3f08f447a3 100644 --- a/synapse/storage/state.py +++ b/synapse/storage/state.py @@ -1114,6 +1114,7 @@ class StateStore(StateGroupWorkerStore, BackgroundUpdateStore): STATE_GROUP_DEDUPLICATION_UPDATE_NAME = "state_group_state_deduplication" STATE_GROUP_INDEX_UPDATE_NAME = "state_group_state_type_index" CURRENT_STATE_INDEX_UPDATE_NAME = "current_state_members_idx" + EVENT_STATE_GROUP_INDEX_UPDATE_NAME = "event_to_state_groups_sg_index" def __init__(self, db_conn, hs): super(StateStore, self).__init__(db_conn, hs) @@ -1132,6 +1133,12 @@ class StateStore(StateGroupWorkerStore, BackgroundUpdateStore): columns=["state_key"], where_clause="type='m.room.member'", ) + self.register_background_index_update( + self.EVENT_STATE_GROUP_INDEX_UPDATE_NAME, + index_name="event_to_state_groups_sg_index", + table="event_to_state_groups", + columns=["state_group"], + ) def _store_event_state_mappings_txn(self, txn, events_and_contexts): state_groups = {} From d9f3db5081a4306520cb3c3208564d3c923fbf0a Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 4 Oct 2018 15:29:30 +0100 Subject: [PATCH 12/84] Newsfile --- changelog.d/4006.misc | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog.d/4006.misc diff --git a/changelog.d/4006.misc b/changelog.d/4006.misc new file mode 100644 index 0000000000..35ffa1c2d2 --- /dev/null +++ b/changelog.d/4006.misc @@ -0,0 +1 @@ +Delete unreferenced state groups during history purge From 67a1e315cc7f8b270ec87e0ab6e58aa2adaee32d Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 12 Oct 2018 13:49:48 +0100 Subject: [PATCH 13/84] Fix up comments --- synapse/storage/state.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/synapse/storage/state.py b/synapse/storage/state.py index 3f08f447a3..f7cf5c86c9 100644 --- a/synapse/storage/state.py +++ b/synapse/storage/state.py @@ -1044,9 +1044,9 @@ class StateGroupWorkerStore(EventsWorkerStore, SQLBaseStore): def _is_state_group_referenced(self, txn, state_group): """Checks if a given state group is referenced, or is safe to delete. - A state groups is referenced if it or any of its descendants are - pointed at by an event. (A descendant is a group which has the given - state_group as a prev group) + A state group is referenced if it or any of its descendants are + pointed at by an event. (A descendant is a state_group whose chain of + prev_groups includes the given state_group.) """ # We check this by doing a depth first search to look for any From 22a20044280d6c0a16ad9e94baf486046d536e5c Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Fri, 12 Oct 2018 17:53:14 -0600 Subject: [PATCH 14/84] Update documentation and templates for new consent --- docs/consent_tracking.md | 13 +++++++++---- docs/privacy_policy_templates/en/1.0.html | 15 +++++++++------ 2 files changed, 18 insertions(+), 10 deletions(-) diff --git a/docs/consent_tracking.md b/docs/consent_tracking.md index 064eae82f7..3634d13d4f 100644 --- a/docs/consent_tracking.md +++ b/docs/consent_tracking.md @@ -31,7 +31,7 @@ Note that the templates must be stored under a name giving the language of the template - currently this must always be `en` (for "English"); internationalisation support is intended for the future. -The template for the policy itself should be versioned and named according to +The template for the policy itself should be versioned and named according to the version: for example `1.0.html`. The version of the policy which the user has agreed to is stored in the database. @@ -81,9 +81,9 @@ should be a matter of `pip install Jinja2`. On debian, try `apt-get install python-jinja2`. Once this is complete, and the server has been restarted, try visiting -`https:///_matrix/consent`. If correctly configured, this should give -an error "Missing string query parameter 'u'". It is now possible to manually -construct URIs where users can give their consent. +`https:///_matrix/consent`. If correctly configured, you should see a +default policy document. It is now possible to manually construct URIs where +users can give their consent. ### Constructing the consent URI @@ -106,6 +106,11 @@ query parameters: `https:///_matrix/consent?u=&h=68a152465a4d...`. +Note that not providing a `u` parameter will be interpreted as wanting to view +the document from an unauthenticated perspective, such as prior to registration. +Therefore, the `h` parameter is not required in this scenario. + + Sending users a server notice asking them to agree to the policy ---------------------------------------------------------------- diff --git a/docs/privacy_policy_templates/en/1.0.html b/docs/privacy_policy_templates/en/1.0.html index 55c5e4b612..321c7e4671 100644 --- a/docs/privacy_policy_templates/en/1.0.html +++ b/docs/privacy_policy_templates/en/1.0.html @@ -12,12 +12,15 @@

All your base are belong to us.

-
- - - - -
+ {% if not public_version %} + +
+ + + + +
+ {% endif %} {% endif %} From 5119818e9d7dac97854868af102476df57f599e5 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Fri, 12 Oct 2018 17:54:28 -0600 Subject: [PATCH 15/84] Rely on the lack of ?u to represent public access also general cleanup --- synapse/rest/client/v2_alpha/auth.py | 4 ++-- synapse/rest/consent/consent_resource.py | 12 ++++-------- 2 files changed, 6 insertions(+), 10 deletions(-) diff --git a/synapse/rest/client/v2_alpha/auth.py b/synapse/rest/client/v2_alpha/auth.py index 77a5ea66f3..ec583ad16a 100644 --- a/synapse/rest/client/v2_alpha/auth.py +++ b/synapse/rest/client/v2_alpha/auth.py @@ -164,7 +164,7 @@ class AuthRestServlet(RestServlet): html = TERMS_TEMPLATE % { 'session': session, - 'terms_url': "%s/_matrix/consent?public=true" % ( + 'terms_url': "%s/_matrix/consent" % ( self.hs.config.public_baseurl, ), 'myurl': "%s/auth/%s/fallback/web" % ( @@ -244,7 +244,7 @@ class AuthRestServlet(RestServlet): else: html = TERMS_TEMPLATE % { 'session': session, - 'terms_url': "%s/_matrix/consent?public=true" % ( + 'terms_url': "%s/_matrix/consent" % ( self.hs.config.public_baseurl, ), 'myurl': "%s/auth/%s/fallback/web" % ( diff --git a/synapse/rest/consent/consent_resource.py b/synapse/rest/consent/consent_resource.py index 7a5786f164..4cadd71d7e 100644 --- a/synapse/rest/consent/consent_resource.py +++ b/synapse/rest/consent/consent_resource.py @@ -30,7 +30,7 @@ from twisted.web.server import NOT_DONE_YET from synapse.api.errors import NotFoundError, StoreError, SynapseError from synapse.config import ConfigError from synapse.http.server import finish_request, wrap_html_request_handler -from synapse.http.servlet import parse_string, parse_boolean +from synapse.http.servlet import parse_string from synapse.types import UserID # language to use for the templates. TODO: figure this out from Accept-Language @@ -137,16 +137,12 @@ class ConsentResource(Resource): request (twisted.web.http.Request): """ - public_version = parse_boolean(request, "public", default=False) - - version = self._default_consent_version - username = None + version = parse_string(request, "v", default=self._default_consent_version) + username = parse_string(request, "u", required=False, default="") userhmac = None has_consented = False + public_version = username != "" if not public_version: - version = parse_string(request, "v", - default=self._default_consent_version) - username = parse_string(request, "u", required=True) userhmac = parse_string(request, "h", required=True, encoding=None) self._check_hash(username, userhmac) From dd99db846d76d511fc7bbea80897b9101782ec1f Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Fri, 12 Oct 2018 18:03:27 -0600 Subject: [PATCH 16/84] Update login terms structure for the proposed language support --- synapse/handlers/auth.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/synapse/handlers/auth.py b/synapse/handlers/auth.py index 42d1336d6e..9038fee264 100644 --- a/synapse/handlers/auth.py +++ b/synapse/handlers/auth.py @@ -468,11 +468,14 @@ class AuthHandler(BaseHandler): def _get_params_terms(self): return { - "policies": [{ - "name": "Privacy Policy", + "policies": { + "privacy_policy": { "version": self.hs.config.user_consent_version, - "url": "%s/_matrix/consent?public=true" % (self.hs.config.public_baseurl,), - }], + "en": { + "name": "Privacy Policy", + "url": "%s/_matrix/consent" % (self.hs.config.public_baseurl,), + }, + }, } def _auth_dict_for_flows(self, flows, session): From 762a0982aab04ebec1e7a00bc03d26aefa8461c4 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Mon, 15 Oct 2018 14:46:09 -0600 Subject: [PATCH 17/84] Python is hard --- synapse/handlers/auth.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/synapse/handlers/auth.py b/synapse/handlers/auth.py index 9038fee264..f1befeb575 100644 --- a/synapse/handlers/auth.py +++ b/synapse/handlers/auth.py @@ -469,11 +469,12 @@ class AuthHandler(BaseHandler): def _get_params_terms(self): return { "policies": { - "privacy_policy": { - "version": self.hs.config.user_consent_version, - "en": { - "name": "Privacy Policy", - "url": "%s/_matrix/consent" % (self.hs.config.public_baseurl,), + "privacy_policy": { + "version": self.hs.config.user_consent_version, + "en": { + "name": "Privacy Policy", + "url": "%s/_matrix/consent" % (self.hs.config.public_baseurl,), + }, }, }, } From 442734ff9e7a4ac09c54a58f8b5467379673914f Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Mon, 15 Oct 2018 14:56:13 -0600 Subject: [PATCH 18/84] Ensure the terms params are actually provided --- synapse/handlers/auth.py | 1 + 1 file changed, 1 insertion(+) diff --git a/synapse/handlers/auth.py b/synapse/handlers/auth.py index f1befeb575..12979f6ed3 100644 --- a/synapse/handlers/auth.py +++ b/synapse/handlers/auth.py @@ -486,6 +486,7 @@ class AuthHandler(BaseHandler): get_params = { LoginType.RECAPTCHA: self._get_params_recaptcha, + LoginType.TERMS: self._get_params_terms, } params = {} From a8ed93a4b55a19a478c9aba929bfea07e691abbf Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Mon, 15 Oct 2018 16:10:29 -0600 Subject: [PATCH 19/84] pep8 --- synapse/handlers/auth.py | 2 +- synapse/rest/client/v2_alpha/auth.py | 3 --- synapse/rest/client/v2_alpha/register.py | 12 ++---------- 3 files changed, 3 insertions(+), 14 deletions(-) diff --git a/synapse/handlers/auth.py b/synapse/handlers/auth.py index 12979f6ed3..bef796fd0c 100644 --- a/synapse/handlers/auth.py +++ b/synapse/handlers/auth.py @@ -469,7 +469,7 @@ class AuthHandler(BaseHandler): def _get_params_terms(self): return { "policies": { - "privacy_policy": { + "privacy_policy": { "version": self.hs.config.user_consent_version, "en": { "name": "Privacy Policy", diff --git a/synapse/rest/client/v2_alpha/auth.py b/synapse/rest/client/v2_alpha/auth.py index ec583ad16a..0b2933fe8e 100644 --- a/synapse/rest/client/v2_alpha/auth.py +++ b/synapse/rest/client/v2_alpha/auth.py @@ -158,9 +158,6 @@ class AuthRestServlet(RestServlet): defer.returnValue(None) elif stagetype == LoginType.TERMS: session = request.args['session'][0] - authdict = { - 'session': session, - } html = TERMS_TEMPLATE % { 'session': session, diff --git a/synapse/rest/client/v2_alpha/register.py b/synapse/rest/client/v2_alpha/register.py index 851ce6e9a4..c5214330ad 100644 --- a/synapse/rest/client/v2_alpha/register.py +++ b/synapse/rest/client/v2_alpha/register.py @@ -359,19 +359,11 @@ class RegisterRestServlet(RestServlet): [LoginType.MSISDN, LoginType.EMAIL_IDENTITY] ]) + # Append m.login.terms to all flows if we're requiring consent if self.hs.config.block_events_without_consent_error is not None: new_flows = [] for flow in flows: - # To only allow registration if completing GDPR auth, - # making clients that don't support it use fallback auth. flow.append(LoginType.TERMS) - - # or to duplicate all the flows above with the GDPR flow on the - # end so clients that support it can use it but clients that don't - # continue to consent via the DM from server notices bot. - #new_flows.extend([ - # flow + [LoginType.TERMS] - #]) flows.extend(new_flows) auth_result, params, session_id = yield self.auth_handler.check_auth( @@ -461,7 +453,7 @@ class RegisterRestServlet(RestServlet): ) if auth_result and LoginType.TERMS in auth_result: - logger.info("User %s has consented to the privacy policy" % registered_user_id) + logger.info("%s has consented to the privacy policy" % registered_user_id) yield self.store.user_set_consent_version( registered_user_id, self.hs.config.user_consent_version, ) From 88c5ffec336116d1d6eafd31b2c91a87d7d43ff7 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Thu, 18 Oct 2018 12:35:30 -0600 Subject: [PATCH 20/84] Test for terms UI auth --- tests/rest/client/v2_alpha/test_register.py | 70 +++++++++++++++++++++ 1 file changed, 70 insertions(+) diff --git a/tests/rest/client/v2_alpha/test_register.py b/tests/rest/client/v2_alpha/test_register.py index 1c128e81f5..a802e1a406 100644 --- a/tests/rest/client/v2_alpha/test_register.py +++ b/tests/rest/client/v2_alpha/test_register.py @@ -174,3 +174,73 @@ class RegisterRestServletTestCase(unittest.TestCase): self.assertEquals(channel.result["code"], b"403", channel.result) self.assertEquals(channel.json_body["error"], "Guest access is disabled") + + def test_POST_terms_auth(self): + self.hs.config.block_events_without_consent_error = True + self.hs.config.public_baseurl = "https://example.org" + self.hs.config.user_consent_version = "1.0" + + # Do a UI auth request + reqest, channel = make_request(b"POST", self.url, b"{}") + render(request, self.resource, self.clock) + + self.assertEquals(channel.result["code"], b"401", channel.result) + + self.assertIsInstance(channel.json_body["session"], str) + + self.assertIsInstance(channel.json_body["flows"], list) + for flow in channel.json_body["flows"]: + self.assertIsInstance(flow["stages"], list) + self.assertTrue(len(flow["stages"]) > 0) + self.assertEquals(flow["stages"][-1], "m.login.terms") + + expected_params = { + "m.login.terms": { + "policies": { + "privacy_policy": { + "en": { + "name": "Privacy Policy", + "url": "https://example.org/_matrix/consent", + }, + "version": "1.0" + }, + }, + }, + } + self.assertIsInstance(channel.json_body["params"], dict) + self.assertDictContainsSubset(channel.json_body["params"], expected_params) + + # Completing the stage should result in the stage being completed + + user_id = "@kermit:muppet" + token = "kermits_access_token" + device_id = "frogfone" + request_data = json.dumps( + { + "username": "kermit", + "password": "monkey", + "device_id": device_id, + "session": channel.json_body["session"], + } + ) + self.registration_handler.check_username = Mock(return_value=True) + self.auth_result = (None, {"username": "kermit", "password": "monkey"}, None) + self.registration_handler.register = Mock(return_value=(user_id, None)) + self.auth_handler.get_access_token_for_user_id = Mock(return_value=token) + self.device_handler.check_device_registered = Mock(return_value=device_id) + + + request, channel = make_request(b"POST", self.url, request_data) + render(request, self.resource, self.clock) + + det_data = { + "user_id": user_id, + "access_token": token, + "home_server": self.hs.hostname, + "device_id": device_id, + } + self.assertEquals(channel.result["code"], b"200", channel.result) + self.assertDictContainsSubset(det_data, channel.json_body) + self.auth_handler.get_login_tuple_for_user_id( + user_id, device_id=device_id, initial_device_display_name=None + ) From dba84fa69c55c61d347169d5210bb63f65849fb5 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Thu, 18 Oct 2018 12:45:21 -0600 Subject: [PATCH 21/84] Fix terms UI auth test --- tests/rest/client/v2_alpha/test_register.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/tests/rest/client/v2_alpha/test_register.py b/tests/rest/client/v2_alpha/test_register.py index a802e1a406..36eaabbad8 100644 --- a/tests/rest/client/v2_alpha/test_register.py +++ b/tests/rest/client/v2_alpha/test_register.py @@ -181,7 +181,7 @@ class RegisterRestServletTestCase(unittest.TestCase): self.hs.config.user_consent_version = "1.0" # Do a UI auth request - reqest, channel = make_request(b"POST", self.url, b"{}") + request, channel = make_request(b"POST", self.url, b"{}") render(request, self.resource, self.clock) self.assertEquals(channel.result["code"], b"401", channel.result) @@ -220,7 +220,10 @@ class RegisterRestServletTestCase(unittest.TestCase): "username": "kermit", "password": "monkey", "device_id": device_id, - "session": channel.json_body["session"], + "auth": { + "session": channel.json_body["session"], + "type": "m.login.terms", + }, } ) self.registration_handler.check_username = Mock(return_value=True) From 47a9da28caa6a4f27d2df31f043971a2c9c7b555 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 12 Oct 2018 20:43:18 +0100 Subject: [PATCH 22/84] Batch process handling state groups --- synapse/storage/events.py | 81 ++++++++---------------------- synapse/storage/state.py | 100 +++++++++++++++++++++++++------------- 2 files changed, 86 insertions(+), 95 deletions(-) diff --git a/synapse/storage/events.py b/synapse/storage/events.py index 0fb190530a..e4d0f8b1a9 100644 --- a/synapse/storage/events.py +++ b/synapse/storage/events.py @@ -37,6 +37,7 @@ from synapse.metrics.background_process_metrics import run_as_background_process from synapse.storage.background_updates import BackgroundUpdateStore from synapse.storage.event_federation import EventFederationStore from synapse.storage.events_worker import EventsWorkerStore +from synapse.storage.state import StateGroupWorkerStore from synapse.types import RoomStreamToken, get_domain_from_id from synapse.util.async_helpers import ObservableDeferred from synapse.util.caches.descriptors import cached, cachedInlineCallbacks @@ -203,7 +204,8 @@ def _retry_on_integrity_error(func): # inherits from EventFederationStore so that we can call _update_backward_extremities # and _handle_mult_prev_events (though arguably those could both be moved in here) -class EventsStore(EventFederationStore, EventsWorkerStore, BackgroundUpdateStore): +class EventsStore(StateGroupWorkerStore, EventFederationStore, EventsWorkerStore, + BackgroundUpdateStore): EVENT_ORIGIN_SERVER_TS_NAME = "event_origin_server_ts" EVENT_FIELDS_SENDER_URL_UPDATE_NAME = "event_fields_sender_url" @@ -1995,70 +1997,29 @@ class EventsStore(EventFederationStore, EventsWorkerStore, BackgroundUpdateStore logger.info("[purge] finding redundant state groups") - # Get all state groups that are only referenced by events that are - # to be deleted. - # This works by first getting state groups that we may want to delete, - # joining against event_to_state_groups to get events that use that - # state group, then left joining against events_to_purge again. Any - # state group where the left join produce *no nulls* are referenced - # only by events that are going to be purged. + # Get all state groups that are referenced by events that are to be + # deleted. We then go and check if they are referenced by other events + # or state groups, and if not we delete them. txn.execute(""" - SELECT state_group FROM - ( - SELECT DISTINCT state_group FROM events_to_purge - INNER JOIN event_to_state_groups USING (event_id) - ) AS sp - INNER JOIN event_to_state_groups USING (state_group) - LEFT JOIN events_to_purge AS ep USING (event_id) - GROUP BY state_group - HAVING SUM(CASE WHEN ep.event_id IS NULL THEN 1 ELSE 0 END) = 0 + SELECT DISTINCT state_group FROM events_to_purge + INNER JOIN event_to_state_groups USING (event_id) """) - state_rows = txn.fetchall() - logger.info("[purge] found %i redundant state groups", len(state_rows)) + referenced_state_groups = set(sg for sg, in txn) + logger.info( + "[purge] found %i referenced state groups", + len(referenced_state_groups), + ) - # make a set of the redundant state groups, so that we can look them up - # efficiently - state_groups_to_delete = set([sg for sg, in state_rows]) + logger.info("[purge] finding state groups that can be deleted") - # Now we get all the state groups that rely on these state groups - logger.info("[purge] finding state groups which depend on redundant" - " state groups") - remaining_state_groups = [] - unreferenced_state_groups = 0 - for i in range(0, len(state_rows), 100): - chunk = [sg for sg, in state_rows[i:i + 100]] - # look for state groups whose prev_state_group is one we are about - # to delete - rows = self._simple_select_many_txn( - txn, - table="state_group_edges", - column="prev_state_group", - iterable=chunk, - retcols=["state_group"], - keyvalues={}, - ) - - for row in rows: - sg = row["state_group"] - - if sg in state_groups_to_delete: - # exclude state groups we are about to delete: no point in - # updating them - continue - - if not self._is_state_group_referenced(txn, sg): - # Let's also delete unreferenced state groups while we're - # here, since otherwise we'd need to de-delta them - state_groups_to_delete.add(sg) - unreferenced_state_groups += 1 - continue - - remaining_state_groups.append(sg) + state_groups_to_delete, remaining_state_groups = self._find_unreferenced_groups( + txn, referenced_state_groups, + ) logger.info( - "[purge] found %i extra unreferenced state groups to delete", - unreferenced_state_groups, + "[purge] found %i state groups to delete", + len(state_groups_to_delete), ) logger.info( @@ -2109,11 +2070,11 @@ class EventsStore(EventFederationStore, EventsWorkerStore, BackgroundUpdateStore logger.info("[purge] removing redundant state groups") txn.executemany( "DELETE FROM state_groups_state WHERE state_group = ?", - state_rows + ((sg,) for sg in state_groups_to_delete), ) txn.executemany( "DELETE FROM state_groups WHERE id = ?", - state_rows + ((sg,) for sg in state_groups_to_delete), ) logger.info("[purge] removing events from event_to_state_groups") diff --git a/synapse/storage/state.py b/synapse/storage/state.py index f7cf5c86c9..0f86311ed4 100644 --- a/synapse/storage/state.py +++ b/synapse/storage/state.py @@ -1041,55 +1041,85 @@ class StateGroupWorkerStore(EventsWorkerStore, SQLBaseStore): return count - def _is_state_group_referenced(self, txn, state_group): - """Checks if a given state group is referenced, or is safe to delete. + def _find_unreferenced_groups(self, txn, state_groups): + """Used when purging history to figure out which state groups can be + deleted and which need to be de-delta'ed (due to one of its prev groups + being scheduled for deletion). - A state group is referenced if it or any of its descendants are - pointed at by an event. (A descendant is a state_group whose chain of - prev_groups includes the given state_group.) + Args: + txn + state_groups (set[int]): Set of state groups referenced by events + that are going to be deleted. + + Returns: + tuple[set[int], set[int]]: The set of state groups that can be + deleted and the set of state groups that need to be de-delta'ed """ + # Graph of state group -> previous group + graph = {} - # We check this by doing a depth first search to look for any - # descendant referenced by `event_to_state_groups`. + # Set of events that we have found to be referenced by events + referenced_groups = set() - # State groups we need to check, contains state groups that are - # descendants of `state_group` - state_groups_to_search = [state_group] + # Set of state groups we've already seen + state_groups_seen = set(state_groups) - # Set of state groups we've already checked - state_groups_searched = set() + # Set of state groups to handle next. + next_to_search = set(state_groups) + while next_to_search: + # We bound size of groups we're looking up at once, to stop the + # SQL query getting too big + if len(next_to_search) < 100: + current_search = next_to_search + next_to_search = set() + else: + lst = list(next_to_search) + current_search = set(lst[:100]) + next_to_search = set(lst[100:]) - while state_groups_to_search: - state_group = state_groups_to_search.pop() # Next state group to check + # Check if state groups are referenced + sql = """ + SELECT state_group, count(*) FROM event_to_state_groups + LEFT JOIN events_to_purge AS ep USING (event_id) + WHERE state_group IN (%s) AND ep.event_id IS NULL + GROUP BY state_group + """ % (",".join("?" for _ in current_search),) + txn.execute(sql, list(current_search)) - is_referenced = self._simple_select_one_onecol_txn( - txn, - table="event_to_state_groups", - keyvalues={"state_group": state_group}, - retcol="event_id", - allow_none=True, - ) - if is_referenced: - # A descendant is referenced by event_to_state_groups, so - # original state group is referenced. - return True + referenced = set(sg for sg, cnt in txn if cnt > 0) + referenced_groups |= referenced - state_groups_searched.add(state_group) + # We don't continue iterating up the state group graphs for state + # groups that are referenced. + current_search -= referenced - # Find all children of current state group and add to search - references = self._simple_select_onecol_txn( + rows = self._simple_select_many_txn( txn, table="state_group_edges", - keyvalues={"prev_state_group": state_group}, - retcol="state_group", + column="prev_state_group", + iterable=current_search, + keyvalues={}, + retcols=("prev_state_group", "state_group",), ) - state_groups_to_search.extend(references) - # Lets be paranoid and check for cycles - if state_groups_searched.intersection(references): - raise Exception("State group %s has cyclic dependency", state_group) + next_to_search.update(row["state_group"] for row in rows) + # We don't bother re-handling groups we've already seen + next_to_search -= state_groups_seen + state_groups_seen |= next_to_search - return False + for row in rows: + # Note: Each state group can have at most one prev group + graph[row["state_group"]] = row["prev_state_group"] + + to_delete = state_groups_seen - referenced_groups + + to_dedelta = set() + for sg in referenced_groups: + prev_sg = graph.get(sg) + if prev_sg and prev_sg in to_delete: + to_dedelta.add(sg) + + return to_delete, to_dedelta class StateStore(StateGroupWorkerStore, BackgroundUpdateStore): From 67f7b9cb50c246226b94027e3c1d4b958ff9f840 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 19 Oct 2018 16:06:59 +0100 Subject: [PATCH 23/84] pep8 --- synapse/storage/events.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/storage/events.py b/synapse/storage/events.py index af822fb69d..379b5c514f 100644 --- a/synapse/storage/events.py +++ b/synapse/storage/events.py @@ -2041,7 +2041,7 @@ class EventsStore(StateGroupWorkerStore, EventFederationStore, EventsWorkerStore INNER JOIN event_to_state_groups USING (event_id) """) - referenced_state_groups = set(sg for sg, in txn) + referenced_state_groups = set(sg for sg, in txn) logger.info( "[purge] found %i referenced state groups", len(referenced_state_groups), From 9283987f7e5c7b678ddf01ffcac888917877ae63 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Thu, 18 Oct 2018 13:03:46 -0600 Subject: [PATCH 24/84] Fix test Debug tests Try printing the channel fix Import and use six Remove debugging Disable captcha Add some mocks Define the URL Fix the clock? Less rendering? use the other render Complete the dummy auth stage Fix last stage of the test Remove mocks we don't need --- tests/rest/client/v2_alpha/test_register.py | 79 ++++++++++++++------- 1 file changed, 52 insertions(+), 27 deletions(-) diff --git a/tests/rest/client/v2_alpha/test_register.py b/tests/rest/client/v2_alpha/test_register.py index 36eaabbad8..5dbd16fd9b 100644 --- a/tests/rest/client/v2_alpha/test_register.py +++ b/tests/rest/client/v2_alpha/test_register.py @@ -1,4 +1,5 @@ import json +import six from mock import Mock @@ -175,18 +176,35 @@ class RegisterRestServletTestCase(unittest.TestCase): self.assertEquals(channel.result["code"], b"403", channel.result) self.assertEquals(channel.json_body["error"], "Guest access is disabled") + +class TermsTestCase(unittest.HomeserverTestCase): + servlets = [register_servlets] + + def prepare(self, reactor, clock, hs): + self.clock = MemoryReactorClock() + self.hs_clock = Clock(self.clock) + self.url = "/_matrix/client/r0/register" + self.registration_handler = Mock() + self.auth_handler = Mock() + self.device_handler = Mock() + hs.config.enable_registration = True + hs.config.registrations_require_3pid = [] + hs.config.auto_join_rooms = [] + hs.config.enable_registration_captcha = False + def test_POST_terms_auth(self): self.hs.config.block_events_without_consent_error = True self.hs.config.public_baseurl = "https://example.org" self.hs.config.user_consent_version = "1.0" # Do a UI auth request - request, channel = make_request(b"POST", self.url, b"{}") - render(request, self.resource, self.clock) + request, channel = self.make_request(b"POST", self.url, b"{}") + self.render(request) self.assertEquals(channel.result["code"], b"401", channel.result) - self.assertIsInstance(channel.json_body["session"], str) + self.assertTrue(channel.json_body is not None) + self.assertIsInstance(channel.json_body["session"], six.text_type) self.assertIsInstance(channel.json_body["flows"], list) for flow in channel.json_body["flows"]: @@ -210,40 +228,47 @@ class RegisterRestServletTestCase(unittest.TestCase): self.assertIsInstance(channel.json_body["params"], dict) self.assertDictContainsSubset(channel.json_body["params"], expected_params) - # Completing the stage should result in the stage being completed - - user_id = "@kermit:muppet" - token = "kermits_access_token" - device_id = "frogfone" + # We have to complete the dummy auth stage before completing the terms stage + request_data = json.dumps( + { + "username": "kermit", + "password": "monkey", + "auth": { + "session": channel.json_body["session"], + "type": "m.login.dummy", + }, + } + ) + + self.registration_handler.check_username = Mock(return_value=True) + + request, channel = make_request(b"POST", self.url, request_data) + self.render(request) + + # We don't bother checking that the response is correct - we'll leave that to + # other tests. We just want to make sure we're on the right path. + self.assertEquals(channel.result["code"], b"401", channel.result) + + # Finish the UI auth for terms request_data = json.dumps( { "username": "kermit", "password": "monkey", - "device_id": device_id, "auth": { "session": channel.json_body["session"], "type": "m.login.terms", }, } ) - self.registration_handler.check_username = Mock(return_value=True) - self.auth_result = (None, {"username": "kermit", "password": "monkey"}, None) - self.registration_handler.register = Mock(return_value=(user_id, None)) - self.auth_handler.get_access_token_for_user_id = Mock(return_value=token) - self.device_handler.check_device_registered = Mock(return_value=device_id) - - request, channel = make_request(b"POST", self.url, request_data) - render(request, self.resource, self.clock) + self.render(request) + + # We're interested in getting a response that looks like a successful registration, + # not so much that the details are exactly what we want. - det_data = { - "user_id": user_id, - "access_token": token, - "home_server": self.hs.hostname, - "device_id": device_id, - } self.assertEquals(channel.result["code"], b"200", channel.result) - self.assertDictContainsSubset(det_data, channel.json_body) - self.auth_handler.get_login_tuple_for_user_id( - user_id, device_id=device_id, initial_device_display_name=None - ) + + self.assertTrue(channel.json_body is not None) + self.assertIsInstance(channel.json_body["user_id"], six.text_type) + self.assertIsInstance(channel.json_body["access_token"], six.text_type) + self.assertIsInstance(channel.json_body["device_id"], six.text_type) From 4acb6fe8a3c09e0b69bca047c5918ce44d5927f1 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Wed, 24 Oct 2018 13:24:24 -0600 Subject: [PATCH 25/84] Move test to where the other integration tests are --- tests/rest/client/v2_alpha/test_register.py | 97 ----------------- tests/test_terms_auth.py | 109 ++++++++++++++++++++ 2 files changed, 109 insertions(+), 97 deletions(-) create mode 100644 tests/test_terms_auth.py diff --git a/tests/rest/client/v2_alpha/test_register.py b/tests/rest/client/v2_alpha/test_register.py index 5dbd16fd9b..5deae1c4dd 100644 --- a/tests/rest/client/v2_alpha/test_register.py +++ b/tests/rest/client/v2_alpha/test_register.py @@ -175,100 +175,3 @@ class RegisterRestServletTestCase(unittest.TestCase): self.assertEquals(channel.result["code"], b"403", channel.result) self.assertEquals(channel.json_body["error"], "Guest access is disabled") - - -class TermsTestCase(unittest.HomeserverTestCase): - servlets = [register_servlets] - - def prepare(self, reactor, clock, hs): - self.clock = MemoryReactorClock() - self.hs_clock = Clock(self.clock) - self.url = "/_matrix/client/r0/register" - self.registration_handler = Mock() - self.auth_handler = Mock() - self.device_handler = Mock() - hs.config.enable_registration = True - hs.config.registrations_require_3pid = [] - hs.config.auto_join_rooms = [] - hs.config.enable_registration_captcha = False - - def test_POST_terms_auth(self): - self.hs.config.block_events_without_consent_error = True - self.hs.config.public_baseurl = "https://example.org" - self.hs.config.user_consent_version = "1.0" - - # Do a UI auth request - request, channel = self.make_request(b"POST", self.url, b"{}") - self.render(request) - - self.assertEquals(channel.result["code"], b"401", channel.result) - - self.assertTrue(channel.json_body is not None) - self.assertIsInstance(channel.json_body["session"], six.text_type) - - self.assertIsInstance(channel.json_body["flows"], list) - for flow in channel.json_body["flows"]: - self.assertIsInstance(flow["stages"], list) - self.assertTrue(len(flow["stages"]) > 0) - self.assertEquals(flow["stages"][-1], "m.login.terms") - - expected_params = { - "m.login.terms": { - "policies": { - "privacy_policy": { - "en": { - "name": "Privacy Policy", - "url": "https://example.org/_matrix/consent", - }, - "version": "1.0" - }, - }, - }, - } - self.assertIsInstance(channel.json_body["params"], dict) - self.assertDictContainsSubset(channel.json_body["params"], expected_params) - - # We have to complete the dummy auth stage before completing the terms stage - request_data = json.dumps( - { - "username": "kermit", - "password": "monkey", - "auth": { - "session": channel.json_body["session"], - "type": "m.login.dummy", - }, - } - ) - - self.registration_handler.check_username = Mock(return_value=True) - - request, channel = make_request(b"POST", self.url, request_data) - self.render(request) - - # We don't bother checking that the response is correct - we'll leave that to - # other tests. We just want to make sure we're on the right path. - self.assertEquals(channel.result["code"], b"401", channel.result) - - # Finish the UI auth for terms - request_data = json.dumps( - { - "username": "kermit", - "password": "monkey", - "auth": { - "session": channel.json_body["session"], - "type": "m.login.terms", - }, - } - ) - request, channel = make_request(b"POST", self.url, request_data) - self.render(request) - - # We're interested in getting a response that looks like a successful registration, - # not so much that the details are exactly what we want. - - self.assertEquals(channel.result["code"], b"200", channel.result) - - self.assertTrue(channel.json_body is not None) - self.assertIsInstance(channel.json_body["user_id"], six.text_type) - self.assertIsInstance(channel.json_body["access_token"], six.text_type) - self.assertIsInstance(channel.json_body["device_id"], six.text_type) diff --git a/tests/test_terms_auth.py b/tests/test_terms_auth.py new file mode 100644 index 0000000000..0d95ae09b7 --- /dev/null +++ b/tests/test_terms_auth.py @@ -0,0 +1,109 @@ +# Copyright 2018 New Vector Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +class TermsTestCase(unittest.HomeserverTestCase): + servlets = [register_servlets] + + def prepare(self, reactor, clock, hs): + self.clock = MemoryReactorClock() + self.hs_clock = Clock(self.clock) + self.url = "/_matrix/client/r0/register" + self.registration_handler = Mock() + self.auth_handler = Mock() + self.device_handler = Mock() + hs.config.enable_registration = True + hs.config.registrations_require_3pid = [] + hs.config.auto_join_rooms = [] + hs.config.enable_registration_captcha = False + + def test_ui_auth(self): + self.hs.config.block_events_without_consent_error = True + self.hs.config.public_baseurl = "https://example.org" + self.hs.config.user_consent_version = "1.0" + + # Do a UI auth request + request, channel = self.make_request(b"POST", self.url, b"{}") + self.render(request) + + self.assertEquals(channel.result["code"], b"401", channel.result) + + self.assertTrue(channel.json_body is not None) + self.assertIsInstance(channel.json_body["session"], six.text_type) + + self.assertIsInstance(channel.json_body["flows"], list) + for flow in channel.json_body["flows"]: + self.assertIsInstance(flow["stages"], list) + self.assertTrue(len(flow["stages"]) > 0) + self.assertEquals(flow["stages"][-1], "m.login.terms") + + expected_params = { + "m.login.terms": { + "policies": { + "privacy_policy": { + "en": { + "name": "Privacy Policy", + "url": "https://example.org/_matrix/consent", + }, + "version": "1.0" + }, + }, + }, + } + self.assertIsInstance(channel.json_body["params"], dict) + self.assertDictContainsSubset(channel.json_body["params"], expected_params) + + # We have to complete the dummy auth stage before completing the terms stage + request_data = json.dumps( + { + "username": "kermit", + "password": "monkey", + "auth": { + "session": channel.json_body["session"], + "type": "m.login.dummy", + }, + } + ) + + self.registration_handler.check_username = Mock(return_value=True) + + request, channel = make_request(b"POST", self.url, request_data) + self.render(request) + + # We don't bother checking that the response is correct - we'll leave that to + # other tests. We just want to make sure we're on the right path. + self.assertEquals(channel.result["code"], b"401", channel.result) + + # Finish the UI auth for terms + request_data = json.dumps( + { + "username": "kermit", + "password": "monkey", + "auth": { + "session": channel.json_body["session"], + "type": "m.login.terms", + }, + } + ) + request, channel = make_request(b"POST", self.url, request_data) + self.render(request) + + # We're interested in getting a response that looks like a successful registration, + # not so much that the details are exactly what we want. + + self.assertEquals(channel.result["code"], b"200", channel.result) + + self.assertTrue(channel.json_body is not None) + self.assertIsInstance(channel.json_body["user_id"], six.text_type) + self.assertIsInstance(channel.json_body["access_token"], six.text_type) + self.assertIsInstance(channel.json_body["device_id"], six.text_type) From 81880beff497c516946e28eb5d119ee60cad69e5 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Wed, 24 Oct 2018 13:32:13 -0600 Subject: [PATCH 26/84] It helps to import things --- tests/test_terms_auth.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/tests/test_terms_auth.py b/tests/test_terms_auth.py index 0d95ae09b7..b1b0f2a8c6 100644 --- a/tests/test_terms_auth.py +++ b/tests/test_terms_auth.py @@ -12,6 +12,22 @@ # See the License for the specific language governing permissions and # limitations under the License. +import json + +import six +from mock import Mock + +from twisted.test.proto_helpers import MemoryReactorClock + +from synapse.api.errors import InteractiveAuthIncompleteError +from synapse.http.server import JsonResource +from synapse.rest.client.v2_alpha.register import register_servlets +from synapse.util import Clock + +from tests import unittest +from tests.server import make_request, render, setup_test_homeserver + + class TermsTestCase(unittest.HomeserverTestCase): servlets = [register_servlets] From a5468eaadfb80588ac10b6e1dd69f1282e65b544 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Wed, 24 Oct 2018 13:54:38 -0600 Subject: [PATCH 27/84] pep8 --- tests/rest/client/v2_alpha/test_register.py | 1 - tests/test_terms_auth.py | 8 +++----- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/tests/rest/client/v2_alpha/test_register.py b/tests/rest/client/v2_alpha/test_register.py index 5deae1c4dd..1c128e81f5 100644 --- a/tests/rest/client/v2_alpha/test_register.py +++ b/tests/rest/client/v2_alpha/test_register.py @@ -1,5 +1,4 @@ import json -import six from mock import Mock diff --git a/tests/test_terms_auth.py b/tests/test_terms_auth.py index b1b0f2a8c6..06b68f0a72 100644 --- a/tests/test_terms_auth.py +++ b/tests/test_terms_auth.py @@ -19,13 +19,11 @@ from mock import Mock from twisted.test.proto_helpers import MemoryReactorClock -from synapse.api.errors import InteractiveAuthIncompleteError -from synapse.http.server import JsonResource from synapse.rest.client.v2_alpha.register import register_servlets from synapse.util import Clock from tests import unittest -from tests.server import make_request, render, setup_test_homeserver +from tests.server import make_request class TermsTestCase(unittest.HomeserverTestCase): @@ -114,8 +112,8 @@ class TermsTestCase(unittest.HomeserverTestCase): request, channel = make_request(b"POST", self.url, request_data) self.render(request) - # We're interested in getting a response that looks like a successful registration, - # not so much that the details are exactly what we want. + # We're interested in getting a response that looks like a successful + # registration, not so much that the details are exactly what we want. self.assertEquals(channel.result["code"], b"200", channel.result) From 871c4abfecfd14acda13e3f25c7d040f848a9a32 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Thu, 25 Oct 2018 17:40:41 +0100 Subject: [PATCH 28/84] Factor _generate_room_id out of create_room we're going to need this for room upgrades. --- synapse/handlers/room.py | 45 ++++++++++++++++++++-------------------- 1 file changed, 23 insertions(+), 22 deletions(-) diff --git a/synapse/handlers/room.py b/synapse/handlers/room.py index 3ba92bdb4c..000a22b07c 100644 --- a/synapse/handlers/room.py +++ b/synapse/handlers/room.py @@ -165,28 +165,7 @@ class RoomCreationHandler(BaseHandler): visibility = config.get("visibility", None) is_public = visibility == "public" - # autogen room IDs and try to create it. We may clash, so just - # try a few times till one goes through, giving up eventually. - attempts = 0 - room_id = None - while attempts < 5: - try: - random_string = stringutils.random_string(18) - gen_room_id = RoomID( - random_string, - self.hs.hostname, - ) - yield self.store.store_room( - room_id=gen_room_id.to_string(), - room_creator_user_id=user_id, - is_public=is_public - ) - room_id = gen_room_id.to_string() - break - except StoreError: - attempts += 1 - if not room_id: - raise StoreError(500, "Couldn't generate a room ID.") + room_id = yield self._generate_room_id(creator_id=user_id, is_public=is_public) if room_alias: directory_handler = self.hs.get_handlers().directory_handler @@ -427,6 +406,28 @@ class RoomCreationHandler(BaseHandler): content=content, ) + @defer.inlineCallbacks + def _generate_room_id(self, creator_id, is_public): + # autogen room IDs and try to create it. We may clash, so just + # try a few times till one goes through, giving up eventually. + attempts = 0 + while attempts < 5: + try: + random_string = stringutils.random_string(18) + gen_room_id = RoomID( + random_string, + self.hs.hostname, + ).to_string() + yield self.store.store_room( + room_id=gen_room_id, + room_creator_user_id=creator_id, + is_public=is_public, + ) + defer.returnValue(gen_room_id) + except StoreError: + attempts += 1 + raise StoreError(500, "Couldn't generate a room ID.") + class RoomContextHandler(object): def __init__(self, hs): From 7f7b2cd3de192816bcb0225774a22617989aec37 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Thu, 25 Oct 2018 17:42:37 +0100 Subject: [PATCH 29/84] Make room_member_handler a member of RoomCreationHandler ... to save passing it into `_send_events_for_new_room` --- synapse/handlers/register.py | 6 ++++-- synapse/handlers/room.py | 9 +++------ 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/synapse/handlers/register.py b/synapse/handlers/register.py index e9d7b25a36..7b4549223f 100644 --- a/synapse/handlers/register.py +++ b/synapse/handlers/register.py @@ -50,7 +50,6 @@ class RegistrationHandler(BaseHandler): self._auth_handler = hs.get_auth_handler() self.profile_handler = hs.get_profile_handler() self.user_directory_handler = hs.get_user_directory_handler() - self.room_creation_handler = self.hs.get_room_creation_handler() self.captcha_client = CaptchaServerHttpClient(hs) self._next_generated_user_id = None @@ -241,7 +240,10 @@ class RegistrationHandler(BaseHandler): else: # create room expects the localpart of the room alias room_alias_localpart = room_alias.localpart - yield self.room_creation_handler.create_room( + + # getting the RoomCreationHandler during init gives a dependency + # loop + yield self.hs.get_room_creation_handler().create_room( fake_requester, config={ "preset": "public_chat", diff --git a/synapse/handlers/room.py b/synapse/handlers/room.py index 000a22b07c..d03d2cd7be 100644 --- a/synapse/handlers/room.py +++ b/synapse/handlers/room.py @@ -73,6 +73,7 @@ class RoomCreationHandler(BaseHandler): self.spam_checker = hs.get_spam_checker() self.event_creation_handler = hs.get_event_creation_handler() + self.room_member_handler = hs.get_room_member_handler() @defer.inlineCallbacks def create_room(self, requester, config, ratelimit=True, @@ -195,12 +196,9 @@ class RoomCreationHandler(BaseHandler): # override any attempt to set room versions via the creation_content creation_content["room_version"] = room_version - room_member_handler = self.hs.get_room_member_handler() - yield self._send_events_for_new_room( requester, room_id, - room_member_handler, preset_config=preset_config, invite_list=invite_list, initial_state=initial_state, @@ -242,7 +240,7 @@ class RoomCreationHandler(BaseHandler): if is_direct: content["is_direct"] = is_direct - yield room_member_handler.update_membership( + yield self.room_member_handler.update_membership( requester, UserID.from_string(invitee), room_id, @@ -280,7 +278,6 @@ class RoomCreationHandler(BaseHandler): self, creator, # A Requester object. room_id, - room_member_handler, preset_config, invite_list, initial_state, @@ -325,7 +322,7 @@ class RoomCreationHandler(BaseHandler): content=creation_content, ) - yield room_member_handler.update_membership( + yield self.room_member_handler.update_membership( creator, creator.user, room_id, From e1948175ee7fc469c985b58a01ecc2eb577e5e0a Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Thu, 25 Oct 2018 17:50:06 +0100 Subject: [PATCH 30/84] Allow power_level_content_override=None for _send_events_for_new_room --- synapse/handlers/room.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/synapse/handlers/room.py b/synapse/handlers/room.py index d03d2cd7be..d42c2c41c4 100644 --- a/synapse/handlers/room.py +++ b/synapse/handlers/room.py @@ -204,7 +204,7 @@ class RoomCreationHandler(BaseHandler): initial_state=initial_state, creation_content=creation_content, room_alias=room_alias, - power_level_content_override=config.get("power_level_content_override", {}), + power_level_content_override=config.get("power_level_content_override"), creator_join_profile=creator_join_profile, ) @@ -282,9 +282,9 @@ class RoomCreationHandler(BaseHandler): invite_list, initial_state, creation_content, - room_alias, - power_level_content_override, - creator_join_profile, + room_alias=None, + power_level_content_override=None, + creator_join_profile=None, ): def create(etype, content, **kwargs): e = { @@ -364,7 +364,8 @@ class RoomCreationHandler(BaseHandler): for invitee in invite_list: power_level_content["users"][invitee] = 100 - power_level_content.update(power_level_content_override) + if power_level_content_override: + power_level_content.update(power_level_content_override) yield send( etype=EventTypes.PowerLevels, From 0f7d1c99061075fe54a37cfe785184f095addf78 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Wed, 22 Aug 2018 10:57:54 +0100 Subject: [PATCH 31/84] Basic initial support for room upgrades Currently just creates a new, empty, room, and sends a tombstone in the old room. --- synapse/api/constants.py | 1 + synapse/handlers/room.py | 121 ++++++++++++++++++ synapse/rest/__init__.py | 2 + .../v2_alpha/room_upgrade_rest_servlet.py | 78 +++++++++++ synapse/server.pyi | 6 + 5 files changed, 208 insertions(+) create mode 100644 synapse/rest/client/v2_alpha/room_upgrade_rest_servlet.py diff --git a/synapse/api/constants.py b/synapse/api/constants.py index c2630c4c64..5565e516d6 100644 --- a/synapse/api/constants.py +++ b/synapse/api/constants.py @@ -61,6 +61,7 @@ class LoginType(object): class EventTypes(object): Member = "m.room.member" Create = "m.room.create" + Tombstone = "m.room.tombstone" JoinRules = "m.room.join_rules" PowerLevels = "m.room.power_levels" Aliases = "m.room.aliases" diff --git a/synapse/handlers/room.py b/synapse/handlers/room.py index d42c2c41c4..3cce6f6150 100644 --- a/synapse/handlers/room.py +++ b/synapse/handlers/room.py @@ -36,6 +36,7 @@ from synapse.api.errors import AuthError, Codes, StoreError, SynapseError from synapse.storage.state import StateFilter from synapse.types import RoomAlias, RoomID, RoomStreamToken, StreamToken, UserID from synapse.util import stringutils +from synapse.util.async_helpers import Linearizer from synapse.visibility import filter_events_for_client from ._base import BaseHandler @@ -75,6 +76,124 @@ class RoomCreationHandler(BaseHandler): self.event_creation_handler = hs.get_event_creation_handler() self.room_member_handler = hs.get_room_member_handler() + # linearizer to stop two upgrades happening at once + self._upgrade_linearizer = Linearizer("room_upgrade_linearizer") + + @defer.inlineCallbacks + def upgrade_room(self, requester, old_room_id, new_version): + """Replace a room with a new room with a different version + + Args: + requester (synapse.types.Requester): the user requesting the upgrade + old_room_id (unicode): the id of the room to be replaced + new_version (unicode): the new room version to use + + Returns: + Deferred[unicode]: the new room id + """ + yield self.ratelimit(requester) + + user_id = requester.user.to_string() + + with (yield self._upgrade_linearizer.queue(old_room_id)): + # start by allocating a new room id + is_public = False # XXX fixme + new_room_id = yield self._generate_room_id( + creator_id=user_id, is_public=is_public, + ) + + # we create and auth the tombstone event before properly creating the new + # room, to check our user has perms in the old room. + tombstone_event, tombstone_context = ( + yield self.event_creation_handler.create_event( + requester, { + "type": EventTypes.Tombstone, + "state_key": "", + "room_id": old_room_id, + "sender": user_id, + "content": { + "body": "This room has been replaced", + "replacement_room": new_room_id, + } + }, + token_id=requester.access_token_id, + ) + ) + yield self.auth.check_from_context(tombstone_event, tombstone_context) + + yield self.clone_exiting_room( + requester, + old_room_id=old_room_id, + new_room_id=new_room_id, + new_room_version=new_version, + tombstone_event_id=tombstone_event.event_id, + ) + + # now send the tombstone + yield self.event_creation_handler.send_nonmember_event( + requester, tombstone_event, tombstone_context, + ) + + # XXX send a power_levels in the old room, if possible + + defer.returnValue(new_room_id) + + @defer.inlineCallbacks + def clone_exiting_room( + self, requester, old_room_id, new_room_id, new_room_version, + tombstone_event_id, + ): + """Populate a new room based on an old room + + Args: + requester (synapse.types.Requester): the user requesting the upgrade + old_room_id (unicode): the id of the room to be replaced + new_room_id (unicode): the id to give the new room (should already have been + created with _gemerate_room_id()) + new_room_version (unicode): the new room version to use + tombstone_event_id (unicode|str): the ID of the tombstone event in the old + room. + Returns: + Deferred[None] + """ + user_id = requester.user.to_string() + + if not self.spam_checker.user_may_create_room(user_id): + raise SynapseError(403, "You are not permitted to create rooms") + + # XXX check alias is free + # canonical_alias = None + + # XXX create association in directory handler + # XXX preset + + preset_config = RoomCreationPreset.PRIVATE_CHAT + + creation_content = { + "room_version": new_room_version, + "predecessor": { + "room_id": old_room_id, + "event_id": tombstone_event_id, + } + } + + initial_state = OrderedDict() + + yield self._send_events_for_new_room( + requester, + new_room_id, + preset_config=preset_config, + invite_list=[], + initial_state=initial_state, + creation_content=creation_content, + ) + + # XXX name + # XXX topic + # XXX invites/joins + # XXX 3pid invites + # XXX directory_handler.send_room_alias_update_event + @defer.inlineCallbacks def create_room(self, requester, config, ratelimit=True, creator_join_profile=None): @@ -416,6 +535,8 @@ class RoomCreationHandler(BaseHandler): random_string, self.hs.hostname, ).to_string() + if isinstance(gen_room_id, bytes): + gen_room_id = gen_room_id.decode('utf-8') yield self.store.store_room( room_id=gen_room_id, room_creator_user_id=creator_id, diff --git a/synapse/rest/__init__.py b/synapse/rest/__init__.py index 4856822a5d..5f35c2d1be 100644 --- a/synapse/rest/__init__.py +++ b/synapse/rest/__init__.py @@ -47,6 +47,7 @@ from synapse.rest.client.v2_alpha import ( register, report_event, room_keys, + room_upgrade_rest_servlet, sendtodevice, sync, tags, @@ -116,3 +117,4 @@ class ClientRestResource(JsonResource): sendtodevice.register_servlets(hs, client_resource) user_directory.register_servlets(hs, client_resource) groups.register_servlets(hs, client_resource) + room_upgrade_rest_servlet.register_servlets(hs, client_resource) diff --git a/synapse/rest/client/v2_alpha/room_upgrade_rest_servlet.py b/synapse/rest/client/v2_alpha/room_upgrade_rest_servlet.py new file mode 100644 index 0000000000..1b195f90c4 --- /dev/null +++ b/synapse/rest/client/v2_alpha/room_upgrade_rest_servlet.py @@ -0,0 +1,78 @@ +# -*- coding: utf-8 -*- +# Copyright 2016 OpenMarket Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging + +from twisted.internet import defer + +from synapse.api.constants import KNOWN_ROOM_VERSIONS +from synapse.api.errors import Codes, SynapseError +from synapse.http.servlet import ( + RestServlet, + assert_params_in_dict, + parse_json_object_from_request, +) + +from ._base import client_v2_patterns + +logger = logging.getLogger(__name__) + + +class RoomUpgradeRestServlet(RestServlet): + PATTERNS = client_v2_patterns( + # /rooms/$roomid/upgrade + "/rooms/(?P[^/]*)/upgrade$", + v2_alpha=False, + ) + + def __init__(self, hs): + """ + + Args: + hs (synapse.server.HomeServer): + """ + super(RoomUpgradeRestServlet, self).__init__() + self._hs = hs + self._room_creation_handler = hs.get_room_creation_handler() + self._auth = hs.get_auth() + + @defer.inlineCallbacks + def on_POST(self, request, room_id): + requester = yield self._auth.get_user_by_req(request) + + content = parse_json_object_from_request(request) + assert_params_in_dict(content, ("new_version", )) + new_version = content["new_version"] + + if new_version not in KNOWN_ROOM_VERSIONS: + raise SynapseError( + 400, + "Your homeserver does not support this room version", + Codes.UNSUPPORTED_ROOM_VERSION, + ) + + new_room_id = yield self._room_creation_handler.upgrade_room( + requester, room_id, new_version + ) + + ret = { + "replacement_room": new_room_id, + } + + defer.returnValue((200, ret)) + + +def register_servlets(hs, http_server): + RoomUpgradeRestServlet(hs).register(http_server) diff --git a/synapse/server.pyi b/synapse/server.pyi index ce28486233..06cd083a74 100644 --- a/synapse/server.pyi +++ b/synapse/server.pyi @@ -7,6 +7,9 @@ import synapse.handlers.auth import synapse.handlers.deactivate_account import synapse.handlers.device import synapse.handlers.e2e_keys +import synapse.handlers.room +import synapse.handlers.room_member +import synapse.handlers.message import synapse.handlers.set_password import synapse.rest.media.v1.media_repository import synapse.server_notices.server_notices_manager @@ -50,6 +53,9 @@ class HomeServer(object): def get_room_creation_handler(self) -> synapse.handlers.room.RoomCreationHandler: pass + def get_room_member_handler(self) -> synapse.handlers.room_member.RoomMemberHandler: + pass + def get_event_creation_handler(self) -> synapse.handlers.message.EventCreationHandler: pass From 4cda300058ba68f97c032923ebf429f437eddd8e Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Fri, 12 Oct 2018 11:13:40 +0100 Subject: [PATCH 32/84] preserve room visibility --- synapse/handlers/room.py | 8 +++++--- synapse/storage/room.py | 2 +- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/synapse/handlers/room.py b/synapse/handlers/room.py index 3cce6f6150..2f9eb8ef4c 100644 --- a/synapse/handlers/room.py +++ b/synapse/handlers/room.py @@ -32,7 +32,7 @@ from synapse.api.constants import ( JoinRules, RoomCreationPreset, ) -from synapse.api.errors import AuthError, Codes, StoreError, SynapseError +from synapse.api.errors import AuthError, Codes, NotFoundError, StoreError, SynapseError from synapse.storage.state import StateFilter from synapse.types import RoomAlias, RoomID, RoomStreamToken, StreamToken, UserID from synapse.util import stringutils @@ -97,9 +97,11 @@ class RoomCreationHandler(BaseHandler): with (yield self._upgrade_linearizer.queue(old_room_id)): # start by allocating a new room id - is_public = False # XXX fixme + r = yield self.store.get_room(old_room_id) + if r is None: + raise NotFoundError("Unknown room id %s" % (old_room_id,)) new_room_id = yield self._generate_room_id( - creator_id=user_id, is_public=is_public, + creator_id=user_id, is_public=r["is_public"], ) # we create and auth the tombstone event before properly creating the new diff --git a/synapse/storage/room.py b/synapse/storage/room.py index 61013b8919..41c65e112a 100644 --- a/synapse/storage/room.py +++ b/synapse/storage/room.py @@ -47,7 +47,7 @@ class RoomWorkerStore(SQLBaseStore): Args: room_id (str): The ID of the room to retrieve. Returns: - A namedtuple containing the room information, or an empty list. + A dict containing the room information, or None if the room is unknown. """ return self._simple_select_one( table="rooms", From 1b9f253e208ea3a471594bde52366e3abf54fc1a Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Fri, 12 Oct 2018 12:05:18 +0100 Subject: [PATCH 33/84] preserve PLs --- synapse/handlers/room.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/synapse/handlers/room.py b/synapse/handlers/room.py index 2f9eb8ef4c..40ca12f1b7 100644 --- a/synapse/handlers/room.py +++ b/synapse/handlers/room.py @@ -179,7 +179,13 @@ class RoomCreationHandler(BaseHandler): } } - initial_state = OrderedDict() + initial_state = dict() + + old_room_state_ids = yield self.store.get_current_state_ids(old_room_id) + pl_event_id = old_room_state_ids.get((EventTypes.PowerLevels, "")) + if pl_event_id: + pl_event = yield self.store.get_event(pl_event_id) + initial_state[(EventTypes.PowerLevels, "")] = pl_event.content yield self._send_events_for_new_room( requester, From 3a263bf3aec6b9709fed391671f8faec334dc739 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Fri, 12 Oct 2018 17:05:48 +0100 Subject: [PATCH 34/84] copy state --- synapse/handlers/room.py | 37 ++++++++++++++++++++++++++----------- 1 file changed, 26 insertions(+), 11 deletions(-) diff --git a/synapse/handlers/room.py b/synapse/handlers/room.py index 40ca12f1b7..ab92ca5e78 100644 --- a/synapse/handlers/room.py +++ b/synapse/handlers/room.py @@ -167,9 +167,6 @@ class RoomCreationHandler(BaseHandler): # canonical_alias = None # XXX create association in directory handler - # XXX preset - - preset_config = RoomCreationPreset.PRIVATE_CHAT creation_content = { "room_version": new_room_version, @@ -181,23 +178,41 @@ class RoomCreationHandler(BaseHandler): initial_state = dict() - old_room_state_ids = yield self.store.get_current_state_ids(old_room_id) - pl_event_id = old_room_state_ids.get((EventTypes.PowerLevels, "")) - if pl_event_id: - pl_event = yield self.store.get_event(pl_event_id) - initial_state[(EventTypes.PowerLevels, "")] = pl_event.content + types_to_copy = ( + (EventTypes.PowerLevels, ""), + (EventTypes.JoinRules, ""), + (EventTypes.Name, ""), + (EventTypes.Topic, ""), + (EventTypes.RoomHistoryVisibility, ""), + (EventTypes.GuestAccess, "") + ) + + old_room_state_ids = yield self.store.get_filtered_current_state_ids( + old_room_id, StateFilter.from_types(types_to_copy), + ) + # map from event_id to BaseEvent + old_room_state_events = yield self.store.get_events(old_room_state_ids.values()) + + for k in types_to_copy: + old_event_id = old_room_state_ids.get(k) + if old_event_id: + old_event = old_room_state_events.get(old_event_id) + if old_event: + initial_state[k] = old_event.content yield self._send_events_for_new_room( requester, new_room_id, - preset_config=preset_config, + + # we expect to override all the presets with initial_state, so this is + # somewhat arbitrary. + preset_config=RoomCreationPreset.PRIVATE_CHAT, + invite_list=[], initial_state=initial_state, creation_content=creation_content, ) - # XXX name - # XXX topic # XXX invites/joins # XXX 3pid invites # XXX directory_handler.send_room_alias_update_event From e6babc27d51c3de04cdaedc40439b7ddb56b2e12 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Wed, 24 Oct 2018 23:14:36 +0100 Subject: [PATCH 35/84] restrict PLs in old room --- synapse/handlers/room.py | 44 +++++++++++++++++++++++++++++++++++++++- 1 file changed, 43 insertions(+), 1 deletion(-) diff --git a/synapse/handlers/room.py b/synapse/handlers/room.py index ab92ca5e78..d016f0e8b8 100644 --- a/synapse/handlers/room.py +++ b/synapse/handlers/room.py @@ -136,7 +136,49 @@ class RoomCreationHandler(BaseHandler): requester, tombstone_event, tombstone_context, ) - # XXX send a power_levels in the old room, if possible + # ... and restrict the PLs in the old room, if possible. + old_room_pl_state = yield self.state_handler.get_current_state( + old_room_id, + event_type=EventTypes.PowerLevels, + latest_event_ids=(tombstone_event.event_id, ), + ) + + if old_room_pl_state is None: + logger.warning( + "Not supported: upgrading a room with no PL event. Not setting PLs " + "in old room.", + ) + else: + pl_content = dict(old_room_pl_state.content) + users_default = int(pl_content.get("users_default", 0)) + restricted_level = max(users_default + 1, 50) + + updated = False + for v in ("invite", "events_default"): + current = int(pl_content.get(v, 0)) + if current < restricted_level: + logger.debug( + "Setting level for %s in %s to %i (was %i)", + v, old_room_id, restricted_level, current, + ) + pl_content[v] = restricted_level + updated = True + else: + logger.debug( + "Not setting level for %s (already %i)", + v, current, + ) + + if updated: + yield self.event_creation_handler.create_and_send_nonmember_event( + requester, { + "type": EventTypes.PowerLevels, + "state_key": '', + "room_id": old_room_id, + "sender": user_id, + "content": pl_content, + }, ratelimit=False, + ) defer.returnValue(new_room_id) From 68c0ce62d839fe3a2014c1df32b761eeb0916b9c Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Thu, 25 Oct 2018 18:17:00 +0100 Subject: [PATCH 36/84] changelog --- changelog.d/4091.feature | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog.d/4091.feature diff --git a/changelog.d/4091.feature b/changelog.d/4091.feature new file mode 100644 index 0000000000..a3f7dbdcdd --- /dev/null +++ b/changelog.d/4091.feature @@ -0,0 +1 @@ +Support for replacing rooms with new ones From 474810d9d545c07cfac567be7b29100a66cb2b7c Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Thu, 25 Oct 2018 23:15:03 +0100 Subject: [PATCH 37/84] fix broken test This test stubbed out some stuff in a very weird way. I have no idea why. It broke. --- .../test_resource_limits_server_notices.py | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/tests/server_notices/test_resource_limits_server_notices.py b/tests/server_notices/test_resource_limits_server_notices.py index 4701eedd45..b1551df7ca 100644 --- a/tests/server_notices/test_resource_limits_server_notices.py +++ b/tests/server_notices/test_resource_limits_server_notices.py @@ -4,7 +4,6 @@ from twisted.internet import defer from synapse.api.constants import EventTypes, ServerNoticeMsgType from synapse.api.errors import ResourceLimitError -from synapse.handlers.auth import AuthHandler from synapse.server_notices.resource_limits_server_notices import ( ResourceLimitsServerNotices, ) @@ -13,17 +12,10 @@ from tests import unittest from tests.utils import setup_test_homeserver -class AuthHandlers(object): - def __init__(self, hs): - self.auth_handler = AuthHandler(hs) - - class TestResourceLimitsServerNotices(unittest.TestCase): @defer.inlineCallbacks def setUp(self): - self.hs = yield setup_test_homeserver(self.addCleanup, handlers=None) - self.hs.handlers = AuthHandlers(self.hs) - self.auth_handler = self.hs.handlers.auth_handler + self.hs = yield setup_test_homeserver(self.addCleanup) self.server_notices_sender = self.hs.get_server_notices_sender() # relying on [1] is far from ideal, but the only case where From 77d70a7646bd30315393b9aec44e452d5739a266 Mon Sep 17 00:00:00 2001 From: Amber Brown Date: Fri, 26 Oct 2018 22:05:22 +1100 Subject: [PATCH 38/84] Port register_new_matrix_user to Python 3 and add tests (#4085) --- changelog.d/4085.feature | 1 + scripts/register_new_matrix_user | 204 +----------------- synapse/_scripts/__init__.py | 0 synapse/_scripts/register_new_matrix_user.py | 215 +++++++++++++++++++ tests/scripts/__init__.py | 0 tests/scripts/test_new_matrix_user.py | 160 ++++++++++++++ 6 files changed, 378 insertions(+), 202 deletions(-) create mode 100644 changelog.d/4085.feature create mode 100644 synapse/_scripts/__init__.py create mode 100644 synapse/_scripts/register_new_matrix_user.py create mode 100644 tests/scripts/__init__.py create mode 100644 tests/scripts/test_new_matrix_user.py diff --git a/changelog.d/4085.feature b/changelog.d/4085.feature new file mode 100644 index 0000000000..4bd3ddcf2c --- /dev/null +++ b/changelog.d/4085.feature @@ -0,0 +1 @@ +The register_new_matrix_user script is now ported to Python 3. diff --git a/scripts/register_new_matrix_user b/scripts/register_new_matrix_user index 89143c5d59..b450712ab7 100755 --- a/scripts/register_new_matrix_user +++ b/scripts/register_new_matrix_user @@ -16,207 +16,7 @@ from __future__ import print_function -import argparse -import getpass -import hashlib -import hmac -import json -import sys -import urllib2 - -from six import input - -import yaml - - -def request_registration(user, password, server_location, shared_secret, admin=False): - req = urllib2.Request( - "%s/_matrix/client/r0/admin/register" % (server_location,), - headers={'Content-Type': 'application/json'}, - ) - - try: - if sys.version_info[:3] >= (2, 7, 9): - # As of version 2.7.9, urllib2 now checks SSL certs - import ssl - - f = urllib2.urlopen(req, context=ssl.SSLContext(ssl.PROTOCOL_SSLv23)) - else: - f = urllib2.urlopen(req) - body = f.read() - f.close() - nonce = json.loads(body)["nonce"] - except urllib2.HTTPError as e: - print("ERROR! Received %d %s" % (e.code, e.reason)) - if 400 <= e.code < 500: - if e.info().type == "application/json": - resp = json.load(e) - if "error" in resp: - print(resp["error"]) - sys.exit(1) - - mac = hmac.new(key=shared_secret, digestmod=hashlib.sha1) - - mac.update(nonce) - mac.update("\x00") - mac.update(user) - mac.update("\x00") - mac.update(password) - mac.update("\x00") - mac.update("admin" if admin else "notadmin") - - mac = mac.hexdigest() - - data = { - "nonce": nonce, - "username": user, - "password": password, - "mac": mac, - "admin": admin, - } - - server_location = server_location.rstrip("/") - - print("Sending registration request...") - - req = urllib2.Request( - "%s/_matrix/client/r0/admin/register" % (server_location,), - data=json.dumps(data), - headers={'Content-Type': 'application/json'}, - ) - try: - if sys.version_info[:3] >= (2, 7, 9): - # As of version 2.7.9, urllib2 now checks SSL certs - import ssl - - f = urllib2.urlopen(req, context=ssl.SSLContext(ssl.PROTOCOL_SSLv23)) - else: - f = urllib2.urlopen(req) - f.read() - f.close() - print("Success.") - except urllib2.HTTPError as e: - print("ERROR! Received %d %s" % (e.code, e.reason)) - if 400 <= e.code < 500: - if e.info().type == "application/json": - resp = json.load(e) - if "error" in resp: - print(resp["error"]) - sys.exit(1) - - -def register_new_user(user, password, server_location, shared_secret, admin): - if not user: - try: - default_user = getpass.getuser() - except Exception: - default_user = None - - if default_user: - user = input("New user localpart [%s]: " % (default_user,)) - if not user: - user = default_user - else: - user = input("New user localpart: ") - - if not user: - print("Invalid user name") - sys.exit(1) - - if not password: - password = getpass.getpass("Password: ") - - if not password: - print("Password cannot be blank.") - sys.exit(1) - - confirm_password = getpass.getpass("Confirm password: ") - - if password != confirm_password: - print("Passwords do not match") - sys.exit(1) - - if admin is None: - admin = input("Make admin [no]: ") - if admin in ("y", "yes", "true"): - admin = True - else: - admin = False - - request_registration(user, password, server_location, shared_secret, bool(admin)) - +from synapse._scripts.register_new_matrix_user import main if __name__ == "__main__": - parser = argparse.ArgumentParser( - description="Used to register new users with a given home server when" - " registration has been disabled. The home server must be" - " configured with the 'registration_shared_secret' option" - " set." - ) - parser.add_argument( - "-u", - "--user", - default=None, - help="Local part of the new user. Will prompt if omitted.", - ) - parser.add_argument( - "-p", - "--password", - default=None, - help="New password for user. Will prompt if omitted.", - ) - admin_group = parser.add_mutually_exclusive_group() - admin_group.add_argument( - "-a", - "--admin", - action="store_true", - help=( - "Register new user as an admin. " - "Will prompt if --no-admin is not set either." - ), - ) - admin_group.add_argument( - "--no-admin", - action="store_true", - help=( - "Register new user as a regular user. " - "Will prompt if --admin is not set either." - ), - ) - - group = parser.add_mutually_exclusive_group(required=True) - group.add_argument( - "-c", - "--config", - type=argparse.FileType('r'), - help="Path to server config file. Used to read in shared secret.", - ) - - group.add_argument( - "-k", "--shared-secret", help="Shared secret as defined in server config file." - ) - - parser.add_argument( - "server_url", - default="https://localhost:8448", - nargs='?', - help="URL to use to talk to the home server. Defaults to " - " 'https://localhost:8448'.", - ) - - args = parser.parse_args() - - if "config" in args and args.config: - config = yaml.safe_load(args.config) - secret = config.get("registration_shared_secret", None) - if not secret: - print("No 'registration_shared_secret' defined in config.") - sys.exit(1) - else: - secret = args.shared_secret - - admin = None - if args.admin or args.no_admin: - admin = args.admin - - register_new_user(args.user, args.password, args.server_url, secret, admin) + main() diff --git a/synapse/_scripts/__init__.py b/synapse/_scripts/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/synapse/_scripts/register_new_matrix_user.py b/synapse/_scripts/register_new_matrix_user.py new file mode 100644 index 0000000000..70cecde486 --- /dev/null +++ b/synapse/_scripts/register_new_matrix_user.py @@ -0,0 +1,215 @@ +# -*- coding: utf-8 -*- +# Copyright 2015, 2016 OpenMarket Ltd +# Copyright 2018 New Vector +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import print_function + +import argparse +import getpass +import hashlib +import hmac +import logging +import sys + +from six.moves import input + +import requests as _requests +import yaml + + +def request_registration( + user, + password, + server_location, + shared_secret, + admin=False, + requests=_requests, + _print=print, + exit=sys.exit, +): + + url = "%s/_matrix/client/r0/admin/register" % (server_location,) + + # Get the nonce + r = requests.get(url, verify=False) + + if r.status_code is not 200: + _print("ERROR! Received %d %s" % (r.status_code, r.reason)) + if 400 <= r.status_code < 500: + try: + _print(r.json()["error"]) + except Exception: + pass + return exit(1) + + nonce = r.json()["nonce"] + + mac = hmac.new(key=shared_secret.encode('utf8'), digestmod=hashlib.sha1) + + mac.update(nonce.encode('utf8')) + mac.update(b"\x00") + mac.update(user.encode('utf8')) + mac.update(b"\x00") + mac.update(password.encode('utf8')) + mac.update(b"\x00") + mac.update(b"admin" if admin else b"notadmin") + + mac = mac.hexdigest() + + data = { + "nonce": nonce, + "username": user, + "password": password, + "mac": mac, + "admin": admin, + } + + _print("Sending registration request...") + r = requests.post(url, json=data, verify=False) + + if r.status_code is not 200: + _print("ERROR! Received %d %s" % (r.status_code, r.reason)) + if 400 <= r.status_code < 500: + try: + _print(r.json()["error"]) + except Exception: + pass + return exit(1) + + _print("Success!") + + +def register_new_user(user, password, server_location, shared_secret, admin): + if not user: + try: + default_user = getpass.getuser() + except Exception: + default_user = None + + if default_user: + user = input("New user localpart [%s]: " % (default_user,)) + if not user: + user = default_user + else: + user = input("New user localpart: ") + + if not user: + print("Invalid user name") + sys.exit(1) + + if not password: + password = getpass.getpass("Password: ") + + if not password: + print("Password cannot be blank.") + sys.exit(1) + + confirm_password = getpass.getpass("Confirm password: ") + + if password != confirm_password: + print("Passwords do not match") + sys.exit(1) + + if admin is None: + admin = input("Make admin [no]: ") + if admin in ("y", "yes", "true"): + admin = True + else: + admin = False + + request_registration(user, password, server_location, shared_secret, bool(admin)) + + +def main(): + + logging.captureWarnings(True) + + parser = argparse.ArgumentParser( + description="Used to register new users with a given home server when" + " registration has been disabled. The home server must be" + " configured with the 'registration_shared_secret' option" + " set." + ) + parser.add_argument( + "-u", + "--user", + default=None, + help="Local part of the new user. Will prompt if omitted.", + ) + parser.add_argument( + "-p", + "--password", + default=None, + help="New password for user. Will prompt if omitted.", + ) + admin_group = parser.add_mutually_exclusive_group() + admin_group.add_argument( + "-a", + "--admin", + action="store_true", + help=( + "Register new user as an admin. " + "Will prompt if --no-admin is not set either." + ), + ) + admin_group.add_argument( + "--no-admin", + action="store_true", + help=( + "Register new user as a regular user. " + "Will prompt if --admin is not set either." + ), + ) + + group = parser.add_mutually_exclusive_group(required=True) + group.add_argument( + "-c", + "--config", + type=argparse.FileType('r'), + help="Path to server config file. Used to read in shared secret.", + ) + + group.add_argument( + "-k", "--shared-secret", help="Shared secret as defined in server config file." + ) + + parser.add_argument( + "server_url", + default="https://localhost:8448", + nargs='?', + help="URL to use to talk to the home server. Defaults to " + " 'https://localhost:8448'.", + ) + + args = parser.parse_args() + + if "config" in args and args.config: + config = yaml.safe_load(args.config) + secret = config.get("registration_shared_secret", None) + if not secret: + print("No 'registration_shared_secret' defined in config.") + sys.exit(1) + else: + secret = args.shared_secret + + admin = None + if args.admin or args.no_admin: + admin = args.admin + + register_new_user(args.user, args.password, args.server_url, secret, admin) + + +if __name__ == "__main__": + main() diff --git a/tests/scripts/__init__.py b/tests/scripts/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/scripts/test_new_matrix_user.py b/tests/scripts/test_new_matrix_user.py new file mode 100644 index 0000000000..6f56893f5e --- /dev/null +++ b/tests/scripts/test_new_matrix_user.py @@ -0,0 +1,160 @@ +# -*- coding: utf-8 -*- +# Copyright 2018 New Vector +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from mock import Mock + +from synapse._scripts.register_new_matrix_user import request_registration + +from tests.unittest import TestCase + + +class RegisterTestCase(TestCase): + def test_success(self): + """ + The script will fetch a nonce, and then generate a MAC with it, and then + post that MAC. + """ + + def get(url, verify=None): + r = Mock() + r.status_code = 200 + r.json = lambda: {"nonce": "a"} + return r + + def post(url, json=None, verify=None): + # Make sure we are sent the correct info + self.assertEqual(json["username"], "user") + self.assertEqual(json["password"], "pass") + self.assertEqual(json["nonce"], "a") + # We want a 40-char hex MAC + self.assertEqual(len(json["mac"]), 40) + + r = Mock() + r.status_code = 200 + return r + + requests = Mock() + requests.get = get + requests.post = post + + # The fake stdout will be written here + out = [] + err_code = [] + + request_registration( + "user", + "pass", + "matrix.org", + "shared", + admin=False, + requests=requests, + _print=out.append, + exit=err_code.append, + ) + + # We should get the success message making sure everything is OK. + self.assertIn("Success!", out) + + # sys.exit shouldn't have been called. + self.assertEqual(err_code, []) + + def test_failure_nonce(self): + """ + If the script fails to fetch a nonce, it throws an error and quits. + """ + + def get(url, verify=None): + r = Mock() + r.status_code = 404 + r.reason = "Not Found" + r.json = lambda: {"not": "error"} + return r + + requests = Mock() + requests.get = get + + # The fake stdout will be written here + out = [] + err_code = [] + + request_registration( + "user", + "pass", + "matrix.org", + "shared", + admin=False, + requests=requests, + _print=out.append, + exit=err_code.append, + ) + + # Exit was called + self.assertEqual(err_code, [1]) + + # We got an error message + self.assertIn("ERROR! Received 404 Not Found", out) + self.assertNotIn("Success!", out) + + def test_failure_post(self): + """ + The script will fetch a nonce, and then if the final POST fails, will + report an error and quit. + """ + + def get(url, verify=None): + r = Mock() + r.status_code = 200 + r.json = lambda: {"nonce": "a"} + return r + + def post(url, json=None, verify=None): + # Make sure we are sent the correct info + self.assertEqual(json["username"], "user") + self.assertEqual(json["password"], "pass") + self.assertEqual(json["nonce"], "a") + # We want a 40-char hex MAC + self.assertEqual(len(json["mac"]), 40) + + r = Mock() + # Then 500 because we're jerks + r.status_code = 500 + r.reason = "Broken" + return r + + requests = Mock() + requests.get = get + requests.post = post + + # The fake stdout will be written here + out = [] + err_code = [] + + request_registration( + "user", + "pass", + "matrix.org", + "shared", + admin=False, + requests=requests, + _print=out.append, + exit=err_code.append, + ) + + # Exit was called + self.assertEqual(err_code, [1]) + + # We got an error message + self.assertIn("ERROR! Received 500 Broken", out) + self.assertNotIn("Success!", out) From 193cadc988801d9035124d1fd3ca23607b9b1f25 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Fri, 26 Oct 2018 17:10:30 +0100 Subject: [PATCH 39/84] Address review comments Improve comments, get old room state from the context we already have --- synapse/handlers/room.py | 16 +++++++------- .../v2_alpha/room_upgrade_rest_servlet.py | 21 ++++++++++++++----- 2 files changed, 25 insertions(+), 12 deletions(-) diff --git a/synapse/handlers/room.py b/synapse/handlers/room.py index d016f0e8b8..145b5b19ee 100644 --- a/synapse/handlers/room.py +++ b/synapse/handlers/room.py @@ -136,19 +136,21 @@ class RoomCreationHandler(BaseHandler): requester, tombstone_event, tombstone_context, ) - # ... and restrict the PLs in the old room, if possible. - old_room_pl_state = yield self.state_handler.get_current_state( - old_room_id, - event_type=EventTypes.PowerLevels, - latest_event_ids=(tombstone_event.event_id, ), - ) + old_room_state = yield tombstone_context.get_current_state_ids(self.store) + old_room_pl_event_id = old_room_state.get((EventTypes.PowerLevels, "")) - if old_room_pl_state is None: + if old_room_pl_event_id is None: logger.warning( "Not supported: upgrading a room with no PL event. Not setting PLs " "in old room.", ) else: + # we try to stop regular users from speaking by setting the PL required + # to send regular events and invites to 'Moderator' level. That's normally + # 50, but if the default PL in a room is 50 or more, then we set the + # required PL above that. + + old_room_pl_state = yield self.store.get_event(old_room_pl_event_id) pl_content = dict(old_room_pl_state.content) users_default = int(pl_content.get("users_default", 0)) restricted_level = max(users_default + 1, 50) diff --git a/synapse/rest/client/v2_alpha/room_upgrade_rest_servlet.py b/synapse/rest/client/v2_alpha/room_upgrade_rest_servlet.py index 1b195f90c4..e6356101fd 100644 --- a/synapse/rest/client/v2_alpha/room_upgrade_rest_servlet.py +++ b/synapse/rest/client/v2_alpha/room_upgrade_rest_servlet.py @@ -31,6 +31,22 @@ logger = logging.getLogger(__name__) class RoomUpgradeRestServlet(RestServlet): + """Handler for room uprade requests. + + Handles requests of the form: + + POST /_matrix/client/r0/rooms/$roomid/upgrade HTTP/1.1 + Content-Type: application/json + + { + "new_version": "2", + } + + Creates a new room and shuts down the old one. Returns the ID of the new room. + + Args: + hs (synapse.server.HomeServer): + """ PATTERNS = client_v2_patterns( # /rooms/$roomid/upgrade "/rooms/(?P[^/]*)/upgrade$", @@ -38,11 +54,6 @@ class RoomUpgradeRestServlet(RestServlet): ) def __init__(self, hs): - """ - - Args: - hs (synapse.server.HomeServer): - """ super(RoomUpgradeRestServlet, self).__init__() self._hs = hs self._room_creation_handler = hs.get_room_creation_handler() From 54bbe71867fb3de2e3984e2b3eb909845c2448b3 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Fri, 26 Oct 2018 22:51:34 +0100 Subject: [PATCH 40/84] optimise state copying --- synapse/handlers/room.py | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/synapse/handlers/room.py b/synapse/handlers/room.py index 145b5b19ee..8e48c1ca6a 100644 --- a/synapse/handlers/room.py +++ b/synapse/handlers/room.py @@ -21,7 +21,7 @@ import math import string from collections import OrderedDict -from six import string_types +from six import iteritems, string_types from twisted.internet import defer @@ -237,12 +237,10 @@ class RoomCreationHandler(BaseHandler): # map from event_id to BaseEvent old_room_state_events = yield self.store.get_events(old_room_state_ids.values()) - for k in types_to_copy: - old_event_id = old_room_state_ids.get(k) - if old_event_id: - old_event = old_room_state_events.get(old_event_id) - if old_event: - initial_state[k] = old_event.content + for k, old_event_id in iteritems(old_room_state_ids): + old_event = old_room_state_events.get(old_event_id) + if old_event: + initial_state[k] = old_event.content yield self._send_events_for_new_room( requester, From 5caf79b312947c823977c89275c1ea5750aeec92 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Fri, 26 Oct 2018 23:56:40 +0100 Subject: [PATCH 41/84] Remember to copy the avatar on room upgrades --- changelog.d/4100.feature | 1 + synapse/handlers/room.py | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) create mode 100644 changelog.d/4100.feature diff --git a/changelog.d/4100.feature b/changelog.d/4100.feature new file mode 100644 index 0000000000..a3f7dbdcdd --- /dev/null +++ b/changelog.d/4100.feature @@ -0,0 +1 @@ +Support for replacing rooms with new ones diff --git a/synapse/handlers/room.py b/synapse/handlers/room.py index 8e48c1ca6a..c59c02527c 100644 --- a/synapse/handlers/room.py +++ b/synapse/handlers/room.py @@ -228,7 +228,8 @@ class RoomCreationHandler(BaseHandler): (EventTypes.Name, ""), (EventTypes.Topic, ""), (EventTypes.RoomHistoryVisibility, ""), - (EventTypes.GuestAccess, "") + (EventTypes.GuestAccess, ""), + (EventTypes.RoomAvatar, ""), ) old_room_state_ids = yield self.store.get_filtered_current_state_ids( From db24d7f15e406390d57b23d48a78fa33604a47e7 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Fri, 26 Oct 2018 23:47:37 +0100 Subject: [PATCH 42/84] Better handling of odd PLs during room upgrades Fixes handling of rooms where we have permission to send the tombstone, but not other state. We need to (a) fail more gracefully when we can't send the PLs in the old room, and (b) not set the PLs in the new room until we are done with the other stuff. --- changelog.d/4099.feature | 1 + synapse/handlers/room.py | 121 +++++++++++++++++++++++++-------------- 2 files changed, 80 insertions(+), 42 deletions(-) create mode 100644 changelog.d/4099.feature diff --git a/changelog.d/4099.feature b/changelog.d/4099.feature new file mode 100644 index 0000000000..a3f7dbdcdd --- /dev/null +++ b/changelog.d/4099.feature @@ -0,0 +1 @@ +Support for replacing rooms with new ones diff --git a/synapse/handlers/room.py b/synapse/handlers/room.py index 8e48c1ca6a..70085db625 100644 --- a/synapse/handlers/room.py +++ b/synapse/handlers/room.py @@ -136,53 +136,91 @@ class RoomCreationHandler(BaseHandler): requester, tombstone_event, tombstone_context, ) + # and finally, shut down the PLs in the old room, and update them in the new + # room. old_room_state = yield tombstone_context.get_current_state_ids(self.store) - old_room_pl_event_id = old_room_state.get((EventTypes.PowerLevels, "")) - if old_room_pl_event_id is None: - logger.warning( - "Not supported: upgrading a room with no PL event. Not setting PLs " - "in old room.", + yield self._update_upgraded_room_pls( + requester, old_room_id, new_room_id, old_room_state, + ) + + defer.returnValue(new_room_id) + + @defer.inlineCallbacks + def _update_upgraded_room_pls( + self, requester, old_room_id, new_room_id, old_room_state, + ): + """Send updated power levels in both rooms after an upgrade + + Args: + requester (synapse.types.Requester): the user requesting the upgrade + old_room_id (unicode): the id of the room to be replaced + new_room_id (unicode): the id of the replacement room + old_room_state (dict[tuple[str, str], str]): the state map for the old room + + Returns: + Deferred + """ + old_room_pl_event_id = old_room_state.get((EventTypes.PowerLevels, "")) + + if old_room_pl_event_id is None: + logger.warning( + "Not supported: upgrading a room with no PL event. Not setting PLs " + "in old room.", + ) + return + + old_room_pl_state = yield self.store.get_event(old_room_pl_event_id) + + # we try to stop regular users from speaking by setting the PL required + # to send regular events and invites to 'Moderator' level. That's normally + # 50, but if the default PL in a room is 50 or more, then we set the + # required PL above that. + + pl_content = dict(old_room_pl_state.content) + users_default = int(pl_content.get("users_default", 0)) + restricted_level = max(users_default + 1, 50) + + updated = False + for v in ("invite", "events_default"): + current = int(pl_content.get(v, 0)) + if current < restricted_level: + logger.info( + "Setting level for %s in %s to %i (was %i)", + v, old_room_id, restricted_level, current, ) + pl_content[v] = restricted_level + updated = True else: - # we try to stop regular users from speaking by setting the PL required - # to send regular events and invites to 'Moderator' level. That's normally - # 50, but if the default PL in a room is 50 or more, then we set the - # required PL above that. + logger.info( + "Not setting level for %s (already %i)", + v, current, + ) - old_room_pl_state = yield self.store.get_event(old_room_pl_event_id) - pl_content = dict(old_room_pl_state.content) - users_default = int(pl_content.get("users_default", 0)) - restricted_level = max(users_default + 1, 50) + if updated: + try: + yield self.event_creation_handler.create_and_send_nonmember_event( + requester, { + "type": EventTypes.PowerLevels, + "state_key": '', + "room_id": old_room_id, + "sender": requester.user.to_string(), + "content": pl_content, + }, ratelimit=False, + ) + except AuthError as e: + logger.warning("Unable to update PLs in old room: %s", e) - updated = False - for v in ("invite", "events_default"): - current = int(pl_content.get(v, 0)) - if current < restricted_level: - logger.debug( - "Setting level for %s in %s to %i (was %i)", - v, old_room_id, restricted_level, current, - ) - pl_content[v] = restricted_level - updated = True - else: - logger.debug( - "Not setting level for %s (already %i)", - v, current, - ) - - if updated: - yield self.event_creation_handler.create_and_send_nonmember_event( - requester, { - "type": EventTypes.PowerLevels, - "state_key": '', - "room_id": old_room_id, - "sender": user_id, - "content": pl_content, - }, ratelimit=False, - ) - - defer.returnValue(new_room_id) + logger.info("Setting correct PLs in new room") + yield self.event_creation_handler.create_and_send_nonmember_event( + requester, { + "type": EventTypes.PowerLevels, + "state_key": '', + "room_id": new_room_id, + "sender": requester.user.to_string(), + "content": old_room_pl_state.content, + }, ratelimit=False, + ) @defer.inlineCallbacks def clone_exiting_room( @@ -223,7 +261,6 @@ class RoomCreationHandler(BaseHandler): initial_state = dict() types_to_copy = ( - (EventTypes.PowerLevels, ""), (EventTypes.JoinRules, ""), (EventTypes.Name, ""), (EventTypes.Topic, ""), From 2b791865c454f3bd08aa16431ef506305f712609 Mon Sep 17 00:00:00 2001 From: Amber Brown Date: Mon, 29 Oct 2018 21:52:52 +1100 Subject: [PATCH 43/84] version bump --- synapse/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/__init__.py b/synapse/__init__.py index 1ddbbbebfb..c8377df7b2 100644 --- a/synapse/__init__.py +++ b/synapse/__init__.py @@ -27,4 +27,4 @@ try: except ImportError: pass -__version__ = "0.33.7" +__version__ = "0.33.8rc1" From 3b0a85fc8ec869a559fbbdb818f254a1a26c9668 Mon Sep 17 00:00:00 2001 From: Amber Brown Date: Mon, 29 Oct 2018 21:54:01 +1100 Subject: [PATCH 44/84] changelog --- CHANGES.md | 47 ++++++++++++++++++++++++++++++++++++++++ changelog.d/3698.misc | 1 - changelog.d/3786.misc | 1 - changelog.d/3969.bugfix | 1 - changelog.d/3975.feature | 1 - changelog.d/4011.misc | 1 - changelog.d/4031.misc | 1 - changelog.d/4041.misc | 1 - changelog.d/4046.bugfix | 1 - changelog.d/4049.misc | 1 - changelog.d/4050.bugfix | 1 - changelog.d/4051.feature | 1 - changelog.d/4057.bugfix | 1 - changelog.d/4060.bugfix | 1 - changelog.d/4061.bugfix | 1 - changelog.d/4063.misc | 1 - changelog.d/4067.bugfix | 1 - changelog.d/4068.bugfix | 1 - changelog.d/4068.misc | 1 - changelog.d/4072.misc | 1 - changelog.d/4073.misc | 1 - changelog.d/4074.bugfix | 1 - changelog.d/4075.misc | 1 - changelog.d/4076.misc | 1 - changelog.d/4077.misc | 1 - changelog.d/4081.bugfix | 2 -- changelog.d/4082.misc | 1 - changelog.d/4083.bugfix | 1 - changelog.d/4085.feature | 1 - changelog.d/4089.feature | 1 - 30 files changed, 47 insertions(+), 30 deletions(-) delete mode 100644 changelog.d/3698.misc delete mode 100644 changelog.d/3786.misc delete mode 100644 changelog.d/3969.bugfix delete mode 100644 changelog.d/3975.feature delete mode 100644 changelog.d/4011.misc delete mode 100644 changelog.d/4031.misc delete mode 100644 changelog.d/4041.misc delete mode 100644 changelog.d/4046.bugfix delete mode 100644 changelog.d/4049.misc delete mode 100644 changelog.d/4050.bugfix delete mode 100644 changelog.d/4051.feature delete mode 100644 changelog.d/4057.bugfix delete mode 100644 changelog.d/4060.bugfix delete mode 100644 changelog.d/4061.bugfix delete mode 100644 changelog.d/4063.misc delete mode 100644 changelog.d/4067.bugfix delete mode 100644 changelog.d/4068.bugfix delete mode 100644 changelog.d/4068.misc delete mode 100644 changelog.d/4072.misc delete mode 100644 changelog.d/4073.misc delete mode 100644 changelog.d/4074.bugfix delete mode 100644 changelog.d/4075.misc delete mode 100644 changelog.d/4076.misc delete mode 100644 changelog.d/4077.misc delete mode 100644 changelog.d/4081.bugfix delete mode 100644 changelog.d/4082.misc delete mode 100644 changelog.d/4083.bugfix delete mode 100644 changelog.d/4085.feature delete mode 100644 changelog.d/4089.feature diff --git a/CHANGES.md b/CHANGES.md index fb98c934c0..dc2a745c8b 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,3 +1,50 @@ +Synapse 0.33.8rc1 (2018-10-29) +============================== + +Features +-------- + +- Servers with auto-join rooms will now automatically create those rooms when the first user registers ([\#3975](https://github.com/matrix-org/synapse/issues/3975)) +- Add config option to control alias creation ([\#4051](https://github.com/matrix-org/synapse/issues/4051)) +- The register_new_matrix_user script is now ported to Python 3. ([\#4085](https://github.com/matrix-org/synapse/issues/4085)) +- Configure Docker image to listen on both ipv4 and ipv6. ([\#4089](https://github.com/matrix-org/synapse/issues/4089)) + + +Bugfixes +-------- + +- Fix HTTP error response codes for federated group requests. ([\#3969](https://github.com/matrix-org/synapse/issues/3969)) +- Fix issue where Python 3 users couldn't paginate /publicRooms ([\#4046](https://github.com/matrix-org/synapse/issues/4046)) +- Fix URL previewing to work in Python 3.7 ([\#4050](https://github.com/matrix-org/synapse/issues/4050)) +- synctl will use the right python executable to run worker processes ([\#4057](https://github.com/matrix-org/synapse/issues/4057)) +- Manhole now works again on Python 3, instead of failing with a "couldn't match all kex parts" when connecting. ([\#4060](https://github.com/matrix-org/synapse/issues/4060), [\#4067](https://github.com/matrix-org/synapse/issues/4067)) +- Fix some metrics being racy and causing exceptions when polled by Prometheus. ([\#4061](https://github.com/matrix-org/synapse/issues/4061)) +- Fix bug which prevented email notifications from being sent unless an absolute path was given for `email_templates`. ([\#4068](https://github.com/matrix-org/synapse/issues/4068)) +- Correctly account for cpu usage by background threads ([\#4074](https://github.com/matrix-org/synapse/issues/4074)) +- Fix race condition where config defined reserved users were not being added to + the monthly active user list prior to the homeserver reactor firing up ([\#4081](https://github.com/matrix-org/synapse/issues/4081)) +- Fix bug which prevented backslashes being used in event field filters ([\#4083](https://github.com/matrix-org/synapse/issues/4083)) + + +Internal Changes +---------------- + +- Add information about the [matrix-docker-ansible-deploy](https://github.com/spantaleev/matrix-docker-ansible-deploy) playbook ([\#3698](https://github.com/matrix-org/synapse/issues/3698)) +- Add initial implementation of new state resolution algorithm ([\#3786](https://github.com/matrix-org/synapse/issues/3786)) +- Reduce database load when fetching state groups ([\#4011](https://github.com/matrix-org/synapse/issues/4011)) +- Various cleanups in the federation client code ([\#4031](https://github.com/matrix-org/synapse/issues/4031)) +- Run the CircleCI builds in docker containers ([\#4041](https://github.com/matrix-org/synapse/issues/4041)) +- Only colourise synctl output when attached to tty ([\#4049](https://github.com/matrix-org/synapse/issues/4049)) +- Refactor room alias creation code ([\#4063](https://github.com/matrix-org/synapse/issues/4063)) +- Make the Python scripts in the top-level scripts folders meet pep8 and pass flake8. ([\#4068](https://github.com/matrix-org/synapse/issues/4068)) +- The README now contains example for the Caddy web server. Contributed by steamp0rt. ([\#4072](https://github.com/matrix-org/synapse/issues/4072)) +- Add psutil as an explicit dependency ([\#4073](https://github.com/matrix-org/synapse/issues/4073)) +- Clean up threading and logcontexts in pushers ([\#4075](https://github.com/matrix-org/synapse/issues/4075)) +- Correctly manage logcontexts during startup to fix some "Unexpected logging context" warnings ([\#4076](https://github.com/matrix-org/synapse/issues/4076)) +- Give some more things logcontexts ([\#4077](https://github.com/matrix-org/synapse/issues/4077)) +- Clean up some bits of code which were flagged by the linter ([\#4082](https://github.com/matrix-org/synapse/issues/4082)) + + Synapse 0.33.7 (2018-10-18) =========================== diff --git a/changelog.d/3698.misc b/changelog.d/3698.misc deleted file mode 100644 index 12537e76f2..0000000000 --- a/changelog.d/3698.misc +++ /dev/null @@ -1 +0,0 @@ -Add information about the [matrix-docker-ansible-deploy](https://github.com/spantaleev/matrix-docker-ansible-deploy) playbook diff --git a/changelog.d/3786.misc b/changelog.d/3786.misc deleted file mode 100644 index a9f9a2bb27..0000000000 --- a/changelog.d/3786.misc +++ /dev/null @@ -1 +0,0 @@ -Add initial implementation of new state resolution algorithm diff --git a/changelog.d/3969.bugfix b/changelog.d/3969.bugfix deleted file mode 100644 index ca2759e91e..0000000000 --- a/changelog.d/3969.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix HTTP error response codes for federated group requests. diff --git a/changelog.d/3975.feature b/changelog.d/3975.feature deleted file mode 100644 index 162f30a532..0000000000 --- a/changelog.d/3975.feature +++ /dev/null @@ -1 +0,0 @@ -Servers with auto-join rooms will now automatically create those rooms when the first user registers diff --git a/changelog.d/4011.misc b/changelog.d/4011.misc deleted file mode 100644 index ad7768c4cd..0000000000 --- a/changelog.d/4011.misc +++ /dev/null @@ -1 +0,0 @@ -Reduce database load when fetching state groups diff --git a/changelog.d/4031.misc b/changelog.d/4031.misc deleted file mode 100644 index 60be8b59fd..0000000000 --- a/changelog.d/4031.misc +++ /dev/null @@ -1 +0,0 @@ -Various cleanups in the federation client code diff --git a/changelog.d/4041.misc b/changelog.d/4041.misc deleted file mode 100644 index 8cce9daac9..0000000000 --- a/changelog.d/4041.misc +++ /dev/null @@ -1 +0,0 @@ -Run the CircleCI builds in docker containers diff --git a/changelog.d/4046.bugfix b/changelog.d/4046.bugfix deleted file mode 100644 index 5046dd1ce3..0000000000 --- a/changelog.d/4046.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix issue where Python 3 users couldn't paginate /publicRooms diff --git a/changelog.d/4049.misc b/changelog.d/4049.misc deleted file mode 100644 index 4370d9dfa6..0000000000 --- a/changelog.d/4049.misc +++ /dev/null @@ -1 +0,0 @@ -Only colourise synctl output when attached to tty diff --git a/changelog.d/4050.bugfix b/changelog.d/4050.bugfix deleted file mode 100644 index 3d1f6af847..0000000000 --- a/changelog.d/4050.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix URL priewing to work in Python 3.7 diff --git a/changelog.d/4051.feature b/changelog.d/4051.feature deleted file mode 100644 index 9c1b3a72a0..0000000000 --- a/changelog.d/4051.feature +++ /dev/null @@ -1 +0,0 @@ -Add config option to control alias creation diff --git a/changelog.d/4057.bugfix b/changelog.d/4057.bugfix deleted file mode 100644 index 7577731255..0000000000 --- a/changelog.d/4057.bugfix +++ /dev/null @@ -1 +0,0 @@ -synctl will use the right python executable to run worker processes \ No newline at end of file diff --git a/changelog.d/4060.bugfix b/changelog.d/4060.bugfix deleted file mode 100644 index 78d69a8819..0000000000 --- a/changelog.d/4060.bugfix +++ /dev/null @@ -1 +0,0 @@ -Manhole now works again on Python 3, instead of failing with a "couldn't match all kex parts" when connecting. diff --git a/changelog.d/4061.bugfix b/changelog.d/4061.bugfix deleted file mode 100644 index 94ffcf7a51..0000000000 --- a/changelog.d/4061.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix some metrics being racy and causing exceptions when polled by Prometheus. diff --git a/changelog.d/4063.misc b/changelog.d/4063.misc deleted file mode 100644 index 677fcb90ad..0000000000 --- a/changelog.d/4063.misc +++ /dev/null @@ -1 +0,0 @@ -Refactor room alias creation code diff --git a/changelog.d/4067.bugfix b/changelog.d/4067.bugfix deleted file mode 100644 index 78d69a8819..0000000000 --- a/changelog.d/4067.bugfix +++ /dev/null @@ -1 +0,0 @@ -Manhole now works again on Python 3, instead of failing with a "couldn't match all kex parts" when connecting. diff --git a/changelog.d/4068.bugfix b/changelog.d/4068.bugfix deleted file mode 100644 index 74bda7491f..0000000000 --- a/changelog.d/4068.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix bug which prevented email notifications from being sent unless an absolute path was given for `email_templates`. \ No newline at end of file diff --git a/changelog.d/4068.misc b/changelog.d/4068.misc deleted file mode 100644 index db6c4ade59..0000000000 --- a/changelog.d/4068.misc +++ /dev/null @@ -1 +0,0 @@ -Make the Python scripts in the top-level scripts folders meet pep8 and pass flake8. diff --git a/changelog.d/4072.misc b/changelog.d/4072.misc deleted file mode 100644 index 9d7279fd2b..0000000000 --- a/changelog.d/4072.misc +++ /dev/null @@ -1 +0,0 @@ -The README now contains example for the Caddy web server. Contributed by steamp0rt. diff --git a/changelog.d/4073.misc b/changelog.d/4073.misc deleted file mode 100644 index fc304bef06..0000000000 --- a/changelog.d/4073.misc +++ /dev/null @@ -1 +0,0 @@ -Add psutil as an explicit dependency diff --git a/changelog.d/4074.bugfix b/changelog.d/4074.bugfix deleted file mode 100644 index b3b6b00243..0000000000 --- a/changelog.d/4074.bugfix +++ /dev/null @@ -1 +0,0 @@ -Correctly account for cpu usage by background threads diff --git a/changelog.d/4075.misc b/changelog.d/4075.misc deleted file mode 100644 index d08b8cc271..0000000000 --- a/changelog.d/4075.misc +++ /dev/null @@ -1 +0,0 @@ -Clean up threading and logcontexts in pushers \ No newline at end of file diff --git a/changelog.d/4076.misc b/changelog.d/4076.misc deleted file mode 100644 index 9dd000decf..0000000000 --- a/changelog.d/4076.misc +++ /dev/null @@ -1 +0,0 @@ -Correctly manage logcontexts during startup to fix some "Unexpected logging context" warnings \ No newline at end of file diff --git a/changelog.d/4077.misc b/changelog.d/4077.misc deleted file mode 100644 index 52ca4c1de2..0000000000 --- a/changelog.d/4077.misc +++ /dev/null @@ -1 +0,0 @@ -Give some more things logcontexts diff --git a/changelog.d/4081.bugfix b/changelog.d/4081.bugfix deleted file mode 100644 index cfe4b3e9d9..0000000000 --- a/changelog.d/4081.bugfix +++ /dev/null @@ -1,2 +0,0 @@ -Fix race condition where config defined reserved users were not being added to -the monthly active user list prior to the homeserver reactor firing up diff --git a/changelog.d/4082.misc b/changelog.d/4082.misc deleted file mode 100644 index a81faf5e9b..0000000000 --- a/changelog.d/4082.misc +++ /dev/null @@ -1 +0,0 @@ -Clean up some bits of code which were flagged by the linter diff --git a/changelog.d/4083.bugfix b/changelog.d/4083.bugfix deleted file mode 100644 index b3b08cdfa6..0000000000 --- a/changelog.d/4083.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix bug which prevented backslashes being used in event field filters \ No newline at end of file diff --git a/changelog.d/4085.feature b/changelog.d/4085.feature deleted file mode 100644 index 4bd3ddcf2c..0000000000 --- a/changelog.d/4085.feature +++ /dev/null @@ -1 +0,0 @@ -The register_new_matrix_user script is now ported to Python 3. diff --git a/changelog.d/4089.feature b/changelog.d/4089.feature deleted file mode 100644 index 62c9d839bb..0000000000 --- a/changelog.d/4089.feature +++ /dev/null @@ -1 +0,0 @@ - Configure Docker image to listen on both ipv4 and ipv6. From c4b3698a80468957c63b2a79685ac06f76cabae1 Mon Sep 17 00:00:00 2001 From: Amber Brown Date: Mon, 29 Oct 2018 22:59:44 +1100 Subject: [PATCH 45/84] Make the replication logger quieter (#4108) --- changelog.d/4108.misc | 1 + synapse/replication/tcp/client.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) create mode 100644 changelog.d/4108.misc diff --git a/changelog.d/4108.misc b/changelog.d/4108.misc new file mode 100644 index 0000000000..85810c3d83 --- /dev/null +++ b/changelog.d/4108.misc @@ -0,0 +1 @@ +The "Received rdata" log messages on workers is now logged at DEBUG, not INFO. diff --git a/synapse/replication/tcp/client.py b/synapse/replication/tcp/client.py index cbe9645817..586dddb40b 100644 --- a/synapse/replication/tcp/client.py +++ b/synapse/replication/tcp/client.py @@ -106,7 +106,7 @@ class ReplicationClientHandler(object): Can be overriden in subclasses to handle more. """ - logger.info("Received rdata %s -> %s", stream_name, token) + logger.debug("Received rdata %s -> %s", stream_name, token) return self.store.process_replication_rows(stream_name, token, rows) def on_position(self, stream_name, token): From 4cd1c9f2ffa46bc8ed258da200ae3b8ba25fcbb5 Mon Sep 17 00:00:00 2001 From: Amber Brown Date: Mon, 29 Oct 2018 23:57:24 +1100 Subject: [PATCH 46/84] Delete the disused & unspecced identicon functionality (#4106) --- changelog.d/4106.removal | 1 + scripts-dev/make_identicons.pl | 39 ------------ synapse/handlers/register.py | 3 - synapse/python_dependencies.py | 1 - synapse/rest/media/v1/identicon_resource.py | 68 --------------------- synapse/rest/media/v1/media_repository.py | 2 - 6 files changed, 1 insertion(+), 113 deletions(-) create mode 100644 changelog.d/4106.removal delete mode 100755 scripts-dev/make_identicons.pl delete mode 100644 synapse/rest/media/v1/identicon_resource.py diff --git a/changelog.d/4106.removal b/changelog.d/4106.removal new file mode 100644 index 0000000000..7e63208daa --- /dev/null +++ b/changelog.d/4106.removal @@ -0,0 +1 @@ +The disused and un-specced identicon generator has been removed. diff --git a/scripts-dev/make_identicons.pl b/scripts-dev/make_identicons.pl deleted file mode 100755 index cbff63e298..0000000000 --- a/scripts-dev/make_identicons.pl +++ /dev/null @@ -1,39 +0,0 @@ -#!/usr/bin/env perl - -use strict; -use warnings; - -use DBI; -use DBD::SQLite; -use JSON; -use Getopt::Long; - -my $db; # = "homeserver.db"; -my $server = "http://localhost:8008"; -my $size = 320; - -GetOptions("db|d=s", \$db, - "server|s=s", \$server, - "width|w=i", \$size) or usage(); - -usage() unless $db; - -my $dbh = DBI->connect("dbi:SQLite:dbname=$db","","") || die $DBI::errstr; - -my $res = $dbh->selectall_arrayref("select token, name from access_tokens, users where access_tokens.user_id = users.id group by user_id") || die $DBI::errstr; - -foreach (@$res) { - my ($token, $mxid) = ($_->[0], $_->[1]); - my ($user_id) = ($mxid =~ m/@(.*):/); - my ($url) = $dbh->selectrow_array("select avatar_url from profiles where user_id=?", undef, $user_id); - if (!$url || $url =~ /#auto$/) { - `curl -s -o tmp.png "$server/_matrix/media/v1/identicon?name=${mxid}&width=$size&height=$size"`; - my $json = `curl -s -X POST -H "Content-Type: image/png" -T "tmp.png" $server/_matrix/media/v1/upload?access_token=$token`; - my $content_uri = from_json($json)->{content_uri}; - `curl -X PUT -H "Content-Type: application/json" --data '{ "avatar_url": "${content_uri}#auto"}' $server/_matrix/client/api/v1/profile/${mxid}/avatar_url?access_token=$token`; - } -} - -sub usage { - die "usage: ./make-identicons.pl\n\t-d database [e.g. homeserver.db]\n\t-s homeserver (default: http://localhost:8008)\n\t-w identicon size in pixels (default 320)"; -} \ No newline at end of file diff --git a/synapse/handlers/register.py b/synapse/handlers/register.py index 7b4549223f..d2beb275cf 100644 --- a/synapse/handlers/register.py +++ b/synapse/handlers/register.py @@ -256,9 +256,6 @@ class RegistrationHandler(BaseHandler): except Exception as e: logger.error("Failed to join new user to %r: %r", r, e) - # We used to generate default identicons here, but nowadays - # we want clients to generate their own as part of their branding - # rather than there being consistent matrix-wide ones, so we don't. defer.returnValue((user_id, token)) @defer.inlineCallbacks diff --git a/synapse/python_dependencies.py b/synapse/python_dependencies.py index 943876456b..ca62ee7637 100644 --- a/synapse/python_dependencies.py +++ b/synapse/python_dependencies.py @@ -51,7 +51,6 @@ REQUIREMENTS = { "daemonize>=2.3.1": ["daemonize"], "bcrypt>=3.1.0": ["bcrypt>=3.1.0"], "pillow>=3.1.2": ["PIL"], - "pydenticon>=0.2": ["pydenticon"], "sortedcontainers>=1.4.4": ["sortedcontainers"], "psutil>=2.0.0": ["psutil>=2.0.0"], "pysaml2>=3.0.0": ["saml2"], diff --git a/synapse/rest/media/v1/identicon_resource.py b/synapse/rest/media/v1/identicon_resource.py deleted file mode 100644 index bdbd8d50dd..0000000000 --- a/synapse/rest/media/v1/identicon_resource.py +++ /dev/null @@ -1,68 +0,0 @@ -# Copyright 2015, 2016 OpenMarket Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from pydenticon import Generator - -from twisted.web.resource import Resource - -from synapse.http.servlet import parse_integer - -FOREGROUND = [ - "rgb(45,79,255)", - "rgb(254,180,44)", - "rgb(226,121,234)", - "rgb(30,179,253)", - "rgb(232,77,65)", - "rgb(49,203,115)", - "rgb(141,69,170)" -] - -BACKGROUND = "rgb(224,224,224)" -SIZE = 5 - - -class IdenticonResource(Resource): - isLeaf = True - - def __init__(self): - Resource.__init__(self) - self.generator = Generator( - SIZE, SIZE, foreground=FOREGROUND, background=BACKGROUND, - ) - - def generate_identicon(self, name, width, height): - v_padding = width % SIZE - h_padding = height % SIZE - top_padding = v_padding // 2 - left_padding = h_padding // 2 - bottom_padding = v_padding - top_padding - right_padding = h_padding - left_padding - width -= v_padding - height -= h_padding - padding = (top_padding, bottom_padding, left_padding, right_padding) - identicon = self.generator.generate( - name, width, height, padding=padding - ) - return identicon - - def render_GET(self, request): - name = "/".join(request.postpath) - width = parse_integer(request, "width", default=96) - height = parse_integer(request, "height", default=96) - identicon_bytes = self.generate_identicon(name, width, height) - request.setHeader(b"Content-Type", b"image/png") - request.setHeader( - b"Cache-Control", b"public,max-age=86400,s-maxage=86400" - ) - return identicon_bytes diff --git a/synapse/rest/media/v1/media_repository.py b/synapse/rest/media/v1/media_repository.py index 08b1867fab..d6c5f07af0 100644 --- a/synapse/rest/media/v1/media_repository.py +++ b/synapse/rest/media/v1/media_repository.py @@ -45,7 +45,6 @@ from ._base import FileInfo, respond_404, respond_with_responder from .config_resource import MediaConfigResource from .download_resource import DownloadResource from .filepath import MediaFilePaths -from .identicon_resource import IdenticonResource from .media_storage import MediaStorage from .preview_url_resource import PreviewUrlResource from .storage_provider import StorageProviderWrapper @@ -769,7 +768,6 @@ class MediaRepositoryResource(Resource): self.putChild(b"thumbnail", ThumbnailResource( hs, media_repo, media_repo.media_storage, )) - self.putChild(b"identicon", IdenticonResource()) if hs.config.url_preview_enabled: self.putChild(b"preview_url", PreviewUrlResource( hs, media_repo, media_repo.media_storage, From b2399f6281d7cd11e7762b683bdd5a4f0c24927e Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 29 Oct 2018 14:01:11 +0000 Subject: [PATCH 47/84] Make SQL a bit cleaner --- synapse/storage/state.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/synapse/storage/state.py b/synapse/storage/state.py index 59a50a5df9..45afd42b3f 100644 --- a/synapse/storage/state.py +++ b/synapse/storage/state.py @@ -1272,14 +1272,13 @@ class StateGroupWorkerStore(EventsWorkerStore, SQLBaseStore): # Check if state groups are referenced sql = """ - SELECT state_group, count(*) FROM event_to_state_groups + SELECT DISTINCT state_group FROM event_to_state_groups LEFT JOIN events_to_purge AS ep USING (event_id) WHERE state_group IN (%s) AND ep.event_id IS NULL - GROUP BY state_group """ % (",".join("?" for _ in current_search),) txn.execute(sql, list(current_search)) - referenced = set(sg for sg, cnt in txn if cnt > 0) + referenced = set(sg for sg, in txn) referenced_groups |= referenced # We don't continue iterating up the state group graphs for state From f4f223aa4455bea3aa642c23ae957932b1168ba3 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 29 Oct 2018 14:01:49 +0000 Subject: [PATCH 48/84] Don't make temporary list --- synapse/storage/state.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/synapse/storage/state.py b/synapse/storage/state.py index 45afd42b3f..947d3fc177 100644 --- a/synapse/storage/state.py +++ b/synapse/storage/state.py @@ -1266,9 +1266,8 @@ class StateGroupWorkerStore(EventsWorkerStore, SQLBaseStore): current_search = next_to_search next_to_search = set() else: - lst = list(next_to_search) - current_search = set(lst[:100]) - next_to_search = set(lst[100:]) + current_search = set(islice(next_to_search, 100)) + next_to_search -= current_search # Check if state groups are referenced sql = """ From 664b192a3b4aa57597d9832361b025bc078ea87a Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 29 Oct 2018 14:21:43 +0000 Subject: [PATCH 49/84] Fix set operations thinko --- synapse/storage/state.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/synapse/storage/state.py b/synapse/storage/state.py index 947d3fc177..dfec57c045 100644 --- a/synapse/storage/state.py +++ b/synapse/storage/state.py @@ -1293,10 +1293,11 @@ class StateGroupWorkerStore(EventsWorkerStore, SQLBaseStore): retcols=("prev_state_group", "state_group",), ) - next_to_search.update(row["state_group"] for row in rows) + prevs = set(row["state_group"] for row in rows) # We don't bother re-handling groups we've already seen - next_to_search -= state_groups_seen - state_groups_seen |= next_to_search + prevs -= state_groups_seen + next_to_search |= prevs + state_groups_seen |= prevs for row in rows: # Note: Each state group can have at most one prev group From ad88460e0d2e0a7c2cf39ec5539d5c4ff030bbbd Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 29 Oct 2018 14:23:34 +0000 Subject: [PATCH 50/84] Move _find_unreferenced_groups --- synapse/storage/events.py | 85 ++++++++++++++++++++++++++++++++++++++- synapse/storage/state.py | 79 ------------------------------------ 2 files changed, 83 insertions(+), 81 deletions(-) diff --git a/synapse/storage/events.py b/synapse/storage/events.py index e791922733..919e855f3b 100644 --- a/synapse/storage/events.py +++ b/synapse/storage/events.py @@ -2052,8 +2052,10 @@ class EventsStore(StateGroupWorkerStore, EventFederationStore, EventsWorkerStore logger.info("[purge] finding state groups that can be deleted") - state_groups_to_delete, remaining_state_groups = self._find_unreferenced_groups( - txn, referenced_state_groups, + state_groups_to_delete, remaining_state_groups = ( + self._find_unreferenced_groups_during_purge( + txn, referenced_state_groups, + ) ) logger.info( @@ -2209,6 +2211,85 @@ class EventsStore(StateGroupWorkerStore, EventFederationStore, EventsWorkerStore logger.info("[purge] done") + def _find_unreferenced_groups_during_purge(self, txn, state_groups): + """Used when purging history to figure out which state groups can be + deleted and which need to be de-delta'ed (due to one of its prev groups + being scheduled for deletion). + + Args: + txn + state_groups (set[int]): Set of state groups referenced by events + that are going to be deleted. + + Returns: + tuple[set[int], set[int]]: The set of state groups that can be + deleted and the set of state groups that need to be de-delta'ed + """ + # Graph of state group -> previous group + graph = {} + + # Set of events that we have found to be referenced by events + referenced_groups = set() + + # Set of state groups we've already seen + state_groups_seen = set(state_groups) + + # Set of state groups to handle next. + next_to_search = set(state_groups) + while next_to_search: + # We bound size of groups we're looking up at once, to stop the + # SQL query getting too big + if len(next_to_search) < 100: + current_search = next_to_search + next_to_search = set() + else: + current_search = set(itertools.islice(next_to_search, 100)) + next_to_search -= current_search + + # Check if state groups are referenced + sql = """ + SELECT DISTINCT state_group FROM event_to_state_groups + LEFT JOIN events_to_purge AS ep USING (event_id) + WHERE state_group IN (%s) AND ep.event_id IS NULL + """ % (",".join("?" for _ in current_search),) + txn.execute(sql, list(current_search)) + + referenced = set(sg for sg, in txn) + referenced_groups |= referenced + + # We don't continue iterating up the state group graphs for state + # groups that are referenced. + current_search -= referenced + + rows = self._simple_select_many_txn( + txn, + table="state_group_edges", + column="prev_state_group", + iterable=current_search, + keyvalues={}, + retcols=("prev_state_group", "state_group",), + ) + + prevs = set(row["state_group"] for row in rows) + # We don't bother re-handling groups we've already seen + prevs -= state_groups_seen + next_to_search |= prevs + state_groups_seen |= prevs + + for row in rows: + # Note: Each state group can have at most one prev group + graph[row["state_group"]] = row["prev_state_group"] + + to_delete = state_groups_seen - referenced_groups + + to_dedelta = set() + for sg in referenced_groups: + prev_sg = graph.get(sg) + if prev_sg and prev_sg in to_delete: + to_dedelta.add(sg) + + return to_delete, to_dedelta + @defer.inlineCallbacks def is_event_after(self, event_id1, event_id2): """Returns True if event_id1 is after event_id2 in the stream diff --git a/synapse/storage/state.py b/synapse/storage/state.py index dfec57c045..d737bd6778 100644 --- a/synapse/storage/state.py +++ b/synapse/storage/state.py @@ -1234,85 +1234,6 @@ class StateGroupWorkerStore(EventsWorkerStore, SQLBaseStore): return count - def _find_unreferenced_groups(self, txn, state_groups): - """Used when purging history to figure out which state groups can be - deleted and which need to be de-delta'ed (due to one of its prev groups - being scheduled for deletion). - - Args: - txn - state_groups (set[int]): Set of state groups referenced by events - that are going to be deleted. - - Returns: - tuple[set[int], set[int]]: The set of state groups that can be - deleted and the set of state groups that need to be de-delta'ed - """ - # Graph of state group -> previous group - graph = {} - - # Set of events that we have found to be referenced by events - referenced_groups = set() - - # Set of state groups we've already seen - state_groups_seen = set(state_groups) - - # Set of state groups to handle next. - next_to_search = set(state_groups) - while next_to_search: - # We bound size of groups we're looking up at once, to stop the - # SQL query getting too big - if len(next_to_search) < 100: - current_search = next_to_search - next_to_search = set() - else: - current_search = set(islice(next_to_search, 100)) - next_to_search -= current_search - - # Check if state groups are referenced - sql = """ - SELECT DISTINCT state_group FROM event_to_state_groups - LEFT JOIN events_to_purge AS ep USING (event_id) - WHERE state_group IN (%s) AND ep.event_id IS NULL - """ % (",".join("?" for _ in current_search),) - txn.execute(sql, list(current_search)) - - referenced = set(sg for sg, in txn) - referenced_groups |= referenced - - # We don't continue iterating up the state group graphs for state - # groups that are referenced. - current_search -= referenced - - rows = self._simple_select_many_txn( - txn, - table="state_group_edges", - column="prev_state_group", - iterable=current_search, - keyvalues={}, - retcols=("prev_state_group", "state_group",), - ) - - prevs = set(row["state_group"] for row in rows) - # We don't bother re-handling groups we've already seen - prevs -= state_groups_seen - next_to_search |= prevs - state_groups_seen |= prevs - - for row in rows: - # Note: Each state group can have at most one prev group - graph[row["state_group"]] = row["prev_state_group"] - - to_delete = state_groups_seen - referenced_groups - - to_dedelta = set() - for sg in referenced_groups: - prev_sg = graph.get(sg) - if prev_sg and prev_sg in to_delete: - to_dedelta.add(sg) - - return to_delete, to_dedelta - class StateStore(StateGroupWorkerStore, BackgroundUpdateStore): """ Keeps track of the state at a given event. From a163b748a5ca37853f440c5c46d2da80f738a9e0 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 29 Oct 2018 17:34:21 +0000 Subject: [PATCH 51/84] Don't truncate command name in metrics --- synapse/replication/tcp/protocol.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/synapse/replication/tcp/protocol.py b/synapse/replication/tcp/protocol.py index 5dc7b3fffc..0b3fe6cbf5 100644 --- a/synapse/replication/tcp/protocol.py +++ b/synapse/replication/tcp/protocol.py @@ -656,7 +656,7 @@ tcp_inbound_commands = LaterGauge( "", ["command", "name"], lambda: { - (k[0], p.name,): count + (k, p.name,): count for p in connected_connections for k, count in iteritems(p.inbound_commands_counter) }, @@ -667,7 +667,7 @@ tcp_outbound_commands = LaterGauge( "", ["command", "name"], lambda: { - (k[0], p.name,): count + (k, p.name,): count for p in connected_connections for k, count in iteritems(p.outbound_commands_counter) }, From 88e5ffe6fe816e54a5471728e93fde63353d9a70 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 29 Oct 2018 17:34:34 +0000 Subject: [PATCH 52/84] Deduplicate device updates sent over replication We currently send several kHz of device list updates over replication occisonally, which often causes the replications streams to lag and then get dropped. A lot of those updates will actually be duplicates, since we don't send e.g. device_ids across replication, so let's deduplicate it when we pull them out of the database. --- synapse/storage/devices.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/synapse/storage/devices.py b/synapse/storage/devices.py index d10ff9e4b9..62497ab63f 100644 --- a/synapse/storage/devices.py +++ b/synapse/storage/devices.py @@ -589,10 +589,14 @@ class DeviceStore(SQLBaseStore): combined list of changes to devices, and which destinations need to be poked. `destination` may be None if no destinations need to be poked. """ + # We do a group by here as there can be a large number of duplicate + # entries, since we throw away device IDs. sql = """ - SELECT stream_id, user_id, destination FROM device_lists_stream + SELECT MAX(stream_id) AS stream_id, user_id, destination + FROM device_lists_stream LEFT JOIN device_lists_outbound_pokes USING (stream_id, user_id, device_id) WHERE ? < stream_id AND stream_id <= ? + GROUP BY user_id, destination """ return self._execute( "get_all_device_list_changes_for_remotes", None, From 39f419868fa8df56e3d9df9ba8e153884fe3ea55 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 29 Oct 2018 17:38:09 +0000 Subject: [PATCH 53/84] Newsfile --- changelog.d/4109.misc | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog.d/4109.misc diff --git a/changelog.d/4109.misc b/changelog.d/4109.misc new file mode 100644 index 0000000000..566c683119 --- /dev/null +++ b/changelog.d/4109.misc @@ -0,0 +1 @@ +Reduce replication traffic for device lists From 4f0fa7a1201b0b76763dc146db9af12c4dd29494 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 29 Oct 2018 18:15:42 +0000 Subject: [PATCH 54/84] Newsfile --- changelog.d/4110.misc | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog.d/4110.misc diff --git a/changelog.d/4110.misc b/changelog.d/4110.misc new file mode 100644 index 0000000000..a50327ae34 --- /dev/null +++ b/changelog.d/4110.misc @@ -0,0 +1 @@ +Fix `synapse_replication_tcp_protocol_*_commands` metric label to be full command name, rather than just the first character From 0dce9e1379ea867c9a00c8e6cf1d42badb52601d Mon Sep 17 00:00:00 2001 From: Amber Brown Date: Tue, 30 Oct 2018 23:55:43 +1100 Subject: [PATCH 55/84] Write some tests for the email pusher (#4095) --- .travis.yml | 11 ++- changelog.d/4095.bugfix | 1 + synapse/push/emailpusher.py | 5 +- synapse/push/mailer.py | 10 +-- synapse/server.py | 5 ++ tests/push/__init__.py | 0 tests/push/test_email.py | 148 ++++++++++++++++++++++++++++++++++++ tests/server.py | 4 +- tests/test_mau.py | 2 +- tests/unittest.py | 9 ++- 10 files changed, 182 insertions(+), 13 deletions(-) create mode 100644 changelog.d/4095.bugfix create mode 100644 tests/push/__init__.py create mode 100644 tests/push/test_email.py diff --git a/.travis.yml b/.travis.yml index fd41841c77..655fab9d8e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -23,6 +23,9 @@ branches: - develop - /^release-v/ +# When running the tox environments that call Twisted Trial, we can pass the -j +# flag to run the tests concurrently. We set this to 2 for CPU bound tests +# (SQLite) and 4 for I/O bound tests (PostgreSQL). matrix: fast_finish: true include: @@ -33,10 +36,10 @@ matrix: env: TOX_ENV="pep8,check_isort" - python: 2.7 - env: TOX_ENV=py27 + env: TOX_ENV=py27 TRIAL_FLAGS="-j 2" - python: 2.7 - env: TOX_ENV=py27-old + env: TOX_ENV=py27-old TRIAL_FLAGS="-j 2" - python: 2.7 env: TOX_ENV=py27-postgres TRIAL_FLAGS="-j 4" @@ -44,10 +47,10 @@ matrix: - postgresql - python: 3.5 - env: TOX_ENV=py35 + env: TOX_ENV=py35 TRIAL_FLAGS="-j 2" - python: 3.6 - env: TOX_ENV=py36 + env: TOX_ENV=py36 TRIAL_FLAGS="-j 2" - python: 3.6 env: TOX_ENV=py36-postgres TRIAL_FLAGS="-j 4" diff --git a/changelog.d/4095.bugfix b/changelog.d/4095.bugfix new file mode 100644 index 0000000000..76ee7148c2 --- /dev/null +++ b/changelog.d/4095.bugfix @@ -0,0 +1 @@ +Fix exceptions when using the email mailer on Python 3. diff --git a/synapse/push/emailpusher.py b/synapse/push/emailpusher.py index f369124258..50e1007d84 100644 --- a/synapse/push/emailpusher.py +++ b/synapse/push/emailpusher.py @@ -85,7 +85,10 @@ class EmailPusher(object): self.timed_call = None def on_new_notifications(self, min_stream_ordering, max_stream_ordering): - self.max_stream_ordering = max(max_stream_ordering, self.max_stream_ordering) + if self.max_stream_ordering: + self.max_stream_ordering = max(max_stream_ordering, self.max_stream_ordering) + else: + self.max_stream_ordering = max_stream_ordering self._start_processing() def on_new_receipts(self, min_stream_id, max_stream_id): diff --git a/synapse/push/mailer.py b/synapse/push/mailer.py index 16fb5e8471..ebcb93bfc7 100644 --- a/synapse/push/mailer.py +++ b/synapse/push/mailer.py @@ -26,7 +26,6 @@ import bleach import jinja2 from twisted.internet import defer -from twisted.mail.smtp import sendmail from synapse.api.constants import EventTypes from synapse.api.errors import StoreError @@ -85,6 +84,7 @@ class Mailer(object): self.notif_template_html = notif_template_html self.notif_template_text = notif_template_text + self.sendmail = self.hs.get_sendmail() self.store = self.hs.get_datastore() self.macaroon_gen = self.hs.get_macaroon_generator() self.state_handler = self.hs.get_state_handler() @@ -191,11 +191,11 @@ class Mailer(object): multipart_msg.attach(html_part) logger.info("Sending email push notification to %s" % email_address) - # logger.debug(html_text) - yield sendmail( + yield self.sendmail( self.hs.config.email_smtp_host, - raw_from, raw_to, multipart_msg.as_string(), + raw_from, raw_to, multipart_msg.as_string().encode('utf8'), + reactor=self.hs.get_reactor(), port=self.hs.config.email_smtp_port, requireAuthentication=self.hs.config.email_smtp_user is not None, username=self.hs.config.email_smtp_user, @@ -333,7 +333,7 @@ class Mailer(object): notif_events, user_id, reason): if len(notifs_by_room) == 1: # Only one room has new stuff - room_id = notifs_by_room.keys()[0] + room_id = list(notifs_by_room.keys())[0] # If the room has some kind of name, use it, but we don't # want the generated-from-names one here otherwise we'll diff --git a/synapse/server.py b/synapse/server.py index cf6b872cbd..9985687b95 100644 --- a/synapse/server.py +++ b/synapse/server.py @@ -23,6 +23,7 @@ import abc import logging from twisted.enterprise import adbapi +from twisted.mail.smtp import sendmail from twisted.web.client import BrowserLikePolicyForHTTPS from synapse.api.auth import Auth @@ -174,6 +175,7 @@ class HomeServer(object): 'message_handler', 'pagination_handler', 'room_context_handler', + 'sendmail', ] # This is overridden in derived application classes @@ -269,6 +271,9 @@ class HomeServer(object): def build_room_creation_handler(self): return RoomCreationHandler(self) + def build_sendmail(self): + return sendmail + def build_state_handler(self): return StateHandler(self) diff --git a/tests/push/__init__.py b/tests/push/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/push/test_email.py b/tests/push/test_email.py new file mode 100644 index 0000000000..50ee6910d1 --- /dev/null +++ b/tests/push/test_email.py @@ -0,0 +1,148 @@ +# -*- coding: utf-8 -*- +# Copyright 2018 New Vector +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +import pkg_resources + +from twisted.internet.defer import Deferred + +from synapse.rest.client.v1 import admin, login, room + +from tests.unittest import HomeserverTestCase + +try: + from synapse.push.mailer import load_jinja2_templates +except Exception: + load_jinja2_templates = None + + +class EmailPusherTests(HomeserverTestCase): + + skip = "No Jinja installed" if not load_jinja2_templates else None + servlets = [ + admin.register_servlets, + room.register_servlets, + login.register_servlets, + ] + user_id = True + hijack_auth = False + + def make_homeserver(self, reactor, clock): + + # List[Tuple[Deferred, args, kwargs]] + self.email_attempts = [] + + def sendmail(*args, **kwargs): + d = Deferred() + self.email_attempts.append((d, args, kwargs)) + return d + + config = self.default_config() + config.email_enable_notifs = True + config.start_pushers = True + + config.email_template_dir = os.path.abspath( + pkg_resources.resource_filename('synapse', 'res/templates') + ) + config.email_notif_template_html = "notif_mail.html" + config.email_notif_template_text = "notif_mail.txt" + config.email_smtp_host = "127.0.0.1" + config.email_smtp_port = 20 + config.require_transport_security = False + config.email_smtp_user = None + config.email_app_name = "Matrix" + config.email_notif_from = "test@example.com" + + hs = self.setup_test_homeserver(config=config, sendmail=sendmail) + + return hs + + def test_sends_email(self): + + # Register the user who gets notified + user_id = self.register_user("user", "pass") + access_token = self.login("user", "pass") + + # Register the user who sends the message + other_user_id = self.register_user("otheruser", "pass") + other_access_token = self.login("otheruser", "pass") + + # Register the pusher + user_tuple = self.get_success( + self.hs.get_datastore().get_user_by_access_token(access_token) + ) + token_id = user_tuple["token_id"] + + self.get_success( + self.hs.get_pusherpool().add_pusher( + user_id=user_id, + access_token=token_id, + kind="email", + app_id="m.email", + app_display_name="Email Notifications", + device_display_name="a@example.com", + pushkey="a@example.com", + lang=None, + data={}, + ) + ) + + # Create a room + room = self.helper.create_room_as(user_id, tok=access_token) + + # Invite the other person + self.helper.invite(room=room, src=user_id, tok=access_token, targ=other_user_id) + + # The other user joins + self.helper.join(room=room, user=other_user_id, tok=other_access_token) + + # The other user sends some messages + self.helper.send(room, body="Hi!", tok=other_access_token) + self.helper.send(room, body="There!", tok=other_access_token) + + # Get the stream ordering before it gets sent + pushers = self.get_success( + self.hs.get_datastore().get_pushers_by(dict(user_name=user_id)) + ) + self.assertEqual(len(pushers), 1) + last_stream_ordering = pushers[0]["last_stream_ordering"] + + # Advance time a bit, so the pusher will register something has happened + self.pump(100) + + # It hasn't succeeded yet, so the stream ordering shouldn't have moved + pushers = self.get_success( + self.hs.get_datastore().get_pushers_by(dict(user_name=user_id)) + ) + self.assertEqual(len(pushers), 1) + self.assertEqual(last_stream_ordering, pushers[0]["last_stream_ordering"]) + + # One email was attempted to be sent + self.assertEqual(len(self.email_attempts), 1) + + # Make the email succeed + self.email_attempts[0][0].callback(True) + self.pump() + + # One email was attempted to be sent + self.assertEqual(len(self.email_attempts), 1) + + # The stream ordering has increased + pushers = self.get_success( + self.hs.get_datastore().get_pushers_by(dict(user_name=user_id)) + ) + self.assertEqual(len(pushers), 1) + self.assertTrue(pushers[0]["last_stream_ordering"] > last_stream_ordering) diff --git a/tests/server.py b/tests/server.py index 7bee58dff1..819c854448 100644 --- a/tests/server.py +++ b/tests/server.py @@ -125,7 +125,9 @@ def make_request(method, path, content=b"", access_token=None, request=SynapseRe req.content = BytesIO(content) if access_token: - req.requestHeaders.addRawHeader(b"Authorization", b"Bearer " + access_token) + req.requestHeaders.addRawHeader( + b"Authorization", b"Bearer " + access_token.encode('ascii') + ) if content: req.requestHeaders.addRawHeader(b"Content-Type", b"application/json") diff --git a/tests/test_mau.py b/tests/test_mau.py index bdbacb8448..5d387851c5 100644 --- a/tests/test_mau.py +++ b/tests/test_mau.py @@ -207,7 +207,7 @@ class TestMauLimit(unittest.TestCase): def do_sync_for_user(self, token): request, channel = make_request( - "GET", "/sync", access_token=token.encode('ascii') + "GET", "/sync", access_token=token ) render(request, self.resource, self.reactor) diff --git a/tests/unittest.py b/tests/unittest.py index a59291cc60..4d40bdb6a5 100644 --- a/tests/unittest.py +++ b/tests/unittest.py @@ -146,6 +146,13 @@ def DEBUG(target): return target +def INFO(target): + """A decorator to set the .loglevel attribute to logging.INFO. + Can apply to either a TestCase or an individual test method.""" + target.loglevel = logging.INFO + return target + + class HomeserverTestCase(TestCase): """ A base TestCase that reduces boilerplate for HomeServer-using test cases. @@ -373,5 +380,5 @@ class HomeserverTestCase(TestCase): self.render(request) self.assertEqual(channel.code, 200) - access_token = channel.json_body["access_token"].encode('ascii') + access_token = channel.json_body["access_token"] return access_token From 2e223a8c22ef7f65aa42fd149f178a842b60e3c7 Mon Sep 17 00:00:00 2001 From: Amber Brown Date: Wed, 31 Oct 2018 04:24:59 +1100 Subject: [PATCH 56/84] Remove the unused /pull federation API (#4118) --- changelog.d/4118.removal | 1 + synapse/federation/federation_server.py | 5 ----- synapse/federation/transport/server.py | 9 --------- 3 files changed, 1 insertion(+), 14 deletions(-) create mode 100644 changelog.d/4118.removal diff --git a/changelog.d/4118.removal b/changelog.d/4118.removal new file mode 100644 index 0000000000..6fb1d67b47 --- /dev/null +++ b/changelog.d/4118.removal @@ -0,0 +1 @@ +The obsolete and non-functional /pull federation endpoint has been removed. diff --git a/synapse/federation/federation_server.py b/synapse/federation/federation_server.py index 0f9302a6a8..fa2cc550e2 100644 --- a/synapse/federation/federation_server.py +++ b/synapse/federation/federation_server.py @@ -323,11 +323,6 @@ class FederationServer(FederationBase): else: defer.returnValue((404, "")) - @defer.inlineCallbacks - @log_function - def on_pull_request(self, origin, versions): - raise NotImplementedError("Pull transactions not implemented") - @defer.inlineCallbacks def on_query_request(self, query_type, args): received_queries_counter.labels(query_type).inc() diff --git a/synapse/federation/transport/server.py b/synapse/federation/transport/server.py index 7288d49074..3553f418f1 100644 --- a/synapse/federation/transport/server.py +++ b/synapse/federation/transport/server.py @@ -362,14 +362,6 @@ class FederationSendServlet(BaseFederationServlet): defer.returnValue((code, response)) -class FederationPullServlet(BaseFederationServlet): - PATH = "/pull/" - - # This is for when someone asks us for everything since version X - def on_GET(self, origin, content, query): - return self.handler.on_pull_request(query["origin"][0], query["v"]) - - class FederationEventServlet(BaseFederationServlet): PATH = "/event/(?P[^/]*)/" @@ -1261,7 +1253,6 @@ class FederationGroupsSettingJoinPolicyServlet(BaseFederationServlet): FEDERATION_SERVLET_CLASSES = ( FederationSendServlet, - FederationPullServlet, FederationEventServlet, FederationStateServlet, FederationStateIdsServlet, From 3bade14ec0aa7e56c84d30241bd86a177f0699d6 Mon Sep 17 00:00:00 2001 From: Amber Brown Date: Wed, 31 Oct 2018 04:33:41 +1100 Subject: [PATCH 57/84] Fix search 500ing (#4122) --- changelog.d/4122.bugfix | 1 + synapse/handlers/search.py | 8 ++- tests/rest/client/v1/test_rooms.py | 106 ++++++++++++++++++++++++++++- 3 files changed, 112 insertions(+), 3 deletions(-) create mode 100644 changelog.d/4122.bugfix diff --git a/changelog.d/4122.bugfix b/changelog.d/4122.bugfix new file mode 100644 index 0000000000..66dcfb18b9 --- /dev/null +++ b/changelog.d/4122.bugfix @@ -0,0 +1 @@ +Searches that request profile info now no longer fail with a 500. diff --git a/synapse/handlers/search.py b/synapse/handlers/search.py index 0c1d52fd11..80e7b15de8 100644 --- a/synapse/handlers/search.py +++ b/synapse/handlers/search.py @@ -24,6 +24,7 @@ from synapse.api.constants import EventTypes, Membership from synapse.api.errors import SynapseError from synapse.api.filtering import Filter from synapse.events.utils import serialize_event +from synapse.storage.state import StateFilter from synapse.visibility import filter_events_for_client from ._base import BaseHandler @@ -324,9 +325,12 @@ class SearchHandler(BaseHandler): else: last_event_id = event.event_id + state_filter = StateFilter.from_types( + [(EventTypes.Member, sender) for sender in senders] + ) + state = yield self.store.get_state_for_event( - last_event_id, - types=[(EventTypes.Member, sender) for sender in senders] + last_event_id, state_filter ) res["profile_info"] = { diff --git a/tests/rest/client/v1/test_rooms.py b/tests/rest/client/v1/test_rooms.py index 359f7777ff..a824be9a62 100644 --- a/tests/rest/client/v1/test_rooms.py +++ b/tests/rest/client/v1/test_rooms.py @@ -23,7 +23,7 @@ from six.moves.urllib import parse as urlparse from twisted.internet import defer from synapse.api.constants import Membership -from synapse.rest.client.v1 import room +from synapse.rest.client.v1 import admin, login, room from tests import unittest @@ -799,3 +799,107 @@ class RoomMessageListTestCase(RoomBase): self.assertEquals(token, channel.json_body['start']) self.assertTrue("chunk" in channel.json_body) self.assertTrue("end" in channel.json_body) + + +class RoomSearchTestCase(unittest.HomeserverTestCase): + servlets = [ + admin.register_servlets, + room.register_servlets, + login.register_servlets, + ] + user_id = True + hijack_auth = False + + def prepare(self, reactor, clock, hs): + + # Register the user who does the searching + self.user_id = self.register_user("user", "pass") + self.access_token = self.login("user", "pass") + + # Register the user who sends the message + self.other_user_id = self.register_user("otheruser", "pass") + self.other_access_token = self.login("otheruser", "pass") + + # Create a room + self.room = self.helper.create_room_as(self.user_id, tok=self.access_token) + + # Invite the other person + self.helper.invite( + room=self.room, + src=self.user_id, + tok=self.access_token, + targ=self.other_user_id, + ) + + # The other user joins + self.helper.join( + room=self.room, user=self.other_user_id, tok=self.other_access_token + ) + + def test_finds_message(self): + """ + The search functionality will search for content in messages if asked to + do so. + """ + # The other user sends some messages + self.helper.send(self.room, body="Hi!", tok=self.other_access_token) + self.helper.send(self.room, body="There!", tok=self.other_access_token) + + request, channel = self.make_request( + "POST", + "/search?access_token=%s" % (self.access_token,), + { + "search_categories": { + "room_events": {"keys": ["content.body"], "search_term": "Hi"} + } + }, + ) + self.render(request) + + # Check we get the results we expect -- one search result, of the sent + # messages + self.assertEqual(channel.code, 200) + results = channel.json_body["search_categories"]["room_events"] + self.assertEqual(results["count"], 1) + self.assertEqual(results["results"][0]["result"]["content"]["body"], "Hi!") + + # No context was requested, so we should get none. + self.assertEqual(results["results"][0]["context"], {}) + + def test_include_context(self): + """ + When event_context includes include_profile, profile information will be + included in the search response. + """ + # The other user sends some messages + self.helper.send(self.room, body="Hi!", tok=self.other_access_token) + self.helper.send(self.room, body="There!", tok=self.other_access_token) + + request, channel = self.make_request( + "POST", + "/search?access_token=%s" % (self.access_token,), + { + "search_categories": { + "room_events": { + "keys": ["content.body"], + "search_term": "Hi", + "event_context": {"include_profile": True}, + } + } + }, + ) + self.render(request) + + # Check we get the results we expect -- one search result, of the sent + # messages + self.assertEqual(channel.code, 200) + results = channel.json_body["search_categories"]["room_events"] + self.assertEqual(results["count"], 1) + self.assertEqual(results["results"][0]["result"]["content"]["body"], "Hi!") + + # We should get context info, like the two users, and the display names. + context = results["results"][0]["context"] + self.assertEqual(len(context["profile_info"].keys()), 2) + self.assertEqual( + context["profile_info"][self.other_user_id]["displayname"], "otheruser" + ) From 086e1a8f3e873351a749cb8b341cd2adbe701049 Mon Sep 17 00:00:00 2001 From: Amber Brown Date: Wed, 31 Oct 2018 04:33:41 +1100 Subject: [PATCH 58/84] Fix search 500ing (#4122) --- changelog.d/4122.bugfix | 1 + synapse/handlers/search.py | 8 ++- tests/rest/client/v1/test_rooms.py | 106 ++++++++++++++++++++++++++++- 3 files changed, 112 insertions(+), 3 deletions(-) create mode 100644 changelog.d/4122.bugfix diff --git a/changelog.d/4122.bugfix b/changelog.d/4122.bugfix new file mode 100644 index 0000000000..66dcfb18b9 --- /dev/null +++ b/changelog.d/4122.bugfix @@ -0,0 +1 @@ +Searches that request profile info now no longer fail with a 500. diff --git a/synapse/handlers/search.py b/synapse/handlers/search.py index 0c1d52fd11..80e7b15de8 100644 --- a/synapse/handlers/search.py +++ b/synapse/handlers/search.py @@ -24,6 +24,7 @@ from synapse.api.constants import EventTypes, Membership from synapse.api.errors import SynapseError from synapse.api.filtering import Filter from synapse.events.utils import serialize_event +from synapse.storage.state import StateFilter from synapse.visibility import filter_events_for_client from ._base import BaseHandler @@ -324,9 +325,12 @@ class SearchHandler(BaseHandler): else: last_event_id = event.event_id + state_filter = StateFilter.from_types( + [(EventTypes.Member, sender) for sender in senders] + ) + state = yield self.store.get_state_for_event( - last_event_id, - types=[(EventTypes.Member, sender) for sender in senders] + last_event_id, state_filter ) res["profile_info"] = { diff --git a/tests/rest/client/v1/test_rooms.py b/tests/rest/client/v1/test_rooms.py index 359f7777ff..a824be9a62 100644 --- a/tests/rest/client/v1/test_rooms.py +++ b/tests/rest/client/v1/test_rooms.py @@ -23,7 +23,7 @@ from six.moves.urllib import parse as urlparse from twisted.internet import defer from synapse.api.constants import Membership -from synapse.rest.client.v1 import room +from synapse.rest.client.v1 import admin, login, room from tests import unittest @@ -799,3 +799,107 @@ class RoomMessageListTestCase(RoomBase): self.assertEquals(token, channel.json_body['start']) self.assertTrue("chunk" in channel.json_body) self.assertTrue("end" in channel.json_body) + + +class RoomSearchTestCase(unittest.HomeserverTestCase): + servlets = [ + admin.register_servlets, + room.register_servlets, + login.register_servlets, + ] + user_id = True + hijack_auth = False + + def prepare(self, reactor, clock, hs): + + # Register the user who does the searching + self.user_id = self.register_user("user", "pass") + self.access_token = self.login("user", "pass") + + # Register the user who sends the message + self.other_user_id = self.register_user("otheruser", "pass") + self.other_access_token = self.login("otheruser", "pass") + + # Create a room + self.room = self.helper.create_room_as(self.user_id, tok=self.access_token) + + # Invite the other person + self.helper.invite( + room=self.room, + src=self.user_id, + tok=self.access_token, + targ=self.other_user_id, + ) + + # The other user joins + self.helper.join( + room=self.room, user=self.other_user_id, tok=self.other_access_token + ) + + def test_finds_message(self): + """ + The search functionality will search for content in messages if asked to + do so. + """ + # The other user sends some messages + self.helper.send(self.room, body="Hi!", tok=self.other_access_token) + self.helper.send(self.room, body="There!", tok=self.other_access_token) + + request, channel = self.make_request( + "POST", + "/search?access_token=%s" % (self.access_token,), + { + "search_categories": { + "room_events": {"keys": ["content.body"], "search_term": "Hi"} + } + }, + ) + self.render(request) + + # Check we get the results we expect -- one search result, of the sent + # messages + self.assertEqual(channel.code, 200) + results = channel.json_body["search_categories"]["room_events"] + self.assertEqual(results["count"], 1) + self.assertEqual(results["results"][0]["result"]["content"]["body"], "Hi!") + + # No context was requested, so we should get none. + self.assertEqual(results["results"][0]["context"], {}) + + def test_include_context(self): + """ + When event_context includes include_profile, profile information will be + included in the search response. + """ + # The other user sends some messages + self.helper.send(self.room, body="Hi!", tok=self.other_access_token) + self.helper.send(self.room, body="There!", tok=self.other_access_token) + + request, channel = self.make_request( + "POST", + "/search?access_token=%s" % (self.access_token,), + { + "search_categories": { + "room_events": { + "keys": ["content.body"], + "search_term": "Hi", + "event_context": {"include_profile": True}, + } + } + }, + ) + self.render(request) + + # Check we get the results we expect -- one search result, of the sent + # messages + self.assertEqual(channel.code, 200) + results = channel.json_body["search_categories"]["room_events"] + self.assertEqual(results["count"], 1) + self.assertEqual(results["results"][0]["result"]["content"]["body"], "Hi!") + + # We should get context info, like the two users, and the display names. + context = results["results"][0]["context"] + self.assertEqual(len(context["profile_info"].keys()), 2) + self.assertEqual( + context["profile_info"][self.other_user_id]["displayname"], "otheruser" + ) From 67c1924899721e57093470d2f449f745a983b1d9 Mon Sep 17 00:00:00 2001 From: Amber Brown Date: Wed, 31 Oct 2018 06:27:05 +1100 Subject: [PATCH 59/84] version bump --- synapse/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/__init__.py b/synapse/__init__.py index c8377df7b2..97a57a7ac1 100644 --- a/synapse/__init__.py +++ b/synapse/__init__.py @@ -27,4 +27,4 @@ try: except ImportError: pass -__version__ = "0.33.8rc1" +__version__ = "0.33.8rc2" From e615e95590caa550de3199234c10761398521e10 Mon Sep 17 00:00:00 2001 From: Amber Brown Date: Wed, 31 Oct 2018 06:28:11 +1100 Subject: [PATCH 60/84] changelog --- CHANGES.md | 10 ++++++++++ changelog.d/4122.bugfix | 1 - 2 files changed, 10 insertions(+), 1 deletion(-) delete mode 100644 changelog.d/4122.bugfix diff --git a/CHANGES.md b/CHANGES.md index dc2a745c8b..560b406297 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,3 +1,13 @@ +Synapse 0.33.8rc2 (2018-10-31) +============================== + +Bugfixes +-------- + +- Searches that request profile info now no longer fail with a 500. Fixes + a regression in 0.33.8rc1. ([\#4122](https://github.com/matrix-org/synapse/issues/4122)) + + Synapse 0.33.8rc1 (2018-10-29) ============================== diff --git a/changelog.d/4122.bugfix b/changelog.d/4122.bugfix deleted file mode 100644 index 66dcfb18b9..0000000000 --- a/changelog.d/4122.bugfix +++ /dev/null @@ -1 +0,0 @@ -Searches that request profile info now no longer fail with a 500. From 0f6ec6d1aedc88a2057f50b77ce9d6a405177096 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Tue, 30 Oct 2018 21:00:31 +0000 Subject: [PATCH 61/84] Attempt to fix tox installs It seems that, at some point, the ability to run tox on old servers (with old setuptools) got broken - and it was only working on our Jenkins instance by dint of reusing the tox environments. Let's try to get tox to do the right thing, and remove the guff from jenkins/prepare_synapse.sh. (There is a separate question about whether the jenkins builds should be using tox to prepare the virtualenv at all here, but that is somewhat orthogonal). --- jenkins/prepare_synapse.sh | 19 ------------------- tox.ini | 14 ++++++++++++++ 2 files changed, 14 insertions(+), 19 deletions(-) diff --git a/jenkins/prepare_synapse.sh b/jenkins/prepare_synapse.sh index d95ca846c4..016afb8baa 100755 --- a/jenkins/prepare_synapse.sh +++ b/jenkins/prepare_synapse.sh @@ -14,22 +14,3 @@ fi # set up the virtualenv tox -e py27 --notest -v - -TOX_BIN=$TOX_DIR/py27/bin - -# cryptography 2.2 requires setuptools >= 18.5. -# -# older versions of virtualenv (?) give us a virtualenv with the same version -# of setuptools as is installed on the system python (and tox runs virtualenv -# under python3, so we get the version of setuptools that is installed on that). -# -# anyway, make sure that we have a recent enough setuptools. -$TOX_BIN/pip install 'setuptools>=18.5' - -# we also need a semi-recent version of pip, because old ones fail to install -# the "enum34" dependency of cryptography. -$TOX_BIN/pip install 'pip>=10' - -{ python synapse/python_dependencies.py - echo lxml -} | xargs $TOX_BIN/pip install diff --git a/tox.ini b/tox.ini index 9de5a5704a..920211bf50 100644 --- a/tox.ini +++ b/tox.ini @@ -11,6 +11,20 @@ deps = # needed by some of the tests lxml + # cyptography 2.2 requires setuptools >= 18.5 + # + # older versions of virtualenv (?) give us a virtualenv with the same + # version of setuptools as is installed on the system python (and tox runs + # virtualenv under python3, so we get the version of setuptools that is + # installed on that). + # + # anyway, make sure that we have a recent enough setuptools. + setuptools>=18.5 + + # we also need a semi-recent version of pip, because old ones fail to + # install the "enum34" dependency of cryptography. + pip>=10 + setenv = PYTHONDONTWRITEBYTECODE = no_byte_code From a2d8bff0dc430f9e0a980535dd4330ff420118ee Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Tue, 30 Oct 2018 21:21:05 +0000 Subject: [PATCH 62/84] changelog --- changelog.d/4124.misc | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog.d/4124.misc diff --git a/changelog.d/4124.misc b/changelog.d/4124.misc new file mode 100644 index 0000000000..28f438b9b2 --- /dev/null +++ b/changelog.d/4124.misc @@ -0,0 +1 @@ +Fix `tox` failure on old systems From f79f45448527f22f3813e38233521a5e13e9223e Mon Sep 17 00:00:00 2001 From: Amber Brown Date: Wed, 31 Oct 2018 22:29:02 +1100 Subject: [PATCH 63/84] Remove deprecated v1 key exchange endpoint (#4119) --- changelog.d/4119.removal | 1 + synapse/api/urls.py | 1 - synapse/app/homeserver.py | 7 +- synapse/rest/key/v1/__init__.py | 14 ---- synapse/rest/key/v1/server_key_resource.py | 92 ---------------------- 5 files changed, 2 insertions(+), 113 deletions(-) create mode 100644 changelog.d/4119.removal delete mode 100644 synapse/rest/key/v1/__init__.py delete mode 100644 synapse/rest/key/v1/server_key_resource.py diff --git a/changelog.d/4119.removal b/changelog.d/4119.removal new file mode 100644 index 0000000000..81383ece6b --- /dev/null +++ b/changelog.d/4119.removal @@ -0,0 +1 @@ +The deprecated v1 key exchange endpoints have been removed. diff --git a/synapse/api/urls.py b/synapse/api/urls.py index 6d9f1ca0ef..f78695b657 100644 --- a/synapse/api/urls.py +++ b/synapse/api/urls.py @@ -28,7 +28,6 @@ FEDERATION_PREFIX = "/_matrix/federation/v1" STATIC_PREFIX = "/_matrix/static" WEB_CLIENT_PREFIX = "/_matrix/client" CONTENT_REPO_PREFIX = "/_matrix/content" -SERVER_KEY_PREFIX = "/_matrix/key/v1" SERVER_KEY_V2_PREFIX = "/_matrix/key/v2" MEDIA_PREFIX = "/_matrix/media/r0" LEGACY_MEDIA_PREFIX = "/_matrix/media/v1" diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index 593e1e75db..415374a2ce 100755 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -37,7 +37,6 @@ from synapse.api.urls import ( FEDERATION_PREFIX, LEGACY_MEDIA_PREFIX, MEDIA_PREFIX, - SERVER_KEY_PREFIX, SERVER_KEY_V2_PREFIX, STATIC_PREFIX, WEB_CLIENT_PREFIX, @@ -59,7 +58,6 @@ from synapse.python_dependencies import CONDITIONAL_REQUIREMENTS, check_requirem from synapse.replication.http import REPLICATION_PREFIX, ReplicationRestResource from synapse.replication.tcp.resource import ReplicationStreamProtocolFactory from synapse.rest import ClientRestResource -from synapse.rest.key.v1.server_key_resource import LocalKey from synapse.rest.key.v2 import KeyApiV2Resource from synapse.rest.media.v0.content_repository import ContentRepoResource from synapse.server import HomeServer @@ -236,10 +234,7 @@ class SynapseHomeServer(HomeServer): ) if name in ["keys", "federation"]: - resources.update({ - SERVER_KEY_PREFIX: LocalKey(self), - SERVER_KEY_V2_PREFIX: KeyApiV2Resource(self), - }) + resources[SERVER_KEY_V2_PREFIX] = KeyApiV2Resource(self) if name == "webclient": resources[WEB_CLIENT_PREFIX] = build_resource_for_web_client(self) diff --git a/synapse/rest/key/v1/__init__.py b/synapse/rest/key/v1/__init__.py deleted file mode 100644 index fe0ac3f8e9..0000000000 --- a/synapse/rest/key/v1/__init__.py +++ /dev/null @@ -1,14 +0,0 @@ -# -*- coding: utf-8 -*- -# Copyright 2015, 2016 OpenMarket Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. diff --git a/synapse/rest/key/v1/server_key_resource.py b/synapse/rest/key/v1/server_key_resource.py deleted file mode 100644 index 38eb2ee23f..0000000000 --- a/synapse/rest/key/v1/server_key_resource.py +++ /dev/null @@ -1,92 +0,0 @@ -# -*- coding: utf-8 -*- -# Copyright 2014-2016 OpenMarket Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -import logging - -from canonicaljson import encode_canonical_json -from signedjson.sign import sign_json -from unpaddedbase64 import encode_base64 - -from OpenSSL import crypto -from twisted.web.resource import Resource - -from synapse.http.server import respond_with_json_bytes - -logger = logging.getLogger(__name__) - - -class LocalKey(Resource): - """HTTP resource containing encoding the TLS X.509 certificate and NACL - signature verification keys for this server:: - - GET /key HTTP/1.1 - - HTTP/1.1 200 OK - Content-Type: application/json - { - "server_name": "this.server.example.com" - "verify_keys": { - "algorithm:version": # base64 encoded NACL verification key. - }, - "tls_certificate": # base64 ASN.1 DER encoded X.509 tls cert. - "signatures": { - "this.server.example.com": { - "algorithm:version": # NACL signature for this server. - } - } - } - """ - - def __init__(self, hs): - self.response_body = encode_canonical_json( - self.response_json_object(hs.config) - ) - Resource.__init__(self) - - @staticmethod - def response_json_object(server_config): - verify_keys = {} - for key in server_config.signing_key: - verify_key_bytes = key.verify_key.encode() - key_id = "%s:%s" % (key.alg, key.version) - verify_keys[key_id] = encode_base64(verify_key_bytes) - - x509_certificate_bytes = crypto.dump_certificate( - crypto.FILETYPE_ASN1, - server_config.tls_certificate - ) - json_object = { - u"server_name": server_config.server_name, - u"verify_keys": verify_keys, - u"tls_certificate": encode_base64(x509_certificate_bytes) - } - for key in server_config.signing_key: - json_object = sign_json( - json_object, - server_config.server_name, - key, - ) - - return json_object - - def render_GET(self, request): - return respond_with_json_bytes( - request, 200, self.response_body, - ) - - def getChild(self, name, request): - if name == b'': - return self From 916efc824950f924c3f7bced09b9cd5759b1532e Mon Sep 17 00:00:00 2001 From: Amber Brown Date: Wed, 31 Oct 2018 23:14:39 +1100 Subject: [PATCH 64/84] Remove fetching keys via the deprecated v1 kex method (#4120) --- changelog.d/4120.removal | 1 + synapse/crypto/keyclient.py | 8 ++- synapse/crypto/keyring.py | 110 +++--------------------------------- 3 files changed, 13 insertions(+), 106 deletions(-) create mode 100644 changelog.d/4120.removal diff --git a/changelog.d/4120.removal b/changelog.d/4120.removal new file mode 100644 index 0000000000..a7a567098f --- /dev/null +++ b/changelog.d/4120.removal @@ -0,0 +1 @@ +Synapse will no longer fetch keys using the fallback deprecated v1 key exchange method and will now always use v2. diff --git a/synapse/crypto/keyclient.py b/synapse/crypto/keyclient.py index 080c81f14b..d40e4b8591 100644 --- a/synapse/crypto/keyclient.py +++ b/synapse/crypto/keyclient.py @@ -15,6 +15,8 @@ import logging +from six.moves import urllib + from canonicaljson import json from twisted.internet import defer, reactor @@ -28,15 +30,15 @@ from synapse.util import logcontext logger = logging.getLogger(__name__) -KEY_API_V1 = b"/_matrix/key/v1/" +KEY_API_V2 = "/_matrix/key/v2/server/%s" @defer.inlineCallbacks -def fetch_server_key(server_name, tls_client_options_factory, path=KEY_API_V1): +def fetch_server_key(server_name, tls_client_options_factory, key_id): """Fetch the keys for a remote server.""" factory = SynapseKeyClientFactory() - factory.path = path + factory.path = KEY_API_V2 % (urllib.parse.quote(key_id), ) factory.host = server_name endpoint = matrix_federation_endpoint( reactor, server_name, tls_client_options_factory, timeout=30 diff --git a/synapse/crypto/keyring.py b/synapse/crypto/keyring.py index d89f94c219..515ebbc148 100644 --- a/synapse/crypto/keyring.py +++ b/synapse/crypto/keyring.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- # Copyright 2014-2016 OpenMarket Ltd -# Copyright 2017 New Vector Ltd. +# Copyright 2017, 2018 New Vector Ltd. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -18,8 +18,6 @@ import hashlib import logging from collections import namedtuple -from six.moves import urllib - from signedjson.key import ( decode_verify_key_bytes, encode_verify_key_base64, @@ -395,32 +393,13 @@ class Keyring(object): @defer.inlineCallbacks def get_keys_from_server(self, server_name_and_key_ids): - @defer.inlineCallbacks - def get_key(server_name, key_ids): - keys = None - try: - keys = yield self.get_server_verify_key_v2_direct( - server_name, key_ids - ) - except Exception as e: - logger.info( - "Unable to get key %r for %r directly: %s %s", - key_ids, server_name, - type(e).__name__, str(e), - ) - - if not keys: - keys = yield self.get_server_verify_key_v1_direct( - server_name, key_ids - ) - - keys = {server_name: keys} - - defer.returnValue(keys) - results = yield logcontext.make_deferred_yieldable(defer.gatherResults( [ - run_in_background(get_key, server_name, key_ids) + run_in_background( + self.get_server_verify_key_v2_direct, + server_name, + key_ids, + ) for server_name, key_ids in server_name_and_key_ids ], consumeErrors=True, @@ -525,10 +504,7 @@ class Keyring(object): continue (response, tls_certificate) = yield fetch_server_key( - server_name, self.hs.tls_client_options_factory, - path=("/_matrix/key/v2/server/%s" % ( - urllib.parse.quote(requested_key_id), - )).encode("ascii"), + server_name, self.hs.tls_client_options_factory, requested_key_id ) if (u"signatures" not in response @@ -657,78 +633,6 @@ class Keyring(object): defer.returnValue(results) - @defer.inlineCallbacks - def get_server_verify_key_v1_direct(self, server_name, key_ids): - """Finds a verification key for the server with one of the key ids. - Args: - server_name (str): The name of the server to fetch a key for. - keys_ids (list of str): The key_ids to check for. - """ - - # Try to fetch the key from the remote server. - - (response, tls_certificate) = yield fetch_server_key( - server_name, self.hs.tls_client_options_factory - ) - - # Check the response. - - x509_certificate_bytes = crypto.dump_certificate( - crypto.FILETYPE_ASN1, tls_certificate - ) - - if ("signatures" not in response - or server_name not in response["signatures"]): - raise KeyLookupError("Key response not signed by remote server") - - if "tls_certificate" not in response: - raise KeyLookupError("Key response missing TLS certificate") - - tls_certificate_b64 = response["tls_certificate"] - - if encode_base64(x509_certificate_bytes) != tls_certificate_b64: - raise KeyLookupError("TLS certificate doesn't match") - - # Cache the result in the datastore. - - time_now_ms = self.clock.time_msec() - - verify_keys = {} - for key_id, key_base64 in response["verify_keys"].items(): - if is_signing_algorithm_supported(key_id): - key_bytes = decode_base64(key_base64) - verify_key = decode_verify_key_bytes(key_id, key_bytes) - verify_key.time_added = time_now_ms - verify_keys[key_id] = verify_key - - for key_id in response["signatures"][server_name]: - if key_id not in response["verify_keys"]: - raise KeyLookupError( - "Key response must include verification keys for all" - " signatures" - ) - if key_id in verify_keys: - verify_signed_json( - response, - server_name, - verify_keys[key_id] - ) - - yield self.store.store_server_certificate( - server_name, - server_name, - time_now_ms, - tls_certificate, - ) - - yield self.store_keys( - server_name=server_name, - from_server=server_name, - verify_keys=verify_keys, - ) - - defer.returnValue(verify_keys) - def store_keys(self, server_name, from_server, verify_keys): """Store a collection of verify keys for a given server Args: From e3758c8c929be11fe38cebb2b4f7d43185f80197 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jonas=20Sch=C3=BCrmann?= Date: Wed, 31 Oct 2018 15:46:47 +0100 Subject: [PATCH 65/84] Fix typo in docker-compose.yml MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Jonas Schürmann --- contrib/docker/docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/docker/docker-compose.yml b/contrib/docker/docker-compose.yml index 3a8dfbae34..b1f6fcb7da 100644 --- a/contrib/docker/docker-compose.yml +++ b/contrib/docker/docker-compose.yml @@ -47,4 +47,4 @@ services: # You may store the database tables in a local folder.. - ./schemas:/var/lib/postgresql/data # .. or store them on some high performance storage for better results - # - /path/to/ssd/storage:/var/lib/postfesql/data + # - /path/to/ssd/storage:/var/lib/postgresql/data From 9b827c40ca71510390c92472f7ec5cfcff9e69b2 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff <1389908+richvdh@users.noreply.github.com> Date: Wed, 31 Oct 2018 15:42:23 +0000 Subject: [PATCH 66/84] Log some bits about event creation (#4121) I found these helpful in debugging my room upgrade tests. --- changelog.d/4121.misc | 1 + synapse/handlers/message.py | 3 +++ synapse/handlers/room.py | 4 ++++ 3 files changed, 8 insertions(+) create mode 100644 changelog.d/4121.misc diff --git a/changelog.d/4121.misc b/changelog.d/4121.misc new file mode 100644 index 0000000000..9c29d80c3f --- /dev/null +++ b/changelog.d/4121.misc @@ -0,0 +1 @@ +Log some bits about room creation diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py index 969e588e73..a7cd779b02 100644 --- a/synapse/handlers/message.py +++ b/synapse/handlers/message.py @@ -427,6 +427,9 @@ class EventCreationHandler(object): if event.is_state(): prev_state = yield self.deduplicate_state_event(event, context) + logger.info( + "Not bothering to persist duplicate state event %s", event.event_id, + ) if prev_state is not None: defer.returnValue(prev_state) diff --git a/synapse/handlers/room.py b/synapse/handlers/room.py index 1d9417ff1a..fe960342b9 100644 --- a/synapse/handlers/room.py +++ b/synapse/handlers/room.py @@ -104,6 +104,8 @@ class RoomCreationHandler(BaseHandler): creator_id=user_id, is_public=r["is_public"], ) + logger.info("Creating new room %s to replace %s", new_room_id, old_room_id) + # we create and auth the tombstone event before properly creating the new # room, to check our user has perms in the old room. tombstone_event, tombstone_context = ( @@ -522,6 +524,7 @@ class RoomCreationHandler(BaseHandler): @defer.inlineCallbacks def send(etype, content, **kwargs): event = create(etype, content, **kwargs) + logger.info("Sending %s in new room", etype) yield self.event_creation_handler.create_and_send_nonmember_event( creator, event, @@ -544,6 +547,7 @@ class RoomCreationHandler(BaseHandler): content=creation_content, ) + logger.info("Sending %s in new room", EventTypes.Member) yield self.room_member_handler.update_membership( creator, creator.user, From 94c7fadc98542d582ff67c5ac788081c0d836e6b Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Fri, 26 Oct 2018 15:11:35 +0100 Subject: [PATCH 67/84] Attempt to move room aliases on room upgrades --- changelog.d/4101.feature | 1 + synapse/handlers/directory.py | 34 ++++++++-- synapse/handlers/room.py | 121 +++++++++++++++++++++++++++++++--- 3 files changed, 142 insertions(+), 14 deletions(-) create mode 100644 changelog.d/4101.feature diff --git a/changelog.d/4101.feature b/changelog.d/4101.feature new file mode 100644 index 0000000000..a3f7dbdcdd --- /dev/null +++ b/changelog.d/4101.feature @@ -0,0 +1 @@ +Support for replacing rooms with new ones diff --git a/synapse/handlers/directory.py b/synapse/handlers/directory.py index 7d67bf803a..0699731c13 100644 --- a/synapse/handlers/directory.py +++ b/synapse/handlers/directory.py @@ -138,9 +138,30 @@ class DirectoryHandler(BaseHandler): ) @defer.inlineCallbacks - def delete_association(self, requester, room_alias): - # association deletion for human users + def delete_association(self, requester, room_alias, send_event=True): + """Remove an alias from the directory + (this is only meant for human users; AS users should call + delete_appservice_association) + + Args: + requester (Requester): + room_alias (RoomAlias): + send_event (bool): Whether to send an updated m.room.aliases event. + Note that, if we delete the canonical alias, we will always attempt + to send an m.room.canonical_alias event + + Returns: + Deferred[unicode]: room id that the alias used to point to + + Raises: + NotFoundError: if the alias doesn't exist + + AuthError: if the user doesn't have perms to delete the alias (ie, the user + is neither the creator of the alias, nor a server admin. + + SynapseError: if the alias belongs to an AS + """ user_id = requester.user.to_string() try: @@ -168,10 +189,11 @@ class DirectoryHandler(BaseHandler): room_id = yield self._delete_association(room_alias) try: - yield self.send_room_alias_update_event( - requester, - room_id - ) + if send_event: + yield self.send_room_alias_update_event( + requester, + room_id + ) yield self._update_canonical_alias( requester, diff --git a/synapse/handlers/room.py b/synapse/handlers/room.py index 1d9417ff1a..76811050a6 100644 --- a/synapse/handlers/room.py +++ b/synapse/handlers/room.py @@ -136,10 +136,15 @@ class RoomCreationHandler(BaseHandler): requester, tombstone_event, tombstone_context, ) - # and finally, shut down the PLs in the old room, and update them in the new - # room. old_room_state = yield tombstone_context.get_current_state_ids(self.store) + # update any aliases + yield self._move_aliases_to_new_room( + requester, old_room_id, new_room_id, old_room_state, + ) + + # and finally, shut down the PLs in the old room, and update them in the new + # room. yield self._update_upgraded_room_pls( requester, old_room_id, new_room_id, old_room_state, ) @@ -245,11 +250,6 @@ class RoomCreationHandler(BaseHandler): if not self.spam_checker.user_may_create_room(user_id): raise SynapseError(403, "You are not permitted to create rooms") - # XXX check alias is free - # canonical_alias = None - - # XXX create association in directory handler - creation_content = { "room_version": new_room_version, "predecessor": { @@ -295,7 +295,112 @@ class RoomCreationHandler(BaseHandler): # XXX invites/joins # XXX 3pid invites - # XXX directory_handler.send_room_alias_update_event + + @defer.inlineCallbacks + def _move_aliases_to_new_room( + self, requester, old_room_id, new_room_id, old_room_state, + ): + directory_handler = self.hs.get_handlers().directory_handler + + aliases = yield self.store.get_aliases_for_room(old_room_id) + + # check to see if we have a canonical alias. + canonical_alias = None + canonical_alias_event_id = old_room_state.get((EventTypes.CanonicalAlias, "")) + if canonical_alias_event_id: + canonical_alias_event = yield self.store.get_event(canonical_alias_event_id) + if canonical_alias_event: + canonical_alias = canonical_alias_event.content.get("alias", "") + + # first we try to remove the aliases from the old room (we suppress sending + # the room_aliases event until the end). + # + # Note that we'll only be able to remove aliases that (a) aren't owned by an AS, + # and (b) unless the user is a server admin, which the user created. + # + # This is probably correct - given we don't allow such aliases to be deleted + # normally, it would be odd to allow it in the case of doing a room upgrade - + # but it makes the upgrade less effective, and you have to wonder why a room + # admin can't remove aliases that point to that room anyway. + # (cf https://github.com/matrix-org/synapse/issues/2360) + # + removed_aliases = [] + for alias_str in aliases: + alias = RoomAlias.from_string(alias_str) + try: + yield directory_handler.delete_association( + requester, alias, send_event=False, + ) + except SynapseError as e: + logger.warning( + "Unable to remove alias %s from old room: %s", + alias, e, + ) + else: + removed_aliases.append(alias_str) + + # if we didn't find any aliases, or couldn't remove anyway, we can skip the rest + # of this. + if not removed_aliases: + return + + try: + # this can fail if, for some reason, our user doesn't have perms to send + # m.room.aliases events in the old room (note that we've already checked that + # they have perms to send a tombstone event, so that's not terribly likely). + # + # If that happens, it's regrettable, but we should carry on: it's the same + # as when you remove an alias from the directory normally - it just means that + # the aliases event gets out of sync with the directory + # (cf https://github.com/vector-im/riot-web/issues/2369) + yield directory_handler.send_room_alias_update_event( + requester, old_room_id, + ) + except AuthError as e: + logger.warning( + "Failed to send updated alias event on old room: %s", e, + ) + + # we can now add any aliases we successfully removed to the new room. + for alias in removed_aliases: + try: + yield directory_handler.create_association( + requester, RoomAlias.from_string(alias), + new_room_id, servers=(self.hs.hostname, ), + send_event=False, + ) + logger.info("Moved alias %s to new room", alias) + except SynapseError as e: + # I'm not really expecting this to happen, but it could if the spam + # checking module decides it shouldn't, or similar. + logger.error( + "Error adding alias %s to new room: %s", + alias, e, + ) + + try: + if canonical_alias and (canonical_alias in removed_aliases): + yield self.event_creation_handler.create_and_send_nonmember_event( + requester, + { + "type": EventTypes.CanonicalAlias, + "state_key": "", + "room_id": new_room_id, + "sender": requester.user.to_string(), + "content": {"alias": canonical_alias, }, + }, + ratelimit=False + ) + + yield directory_handler.send_room_alias_update_event( + requester, new_room_id, + ) + except SynapseError as e: + # again I'm not really expecting this to fail, but if it does, I'd rather + # we returned the new room to the client at this point. + logger.error( + "Unable to send updated alias events in new room: %s", e, + ) @defer.inlineCallbacks def create_room(self, requester, config, ratelimit=True, From 0f8591a5a8695aa176736c651a361c40cf228b6d Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Mon, 29 Oct 2018 15:20:19 +0000 Subject: [PATCH 68/84] Avoid else clause on exception for clarity --- synapse/handlers/room.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/synapse/handlers/room.py b/synapse/handlers/room.py index 76811050a6..9ff4656717 100644 --- a/synapse/handlers/room.py +++ b/synapse/handlers/room.py @@ -331,13 +331,12 @@ class RoomCreationHandler(BaseHandler): yield directory_handler.delete_association( requester, alias, send_event=False, ) + removed_aliases.append(alias_str) except SynapseError as e: logger.warning( "Unable to remove alias %s from old room: %s", alias, e, ) - else: - removed_aliases.append(alias_str) # if we didn't find any aliases, or couldn't remove anyway, we can skip the rest # of this. From a8d41c6aff0e58fc24fae1fe4ae89d28541a63cb Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Wed, 31 Oct 2018 13:19:28 -0600 Subject: [PATCH 69/84] Include a version query string arg for the consent route --- synapse/handlers/auth.py | 5 ++++- synapse/rest/client/v2_alpha/auth.py | 6 ++++-- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/synapse/handlers/auth.py b/synapse/handlers/auth.py index d143522d9a..85fc1fc525 100644 --- a/synapse/handlers/auth.py +++ b/synapse/handlers/auth.py @@ -473,7 +473,10 @@ class AuthHandler(BaseHandler): "version": self.hs.config.user_consent_version, "en": { "name": "Privacy Policy", - "url": "%s/_matrix/consent" % (self.hs.config.public_baseurl,), + "url": "%s/_matrix/consent?v=%s" % ( + self.hs.config.public_baseurl, + self.hs.config.user_consent_version, + ), }, }, }, diff --git a/synapse/rest/client/v2_alpha/auth.py b/synapse/rest/client/v2_alpha/auth.py index 6f90935b22..a8d8ed6590 100644 --- a/synapse/rest/client/v2_alpha/auth.py +++ b/synapse/rest/client/v2_alpha/auth.py @@ -161,8 +161,9 @@ class AuthRestServlet(RestServlet): html = TERMS_TEMPLATE % { 'session': session, - 'terms_url': "%s/_matrix/consent" % ( + 'terms_url': "%s/_matrix/consent?v=%s" % ( self.hs.config.public_baseurl, + self.hs.config.user_consent_version, ), 'myurl': "%s/auth/%s/fallback/web" % ( CLIENT_V2_ALPHA_PREFIX, LoginType.TERMS @@ -241,8 +242,9 @@ class AuthRestServlet(RestServlet): else: html = TERMS_TEMPLATE % { 'session': session, - 'terms_url': "%s/_matrix/consent" % ( + 'terms_url': "%s/_matrix/consent?v=%s" % ( self.hs.config.public_baseurl, + self.hs.config.user_consent_version, ), 'myurl': "%s/auth/%s/fallback/web" % ( CLIENT_V2_ALPHA_PREFIX, LoginType.TERMS From a8c9faa9a2517f7d733f58bea574e5c147c5b328 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Wed, 31 Oct 2018 13:28:08 -0600 Subject: [PATCH 70/84] The tests also need a version parameter --- tests/test_terms_auth.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_terms_auth.py b/tests/test_terms_auth.py index 06b68f0a72..7deab5266f 100644 --- a/tests/test_terms_auth.py +++ b/tests/test_terms_auth.py @@ -67,7 +67,7 @@ class TermsTestCase(unittest.HomeserverTestCase): "privacy_policy": { "en": { "name": "Privacy Policy", - "url": "https://example.org/_matrix/consent", + "url": "https://example.org/_matrix/consent?v=1.0", }, "version": "1.0" }, From aa98e3889667bbbd4186f25336e1a9a5666113d2 Mon Sep 17 00:00:00 2001 From: Amber Brown Date: Thu, 1 Nov 2018 21:28:35 +1100 Subject: [PATCH 71/84] version --- synapse/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/__init__.py b/synapse/__init__.py index 97a57a7ac1..89ea9a9775 100644 --- a/synapse/__init__.py +++ b/synapse/__init__.py @@ -27,4 +27,4 @@ try: except ImportError: pass -__version__ = "0.33.8rc2" +__version__ = "0.33.8" From d0ebe8287196b9a7798503c1dfe174a6920bdc81 Mon Sep 17 00:00:00 2001 From: Amber Brown Date: Thu, 1 Nov 2018 21:29:11 +1100 Subject: [PATCH 72/84] changelog --- CHANGES.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/CHANGES.md b/CHANGES.md index 560b406297..8302610585 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,3 +1,9 @@ +Synapse 0.33.8 (2018-11-01) +=========================== + +No significant changes. + + Synapse 0.33.8rc2 (2018-10-31) ============================== From b3dd6fa98199f58804df903195b6c5a1cc4686d7 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 1 Nov 2018 11:43:46 +0000 Subject: [PATCH 73/84] Add STATE_V2_TEST room version --- synapse/api/constants.py | 7 ++++++- synapse/state/__init__.py | 2 +- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/synapse/api/constants.py b/synapse/api/constants.py index 5565e516d6..e63b1e8a38 100644 --- a/synapse/api/constants.py +++ b/synapse/api/constants.py @@ -102,6 +102,7 @@ class ThirdPartyEntityKind(object): class RoomVersions(object): V1 = "1" VDH_TEST = "vdh-test-version" + STATE_V2_TEST = "state-v2-test" # the version we will give rooms which are created on this server @@ -109,7 +110,11 @@ DEFAULT_ROOM_VERSION = RoomVersions.V1 # vdh-test-version is a placeholder to get room versioning support working and tested # until we have a working v2. -KNOWN_ROOM_VERSIONS = {RoomVersions.V1, RoomVersions.VDH_TEST} +KNOWN_ROOM_VERSIONS = { + RoomVersions.V1, + RoomVersions.VDH_TEST, + RoomVersions.STATE_V2_TEST, +} ServerNoticeMsgType = "m.server_notice" ServerNoticeLimitReached = "m.server_notice.usage_limit_reached" diff --git a/synapse/state/__init__.py b/synapse/state/__init__.py index 9b40b18d5b..943d5d6bb5 100644 --- a/synapse/state/__init__.py +++ b/synapse/state/__init__.py @@ -607,7 +607,7 @@ def resolve_events_with_store(room_version, state_sets, event_map, state_res_sto return v1.resolve_events_with_store( state_sets, event_map, state_res_store.get_events, ) - elif room_version == RoomVersions.VDH_TEST: + elif room_version in (RoomVersions.VDH_TEST, RoomVersions.STATE_V2_TEST): return v2.resolve_events_with_store( state_sets, event_map, state_res_store, ) From 62d683161eb2548cd4e81659519229d24ba48f37 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 1 Nov 2018 11:44:44 +0000 Subject: [PATCH 74/84] Newsfile --- changelog.d/4128.misc | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog.d/4128.misc diff --git a/changelog.d/4128.misc b/changelog.d/4128.misc new file mode 100644 index 0000000000..76ab4b085c --- /dev/null +++ b/changelog.d/4128.misc @@ -0,0 +1 @@ +Add STATE_V2_TEST room version From 642505abc385afe7849f60c37f8ef99592f8f7b4 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Thu, 1 Nov 2018 16:47:05 -0600 Subject: [PATCH 75/84] Fix logic error that prevented guests from seeing the privacy policy --- synapse/rest/consent/consent_resource.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/rest/consent/consent_resource.py b/synapse/rest/consent/consent_resource.py index 4cadd71d7e..89b82b0591 100644 --- a/synapse/rest/consent/consent_resource.py +++ b/synapse/rest/consent/consent_resource.py @@ -141,7 +141,7 @@ class ConsentResource(Resource): username = parse_string(request, "u", required=False, default="") userhmac = None has_consented = False - public_version = username != "" + public_version = username == "" if not public_version: userhmac = parse_string(request, "h", required=True, encoding=None) From 552f090f62e2f48b6a71aaf547fae7a33e03e935 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Thu, 1 Nov 2018 16:51:11 -0600 Subject: [PATCH 76/84] Changelog --- changelog.d/4133.feature | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog.d/4133.feature diff --git a/changelog.d/4133.feature b/changelog.d/4133.feature new file mode 100644 index 0000000000..ef5cdaf5ec --- /dev/null +++ b/changelog.d/4133.feature @@ -0,0 +1 @@ +Add `m.login.terms` to the registration flow when consent tracking is enabled. **This makes the template arguments conditionally optional on a new `public_version` variable - update your privacy templates to support this.** From 54aec35867d24bafa870e5b437a11b9a0c502658 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 2 Nov 2018 10:29:19 +0000 Subject: [PATCH 77/84] Fix None exception in state res v2 --- synapse/state/v2.py | 4 ++ tests/state/test_v2.py | 100 ++++++++++++++++++++++++++++++++++++++++- 2 files changed, 102 insertions(+), 2 deletions(-) diff --git a/synapse/state/v2.py b/synapse/state/v2.py index 5d06f7e928..dbc9688c56 100644 --- a/synapse/state/v2.py +++ b/synapse/state/v2.py @@ -53,6 +53,10 @@ def resolve_events_with_store(state_sets, event_map, state_res_store): logger.debug("Computing conflicted state") + # We use event_map as a cache, so if its None we need to initialize it + if event_map is None: + event_map = {} + # First split up the un/conflicted state unconflicted_state, conflicted_state = _seperate(state_sets) diff --git a/tests/state/test_v2.py b/tests/state/test_v2.py index efd85ebe6c..d67f59b2c7 100644 --- a/tests/state/test_v2.py +++ b/tests/state/test_v2.py @@ -544,8 +544,7 @@ class StateTestCase(unittest.TestCase): state_res_store=TestStateResolutionStore(event_map), ) - self.assertTrue(state_d.called) - state_before = state_d.result + state_before = self.successResultOf(state_d) state_after = dict(state_before) if fake_event.state_key is not None: @@ -599,6 +598,103 @@ class LexicographicalTestCase(unittest.TestCase): self.assertEqual(["o", "l", "n", "m", "p"], res) +class SimpleParamStateTestCase(unittest.TestCase): + def setUp(self): + # We build up a simple DAG. + + event_map = {} + + create_event = FakeEvent( + id="CREATE", + sender=ALICE, + type=EventTypes.Create, + state_key="", + content={"creator": ALICE}, + ).to_event([], []) + event_map[create_event.event_id] = create_event + + alice_member = FakeEvent( + id="IMA", + sender=ALICE, + type=EventTypes.Member, + state_key=ALICE, + content=MEMBERSHIP_CONTENT_JOIN, + ).to_event([create_event.event_id], [create_event.event_id]) + event_map[alice_member.event_id] = alice_member + + join_rules = FakeEvent( + id="IJR", + sender=ALICE, + type=EventTypes.JoinRules, + state_key="", + content={"join_rule": JoinRules.PUBLIC}, + ).to_event( + auth_events=[create_event.event_id, alice_member.event_id], + prev_events=[alice_member.event_id], + ) + event_map[join_rules.event_id] = join_rules + + # Bob and Charlie join at the same time, so there is a fork + bob_member = FakeEvent( + id="IMB", + sender=BOB, + type=EventTypes.Member, + state_key=BOB, + content=MEMBERSHIP_CONTENT_JOIN, + ).to_event( + auth_events=[create_event.event_id, join_rules.event_id], + prev_events=[join_rules.event_id], + ) + event_map[bob_member.event_id] = bob_member + + charlie_member = FakeEvent( + id="IMC", + sender=CHARLIE, + type=EventTypes.Member, + state_key=CHARLIE, + content=MEMBERSHIP_CONTENT_JOIN, + ).to_event( + auth_events=[create_event.event_id, join_rules.event_id], + prev_events=[join_rules.event_id], + ) + event_map[charlie_member.event_id] = charlie_member + + self.event_map = event_map + self.create_event = create_event + self.alice_member = alice_member + self.join_rules = join_rules + self.bob_member = bob_member + self.charlie_member = charlie_member + + self.state_at_bob = { + (e.type, e.state_key): e.event_id + for e in [create_event, alice_member, join_rules, bob_member] + } + + self.state_at_charlie = { + (e.type, e.state_key): e.event_id + for e in [create_event, alice_member, join_rules, charlie_member] + } + + self.expected_combined_state = { + (e.type, e.state_key): e.event_id + for e in [create_event, alice_member, join_rules, bob_member, charlie_member] + } + + def test_event_map_none(self): + # Test that we correctly handle passing `None` as the event_map + + state_d = resolve_events_with_store( + [self.state_at_bob, self.state_at_charlie], + event_map=None, + state_res_store=TestStateResolutionStore(self.event_map), + ) + + state = self.successResultOf(state_d) + + self.assert_dict(self.expected_combined_state, state) + + def pairwise(iterable): "s -> (s0,s1), (s1,s2), (s2, s3), ..." a, b = itertools.tee(iterable) From f05d97e28331f83af374b81f0862479d3af34722 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 2 Nov 2018 10:32:06 +0000 Subject: [PATCH 78/84] Newsfile --- changelog.d/4135.bugfix | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog.d/4135.bugfix diff --git a/changelog.d/4135.bugfix b/changelog.d/4135.bugfix new file mode 100644 index 0000000000..6879b1c162 --- /dev/null +++ b/changelog.d/4135.bugfix @@ -0,0 +1 @@ +Fix exception when using state res v2 algorithm From 50e328d1e7a61268dbf274f10798ea5994c6c25a Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Thu, 1 Nov 2018 19:01:29 +0000 Subject: [PATCH 79/84] Remove redundant database locks for device list updates We can rely on the application-level per-user linearizer. --- synapse/storage/devices.py | 45 +++++++++++++++++++++++++++++++++----- 1 file changed, 40 insertions(+), 5 deletions(-) diff --git a/synapse/storage/devices.py b/synapse/storage/devices.py index 62497ab63f..bc3f575b1e 100644 --- a/synapse/storage/devices.py +++ b/synapse/storage/devices.py @@ -239,7 +239,19 @@ class DeviceStore(SQLBaseStore): def update_remote_device_list_cache_entry(self, user_id, device_id, content, stream_id): - """Updates a single user's device in the cache. + """Updates a single device in the cache of a remote user's devicelist. + + Note: assumes that we are the only thread that can be updating this user's + device list. + + Args: + user_id (str): User to update device list for + device_id (str): ID of decivice being updated + content (dict): new data on this device + stream_id (int): the version of the device list + + Returns: + Deferred[None] """ return self.runInteraction( "update_remote_device_list_cache_entry", @@ -272,7 +284,11 @@ class DeviceStore(SQLBaseStore): }, values={ "content": json.dumps(content), - } + }, + + # we don't need to lock, because we assume we are the only thread + # updating this user's devices. + lock=False, ) txn.call_after(self._get_cached_user_device.invalidate, (user_id, device_id,)) @@ -289,11 +305,26 @@ class DeviceStore(SQLBaseStore): }, values={ "stream_id": stream_id, - } + }, + + # again, we can assume we are the only thread updating this user's + # extremity. + lock=False, ) def update_remote_device_list_cache(self, user_id, devices, stream_id): - """Replace the cache of the remote user's devices. + """Replace the entire cache of the remote user's devices. + + Note: assumes that we are the only thread that can be updating this user's + device list. + + Args: + user_id (str): User to update device list for + devices (list[dict]): list of device objects supplied over federation + stream_id (int): the version of the device list + + Returns: + Deferred[None] """ return self.runInteraction( "update_remote_device_list_cache", @@ -338,7 +369,11 @@ class DeviceStore(SQLBaseStore): }, values={ "stream_id": stream_id, - } + }, + + # we don't need to lock, because we can assume we are the only thread + # updating this user's extremity. + lock=False, ) def get_devices_by_remote(self, destination, from_stream_id): From 350f654e7b80ff19d1fdf3861093cef09ccf3ab1 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Thu, 1 Nov 2018 19:10:33 +0000 Subject: [PATCH 80/84] Add unique indexes to a couple of tables The indexes on device_lists_remote_extremeties can be unique, and they therefore should, to ensure that the db remains consistent. --- synapse/storage/devices.py | 49 ++++++++++++++++++- .../schema/delta/40/device_list_streams.sql | 9 ++-- .../52/device_list_streams_unique_idx.sql | 36 ++++++++++++++ 3 files changed, 88 insertions(+), 6 deletions(-) create mode 100644 synapse/storage/schema/delta/52/device_list_streams_unique_idx.sql diff --git a/synapse/storage/devices.py b/synapse/storage/devices.py index bc3f575b1e..ecdab34e7d 100644 --- a/synapse/storage/devices.py +++ b/synapse/storage/devices.py @@ -22,14 +22,19 @@ from twisted.internet import defer from synapse.api.errors import StoreError from synapse.metrics.background_process_metrics import run_as_background_process +from synapse.storage.background_updates import BackgroundUpdateStore from synapse.util.caches.descriptors import cached, cachedInlineCallbacks, cachedList -from ._base import Cache, SQLBaseStore, db_to_json +from ._base import Cache, db_to_json logger = logging.getLogger(__name__) +DROP_DEVICE_LIST_STREAMS_NON_UNIQUE_INDEXES = ( + "drop_device_list_streams_non_unique_indexes" +) -class DeviceStore(SQLBaseStore): + +class DeviceStore(BackgroundUpdateStore): def __init__(self, db_conn, hs): super(DeviceStore, self).__init__(db_conn, hs) @@ -52,6 +57,30 @@ class DeviceStore(SQLBaseStore): columns=["user_id", "device_id"], ) + # create a unique index on device_lists_remote_cache + self.register_background_index_update( + "device_lists_remote_cache_unique_idx", + index_name="device_lists_remote_cache_unique_id", + table="device_lists_remote_cache", + columns=["user_id", "device_id"], + unique=True, + ) + + # And one on device_lists_remote_extremeties + self.register_background_index_update( + "device_lists_remote_extremeties_unique_idx", + index_name="device_lists_remote_extremeties_unique_idx", + table="device_lists_remote_extremeties", + columns=["user_id"], + unique=True, + ) + + # once they complete, we can remove the old non-unique indexes. + self.register_background_update_handler( + DROP_DEVICE_LIST_STREAMS_NON_UNIQUE_INDEXES, + self._drop_device_list_streams_non_unique_indexes, + ) + @defer.inlineCallbacks def store_device(self, user_id, device_id, initial_device_display_name): @@ -757,3 +786,19 @@ class DeviceStore(SQLBaseStore): "_prune_old_outbound_device_pokes", _prune_txn, ) + + @defer.inlineCallbacks + def _drop_device_list_streams_non_unique_indexes(self, progress, batch_size): + def f(conn): + txn = conn.cursor() + txn.execute( + "DROP INDEX IF EXISTS device_lists_remote_cache_id" + ) + txn.execute( + "DROP INDEX IF EXISTS device_lists_remote_extremeties_id" + ) + txn.close() + + yield self.runWithConnection(f) + yield self._end_background_update(DROP_DEVICE_LIST_STREAMS_NON_UNIQUE_INDEXES) + defer.returnValue(1) diff --git a/synapse/storage/schema/delta/40/device_list_streams.sql b/synapse/storage/schema/delta/40/device_list_streams.sql index 54841b3843..dd6dcb65f1 100644 --- a/synapse/storage/schema/delta/40/device_list_streams.sql +++ b/synapse/storage/schema/delta/40/device_list_streams.sql @@ -20,9 +20,6 @@ CREATE TABLE device_lists_remote_cache ( content TEXT NOT NULL ); -CREATE INDEX device_lists_remote_cache_id ON device_lists_remote_cache(user_id, device_id); - - -- The last update we got for a user. Empty if we're not receiving updates for -- that user. CREATE TABLE device_lists_remote_extremeties ( @@ -30,7 +27,11 @@ CREATE TABLE device_lists_remote_extremeties ( stream_id TEXT NOT NULL ); -CREATE INDEX device_lists_remote_extremeties_id ON device_lists_remote_extremeties(user_id, stream_id); +-- we used to create non-unique indexes on these tables, but as of update 52 we create +-- unique indexes concurrently: +-- +-- CREATE INDEX device_lists_remote_cache_id ON device_lists_remote_cache(user_id, device_id); +-- CREATE INDEX device_lists_remote_extremeties_id ON device_lists_remote_extremeties(user_id, stream_id); -- Stream of device lists updates. Includes both local and remotes diff --git a/synapse/storage/schema/delta/52/device_list_streams_unique_idx.sql b/synapse/storage/schema/delta/52/device_list_streams_unique_idx.sql new file mode 100644 index 0000000000..bfa49e6f92 --- /dev/null +++ b/synapse/storage/schema/delta/52/device_list_streams_unique_idx.sql @@ -0,0 +1,36 @@ +/* Copyright 2018 New Vector Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +-- register a background update which will create a unique index on +-- device_lists_remote_cache +INSERT into background_updates (update_name, progress_json) + VALUES ('device_lists_remote_cache_unique_idx', '{}'); + +-- and one on device_lists_remote_extremeties +INSERT into background_updates (update_name, progress_json, depends_on) + VALUES ( + 'device_lists_remote_extremeties_unique_idx', '{}', + + -- doesn't really depend on this, but we need to make sure both happen + -- before we drop the old indexes. + 'device_lists_remote_cache_unique_idx' + ); + +-- once they complete, we can drop the old indexes. +INSERT into background_updates (update_name, progress_json, depends_on) + VALUES ( + 'drop_device_list_streams_non_unique_indexes', '{}', + 'device_lists_remote_extremeties_unique_idx' + ); From 1cc6671ec4157f363d9c476825693b3ad318ffda Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Thu, 1 Nov 2018 19:14:04 +0000 Subject: [PATCH 81/84] changelog --- changelog.d/4132.bugfix | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog.d/4132.bugfix diff --git a/changelog.d/4132.bugfix b/changelog.d/4132.bugfix new file mode 100644 index 0000000000..2304a40f05 --- /dev/null +++ b/changelog.d/4132.bugfix @@ -0,0 +1 @@ +Fix table lock of device_lists_remote_cache which could freeze the application \ No newline at end of file From cb7a6b2379e0e0a4ba8043da98e376b45d05b977 Mon Sep 17 00:00:00 2001 From: Amber Brown Date: Sat, 3 Nov 2018 00:19:23 +1100 Subject: [PATCH 82/84] Fix typing being reset causing infinite syncs (#4127) --- changelog.d/4127.bugfix | 1 + synapse/app/synchrotron.py | 14 +++ synapse/handlers/typing.py | 14 ++- tests/rest/client/v2_alpha/test_sync.py | 123 ++++++++++++++++++++++++ tests/server.py | 8 +- 5 files changed, 155 insertions(+), 5 deletions(-) create mode 100644 changelog.d/4127.bugfix diff --git a/changelog.d/4127.bugfix b/changelog.d/4127.bugfix new file mode 100644 index 0000000000..0701d2ceaa --- /dev/null +++ b/changelog.d/4127.bugfix @@ -0,0 +1 @@ +If the typing stream ID goes backwards (as on a worker when the master restarts), the worker's typing handler will no longer erroneously report rooms containing new typing events. diff --git a/synapse/app/synchrotron.py b/synapse/app/synchrotron.py index 3926c7f263..0354e82bf8 100644 --- a/synapse/app/synchrotron.py +++ b/synapse/app/synchrotron.py @@ -226,7 +226,15 @@ class SynchrotronPresence(object): class SynchrotronTyping(object): def __init__(self, hs): self._latest_room_serial = 0 + self._reset() + + def _reset(self): + """ + Reset the typing handler's data caches. + """ + # map room IDs to serial numbers self._room_serials = {} + # map room IDs to sets of users currently typing self._room_typing = {} def stream_positions(self): @@ -236,6 +244,12 @@ class SynchrotronTyping(object): return {"typing": self._latest_room_serial} def process_replication_rows(self, token, rows): + if self._latest_room_serial > token: + # The master has gone backwards. To prevent inconsistent data, just + # clear everything. + self._reset() + + # Set the latest serial token to whatever the server gave us. self._latest_room_serial = token for row in rows: diff --git a/synapse/handlers/typing.py b/synapse/handlers/typing.py index c610933dd4..a61bbf9392 100644 --- a/synapse/handlers/typing.py +++ b/synapse/handlers/typing.py @@ -63,11 +63,8 @@ class TypingHandler(object): self._member_typing_until = {} # clock time we expect to stop self._member_last_federation_poke = {} - # map room IDs to serial numbers - self._room_serials = {} self._latest_room_serial = 0 - # map room IDs to sets of users currently typing - self._room_typing = {} + self._reset() # caches which room_ids changed at which serials self._typing_stream_change_cache = StreamChangeCache( @@ -79,6 +76,15 @@ class TypingHandler(object): 5000, ) + def _reset(self): + """ + Reset the typing handler's data caches. + """ + # map room IDs to serial numbers + self._room_serials = {} + # map room IDs to sets of users currently typing + self._room_typing = {} + def _handle_timeouts(self): logger.info("Checking for typing timeouts") diff --git a/tests/rest/client/v2_alpha/test_sync.py b/tests/rest/client/v2_alpha/test_sync.py index 4c30c5f258..99b716f00a 100644 --- a/tests/rest/client/v2_alpha/test_sync.py +++ b/tests/rest/client/v2_alpha/test_sync.py @@ -15,9 +15,11 @@ from mock import Mock +from synapse.rest.client.v1 import admin, login, room from synapse.rest.client.v2_alpha import sync from tests import unittest +from tests.server import TimedOutException class FilterTestCase(unittest.HomeserverTestCase): @@ -65,3 +67,124 @@ class FilterTestCase(unittest.HomeserverTestCase): ["next_batch", "rooms", "account_data", "to_device", "device_lists"] ).issubset(set(channel.json_body.keys())) ) + + +class SyncTypingTests(unittest.HomeserverTestCase): + + servlets = [ + admin.register_servlets, + room.register_servlets, + login.register_servlets, + sync.register_servlets, + ] + user_id = True + hijack_auth = False + + def test_sync_backwards_typing(self): + """ + If the typing serial goes backwards and the typing handler is then reset + (such as when the master restarts and sets the typing serial to 0), we + do not incorrectly return typing information that had a serial greater + than the now-reset serial. + """ + typing_url = "/rooms/%s/typing/%s?access_token=%s" + sync_url = "/sync?timeout=3000000&access_token=%s&since=%s" + + # Register the user who gets notified + user_id = self.register_user("user", "pass") + access_token = self.login("user", "pass") + + # Register the user who sends the message + other_user_id = self.register_user("otheruser", "pass") + other_access_token = self.login("otheruser", "pass") + + # Create a room + room = self.helper.create_room_as(user_id, tok=access_token) + + # Invite the other person + self.helper.invite(room=room, src=user_id, tok=access_token, targ=other_user_id) + + # The other user joins + self.helper.join(room=room, user=other_user_id, tok=other_access_token) + + # The other user sends some messages + self.helper.send(room, body="Hi!", tok=other_access_token) + self.helper.send(room, body="There!", tok=other_access_token) + + # Start typing. + request, channel = self.make_request( + "PUT", + typing_url % (room, other_user_id, other_access_token), + b'{"typing": true, "timeout": 30000}', + ) + self.render(request) + self.assertEquals(200, channel.code) + + request, channel = self.make_request( + "GET", "/sync?access_token=%s" % (access_token,) + ) + self.render(request) + self.assertEquals(200, channel.code) + next_batch = channel.json_body["next_batch"] + + # Stop typing. + request, channel = self.make_request( + "PUT", + typing_url % (room, other_user_id, other_access_token), + b'{"typing": false}', + ) + self.render(request) + self.assertEquals(200, channel.code) + + # Start typing. + request, channel = self.make_request( + "PUT", + typing_url % (room, other_user_id, other_access_token), + b'{"typing": true, "timeout": 30000}', + ) + self.render(request) + self.assertEquals(200, channel.code) + + # Should return immediately + request, channel = self.make_request( + "GET", sync_url % (access_token, next_batch) + ) + self.render(request) + self.assertEquals(200, channel.code) + next_batch = channel.json_body["next_batch"] + + # Reset typing serial back to 0, as if the master had. + typing = self.hs.get_typing_handler() + typing._latest_room_serial = 0 + + # Since it checks the state token, we need some state to update to + # invalidate the stream token. + self.helper.send(room, body="There!", tok=other_access_token) + + request, channel = self.make_request( + "GET", sync_url % (access_token, next_batch) + ) + self.render(request) + self.assertEquals(200, channel.code) + next_batch = channel.json_body["next_batch"] + + # This should time out! But it does not, because our stream token is + # ahead, and therefore it's saying the typing (that we've actually + # already seen) is new, since it's got a token above our new, now-reset + # stream token. + request, channel = self.make_request( + "GET", sync_url % (access_token, next_batch) + ) + self.render(request) + self.assertEquals(200, channel.code) + next_batch = channel.json_body["next_batch"] + + # Clear the typing information, so that it doesn't think everything is + # in the future. + typing._reset() + + # Now it SHOULD fail as it never completes! + request, channel = self.make_request( + "GET", sync_url % (access_token, next_batch) + ) + self.assertRaises(TimedOutException, self.render, request) diff --git a/tests/server.py b/tests/server.py index 819c854448..cc6dbe04ac 100644 --- a/tests/server.py +++ b/tests/server.py @@ -21,6 +21,12 @@ from synapse.util import Clock from tests.utils import setup_test_homeserver as _sth +class TimedOutException(Exception): + """ + A web query timed out. + """ + + @attr.s class FakeChannel(object): """ @@ -153,7 +159,7 @@ def wait_until_result(clock, request, timeout=100): x += 1 if x > timeout: - raise Exception("Timed out waiting for request to finish.") + raise TimedOutException("Timed out waiting for request to finish.") clock.advance(0.1) From b86d05a279051b59d711c5a1982474994f302803 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 2 Nov 2018 13:44:12 +0000 Subject: [PATCH 83/84] Clean up event accesses and tests This is in preparation to refactor FrozenEvent to support different event formats for different room versions --- synapse/federation/units.py | 3 --- synapse/push/httppusher.py | 4 ++-- synapse/push/push_rule_evaluator.py | 4 ++-- tests/replication/slave/storage/test_events.py | 4 ++-- tests/test_federation.py | 2 +- 5 files changed, 7 insertions(+), 10 deletions(-) diff --git a/synapse/federation/units.py b/synapse/federation/units.py index c5ab14314e..025a79c022 100644 --- a/synapse/federation/units.py +++ b/synapse/federation/units.py @@ -117,9 +117,6 @@ class Transaction(JsonEncodedObject): "Require 'transaction_id' to construct a Transaction" ) - for p in pdus: - p.transaction_id = kwargs["transaction_id"] - kwargs["pdus"] = [p.get_pdu_json() for p in pdus] return Transaction(**kwargs) diff --git a/synapse/push/httppusher.py b/synapse/push/httppusher.py index 6bd703632d..87fa7f006a 100644 --- a/synapse/push/httppusher.py +++ b/synapse/push/httppusher.py @@ -311,10 +311,10 @@ class HttpPusher(object): ] } } - if event.type == 'm.room.member': + if event.type == 'm.room.member' and event.is_state(): d['notification']['membership'] = event.content['membership'] d['notification']['user_is_target'] = event.state_key == self.user_id - if self.hs.config.push_include_content and 'content' in event: + if self.hs.config.push_include_content and event.content: d['notification']['content'] = event.content # We no longer send aliases separately, instead, we send the human diff --git a/synapse/push/push_rule_evaluator.py b/synapse/push/push_rule_evaluator.py index 2bd321d530..cf6c8b875e 100644 --- a/synapse/push/push_rule_evaluator.py +++ b/synapse/push/push_rule_evaluator.py @@ -124,7 +124,7 @@ class PushRuleEvaluatorForEvent(object): # XXX: optimisation: cache our pattern regexps if condition['key'] == 'content.body': - body = self._event["content"].get("body", None) + body = self._event.content.get("body", None) if not body: return False @@ -140,7 +140,7 @@ class PushRuleEvaluatorForEvent(object): if not display_name: return False - body = self._event["content"].get("body", None) + body = self._event.content.get("body", None) if not body: return False diff --git a/tests/replication/slave/storage/test_events.py b/tests/replication/slave/storage/test_events.py index 41be5d5a1a..1688a741d1 100644 --- a/tests/replication/slave/storage/test_events.py +++ b/tests/replication/slave/storage/test_events.py @@ -28,8 +28,8 @@ ROOM_ID = "!room:blue" def dict_equals(self, other): - me = encode_canonical_json(self._event_dict) - them = encode_canonical_json(other._event_dict) + me = encode_canonical_json(self.get_pdu_json()) + them = encode_canonical_json(other.get_pdu_json()) return me == them diff --git a/tests/test_federation.py b/tests/test_federation.py index 952a0a7b51..e1a34ccffd 100644 --- a/tests/test_federation.py +++ b/tests/test_federation.py @@ -112,7 +112,7 @@ class MessageAcceptTests(unittest.TestCase): "origin_server_ts": 1, "type": "m.room.message", "origin": "test.serv", - "content": "hewwo?", + "content": {"body": "hewwo?"}, "auth_events": [], "prev_events": [("two:test.serv", {}), (most_recent, {})], } From 76cd7de108eacda49ab95308a3e89f65c8392d57 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 2 Nov 2018 13:45:56 +0000 Subject: [PATCH 84/84] Newsfile --- changelog.d/4137.misc | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog.d/4137.misc diff --git a/changelog.d/4137.misc b/changelog.d/4137.misc new file mode 100644 index 0000000000..4fe933e33c --- /dev/null +++ b/changelog.d/4137.misc @@ -0,0 +1 @@ +Clean up event accesses and tests