From 563f9b61b1439e7a6a8a250fd068659092583dbf Mon Sep 17 00:00:00 2001 From: David Baker Date: Mon, 29 Oct 2018 21:01:22 +0000 Subject: [PATCH 01/32] Make e2e backup versions numeric in the DB We were doing max(version) which does not do what we wanted on a column of type TEXT. --- synapse/handlers/e2e_room_keys.py | 2 +- synapse/storage/prepare_database.py | 2 +- .../storage/schema/delta/52/e2e_room_keys.sql | 53 +++++++++++++++++++ 3 files changed, 55 insertions(+), 2 deletions(-) create mode 100644 synapse/storage/schema/delta/52/e2e_room_keys.sql diff --git a/synapse/handlers/e2e_room_keys.py b/synapse/handlers/e2e_room_keys.py index 5edb3cfe04..b6d302cd6b 100644 --- a/synapse/handlers/e2e_room_keys.py +++ b/synapse/handlers/e2e_room_keys.py @@ -138,7 +138,7 @@ class E2eRoomKeysHandler(object): else: raise - if version_info['version'] != version: + if str(version_info['version']) != version: # Check that the version we're trying to upload actually exists try: version_info = yield self.store.get_e2e_room_keys_version_info( diff --git a/synapse/storage/prepare_database.py b/synapse/storage/prepare_database.py index b364719312..bd740e1e45 100644 --- a/synapse/storage/prepare_database.py +++ b/synapse/storage/prepare_database.py @@ -25,7 +25,7 @@ logger = logging.getLogger(__name__) # Remember to update this number every time a change is made to database # schema files, so the users will be informed on server restarts. -SCHEMA_VERSION = 51 +SCHEMA_VERSION = 52 dir_path = os.path.abspath(os.path.dirname(__file__)) diff --git a/synapse/storage/schema/delta/52/e2e_room_keys.sql b/synapse/storage/schema/delta/52/e2e_room_keys.sql new file mode 100644 index 0000000000..754985187e --- /dev/null +++ b/synapse/storage/schema/delta/52/e2e_room_keys.sql @@ -0,0 +1,53 @@ +/* Copyright 2018 New Vector Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* Change version column to an integer so we can do MAX() sensibly + */ +CREATE TABLE e2e_room_keys_versions_new ( + user_id TEXT NOT NULL, + version BIGINT NOT NULL, + algorithm TEXT NOT NULL, + auth_data TEXT NOT NULL, + deleted SMALLINT DEFAULT 0 NOT NULL +); + +INSERT INTO e2e_room_keys_versions_new + SELECT user_id, version, algorithm, auth_data, deleted FROM e2e_room_keys_versions; + +DROP TABLE e2e_room_keys_versions; +ALTER TABLE e2e_room_keys_versions_new RENAME TO e2e_room_keys_versions; + +CREATE UNIQUE INDEX e2e_room_keys_versions_idx ON e2e_room_keys_versions(user_id, version); + +/* Change e2e_rooms_keys to match + */ +CREATE TABLE e2e_room_keys_new ( + user_id TEXT NOT NULL, + room_id TEXT NOT NULL, + session_id TEXT NOT NULL, + version BIGINT NOT NULL, + first_message_index INT, + forwarded_count INT, + is_verified BOOLEAN, + session_data TEXT NOT NULL +); + +INSERT INTO e2e_room_keys_new + SELECT user_id, room_id, session_id, version, first_message_index, forwarded_count, is_verified, session_data FROM e2e_room_keys; + +DROP TABLE e2e_room_keys; +ALTER TABLE e2e_room_keys_new RENAME TO e2e_room_keys; + +CREATE UNIQUE INDEX e2e_room_keys_idx ON e2e_room_keys(user_id, room_id, session_id); From 64fa557f80edc99318aa6152c3b84c76dd455c8a Mon Sep 17 00:00:00 2001 From: David Baker Date: Tue, 30 Oct 2018 09:51:04 +0000 Subject: [PATCH 02/32] Try & make it work on postgres --- synapse/storage/schema/delta/52/e2e_room_keys.sql | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/synapse/storage/schema/delta/52/e2e_room_keys.sql b/synapse/storage/schema/delta/52/e2e_room_keys.sql index 754985187e..db687cccae 100644 --- a/synapse/storage/schema/delta/52/e2e_room_keys.sql +++ b/synapse/storage/schema/delta/52/e2e_room_keys.sql @@ -24,7 +24,7 @@ CREATE TABLE e2e_room_keys_versions_new ( ); INSERT INTO e2e_room_keys_versions_new - SELECT user_id, version, algorithm, auth_data, deleted FROM e2e_room_keys_versions; + SELECT user_id, CAST(version as BIGINT), algorithm, auth_data, deleted FROM e2e_room_keys_versions; DROP TABLE e2e_room_keys_versions; ALTER TABLE e2e_room_keys_versions_new RENAME TO e2e_room_keys_versions; @@ -45,7 +45,7 @@ CREATE TABLE e2e_room_keys_new ( ); INSERT INTO e2e_room_keys_new - SELECT user_id, room_id, session_id, version, first_message_index, forwarded_count, is_verified, session_data FROM e2e_room_keys; + SELECT user_id, room_id, session_id, CAST(version as BIGINT), first_message_index, forwarded_count, is_verified, session_data FROM e2e_room_keys; DROP TABLE e2e_room_keys; ALTER TABLE e2e_room_keys_new RENAME TO e2e_room_keys; From 4eacf0f2008573ae96c2c693427f3098730691b7 Mon Sep 17 00:00:00 2001 From: David Baker Date: Tue, 30 Oct 2018 10:05:51 +0000 Subject: [PATCH 03/32] news fragment --- changelog.d/4113.bugfix | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog.d/4113.bugfix diff --git a/changelog.d/4113.bugfix b/changelog.d/4113.bugfix new file mode 100644 index 0000000000..7f9d8ee032 --- /dev/null +++ b/changelog.d/4113.bugfix @@ -0,0 +1 @@ +Fix e2e key backup with more than 9 backup versions From 2f0f911c52f6de48e47fbb5624a3e3beee0dd6a4 Mon Sep 17 00:00:00 2001 From: David Baker Date: Tue, 30 Oct 2018 10:35:18 +0000 Subject: [PATCH 04/32] Convert version back to a string --- synapse/storage/e2e_room_keys.py | 1 + 1 file changed, 1 insertion(+) diff --git a/synapse/storage/e2e_room_keys.py b/synapse/storage/e2e_room_keys.py index f25ded2295..9f826b292c 100644 --- a/synapse/storage/e2e_room_keys.py +++ b/synapse/storage/e2e_room_keys.py @@ -236,6 +236,7 @@ class EndToEndRoomKeyStore(SQLBaseStore): ), ) result["auth_data"] = json.loads(result["auth_data"]) + result["version"] = str(result["version"]) return result return self.runInteraction( From 12941f5f8b1f38f273e301104203149b10e9e214 Mon Sep 17 00:00:00 2001 From: David Baker Date: Tue, 30 Oct 2018 11:01:07 +0000 Subject: [PATCH 05/32] Cast bacjup version to int when querying --- synapse/storage/e2e_room_keys.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/synapse/storage/e2e_room_keys.py b/synapse/storage/e2e_room_keys.py index 9f826b292c..2a012e9487 100644 --- a/synapse/storage/e2e_room_keys.py +++ b/synapse/storage/e2e_room_keys.py @@ -219,7 +219,12 @@ class EndToEndRoomKeyStore(SQLBaseStore): if version is None: this_version = self._get_current_version(txn, user_id) else: - this_version = version + try: + this_version = int(version) + except ValueError: + # Our versions are all ints so if we can't convert it to an integer, + # it isn't there. + raise StoreError(404, "No row found") result = self._simple_select_one_txn( txn, From e0934acdbbd497ee1330efe42feccb11382639ec Mon Sep 17 00:00:00 2001 From: David Baker Date: Tue, 30 Oct 2018 11:12:23 +0000 Subject: [PATCH 06/32] Cast to int here too --- synapse/storage/e2e_room_keys.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/synapse/storage/e2e_room_keys.py b/synapse/storage/e2e_room_keys.py index 2a012e9487..a1d203fd61 100644 --- a/synapse/storage/e2e_room_keys.py +++ b/synapse/storage/e2e_room_keys.py @@ -118,6 +118,11 @@ class EndToEndRoomKeyStore(SQLBaseStore): these room keys. """ + try: + version = int(version) + except ValueError: + defer.returnValue({'rooms': {}}) + keyvalues = { "user_id": user_id, "version": version, From f1087106cf637e3c108c096ff789100bcbcc461c Mon Sep 17 00:00:00 2001 From: Hubert Chathi Date: Mon, 5 Nov 2018 17:59:29 -0500 Subject: [PATCH 07/32] handle empty backups according to latest spec proposal (#4123) fixes #4056 --- changelog.d/4123.bugfix | 1 + synapse/handlers/e2e_room_keys.py | 22 +++++-- synapse/rest/client/v2_alpha/room_keys.py | 21 +++++- tests/handlers/test_e2e_room_keys.py | 79 +++++++++++------------ 4 files changed, 71 insertions(+), 52 deletions(-) create mode 100644 changelog.d/4123.bugfix diff --git a/changelog.d/4123.bugfix b/changelog.d/4123.bugfix new file mode 100644 index 0000000000..b82bc2aad3 --- /dev/null +++ b/changelog.d/4123.bugfix @@ -0,0 +1 @@ +fix return code of empty key backups diff --git a/synapse/handlers/e2e_room_keys.py b/synapse/handlers/e2e_room_keys.py index 5edb3cfe04..42b040375f 100644 --- a/synapse/handlers/e2e_room_keys.py +++ b/synapse/handlers/e2e_room_keys.py @@ -19,7 +19,7 @@ from six import iteritems from twisted.internet import defer -from synapse.api.errors import RoomKeysVersionError, StoreError, SynapseError +from synapse.api.errors import NotFoundError, RoomKeysVersionError, StoreError from synapse.util.async_helpers import Linearizer logger = logging.getLogger(__name__) @@ -55,6 +55,8 @@ class E2eRoomKeysHandler(object): room_id(string): room ID to get keys for, for None to get keys for all rooms session_id(string): session ID to get keys for, for None to get keys for all sessions + Raises: + NotFoundError: if the backup version does not exist Returns: A deferred list of dicts giving the session_data and message metadata for these room keys. @@ -63,13 +65,19 @@ class E2eRoomKeysHandler(object): # we deliberately take the lock to get keys so that changing the version # works atomically with (yield self._upload_linearizer.queue(user_id)): + # make sure the backup version exists + try: + yield self.store.get_e2e_room_keys_version_info(user_id, version) + except StoreError as e: + if e.code == 404: + raise NotFoundError("Unknown backup version") + else: + raise + results = yield self.store.get_e2e_room_keys( user_id, version, room_id, session_id ) - if results['rooms'] == {}: - raise SynapseError(404, "No room_keys found") - defer.returnValue(results) @defer.inlineCallbacks @@ -120,7 +128,7 @@ class E2eRoomKeysHandler(object): } Raises: - SynapseError: with code 404 if there are no versions defined + NotFoundError: if there are no versions defined RoomKeysVersionError: if the uploaded version is not the current version """ @@ -134,7 +142,7 @@ class E2eRoomKeysHandler(object): version_info = yield self.store.get_e2e_room_keys_version_info(user_id) except StoreError as e: if e.code == 404: - raise SynapseError(404, "Version '%s' not found" % (version,)) + raise NotFoundError("Version '%s' not found" % (version,)) else: raise @@ -148,7 +156,7 @@ class E2eRoomKeysHandler(object): raise RoomKeysVersionError(current_version=version_info['version']) except StoreError as e: if e.code == 404: - raise SynapseError(404, "Version '%s' not found" % (version,)) + raise NotFoundError("Version '%s' not found" % (version,)) else: raise diff --git a/synapse/rest/client/v2_alpha/room_keys.py b/synapse/rest/client/v2_alpha/room_keys.py index 45b5817d8b..ab3f1bd21a 100644 --- a/synapse/rest/client/v2_alpha/room_keys.py +++ b/synapse/rest/client/v2_alpha/room_keys.py @@ -17,7 +17,7 @@ import logging from twisted.internet import defer -from synapse.api.errors import Codes, SynapseError +from synapse.api.errors import Codes, NotFoundError, SynapseError from synapse.http.servlet import ( RestServlet, parse_json_object_from_request, @@ -208,10 +208,25 @@ class RoomKeysServlet(RestServlet): user_id, version, room_id, session_id ) + # Convert room_keys to the right format to return. if session_id: - room_keys = room_keys['rooms'][room_id]['sessions'][session_id] + # If the client requests a specific session, but that session was + # not backed up, then return an M_NOT_FOUND. + if room_keys['rooms'] == {}: + raise NotFoundError("No room_keys found") + else: + room_keys = room_keys['rooms'][room_id]['sessions'][session_id] elif room_id: - room_keys = room_keys['rooms'][room_id] + # If the client requests all sessions from a room, but no sessions + # are found, then return an empty result rather than an error, so + # that clients don't have to handle an error condition, and an + # empty result is valid. (Similarly if the client requests all + # sessions from the backup, but in that case, room_keys is already + # in the right format, so we don't need to do anything about it.) + if room_keys['rooms'] == {}: + room_keys = {'sessions': {}} + else: + room_keys = room_keys['rooms'][room_id] defer.returnValue((200, room_keys)) diff --git a/tests/handlers/test_e2e_room_keys.py b/tests/handlers/test_e2e_room_keys.py index 9e08eac0a5..c8994f416e 100644 --- a/tests/handlers/test_e2e_room_keys.py +++ b/tests/handlers/test_e2e_room_keys.py @@ -169,8 +169,8 @@ class E2eRoomKeysHandlerTestCase(unittest.TestCase): self.assertEqual(res, 404) @defer.inlineCallbacks - def test_get_missing_room_keys(self): - """Check that we get a 404 on querying missing room_keys + def test_get_missing_backup(self): + """Check that we get a 404 on querying missing backup """ res = None try: @@ -179,19 +179,20 @@ class E2eRoomKeysHandlerTestCase(unittest.TestCase): res = e.code self.assertEqual(res, 404) - # check we also get a 404 even if the version is valid + @defer.inlineCallbacks + def test_get_missing_room_keys(self): + """Check we get an empty response from an empty backup + """ version = yield self.handler.create_version(self.local_user, { "algorithm": "m.megolm_backup.v1", "auth_data": "first_version_auth_data", }) self.assertEqual(version, "1") - res = None - try: - yield self.handler.get_room_keys(self.local_user, version) - except errors.SynapseError as e: - res = e.code - self.assertEqual(res, 404) + res = yield self.handler.get_room_keys(self.local_user, version) + self.assertDictEqual(res, { + "rooms": {} + }) # TODO: test the locking semantics when uploading room_keys, # although this is probably best done in sytest @@ -345,17 +346,15 @@ class E2eRoomKeysHandlerTestCase(unittest.TestCase): # check for bulk-delete yield self.handler.upload_room_keys(self.local_user, version, room_keys) yield self.handler.delete_room_keys(self.local_user, version) - res = None - try: - yield self.handler.get_room_keys( - self.local_user, - version, - room_id="!abc:matrix.org", - session_id="c0ff33", - ) - except errors.SynapseError as e: - res = e.code - self.assertEqual(res, 404) + res = yield self.handler.get_room_keys( + self.local_user, + version, + room_id="!abc:matrix.org", + session_id="c0ff33", + ) + self.assertDictEqual(res, { + "rooms": {} + }) # check for bulk-delete per room yield self.handler.upload_room_keys(self.local_user, version, room_keys) @@ -364,17 +363,15 @@ class E2eRoomKeysHandlerTestCase(unittest.TestCase): version, room_id="!abc:matrix.org", ) - res = None - try: - yield self.handler.get_room_keys( - self.local_user, - version, - room_id="!abc:matrix.org", - session_id="c0ff33", - ) - except errors.SynapseError as e: - res = e.code - self.assertEqual(res, 404) + res = yield self.handler.get_room_keys( + self.local_user, + version, + room_id="!abc:matrix.org", + session_id="c0ff33", + ) + self.assertDictEqual(res, { + "rooms": {} + }) # check for bulk-delete per session yield self.handler.upload_room_keys(self.local_user, version, room_keys) @@ -384,14 +381,12 @@ class E2eRoomKeysHandlerTestCase(unittest.TestCase): room_id="!abc:matrix.org", session_id="c0ff33", ) - res = None - try: - yield self.handler.get_room_keys( - self.local_user, - version, - room_id="!abc:matrix.org", - session_id="c0ff33", - ) - except errors.SynapseError as e: - res = e.code - self.assertEqual(res, 404) + res = yield self.handler.get_room_keys( + self.local_user, + version, + room_id="!abc:matrix.org", + session_id="c0ff33", + ) + self.assertDictEqual(res, { + "rooms": {} + }) From 0f5e51f726756318f355d988856730a9930e2d2f Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Tue, 6 Nov 2018 03:32:34 -0700 Subject: [PATCH 08/32] Add config variables for enabling terms auth and the policy name (#4142) So people can still collect consent the old way if they want to. --- changelog.d/4004.feature | 2 +- changelog.d/4133.feature | 2 +- changelog.d/4142.feature | 1 + docs/consent_tracking.md | 40 +++++++++++++++++++++--- synapse/config/consent_config.py | 18 +++++++++++ synapse/handlers/auth.py | 2 +- synapse/rest/client/v2_alpha/register.py | 2 +- synapse/rest/consent/consent_resource.py | 2 +- tests/test_terms_auth.py | 5 +-- tests/utils.py | 2 ++ 10 files changed, 65 insertions(+), 11 deletions(-) create mode 100644 changelog.d/4142.feature diff --git a/changelog.d/4004.feature b/changelog.d/4004.feature index ef5cdaf5ec..89975f4c6e 100644 --- a/changelog.d/4004.feature +++ b/changelog.d/4004.feature @@ -1 +1 @@ -Add `m.login.terms` to the registration flow when consent tracking is enabled. **This makes the template arguments conditionally optional on a new `public_version` variable - update your privacy templates to support this.** +Include flags to optionally add `m.login.terms` to the registration flow when consent tracking is enabled. diff --git a/changelog.d/4133.feature b/changelog.d/4133.feature index ef5cdaf5ec..89975f4c6e 100644 --- a/changelog.d/4133.feature +++ b/changelog.d/4133.feature @@ -1 +1 @@ -Add `m.login.terms` to the registration flow when consent tracking is enabled. **This makes the template arguments conditionally optional on a new `public_version` variable - update your privacy templates to support this.** +Include flags to optionally add `m.login.terms` to the registration flow when consent tracking is enabled. diff --git a/changelog.d/4142.feature b/changelog.d/4142.feature new file mode 100644 index 0000000000..89975f4c6e --- /dev/null +++ b/changelog.d/4142.feature @@ -0,0 +1 @@ +Include flags to optionally add `m.login.terms` to the registration flow when consent tracking is enabled. diff --git a/docs/consent_tracking.md b/docs/consent_tracking.md index 3634d13d4f..c586b5f0b6 100644 --- a/docs/consent_tracking.md +++ b/docs/consent_tracking.md @@ -81,9 +81,40 @@ should be a matter of `pip install Jinja2`. On debian, try `apt-get install python-jinja2`. Once this is complete, and the server has been restarted, try visiting -`https:///_matrix/consent`. If correctly configured, you should see a -default policy document. It is now possible to manually construct URIs where -users can give their consent. +`https:///_matrix/consent`. If correctly configured, this should give +an error "Missing string query parameter 'u'". It is now possible to manually +construct URIs where users can give their consent. + +### Enabling consent tracking at registration + +1. Add the following to your configuration: + + ```yaml + user_consent: + require_at_registration: true + policy_name: "Privacy Policy" # or whatever you'd like to call the policy + ``` + +2. In your consent templates, make use of the `public_version` variable to + see if an unauthenticated user is viewing the page. This is typically + wrapped around the form that would be used to actually agree to the document: + + ``` + {% if not public_version %} + +
+ + + + +
+ {% endif %} + ``` + +3. Restart Synapse to apply the changes. + +Visiting `https:///_matrix/consent` should now give you a view of the privacy +document. This is what users will be able to see when registering for accounts. ### Constructing the consent URI @@ -108,7 +139,8 @@ query parameters: Note that not providing a `u` parameter will be interpreted as wanting to view the document from an unauthenticated perspective, such as prior to registration. -Therefore, the `h` parameter is not required in this scenario. +Therefore, the `h` parameter is not required in this scenario. To enable this +behaviour, set `require_at_registration` to `true` in your `user_consent` config. Sending users a server notice asking them to agree to the policy diff --git a/synapse/config/consent_config.py b/synapse/config/consent_config.py index e22c731aad..f193a090ae 100644 --- a/synapse/config/consent_config.py +++ b/synapse/config/consent_config.py @@ -42,6 +42,14 @@ DEFAULT_CONFIG = """\ # until the user consents to the privacy policy. The value of the setting is # used as the text of the error. # +# 'require_at_registration', if enabled, will add a step to the registration +# process, similar to how captcha works. Users will be required to accept the +# policy before their account is created. +# +# 'policy_name' is the display name of the policy users will see when registering +# for an account. Has no effect unless `require_at_registration` is enabled. +# Defaults to "Privacy Policy". +# # user_consent: # template_dir: res/templates/privacy # version: 1.0 @@ -54,6 +62,8 @@ DEFAULT_CONFIG = """\ # block_events_error: >- # To continue using this homeserver you must review and agree to the # terms and conditions at %(consent_uri)s +# require_at_registration: False +# policy_name: Privacy Policy # """ @@ -67,6 +77,8 @@ class ConsentConfig(Config): self.user_consent_server_notice_content = None self.user_consent_server_notice_to_guests = False self.block_events_without_consent_error = None + self.user_consent_at_registration = False + self.user_consent_policy_name = "Privacy Policy" def read_config(self, config): consent_config = config.get("user_consent") @@ -83,6 +95,12 @@ class ConsentConfig(Config): self.user_consent_server_notice_to_guests = bool(consent_config.get( "send_server_notice_to_guests", False, )) + self.user_consent_at_registration = bool(consent_config.get( + "require_at_registration", False, + )) + self.user_consent_policy_name = consent_config.get( + "policy_name", "Privacy Policy", + ) def default_config(self, **kwargs): return DEFAULT_CONFIG diff --git a/synapse/handlers/auth.py b/synapse/handlers/auth.py index 85fc1fc525..a958c45271 100644 --- a/synapse/handlers/auth.py +++ b/synapse/handlers/auth.py @@ -472,7 +472,7 @@ class AuthHandler(BaseHandler): "privacy_policy": { "version": self.hs.config.user_consent_version, "en": { - "name": "Privacy Policy", + "name": self.hs.config.user_consent_policy_name, "url": "%s/_matrix/consent?v=%s" % ( self.hs.config.public_baseurl, self.hs.config.user_consent_version, diff --git a/synapse/rest/client/v2_alpha/register.py b/synapse/rest/client/v2_alpha/register.py index c5214330ad..0515715f7c 100644 --- a/synapse/rest/client/v2_alpha/register.py +++ b/synapse/rest/client/v2_alpha/register.py @@ -360,7 +360,7 @@ class RegisterRestServlet(RestServlet): ]) # Append m.login.terms to all flows if we're requiring consent - if self.hs.config.block_events_without_consent_error is not None: + if self.hs.config.user_consent_at_registration: new_flows = [] for flow in flows: flow.append(LoginType.TERMS) diff --git a/synapse/rest/consent/consent_resource.py b/synapse/rest/consent/consent_resource.py index c85e84b465..e0f7de5d5c 100644 --- a/synapse/rest/consent/consent_resource.py +++ b/synapse/rest/consent/consent_resource.py @@ -142,7 +142,7 @@ class ConsentResource(Resource): userhmac = None has_consented = False public_version = username == "" - if not public_version: + if not public_version or not self.hs.config.user_consent_at_registration: userhmac = parse_string(request, "h", required=True, encoding=None) self._check_hash(username, userhmac) diff --git a/tests/test_terms_auth.py b/tests/test_terms_auth.py index 7deab5266f..0b71c6feb9 100644 --- a/tests/test_terms_auth.py +++ b/tests/test_terms_auth.py @@ -42,7 +42,8 @@ class TermsTestCase(unittest.HomeserverTestCase): hs.config.enable_registration_captcha = False def test_ui_auth(self): - self.hs.config.block_events_without_consent_error = True + self.hs.config.user_consent_at_registration = True + self.hs.config.user_consent_policy_name = "My Cool Privacy Policy" self.hs.config.public_baseurl = "https://example.org" self.hs.config.user_consent_version = "1.0" @@ -66,7 +67,7 @@ class TermsTestCase(unittest.HomeserverTestCase): "policies": { "privacy_policy": { "en": { - "name": "Privacy Policy", + "name": "My Cool Privacy Policy", "url": "https://example.org/_matrix/consent?v=1.0", }, "version": "1.0" diff --git a/tests/utils.py b/tests/utils.py index 565bb60d08..67ab916f30 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -123,6 +123,8 @@ def default_config(name): config.user_directory_search_all_users = False config.user_consent_server_notice_content = None config.block_events_without_consent_error = None + config.user_consent_at_registration = False + config.user_consent_policy_name = "Privacy Policy" config.media_storage_providers = [] config.autocreate_auto_join_rooms = True config.auto_join_rooms = [] From e62f7f17b32fa51a818fdcfc756464692ddb9194 Mon Sep 17 00:00:00 2001 From: Amber Brown Date: Wed, 7 Nov 2018 03:00:00 +1100 Subject: [PATCH 09/32] Remove some boilerplate in tests (#4156) --- changelog.d/4156.misc | 1 + tests/rest/client/v1/test_admin.py | 116 +++++++++----------- tests/rest/client/v1/test_register.py | 10 +- tests/rest/client/v1/utils.py | 22 ++-- tests/rest/client/v2_alpha/test_filter.py | 95 +++++----------- tests/rest/client/v2_alpha/test_register.py | 52 ++++----- tests/server.py | 20 +++- tests/test_mau.py | 35 ++---- tests/test_server.py | 12 +- tests/test_terms_auth.py | 5 +- tests/unittest.py | 12 +- 11 files changed, 163 insertions(+), 217 deletions(-) create mode 100644 changelog.d/4156.misc diff --git a/changelog.d/4156.misc b/changelog.d/4156.misc new file mode 100644 index 0000000000..20d404406c --- /dev/null +++ b/changelog.d/4156.misc @@ -0,0 +1 @@ +HTTP tests have been refactored to contain less boilerplate. diff --git a/tests/rest/client/v1/test_admin.py b/tests/rest/client/v1/test_admin.py index 1a553fa3f9..e38eb628a9 100644 --- a/tests/rest/client/v1/test_admin.py +++ b/tests/rest/client/v1/test_admin.py @@ -19,24 +19,17 @@ import json from mock import Mock -from synapse.http.server import JsonResource from synapse.rest.client.v1.admin import register_servlets -from synapse.util import Clock from tests import unittest -from tests.server import ( - ThreadedMemoryReactorClock, - make_request, - render, - setup_test_homeserver, -) -class UserRegisterTestCase(unittest.TestCase): - def setUp(self): +class UserRegisterTestCase(unittest.HomeserverTestCase): + + servlets = [register_servlets] + + def make_homeserver(self, reactor, clock): - self.clock = ThreadedMemoryReactorClock() - self.hs_clock = Clock(self.clock) self.url = "/_matrix/client/r0/admin/register" self.registration_handler = Mock() @@ -50,17 +43,14 @@ class UserRegisterTestCase(unittest.TestCase): self.secrets = Mock() - self.hs = setup_test_homeserver( - self.addCleanup, http_client=None, clock=self.hs_clock, reactor=self.clock - ) + self.hs = self.setup_test_homeserver() self.hs.config.registration_shared_secret = u"shared" self.hs.get_media_repository = Mock() self.hs.get_deactivate_account_handler = Mock() - self.resource = JsonResource(self.hs) - register_servlets(self.hs, self.resource) + return self.hs def test_disabled(self): """ @@ -69,8 +59,8 @@ class UserRegisterTestCase(unittest.TestCase): """ self.hs.config.registration_shared_secret = None - request, channel = make_request("POST", self.url, b'{}') - render(request, self.resource, self.clock) + request, channel = self.make_request("POST", self.url, b'{}') + self.render(request) self.assertEqual(400, int(channel.result["code"]), msg=channel.result["body"]) self.assertEqual( @@ -87,8 +77,8 @@ class UserRegisterTestCase(unittest.TestCase): self.hs.get_secrets = Mock(return_value=secrets) - request, channel = make_request("GET", self.url) - render(request, self.resource, self.clock) + request, channel = self.make_request("GET", self.url) + self.render(request) self.assertEqual(channel.json_body, {"nonce": "abcd"}) @@ -97,25 +87,25 @@ class UserRegisterTestCase(unittest.TestCase): Calling GET on the endpoint will return a randomised nonce, which will only last for SALT_TIMEOUT (60s). """ - request, channel = make_request("GET", self.url) - render(request, self.resource, self.clock) + request, channel = self.make_request("GET", self.url) + self.render(request) nonce = channel.json_body["nonce"] # 59 seconds - self.clock.advance(59) + self.reactor.advance(59) body = json.dumps({"nonce": nonce}) - request, channel = make_request("POST", self.url, body.encode('utf8')) - render(request, self.resource, self.clock) + request, channel = self.make_request("POST", self.url, body.encode('utf8')) + self.render(request) self.assertEqual(400, int(channel.result["code"]), msg=channel.result["body"]) self.assertEqual('username must be specified', channel.json_body["error"]) # 61 seconds - self.clock.advance(2) + self.reactor.advance(2) - request, channel = make_request("POST", self.url, body.encode('utf8')) - render(request, self.resource, self.clock) + request, channel = self.make_request("POST", self.url, body.encode('utf8')) + self.render(request) self.assertEqual(400, int(channel.result["code"]), msg=channel.result["body"]) self.assertEqual('unrecognised nonce', channel.json_body["error"]) @@ -124,8 +114,8 @@ class UserRegisterTestCase(unittest.TestCase): """ Only the provided nonce can be used, as it's checked in the MAC. """ - request, channel = make_request("GET", self.url) - render(request, self.resource, self.clock) + request, channel = self.make_request("GET", self.url) + self.render(request) nonce = channel.json_body["nonce"] want_mac = hmac.new(key=b"shared", digestmod=hashlib.sha1) @@ -141,8 +131,8 @@ class UserRegisterTestCase(unittest.TestCase): "mac": want_mac, } ) - request, channel = make_request("POST", self.url, body.encode('utf8')) - render(request, self.resource, self.clock) + request, channel = self.make_request("POST", self.url, body.encode('utf8')) + self.render(request) self.assertEqual(403, int(channel.result["code"]), msg=channel.result["body"]) self.assertEqual("HMAC incorrect", channel.json_body["error"]) @@ -152,8 +142,8 @@ class UserRegisterTestCase(unittest.TestCase): When the correct nonce is provided, and the right key is provided, the user is registered. """ - request, channel = make_request("GET", self.url) - render(request, self.resource, self.clock) + request, channel = self.make_request("GET", self.url) + self.render(request) nonce = channel.json_body["nonce"] want_mac = hmac.new(key=b"shared", digestmod=hashlib.sha1) @@ -169,8 +159,8 @@ class UserRegisterTestCase(unittest.TestCase): "mac": want_mac, } ) - request, channel = make_request("POST", self.url, body.encode('utf8')) - render(request, self.resource, self.clock) + request, channel = self.make_request("POST", self.url, body.encode('utf8')) + self.render(request) self.assertEqual(200, int(channel.result["code"]), msg=channel.result["body"]) self.assertEqual("@bob:test", channel.json_body["user_id"]) @@ -179,8 +169,8 @@ class UserRegisterTestCase(unittest.TestCase): """ A valid unrecognised nonce. """ - request, channel = make_request("GET", self.url) - render(request, self.resource, self.clock) + request, channel = self.make_request("GET", self.url) + self.render(request) nonce = channel.json_body["nonce"] want_mac = hmac.new(key=b"shared", digestmod=hashlib.sha1) @@ -196,15 +186,15 @@ class UserRegisterTestCase(unittest.TestCase): "mac": want_mac, } ) - request, channel = make_request("POST", self.url, body.encode('utf8')) - render(request, self.resource, self.clock) + request, channel = self.make_request("POST", self.url, body.encode('utf8')) + self.render(request) self.assertEqual(200, int(channel.result["code"]), msg=channel.result["body"]) self.assertEqual("@bob:test", channel.json_body["user_id"]) # Now, try and reuse it - request, channel = make_request("POST", self.url, body.encode('utf8')) - render(request, self.resource, self.clock) + request, channel = self.make_request("POST", self.url, body.encode('utf8')) + self.render(request) self.assertEqual(400, int(channel.result["code"]), msg=channel.result["body"]) self.assertEqual('unrecognised nonce', channel.json_body["error"]) @@ -217,8 +207,8 @@ class UserRegisterTestCase(unittest.TestCase): """ def nonce(): - request, channel = make_request("GET", self.url) - render(request, self.resource, self.clock) + request, channel = self.make_request("GET", self.url) + self.render(request) return channel.json_body["nonce"] # @@ -227,8 +217,8 @@ class UserRegisterTestCase(unittest.TestCase): # Must be present body = json.dumps({}) - request, channel = make_request("POST", self.url, body.encode('utf8')) - render(request, self.resource, self.clock) + request, channel = self.make_request("POST", self.url, body.encode('utf8')) + self.render(request) self.assertEqual(400, int(channel.result["code"]), msg=channel.result["body"]) self.assertEqual('nonce must be specified', channel.json_body["error"]) @@ -239,32 +229,32 @@ class UserRegisterTestCase(unittest.TestCase): # Must be present body = json.dumps({"nonce": nonce()}) - request, channel = make_request("POST", self.url, body.encode('utf8')) - render(request, self.resource, self.clock) + request, channel = self.make_request("POST", self.url, body.encode('utf8')) + self.render(request) self.assertEqual(400, int(channel.result["code"]), msg=channel.result["body"]) self.assertEqual('username must be specified', channel.json_body["error"]) # Must be a string body = json.dumps({"nonce": nonce(), "username": 1234}) - request, channel = make_request("POST", self.url, body.encode('utf8')) - render(request, self.resource, self.clock) + request, channel = self.make_request("POST", self.url, body.encode('utf8')) + self.render(request) self.assertEqual(400, int(channel.result["code"]), msg=channel.result["body"]) self.assertEqual('Invalid username', channel.json_body["error"]) # Must not have null bytes body = json.dumps({"nonce": nonce(), "username": u"abcd\u0000"}) - request, channel = make_request("POST", self.url, body.encode('utf8')) - render(request, self.resource, self.clock) + request, channel = self.make_request("POST", self.url, body.encode('utf8')) + self.render(request) self.assertEqual(400, int(channel.result["code"]), msg=channel.result["body"]) self.assertEqual('Invalid username', channel.json_body["error"]) # Must not have null bytes body = json.dumps({"nonce": nonce(), "username": "a" * 1000}) - request, channel = make_request("POST", self.url, body.encode('utf8')) - render(request, self.resource, self.clock) + request, channel = self.make_request("POST", self.url, body.encode('utf8')) + self.render(request) self.assertEqual(400, int(channel.result["code"]), msg=channel.result["body"]) self.assertEqual('Invalid username', channel.json_body["error"]) @@ -275,16 +265,16 @@ class UserRegisterTestCase(unittest.TestCase): # Must be present body = json.dumps({"nonce": nonce(), "username": "a"}) - request, channel = make_request("POST", self.url, body.encode('utf8')) - render(request, self.resource, self.clock) + request, channel = self.make_request("POST", self.url, body.encode('utf8')) + self.render(request) self.assertEqual(400, int(channel.result["code"]), msg=channel.result["body"]) self.assertEqual('password must be specified', channel.json_body["error"]) # Must be a string body = json.dumps({"nonce": nonce(), "username": "a", "password": 1234}) - request, channel = make_request("POST", self.url, body.encode('utf8')) - render(request, self.resource, self.clock) + request, channel = self.make_request("POST", self.url, body.encode('utf8')) + self.render(request) self.assertEqual(400, int(channel.result["code"]), msg=channel.result["body"]) self.assertEqual('Invalid password', channel.json_body["error"]) @@ -293,16 +283,16 @@ class UserRegisterTestCase(unittest.TestCase): body = json.dumps( {"nonce": nonce(), "username": "a", "password": u"abcd\u0000"} ) - request, channel = make_request("POST", self.url, body.encode('utf8')) - render(request, self.resource, self.clock) + request, channel = self.make_request("POST", self.url, body.encode('utf8')) + self.render(request) self.assertEqual(400, int(channel.result["code"]), msg=channel.result["body"]) self.assertEqual('Invalid password', channel.json_body["error"]) # Super long body = json.dumps({"nonce": nonce(), "username": "a", "password": "A" * 1000}) - request, channel = make_request("POST", self.url, body.encode('utf8')) - render(request, self.resource, self.clock) + request, channel = self.make_request("POST", self.url, body.encode('utf8')) + self.render(request) self.assertEqual(400, int(channel.result["code"]), msg=channel.result["body"]) self.assertEqual('Invalid password', channel.json_body["error"]) diff --git a/tests/rest/client/v1/test_register.py b/tests/rest/client/v1/test_register.py index 6b7ff813d5..f973eff8cf 100644 --- a/tests/rest/client/v1/test_register.py +++ b/tests/rest/client/v1/test_register.py @@ -45,11 +45,11 @@ class CreateUserServletTestCase(unittest.TestCase): ) handlers = Mock(registration_handler=self.registration_handler) - self.clock = MemoryReactorClock() - self.hs_clock = Clock(self.clock) + self.reactor = MemoryReactorClock() + self.hs_clock = Clock(self.reactor) self.hs = self.hs = setup_test_homeserver( - self.addCleanup, http_client=None, clock=self.hs_clock, reactor=self.clock + self.addCleanup, http_client=None, clock=self.hs_clock, reactor=self.reactor ) self.hs.get_datastore = Mock(return_value=self.datastore) self.hs.get_handlers = Mock(return_value=handlers) @@ -76,8 +76,8 @@ class CreateUserServletTestCase(unittest.TestCase): return_value=(user_id, token) ) - request, channel = make_request(b"POST", url, request_data) - render(request, res, self.clock) + request, channel = make_request(self.reactor, b"POST", url, request_data) + render(request, res, self.reactor) self.assertEquals(channel.result["code"], b"200") diff --git a/tests/rest/client/v1/utils.py b/tests/rest/client/v1/utils.py index 530dc8ba6d..9c401bf300 100644 --- a/tests/rest/client/v1/utils.py +++ b/tests/rest/client/v1/utils.py @@ -169,7 +169,7 @@ class RestHelper(object): path = path + "?access_token=%s" % tok request, channel = make_request( - "POST", path, json.dumps(content).encode('utf8') + self.hs.get_reactor(), "POST", path, json.dumps(content).encode('utf8') ) render(request, self.resource, self.hs.get_reactor()) @@ -217,7 +217,9 @@ class RestHelper(object): data = {"membership": membership} - request, channel = make_request("PUT", path, json.dumps(data).encode('utf8')) + request, channel = make_request( + self.hs.get_reactor(), "PUT", path, json.dumps(data).encode('utf8') + ) render(request, self.resource, self.hs.get_reactor()) @@ -228,18 +230,6 @@ class RestHelper(object): self.auth_user_id = temp_id - @defer.inlineCallbacks - def register(self, user_id): - (code, response) = yield self.mock_resource.trigger( - "POST", - "/_matrix/client/r0/register", - json.dumps( - {"user": user_id, "password": "test", "type": "m.login.password"} - ), - ) - self.assertEquals(200, code) - defer.returnValue(response) - def send(self, room_id, body=None, txn_id=None, tok=None, expect_code=200): if txn_id is None: txn_id = "m%s" % (str(time.time())) @@ -251,7 +241,9 @@ class RestHelper(object): if tok: path = path + "?access_token=%s" % tok - request, channel = make_request("PUT", path, json.dumps(content).encode('utf8')) + request, channel = make_request( + self.hs.get_reactor(), "PUT", path, json.dumps(content).encode('utf8') + ) render(request, self.resource, self.hs.get_reactor()) assert int(channel.result["code"]) == expect_code, ( diff --git a/tests/rest/client/v2_alpha/test_filter.py b/tests/rest/client/v2_alpha/test_filter.py index 6a886ee3b8..f42a8efbf4 100644 --- a/tests/rest/client/v2_alpha/test_filter.py +++ b/tests/rest/client/v2_alpha/test_filter.py @@ -13,84 +13,47 @@ # See the License for the specific language governing permissions and # limitations under the License. -import synapse.types from synapse.api.errors import Codes -from synapse.http.server import JsonResource from synapse.rest.client.v2_alpha import filter -from synapse.types import UserID -from synapse.util import Clock from tests import unittest -from tests.server import ( - ThreadedMemoryReactorClock as MemoryReactorClock, - make_request, - render, - setup_test_homeserver, -) PATH_PREFIX = "/_matrix/client/v2_alpha" -class FilterTestCase(unittest.TestCase): +class FilterTestCase(unittest.HomeserverTestCase): - USER_ID = "@apple:test" + user_id = "@apple:test" + hijack_auth = True EXAMPLE_FILTER = {"room": {"timeline": {"types": ["m.room.message"]}}} EXAMPLE_FILTER_JSON = b'{"room": {"timeline": {"types": ["m.room.message"]}}}' - TO_REGISTER = [filter] + servlets = [filter.register_servlets] - def setUp(self): - self.clock = MemoryReactorClock() - self.hs_clock = Clock(self.clock) - - self.hs = setup_test_homeserver( - self.addCleanup, http_client=None, clock=self.hs_clock, reactor=self.clock - ) - - self.auth = self.hs.get_auth() - - def get_user_by_access_token(token=None, allow_guest=False): - return { - "user": UserID.from_string(self.USER_ID), - "token_id": 1, - "is_guest": False, - } - - def get_user_by_req(request, allow_guest=False, rights="access"): - return synapse.types.create_requester( - UserID.from_string(self.USER_ID), 1, False, None - ) - - self.auth.get_user_by_access_token = get_user_by_access_token - self.auth.get_user_by_req = get_user_by_req - - self.store = self.hs.get_datastore() - self.filtering = self.hs.get_filtering() - self.resource = JsonResource(self.hs) - - for r in self.TO_REGISTER: - r.register_servlets(self.hs, self.resource) + def prepare(self, reactor, clock, hs): + self.filtering = hs.get_filtering() + self.store = hs.get_datastore() def test_add_filter(self): - request, channel = make_request( + request, channel = self.make_request( "POST", - "/_matrix/client/r0/user/%s/filter" % (self.USER_ID), + "/_matrix/client/r0/user/%s/filter" % (self.user_id), self.EXAMPLE_FILTER_JSON, ) - render(request, self.resource, self.clock) + self.render(request) self.assertEqual(channel.result["code"], b"200") self.assertEqual(channel.json_body, {"filter_id": "0"}) filter = self.store.get_user_filter(user_localpart="apple", filter_id=0) - self.clock.advance(0) + self.pump() self.assertEquals(filter.result, self.EXAMPLE_FILTER) def test_add_filter_for_other_user(self): - request, channel = make_request( + request, channel = self.make_request( "POST", "/_matrix/client/r0/user/%s/filter" % ("@watermelon:test"), self.EXAMPLE_FILTER_JSON, ) - render(request, self.resource, self.clock) + self.render(request) self.assertEqual(channel.result["code"], b"403") self.assertEquals(channel.json_body["errcode"], Codes.FORBIDDEN) @@ -98,12 +61,12 @@ class FilterTestCase(unittest.TestCase): def test_add_filter_non_local_user(self): _is_mine = self.hs.is_mine self.hs.is_mine = lambda target_user: False - request, channel = make_request( + request, channel = self.make_request( "POST", - "/_matrix/client/r0/user/%s/filter" % (self.USER_ID), + "/_matrix/client/r0/user/%s/filter" % (self.user_id), self.EXAMPLE_FILTER_JSON, ) - render(request, self.resource, self.clock) + self.render(request) self.hs.is_mine = _is_mine self.assertEqual(channel.result["code"], b"403") @@ -113,21 +76,21 @@ class FilterTestCase(unittest.TestCase): filter_id = self.filtering.add_user_filter( user_localpart="apple", user_filter=self.EXAMPLE_FILTER ) - self.clock.advance(1) + self.reactor.advance(1) filter_id = filter_id.result - request, channel = make_request( - "GET", "/_matrix/client/r0/user/%s/filter/%s" % (self.USER_ID, filter_id) + request, channel = self.make_request( + "GET", "/_matrix/client/r0/user/%s/filter/%s" % (self.user_id, filter_id) ) - render(request, self.resource, self.clock) + self.render(request) self.assertEqual(channel.result["code"], b"200") self.assertEquals(channel.json_body, self.EXAMPLE_FILTER) def test_get_filter_non_existant(self): - request, channel = make_request( - "GET", "/_matrix/client/r0/user/%s/filter/12382148321" % (self.USER_ID) + request, channel = self.make_request( + "GET", "/_matrix/client/r0/user/%s/filter/12382148321" % (self.user_id) ) - render(request, self.resource, self.clock) + self.render(request) self.assertEqual(channel.result["code"], b"400") self.assertEquals(channel.json_body["errcode"], Codes.NOT_FOUND) @@ -135,18 +98,18 @@ class FilterTestCase(unittest.TestCase): # Currently invalid params do not have an appropriate errcode # in errors.py def test_get_filter_invalid_id(self): - request, channel = make_request( - "GET", "/_matrix/client/r0/user/%s/filter/foobar" % (self.USER_ID) + request, channel = self.make_request( + "GET", "/_matrix/client/r0/user/%s/filter/foobar" % (self.user_id) ) - render(request, self.resource, self.clock) + self.render(request) self.assertEqual(channel.result["code"], b"400") # No ID also returns an invalid_id error def test_get_filter_no_id(self): - request, channel = make_request( - "GET", "/_matrix/client/r0/user/%s/filter/" % (self.USER_ID) + request, channel = self.make_request( + "GET", "/_matrix/client/r0/user/%s/filter/" % (self.user_id) ) - render(request, self.resource, self.clock) + self.render(request) self.assertEqual(channel.result["code"], b"400") diff --git a/tests/rest/client/v2_alpha/test_register.py b/tests/rest/client/v2_alpha/test_register.py index 1c128e81f5..753d5c3e80 100644 --- a/tests/rest/client/v2_alpha/test_register.py +++ b/tests/rest/client/v2_alpha/test_register.py @@ -3,22 +3,19 @@ import json from mock import Mock from twisted.python import failure -from twisted.test.proto_helpers import MemoryReactorClock from synapse.api.errors import InteractiveAuthIncompleteError -from synapse.http.server import JsonResource from synapse.rest.client.v2_alpha.register import register_servlets -from synapse.util import Clock from tests import unittest -from tests.server import make_request, render, setup_test_homeserver -class RegisterRestServletTestCase(unittest.TestCase): - def setUp(self): +class RegisterRestServletTestCase(unittest.HomeserverTestCase): + + servlets = [register_servlets] + + def make_homeserver(self, reactor, clock): - self.clock = MemoryReactorClock() - self.hs_clock = Clock(self.clock) self.url = b"/_matrix/client/r0/register" self.appservice = None @@ -46,9 +43,7 @@ class RegisterRestServletTestCase(unittest.TestCase): identity_handler=self.identity_handler, login_handler=self.login_handler, ) - self.hs = setup_test_homeserver( - self.addCleanup, http_client=None, clock=self.hs_clock, reactor=self.clock - ) + self.hs = self.setup_test_homeserver() self.hs.get_auth = Mock(return_value=self.auth) self.hs.get_handlers = Mock(return_value=self.handlers) self.hs.get_auth_handler = Mock(return_value=self.auth_handler) @@ -58,8 +53,7 @@ class RegisterRestServletTestCase(unittest.TestCase): self.hs.config.registrations_require_3pid = [] self.hs.config.auto_join_rooms = [] - self.resource = JsonResource(self.hs) - register_servlets(self.hs, self.resource) + return self.hs def test_POST_appservice_registration_valid(self): user_id = "@kermit:muppet" @@ -69,10 +63,10 @@ class RegisterRestServletTestCase(unittest.TestCase): self.auth_handler.get_access_token_for_user_id = Mock(return_value=token) request_data = json.dumps({"username": "kermit"}) - request, channel = make_request( + request, channel = self.make_request( b"POST", self.url + b"?access_token=i_am_an_app_service", request_data ) - render(request, self.resource, self.clock) + self.render(request) self.assertEquals(channel.result["code"], b"200", channel.result) det_data = { @@ -85,25 +79,25 @@ class RegisterRestServletTestCase(unittest.TestCase): def test_POST_appservice_registration_invalid(self): self.appservice = None # no application service exists request_data = json.dumps({"username": "kermit"}) - request, channel = make_request( + request, channel = self.make_request( b"POST", self.url + b"?access_token=i_am_an_app_service", request_data ) - render(request, self.resource, self.clock) + self.render(request) self.assertEquals(channel.result["code"], b"401", channel.result) def test_POST_bad_password(self): request_data = json.dumps({"username": "kermit", "password": 666}) - request, channel = make_request(b"POST", self.url, request_data) - render(request, self.resource, self.clock) + request, channel = self.make_request(b"POST", self.url, request_data) + self.render(request) self.assertEquals(channel.result["code"], b"400", channel.result) self.assertEquals(channel.json_body["error"], "Invalid password") def test_POST_bad_username(self): request_data = json.dumps({"username": 777, "password": "monkey"}) - request, channel = make_request(b"POST", self.url, request_data) - render(request, self.resource, self.clock) + request, channel = self.make_request(b"POST", self.url, request_data) + self.render(request) self.assertEquals(channel.result["code"], b"400", channel.result) self.assertEquals(channel.json_body["error"], "Invalid username") @@ -121,8 +115,8 @@ class RegisterRestServletTestCase(unittest.TestCase): self.auth_handler.get_access_token_for_user_id = Mock(return_value=token) self.device_handler.check_device_registered = Mock(return_value=device_id) - request, channel = make_request(b"POST", self.url, request_data) - render(request, self.resource, self.clock) + request, channel = self.make_request(b"POST", self.url, request_data) + self.render(request) det_data = { "user_id": user_id, @@ -143,8 +137,8 @@ class RegisterRestServletTestCase(unittest.TestCase): self.auth_result = (None, {"username": "kermit", "password": "monkey"}, None) self.registration_handler.register = Mock(return_value=("@user:id", "t")) - request, channel = make_request(b"POST", self.url, request_data) - render(request, self.resource, self.clock) + request, channel = self.make_request(b"POST", self.url, request_data) + self.render(request) self.assertEquals(channel.result["code"], b"403", channel.result) self.assertEquals(channel.json_body["error"], "Registration has been disabled") @@ -155,8 +149,8 @@ class RegisterRestServletTestCase(unittest.TestCase): self.hs.config.allow_guest_access = True self.registration_handler.register = Mock(return_value=(user_id, None)) - request, channel = make_request(b"POST", self.url + b"?kind=guest", b"{}") - render(request, self.resource, self.clock) + request, channel = self.make_request(b"POST", self.url + b"?kind=guest", b"{}") + self.render(request) det_data = { "user_id": user_id, @@ -169,8 +163,8 @@ class RegisterRestServletTestCase(unittest.TestCase): def test_POST_disabled_guest_registration(self): self.hs.config.allow_guest_access = False - request, channel = make_request(b"POST", self.url + b"?kind=guest", b"{}") - render(request, self.resource, self.clock) + request, channel = self.make_request(b"POST", self.url + b"?kind=guest", b"{}") + self.render(request) self.assertEquals(channel.result["code"], b"403", channel.result) self.assertEquals(channel.json_body["error"], "Guest access is disabled") diff --git a/tests/server.py b/tests/server.py index f63f33c94f..984cfe26d4 100644 --- a/tests/server.py +++ b/tests/server.py @@ -34,6 +34,7 @@ class FakeChannel(object): wire). """ + _reactor = attr.ib() result = attr.ib(default=attr.Factory(dict)) _producer = None @@ -63,6 +64,15 @@ class FakeChannel(object): def registerProducer(self, producer, streaming): self._producer = producer + self.producerStreaming = streaming + + def _produce(): + if self._producer: + self._producer.resumeProducing() + self._reactor.callLater(0.1, _produce) + + if not streaming: + self._reactor.callLater(0.0, _produce) def unregisterProducer(self): if self._producer is None: @@ -105,7 +115,13 @@ class FakeSite: def make_request( - method, path, content=b"", access_token=None, request=SynapseRequest, shorthand=True + reactor, + method, + path, + content=b"", + access_token=None, + request=SynapseRequest, + shorthand=True, ): """ Make a web request using the given method and path, feed it the @@ -138,7 +154,7 @@ def make_request( content = content.encode('utf8') site = FakeSite() - channel = FakeChannel() + channel = FakeChannel(reactor) req = request(site, channel) req.process = lambda: b"" diff --git a/tests/test_mau.py b/tests/test_mau.py index 5d387851c5..0afdeb0818 100644 --- a/tests/test_mau.py +++ b/tests/test_mau.py @@ -21,30 +21,20 @@ from mock import Mock, NonCallableMock from synapse.api.constants import LoginType from synapse.api.errors import Codes, HttpResponseException, SynapseError -from synapse.http.server import JsonResource from synapse.rest.client.v2_alpha import register, sync -from synapse.util import Clock from tests import unittest -from tests.server import ( - ThreadedMemoryReactorClock, - make_request, - render, - setup_test_homeserver, -) -class TestMauLimit(unittest.TestCase): - def setUp(self): - self.reactor = ThreadedMemoryReactorClock() - self.clock = Clock(self.reactor) +class TestMauLimit(unittest.HomeserverTestCase): - self.hs = setup_test_homeserver( - self.addCleanup, + servlets = [register.register_servlets, sync.register_servlets] + + def make_homeserver(self, reactor, clock): + + self.hs = self.setup_test_homeserver( "red", http_client=None, - clock=self.clock, - reactor=self.reactor, federation_client=Mock(), ratelimiter=NonCallableMock(spec_set=["send_message"]), ) @@ -63,10 +53,7 @@ class TestMauLimit(unittest.TestCase): self.hs.config.server_notices_mxid_display_name = None self.hs.config.server_notices_mxid_avatar_url = None self.hs.config.server_notices_room_name = "Test Server Notice Room" - - self.resource = JsonResource(self.hs) - register.register_servlets(self.hs, self.resource) - sync.register_servlets(self.hs, self.resource) + return self.hs def test_simple_deny_mau(self): # Create and sync so that the MAU counts get updated @@ -193,8 +180,8 @@ class TestMauLimit(unittest.TestCase): } ) - request, channel = make_request("POST", "/register", request_data) - render(request, self.resource, self.reactor) + request, channel = self.make_request("POST", "/register", request_data) + self.render(request) if channel.code != 200: raise HttpResponseException( @@ -206,10 +193,10 @@ class TestMauLimit(unittest.TestCase): return access_token def do_sync_for_user(self, token): - request, channel = make_request( + request, channel = self.make_request( "GET", "/sync", access_token=token ) - render(request, self.resource, self.reactor) + self.render(request) if channel.code != 200: raise HttpResponseException( diff --git a/tests/test_server.py b/tests/test_server.py index 4045fdadc3..f0e6291b7e 100644 --- a/tests/test_server.py +++ b/tests/test_server.py @@ -57,7 +57,9 @@ class JsonResourceTests(unittest.TestCase): "GET", [re.compile("^/_matrix/foo/(?P[^/]*)$")], _callback ) - request, channel = make_request(b"GET", b"/_matrix/foo/%E2%98%83?a=%E2%98%83") + request, channel = make_request( + self.reactor, b"GET", b"/_matrix/foo/%E2%98%83?a=%E2%98%83" + ) render(request, res, self.reactor) self.assertEqual(request.args, {b'a': [u"\N{SNOWMAN}".encode('utf8')]}) @@ -75,7 +77,7 @@ class JsonResourceTests(unittest.TestCase): res = JsonResource(self.homeserver) res.register_paths("GET", [re.compile("^/_matrix/foo$")], _callback) - request, channel = make_request(b"GET", b"/_matrix/foo") + request, channel = make_request(self.reactor, b"GET", b"/_matrix/foo") render(request, res, self.reactor) self.assertEqual(channel.result["code"], b'500') @@ -98,7 +100,7 @@ class JsonResourceTests(unittest.TestCase): res = JsonResource(self.homeserver) res.register_paths("GET", [re.compile("^/_matrix/foo$")], _callback) - request, channel = make_request(b"GET", b"/_matrix/foo") + request, channel = make_request(self.reactor, b"GET", b"/_matrix/foo") render(request, res, self.reactor) self.assertEqual(channel.result["code"], b'500') @@ -115,7 +117,7 @@ class JsonResourceTests(unittest.TestCase): res = JsonResource(self.homeserver) res.register_paths("GET", [re.compile("^/_matrix/foo$")], _callback) - request, channel = make_request(b"GET", b"/_matrix/foo") + request, channel = make_request(self.reactor, b"GET", b"/_matrix/foo") render(request, res, self.reactor) self.assertEqual(channel.result["code"], b'403') @@ -136,7 +138,7 @@ class JsonResourceTests(unittest.TestCase): res = JsonResource(self.homeserver) res.register_paths("GET", [re.compile("^/_matrix/foo$")], _callback) - request, channel = make_request(b"GET", b"/_matrix/foobar") + request, channel = make_request(self.reactor, b"GET", b"/_matrix/foobar") render(request, res, self.reactor) self.assertEqual(channel.result["code"], b'400') diff --git a/tests/test_terms_auth.py b/tests/test_terms_auth.py index 0b71c6feb9..9ecc3ef14f 100644 --- a/tests/test_terms_auth.py +++ b/tests/test_terms_auth.py @@ -23,7 +23,6 @@ from synapse.rest.client.v2_alpha.register import register_servlets from synapse.util import Clock from tests import unittest -from tests.server import make_request class TermsTestCase(unittest.HomeserverTestCase): @@ -92,7 +91,7 @@ class TermsTestCase(unittest.HomeserverTestCase): self.registration_handler.check_username = Mock(return_value=True) - request, channel = make_request(b"POST", self.url, request_data) + request, channel = self.make_request(b"POST", self.url, request_data) self.render(request) # We don't bother checking that the response is correct - we'll leave that to @@ -110,7 +109,7 @@ class TermsTestCase(unittest.HomeserverTestCase): }, } ) - request, channel = make_request(b"POST", self.url, request_data) + request, channel = self.make_request(b"POST", self.url, request_data) self.render(request) # We're interested in getting a response that looks like a successful diff --git a/tests/unittest.py b/tests/unittest.py index 5e35c943d7..a9ce57da9a 100644 --- a/tests/unittest.py +++ b/tests/unittest.py @@ -189,11 +189,11 @@ class HomeserverTestCase(TestCase): for servlet in self.servlets: servlet(self.hs, self.resource) + from tests.rest.client.v1.utils import RestHelper + + self.helper = RestHelper(self.hs, self.resource, getattr(self, "user_id", None)) + if hasattr(self, "user_id"): - from tests.rest.client.v1.utils import RestHelper - - self.helper = RestHelper(self.hs, self.resource, self.user_id) - if self.hijack_auth: def get_user_by_access_token(token=None, allow_guest=False): @@ -285,7 +285,9 @@ class HomeserverTestCase(TestCase): if isinstance(content, dict): content = json.dumps(content).encode('utf8') - return make_request(method, path, content, access_token, request, shorthand) + return make_request( + self.reactor, method, path, content, access_token, request, shorthand + ) def render(self, request): """ From 2904d133f34981cbccbc6b5e615ccd9ba511ab0d Mon Sep 17 00:00:00 2001 From: rubo77 Date: Tue, 6 Nov 2018 23:16:22 +0100 Subject: [PATCH 10/32] add purge_history.sh and purge_remote_media.sh scripts to contrib/purge_api/ Signed-off-by: Ruben Barkow --- changelog.d/4155.misc | 1 + contrib/purge_api/README.md | 16 +++ contrib/purge_api/purge_history.sh | 141 ++++++++++++++++++++++++ contrib/purge_api/purge_remote_media.sh | 54 +++++++++ 4 files changed, 212 insertions(+) create mode 100644 changelog.d/4155.misc create mode 100644 contrib/purge_api/README.md create mode 100644 contrib/purge_api/purge_history.sh create mode 100644 contrib/purge_api/purge_remote_media.sh diff --git a/changelog.d/4155.misc b/changelog.d/4155.misc new file mode 100644 index 0000000000..4a7d5acb66 --- /dev/null +++ b/changelog.d/4155.misc @@ -0,0 +1 @@ +add purge_history.sh and purge_remote_media.sh scripts to contrib/ diff --git a/contrib/purge_api/README.md b/contrib/purge_api/README.md new file mode 100644 index 0000000000..000bf35ca7 --- /dev/null +++ b/contrib/purge_api/README.md @@ -0,0 +1,16 @@ +Purge history API examples +========================== + +# `purge_history.sh` + +A bash file, that uses the [purge history API](/docs/admin_api/README.rst) to +purge all messages in a list of rooms up to a certain event. You can select a +timeframe or a number of messages that you want to keep in the room. + +Just configure the variables DOMAIN, ADMIN, ROOMS_ARRAY and TIME at the top of +the script. + +# `purge_remote_media.sh` + +A bash file, that uses the [purge history API](/docs/admin_api/README.rst) to +purge all old cached remote media. diff --git a/contrib/purge_api/purge_history.sh b/contrib/purge_api/purge_history.sh new file mode 100644 index 0000000000..e7dd5d6468 --- /dev/null +++ b/contrib/purge_api/purge_history.sh @@ -0,0 +1,141 @@ +#!/bin/bash + +# this script will use the api: +# https://github.com/matrix-org/synapse/blob/master/docs/admin_api/purge_history_api.rst +# +# It will purge all messages in a list of rooms up to a cetrain event + +################################################################################################### +# define your domain and admin user +################################################################################################### +# add this user as admin in your home server: +DOMAIN=yourserver.tld +# add this user as admin in your home server: +ADMIN="@you_admin_username:$DOMAIN" + +API_URL="$DOMAIN:8008/_matrix/client/r0" + +################################################################################################### +#choose the rooms to prune old messages from (add a free comment at the end) +################################################################################################### +# the room_id's you can get e.g. from your Riot clients "View Source" button on each message +ROOMS_ARRAY=( +'!DgvjtOljKujDBrxyHk:matrix.org#riot:matrix.org' +'!QtykxKocfZaZOUrTwp:matrix.org#Matrix HQ' +) + +# ALTERNATIVELY: +# you can select all the rooms that are not encrypted and loop over the result: +# SELECT room_id FROM rooms WHERE room_id NOT IN (SELECT DISTINCT room_id FROM events WHERE type ='m.room.encrypted') +# or +# select all rooms with at least 100 members: +# SELECT q.room_id FROM (select count(*) as numberofusers, room_id FROM current_state_events WHERE type ='m.room.member' +# GROUP BY room_id) AS q LEFT JOIN room_aliases a ON q.room_id=a.room_id WHERE q.numberofusers > 100 ORDER BY numberofusers desc + +################################################################################################### +# evaluate the EVENT_ID before which should be pruned +################################################################################################### +# choose a time before which the messages should be pruned: +TIME='12 months ago' +# ALTERNATIVELY: +# a certain time: +# TIME='2016-08-31 23:59:59' + +# creates a timestamp from the given time string: +UNIX_TIMESTAMP=$(date +%s%3N --date='TZ="UTC+2" '"$TIME") + +# ALTERNATIVELY: +# prune all messages that are older than 1000 messages ago: +# LAST_MESSAGES=1000 +# SQL_GET_EVENT="SELECT event_id from events WHERE type='m.room.message' AND room_id ='$ROOM' ORDER BY received_ts DESC LIMIT 1 offset $(($LAST_MESSAGES - 1))" + +# ALTERNATIVELY: +# select the EVENT_ID manually: +#EVENT_ID='$1471814088343495zpPNI:matrix.org' # an example event from 21st of Aug 2016 by Matthew + +################################################################################################### +# make the admin user a server admin in the database with +################################################################################################### +# psql -A -t --dbname=synapse -c "UPDATE users SET admin=1 WHERE name LIKE '$ADMIN'" + +################################################################################################### +# database function +################################################################################################### +sql (){ + # for sqlite3: + #sqlite3 homeserver.db "pragma busy_timeout=20000;$1" | awk '{print $2}' + # for postgres: + psql -A -t --dbname=synapse -c "$1" | grep -v 'Pager' +} + +################################################################################################### +# get an access token +################################################################################################### +# for example externally by watching Riot in your browser's network inspector +# or internally on the server locally, use this: +TOKEN=$(sql "SELECT token FROM access_tokens WHERE user_id='$ADMIN' ORDER BY id DESC LIMIT 1") +AUTH="Authorization: Bearer $TOKEN" + +################################################################################################### +# check, if your TOKEN works. For example this works: +################################################################################################### +# $ curl --header "$AUTH" "$API_URL/rooms/$ROOM/state/m.room.power_levels" + +################################################################################################### +# finally start pruning the room: +################################################################################################### +POSTDATA='{"delete_local_events":"true"}' # this will really delete local events, so the messages in the room really disappear unless they are restored by remote federation + +for ROOM in "${ROOMS_ARRAY[@]}"; do + echo "########################################### $(date) ################# " + echo "pruning room: $ROOM ..." + ROOM=${ROOM%#*} + #set -x + echo "check for alias in db..." + # for postgres: + sql "SELECT * FROM room_aliases WHERE room_id='$ROOM'" + echo "get event..." + # for postgres: + EVENT_ID=$(sql "SELECT event_id FROM events WHERE type='m.room.message' AND received_ts<'$UNIX_TIMESTAMP' AND room_id='$ROOM' ORDER BY received_ts DESC LIMIT 1;") + if [ "$EVENT_ID" == "" ]; then + echo "no event $TIME" + else + echo "event: $EVENT_ID" + SLEEP=2 + set -x + # call purge + OUT=$(curl --header "$AUTH" -s -d $POSTDATA POST "$API_URL/admin/purge_history/$ROOM/$EVENT_ID") + PURGE_ID=$(echo "$OUT" |grep purge_id|cut -d'"' -f4 ) + if [ "$PURGE_ID" == "" ]; then + # probably the history purge is already in progress for $ROOM + : "continuing with next room" + else + while : ; do + # get status of purge and sleep longer each time if still active + sleep $SLEEP + STATUS=$(curl --header "$AUTH" -s GET "$API_URL/admin/purge_history_status/$PURGE_ID" |grep status|cut -d'"' -f4) + : "$ROOM --> Status: $STATUS" + [[ "$STATUS" == "active" ]] || break + SLEEP=$((SLEEP + 1)) + done + fi + set +x + sleep 1 + fi +done + + +################################################################################################### +# additionally +################################################################################################### +# to benefit from pruning large amounts of data, you need to call VACUUM to free the unused space. +# This can take a very long time (hours) and the client have to be stopped while you do so: +# $ synctl stop +# $ sqlite3 -line homeserver.db "vacuum;" +# $ synctl start + +# This could be set, so you don't need to prune every time after deleting some rows: +# $ sqlite3 homeserver.db "PRAGMA auto_vacuum = FULL;" +# be cautious, it could make the database somewhat slow if there are a lot of deletions + +exit diff --git a/contrib/purge_api/purge_remote_media.sh b/contrib/purge_api/purge_remote_media.sh new file mode 100644 index 0000000000..99c07c663d --- /dev/null +++ b/contrib/purge_api/purge_remote_media.sh @@ -0,0 +1,54 @@ +#!/bin/bash + +DOMAIN=yourserver.tld +# add this user as admin in your home server: +ADMIN="@you_admin_username:$DOMAIN" + +API_URL="$DOMAIN:8008/_matrix/client/r0" + +# choose a time before which the messages should be pruned: +# TIME='2016-08-31 23:59:59' +TIME='12 months ago' + +# creates a timestamp from the given time string: +UNIX_TIMESTAMP=$(date +%s%3N --date='TZ="UTC+2" '"$TIME") + + +################################################################################################### +# database function +################################################################################################### +sql (){ + # for sqlite3: + #sqlite3 homeserver.db "pragma busy_timeout=20000;$1" | awk '{print $2}' + # for postgres: + psql -A -t --dbname=synapse -c "$1" | grep -v 'Pager' +} + +############################################################################### +# make the admin user a server admin in the database with +############################################################################### +# sql "UPDATE users SET admin=1 WHERE name LIKE '$ADMIN'" + +############################################################################### +# get an access token +############################################################################### +# for example externally by watching Riot in your browser's network inspector +# or internally on the server locally, use this: +TOKEN=$(sql "SELECT token FROM access_tokens WHERE user_id='$ADMIN' ORDER BY id DESC LIMIT 1") + +############################################################################### +# check, if your TOKEN works. For example this works: +############################################################################### +# curl --header "Authorization: Bearer $TOKEN" "$API_URL/rooms/$ROOM/state/m.room.power_levels" + +############################################################################### +# optional check size before +############################################################################### +# echo calculate used storage before ... +# du -shc ../.synapse/media_store/* + +############################################################################### +# finally start pruning media: +############################################################################### +set -x # for debugging the generated string +curl --header "Authorization: Bearer $TOKEN" -v POST "$API_URL/admin/purge_media_cache/?before_ts=$UNIX_TIMESTAMP" From b3708830b847245a5d559a099fcaf738250b7cbe Mon Sep 17 00:00:00 2001 From: Amber Brown Date: Thu, 8 Nov 2018 01:37:43 +1100 Subject: [PATCH 11/32] Fix URL preview bugs (type error when loading cache from db, content-type including quotes) (#4157) --- changelog.d/4157.bugfix | 1 + synapse/http/server.py | 8 +- synapse/rest/media/v1/preview_url_resource.py | 22 ++- tests/rest/media/v1/test_url_preview.py | 164 ++++++++++++++++++ tests/server.py | 2 + 5 files changed, 187 insertions(+), 10 deletions(-) create mode 100644 changelog.d/4157.bugfix create mode 100644 tests/rest/media/v1/test_url_preview.py diff --git a/changelog.d/4157.bugfix b/changelog.d/4157.bugfix new file mode 100644 index 0000000000..265514c3af --- /dev/null +++ b/changelog.d/4157.bugfix @@ -0,0 +1 @@ +Loading URL previews from the DB cache on Postgres will no longer cause Unicode type errors when responding to the request, and URL previews will no longer fail if the remote server returns a Content-Type header with the chartype in quotes. \ No newline at end of file diff --git a/synapse/http/server.py b/synapse/http/server.py index b4b25cab19..6a427d96a6 100644 --- a/synapse/http/server.py +++ b/synapse/http/server.py @@ -468,13 +468,13 @@ def set_cors_headers(request): Args: request (twisted.web.http.Request): The http request to add CORs to. """ - request.setHeader("Access-Control-Allow-Origin", "*") + request.setHeader(b"Access-Control-Allow-Origin", b"*") request.setHeader( - "Access-Control-Allow-Methods", "GET, POST, PUT, DELETE, OPTIONS" + b"Access-Control-Allow-Methods", b"GET, POST, PUT, DELETE, OPTIONS" ) request.setHeader( - "Access-Control-Allow-Headers", - "Origin, X-Requested-With, Content-Type, Accept, Authorization" + b"Access-Control-Allow-Headers", + b"Origin, X-Requested-With, Content-Type, Accept, Authorization" ) diff --git a/synapse/rest/media/v1/preview_url_resource.py b/synapse/rest/media/v1/preview_url_resource.py index 1a7bfd6b56..91d1dafe64 100644 --- a/synapse/rest/media/v1/preview_url_resource.py +++ b/synapse/rest/media/v1/preview_url_resource.py @@ -12,6 +12,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. + import cgi import datetime import errno @@ -24,6 +25,7 @@ import shutil import sys import traceback +import six from six import string_types from six.moves import urllib_parse as urlparse @@ -98,7 +100,7 @@ class PreviewUrlResource(Resource): # XXX: if get_user_by_req fails, what should we do in an async render? requester = yield self.auth.get_user_by_req(request) url = parse_string(request, "url") - if "ts" in request.args: + if b"ts" in request.args: ts = parse_integer(request, "ts") else: ts = self.clock.time_msec() @@ -180,7 +182,12 @@ class PreviewUrlResource(Resource): cache_result["expires_ts"] > ts and cache_result["response_code"] / 100 == 2 ): - defer.returnValue(cache_result["og"]) + # It may be stored as text in the database, not as bytes (such as + # PostgreSQL). If so, encode it back before handing it on. + og = cache_result["og"] + if isinstance(og, six.text_type): + og = og.encode('utf8') + defer.returnValue(og) return media_info = yield self._download_url(url, user) @@ -213,14 +220,17 @@ class PreviewUrlResource(Resource): elif _is_html(media_info['media_type']): # TODO: somehow stop a big HTML tree from exploding synapse's RAM - file = open(media_info['filename']) - body = file.read() - file.close() + with open(media_info['filename'], 'rb') as file: + body = file.read() # clobber the encoding from the content-type, or default to utf-8 # XXX: this overrides any or XML charset headers in the body # which may pose problems, but so far seems to work okay. - match = re.match(r'.*; *charset=(.*?)(;|$)', media_info['media_type'], re.I) + match = re.match( + r'.*; *charset="?(.*?)"?(;|$)', + media_info['media_type'], + re.I + ) encoding = match.group(1) if match else "utf-8" og = decode_and_calc_og(body, media_info['uri'], encoding) diff --git a/tests/rest/media/v1/test_url_preview.py b/tests/rest/media/v1/test_url_preview.py new file mode 100644 index 0000000000..29579cf091 --- /dev/null +++ b/tests/rest/media/v1/test_url_preview.py @@ -0,0 +1,164 @@ +# -*- coding: utf-8 -*- +# Copyright 2018 New Vector Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +from mock import Mock + +from twisted.internet.defer import Deferred + +from synapse.config.repository import MediaStorageProviderConfig +from synapse.util.module_loader import load_module + +from tests import unittest + + +class URLPreviewTests(unittest.HomeserverTestCase): + + hijack_auth = True + user_id = "@test:user" + + def make_homeserver(self, reactor, clock): + + self.storage_path = self.mktemp() + os.mkdir(self.storage_path) + + config = self.default_config() + config.url_preview_enabled = True + config.max_spider_size = 9999999 + config.url_preview_url_blacklist = [] + config.media_store_path = self.storage_path + + provider_config = { + "module": "synapse.rest.media.v1.storage_provider.FileStorageProviderBackend", + "store_local": True, + "store_synchronous": False, + "store_remote": True, + "config": {"directory": self.storage_path}, + } + + loaded = list(load_module(provider_config)) + [ + MediaStorageProviderConfig(False, False, False) + ] + + config.media_storage_providers = [loaded] + + hs = self.setup_test_homeserver(config=config) + + return hs + + def prepare(self, reactor, clock, hs): + + self.fetches = [] + + def get_file(url, output_stream, max_size): + """ + Returns tuple[int,dict,str,int] of file length, response headers, + absolute URI, and response code. + """ + + def write_to(r): + data, response = r + output_stream.write(data) + return response + + d = Deferred() + d.addCallback(write_to) + self.fetches.append((d, url)) + return d + + client = Mock() + client.get_file = get_file + + self.media_repo = hs.get_media_repository_resource() + preview_url = self.media_repo.children[b'preview_url'] + preview_url.client = client + self.preview_url = preview_url + + def test_cache_returns_correct_type(self): + + request, channel = self.make_request( + "GET", "url_preview?url=matrix.org", shorthand=False + ) + request.render(self.preview_url) + self.pump() + + # We've made one fetch + self.assertEqual(len(self.fetches), 1) + + end_content = ( + b'' + b'' + b'' + b'' + ) + + self.fetches[0][0].callback( + ( + end_content, + ( + len(end_content), + { + b"Content-Length": [b"%d" % (len(end_content))], + b"Content-Type": [b'text/html; charset="utf8"'], + }, + "https://example.com", + 200, + ), + ) + ) + + self.pump() + self.assertEqual(channel.code, 200) + self.assertEqual( + channel.json_body, {"og:title": "~matrix~", "og:description": "hi"} + ) + + # Check the cache returns the correct response + request, channel = self.make_request( + "GET", "url_preview?url=matrix.org", shorthand=False + ) + request.render(self.preview_url) + self.pump() + + # Only one fetch, still, since we'll lean on the cache + self.assertEqual(len(self.fetches), 1) + + # Check the cache response has the same content + self.assertEqual(channel.code, 200) + self.assertEqual( + channel.json_body, {"og:title": "~matrix~", "og:description": "hi"} + ) + + # Clear the in-memory cache + self.assertIn("matrix.org", self.preview_url._cache) + self.preview_url._cache.pop("matrix.org") + self.assertNotIn("matrix.org", self.preview_url._cache) + + # Check the database cache returns the correct response + request, channel = self.make_request( + "GET", "url_preview?url=matrix.org", shorthand=False + ) + request.render(self.preview_url) + self.pump() + + # Only one fetch, still, since we'll lean on the cache + self.assertEqual(len(self.fetches), 1) + + # Check the cache response has the same content + self.assertEqual(channel.code, 200) + self.assertEqual( + channel.json_body, {"og:title": "~matrix~", "og:description": "hi"} + ) diff --git a/tests/server.py b/tests/server.py index 984cfe26d4..7919a1f124 100644 --- a/tests/server.py +++ b/tests/server.py @@ -57,6 +57,8 @@ class FakeChannel(object): self.result["headers"] = headers def write(self, content): + assert isinstance(content, bytes), "Should be bytes! " + repr(content) + if "body" not in self.result: self.result["body"] = b"" From 264cb144021f977fe4958f8faa358f9baa48cc7f Mon Sep 17 00:00:00 2001 From: Amber Brown Date: Thu, 8 Nov 2018 04:57:28 +1100 Subject: [PATCH 12/32] Port hash_password to Python 3 (#4161) * port hash_password * changelog --- changelog.d/4161.bugfix | 1 + scripts/hash_password | 41 ++++++++++++++++++++++++++++++++++------- tox.ini | 2 +- 3 files changed, 36 insertions(+), 8 deletions(-) create mode 100644 changelog.d/4161.bugfix diff --git a/changelog.d/4161.bugfix b/changelog.d/4161.bugfix new file mode 100644 index 0000000000..252a40376b --- /dev/null +++ b/changelog.d/4161.bugfix @@ -0,0 +1 @@ +The hash_password script now works on Python 3. diff --git a/scripts/hash_password b/scripts/hash_password index a62bb5aa83..a1eb0769da 100755 --- a/scripts/hash_password +++ b/scripts/hash_password @@ -3,13 +3,15 @@ import argparse import getpass import sys +import unicodedata import bcrypt import yaml -bcrypt_rounds=12 +bcrypt_rounds = 12 password_pepper = "" + def prompt_for_pass(): password = getpass.getpass("Password: ") @@ -23,19 +25,27 @@ def prompt_for_pass(): return password + if __name__ == "__main__": parser = argparse.ArgumentParser( - description="Calculate the hash of a new password, so that passwords" - " can be reset") + description=( + "Calculate the hash of a new password, so that passwords can be reset" + ) + ) parser.add_argument( - "-p", "--password", + "-p", + "--password", default=None, help="New password for user. Will prompt if omitted.", ) parser.add_argument( - "-c", "--config", + "-c", + "--config", type=argparse.FileType('r'), - help="Path to server config file. Used to read in bcrypt_rounds and password_pepper.", + help=( + "Path to server config file. " + "Used to read in bcrypt_rounds and password_pepper." + ), ) args = parser.parse_args() @@ -49,4 +59,21 @@ if __name__ == "__main__": if not password: password = prompt_for_pass() - print bcrypt.hashpw(password + password_pepper, bcrypt.gensalt(bcrypt_rounds)) + # On Python 2, make sure we decode it to Unicode before we normalise it + if isinstance(password, bytes): + try: + password = password.decode(sys.stdin.encoding) + except UnicodeDecodeError: + print( + "ERROR! Your password is not decodable using your terminal encoding (%s)." + % (sys.stdin.encoding,) + ) + + pw = unicodedata.normalize("NFKC", password) + + hashed = bcrypt.hashpw( + pw.encode('utf8') + password_pepper.encode("utf8"), + bcrypt.gensalt(bcrypt_rounds), + ).decode('ascii') + + print(hashed) diff --git a/tox.ini b/tox.ini index 920211bf50..03ddaeb0b7 100644 --- a/tox.ini +++ b/tox.ini @@ -122,7 +122,7 @@ skip_install = True basepython = python3.6 deps = flake8 -commands = /bin/sh -c "flake8 synapse tests scripts scripts-dev scripts/register_new_matrix_user scripts/synapse_port_db synctl {env:PEP8SUFFIX:}" +commands = /bin/sh -c "flake8 synapse tests scripts scripts-dev scripts/hash_password scripts/register_new_matrix_user scripts/synapse_port_db synctl {env:PEP8SUFFIX:}" [testenv:check_isort] skip_install = True From 2b075fb03a704865a759693989127cd5800e7fe6 Mon Sep 17 00:00:00 2001 From: hera Date: Thu, 8 Nov 2018 11:03:08 +0000 Subject: [PATCH 13/32] Fix encoding error for consent form on python3 The form was rendering this as "b'01234....'". -- richvdh --- synapse/rest/consent/consent_resource.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/synapse/rest/consent/consent_resource.py b/synapse/rest/consent/consent_resource.py index e0f7de5d5c..8009b7ff1c 100644 --- a/synapse/rest/consent/consent_resource.py +++ b/synapse/rest/consent/consent_resource.py @@ -160,7 +160,9 @@ class ConsentResource(Resource): try: self._render_template( request, "%s.html" % (version,), - user=username, userhmac=userhmac, version=version, + user=username, + userhmac=userhmac.decode('ascii'), + version=version, has_consented=has_consented, public_version=public_version, ) except TemplateNotFound: From de6223836ed5094388fd430465beeab905a4e501 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Thu, 8 Nov 2018 11:06:28 +0000 Subject: [PATCH 14/32] changelog --- changelog.d/4163.bugfix | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog.d/4163.bugfix diff --git a/changelog.d/4163.bugfix b/changelog.d/4163.bugfix new file mode 100644 index 0000000000..c7e0ee229d --- /dev/null +++ b/changelog.d/4163.bugfix @@ -0,0 +1 @@ +Generating the user consent URI no longer fails on Python 3. From 0a1fc5297148224b33013ad6b3047933ede03e15 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Thu, 8 Nov 2018 11:12:29 +0000 Subject: [PATCH 15/32] fix parse_string docstring --- synapse/http/servlet.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/synapse/http/servlet.py b/synapse/http/servlet.py index a1e4b88e6d..528125e737 100644 --- a/synapse/http/servlet.py +++ b/synapse/http/servlet.py @@ -121,16 +121,15 @@ def parse_string(request, name, default=None, required=False, Args: request: the twisted HTTP request. - name (bytes/unicode): the name of the query parameter. - default (bytes/unicode|None): value to use if the parameter is absent, + name (bytes|unicode): the name of the query parameter. + default (bytes|unicode|None): value to use if the parameter is absent, defaults to None. Must be bytes if encoding is None. required (bool): whether to raise a 400 SynapseError if the parameter is absent, defaults to False. - allowed_values (list[bytes/unicode]): List of allowed values for the + allowed_values (list[bytes|unicode]): List of allowed values for the string, or None if any value is allowed, defaults to None. Must be the same type as name, if given. - encoding: The encoding to decode the name to, and decode the string - content with. + encoding (str|None): The encoding to decode the string content with. Returns: bytes/unicode|None: A string value or the default. Unicode if encoding From 9417986f7799bdcd609db16c2c964af09d9fe9b5 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 8 Nov 2018 12:11:20 +0000 Subject: [PATCH 16/32] Drop PDUs of unknown rooms When we receive events over federation we will need to know the room version to be able to correctly handle them, e.g. once we start changing event formats. Currently, we attempt to handle events in unknown rooms. --- synapse/federation/federation_server.py | 24 +++++++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/synapse/federation/federation_server.py b/synapse/federation/federation_server.py index fa2cc550e2..38949fb49a 100644 --- a/synapse/federation/federation_server.py +++ b/synapse/federation/federation_server.py @@ -162,8 +162,30 @@ class FederationServer(FederationBase): p["age_ts"] = request_time - int(p["age"]) del p["age"] + # We try and pull out an event ID so that if latter checks fail we + # can log something sensible. We don't mandate an event ID here in + # case future event formats get rid of the key. + possible_event_id = p.get("event_id", "") + + # Now we get the room ID so that we can check that we know the + # version of the room. + room_id = p.get("room_id") + if not room_id: + logger.info( + "Ignoring PDU as does not have a room_id. Event ID: %s", + possible_event_id, + ) + continue + + try: + # In future we will actually use the room version to parse the + # PDU into an event. + yield self.store.get_room_version(room_id) + except NotFoundError: + logger.info("Ignoring PUD for unknown room_id: %s", room_id) + continue + event = event_from_pdu_json(p) - room_id = event.room_id pdus_by_room.setdefault(room_id, []).append(event) pdu_results = {} From b1a22b24ab7532da993e673f353dd87eeef49151 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 8 Nov 2018 12:14:20 +0000 Subject: [PATCH 17/32] Fix noop checks when updating device keys Clients often reupload their device keys (for some reason) so its important for the server to check for no-ops before sending out device list update notifications. The check is broken in python 3 due to the fact comparing bytes and unicode always fails, and that we write bytes to the DB but get unicode when we read. --- synapse/storage/end_to_end_keys.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/synapse/storage/end_to_end_keys.py b/synapse/storage/end_to_end_keys.py index 1f1721e820..29281630c0 100644 --- a/synapse/storage/end_to_end_keys.py +++ b/synapse/storage/end_to_end_keys.py @@ -40,6 +40,11 @@ class EndToEndKeyStore(SQLBaseStore): allow_none=True, ) + if old_key_json and not isinstance(old_key_json, bytes): + # In py3 we need old_key_json to match new_key_json type. The DB + # returns unicode while encode_canonical_json returns bytes + old_key_json = old_key_json.encode("utf-8") + new_key_json = encode_canonical_json(device_keys) if old_key_json == new_key_json: return False From 06c3d8050f471a8aa94736ca54b7cb910099459e Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 8 Nov 2018 12:17:21 +0000 Subject: [PATCH 18/32] Newsfile --- changelog.d/4164.bugfix | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog.d/4164.bugfix diff --git a/changelog.d/4164.bugfix b/changelog.d/4164.bugfix new file mode 100644 index 0000000000..f70e0b2056 --- /dev/null +++ b/changelog.d/4164.bugfix @@ -0,0 +1 @@ +Fix noop checks when updating device keys, reducing spurious device list update notifications. From 94896d7ffe775f48bc75994fb2caba85daac96de Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 8 Nov 2018 12:30:25 +0000 Subject: [PATCH 19/32] Newsfile --- changelog.d/4165.misc | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog.d/4165.misc diff --git a/changelog.d/4165.misc b/changelog.d/4165.misc new file mode 100644 index 0000000000..415d80aeab --- /dev/null +++ b/changelog.d/4165.misc @@ -0,0 +1 @@ +Drop incoming events from federation for unknown rooms From 5ebed186926eee77844730f5270a926417a0be09 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 8 Nov 2018 12:33:13 +0000 Subject: [PATCH 20/32] Lets convert bytes to unicode instead --- synapse/storage/end_to_end_keys.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/synapse/storage/end_to_end_keys.py b/synapse/storage/end_to_end_keys.py index 29281630c0..2a0f6cfca9 100644 --- a/synapse/storage/end_to_end_keys.py +++ b/synapse/storage/end_to_end_keys.py @@ -40,12 +40,10 @@ class EndToEndKeyStore(SQLBaseStore): allow_none=True, ) - if old_key_json and not isinstance(old_key_json, bytes): - # In py3 we need old_key_json to match new_key_json type. The DB - # returns unicode while encode_canonical_json returns bytes - old_key_json = old_key_json.encode("utf-8") + # In py3 we need old_key_json to match new_key_json type. The DB + # returns unicode while encode_canonical_json returns bytes. + new_key_json = encode_canonical_json(device_keys).decode("utf-8") - new_key_json = encode_canonical_json(device_keys) if old_key_json == new_key_json: return False From abaa93c1583800155093ab75f6c072c8d83200d0 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 8 Nov 2018 14:06:44 +0000 Subject: [PATCH 21/32] Add test to assert set_e2e_device_keys correctly returns False on no-op --- tests/storage/test_end_to_end_keys.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/tests/storage/test_end_to_end_keys.py b/tests/storage/test_end_to_end_keys.py index 8f0aaece40..b83f7336d3 100644 --- a/tests/storage/test_end_to_end_keys.py +++ b/tests/storage/test_end_to_end_keys.py @@ -44,6 +44,21 @@ class EndToEndKeyStoreTestCase(tests.unittest.TestCase): dev = res["user"]["device"] self.assertDictContainsSubset({"keys": json, "device_display_name": None}, dev) + @defer.inlineCallbacks + def test_reupload_key(self): + now = 1470174257070 + json = {"key": "value"} + + yield self.store.store_device("user", "device", None) + + changed = yield self.store.set_e2e_device_keys("user", "device", now, json) + self.assertTrue(changed) + + # If we try to upload the same key then we should be told nothing + # changed + changed = yield self.store.set_e2e_device_keys("user", "device", now, json) + self.assertFalse(changed) + @defer.inlineCallbacks def test_get_key_with_device_name(self): now = 1470174257070 From 91d96759c983d7ea96e8513994d051c82892a2c0 Mon Sep 17 00:00:00 2001 From: Brendan Abolivier Date: Fri, 9 Nov 2018 10:41:34 +0000 Subject: [PATCH 22/32] Add a Content-Type header on POST requests to the federation client --- scripts-dev/federation_client.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/scripts-dev/federation_client.py b/scripts-dev/federation_client.py index 2566ce7cef..e0287c8c6c 100755 --- a/scripts-dev/federation_client.py +++ b/scripts-dev/federation_client.py @@ -154,10 +154,15 @@ def request_json(method, origin_name, origin_key, destination, path, content): s = requests.Session() s.mount("matrix://", MatrixConnectionAdapter()) + headers = {"Host": destination, "Authorization": authorization_headers[0]} + + if method == "POST": + headers["Content-Type"] = "application/json" + result = s.request( method=method, url=dest, - headers={"Host": destination, "Authorization": authorization_headers[0]}, + headers=headers, verify=False, data=content, ) @@ -203,7 +208,7 @@ def main(): parser.add_argument( "-X", "--method", - help="HTTP method to use for the request. Defaults to GET if --data is" + help="HTTP method to use for the request. Defaults to GET if --body is" "unspecified, POST if it is.", ) From d3fa6194f7cce262b39ab27523565ca0e6880263 Mon Sep 17 00:00:00 2001 From: David Baker Date: Fri, 9 Nov 2018 11:11:31 +0000 Subject: [PATCH 23/32] Remove unnecessary str() --- synapse/handlers/e2e_room_keys.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/handlers/e2e_room_keys.py b/synapse/handlers/e2e_room_keys.py index b6d302cd6b..5edb3cfe04 100644 --- a/synapse/handlers/e2e_room_keys.py +++ b/synapse/handlers/e2e_room_keys.py @@ -138,7 +138,7 @@ class E2eRoomKeysHandler(object): else: raise - if str(version_info['version']) != version: + if version_info['version'] != version: # Check that the version we're trying to upload actually exists try: version_info = yield self.store.get_e2e_room_keys_version_info( From 9bce065a53799eb6403480736630ca5d7fdfe19c Mon Sep 17 00:00:00 2001 From: Richard van der Hoff <1389908+richvdh@users.noreply.github.com> Date: Fri, 9 Nov 2018 11:28:22 +0000 Subject: [PATCH 24/32] Update synapse/federation/federation_server.py Co-Authored-By: erikjohnston --- synapse/federation/federation_server.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/federation/federation_server.py b/synapse/federation/federation_server.py index 38949fb49a..3bdc6024f8 100644 --- a/synapse/federation/federation_server.py +++ b/synapse/federation/federation_server.py @@ -162,7 +162,7 @@ class FederationServer(FederationBase): p["age_ts"] = request_time - int(p["age"]) del p["age"] - # We try and pull out an event ID so that if latter checks fail we + # We try and pull out an event ID so that if later checks fail we # can log something sensible. We don't mandate an event ID here in # case future event formats get rid of the key. possible_event_id = p.get("event_id", "") From 3cecf5340d4e2ad8944a79b0fd4213e242791cbe Mon Sep 17 00:00:00 2001 From: Richard van der Hoff <1389908+richvdh@users.noreply.github.com> Date: Fri, 9 Nov 2018 11:28:25 +0000 Subject: [PATCH 25/32] Update synapse/federation/federation_server.py Co-Authored-By: erikjohnston --- synapse/federation/federation_server.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/federation/federation_server.py b/synapse/federation/federation_server.py index 3bdc6024f8..98722ae543 100644 --- a/synapse/federation/federation_server.py +++ b/synapse/federation/federation_server.py @@ -182,7 +182,7 @@ class FederationServer(FederationBase): # PDU into an event. yield self.store.get_room_version(room_id) except NotFoundError: - logger.info("Ignoring PUD for unknown room_id: %s", room_id) + logger.info("Ignoring PDU for unknown room_id: %s", room_id) continue event = event_from_pdu_json(p) From 30dd27afff3889293ec05b82c7f7e05d61c5b609 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 9 Nov 2018 11:34:45 +0000 Subject: [PATCH 26/32] Simplify to always drop events if server isn't in the room --- synapse/handlers/federation.py | 23 +++++++++-------------- 1 file changed, 9 insertions(+), 14 deletions(-) diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index 9ca5fd8724..e859ff5ffa 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -202,27 +202,22 @@ class FederationHandler(BaseHandler): self.room_queues[room_id].append((pdu, origin)) return - # If we're no longer in the room just ditch the event entirely. This - # is probably an old server that has come back and thinks we're still - # in the room (or we've been rejoined to the room by a state reset). + # If we're not in the room just ditch the event entirely. This is + # probably an old server that has come back and thinks we're still in + # the room (or we've been rejoined to the room by a state reset). # - # If we were never in the room then maybe our database got vaped and - # we should check if we *are* in fact in the room. If we are then we - # can magically rejoin the room. + # Note that if we were never in the room then we would have already + # dropped the event, since we wouldn't know the room version. is_in_room = yield self.auth.check_host_in_room( room_id, self.server_name ) if not is_in_room: - was_in_room = yield self.store.was_host_joined( - pdu.room_id, self.server_name, + logger.info( + "[%s %s] Ignoring PDU from %s as we're not in the room", + room_id, event_id, origin, ) - if was_in_room: - logger.info( - "[%s %s] Ignoring PDU from %s as we've left the room", - room_id, event_id, origin, - ) - defer.returnValue(None) + defer.returnValue(None) state = None auth_chain = [] From 4f93abd62d6800c00c9e943f75e19e09c98c8f30 Mon Sep 17 00:00:00 2001 From: David Baker Date: Fri, 9 Nov 2018 13:25:38 +0000 Subject: [PATCH 27/32] add docs --- synapse/storage/e2e_room_keys.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/synapse/storage/e2e_room_keys.py b/synapse/storage/e2e_room_keys.py index a1d203fd61..5d83707fa0 100644 --- a/synapse/storage/e2e_room_keys.py +++ b/synapse/storage/e2e_room_keys.py @@ -217,7 +217,10 @@ class EndToEndRoomKeyStore(SQLBaseStore): Raises: StoreError: with code 404 if there are no e2e_room_keys_versions present Returns: - A deferred dict giving the info metadata for this backup version + A deferred dict giving the info metadata for this backup version, with fields including: + version(str) + algorithm(str) + auth_data(object): opaque dict supplied by the client """ def _get_e2e_room_keys_version_info_txn(txn): From d44dea0223c16f86a57647e670b8a7651b93aa2a Mon Sep 17 00:00:00 2001 From: David Baker Date: Fri, 9 Nov 2018 14:38:31 +0000 Subject: [PATCH 28/32] pep8 --- synapse/storage/e2e_room_keys.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/synapse/storage/e2e_room_keys.py b/synapse/storage/e2e_room_keys.py index 5d83707fa0..16b7f005aa 100644 --- a/synapse/storage/e2e_room_keys.py +++ b/synapse/storage/e2e_room_keys.py @@ -217,7 +217,8 @@ class EndToEndRoomKeyStore(SQLBaseStore): Raises: StoreError: with code 404 if there are no e2e_room_keys_versions present Returns: - A deferred dict giving the info metadata for this backup version, with fields including: + A deferred dict giving the info metadata for this backup version, with + fields including: version(str) algorithm(str) auth_data(object): opaque dict supplied by the client From dc59ad533414d63a2b87da705d9792fca4ee8b36 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 9 Nov 2018 14:58:09 +0000 Subject: [PATCH 29/32] Remove hack to support rejoining rooms --- synapse/handlers/federation.py | 120 ++++++++++++--------------------- 1 file changed, 44 insertions(+), 76 deletions(-) diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index e859ff5ffa..a3bb864bb2 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -552,86 +552,54 @@ class FederationHandler(BaseHandler): room_id, event_id, event, ) - # FIXME (erikj): Awful hack to make the case where we are not currently - # in the room work - # If state and auth_chain are None, then we don't need to do this check - # as we already know we have enough state in the DB to handle this - # event. - if state and auth_chain and not event.internal_metadata.is_outlier(): - is_in_room = yield self.auth.check_host_in_room( - room_id, - self.server_name - ) - else: - is_in_room = True + event_ids = set() + if state: + event_ids |= {e.event_id for e in state} + if auth_chain: + event_ids |= {e.event_id for e in auth_chain} + + seen_ids = yield self.store.have_seen_events(event_ids) + + if state and auth_chain is not None: + # If we have any state or auth_chain given to us by the replication + # layer, then we should handle them (if we haven't before.) + + event_infos = [] + + for e in itertools.chain(auth_chain, state): + if e.event_id in seen_ids: + continue + e.internal_metadata.outlier = True + auth_ids = e.auth_event_ids() + auth = { + (e.type, e.state_key): e for e in auth_chain + if e.event_id in auth_ids or e.type == EventTypes.Create + } + event_infos.append({ + "event": e, + "auth_events": auth, + }) + seen_ids.add(e.event_id) - if not is_in_room: logger.info( - "[%s %s] Got event for room we're not in", - room_id, event_id, + "[%s %s] persisting newly-received auth/state events %s", + room_id, event_id, [e["event"].event_id for e in event_infos] ) + yield self._handle_new_events(origin, event_infos) - try: - yield self._persist_auth_tree( - origin, auth_chain, state, event - ) - except AuthError as e: - raise FederationError( - "ERROR", - e.code, - e.msg, - affected=event_id, - ) - - else: - event_ids = set() - if state: - event_ids |= {e.event_id for e in state} - if auth_chain: - event_ids |= {e.event_id for e in auth_chain} - - seen_ids = yield self.store.have_seen_events(event_ids) - - if state and auth_chain is not None: - # If we have any state or auth_chain given to us by the replication - # layer, then we should handle them (if we haven't before.) - - event_infos = [] - - for e in itertools.chain(auth_chain, state): - if e.event_id in seen_ids: - continue - e.internal_metadata.outlier = True - auth_ids = e.auth_event_ids() - auth = { - (e.type, e.state_key): e for e in auth_chain - if e.event_id in auth_ids or e.type == EventTypes.Create - } - event_infos.append({ - "event": e, - "auth_events": auth, - }) - seen_ids.add(e.event_id) - - logger.info( - "[%s %s] persisting newly-received auth/state events %s", - room_id, event_id, [e["event"].event_id for e in event_infos] - ) - yield self._handle_new_events(origin, event_infos) - - try: - context = yield self._handle_new_event( - origin, - event, - state=state, - ) - except AuthError as e: - raise FederationError( - "ERROR", - e.code, - e.msg, - affected=event.event_id, - ) + try: + context = yield self._handle_new_event( + origin, + event, + state=state, + ) + except AuthError as e: + raise FederationError( + "ERROR", + e.code, + e.msg, + affected=event.event_id, + ) room = yield self.store.get_room(room_id) From 83a5f459aad05905abe9f455cb98f3b6f8c25549 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Wed, 14 Nov 2018 10:21:07 +0000 Subject: [PATCH 30/32] Fix an internal server error when viewing the public privacy policy --- synapse/rest/consent/consent_resource.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/synapse/rest/consent/consent_resource.py b/synapse/rest/consent/consent_resource.py index 8009b7ff1c..53d5fab914 100644 --- a/synapse/rest/consent/consent_resource.py +++ b/synapse/rest/consent/consent_resource.py @@ -143,9 +143,9 @@ class ConsentResource(Resource): has_consented = False public_version = username == "" if not public_version or not self.hs.config.user_consent_at_registration: - userhmac = parse_string(request, "h", required=True, encoding=None) + userhmac_bytes = parse_string(request, "h", required=True, encoding=None) - self._check_hash(username, userhmac) + self._check_hash(username, userhmac_bytes) if username.startswith('@'): qualified_user_id = username @@ -155,15 +155,18 @@ class ConsentResource(Resource): u = yield self.store.get_user_by_id(qualified_user_id) if u is None: raise NotFoundError("Unknown user") + has_consented = u["consent_version"] == version + userhmac = userhmac_bytes.decode("ascii") try: self._render_template( request, "%s.html" % (version,), user=username, - userhmac=userhmac.decode('ascii'), + userhmac=userhmac, version=version, - has_consented=has_consented, public_version=public_version, + has_consented=has_consented, + public_version=public_version, ) except TemplateNotFound: raise NotFoundError("Unknown policy version") From c1efcd7c6a155cb6732428173f7e78df2ab4321d Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Wed, 14 Nov 2018 10:44:46 +0000 Subject: [PATCH 31/32] Add a test for the public T&Cs form --- synapse/rest/consent/consent_resource.py | 2 +- tests/rest/client/test_consent.py | 7 +++++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/synapse/rest/consent/consent_resource.py b/synapse/rest/consent/consent_resource.py index 53d5fab914..ad525b22e1 100644 --- a/synapse/rest/consent/consent_resource.py +++ b/synapse/rest/consent/consent_resource.py @@ -142,7 +142,7 @@ class ConsentResource(Resource): userhmac = None has_consented = False public_version = username == "" - if not public_version or not self.hs.config.user_consent_at_registration: + if not public_version: userhmac_bytes = parse_string(request, "h", required=True, encoding=None) self._check_hash(username, userhmac_bytes) diff --git a/tests/rest/client/test_consent.py b/tests/rest/client/test_consent.py index df3f1cde6e..4294bbec2a 100644 --- a/tests/rest/client/test_consent.py +++ b/tests/rest/client/test_consent.py @@ -60,6 +60,13 @@ class ConsentResourceTestCase(unittest.HomeserverTestCase): hs = self.setup_test_homeserver(config=config) return hs + def test_render_public_consent(self): + """You can observe the terms form without specifying a user""" + resource = consent_resource.ConsentResource(self.hs) + request, channel = self.make_request("GET", "/consent?v=1", shorthand=False) + render(request, resource, self.reactor) + self.assertEqual(channel.code, 200) + def test_accept_consent(self): """ A user can use the consent form to accept the terms. From 0c4dc6fd76397b8e73a1092d1fab38fa79341818 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff Date: Wed, 14 Nov 2018 10:48:08 +0000 Subject: [PATCH 32/32] changelog --- changelog.d/4184.feature | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog.d/4184.feature diff --git a/changelog.d/4184.feature b/changelog.d/4184.feature new file mode 100644 index 0000000000..89975f4c6e --- /dev/null +++ b/changelog.d/4184.feature @@ -0,0 +1 @@ +Include flags to optionally add `m.login.terms` to the registration flow when consent tracking is enabled.