From 6b95e35e969f730f7f92f6c3f814691c4bedaeff Mon Sep 17 00:00:00 2001 From: Krombel Date: Thu, 11 May 2017 16:05:30 +0200 Subject: [PATCH 1/7] add check to only add a new filter if the same filter does not exist previously Signed-off-by: Matthias Kesler --- synapse/storage/filtering.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/synapse/storage/filtering.py b/synapse/storage/filtering.py index a2ccc66ea7..e7ec43cf97 100644 --- a/synapse/storage/filtering.py +++ b/synapse/storage/filtering.py @@ -51,6 +51,15 @@ class FilteringStore(SQLBaseStore): # Need an atomic transaction to SELECT the maximal ID so far then # INSERT a new one def _do_txn(txn): + sql = ( + "SELECT filter_id FROM user_filters " + "WHERE user_id = ? AND filter_json = ?" + ) + txn.execute(sql, (user_localpart,def_json)) + filter_id = txn.fetchone()[0] + if filter_id is not None: + return filter_id + sql = ( "SELECT MAX(filter_id) FROM user_filters " "WHERE user_id = ?" From eb7cbf27bc8723f5761f0b8bb4c57a7ceb0fc3e5 Mon Sep 17 00:00:00 2001 From: Krombel Date: Fri, 12 May 2017 12:09:42 +0200 Subject: [PATCH 2/7] insert whitespace to fix travis build --- synapse/storage/filtering.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/storage/filtering.py b/synapse/storage/filtering.py index e7ec43cf97..9101f57288 100644 --- a/synapse/storage/filtering.py +++ b/synapse/storage/filtering.py @@ -55,7 +55,7 @@ class FilteringStore(SQLBaseStore): "SELECT filter_id FROM user_filters " "WHERE user_id = ? AND filter_json = ?" ) - txn.execute(sql, (user_localpart,def_json)) + txn.execute(sql, (user_localpart, def_json)) filter_id = txn.fetchone()[0] if filter_id is not None: return filter_id From 64953c8ed235b38a31f7b2909493e63267bd2cfd Mon Sep 17 00:00:00 2001 From: Krombel Date: Mon, 15 May 2017 18:36:37 +0200 Subject: [PATCH 3/7] avoid access-error if no filter_id matches --- synapse/storage/filtering.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/synapse/storage/filtering.py b/synapse/storage/filtering.py index 9101f57288..73f7a078fa 100644 --- a/synapse/storage/filtering.py +++ b/synapse/storage/filtering.py @@ -56,9 +56,9 @@ class FilteringStore(SQLBaseStore): "WHERE user_id = ? AND filter_json = ?" ) txn.execute(sql, (user_localpart, def_json)) - filter_id = txn.fetchone()[0] - if filter_id is not None: - return filter_id + filter_id_response = txn.fetchone() + if filter_id_response is not None: + return filter_id_response[0] sql = ( "SELECT MAX(filter_id) FROM user_filters " From 1217c7da911fe534b7804088bedc375854828ad1 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 21 Jun 2017 12:00:41 +0100 Subject: [PATCH 4/7] Don't work out users who share room with appservice users --- synapse/handlers/user_directory.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/synapse/handlers/user_directory.py b/synapse/handlers/user_directory.py index d33a20a1f2..c7a3acafcb 100644 --- a/synapse/handlers/user_directory.py +++ b/synapse/handlers/user_directory.py @@ -205,6 +205,10 @@ class UserDirectoyHandler(object): count += 1 continue + if self.store.get_if_app_services_interested_in_user(user_id): + count += 1 + continue + for other_user_id in user_ids: if user_id == other_user_id: continue @@ -411,8 +415,10 @@ class UserDirectoyHandler(object): to_insert = set() to_update = set() + is_appservice = self.store.get_if_app_services_interested_in_user(user_id) + # First, if they're our user then we need to update for every user - if self.is_mine_id(user_id): + if self.is_mine_id(user_id) and not is_appservice: # Returns a map of other_user_id -> shared_private. We only need # to update mappings if for users that either don't share a room # already (aren't in the map) or, if the room is private, those that @@ -443,7 +449,10 @@ class UserDirectoyHandler(object): if user_id == other_user_id: continue - if self.is_mine_id(other_user_id): + is_appservice = self.store.get_if_app_services_interested_in_user( + other_user_id + ) + if self.is_mine_id(other_user_id) and not is_appservice: shared_is_private = yield self.store.get_if_users_share_a_room( other_user_id, user_id, ) From 812c030e87f4a31e13496dbf2fa0c2e61aa918d2 Mon Sep 17 00:00:00 2001 From: Krombel Date: Wed, 21 Jun 2017 14:48:12 +0200 Subject: [PATCH 5/7] replaced json.dumps with encode_canonical_json --- synapse/storage/filtering.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/synapse/storage/filtering.py b/synapse/storage/filtering.py index 73f7a078fa..78b1e30945 100644 --- a/synapse/storage/filtering.py +++ b/synapse/storage/filtering.py @@ -19,6 +19,7 @@ from ._base import SQLBaseStore from synapse.api.errors import SynapseError, Codes from synapse.util.caches.descriptors import cachedInlineCallbacks +from canonicaljson import encode_canonical_json import simplejson as json @@ -46,7 +47,7 @@ class FilteringStore(SQLBaseStore): defer.returnValue(json.loads(str(def_json).decode("utf-8"))) def add_user_filter(self, user_localpart, user_filter): - def_json = json.dumps(user_filter).encode("utf-8") + def_json = encode_canonical_json(user_filter) # Need an atomic transaction to SELECT the maximal ID so far then # INSERT a new one From 8177563ebed92382a3d08943c8a1a8c5f3c28654 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 21 Jun 2017 13:51:38 +0100 Subject: [PATCH 6/7] Fix for workers --- synapse/app/user_dir.py | 2 +- synapse/replication/slave/storage/appservice.py | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/synapse/app/user_dir.py b/synapse/app/user_dir.py index 9d8edaa8e3..6d2aebe8de 100644 --- a/synapse/app/user_dir.py +++ b/synapse/app/user_dir.py @@ -25,8 +25,8 @@ from synapse.http.site import SynapseSite from synapse.http.server import JsonResource from synapse.metrics.resource import MetricsResource, METRICS_PREFIX from synapse.replication.slave.storage._base import BaseSlavedStore -from synapse.replication.slave.storage.events import SlavedEventStore from synapse.replication.slave.storage.appservice import SlavedApplicationServiceStore +from synapse.replication.slave.storage.events import SlavedEventStore from synapse.replication.slave.storage.registration import SlavedRegistrationStore from synapse.replication.tcp.client import ReplicationClientHandler from synapse.rest.client.v2_alpha import user_directory diff --git a/synapse/replication/slave/storage/appservice.py b/synapse/replication/slave/storage/appservice.py index a374f2f1a2..73048327c5 100644 --- a/synapse/replication/slave/storage/appservice.py +++ b/synapse/replication/slave/storage/appservice.py @@ -38,3 +38,6 @@ class SlavedApplicationServiceStore(BaseSlavedStore): get_appservice_state = DataStore.get_appservice_state.__func__ set_appservice_last_pos = DataStore.set_appservice_last_pos.__func__ set_appservice_state = DataStore.set_appservice_state.__func__ + get_if_app_services_interested_in_user = ( + DataStore.get_if_app_services_interested_in_user.__func__ + ) From dae9a00a28cbd9e40be521effb25b0b2528caa4f Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 21 Jun 2017 14:19:33 +0100 Subject: [PATCH 7/7] Initialise exclusive_user_regex --- .../replication/slave/storage/appservice.py | 2 ++ synapse/storage/appservice.py | 35 +++++++++++-------- 2 files changed, 22 insertions(+), 15 deletions(-) diff --git a/synapse/replication/slave/storage/appservice.py b/synapse/replication/slave/storage/appservice.py index 73048327c5..0d3f31a50c 100644 --- a/synapse/replication/slave/storage/appservice.py +++ b/synapse/replication/slave/storage/appservice.py @@ -16,6 +16,7 @@ from ._base import BaseSlavedStore from synapse.storage import DataStore from synapse.config.appservice import load_appservices +from synapse.storage.appservice import _make_exclusive_regex class SlavedApplicationServiceStore(BaseSlavedStore): @@ -25,6 +26,7 @@ class SlavedApplicationServiceStore(BaseSlavedStore): hs.config.server_name, hs.config.app_service_config_files ) + self.exclusive_user_regex = _make_exclusive_regex(self.services_cache) get_app_service_by_token = DataStore.get_app_service_by_token.__func__ get_app_service_by_user_id = DataStore.get_app_service_by_user_id.__func__ diff --git a/synapse/storage/appservice.py b/synapse/storage/appservice.py index 532df736a5..c63935cb07 100644 --- a/synapse/storage/appservice.py +++ b/synapse/storage/appservice.py @@ -27,6 +27,25 @@ from ._base import SQLBaseStore logger = logging.getLogger(__name__) +def _make_exclusive_regex(services_cache): + # We precompie a regex constructed from all the regexes that the AS's + # have registered for exclusive users. + exclusive_user_regexes = [ + regex.pattern + for service in services_cache + for regex in service.get_exlusive_user_regexes() + ] + if exclusive_user_regexes: + exclusive_user_regex = "|".join("(" + r + ")" for r in exclusive_user_regexes) + exclusive_user_regex = re.compile(exclusive_user_regex) + else: + # We handle this case specially otherwise the constructed regex + # will always match + exclusive_user_regex = None + + return exclusive_user_regex + + class ApplicationServiceStore(SQLBaseStore): def __init__(self, hs): @@ -36,21 +55,7 @@ class ApplicationServiceStore(SQLBaseStore): hs.hostname, hs.config.app_service_config_files ) - - # We precompie a regex constructed from all the regexes that the AS's - # have registered for exclusive users. - exclusive_user_regexes = [ - regex.pattern - for service in self.services_cache - for regex in service.get_exlusive_user_regexes() - ] - if exclusive_user_regexes: - exclusive_user_regex = "|".join("(" + r + ")" for r in exclusive_user_regexes) - self.exclusive_user_regex = re.compile(exclusive_user_regex) - else: - # We handle this case specially otherwise the constructed regex - # will always match - self.exclusive_user_regex = None + self.exclusive_user_regex = _make_exclusive_regex(self.services_cache) def get_app_services(self): return self.services_cache