cleanup so it can refer to late-config

synapse/app/homeserver.py

@@ -484,7 +484,7 @@ def phone_stats_home(hs, stats, stats_process=_stats_process):
     daily_sent_messages = yield hs.get_datastore().count_daily_sent_messages()
     stats["daily_sent_messages"] = daily_sent_messages
     stats["cache_factor"] = hs.config.caches.global_factor
-    stats["event_cache_size"] = hs.config.event_cache_size
+    stats["event_cache_size"] = hs.config.caches.event_cache_size
 
     #
     # Performance statistics

synapse/config/cache.py

@@ -53,6 +53,8 @@ class CacheConfig(Config):
     section = "caches"
 
     def read_config(self, config, **kwargs):
+        self.event_cache_size = self.parse_size(config.get("event_cache_size", "10K"))
+
         global DEFAULT_CACHE_SIZE_FACTOR
 
         cache_config = config.get("caches", {})

synapse/config/database.py

@@ -57,8 +57,6 @@ class DatabaseConfig(Config):
     section = "database"
 
     def read_config(self, config, **kwargs):
-        self.event_cache_size = self.parse_size(config.get("event_cache_size", "10K"))
-
         # We *experimentally* support specifying multiple databases via the
         # `databases` key. This is a map from a label to database config in the
         # same format as the `database` config option, plus an extra

synapse/storage/data_stores/main/events_worker.py

@@ -72,7 +72,7 @@ class EventsWorkerStore(SQLBaseStore):
         super(EventsWorkerStore, self).__init__(database, db_conn, hs)
 
         self._get_event_cache = Cache(
-            "*getEvent*", keylen=3, max_entries=hs.config.event_cache_size
+            "*getEvent*", keylen=3, max_entries=hs.config.caches.event_cache_size
         )
 
         self._event_fetch_lock = threading.Condition()

synapse/storage/data_stores/main/relations.py

@@ -31,7 +31,7 @@ logger = logging.getLogger(__name__)
 
 
 class RelationsWorkerStore(SQLBaseStore):
-    @cached(tree=True)
+    @cached(tree=True, max_entries="event_cache_size")
     def get_relations_for_event(
         self,
         event_id,
@@ -133,7 +133,7 @@ class RelationsWorkerStore(SQLBaseStore):
             "get_recent_references_for_event", _get_recent_references_for_event_txn
         )
 
-    @cached(tree=True)
+    @cached(tree=True, max_entries="event_cache_size")
     def get_aggregation_groups_for_event(
         self,
         event_id,

synapse/storage/data_stores/state/store.py

@@ -28,7 +28,6 @@ from synapse.storage.data_stores.state.bg_updates import StateBackgroundUpdateSt
 from synapse.storage.database import Database
 from synapse.storage.state import StateFilter
 from synapse.types import StateMap
-from synapse.util.caches import get_cache_factor_for
 from synapse.util.caches.descriptors import cached
 from synapse.util.caches.dictionary_cache import DictionaryCache
 
@@ -90,11 +89,10 @@ class StateGroupDataStore(StateBackgroundUpdateStore, SQLBaseStore):
         self._state_group_cache = DictionaryCache(
             "*stateGroupCache*",
             # TODO: this hasn't been tuned yet
-            50000 * get_cache_factor_for("stateGroupCache"),
+            50000,
         )
         self._state_group_members_cache = DictionaryCache(
-            "*stateGroupMembersCache*",
-            500000 * get_cache_factor_for("stateGroupMembersCache"),
+            "*stateGroupMembersCache*", 500000,
         )
 
     @cached(max_entries=10000, iterable=True)

synapse/util/caches/descriptors.py

@@ -377,7 +377,10 @@ class CacheDescriptor(_CacheDescriptorBase):
         self.tree = tree
         self.iterable = iterable
 
-    def __get__(self, obj, objtype=None):
+    def __get__(self, obj, owner):
+        if isinstance(self.max_entries, str):
+            self.max_entries = getattr(obj.hs.config.caches, self.max_entries)
+
         cache = Cache(
             name=self.orig.__name__,
             max_entries=self.max_entries,

tests/storage/test_appservice.py

@@ -43,7 +43,7 @@ class ApplicationServiceStoreTestCase(unittest.TestCase):
         )
 
         hs.config.app_service_config_files = self.as_yaml_files
-        hs.config.event_cache_size = 1
+        hs.config.caches.event_cache_size = 1
         hs.config.password_providers = []
 
         self.as_token = "token1"
@@ -110,7 +110,7 @@ class ApplicationServiceTransactionStoreTestCase(unittest.TestCase):
         )
 
         hs.config.app_service_config_files = self.as_yaml_files
-        hs.config.event_cache_size = 1
+        hs.config.caches.event_cache_size = 1
         hs.config.password_providers = []
 
         self.as_list = [
@@ -422,7 +422,7 @@ class ApplicationServiceStoreConfigTestCase(unittest.TestCase):
         )
 
         hs.config.app_service_config_files = [f1, f2]
-        hs.config.event_cache_size = 1
+        hs.config.caches.event_cache_size = 1
         hs.config.password_providers = []
 
         database = hs.get_datastores().databases[0]
@@ -440,7 +440,7 @@ class ApplicationServiceStoreConfigTestCase(unittest.TestCase):
         )
 
         hs.config.app_service_config_files = [f1, f2]
-        hs.config.event_cache_size = 1
+        hs.config.caches.event_cache_size = 1
         hs.config.password_providers = []
 
         with self.assertRaises(ConfigError) as cm:
@@ -464,7 +464,7 @@ class ApplicationServiceStoreConfigTestCase(unittest.TestCase):
         )
 
         hs.config.app_service_config_files = [f1, f2]
-        hs.config.event_cache_size = 1
+        hs.config.caches.event_cache_size = 1
         hs.config.password_providers = []
 
         with self.assertRaises(ConfigError) as cm:

tests/storage/test_base.py

@@ -51,7 +51,8 @@ class SQLBaseStoreTestCase(unittest.TestCase):
 
         config = Mock()
         config._disable_native_upserts = True
-        config.event_cache_size = 1
+        config.caches = Mock()
+        config.caches.event_cache_size = 1
         hs = TestHomeServer("test", config=config)
 
         sqlite_config = {"name": "sqlite3"}

tests/util/test_lrucache.py

@@ -84,7 +84,7 @@ class LruCacheTestCase(unittest.HomeserverTestCase):
         self.assertEquals(len(cache), 0)
 
 
-class LruCacheCallbacksTestCase(unittest.TestCase):
+class LruCacheCallbacksTestCase(unittest.HomeserverTestCase):
     def test_get(self):
         m = Mock()
         cache = LruCache(1)
@@ -233,7 +233,7 @@ class LruCacheCallbacksTestCase(unittest.TestCase):
         self.assertEquals(m3.call_count, 1)
 
 
-class LruCacheSizedTestCase(unittest.TestCase):
+class LruCacheSizedTestCase(unittest.HomeserverTestCase):
     def test_evict(self):
         cache = LruCache(5, size_callback=len)
         cache["key1"] = [0]