From 945e22303c3dd5ca89b84292ec0a86d887ebaa1c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 15 Jul 2025 11:47:59 +0100 Subject: [PATCH 001/185] Bump phonenumbers from 9.0.8 to 9.0.9 (#18681) --- poetry.lock | 70 ++++++++++++++++++++++++++++++++++------------------- 1 file changed, 45 insertions(+), 25 deletions(-) diff --git a/poetry.lock b/poetry.lock index 2218b36e71..711dd2f016 100644 --- a/poetry.lock +++ b/poetry.lock @@ -39,7 +39,7 @@ description = "The ultimate Python library in building OAuth and OpenID Connect optional = true python-versions = ">=3.9" groups = ["main"] -markers = "extra == \"oidc\" or extra == \"jwt\" or extra == \"all\"" +markers = "extra == \"all\" or extra == \"jwt\" or extra == \"oidc\"" files = [ {file = "authlib-1.6.0-py2.py3-none-any.whl", hash = "sha256:91685589498f79e8655e8a8947431ad6288831d643f11c55c2143ffcc738048d"}, {file = "authlib-1.6.0.tar.gz", hash = "sha256:4367d32031b7af175ad3a323d571dc7257b7099d55978087ceae4a0d88cd3210"}, @@ -435,7 +435,7 @@ description = "XML bomb protection for Python stdlib modules" optional = true python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" groups = ["main"] -markers = "extra == \"saml2\" or extra == \"all\"" +markers = "extra == \"all\" or extra == \"saml2\"" files = [ {file = "defusedxml-0.7.1-py2.py3-none-any.whl", hash = "sha256:a352e7e428770286cc899e2542b6cdaedb2b4953ff269a210103ec58f6198a61"}, {file = "defusedxml-0.7.1.tar.gz", hash = "sha256:1bb3032db185915b62d7c6209c5a8792be6a32ab2fedacc84e01b52c51aa3e69"}, @@ -478,7 +478,7 @@ description = "XPath 1.0/2.0/3.0/3.1 parsers and selectors for ElementTree and l optional = true python-versions = ">=3.7" groups = ["main"] -markers = "extra == \"saml2\" or extra == \"all\"" +markers = "extra == \"all\" or extra == \"saml2\"" files = [ {file = "elementpath-4.1.5-py3-none-any.whl", hash = "sha256:2ac1a2fb31eb22bbbf817f8cf6752f844513216263f0e3892c8e79782fe4bb55"}, {file = "elementpath-4.1.5.tar.gz", hash = "sha256:c2d6dc524b29ef751ecfc416b0627668119d8812441c555d7471da41d4bacb8d"}, @@ -528,7 +528,7 @@ description = "Python wrapper for hiredis" optional = true python-versions = ">=3.8" groups = ["main"] -markers = "extra == \"redis\" or extra == \"all\"" +markers = "extra == \"all\" or extra == \"redis\"" files = [ {file = "hiredis-3.2.1-cp310-cp310-macosx_10_15_universal2.whl", hash = "sha256:add17efcbae46c5a6a13b244ff0b4a8fa079602ceb62290095c941b42e9d5dec"}, {file = "hiredis-3.2.1-cp310-cp310-macosx_10_15_x86_64.whl", hash = "sha256:5fe955cc4f66c57df1ae8e5caf4de2925d43b5efab4e40859662311d1bcc5f54"}, @@ -865,7 +865,7 @@ description = "Jaeger Python OpenTracing Tracer implementation" optional = true python-versions = ">=3.7" groups = ["main"] -markers = "extra == \"opentracing\" or extra == \"all\"" +markers = "extra == \"all\" or extra == \"opentracing\"" files = [ {file = "jaeger-client-4.8.0.tar.gz", hash = "sha256:3157836edab8e2c209bd2d6ae61113db36f7ee399e66b1dcbb715d87ab49bfe0"}, ] @@ -1003,7 +1003,7 @@ description = "A strictly RFC 4510 conforming LDAP V3 pure Python client library optional = true python-versions = "*" groups = ["main"] -markers = "extra == \"matrix-synapse-ldap3\" or extra == \"all\"" +markers = "extra == \"all\" or extra == \"matrix-synapse-ldap3\"" files = [ {file = "ldap3-2.9.1-py2.py3-none-any.whl", hash = "sha256:5869596fc4948797020d3f03b7939da938778a0f9e2009f7a072ccf92b8e8d70"}, {file = "ldap3-2.9.1.tar.gz", hash = "sha256:f3e7fc4718e3f09dda568b57100095e0ce58633bcabbed8667ce3f8fbaa4229f"}, @@ -1019,7 +1019,7 @@ description = "Powerful and Pythonic XML processing library combining libxml2/li optional = true python-versions = ">=3.6" groups = ["main"] -markers = "extra == \"url-preview\" or extra == \"all\"" +markers = "extra == \"all\" or extra == \"url-preview\"" files = [ {file = "lxml-5.4.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:e7bc6df34d42322c5289e37e9971d6ed114e3776b45fa879f734bded9d1fea9c"}, {file = "lxml-5.4.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:6854f8bd8a1536f8a1d9a3655e6354faa6406621cf857dc27b681b69860645c7"}, @@ -1299,7 +1299,7 @@ description = "An LDAP3 auth provider for Synapse" optional = true python-versions = ">=3.7" groups = ["main"] -markers = "extra == \"matrix-synapse-ldap3\" or extra == \"all\"" +markers = "extra == \"all\" or extra == \"matrix-synapse-ldap3\"" files = [ {file = "matrix-synapse-ldap3-0.3.0.tar.gz", hash = "sha256:8bb6517173164d4b9cc44f49de411d8cebdb2e705d5dd1ea1f38733c4a009e1d"}, {file = "matrix_synapse_ldap3-0.3.0-py3-none-any.whl", hash = "sha256:8b4d701f8702551e98cc1d8c20dbed532de5613584c08d0df22de376ba99159d"}, @@ -1531,7 +1531,7 @@ description = "OpenTracing API for Python. See documentation at http://opentraci optional = true python-versions = "*" groups = ["main"] -markers = "extra == \"opentracing\" or extra == \"all\"" +markers = "extra == \"all\" or extra == \"opentracing\"" files = [ {file = "opentracing-2.4.0.tar.gz", hash = "sha256:a173117e6ef580d55874734d1fa7ecb6f3655160b8b8974a2a1e98e5ec9c840d"}, ] @@ -1568,14 +1568,14 @@ dev = ["jinja2"] [[package]] name = "phonenumbers" -version = "9.0.8" +version = "9.0.9" description = "Python version of Google's common library for parsing, formatting, storing and validating international phone numbers." optional = false python-versions = "*" groups = ["main"] files = [ - {file = "phonenumbers-9.0.8-py2.py3-none-any.whl", hash = "sha256:53d357111c0ead0d6408ae443613b18d3a053431ca1ddf7e881457c0969afcf9"}, - {file = "phonenumbers-9.0.8.tar.gz", hash = "sha256:16f03f2cf65b5eee99ed25827d810febcab92b5d76f977e425fcd2e4ca6d4865"}, + {file = "phonenumbers-9.0.9-py2.py3-none-any.whl", hash = "sha256:13b91aa153f87675902829b38a556bad54824f9c121b89588bbb5fa8550d97ef"}, + {file = "phonenumbers-9.0.9.tar.gz", hash = "sha256:c640545019a07e68b0bea57a5fede6eef45c7391165d28935f45615f9a567a5b"}, ] [[package]] @@ -1588,6 +1588,8 @@ groups = ["main"] files = [ {file = "pillow-11.3.0-cp310-cp310-macosx_10_10_x86_64.whl", hash = "sha256:1b9c17fd4ace828b3003dfd1e30bff24863e0eb59b535e8f80194d9cc7ecf860"}, {file = "pillow-11.3.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:65dc69160114cdd0ca0f35cb434633c75e8e7fad4cf855177a05bf38678f73ad"}, + {file = "pillow-11.3.0-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:7107195ddc914f656c7fc8e4a5e1c25f32e9236ea3ea860f257b0436011fddd0"}, + {file = "pillow-11.3.0-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:cc3e831b563b3114baac7ec2ee86819eb03caa1a2cef0b481a5675b59c4fe23b"}, {file = "pillow-11.3.0-cp310-cp310-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f1f182ebd2303acf8c380a54f615ec883322593320a9b00438eb842c1f37ae50"}, {file = "pillow-11.3.0-cp310-cp310-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:4445fa62e15936a028672fd48c4c11a66d641d2c05726c7ec1f8ba6a572036ae"}, {file = "pillow-11.3.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:71f511f6b3b91dd543282477be45a033e4845a40278fa8dcdbfdb07109bf18f9"}, @@ -1597,6 +1599,8 @@ files = [ {file = "pillow-11.3.0-cp310-cp310-win_arm64.whl", hash = "sha256:819931d25e57b513242859ce1876c58c59dc31587847bf74cfe06b2e0cb22d2f"}, {file = "pillow-11.3.0-cp311-cp311-macosx_10_10_x86_64.whl", hash = "sha256:1cd110edf822773368b396281a2293aeb91c90a2db00d78ea43e7e861631b722"}, {file = "pillow-11.3.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:9c412fddd1b77a75aa904615ebaa6001f169b26fd467b4be93aded278266b288"}, + {file = "pillow-11.3.0-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:7d1aa4de119a0ecac0a34a9c8bde33f34022e2e8f99104e47a3ca392fd60e37d"}, + {file = "pillow-11.3.0-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:91da1d88226663594e3f6b4b8c3c8d85bd504117d043740a8e0ec449087cc494"}, {file = "pillow-11.3.0-cp311-cp311-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:643f189248837533073c405ec2f0bb250ba54598cf80e8c1e043381a60632f58"}, {file = "pillow-11.3.0-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:106064daa23a745510dabce1d84f29137a37224831d88eb4ce94bb187b1d7e5f"}, {file = "pillow-11.3.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:cd8ff254faf15591e724dc7c4ddb6bf4793efcbe13802a4ae3e863cd300b493e"}, @@ -1606,6 +1610,8 @@ files = [ {file = "pillow-11.3.0-cp311-cp311-win_arm64.whl", hash = "sha256:30807c931ff7c095620fe04448e2c2fc673fcbb1ffe2a7da3fb39613489b1ddd"}, {file = "pillow-11.3.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:fdae223722da47b024b867c1ea0be64e0df702c5e0a60e27daad39bf960dd1e4"}, {file = "pillow-11.3.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:921bd305b10e82b4d1f5e802b6850677f965d8394203d182f078873851dada69"}, + {file = "pillow-11.3.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:eb76541cba2f958032d79d143b98a3a6b3ea87f0959bbe256c0b5e416599fd5d"}, + {file = "pillow-11.3.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:67172f2944ebba3d4a7b54f2e95c786a3a50c21b88456329314caaa28cda70f6"}, {file = "pillow-11.3.0-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:97f07ed9f56a3b9b5f49d3661dc9607484e85c67e27f3e8be2c7d28ca032fec7"}, {file = "pillow-11.3.0-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:676b2815362456b5b3216b4fd5bd89d362100dc6f4945154ff172e206a22c024"}, {file = "pillow-11.3.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:3e184b2f26ff146363dd07bde8b711833d7b0202e27d13540bfe2e35a323a809"}, @@ -1618,6 +1624,8 @@ files = [ {file = "pillow-11.3.0-cp313-cp313-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:7859a4cc7c9295f5838015d8cc0a9c215b77e43d07a25e460f35cf516df8626f"}, {file = "pillow-11.3.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:ec1ee50470b0d050984394423d96325b744d55c701a439d2bd66089bff963d3c"}, {file = "pillow-11.3.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:7db51d222548ccfd274e4572fdbf3e810a5e66b00608862f947b163e613b67dd"}, + {file = "pillow-11.3.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:2d6fcc902a24ac74495df63faad1884282239265c6839a0a6416d33faedfae7e"}, + {file = "pillow-11.3.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:f0f5d8f4a08090c6d6d578351a2b91acf519a54986c055af27e7a93feae6d3f1"}, {file = "pillow-11.3.0-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c37d8ba9411d6003bba9e518db0db0c58a680ab9fe5179f040b0463644bc9805"}, {file = "pillow-11.3.0-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:13f87d581e71d9189ab21fe0efb5a23e9f28552d5be6979e84001d3b8505abe8"}, {file = "pillow-11.3.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:023f6d2d11784a465f09fd09a34b150ea4672e85fb3d05931d89f373ab14abb2"}, @@ -1627,6 +1635,8 @@ files = [ {file = "pillow-11.3.0-cp313-cp313-win_arm64.whl", hash = "sha256:1904e1264881f682f02b7f8167935cce37bc97db457f8e7849dc3a6a52b99580"}, {file = "pillow-11.3.0-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:4c834a3921375c48ee6b9624061076bc0a32a60b5532b322cc0ea64e639dd50e"}, {file = "pillow-11.3.0-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:5e05688ccef30ea69b9317a9ead994b93975104a677a36a8ed8106be9260aa6d"}, + {file = "pillow-11.3.0-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:1019b04af07fc0163e2810167918cb5add8d74674b6267616021ab558dc98ced"}, + {file = "pillow-11.3.0-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:f944255db153ebb2b19c51fe85dd99ef0ce494123f21b9db4877ffdfc5590c7c"}, {file = "pillow-11.3.0-cp313-cp313t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1f85acb69adf2aaee8b7da124efebbdb959a104db34d3a2cb0f3793dbae422a8"}, {file = "pillow-11.3.0-cp313-cp313t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:05f6ecbeff5005399bb48d198f098a9b4b6bdf27b8487c7f38ca16eeb070cd59"}, {file = "pillow-11.3.0-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:a7bc6e6fd0395bc052f16b1a8670859964dbd7003bd0af2ff08342eb6e442cfe"}, @@ -1636,6 +1646,8 @@ files = [ {file = "pillow-11.3.0-cp313-cp313t-win_arm64.whl", hash = "sha256:8797edc41f3e8536ae4b10897ee2f637235c94f27404cac7297f7b607dd0716e"}, {file = "pillow-11.3.0-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:d9da3df5f9ea2a89b81bb6087177fb1f4d1c7146d583a3fe5c672c0d94e55e12"}, {file = "pillow-11.3.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:0b275ff9b04df7b640c59ec5a3cb113eefd3795a8df80bac69646ef699c6981a"}, + {file = "pillow-11.3.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:0743841cabd3dba6a83f38a92672cccbd69af56e3e91777b0ee7f4dba4385632"}, + {file = "pillow-11.3.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:2465a69cf967b8b49ee1b96d76718cd98c4e925414ead59fdf75cf0fd07df673"}, {file = "pillow-11.3.0-cp314-cp314-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:41742638139424703b4d01665b807c6468e23e699e8e90cffefe291c5832b027"}, {file = "pillow-11.3.0-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:93efb0b4de7e340d99057415c749175e24c8864302369e05914682ba642e5d77"}, {file = "pillow-11.3.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:7966e38dcd0fa11ca390aed7c6f20454443581d758242023cf36fcb319b1a874"}, @@ -1645,6 +1657,8 @@ files = [ {file = "pillow-11.3.0-cp314-cp314-win_arm64.whl", hash = "sha256:155658efb5e044669c08896c0c44231c5e9abcaadbc5cd3648df2f7c0b96b9a6"}, {file = "pillow-11.3.0-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:59a03cdf019efbfeeed910bf79c7c93255c3d54bc45898ac2a4140071b02b4ae"}, {file = "pillow-11.3.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:f8a5827f84d973d8636e9dc5764af4f0cf2318d26744b3d902931701b0d46653"}, + {file = "pillow-11.3.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:ee92f2fd10f4adc4b43d07ec5e779932b4eb3dbfbc34790ada5a6669bc095aa6"}, + {file = "pillow-11.3.0-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:c96d333dcf42d01f47b37e0979b6bd73ec91eae18614864622d9b87bbd5bbf36"}, {file = "pillow-11.3.0-cp314-cp314t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4c96f993ab8c98460cd0c001447bff6194403e8b1d7e149ade5f00594918128b"}, {file = "pillow-11.3.0-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:41342b64afeba938edb034d122b2dda5db2139b9a4af999729ba8818e0056477"}, {file = "pillow-11.3.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:068d9c39a2d1b358eb9f245ce7ab1b5c3246c7c8c7d9ba58cfa5b43146c06e50"}, @@ -1654,6 +1668,8 @@ files = [ {file = "pillow-11.3.0-cp314-cp314t-win_arm64.whl", hash = "sha256:79ea0d14d3ebad43ec77ad5272e6ff9bba5b679ef73375ea760261207fa8e0aa"}, {file = "pillow-11.3.0-cp39-cp39-macosx_10_10_x86_64.whl", hash = "sha256:48d254f8a4c776de343051023eb61ffe818299eeac478da55227d96e241de53f"}, {file = "pillow-11.3.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:7aee118e30a4cf54fdd873bd3a29de51e29105ab11f9aad8c32123f58c8f8081"}, + {file = "pillow-11.3.0-cp39-cp39-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:23cff760a9049c502721bdb743a7cb3e03365fafcdfc2ef9784610714166e5a4"}, + {file = "pillow-11.3.0-cp39-cp39-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:6359a3bc43f57d5b375d1ad54a0074318a0844d11b76abccf478c37c986d3cfc"}, {file = "pillow-11.3.0-cp39-cp39-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:092c80c76635f5ecb10f3f83d76716165c96f5229addbd1ec2bdbbda7d496e06"}, {file = "pillow-11.3.0-cp39-cp39-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:cadc9e0ea0a2431124cde7e1697106471fc4c1da01530e679b2391c37d3fbb3a"}, {file = "pillow-11.3.0-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:6a418691000f2a418c9135a7cf0d797c1bb7d9a485e61fe8e7722845b95ef978"}, @@ -1663,11 +1679,15 @@ files = [ {file = "pillow-11.3.0-cp39-cp39-win_arm64.whl", hash = "sha256:6abdbfd3aea42be05702a8dd98832329c167ee84400a1d1f61ab11437f1717eb"}, {file = "pillow-11.3.0-pp310-pypy310_pp73-macosx_10_15_x86_64.whl", hash = "sha256:3cee80663f29e3843b68199b9d6f4f54bd1d4a6b59bdd91bceefc51238bcb967"}, {file = "pillow-11.3.0-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:b5f56c3f344f2ccaf0dd875d3e180f631dc60a51b314295a3e681fe8cf851fbe"}, + {file = "pillow-11.3.0-pp310-pypy310_pp73-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:e67d793d180c9df62f1f40aee3accca4829d3794c95098887edc18af4b8b780c"}, + {file = "pillow-11.3.0-pp310-pypy310_pp73-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:d000f46e2917c705e9fb93a3606ee4a819d1e3aa7a9b442f6444f07e77cf5e25"}, {file = "pillow-11.3.0-pp310-pypy310_pp73-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:527b37216b6ac3a12d7838dc3bd75208ec57c1c6d11ef01902266a5a0c14fc27"}, {file = "pillow-11.3.0-pp310-pypy310_pp73-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:be5463ac478b623b9dd3937afd7fb7ab3d79dd290a28e2b6df292dc75063eb8a"}, {file = "pillow-11.3.0-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:8dc70ca24c110503e16918a658b869019126ecfe03109b754c402daff12b3d9f"}, {file = "pillow-11.3.0-pp311-pypy311_pp73-macosx_10_15_x86_64.whl", hash = "sha256:7c8ec7a017ad1bd562f93dbd8505763e688d388cde6e4a010ae1486916e713e6"}, {file = "pillow-11.3.0-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:9ab6ae226de48019caa8074894544af5b53a117ccb9d3b3dcb2871464c829438"}, + {file = "pillow-11.3.0-pp311-pypy311_pp73-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:fe27fb049cdcca11f11a7bfda64043c37b30e6b91f10cb5bab275806c32f6ab3"}, + {file = "pillow-11.3.0-pp311-pypy311_pp73-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:465b9e8844e3c3519a983d58b80be3f668e2a7a5db97f2784e7079fbc9f9822c"}, {file = "pillow-11.3.0-pp311-pypy311_pp73-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5418b53c0d59b3824d05e029669efa023bbef0f3e92e75ec8428f3799487f361"}, {file = "pillow-11.3.0-pp311-pypy311_pp73-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:504b6f59505f08ae014f724b6207ff6222662aab5cc9542577fb084ed0676ac7"}, {file = "pillow-11.3.0-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:c84d689db21a1c397d001aa08241044aa2069e7587b398c8cc63020390b1c1b8"}, @@ -1705,7 +1725,7 @@ description = "psycopg2 - Python-PostgreSQL Database Adapter" optional = true python-versions = ">=3.8" groups = ["main"] -markers = "extra == \"postgres\" or extra == \"all\"" +markers = "extra == \"all\" or extra == \"postgres\"" files = [ {file = "psycopg2-2.9.10-cp310-cp310-win32.whl", hash = "sha256:5df2b672140f95adb453af93a7d669d7a7bf0a56bcd26f1502329166f4a61716"}, {file = "psycopg2-2.9.10-cp310-cp310-win_amd64.whl", hash = "sha256:c6f7b8561225f9e711a9c47087388a97fdc948211c10a4bccbf0ba68ab7b3b5a"}, @@ -1726,7 +1746,7 @@ description = ".. image:: https://travis-ci.org/chtd/psycopg2cffi.svg?branch=mas optional = true python-versions = "*" groups = ["main"] -markers = "platform_python_implementation == \"PyPy\" and (extra == \"postgres\" or extra == \"all\")" +markers = "platform_python_implementation == \"PyPy\" and (extra == \"all\" or extra == \"postgres\")" files = [ {file = "psycopg2cffi-2.9.0.tar.gz", hash = "sha256:7e272edcd837de3a1d12b62185eb85c45a19feda9e62fa1b120c54f9e8d35c52"}, ] @@ -1742,7 +1762,7 @@ description = "A Simple library to enable psycopg2 compatability" optional = true python-versions = "*" groups = ["main"] -markers = "platform_python_implementation == \"PyPy\" and (extra == \"postgres\" or extra == \"all\")" +markers = "platform_python_implementation == \"PyPy\" and (extra == \"all\" or extra == \"postgres\")" files = [ {file = "psycopg2cffi-compat-1.1.tar.gz", hash = "sha256:d25e921748475522b33d13420aad5c2831c743227dc1f1f2585e0fdb5c914e05"}, ] @@ -2002,7 +2022,7 @@ description = "A development tool to measure, monitor and analyze the memory beh optional = true python-versions = ">=3.6" groups = ["main"] -markers = "extra == \"cache-memory\" or extra == \"all\"" +markers = "extra == \"all\" or extra == \"cache-memory\"" files = [ {file = "Pympler-1.0.1-py3-none-any.whl", hash = "sha256:d260dda9ae781e1eab6ea15bacb84015849833ba5555f141d2d9b7b7473b307d"}, {file = "Pympler-1.0.1.tar.gz", hash = "sha256:993f1a3599ca3f4fcd7160c7545ad06310c9e12f70174ae7ae8d4e25f6c5d3fa"}, @@ -2062,7 +2082,7 @@ description = "Python implementation of SAML Version 2 Standard" optional = true python-versions = ">=3.9,<4.0" groups = ["main"] -markers = "extra == \"saml2\" or extra == \"all\"" +markers = "extra == \"all\" or extra == \"saml2\"" files = [ {file = "pysaml2-7.5.0-py3-none-any.whl", hash = "sha256:bc6627cc344476a83c757f440a73fda1369f13b6fda1b4e16bca63ffbabb5318"}, {file = "pysaml2-7.5.0.tar.gz", hash = "sha256:f36871d4e5ee857c6b85532e942550d2cf90ea4ee943d75eb681044bbc4f54f7"}, @@ -2087,7 +2107,7 @@ description = "Extensions to the standard Python datetime module" optional = true python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" groups = ["main"] -markers = "extra == \"saml2\" or extra == \"all\"" +markers = "extra == \"all\" or extra == \"saml2\"" files = [ {file = "python-dateutil-2.8.2.tar.gz", hash = "sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86"}, {file = "python_dateutil-2.8.2-py2.py3-none-any.whl", hash = "sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9"}, @@ -2115,7 +2135,7 @@ description = "World timezone definitions, modern and historical" optional = true python-versions = "*" groups = ["main"] -markers = "extra == \"saml2\" or extra == \"all\"" +markers = "extra == \"all\" or extra == \"saml2\"" files = [ {file = "pytz-2022.7.1-py2.py3-none-any.whl", hash = "sha256:78f4f37d8198e0627c5f1143240bb0206b8691d8d7ac6d78fee88b78733f8c4a"}, {file = "pytz-2022.7.1.tar.gz", hash = "sha256:01a0681c4b9684a28304615eba55d1ab31ae00bf68ec157ec3708a8182dbbcd0"}, @@ -2480,7 +2500,7 @@ description = "Python client for Sentry (https://sentry.io)" optional = true python-versions = ">=3.6" groups = ["main"] -markers = "extra == \"sentry\" or extra == \"all\"" +markers = "extra == \"all\" or extra == \"sentry\"" files = [ {file = "sentry_sdk-2.32.0-py2.py3-none-any.whl", hash = "sha256:6cf51521b099562d7ce3606da928c473643abe99b00ce4cb5626ea735f4ec345"}, {file = "sentry_sdk-2.32.0.tar.gz", hash = "sha256:9016c75d9316b0f6921ac14c8cd4fb938f26002430ac5be9945ab280f78bec6b"}, @@ -2668,7 +2688,7 @@ description = "Tornado IOLoop Backed Concurrent Futures" optional = true python-versions = "*" groups = ["main"] -markers = "extra == \"opentracing\" or extra == \"all\"" +markers = "extra == \"all\" or extra == \"opentracing\"" files = [ {file = "threadloop-1.0.2-py2-none-any.whl", hash = "sha256:5c90dbefab6ffbdba26afb4829d2a9df8275d13ac7dc58dccb0e279992679599"}, {file = "threadloop-1.0.2.tar.gz", hash = "sha256:8b180aac31013de13c2ad5c834819771992d350267bddb854613ae77ef571944"}, @@ -2684,7 +2704,7 @@ description = "Python bindings for the Apache Thrift RPC system" optional = true python-versions = "*" groups = ["main"] -markers = "extra == \"opentracing\" or extra == \"all\"" +markers = "extra == \"all\" or extra == \"opentracing\"" files = [ {file = "thrift-0.16.0.tar.gz", hash = "sha256:2b5b6488fcded21f9d312aa23c9ff6a0195d0f6ae26ddbd5ad9e3e25dfc14408"}, ] @@ -2746,7 +2766,7 @@ description = "Tornado is a Python web framework and asynchronous networking lib optional = true python-versions = ">=3.9" groups = ["main"] -markers = "extra == \"opentracing\" or extra == \"all\"" +markers = "extra == \"all\" or extra == \"opentracing\"" files = [ {file = "tornado-6.5-cp39-abi3-macosx_10_9_universal2.whl", hash = "sha256:f81067dad2e4443b015368b24e802d0083fecada4f0a4572fdb72fc06e54a9a6"}, {file = "tornado-6.5-cp39-abi3-macosx_10_9_x86_64.whl", hash = "sha256:9ac1cbe1db860b3cbb251e795c701c41d343f06a96049d6274e7c77559117e41"}, @@ -2883,7 +2903,7 @@ description = "non-blocking redis client for python" optional = true python-versions = "*" groups = ["main"] -markers = "extra == \"redis\" or extra == \"all\"" +markers = "extra == \"all\" or extra == \"redis\"" files = [ {file = "txredisapi-1.4.11-py3-none-any.whl", hash = "sha256:ac64d7a9342b58edca13ef267d4fa7637c1aa63f8595e066801c1e8b56b22d0b"}, {file = "txredisapi-1.4.11.tar.gz", hash = "sha256:3eb1af99aefdefb59eb877b1dd08861efad60915e30ad5bf3d5bf6c5cedcdbc6"}, @@ -3214,7 +3234,7 @@ description = "An XML Schema validator and decoder" optional = true python-versions = ">=3.7" groups = ["main"] -markers = "extra == \"saml2\" or extra == \"all\"" +markers = "extra == \"all\" or extra == \"saml2\"" files = [ {file = "xmlschema-2.4.0-py3-none-any.whl", hash = "sha256:dc87be0caaa61f42649899189aab2fd8e0d567f2cf548433ba7b79278d231a4a"}, {file = "xmlschema-2.4.0.tar.gz", hash = "sha256:d74cd0c10866ac609e1ef94a5a69b018ad16e39077bc6393408b40c6babee793"}, From 97d2738eef1c966c841457e9628d3fadb4bf1a9d Mon Sep 17 00:00:00 2001 From: reivilibre Date: Tue, 15 Jul 2025 11:01:41 +0000 Subject: [PATCH 002/185] Fix CPU and database spinning when retrying sending events to servers whilst at the same time purging those events. (#18499) Fixes: #18491 Fix hotlooping due to skipped PDUs if there is still no progress to be made. This could bite if the event was purged since being skipped during catch-up. Signed-off-by: Olivier 'reivilibre --- changelog.d/18499.bugfix | 1 + synapse/federation/sender/per_destination_queue.py | 14 +++++++++++++- 2 files changed, 14 insertions(+), 1 deletion(-) create mode 100644 changelog.d/18499.bugfix diff --git a/changelog.d/18499.bugfix b/changelog.d/18499.bugfix new file mode 100644 index 0000000000..b07601a35d --- /dev/null +++ b/changelog.d/18499.bugfix @@ -0,0 +1 @@ +Fix CPU and database spinning when retrying sending events to servers whilst at the same time purging those events. \ No newline at end of file diff --git a/synapse/federation/sender/per_destination_queue.py b/synapse/federation/sender/per_destination_queue.py index b3f65e8237..8d6c77faee 100644 --- a/synapse/federation/sender/per_destination_queue.py +++ b/synapse/federation/sender/per_destination_queue.py @@ -129,6 +129,8 @@ class PerDestinationQueue: # The stream_ordering of the most recent PDU that was discarded due to # being in catch-up mode. + # Can be set to zero if no PDU has been discarded since the last time + # we queried for new PDUs during catch-up. self._catchup_last_skipped: int = 0 # Cache of the last successfully-transmitted stream ordering for this @@ -462,8 +464,18 @@ class PerDestinationQueue: # of a race condition, so we check that no new events have been # skipped due to us being in catch-up mode - if self._catchup_last_skipped > last_successful_stream_ordering: + if ( + self._catchup_last_skipped != 0 + and self._catchup_last_skipped > last_successful_stream_ordering + ): # another event has been skipped because we were in catch-up mode + # As an exception to this case: we can hit this branch if the + # room has been purged whilst we have been looping. + # In that case we avoid hot-looping by resetting the 'catch-up skipped + # PDU' flag. + # Then if there is still no progress to be made at the next iteration, + # we can exit catch-up mode. + self._catchup_last_skipped = 0 continue # we are done catching up! From 60be549c0ceb826946ea06a1ca6fd4a0ebcfc106 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 15 Jul 2025 14:22:54 +0100 Subject: [PATCH 003/185] 1.134.0 --- CHANGES.md | 7 +++++++ debian/changelog | 6 ++++++ pyproject.toml | 2 +- 3 files changed, 14 insertions(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index 8948cf1e29..220e7b5613 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,3 +1,10 @@ +# Synapse 1.134.0 (2025-07-15) + +No significant changes since 1.134.0rc1. + + + + # Synapse 1.134.0rc1 (2025-07-09) ### Features diff --git a/debian/changelog b/debian/changelog index 5b1f1f339a..49e27aed01 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,9 @@ +matrix-synapse-py3 (1.134.0) stable; urgency=medium + + * New Synapse release 1.134.0. + + -- Synapse Packaging team Tue, 15 Jul 2025 14:22:50 +0100 + matrix-synapse-py3 (1.134.0~rc1) stable; urgency=medium * New Synapse release 1.134.0rc1. diff --git a/pyproject.toml b/pyproject.toml index 79b2cf9690..28de934602 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -101,7 +101,7 @@ module-name = "synapse.synapse_rust" [tool.poetry] name = "matrix-synapse" -version = "1.134.0rc1" +version = "1.134.0" description = "Homeserver for the Matrix decentralised comms protocol" authors = ["Matrix.org Team and Contributors "] license = "AGPL-3.0-or-later" From 78ce4dc26fc09001338a2d95817cdfe686cf19db Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 15 Jul 2025 14:42:54 +0100 Subject: [PATCH 004/185] Bump mypy from 1.13.0 to 1.16.1 (#18653) --- changelog.d/18653.misc | 1 + poetry.lock | 85 +++++++++++++--------- synapse/config/cas.py | 3 + synapse/config/key.py | 5 +- synapse/config/workers.py | 8 +- synapse/handlers/auth.py | 1 + synapse/handlers/cas.py | 3 +- synapse/handlers/presence.py | 2 +- synapse/handlers/register.py | 2 +- synapse/handlers/send_email.py | 2 +- synapse/handlers/sso.py | 2 +- synapse/http/connectproxyclient.py | 33 +++++---- synapse/metrics/_reactor_metrics.py | 2 +- synapse/module_api/__init__.py | 2 +- synapse/storage/databases/main/state.py | 7 +- synapse/storage/engines/_base.py | 6 +- synapse/util/gai_resolver.py | 2 +- synapse/util/patch_inline_callbacks.py | 7 ++ tests/federation/test_federation_sender.py | 2 +- 19 files changed, 109 insertions(+), 66 deletions(-) create mode 100644 changelog.d/18653.misc diff --git a/changelog.d/18653.misc b/changelog.d/18653.misc new file mode 100644 index 0000000000..d3a4e7c438 --- /dev/null +++ b/changelog.d/18653.misc @@ -0,0 +1 @@ +Fix typing errors with upgraded mypy version. diff --git a/poetry.lock b/poetry.lock index 711dd2f016..ed5028c017 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1425,50 +1425,51 @@ docs = ["sphinx (>=8,<9)", "sphinx-autobuild"] [[package]] name = "mypy" -version = "1.13.0" +version = "1.16.1" description = "Optional static typing for Python" optional = false -python-versions = ">=3.8" +python-versions = ">=3.9" groups = ["dev"] files = [ - {file = "mypy-1.13.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:6607e0f1dd1fb7f0aca14d936d13fd19eba5e17e1cd2a14f808fa5f8f6d8f60a"}, - {file = "mypy-1.13.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:8a21be69bd26fa81b1f80a61ee7ab05b076c674d9b18fb56239d72e21d9f4c80"}, - {file = "mypy-1.13.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7b2353a44d2179846a096e25691d54d59904559f4232519d420d64da6828a3a7"}, - {file = "mypy-1.13.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:0730d1c6a2739d4511dc4253f8274cdd140c55c32dfb0a4cf8b7a43f40abfa6f"}, - {file = "mypy-1.13.0-cp310-cp310-win_amd64.whl", hash = "sha256:c5fc54dbb712ff5e5a0fca797e6e0aa25726c7e72c6a5850cfd2adbc1eb0a372"}, - {file = "mypy-1.13.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:581665e6f3a8a9078f28d5502f4c334c0c8d802ef55ea0e7276a6e409bc0d82d"}, - {file = "mypy-1.13.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:3ddb5b9bf82e05cc9a627e84707b528e5c7caaa1c55c69e175abb15a761cec2d"}, - {file = "mypy-1.13.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:20c7ee0bc0d5a9595c46f38beb04201f2620065a93755704e141fcac9f59db2b"}, - {file = "mypy-1.13.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:3790ded76f0b34bc9c8ba4def8f919dd6a46db0f5a6610fb994fe8efdd447f73"}, - {file = "mypy-1.13.0-cp311-cp311-win_amd64.whl", hash = "sha256:51f869f4b6b538229c1d1bcc1dd7d119817206e2bc54e8e374b3dfa202defcca"}, - {file = "mypy-1.13.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:5c7051a3461ae84dfb5dd15eff5094640c61c5f22257c8b766794e6dd85e72d5"}, - {file = "mypy-1.13.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:39bb21c69a5d6342f4ce526e4584bc5c197fd20a60d14a8624d8743fffb9472e"}, - {file = "mypy-1.13.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:164f28cb9d6367439031f4c81e84d3ccaa1e19232d9d05d37cb0bd880d3f93c2"}, - {file = "mypy-1.13.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:a4c1bfcdbce96ff5d96fc9b08e3831acb30dc44ab02671eca5953eadad07d6d0"}, - {file = "mypy-1.13.0-cp312-cp312-win_amd64.whl", hash = "sha256:a0affb3a79a256b4183ba09811e3577c5163ed06685e4d4b46429a271ba174d2"}, - {file = "mypy-1.13.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:a7b44178c9760ce1a43f544e595d35ed61ac2c3de306599fa59b38a6048e1aa7"}, - {file = "mypy-1.13.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:5d5092efb8516d08440e36626f0153b5006d4088c1d663d88bf79625af3d1d62"}, - {file = "mypy-1.13.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:de2904956dac40ced10931ac967ae63c5089bd498542194b436eb097a9f77bc8"}, - {file = "mypy-1.13.0-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:7bfd8836970d33c2105562650656b6846149374dc8ed77d98424b40b09340ba7"}, - {file = "mypy-1.13.0-cp313-cp313-win_amd64.whl", hash = "sha256:9f73dba9ec77acb86457a8fc04b5239822df0c14a082564737833d2963677dbc"}, - {file = "mypy-1.13.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:100fac22ce82925f676a734af0db922ecfea991e1d7ec0ceb1e115ebe501301a"}, - {file = "mypy-1.13.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:7bcb0bb7f42a978bb323a7c88f1081d1b5dee77ca86f4100735a6f541299d8fb"}, - {file = "mypy-1.13.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:bde31fc887c213e223bbfc34328070996061b0833b0a4cfec53745ed61f3519b"}, - {file = "mypy-1.13.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:07de989f89786f62b937851295ed62e51774722e5444a27cecca993fc3f9cd74"}, - {file = "mypy-1.13.0-cp38-cp38-win_amd64.whl", hash = "sha256:4bde84334fbe19bad704b3f5b78c4abd35ff1026f8ba72b29de70dda0916beb6"}, - {file = "mypy-1.13.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:0246bcb1b5de7f08f2826451abd947bf656945209b140d16ed317f65a17dc7dc"}, - {file = "mypy-1.13.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:7f5b7deae912cf8b77e990b9280f170381fdfbddf61b4ef80927edd813163732"}, - {file = "mypy-1.13.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7029881ec6ffb8bc233a4fa364736789582c738217b133f1b55967115288a2bc"}, - {file = "mypy-1.13.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:3e38b980e5681f28f033f3be86b099a247b13c491f14bb8b1e1e134d23bb599d"}, - {file = "mypy-1.13.0-cp39-cp39-win_amd64.whl", hash = "sha256:a6789be98a2017c912ae6ccb77ea553bbaf13d27605d2ca20a76dfbced631b24"}, - {file = "mypy-1.13.0-py3-none-any.whl", hash = "sha256:9c250883f9fd81d212e0952c92dbfcc96fc237f4b7c92f56ac81fd48460b3e5a"}, - {file = "mypy-1.13.0.tar.gz", hash = "sha256:0291a61b6fbf3e6673e3405cfcc0e7650bebc7939659fdca2702958038bd835e"}, + {file = "mypy-1.16.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:b4f0fed1022a63c6fec38f28b7fc77fca47fd490445c69d0a66266c59dd0b88a"}, + {file = "mypy-1.16.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:86042bbf9f5a05ea000d3203cf87aa9d0ccf9a01f73f71c58979eb9249f46d72"}, + {file = "mypy-1.16.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ea7469ee5902c95542bea7ee545f7006508c65c8c54b06dc2c92676ce526f3ea"}, + {file = "mypy-1.16.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:352025753ef6a83cb9e7f2427319bb7875d1fdda8439d1e23de12ab164179574"}, + {file = "mypy-1.16.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:ff9fa5b16e4c1364eb89a4d16bcda9987f05d39604e1e6c35378a2987c1aac2d"}, + {file = "mypy-1.16.1-cp310-cp310-win_amd64.whl", hash = "sha256:1256688e284632382f8f3b9e2123df7d279f603c561f099758e66dd6ed4e8bd6"}, + {file = "mypy-1.16.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:472e4e4c100062488ec643f6162dd0d5208e33e2f34544e1fc931372e806c0cc"}, + {file = "mypy-1.16.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:ea16e2a7d2714277e349e24d19a782a663a34ed60864006e8585db08f8ad1782"}, + {file = "mypy-1.16.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:08e850ea22adc4d8a4014651575567b0318ede51e8e9fe7a68f25391af699507"}, + {file = "mypy-1.16.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:22d76a63a42619bfb90122889b903519149879ddbf2ba4251834727944c8baca"}, + {file = "mypy-1.16.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:2c7ce0662b6b9dc8f4ed86eb7a5d505ee3298c04b40ec13b30e572c0e5ae17c4"}, + {file = "mypy-1.16.1-cp311-cp311-win_amd64.whl", hash = "sha256:211287e98e05352a2e1d4e8759c5490925a7c784ddc84207f4714822f8cf99b6"}, + {file = "mypy-1.16.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:af4792433f09575d9eeca5c63d7d90ca4aeceda9d8355e136f80f8967639183d"}, + {file = "mypy-1.16.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:66df38405fd8466ce3517eda1f6640611a0b8e70895e2a9462d1d4323c5eb4b9"}, + {file = "mypy-1.16.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:44e7acddb3c48bd2713994d098729494117803616e116032af192871aed80b79"}, + {file = "mypy-1.16.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0ab5eca37b50188163fa7c1b73c685ac66c4e9bdee4a85c9adac0e91d8895e15"}, + {file = "mypy-1.16.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:dedb6229b2c9086247e21a83c309754b9058b438704ad2f6807f0d8227f6ebdd"}, + {file = "mypy-1.16.1-cp312-cp312-win_amd64.whl", hash = "sha256:1f0435cf920e287ff68af3d10a118a73f212deb2ce087619eb4e648116d1fe9b"}, + {file = "mypy-1.16.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:ddc91eb318c8751c69ddb200a5937f1232ee8efb4e64e9f4bc475a33719de438"}, + {file = "mypy-1.16.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:87ff2c13d58bdc4bbe7dc0dedfe622c0f04e2cb2a492269f3b418df2de05c536"}, + {file = "mypy-1.16.1-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0a7cfb0fe29fe5a9841b7c8ee6dffb52382c45acdf68f032145b75620acfbd6f"}, + {file = "mypy-1.16.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:051e1677689c9d9578b9c7f4d206d763f9bbd95723cd1416fad50db49d52f359"}, + {file = "mypy-1.16.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:d5d2309511cc56c021b4b4e462907c2b12f669b2dbeb68300110ec27723971be"}, + {file = "mypy-1.16.1-cp313-cp313-win_amd64.whl", hash = "sha256:4f58ac32771341e38a853c5d0ec0dfe27e18e27da9cdb8bbc882d2249c71a3ee"}, + {file = "mypy-1.16.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:7fc688329af6a287567f45cc1cefb9db662defeb14625213a5b7da6e692e2069"}, + {file = "mypy-1.16.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:5e198ab3f55924c03ead626ff424cad1732d0d391478dfbf7bb97b34602395da"}, + {file = "mypy-1.16.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:09aa4f91ada245f0a45dbc47e548fd94e0dd5a8433e0114917dc3b526912a30c"}, + {file = "mypy-1.16.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:13c7cd5b1cb2909aa318a90fd1b7e31f17c50b242953e7dd58345b2a814f6383"}, + {file = "mypy-1.16.1-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:58e07fb958bc5d752a280da0e890c538f1515b79a65757bbdc54252ba82e0b40"}, + {file = "mypy-1.16.1-cp39-cp39-win_amd64.whl", hash = "sha256:f895078594d918f93337a505f8add9bd654d1a24962b4c6ed9390e12531eb31b"}, + {file = "mypy-1.16.1-py3-none-any.whl", hash = "sha256:5fc2ac4027d0ef28d6ba69a0343737a23c4d1b83672bf38d1fe237bdc0643b37"}, + {file = "mypy-1.16.1.tar.gz", hash = "sha256:6bd00a0a2094841c5e47e7374bb42b83d64c527a502e3334e1173a0c24437bab"}, ] [package.dependencies] -mypy-extensions = ">=1.0.0" +mypy_extensions = ">=1.0.0" +pathspec = ">=0.9.0" tomli = {version = ">=1.1.0", markers = "python_version < \"3.11\""} -typing-extensions = ">=4.6.0" +typing_extensions = ">=4.6.0" [package.extras] dmypy = ["psutil (>=4.0)"] @@ -1566,6 +1567,18 @@ files = [ [package.extras] dev = ["jinja2"] +[[package]] +name = "pathspec" +version = "0.12.1" +description = "Utility library for gitignore style pattern matching of file paths." +optional = false +python-versions = ">=3.8" +groups = ["dev"] +files = [ + {file = "pathspec-0.12.1-py3-none-any.whl", hash = "sha256:a0d503e138a4c123b27490a4f7beda6a01c6f288df0e4a8b79c7eb0dc7b4cc08"}, + {file = "pathspec-0.12.1.tar.gz", hash = "sha256:a482d51503a1ab33b1c67a6c3813a26953dbdc71c31dacaef9a838c4e29f5712"}, +] + [[package]] name = "phonenumbers" version = "9.0.9" diff --git a/synapse/config/cas.py b/synapse/config/cas.py index c32bf36951..60d66d7019 100644 --- a/synapse/config/cas.py +++ b/synapse/config/cas.py @@ -42,6 +42,9 @@ class CasConfig(Config): self.cas_enabled = cas_config and cas_config.get("enabled", True) if self.cas_enabled: + if not isinstance(cas_config, dict): + raise ConfigError("Must be a dictionary", ("cas_config",)) + self.cas_server_url = cas_config["server_url"] # TODO Update this to a _synapse URL. diff --git a/synapse/config/key.py b/synapse/config/key.py index 29c558448b..f78ff5114f 100644 --- a/synapse/config/key.py +++ b/synapse/config/key.py @@ -212,11 +212,14 @@ class KeyConfig(Config): "Config options that expect an in-line secret as value are disabled", ("form_secret",), ) + if form_secret is not None and not isinstance(form_secret, str): + raise ConfigError("Config option must be a string", ("form_secret",)) + form_secret_path = config.get("form_secret_path", None) if form_secret_path: if form_secret: raise ConfigError(CONFLICTING_FORM_SECRET_OPTS_ERROR) - self.form_secret = read_file( + self.form_secret: Optional[str] = read_file( form_secret_path, ("form_secret_path",) ).strip() else: diff --git a/synapse/config/workers.py b/synapse/config/workers.py index c486c81274..69036f9b52 100644 --- a/synapse/config/workers.py +++ b/synapse/config/workers.py @@ -238,10 +238,16 @@ class WorkerConfig(Config): if worker_replication_secret_path: if worker_replication_secret: raise ConfigError(CONFLICTING_WORKER_REPLICATION_SECRET_OPTS_ERROR) - self.worker_replication_secret = read_file( + self.worker_replication_secret: Optional[str] = read_file( worker_replication_secret_path, ("worker_replication_secret_path",) ).strip() else: + if worker_replication_secret is not None and not isinstance( + worker_replication_secret, str + ): + raise ConfigError( + "Config option must be a string", ("worker_replication_secret",) + ) self.worker_replication_secret = worker_replication_secret self.worker_name = config.get("worker_name", self.worker_app) diff --git a/synapse/handlers/auth.py b/synapse/handlers/auth.py index 9e3e70ec1c..257453674c 100644 --- a/synapse/handlers/auth.py +++ b/synapse/handlers/auth.py @@ -174,6 +174,7 @@ def login_id_phone_to_thirdparty(identifier: JsonDict) -> Dict[str, str]: # Accept both "phone" and "number" as valid keys in m.id.phone phone_number = identifier.get("phone", identifier["number"]) + assert isinstance(phone_number, str) # Convert user-provided phone number to a consistent representation msisdn = phone_number_to_msisdn(identifier["country"], phone_number) diff --git a/synapse/handlers/cas.py b/synapse/handlers/cas.py index cc3d641b7d..fbe79c2e4c 100644 --- a/synapse/handlers/cas.py +++ b/synapse/handlers/cas.py @@ -378,7 +378,8 @@ class CasHandler: # Arbitrarily use the first attribute found. display_name = cas_response.attributes.get( - self._cas_displayname_attribute, [None] + self._cas_displayname_attribute, # type: ignore[arg-type] + [None], )[0] return UserAttributes(localpart=localpart, display_name=display_name) diff --git a/synapse/handlers/presence.py b/synapse/handlers/presence.py index 390cafa8f6..fb5d691d65 100644 --- a/synapse/handlers/presence.py +++ b/synapse/handlers/presence.py @@ -1405,7 +1405,7 @@ class PresenceHandler(BasePresenceHandler): # Based on the state of each user's device calculate the new presence state. presence = _combine_device_states(devices.values()) - new_fields = {"state": presence} + new_fields: JsonDict = {"state": presence} if presence == PresenceState.ONLINE or presence == PresenceState.BUSY: new_fields["last_active_ts"] = now diff --git a/synapse/handlers/register.py b/synapse/handlers/register.py index b90126f9c7..1e1f0c79c8 100644 --- a/synapse/handlers/register.py +++ b/synapse/handlers/register.py @@ -557,7 +557,7 @@ class RegistrationHandler: if join_rules_event: join_rule = join_rules_event.content.get("join_rule", None) requires_invite = ( - join_rule and join_rule != JoinRules.PUBLIC + join_rule is not None and join_rule != JoinRules.PUBLIC ) # Send the invite, if necessary. diff --git a/synapse/handlers/send_email.py b/synapse/handlers/send_email.py index 353991573f..92fed980e6 100644 --- a/synapse/handlers/send_email.py +++ b/synapse/handlers/send_email.py @@ -197,7 +197,7 @@ class SendEmailHandler: additional_headers: A map of additional headers to include. """ try: - from_string = self._from % {"app": app_name} + from_string = self._from % {"app": app_name} # type: ignore[operator] except (KeyError, TypeError): from_string = self._from diff --git a/synapse/handlers/sso.py b/synapse/handlers/sso.py index 1cdf632649..0dd64618e0 100644 --- a/synapse/handlers/sso.py +++ b/synapse/handlers/sso.py @@ -818,7 +818,7 @@ class SsoHandler: server_name = avatar_url_parts[-2] media_id = avatar_url_parts[-1] if self._is_mine_server_name(server_name): - media = await self._media_repo.store.get_local_media(media_id) # type: ignore[has-type] + media = await self._media_repo.store.get_local_media(media_id) if media is not None and upload_name == media.upload_name: logger.info("skipping saving the user avatar") return True diff --git a/synapse/http/connectproxyclient.py b/synapse/http/connectproxyclient.py index 4e4d78cb88..db803bc75a 100644 --- a/synapse/http/connectproxyclient.py +++ b/synapse/http/connectproxyclient.py @@ -33,10 +33,11 @@ from twisted.internet.interfaces import ( IAddress, IConnector, IProtocol, + IProtocolFactory, IReactorCore, IStreamClientEndpoint, ) -from twisted.internet.protocol import ClientFactory, Protocol, connectionDone +from twisted.internet.protocol import ClientFactory, connectionDone from twisted.python.failure import Failure from twisted.web import http @@ -116,11 +117,7 @@ class HTTPConnectProxyEndpoint: def __repr__(self) -> str: return "" % (self._proxy_endpoint,) - # Mypy encounters a false positive here: it complains that ClientFactory - # is incompatible with IProtocolFactory. But ClientFactory inherits from - # Factory, which implements IProtocolFactory. So I think this is a bug - # in mypy-zope. - def connect(self, protocolFactory: ClientFactory) -> "defer.Deferred[IProtocol]": # type: ignore[override] + def connect(self, protocolFactory: IProtocolFactory) -> "defer.Deferred[IProtocol]": f = HTTPProxiedClientFactory( self._host, self._port, protocolFactory, self._proxy_creds ) @@ -148,7 +145,7 @@ class HTTPProxiedClientFactory(protocol.ClientFactory): self, dst_host: bytes, dst_port: int, - wrapped_factory: ClientFactory, + wrapped_factory: IProtocolFactory, proxy_creds: Optional[ProxyCredentials], ): self.dst_host = dst_host @@ -158,7 +155,10 @@ class HTTPProxiedClientFactory(protocol.ClientFactory): self.on_connection: "defer.Deferred[None]" = defer.Deferred() def startedConnecting(self, connector: IConnector) -> None: - return self.wrapped_factory.startedConnecting(connector) + # We expect the wrapped factory to be a ClientFactory, but the generic + # interfaces only guarantee that it implements IProtocolFactory. + if isinstance(self.wrapped_factory, ClientFactory): + return self.wrapped_factory.startedConnecting(connector) def buildProtocol(self, addr: IAddress) -> "HTTPConnectProtocol": wrapped_protocol = self.wrapped_factory.buildProtocol(addr) @@ -177,13 +177,15 @@ class HTTPProxiedClientFactory(protocol.ClientFactory): logger.debug("Connection to proxy failed: %s", reason) if not self.on_connection.called: self.on_connection.errback(reason) - return self.wrapped_factory.clientConnectionFailed(connector, reason) + if isinstance(self.wrapped_factory, ClientFactory): + return self.wrapped_factory.clientConnectionFailed(connector, reason) def clientConnectionLost(self, connector: IConnector, reason: Failure) -> None: logger.debug("Connection to proxy lost: %s", reason) if not self.on_connection.called: self.on_connection.errback(reason) - return self.wrapped_factory.clientConnectionLost(connector, reason) + if isinstance(self.wrapped_factory, ClientFactory): + return self.wrapped_factory.clientConnectionLost(connector, reason) class HTTPConnectProtocol(protocol.Protocol): @@ -208,7 +210,7 @@ class HTTPConnectProtocol(protocol.Protocol): self, host: bytes, port: int, - wrapped_protocol: Protocol, + wrapped_protocol: IProtocol, connected_deferred: defer.Deferred, proxy_creds: Optional[ProxyCredentials], ): @@ -223,11 +225,14 @@ class HTTPConnectProtocol(protocol.Protocol): ) self.http_setup_client.on_connected.addCallback(self.proxyConnected) + # Set once we start connecting to the wrapped protocol + self.wrapped_connection_started = False + def connectionMade(self) -> None: self.http_setup_client.makeConnection(self.transport) def connectionLost(self, reason: Failure = connectionDone) -> None: - if self.wrapped_protocol.connected: + if self.wrapped_connection_started: self.wrapped_protocol.connectionLost(reason) self.http_setup_client.connectionLost(reason) @@ -236,6 +241,8 @@ class HTTPConnectProtocol(protocol.Protocol): self.connected_deferred.errback(reason) def proxyConnected(self, _: Union[None, "defer.Deferred[None]"]) -> None: + self.wrapped_connection_started = True + assert self.transport is not None self.wrapped_protocol.makeConnection(self.transport) self.connected_deferred.callback(self.wrapped_protocol) @@ -247,7 +254,7 @@ class HTTPConnectProtocol(protocol.Protocol): def dataReceived(self, data: bytes) -> None: # if we've set up the HTTP protocol, we can send the data there - if self.wrapped_protocol.connected: + if self.wrapped_connection_started: return self.wrapped_protocol.dataReceived(data) # otherwise, we must still be setting up the connection: send the data to the diff --git a/synapse/metrics/_reactor_metrics.py b/synapse/metrics/_reactor_metrics.py index c0a4ee16ee..fda0cd018b 100644 --- a/synapse/metrics/_reactor_metrics.py +++ b/synapse/metrics/_reactor_metrics.py @@ -33,7 +33,7 @@ from twisted.internet.asyncioreactor import AsyncioSelectorReactor from synapse.metrics._types import Collector try: - from selectors import KqueueSelector + from selectors import KqueueSelector # type: ignore[attr-defined] except ImportError: class KqueueSelector: # type: ignore[no-redef] diff --git a/synapse/module_api/__init__.py b/synapse/module_api/__init__.py index b1a2476da8..dd58d3aedc 100644 --- a/synapse/module_api/__init__.py +++ b/synapse/module_api/__init__.py @@ -284,7 +284,7 @@ class ModuleApi: try: app_name = self._hs.config.email.email_app_name - self._from_string = self._hs.config.email.email_notif_from % { + self._from_string = self._hs.config.email.email_notif_from % { # type: ignore[operator] "app": app_name } except (KeyError, TypeError): diff --git a/synapse/storage/databases/main/state.py b/synapse/storage/databases/main/state.py index 788f7d1e32..cfcc731f86 100644 --- a/synapse/storage/databases/main/state.py +++ b/synapse/storage/databases/main/state.py @@ -990,11 +990,12 @@ class StateMapWrapper(Dict[StateKey, str]): raise Exception("State map was filtered and doesn't include: %s", key) return super().__getitem__(key) + @overload # type: ignore[override] + def get(self, key: StateKey, default: None = None, /) -> Optional[str]: ... @overload - def get(self, key: Tuple[str, str]) -> Optional[str]: ... - + def get(self, key: StateKey, default: str, /) -> str: ... @overload - def get(self, key: Tuple[str, str], default: Union[str, _T]) -> Union[str, _T]: ... + def get(self, key: StateKey, default: _T, /) -> Union[str, _T]: ... def get( self, key: StateKey, default: Union[str, _T, None] = None diff --git a/synapse/storage/engines/_base.py b/synapse/storage/engines/_base.py index 9d82c59384..9fec42c2e0 100644 --- a/synapse/storage/engines/_base.py +++ b/synapse/storage/engines/_base.py @@ -34,9 +34,9 @@ AUTO_INCREMENT_PRIMARY_KEYPLACEHOLDER = "$%AUTO_INCREMENT_PRIMARY_KEY%$" class IsolationLevel(IntEnum): - READ_COMMITTED: int = 1 - REPEATABLE_READ: int = 2 - SERIALIZABLE: int = 3 + READ_COMMITTED = 1 + REPEATABLE_READ = 2 + SERIALIZABLE = 3 class IncorrectDatabaseSetup(RuntimeError): diff --git a/synapse/util/gai_resolver.py b/synapse/util/gai_resolver.py index fecf829ade..3c7a966e87 100644 --- a/synapse/util/gai_resolver.py +++ b/synapse/util/gai_resolver.py @@ -97,7 +97,7 @@ _GETADDRINFO_RESULT = List[ SocketKind, int, str, - Union[Tuple[str, int], Tuple[str, int, int, int]], + Union[Tuple[str, int], Tuple[str, int, int, int], Tuple[int, bytes]], ] ] diff --git a/synapse/util/patch_inline_callbacks.py b/synapse/util/patch_inline_callbacks.py index beea4d2888..c776ad65b3 100644 --- a/synapse/util/patch_inline_callbacks.py +++ b/synapse/util/patch_inline_callbacks.py @@ -20,6 +20,7 @@ import functools import sys +from types import GeneratorType from typing import Any, Callable, Generator, List, TypeVar, cast from typing_extensions import ParamSpec @@ -151,6 +152,12 @@ def _check_yield_points( ) -> Generator["Deferred[object]", object, T]: gen = f(*args, **kwargs) + # We only patch if we have a native generator function, as we rely on + # `gen.gi_frame`. + if not isinstance(gen, GeneratorType): + ret = yield from gen + return ret + last_yield_line_no = gen.gi_frame.f_lineno result: Any = None while True: diff --git a/tests/federation/test_federation_sender.py b/tests/federation/test_federation_sender.py index cd906bbbc7..64e8c12817 100644 --- a/tests/federation/test_federation_sender.py +++ b/tests/federation/test_federation_sender.py @@ -672,7 +672,7 @@ class FederationSenderDevicesTestCases(HomeserverTestCase): self.assertEqual(edu["edu_type"], EduTypes.DEVICE_LIST_UPDATE) c = edu["content"] if stream_id is not None: - self.assertEqual(c["prev_id"], [stream_id]) # type: ignore[unreachable] + self.assertEqual(c["prev_id"], [stream_id]) self.assertGreaterEqual(c["stream_id"], stream_id) stream_id = c["stream_id"] devices = {edu["content"]["device_id"] for edu in self.edus} From 5f027adb33bc131aaa9618602497d01c9dcd59e9 Mon Sep 17 00:00:00 2001 From: Andrew Morgan <1342360+anoadragon453@users.noreply.github.com> Date: Tue, 15 Jul 2025 15:04:29 +0100 Subject: [PATCH 005/185] Update URL Preview code to work with `lxml` 6.0.0 (#18622) --- changelog.d/18622.misc | 1 + synapse/media/preview_html.py | 30 ++++++++++++++++++++++++++---- 2 files changed, 27 insertions(+), 4 deletions(-) create mode 100644 changelog.d/18622.misc diff --git a/changelog.d/18622.misc b/changelog.d/18622.misc new file mode 100644 index 0000000000..9a8f36ae2c --- /dev/null +++ b/changelog.d/18622.misc @@ -0,0 +1 @@ +Update URL Preview code to work with `lxml` 6.0.0+. \ No newline at end of file diff --git a/synapse/media/preview_html.py b/synapse/media/preview_html.py index 62ce7789be..38ae126a23 100644 --- a/synapse/media/preview_html.py +++ b/synapse/media/preview_html.py @@ -133,7 +133,7 @@ def decode_body( content_type: The Content-Type header. Returns: - The parsed HTML body, or None if an error occurred during processed. + The parsed HTML body, or None if an error occurred during processing. """ # If there's no body, nothing useful is going to be found. if not body: @@ -158,9 +158,31 @@ def decode_body( # Create an HTML parser. parser = etree.HTMLParser(recover=True, encoding=encoding) - # Attempt to parse the body. Returns None if the body was successfully - # parsed, but no tree was found. - return etree.fromstring(body, parser) + # Attempt to parse the body. With `lxml` 6.0.0+, this will be an empty HTML + # tree if the body was successfully parsed, but no tree was found. In + # previous `lxml` versions, `etree.fromstring` would return `None` in that + # case. + html_tree = etree.fromstring(body, parser) + + # Account for the above referenced case where `html_tree` is an HTML tree + # with an empty body. If so, return None. + if html_tree is not None and html_tree.tag == "html": + # If the tree has only a single element and it's empty, then + # return None. + body_el = html_tree.find("body") + if body_el is not None and len(html_tree) == 1: + # Extract the content of the body tag as text. + body_text = "".join(cast(Iterable[str], body_el.itertext())) + + # Strip any undecodable Unicode characters and whitespace. + body_text = body_text.strip("\ufffd").strip() + + # If there's no text left, and there were no child tags, + # then we consider the tag empty. + if not body_text and len(body_el) == 0: + return None + + return html_tree def _get_meta_tags( From 88f38ea1499665027e6b2bed6b6179e9c43a3e44 Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Tue, 15 Jul 2025 09:05:45 -0500 Subject: [PATCH 006/185] Correct version that `recaptcha_{private,public}_key_path` config options were introduced (#18684) Introduced in https://github.com/element-hq/synapse/pull/17984 I already see a [`v1.134.0rc1`](https://github.com/element-hq/synapse/releases/tag/v1.134.0rc1) tag from 5 days ago so I assume https://github.com/element-hq/synapse/pull/17984 will actually ship in the next release (which will be `v1.135.0`) --- changelog.d/18684.feature | 1 + docs/usage/configuration/config_documentation.md | 4 ++-- schema/synapse-config.schema.yaml | 4 ++-- 3 files changed, 5 insertions(+), 4 deletions(-) create mode 100644 changelog.d/18684.feature diff --git a/changelog.d/18684.feature b/changelog.d/18684.feature new file mode 100644 index 0000000000..5099881d3a --- /dev/null +++ b/changelog.d/18684.feature @@ -0,0 +1 @@ +Add `recaptcha_private_key_path` and `recaptcha_public_key_path` config option. diff --git a/docs/usage/configuration/config_documentation.md b/docs/usage/configuration/config_documentation.md index 1d48f8110e..0a3adf021a 100644 --- a/docs/usage/configuration/config_documentation.md +++ b/docs/usage/configuration/config_documentation.md @@ -2363,7 +2363,7 @@ recaptcha_public_key: YOUR_PUBLIC_KEY The file should be a plain text file, containing only the public key. Synapse reads the public key from the given file once at startup. -_Added in Synapse 1.134.0._ +_Added in Synapse 1.135.0._ Defaults to `null`. @@ -2387,7 +2387,7 @@ recaptcha_private_key: YOUR_PRIVATE_KEY The file should be a plain text file, containing only the private key. Synapse reads the private key from the given file once at startup. -_Added in Synapse 1.134.0._ +_Added in Synapse 1.135.0._ Defaults to `null`. diff --git a/schema/synapse-config.schema.yaml b/schema/synapse-config.schema.yaml index 24fbf2419a..a2ba557fe0 100644 --- a/schema/synapse-config.schema.yaml +++ b/schema/synapse-config.schema.yaml @@ -2703,7 +2703,7 @@ properties: Synapse reads the public key from the given file once at startup. - _Added in Synapse 1.134.0._ + _Added in Synapse 1.135.0._ default: null examples: - /path/to/key/file @@ -2726,7 +2726,7 @@ properties: Synapse reads the private key from the given file once at startup. - _Added in Synapse 1.134.0._ + _Added in Synapse 1.135.0._ default: null examples: - /path/to/key/file From 49cb78376ec28943f4ed8bbf4aadcb34c3091413 Mon Sep 17 00:00:00 2001 From: Andrew Morgan <1342360+anoadragon453@users.noreply.github.com> Date: Tue, 15 Jul 2025 15:07:20 +0100 Subject: [PATCH 007/185] Advertise support for Matrix v1.12 (#18647) --- changelog.d/18647.feature | 1 + synapse/rest/client/versions.py | 1 + 2 files changed, 2 insertions(+) create mode 100644 changelog.d/18647.feature diff --git a/changelog.d/18647.feature b/changelog.d/18647.feature new file mode 100644 index 0000000000..a9c273bd29 --- /dev/null +++ b/changelog.d/18647.feature @@ -0,0 +1 @@ +Advertise support for Matrix v1.12. \ No newline at end of file diff --git a/synapse/rest/client/versions.py b/synapse/rest/client/versions.py index f58f11e5cc..fa39eb9e6d 100644 --- a/synapse/rest/client/versions.py +++ b/synapse/rest/client/versions.py @@ -112,6 +112,7 @@ class VersionsRestServlet(RestServlet): "v1.9", "v1.10", "v1.11", + "v1.12", ], # as per MSC1497: "unstable_features": { From b274d6561c1ebcc8a464caa9e899a849d5d0d813 Mon Sep 17 00:00:00 2001 From: Johannes Marbach Date: Tue, 15 Jul 2025 20:25:25 +0200 Subject: [PATCH 008/185] Document that some config options for the user directory are in violation of the Matrix spec (#18548) Fix #17534 Signed-off-by: Johannes Marbach --- changelog.d/18548.doc | 1 + docs/usage/configuration/config_documentation.md | 6 +++++- schema/synapse-config.schema.yaml | 9 ++++++++- 3 files changed, 14 insertions(+), 2 deletions(-) create mode 100644 changelog.d/18548.doc diff --git a/changelog.d/18548.doc b/changelog.d/18548.doc new file mode 100644 index 0000000000..69b073221b --- /dev/null +++ b/changelog.d/18548.doc @@ -0,0 +1 @@ +Document that some config options for the user directory are in violation of the Matrix spec. diff --git a/docs/usage/configuration/config_documentation.md b/docs/usage/configuration/config_documentation.md index 0a3adf021a..6918559dea 100644 --- a/docs/usage/configuration/config_documentation.md +++ b/docs/usage/configuration/config_documentation.md @@ -3808,7 +3808,11 @@ encryption_enabled_by_default_for_room_type: invite This setting has the following sub-options: -* `enabled` (boolean): Defines whether users can search the user directory. If false then empty responses are returned to all queries. Defaults to `true`. +* `enabled` (boolean): Defines whether users can search the user directory. If `false` then empty responses are returned to all queries. + + *Warning: While the homeserver may determine which subset of users are searched, the Matrix specification requires homeservers to include (at minimum) users visible in public rooms and users sharing a room with the requester. Using `false` improves performance but violates this requirement.* + + Defaults to `true`. * `search_all_users` (boolean): Defines whether to search all users visible to your homeserver at the time the search is performed. If set to true, will return all users known to the homeserver matching the search query. If false, search results will only contain users visible in public rooms and users sharing a room with the requester. diff --git a/schema/synapse-config.schema.yaml b/schema/synapse-config.schema.yaml index a2ba557fe0..d9a5a98496 100644 --- a/schema/synapse-config.schema.yaml +++ b/schema/synapse-config.schema.yaml @@ -4719,8 +4719,15 @@ properties: enabled: type: boolean description: >- - Defines whether users can search the user directory. If false then + Defines whether users can search the user directory. If `false` then empty responses are returned to all queries. + + + *Warning: While the homeserver may determine which subset of users are + searched, the Matrix specification requires homeservers to include (at + minimum) users visible in public rooms and users sharing a room with + the requester. Using `false` improves performance but violates this + requirement.* default: true search_all_users: type: boolean From d72c278a07ac172959dd8d121ce02f7e52446395 Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Tue, 15 Jul 2025 15:53:56 -0500 Subject: [PATCH 009/185] Remove `allow_no_prev_events` option (MSC2716 cleanup) (#18676) This option is no longer used since we backed out the MSC2716 changes in https://github.com/matrix-org/synapse/pull/15748 and is even mentioned as a follow-up task in the PR description there. The `allow_no_prev_events` option was first introduced in https://github.com/matrix-org/synapse/pull/11243 to support MSC2716 back in the day. --- changelog.d/18676.misc | 1 + synapse/handlers/message.py | 40 +++----------------- synapse/handlers/room_member.py | 21 +---------- tests/handlers/test_message.py | 66 +++------------------------------ 4 files changed, 12 insertions(+), 116 deletions(-) create mode 100644 changelog.d/18676.misc diff --git a/changelog.d/18676.misc b/changelog.d/18676.misc new file mode 100644 index 0000000000..81306954b6 --- /dev/null +++ b/changelog.d/18676.misc @@ -0,0 +1 @@ +Remove unused `allow_no_prev_events` option when creating an event. diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py index 5d6ee6996f..6f68b8f603 100644 --- a/synapse/handlers/message.py +++ b/synapse/handlers/message.py @@ -568,7 +568,6 @@ class EventCreationHandler: requester: Requester, event_dict: dict, txn_id: Optional[str] = None, - allow_no_prev_events: bool = False, prev_event_ids: Optional[List[str]] = None, auth_event_ids: Optional[List[str]] = None, state_event_ids: Optional[List[str]] = None, @@ -594,10 +593,6 @@ class EventCreationHandler: requester event_dict: An entire event txn_id - allow_no_prev_events: Whether to allow this event to be created an empty - list of prev_events. Normally this is prohibited just because most - events should have a prev_event and we should only use this in special - cases (previously useful for MSC2716). prev_event_ids: the forward extremities to use as the prev_events for the new event. @@ -717,7 +712,6 @@ class EventCreationHandler: event, unpersisted_context = await self.create_new_client_event( builder=builder, requester=requester, - allow_no_prev_events=allow_no_prev_events, prev_event_ids=prev_event_ids, auth_event_ids=auth_event_ids, state_event_ids=state_event_ids, @@ -945,7 +939,6 @@ class EventCreationHandler: self, requester: Requester, event_dict: dict, - allow_no_prev_events: bool = False, prev_event_ids: Optional[List[str]] = None, state_event_ids: Optional[List[str]] = None, ratelimit: bool = True, @@ -962,10 +955,6 @@ class EventCreationHandler: Args: requester: The requester sending the event. event_dict: An entire event. - allow_no_prev_events: Whether to allow this event to be created an empty - list of prev_events. Normally this is prohibited just because most - events should have a prev_event and we should only use this in special - cases (previously useful for MSC2716). prev_event_ids: The event IDs to use as the prev events. Should normally be left as None to automatically request them @@ -1051,7 +1040,6 @@ class EventCreationHandler: return await self._create_and_send_nonmember_event_locked( requester=requester, event_dict=event_dict, - allow_no_prev_events=allow_no_prev_events, prev_event_ids=prev_event_ids, state_event_ids=state_event_ids, ratelimit=ratelimit, @@ -1065,7 +1053,6 @@ class EventCreationHandler: self, requester: Requester, event_dict: dict, - allow_no_prev_events: bool = False, prev_event_ids: Optional[List[str]] = None, state_event_ids: Optional[List[str]] = None, ratelimit: bool = True, @@ -1097,7 +1084,6 @@ class EventCreationHandler: requester, event_dict, txn_id=txn_id, - allow_no_prev_events=allow_no_prev_events, prev_event_ids=prev_event_ids, state_event_ids=state_event_ids, outlier=outlier, @@ -1180,7 +1166,6 @@ class EventCreationHandler: self, builder: EventBuilder, requester: Optional[Requester] = None, - allow_no_prev_events: bool = False, prev_event_ids: Optional[List[str]] = None, auth_event_ids: Optional[List[str]] = None, state_event_ids: Optional[List[str]] = None, @@ -1200,10 +1185,6 @@ class EventCreationHandler: Args: builder: requester: - allow_no_prev_events: Whether to allow this event to be created an empty - list of prev_events. Normally this is prohibited just because most - events should have a prev_event and we should only use this in special - cases (previously useful for MSC2716). prev_event_ids: the forward extremities to use as the prev_events for the new event. @@ -1241,7 +1222,6 @@ class EventCreationHandler: if state_event_ids is not None: # Do a quick check to make sure that prev_event_ids is present to # make the type-checking around `builder.build` happy. - # prev_event_ids could be an empty array though. assert prev_event_ids is not None temp_event = await builder.build( @@ -1269,24 +1249,14 @@ class EventCreationHandler: else: prev_event_ids = await self.store.get_prev_events_for_room(builder.room_id) + # We now ought to have some `prev_events` (unless it's a create event). + # # Do a quick sanity check here, rather than waiting until we've created the # event and then try to auth it (which fails with a somewhat confusing "No # create event in auth events") - if allow_no_prev_events: - # We allow events with no `prev_events` but it better have some `auth_events` - assert ( - builder.type == EventTypes.Create - # Allow an event to have empty list of prev_event_ids - # only if it has auth_event_ids. - or auth_event_ids - ), ( - "Attempting to create a non-m.room.create event with no prev_events or auth_event_ids" - ) - else: - # we now ought to have some prev_events (unless it's a create event). - assert builder.type == EventTypes.Create or prev_event_ids, ( - "Attempting to create a non-m.room.create event with no prev_events" - ) + assert builder.type == EventTypes.Create or len(prev_event_ids) > 0, ( + "Attempting to create an event with no prev_events" + ) if for_batch: assert prev_event_ids is not None diff --git a/synapse/handlers/room_member.py b/synapse/handlers/room_member.py index cf9db7b018..b6800a9f63 100644 --- a/synapse/handlers/room_member.py +++ b/synapse/handlers/room_member.py @@ -388,11 +388,11 @@ class RoomMemberHandler(metaclass=abc.ABCMeta): async def _local_membership_update( self, + *, requester: Requester, target: UserID, room_id: str, membership: str, - allow_no_prev_events: bool = False, prev_event_ids: Optional[List[str]] = None, state_event_ids: Optional[List[str]] = None, depth: Optional[int] = None, @@ -414,11 +414,6 @@ class RoomMemberHandler(metaclass=abc.ABCMeta): desired membership event. room_id: membership: - - allow_no_prev_events: Whether to allow this event to be created an empty - list of prev_events. Normally this is prohibited just because most - events should have a prev_event and we should only use this in special - cases (previously useful for MSC2716). prev_event_ids: The event IDs to use as the prev events state_event_ids: The full state at a given event. This was previously used particularly @@ -486,7 +481,6 @@ class RoomMemberHandler(metaclass=abc.ABCMeta): "origin_server_ts": origin_server_ts, }, txn_id=txn_id, - allow_no_prev_events=allow_no_prev_events, prev_event_ids=prev_event_ids, state_event_ids=state_event_ids, depth=depth, @@ -583,7 +577,6 @@ class RoomMemberHandler(metaclass=abc.ABCMeta): new_room: bool = False, require_consent: bool = True, outlier: bool = False, - allow_no_prev_events: bool = False, prev_event_ids: Optional[List[str]] = None, state_event_ids: Optional[List[str]] = None, depth: Optional[int] = None, @@ -607,10 +600,6 @@ class RoomMemberHandler(metaclass=abc.ABCMeta): outlier: Indicates whether the event is an `outlier`, i.e. if it's from an arbitrary point and floating in the DAG as opposed to being inline with the current DAG. - allow_no_prev_events: Whether to allow this event to be created an empty - list of prev_events. Normally this is prohibited just because most - events should have a prev_event and we should only use this in special - cases (previously useful for MSC2716). prev_event_ids: The event IDs to use as the prev events state_event_ids: The full state at a given event. This was previously used particularly @@ -680,7 +669,6 @@ class RoomMemberHandler(metaclass=abc.ABCMeta): new_room=new_room, require_consent=require_consent, outlier=outlier, - allow_no_prev_events=allow_no_prev_events, prev_event_ids=prev_event_ids, state_event_ids=state_event_ids, depth=depth, @@ -703,7 +691,6 @@ class RoomMemberHandler(metaclass=abc.ABCMeta): new_room: bool = False, require_consent: bool = True, outlier: bool = False, - allow_no_prev_events: bool = False, prev_event_ids: Optional[List[str]] = None, state_event_ids: Optional[List[str]] = None, depth: Optional[int] = None, @@ -729,10 +716,6 @@ class RoomMemberHandler(metaclass=abc.ABCMeta): outlier: Indicates whether the event is an `outlier`, i.e. if it's from an arbitrary point and floating in the DAG as opposed to being inline with the current DAG. - allow_no_prev_events: Whether to allow this event to be created an empty - list of prev_events. Normally this is prohibited just because most - events should have a prev_event and we should only use this in special - cases (previously useful for MSC2716). prev_event_ids: The event IDs to use as the prev events state_event_ids: The full state at a given event. This was previously used particularly @@ -933,7 +916,6 @@ class RoomMemberHandler(metaclass=abc.ABCMeta): ) # InviteRule.IGNORE is handled at the sync layer. - # An empty prev_events list is allowed as long as the auth_event_ids are present if prev_event_ids is not None: return await self._local_membership_update( requester=requester, @@ -942,7 +924,6 @@ class RoomMemberHandler(metaclass=abc.ABCMeta): membership=effective_membership_state, txn_id=txn_id, ratelimit=ratelimit, - allow_no_prev_events=allow_no_prev_events, prev_event_ids=prev_event_ids, state_event_ids=state_event_ids, depth=depth, diff --git a/tests/handlers/test_message.py b/tests/handlers/test_message.py index 76ab83d1f7..990c906d2c 100644 --- a/tests/handlers/test_message.py +++ b/tests/handlers/test_message.py @@ -204,46 +204,19 @@ class EventCreationTestCase(unittest.HomeserverTestCase): self.assertEqual(len(events), 2) self.assertEqual(events[0].event_id, events[1].event_id) - def test_when_empty_prev_events_allowed_create_event_with_empty_prev_events( + def test_reject_event_with_empty_prev_events( self, ) -> None: - """When we set allow_no_prev_events=True, should be able to create a - event without any prev_events (only auth_events). """ - # Create a member event we can use as an auth_event - memberEvent, _ = self._create_and_persist_member_event() - - # Try to create the event with empty prev_events bit with some auth_events - event, _ = self.get_success( - self.handler.create_event( - self.requester, - { - "type": EventTypes.Message, - "room_id": self.room_id, - "sender": self.requester.user.to_string(), - "content": {"msgtype": "m.text", "body": random_string(5)}, - }, - # Empty prev_events is the key thing we're testing here - prev_event_ids=[], - # But with some auth_events - auth_event_ids=[memberEvent.event_id], - # Allow no prev_events! - allow_no_prev_events=True, - ) - ) - self.assertIsNotNone(event) - - def test_when_empty_prev_events_not_allowed_reject_event_with_empty_prev_events( - self, - ) -> None: - """When we set allow_no_prev_events=False, shouldn't be able to create a - event without any prev_events even if it has auth_events. Expect an - exception to be raised. + Shouldn't be able to create an event without any `prev_events` even if it has + `auth_events`. Expect an exception to be raised. """ # Create a member event we can use as an auth_event memberEvent, _ = self._create_and_persist_member_event() # Try to create the event with empty prev_events but with some auth_events + # + # We expect the test to fail because empty prev_events are not allowed self.get_failure( self.handler.create_event( self.requester, @@ -257,35 +230,6 @@ class EventCreationTestCase(unittest.HomeserverTestCase): prev_event_ids=[], # But with some auth_events auth_event_ids=[memberEvent.event_id], - # We expect the test to fail because empty prev_events are not - # allowed here! - allow_no_prev_events=False, - ), - AssertionError, - ) - - def test_when_empty_prev_events_allowed_reject_event_with_empty_prev_events_and_auth_events( - self, - ) -> None: - """When we set allow_no_prev_events=True, should be able to create a - event without any prev_events or auth_events. Expect an exception to be - raised. - """ - # Try to create the event with empty prev_events and empty auth_events - self.get_failure( - self.handler.create_event( - self.requester, - { - "type": EventTypes.Message, - "room_id": self.room_id, - "sender": self.requester.user.to_string(), - "content": {"msgtype": "m.text", "body": random_string(5)}, - }, - prev_event_ids=[], - # The event should be rejected when there are no auth_events - auth_event_ids=[], - # Allow no prev_events! - allow_no_prev_events=True, ), AssertionError, ) From fc10a5ee29825e2eea76bc073669c5494f336d33 Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Tue, 15 Jul 2025 15:55:23 -0500 Subject: [PATCH 010/185] Refactor `Measure` block metrics to be homeserver-scoped (v2) (#18601) Refactor `Measure` block metrics to be homeserver-scoped (add `server_name` label to block metrics). Part of https://github.com/element-hq/synapse/issues/18592 ### Testing strategy #### See behavior of previous `metrics` listener 1. Add the `metrics` listener in your `homeserver.yaml` ```yaml listeners: - port: 9323 type: metrics bind_addresses: ['127.0.0.1'] ``` 1. Start the homeserver: `poetry run synapse_homeserver --config-path homeserver.yaml` 1. Fetch `http://localhost:9323/metrics` 1. Observe response includes the block metrics (`synapse_util_metrics_block_count`, `synapse_util_metrics_block_in_flight`, etc) #### See behavior of the `http` `metrics` resource 1. Add the `metrics` resource to a new or existing `http` listeners in your `homeserver.yaml` ```yaml listeners: - port: 9322 type: http bind_addresses: ['127.0.0.1'] resources: - names: [metrics] compress: false ``` 1. Start the homeserver: `poetry run synapse_homeserver --config-path homeserver.yaml` 1. Fetch `http://localhost:9322/_synapse/metrics` (it's just a `GET` request so you can even do in the browser) 1. Observe response includes the block metrics (`synapse_util_metrics_block_count`, `synapse_util_metrics_block_in_flight`, etc) --- changelog.d/18601.misc | 1 + synapse/federation/send_queue.py | 4 +- synapse/federation/sender/__init__.py | 6 +- .../federation/sender/transaction_manager.py | 4 +- synapse/handlers/appservice.py | 11 +- synapse/handlers/delayed_events.py | 5 +- synapse/handlers/device.py | 5 +- synapse/handlers/message.py | 8 +- synapse/handlers/presence.py | 14 +- synapse/handlers/sync.py | 17 ++- synapse/handlers/typing.py | 9 +- synapse/handlers/user_directory.py | 4 +- .../federation/matrix_federation_agent.py | 7 + .../http/federation/well_known_resolver.py | 15 ++- synapse/http/matrixfederationclient.py | 7 +- synapse/metrics/__init__.py | 40 +++++- .../callbacks/media_repository_callbacks.py | 13 +- .../callbacks/ratelimit_callbacks.py | 7 +- .../callbacks/spamchecker_callbacks.py | 85 ++++++++++-- synapse/push/bulk_push_rule_evaluator.py | 3 +- synapse/replication/http/federation.py | 5 +- synapse/replication/http/send_event.py | 5 +- synapse/replication/http/send_events.py | 5 +- synapse/replication/tcp/client.py | 7 +- synapse/replication/tcp/resource.py | 7 +- synapse/state/__init__.py | 12 +- synapse/storage/_base.py | 1 + synapse/storage/controllers/persist_events.py | 25 +++- synapse/storage/controllers/state.py | 5 +- .../storage/databases/main/events_worker.py | 4 +- synapse/storage/databases/main/roommember.py | 6 +- synapse/util/metrics.py | 121 +++++++++++++----- tests/handlers/test_typing.py | 1 + .../test_matrix_federation_agent.py | 4 + .../test_federation_sender_shard.py | 1 + 35 files changed, 380 insertions(+), 94 deletions(-) create mode 100644 changelog.d/18601.misc diff --git a/changelog.d/18601.misc b/changelog.d/18601.misc new file mode 100644 index 0000000000..c8893c2806 --- /dev/null +++ b/changelog.d/18601.misc @@ -0,0 +1 @@ +Refactor `Measure` block metrics to be homeserver-scoped. diff --git a/synapse/federation/send_queue.py b/synapse/federation/send_queue.py index b5c9fcff7c..e309836a52 100644 --- a/synapse/federation/send_queue.py +++ b/synapse/federation/send_queue.py @@ -156,7 +156,9 @@ class FederationRemoteSendQueue(AbstractFederationSender): def _clear_queue_before_pos(self, position_to_delete: int) -> None: """Clear all the queues from before a given position""" - with Measure(self.clock, "send_queue._clear"): + with Measure( + self.clock, name="send_queue._clear", server_name=self.server_name + ): # Delete things out of presence maps keys = self.presence_destinations.keys() i = self.presence_destinations.bisect_left(position_to_delete) diff --git a/synapse/federation/sender/__init__.py b/synapse/federation/sender/__init__.py index 2eef7b707d..8010cc62f3 100644 --- a/synapse/federation/sender/__init__.py +++ b/synapse/federation/sender/__init__.py @@ -657,7 +657,11 @@ class FederationSender(AbstractFederationSender): logger.debug( "Handling %i events in room %s", len(events), events[0].room_id ) - with Measure(self.clock, "handle_room_events"): + with Measure( + self.clock, + name="handle_room_events", + server_name=self.server_name, + ): for event in events: await handle_event(event) diff --git a/synapse/federation/sender/transaction_manager.py b/synapse/federation/sender/transaction_manager.py index d8a3eaa525..21e2fed085 100644 --- a/synapse/federation/sender/transaction_manager.py +++ b/synapse/federation/sender/transaction_manager.py @@ -58,7 +58,7 @@ class TransactionManager: """ def __init__(self, hs: "synapse.server.HomeServer"): - self._server_name = hs.hostname + self.server_name = hs.hostname # nb must be called this for @measure_func self.clock = hs.get_clock() # nb must be called this for @measure_func self._store = hs.get_datastores().main self._transaction_actions = TransactionActions(self._store) @@ -116,7 +116,7 @@ class TransactionManager: transaction = Transaction( origin_server_ts=int(self.clock.time_msec()), transaction_id=txn_id, - origin=self._server_name, + origin=self.server_name, destination=destination, pdus=[p.get_pdu_json() for p in pdus], edus=[edu.get_dict() for edu in edus], diff --git a/synapse/handlers/appservice.py b/synapse/handlers/appservice.py index f3bbdb5a05..5aefc73aba 100644 --- a/synapse/handlers/appservice.py +++ b/synapse/handlers/appservice.py @@ -73,6 +73,7 @@ events_processed_counter = Counter("synapse_handlers_appservice_events_processed class ApplicationServicesHandler: def __init__(self, hs: "HomeServer"): + self.server_name = hs.hostname self.store = hs.get_datastores().main self.is_mine_id = hs.is_mine_id self.appservice_api = hs.get_application_service_api() @@ -120,7 +121,9 @@ class ApplicationServicesHandler: @wrap_as_background_process("notify_interested_services") async def _notify_interested_services(self, max_token: RoomStreamToken) -> None: - with Measure(self.clock, "notify_interested_services"): + with Measure( + self.clock, name="notify_interested_services", server_name=self.server_name + ): self.is_processing = True try: upper_bound = -1 @@ -329,7 +332,11 @@ class ApplicationServicesHandler: users: Collection[Union[str, UserID]], ) -> None: logger.debug("Checking interested services for %s", stream_key) - with Measure(self.clock, "notify_interested_services_ephemeral"): + with Measure( + self.clock, + name="notify_interested_services_ephemeral", + server_name=self.server_name, + ): for service in services: if stream_key == StreamKeyType.TYPING: # Note that we don't persist the token (via set_appservice_stream_type_pos) diff --git a/synapse/handlers/delayed_events.py b/synapse/handlers/delayed_events.py index 80cb1cec9b..beb0e819c2 100644 --- a/synapse/handlers/delayed_events.py +++ b/synapse/handlers/delayed_events.py @@ -54,6 +54,7 @@ logger = logging.getLogger(__name__) class DelayedEventsHandler: def __init__(self, hs: "HomeServer"): + self.server_name = hs.hostname self._store = hs.get_datastores().main self._storage_controllers = hs.get_storage_controllers() self._config = hs.config @@ -159,7 +160,9 @@ class DelayedEventsHandler: # Loop round handling deltas until we're up to date while True: - with Measure(self._clock, "delayed_events_delta"): + with Measure( + self._clock, name="delayed_events_delta", server_name=self.server_name + ): room_max_stream_ordering = self._store.get_room_max_stream_ordering() if self._event_pos == room_max_stream_ordering: return diff --git a/synapse/handlers/device.py b/synapse/handlers/device.py index c6e44dae6a..e825626558 100644 --- a/synapse/handlers/device.py +++ b/synapse/handlers/device.py @@ -526,6 +526,8 @@ class DeviceHandler(DeviceWorkerHandler): def __init__(self, hs: "HomeServer"): super().__init__(hs) + self.server_name = hs.hostname # nb must be called this for @measure_func + self.clock = hs.get_clock() # nb must be called this for @measure_func self.federation_sender = hs.get_federation_sender() self._account_data_handler = hs.get_account_data_handler() self._storage_controllers = hs.get_storage_controllers() @@ -1215,7 +1217,8 @@ class DeviceListUpdater(DeviceListWorkerUpdater): def __init__(self, hs: "HomeServer", device_handler: DeviceHandler): self.store = hs.get_datastores().main self.federation = hs.get_federation_client() - self.clock = hs.get_clock() + self.server_name = hs.hostname # nb must be called this for @measure_func + self.clock = hs.get_clock() # nb must be called this for @measure_func self.device_handler = device_handler self._notifier = hs.get_notifier() diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py index 6f68b8f603..7c76c187bc 100644 --- a/synapse/handlers/message.py +++ b/synapse/handlers/message.py @@ -476,16 +476,16 @@ _DUMMY_EVENT_ROOM_EXCLUSION_EXPIRY = 7 * 24 * 60 * 60 * 1000 class EventCreationHandler: def __init__(self, hs: "HomeServer"): self.hs = hs + self.validator = EventValidator() + self.event_builder_factory = hs.get_event_builder_factory() + self.server_name = hs.hostname # nb must be called this for @measure_func + self.clock = hs.get_clock() # nb must be called this for @measure_func self.auth_blocking = hs.get_auth_blocking() self._event_auth_handler = hs.get_event_auth_handler() self.store = hs.get_datastores().main self._storage_controllers = hs.get_storage_controllers() self.state = hs.get_state_handler() - self.clock = hs.get_clock() - self.validator = EventValidator() self.profile_handler = hs.get_profile_handler() - self.event_builder_factory = hs.get_event_builder_factory() - self.server_name = hs.hostname self.notifier = hs.get_notifier() self.config = hs.config self.require_membership_for_aliases = ( diff --git a/synapse/handlers/presence.py b/synapse/handlers/presence.py index fb5d691d65..c652e333a6 100644 --- a/synapse/handlers/presence.py +++ b/synapse/handlers/presence.py @@ -747,6 +747,7 @@ class WorkerPresenceHandler(BasePresenceHandler): class PresenceHandler(BasePresenceHandler): def __init__(self, hs: "HomeServer"): super().__init__(hs) + self.server_name = hs.hostname self.wheel_timer: WheelTimer[str] = WheelTimer() self.notifier = hs.get_notifier() @@ -941,7 +942,9 @@ class PresenceHandler(BasePresenceHandler): now = self.clock.time_msec() - with Measure(self.clock, "presence_update_states"): + with Measure( + self.clock, name="presence_update_states", server_name=self.server_name + ): # NOTE: We purposefully don't await between now and when we've # calculated what we want to do with the new states, to avoid races. @@ -1497,7 +1500,9 @@ class PresenceHandler(BasePresenceHandler): async def _unsafe_process(self) -> None: # Loop round handling deltas until we're up to date while True: - with Measure(self.clock, "presence_delta"): + with Measure( + self.clock, name="presence_delta", server_name=self.server_name + ): room_max_stream_ordering = self.store.get_room_max_stream_ordering() if self._event_pos == room_max_stream_ordering: return @@ -1759,6 +1764,7 @@ class PresenceEventSource(EventSource[int, UserPresenceState]): # Same with get_presence_router: # # AuthHandler -> Notifier -> PresenceEventSource -> ModuleApi -> AuthHandler + self.server_name = hs.hostname self.get_presence_handler = hs.get_presence_handler self.get_presence_router = hs.get_presence_router self.clock = hs.get_clock() @@ -1792,7 +1798,9 @@ class PresenceEventSource(EventSource[int, UserPresenceState]): user_id = user.to_string() stream_change_cache = self.store.presence_stream_cache - with Measure(self.clock, "presence.get_new_events"): + with Measure( + self.clock, name="presence.get_new_events", server_name=self.server_name + ): if from_key is not None: from_key = int(from_key) diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py index a400e63fd5..7b99defac1 100644 --- a/synapse/handlers/sync.py +++ b/synapse/handlers/sync.py @@ -329,6 +329,7 @@ class E2eeSyncResult: class SyncHandler: def __init__(self, hs: "HomeServer"): + self.server_name = hs.hostname self.hs_config = hs.config self.store = hs.get_datastores().main self.notifier = hs.get_notifier() @@ -710,7 +711,9 @@ class SyncHandler: sync_config = sync_result_builder.sync_config - with Measure(self.clock, "ephemeral_by_room"): + with Measure( + self.clock, name="ephemeral_by_room", server_name=self.server_name + ): typing_key = since_token.typing_key if since_token else 0 room_ids = sync_result_builder.joined_room_ids @@ -783,7 +786,9 @@ class SyncHandler: and current token to send down to clients. newly_joined_room """ - with Measure(self.clock, "load_filtered_recents"): + with Measure( + self.clock, name="load_filtered_recents", server_name=self.server_name + ): timeline_limit = sync_config.filter_collection.timeline_limit() block_all_timeline = ( sync_config.filter_collection.blocks_all_room_timeline() @@ -1174,7 +1179,9 @@ class SyncHandler: # updates even if they occurred logically before the previous event. # TODO(mjark) Check for new redactions in the state events. - with Measure(self.clock, "compute_state_delta"): + with Measure( + self.clock, name="compute_state_delta", server_name=self.server_name + ): # The memberships needed for events in the timeline. # Only calculated when `lazy_load_members` is on. members_to_fetch: Optional[Set[str]] = None @@ -1791,7 +1798,9 @@ class SyncHandler: # the DB. return RoomNotifCounts.empty() - with Measure(self.clock, "unread_notifs_for_room_id"): + with Measure( + self.clock, name="unread_notifs_for_room_id", server_name=self.server_name + ): return await self.store.get_unread_event_push_actions_by_room_for_user( room_id, sync_config.user.to_string(), diff --git a/synapse/handlers/typing.py b/synapse/handlers/typing.py index 8d693fee30..bbef3a59a5 100644 --- a/synapse/handlers/typing.py +++ b/synapse/handlers/typing.py @@ -503,6 +503,7 @@ class TypingWriterHandler(FollowerTypingHandler): class TypingNotificationEventSource(EventSource[int, JsonMapping]): def __init__(self, hs: "HomeServer"): + self.server_name = hs.hostname self._main_store = hs.get_datastores().main self.clock = hs.get_clock() # We can't call get_typing_handler here because there's a cycle: @@ -535,7 +536,9 @@ class TypingNotificationEventSource(EventSource[int, JsonMapping]): appservice may be interested in. * The latest known room serial. """ - with Measure(self.clock, "typing.get_new_events_as"): + with Measure( + self.clock, name="typing.get_new_events_as", server_name=self.server_name + ): handler = self.get_typing_handler() events = [] @@ -571,7 +574,9 @@ class TypingNotificationEventSource(EventSource[int, JsonMapping]): Find typing notifications for given rooms (> `from_token` and <= `to_token`) """ - with Measure(self.clock, "typing.get_new_events"): + with Measure( + self.clock, name="typing.get_new_events", server_name=self.server_name + ): from_key = int(from_key) handler = self.get_typing_handler() diff --git a/synapse/handlers/user_directory.py b/synapse/handlers/user_directory.py index 1f692c79a0..5f9e96706a 100644 --- a/synapse/handlers/user_directory.py +++ b/synapse/handlers/user_directory.py @@ -237,7 +237,9 @@ class UserDirectoryHandler(StateDeltasHandler): # Loop round handling deltas until we're up to date while True: - with Measure(self.clock, "user_dir_delta"): + with Measure( + self.clock, name="user_dir_delta", server_name=self.server_name + ): room_max_stream_ordering = self.store.get_room_max_stream_ordering() if self.pos == room_max_stream_ordering: return diff --git a/synapse/http/federation/matrix_federation_agent.py b/synapse/http/federation/matrix_federation_agent.py index a7742fcea8..4a47665abd 100644 --- a/synapse/http/federation/matrix_federation_agent.py +++ b/synapse/http/federation/matrix_federation_agent.py @@ -92,6 +92,7 @@ class MatrixFederationAgent: def __init__( self, + server_name: str, reactor: ISynapseReactor, tls_client_options_factory: Optional[FederationPolicyForHTTPS], user_agent: bytes, @@ -100,6 +101,11 @@ class MatrixFederationAgent: _srv_resolver: Optional[SrvResolver] = None, _well_known_resolver: Optional[WellKnownResolver] = None, ): + """ + Args: + server_name: Our homeserver name (used to label metrics) (`hs.hostname`). + """ + # proxy_reactor is not blocklisting reactor proxy_reactor = reactor @@ -127,6 +133,7 @@ class MatrixFederationAgent: if _well_known_resolver is None: _well_known_resolver = WellKnownResolver( + server_name, reactor, agent=BlocklistingAgentWrapper( ProxyAgent( diff --git a/synapse/http/federation/well_known_resolver.py b/synapse/http/federation/well_known_resolver.py index 9a6bac7281..911cbac7ea 100644 --- a/synapse/http/federation/well_known_resolver.py +++ b/synapse/http/federation/well_known_resolver.py @@ -91,12 +91,19 @@ class WellKnownResolver: def __init__( self, + server_name: str, reactor: IReactorTime, agent: IAgent, user_agent: bytes, well_known_cache: Optional[TTLCache[bytes, Optional[bytes]]] = None, had_well_known_cache: Optional[TTLCache[bytes, bool]] = None, ): + """ + Args: + server_name: Our homeserver name (used to label metrics) (`hs.hostname`). + """ + + self.server_name = server_name self._reactor = reactor self._clock = Clock(reactor) @@ -134,7 +141,13 @@ class WellKnownResolver: # TODO: should we linearise so that we don't end up doing two .well-known # requests for the same server in parallel? try: - with Measure(self._clock, "get_well_known"): + with Measure( + self._clock, + name="get_well_known", + # This should be our homeserver where the the code is running (used to + # label metrics) + server_name=self.server_name, + ): result: Optional[bytes] cache_period: float diff --git a/synapse/http/matrixfederationclient.py b/synapse/http/matrixfederationclient.py index 97a863a118..67ea9cdb81 100644 --- a/synapse/http/matrixfederationclient.py +++ b/synapse/http/matrixfederationclient.py @@ -417,6 +417,7 @@ class MatrixFederationHttpClient: if hs.get_instance_name() in outbound_federation_restricted_to: # Talk to federation directly federation_agent: IAgent = MatrixFederationAgent( + self.server_name, self.reactor, tls_client_options_factory, user_agent.encode("ascii"), @@ -697,7 +698,11 @@ class MatrixFederationHttpClient: outgoing_requests_counter.labels(request.method).inc() try: - with Measure(self.clock, "outbound_request"): + with Measure( + self.clock, + name="outbound_request", + server_name=self.server_name, + ): # we don't want all the fancy cookie and redirect handling # that treq.request gives: just use the raw Agent. diff --git a/synapse/metrics/__init__.py b/synapse/metrics/__init__.py index 86ac2c2395..7e508dba05 100644 --- a/synapse/metrics/__init__.py +++ b/synapse/metrics/__init__.py @@ -66,6 +66,21 @@ all_gauges: Dict[str, Collector] = {} HAVE_PROC_SELF_STAT = os.path.exists("/proc/self/stat") +SERVER_NAME_LABEL = "server_name" +""" +The `server_name` label is used to identify the homeserver that the metrics correspond +to. Because we support multiple instances of Synapse running in the same process and all +metrics are in a single global `REGISTRY`, we need to manually label any metrics. + +In the case of a Synapse homeserver, this should be set to the homeserver name +(`hs.hostname`). + +We're purposely not using the `instance` label for this purpose as that should be "The +: part of the target's URL that was scraped.". Also: "In Prometheus +terms, an endpoint you can scrape is called an *instance*, usually corresponding to a +single process." (source: https://prometheus.io/docs/concepts/jobs_instances/) +""" + class _RegistryProxy: @staticmethod @@ -192,7 +207,16 @@ class InFlightGauge(Generic[MetricsEntry], Collector): same key. Note that `callback` may be called on a separate thread. + + Args: + key: A tuple of label values, which must match the order of the + `labels` given to the constructor. + callback """ + assert len(key) == len(self.labels), ( + f"Expected {len(self.labels)} labels in `key`, got {len(key)}: {key}" + ) + with self._lock: self._registrations.setdefault(key, set()).add(callback) @@ -201,7 +225,17 @@ class InFlightGauge(Generic[MetricsEntry], Collector): key: Tuple[str, ...], callback: Callable[[MetricsEntry], None], ) -> None: - """Registers that we've exited a block with labels `key`.""" + """ + Registers that we've exited a block with labels `key`. + + Args: + key: A tuple of label values, which must match the order of the + `labels` given to the constructor. + callback + """ + assert len(key) == len(self.labels), ( + f"Expected {len(self.labels)} labels in `key`, got {len(key)}: {key}" + ) with self._lock: self._registrations.setdefault(key, set()).discard(callback) @@ -225,7 +259,7 @@ class InFlightGauge(Generic[MetricsEntry], Collector): with self._lock: callbacks = set(self._registrations[key]) - in_flight.add_metric(key, len(callbacks)) + in_flight.add_metric(labels=key, value=len(callbacks)) metrics = self._metrics_class() metrics_by_key[key] = metrics @@ -239,7 +273,7 @@ class InFlightGauge(Generic[MetricsEntry], Collector): "_".join([self.name, name]), "", labels=self.labels ) for key, metrics in metrics_by_key.items(): - gauge.add_metric(key, getattr(metrics, name)) + gauge.add_metric(labels=key, value=getattr(metrics, name)) yield gauge def _register_with_collector(self) -> None: diff --git a/synapse/module_api/callbacks/media_repository_callbacks.py b/synapse/module_api/callbacks/media_repository_callbacks.py index 6fa80a8eab..2ab65f9fd6 100644 --- a/synapse/module_api/callbacks/media_repository_callbacks.py +++ b/synapse/module_api/callbacks/media_repository_callbacks.py @@ -31,6 +31,7 @@ IS_USER_ALLOWED_TO_UPLOAD_MEDIA_OF_SIZE_CALLBACK = Callable[[str, int], Awaitabl class MediaRepositoryModuleApiCallbacks: def __init__(self, hs: "HomeServer") -> None: + self.server_name = hs.hostname self.clock = hs.get_clock() self._get_media_config_for_user_callbacks: List[ GET_MEDIA_CONFIG_FOR_USER_CALLBACK @@ -57,7 +58,11 @@ class MediaRepositoryModuleApiCallbacks: async def get_media_config_for_user(self, user_id: str) -> Optional[JsonDict]: for callback in self._get_media_config_for_user_callbacks: - with Measure(self.clock, f"{callback.__module__}.{callback.__qualname__}"): + with Measure( + self.clock, + name=f"{callback.__module__}.{callback.__qualname__}", + server_name=self.server_name, + ): res: Optional[JsonDict] = await delay_cancellation(callback(user_id)) if res: return res @@ -68,7 +73,11 @@ class MediaRepositoryModuleApiCallbacks: self, user_id: str, size: int ) -> bool: for callback in self._is_user_allowed_to_upload_media_of_size_callbacks: - with Measure(self.clock, f"{callback.__module__}.{callback.__qualname__}"): + with Measure( + self.clock, + name=f"{callback.__module__}.{callback.__qualname__}", + server_name=self.server_name, + ): res: bool = await delay_cancellation(callback(user_id, size)) if not res: return res diff --git a/synapse/module_api/callbacks/ratelimit_callbacks.py b/synapse/module_api/callbacks/ratelimit_callbacks.py index 64f9cc81e8..a580ea7d7c 100644 --- a/synapse/module_api/callbacks/ratelimit_callbacks.py +++ b/synapse/module_api/callbacks/ratelimit_callbacks.py @@ -43,6 +43,7 @@ GET_RATELIMIT_OVERRIDE_FOR_USER_CALLBACK = Callable[ class RatelimitModuleApiCallbacks: def __init__(self, hs: "HomeServer") -> None: + self.server_name = hs.hostname self.clock = hs.get_clock() self._get_ratelimit_override_for_user_callbacks: List[ GET_RATELIMIT_OVERRIDE_FOR_USER_CALLBACK @@ -64,7 +65,11 @@ class RatelimitModuleApiCallbacks: self, user_id: str, limiter_name: str ) -> Optional[RatelimitOverride]: for callback in self._get_ratelimit_override_for_user_callbacks: - with Measure(self.clock, f"{callback.__module__}.{callback.__qualname__}"): + with Measure( + self.clock, + name=f"{callback.__module__}.{callback.__qualname__}", + server_name=self.server_name, + ): res: Optional[RatelimitOverride] = await delay_cancellation( callback(user_id, limiter_name) ) diff --git a/synapse/module_api/callbacks/spamchecker_callbacks.py b/synapse/module_api/callbacks/spamchecker_callbacks.py index c43824f213..428e733979 100644 --- a/synapse/module_api/callbacks/spamchecker_callbacks.py +++ b/synapse/module_api/callbacks/spamchecker_callbacks.py @@ -356,6 +356,7 @@ class SpamCheckerModuleApiCallbacks: NOT_SPAM: Literal["NOT_SPAM"] = "NOT_SPAM" def __init__(self, hs: "synapse.server.HomeServer") -> None: + self.server_name = hs.hostname self.clock = hs.get_clock() self._check_event_for_spam_callbacks: List[CHECK_EVENT_FOR_SPAM_CALLBACK] = [] @@ -490,7 +491,11 @@ class SpamCheckerModuleApiCallbacks: generally discouraged as it doesn't support internationalization. """ for callback in self._check_event_for_spam_callbacks: - with Measure(self.clock, f"{callback.__module__}.{callback.__qualname__}"): + with Measure( + self.clock, + name=f"{callback.__module__}.{callback.__qualname__}", + server_name=self.server_name, + ): res = await delay_cancellation(callback(event)) if res is False or res == self.NOT_SPAM: # This spam-checker accepts the event. @@ -543,7 +548,11 @@ class SpamCheckerModuleApiCallbacks: True if the event should be silently dropped """ for callback in self._should_drop_federated_event_callbacks: - with Measure(self.clock, f"{callback.__module__}.{callback.__qualname__}"): + with Measure( + self.clock, + name=f"{callback.__module__}.{callback.__qualname__}", + server_name=self.server_name, + ): res: Union[bool, str] = await delay_cancellation(callback(event)) if res: return res @@ -565,7 +574,11 @@ class SpamCheckerModuleApiCallbacks: NOT_SPAM if the operation is permitted, [Codes, Dict] otherwise. """ for callback in self._user_may_join_room_callbacks: - with Measure(self.clock, f"{callback.__module__}.{callback.__qualname__}"): + with Measure( + self.clock, + name=f"{callback.__module__}.{callback.__qualname__}", + server_name=self.server_name, + ): res = await delay_cancellation(callback(user_id, room_id, is_invited)) # Normalize return values to `Codes` or `"NOT_SPAM"`. if res is True or res is self.NOT_SPAM: @@ -604,7 +617,11 @@ class SpamCheckerModuleApiCallbacks: NOT_SPAM if the operation is permitted, Codes otherwise. """ for callback in self._user_may_invite_callbacks: - with Measure(self.clock, f"{callback.__module__}.{callback.__qualname__}"): + with Measure( + self.clock, + name=f"{callback.__module__}.{callback.__qualname__}", + server_name=self.server_name, + ): res = await delay_cancellation( callback(inviter_userid, invitee_userid, room_id) ) @@ -643,7 +660,11 @@ class SpamCheckerModuleApiCallbacks: NOT_SPAM if the operation is permitted, Codes otherwise. """ for callback in self._federated_user_may_invite_callbacks: - with Measure(self.clock, f"{callback.__module__}.{callback.__qualname__}"): + with Measure( + self.clock, + name=f"{callback.__module__}.{callback.__qualname__}", + server_name=self.server_name, + ): res = await delay_cancellation(callback(event)) # Normalize return values to `Codes` or `"NOT_SPAM"`. if res is True or res is self.NOT_SPAM: @@ -686,7 +707,11 @@ class SpamCheckerModuleApiCallbacks: NOT_SPAM if the operation is permitted, Codes otherwise. """ for callback in self._user_may_send_3pid_invite_callbacks: - with Measure(self.clock, f"{callback.__module__}.{callback.__qualname__}"): + with Measure( + self.clock, + name=f"{callback.__module__}.{callback.__qualname__}", + server_name=self.server_name, + ): res = await delay_cancellation( callback(inviter_userid, medium, address, room_id) ) @@ -722,7 +747,11 @@ class SpamCheckerModuleApiCallbacks: room_config: The room creation configuration which is the body of the /createRoom request """ for callback in self._user_may_create_room_callbacks: - with Measure(self.clock, f"{callback.__module__}.{callback.__qualname__}"): + with Measure( + self.clock, + name=f"{callback.__module__}.{callback.__qualname__}", + server_name=self.server_name, + ): checker_args = inspect.signature(callback) # Also ensure backwards compatibility with spam checker callbacks # that don't expect the room_config argument. @@ -786,7 +815,11 @@ class SpamCheckerModuleApiCallbacks: content: The content of the state event """ for callback in self._user_may_send_state_event_callbacks: - with Measure(self.clock, f"{callback.__module__}.{callback.__qualname__}"): + with Measure( + self.clock, + name=f"{callback.__module__}.{callback.__qualname__}", + server_name=self.server_name, + ): # We make a copy of the content to ensure that the spam checker cannot modify it. res = await delay_cancellation( callback(user_id, room_id, event_type, state_key, deepcopy(content)) @@ -814,7 +847,11 @@ class SpamCheckerModuleApiCallbacks: """ for callback in self._user_may_create_room_alias_callbacks: - with Measure(self.clock, f"{callback.__module__}.{callback.__qualname__}"): + with Measure( + self.clock, + name=f"{callback.__module__}.{callback.__qualname__}", + server_name=self.server_name, + ): res = await delay_cancellation(callback(userid, room_alias)) if res is True or res is self.NOT_SPAM: continue @@ -847,7 +884,11 @@ class SpamCheckerModuleApiCallbacks: room_id: The ID of the room that would be published """ for callback in self._user_may_publish_room_callbacks: - with Measure(self.clock, f"{callback.__module__}.{callback.__qualname__}"): + with Measure( + self.clock, + name=f"{callback.__module__}.{callback.__qualname__}", + server_name=self.server_name, + ): res = await delay_cancellation(callback(userid, room_id)) if res is True or res is self.NOT_SPAM: continue @@ -889,7 +930,11 @@ class SpamCheckerModuleApiCallbacks: True if the user is spammy. """ for callback in self._check_username_for_spam_callbacks: - with Measure(self.clock, f"{callback.__module__}.{callback.__qualname__}"): + with Measure( + self.clock, + name=f"{callback.__module__}.{callback.__qualname__}", + server_name=self.server_name, + ): checker_args = inspect.signature(callback) # Make a copy of the user profile object to ensure the spam checker cannot # modify it. @@ -938,7 +983,11 @@ class SpamCheckerModuleApiCallbacks: """ for callback in self._check_registration_for_spam_callbacks: - with Measure(self.clock, f"{callback.__module__}.{callback.__qualname__}"): + with Measure( + self.clock, + name=f"{callback.__module__}.{callback.__qualname__}", + server_name=self.server_name, + ): behaviour = await delay_cancellation( callback(email_threepid, username, request_info, auth_provider_id) ) @@ -980,7 +1029,11 @@ class SpamCheckerModuleApiCallbacks: """ for callback in self._check_media_file_for_spam_callbacks: - with Measure(self.clock, f"{callback.__module__}.{callback.__qualname__}"): + with Measure( + self.clock, + name=f"{callback.__module__}.{callback.__qualname__}", + server_name=self.server_name, + ): res = await delay_cancellation(callback(file_wrapper, file_info)) # Normalize return values to `Codes` or `"NOT_SPAM"`. if res is False or res is self.NOT_SPAM: @@ -1027,7 +1080,11 @@ class SpamCheckerModuleApiCallbacks: """ for callback in self._check_login_for_spam_callbacks: - with Measure(self.clock, f"{callback.__module__}.{callback.__qualname__}"): + with Measure( + self.clock, + name=f"{callback.__module__}.{callback.__qualname__}", + server_name=self.server_name, + ): res = await delay_cancellation( callback( user_id, diff --git a/synapse/push/bulk_push_rule_evaluator.py b/synapse/push/bulk_push_rule_evaluator.py index 1f4f5b90c3..f20b98f73f 100644 --- a/synapse/push/bulk_push_rule_evaluator.py +++ b/synapse/push/bulk_push_rule_evaluator.py @@ -129,7 +129,8 @@ class BulkPushRuleEvaluator: def __init__(self, hs: "HomeServer"): self.hs = hs self.store = hs.get_datastores().main - self.clock = hs.get_clock() + self.server_name = hs.hostname # nb must be called this for @measure_func + self.clock = hs.get_clock() # nb must be called this for @measure_func self._event_auth_handler = hs.get_event_auth_handler() self.should_calculate_push_rules = self.hs.config.push.enable_push diff --git a/synapse/replication/http/federation.py b/synapse/replication/http/federation.py index f3f8ddfd65..c29ed8d149 100644 --- a/synapse/replication/http/federation.py +++ b/synapse/replication/http/federation.py @@ -76,6 +76,7 @@ class ReplicationFederationSendEventsRestServlet(ReplicationEndpoint): def __init__(self, hs: "HomeServer"): super().__init__(hs) + self.server_name = hs.hostname self.store = hs.get_datastores().main self._storage_controllers = hs.get_storage_controllers() self.clock = hs.get_clock() @@ -122,7 +123,9 @@ class ReplicationFederationSendEventsRestServlet(ReplicationEndpoint): async def _handle_request( # type: ignore[override] self, request: Request, content: JsonDict ) -> Tuple[int, JsonDict]: - with Measure(self.clock, "repl_fed_send_events_parse"): + with Measure( + self.clock, name="repl_fed_send_events_parse", server_name=self.server_name + ): room_id = content["room_id"] backfilled = content["backfilled"] diff --git a/synapse/replication/http/send_event.py b/synapse/replication/http/send_event.py index 01952a8d59..edda419a03 100644 --- a/synapse/replication/http/send_event.py +++ b/synapse/replication/http/send_event.py @@ -76,6 +76,7 @@ class ReplicationSendEventRestServlet(ReplicationEndpoint): def __init__(self, hs: "HomeServer"): super().__init__(hs) + self.server_name = hs.hostname self.event_creation_handler = hs.get_event_creation_handler() self.store = hs.get_datastores().main self._storage_controllers = hs.get_storage_controllers() @@ -121,7 +122,9 @@ class ReplicationSendEventRestServlet(ReplicationEndpoint): async def _handle_request( # type: ignore[override] self, request: Request, content: JsonDict, event_id: str ) -> Tuple[int, JsonDict]: - with Measure(self.clock, "repl_send_event_parse"): + with Measure( + self.clock, name="repl_send_event_parse", server_name=self.server_name + ): event_dict = content["event"] room_ver = KNOWN_ROOM_VERSIONS[content["room_version"]] internal_metadata = content["internal_metadata"] diff --git a/synapse/replication/http/send_events.py b/synapse/replication/http/send_events.py index d965ce5492..15e363b3eb 100644 --- a/synapse/replication/http/send_events.py +++ b/synapse/replication/http/send_events.py @@ -77,6 +77,7 @@ class ReplicationSendEventsRestServlet(ReplicationEndpoint): def __init__(self, hs: "HomeServer"): super().__init__(hs) + self.server_name = hs.hostname self.event_creation_handler = hs.get_event_creation_handler() self.store = hs.get_datastores().main self._storage_controllers = hs.get_storage_controllers() @@ -122,7 +123,9 @@ class ReplicationSendEventsRestServlet(ReplicationEndpoint): async def _handle_request( # type: ignore[override] self, request: Request, payload: JsonDict ) -> Tuple[int, JsonDict]: - with Measure(self.clock, "repl_send_events_parse"): + with Measure( + self.clock, name="repl_send_events_parse", server_name=self.server_name + ): events_and_context = [] events = payload["events"] rooms = set() diff --git a/synapse/replication/tcp/client.py b/synapse/replication/tcp/client.py index 0bd5478cd3..e71588f3de 100644 --- a/synapse/replication/tcp/client.py +++ b/synapse/replication/tcp/client.py @@ -75,6 +75,7 @@ class ReplicationDataHandler: """ def __init__(self, hs: "HomeServer"): + self.server_name = hs.hostname self.store = hs.get_datastores().main self.notifier = hs.get_notifier() self._reactor = hs.get_reactor() @@ -342,7 +343,11 @@ class ReplicationDataHandler: waiting_list.add((position, deferred)) # We measure here to get in flight counts and average waiting time. - with Measure(self._clock, "repl.wait_for_stream_position"): + with Measure( + self._clock, + name="repl.wait_for_stream_position", + server_name=self.server_name, + ): logger.info( "Waiting for repl stream %r to reach %s (%s); currently at: %s", stream_name, diff --git a/synapse/replication/tcp/resource.py b/synapse/replication/tcp/resource.py index d647a2b332..0080a76f6f 100644 --- a/synapse/replication/tcp/resource.py +++ b/synapse/replication/tcp/resource.py @@ -78,6 +78,7 @@ class ReplicationStreamer: """ def __init__(self, hs: "HomeServer"): + self.server_name = hs.hostname self.store = hs.get_datastores().main self.clock = hs.get_clock() self.notifier = hs.get_notifier() @@ -155,7 +156,11 @@ class ReplicationStreamer: while self.pending_updates: self.pending_updates = False - with Measure(self.clock, "repl.stream.get_updates"): + with Measure( + self.clock, + name="repl.stream.get_updates", + server_name=self.server_name, + ): all_streams = self.streams if self._replication_torture_level is not None: diff --git a/synapse/state/__init__.py b/synapse/state/__init__.py index 1c3e5d00a9..9c24525845 100644 --- a/synapse/state/__init__.py +++ b/synapse/state/__init__.py @@ -189,7 +189,8 @@ class StateHandler: """ def __init__(self, hs: "HomeServer"): - self.clock = hs.get_clock() + self.server_name = hs.hostname # nb must be called this for @measure_func + self.clock = hs.get_clock() # nb must be called this for @measure_func self.store = hs.get_datastores().main self._state_storage_controller = hs.get_storage_controllers().state self.hs = hs @@ -631,6 +632,7 @@ class StateResolutionHandler: """ def __init__(self, hs: "HomeServer"): + self.server_name = hs.hostname self.clock = hs.get_clock() self.resolve_linearizer = Linearizer(name="state_resolve_lock") @@ -747,7 +749,9 @@ class StateResolutionHandler: # which will be used as a cache key for future resolutions, but # not get persisted. - with Measure(self.clock, "state.create_group_ids"): + with Measure( + self.clock, name="state.create_group_ids", server_name=self.server_name + ): cache = _make_state_cache_entry(new_state, state_groups_ids) self._state_cache[group_names] = cache @@ -785,7 +789,9 @@ class StateResolutionHandler: a map from (type, state_key) to event_id. """ try: - with Measure(self.clock, "state._resolve_events") as m: + with Measure( + self.clock, name="state._resolve_events", server_name=self.server_name + ) as m: room_version_obj = KNOWN_ROOM_VERSIONS[room_version] if room_version_obj.state_res == StateResolutionVersions.V1: return await v1.resolve_events_with_store( diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index 2309b1648e..548e7df930 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -55,6 +55,7 @@ class SQLBaseStore(metaclass=ABCMeta): hs: "HomeServer", ): self.hs = hs + self.server_name = hs.hostname self._clock = hs.get_clock() self.database_engine = database.engine self.db_pool = database diff --git a/synapse/storage/controllers/persist_events.py b/synapse/storage/controllers/persist_events.py index f5131fe291..9f54430a22 100644 --- a/synapse/storage/controllers/persist_events.py +++ b/synapse/storage/controllers/persist_events.py @@ -337,6 +337,7 @@ class EventsPersistenceStorageController: assert stores.persist_events self.persist_events_store = stores.persist_events + self.server_name = hs.hostname self._clock = hs.get_clock() self._instance_name = hs.get_instance_name() self.is_mine_id = hs.is_mine_id @@ -616,7 +617,11 @@ class EventsPersistenceStorageController: state_delta_for_room = None if not backfilled: - with Measure(self._clock, "_calculate_state_and_extrem"): + with Measure( + self._clock, + name="_calculate_state_and_extrem", + server_name=self.server_name, + ): # Work out the new "current state" for the room. # We do this by working out what the new extremities are and then # calculating the state from that. @@ -627,7 +632,11 @@ class EventsPersistenceStorageController: room_id, chunk ) - with Measure(self._clock, "calculate_chain_cover_index_for_events"): + with Measure( + self._clock, + name="calculate_chain_cover_index_for_events", + server_name=self.server_name, + ): # We now calculate chain ID/sequence numbers for any state events we're # persisting. We ignore out of band memberships as we're not in the room # and won't have their auth chain (we'll fix it up later if we join the @@ -719,7 +728,11 @@ class EventsPersistenceStorageController: break logger.debug("Calculating state delta for room %s", room_id) - with Measure(self._clock, "persist_events.get_new_state_after_events"): + with Measure( + self._clock, + name="persist_events.get_new_state_after_events", + server_name=self.server_name, + ): res = await self._get_new_state_after_events( room_id, ev_ctx_rm, @@ -746,7 +759,11 @@ class EventsPersistenceStorageController: # removed keys entirely. delta = DeltaState([], delta_ids) elif current_state is not None: - with Measure(self._clock, "persist_events.calculate_state_delta"): + with Measure( + self._clock, + name="persist_events.calculate_state_delta", + server_name=self.server_name, + ): delta = await self._calculate_state_delta(room_id, current_state) if delta: diff --git a/synapse/storage/controllers/state.py b/synapse/storage/controllers/state.py index f28f5d7e03..d79791fed4 100644 --- a/synapse/storage/controllers/state.py +++ b/synapse/storage/controllers/state.py @@ -68,6 +68,7 @@ class StateStorageController: """ def __init__(self, hs: "HomeServer", stores: "Databases"): + self.server_name = hs.hostname self._is_mine_id = hs.is_mine_id self._clock = hs.get_clock() self.stores = stores @@ -812,7 +813,9 @@ class StateStorageController: state_group = object() assert state_group is not None - with Measure(self._clock, "get_joined_hosts"): + with Measure( + self._clock, name="get_joined_hosts", server_name=self.server_name + ): return await self._get_joined_hosts( room_id, state_group, state_entry=state_entry ) diff --git a/synapse/storage/databases/main/events_worker.py b/synapse/storage/databases/main/events_worker.py index 9cc0493307..2929b1d57a 100644 --- a/synapse/storage/databases/main/events_worker.py +++ b/synapse/storage/databases/main/events_worker.py @@ -1246,7 +1246,9 @@ class EventsWorkerStore(SQLBaseStore): to event row. Note that it may well contain additional events that were not part of this request. """ - with Measure(self._clock, "_fetch_event_list"): + with Measure( + self._clock, name="_fetch_event_list", server_name=self.server_name + ): try: events_to_fetch = { event_id for events, _ in event_list for event_id in events diff --git a/synapse/storage/databases/main/roommember.py b/synapse/storage/databases/main/roommember.py index 7ca73abb83..ce77e0b0d6 100644 --- a/synapse/storage/databases/main/roommember.py +++ b/synapse/storage/databases/main/roommember.py @@ -983,7 +983,11 @@ class RoomMemberWorkerStore(EventsWorkerStore, CacheInvalidationWorkerStore): `_get_user_ids_from_membership_event_ids` for any uncached events. """ - with Measure(self._clock, "get_joined_user_ids_from_state"): + with Measure( + self._clock, + name="get_joined_user_ids_from_state", + server_name=self.server_name, + ): users_in_room = set() member_event_ids = [ e_id for key, e_id in state.items() if key[0] == EventTypes.Member diff --git a/synapse/util/metrics.py b/synapse/util/metrics.py index 6a389f7a7e..608a4d4848 100644 --- a/synapse/util/metrics.py +++ b/synapse/util/metrics.py @@ -41,59 +41,100 @@ from synapse.logging.context import ( LoggingContext, current_context, ) -from synapse.metrics import InFlightGauge +from synapse.metrics import SERVER_NAME_LABEL, InFlightGauge from synapse.util import Clock logger = logging.getLogger(__name__) -block_counter = Counter("synapse_util_metrics_block_count", "", ["block_name"]) +# Metrics to see the number of and how much time is spend in various blocks of code. +# +block_counter = Counter( + "synapse_util_metrics_block_count", + documentation="The number of times this block has been called.", + labelnames=["block_name", SERVER_NAME_LABEL], +) +"""The number of times this block has been called.""" -block_timer = Counter("synapse_util_metrics_block_time_seconds", "", ["block_name"]) +block_timer = Counter( + "synapse_util_metrics_block_time_seconds", + documentation="The cumulative time spent executing this block across all calls, in seconds.", + labelnames=["block_name", SERVER_NAME_LABEL], +) +"""The cumulative time spent executing this block across all calls, in seconds.""" block_ru_utime = Counter( - "synapse_util_metrics_block_ru_utime_seconds", "", ["block_name"] + "synapse_util_metrics_block_ru_utime_seconds", + documentation="Resource usage: user CPU time in seconds used in this block", + labelnames=["block_name", SERVER_NAME_LABEL], ) +"""Resource usage: user CPU time in seconds used in this block""" block_ru_stime = Counter( - "synapse_util_metrics_block_ru_stime_seconds", "", ["block_name"] + "synapse_util_metrics_block_ru_stime_seconds", + documentation="Resource usage: system CPU time in seconds used in this block", + labelnames=["block_name", SERVER_NAME_LABEL], ) +"""Resource usage: system CPU time in seconds used in this block""" block_db_txn_count = Counter( - "synapse_util_metrics_block_db_txn_count", "", ["block_name"] + "synapse_util_metrics_block_db_txn_count", + documentation="Number of database transactions completed in this block", + labelnames=["block_name", SERVER_NAME_LABEL], ) +"""Number of database transactions completed in this block""" # seconds spent waiting for db txns, excluding scheduling time, in this block block_db_txn_duration = Counter( - "synapse_util_metrics_block_db_txn_duration_seconds", "", ["block_name"] + "synapse_util_metrics_block_db_txn_duration_seconds", + documentation="Seconds spent waiting for database txns, excluding scheduling time, in this block", + labelnames=["block_name", SERVER_NAME_LABEL], ) +"""Seconds spent waiting for database txns, excluding scheduling time, in this block""" # seconds spent waiting for a db connection, in this block block_db_sched_duration = Counter( - "synapse_util_metrics_block_db_sched_duration_seconds", "", ["block_name"] + "synapse_util_metrics_block_db_sched_duration_seconds", + documentation="Seconds spent waiting for a db connection, in this block", + labelnames=["block_name", SERVER_NAME_LABEL], ) +"""Seconds spent waiting for a db connection, in this block""" # This is dynamically created in InFlightGauge.__init__. -class _InFlightMetric(Protocol): +class _BlockInFlightMetric(Protocol): + """ + Sub-metrics used for the `InFlightGauge` for blocks. + """ + real_time_max: float + """The longest observed duration of any single execution of this block, in seconds.""" real_time_sum: float + """The cumulative time spent executing this block across all calls, in seconds.""" -# Tracks the number of blocks currently active -in_flight: InFlightGauge[_InFlightMetric] = InFlightGauge( +in_flight: InFlightGauge[_BlockInFlightMetric] = InFlightGauge( "synapse_util_metrics_block_in_flight", - "", - labels=["block_name"], + desc="Tracks the number of blocks currently active", + labels=["block_name", SERVER_NAME_LABEL], + # Matches the fields in the `_BlockInFlightMetric` sub_metrics=["real_time_max", "real_time_sum"], ) - +"""Tracks the number of blocks currently active""" P = ParamSpec("P") R = TypeVar("R") -class HasClock(Protocol): +class HasClockAndServerName(Protocol): clock: Clock + """ + Used to measure functions + """ + server_name: str + """ + The homeserver name that this Measure is associated with (used to label the metric) + (`hs.hostname`). + """ def measure_func( @@ -101,8 +142,9 @@ def measure_func( ) -> Callable[[Callable[P, Awaitable[R]]], Callable[P, Awaitable[R]]]: """Decorate an async method with a `Measure` context manager. - The Measure is created using `self.clock`; it should only be used to decorate - methods in classes defining an instance-level `clock` attribute. + The Measure is created using `self.clock` and `self.server_name; it should only be + used to decorate methods in classes defining an instance-level `clock` and + `server_name` attributes. Usage: @@ -116,16 +158,21 @@ def measure_func( with Measure(...): ... + Args: + name: The name of the metric to report (the block name) (used to label the + metric). Defaults to the name of the decorated function. """ def wrapper( - func: Callable[Concatenate[HasClock, P], Awaitable[R]], + func: Callable[Concatenate[HasClockAndServerName, P], Awaitable[R]], ) -> Callable[P, Awaitable[R]]: block_name = func.__name__ if name is None else name @wraps(func) - async def measured_func(self: HasClock, *args: P.args, **kwargs: P.kwargs) -> R: - with Measure(self.clock, block_name): + async def measured_func( + self: HasClockAndServerName, *args: P.args, **kwargs: P.kwargs + ) -> R: + with Measure(self.clock, name=block_name, server_name=self.server_name): r = await func(self, *args, **kwargs) return r @@ -142,19 +189,24 @@ class Measure: __slots__ = [ "clock", "name", + "server_name", "_logging_context", "start", ] - def __init__(self, clock: Clock, name: str) -> None: + def __init__(self, clock: Clock, *, name: str, server_name: str) -> None: """ Args: clock: An object with a "time()" method, which returns the current time in seconds. - name: The name of the metric to report. + name: The name of the metric to report (the block name) (used to label the + metric). + server_name: The homeserver name that this Measure is associated with (used to + label the metric) (`hs.hostname`). """ self.clock = clock self.name = name + self.server_name = server_name curr_context = current_context() if not curr_context: logger.warning( @@ -174,7 +226,7 @@ class Measure: self.start = self.clock.time() self._logging_context.__enter__() - in_flight.register((self.name,), self._update_in_flight) + in_flight.register((self.name, self.server_name), self._update_in_flight) logger.debug("Entering block %s", self.name) @@ -194,19 +246,20 @@ class Measure: duration = self.clock.time() - self.start usage = self.get_resource_usage() - in_flight.unregister((self.name,), self._update_in_flight) + in_flight.unregister((self.name, self.server_name), self._update_in_flight) self._logging_context.__exit__(exc_type, exc_val, exc_tb) try: - block_counter.labels(self.name).inc() - block_timer.labels(self.name).inc(duration) - block_ru_utime.labels(self.name).inc(usage.ru_utime) - block_ru_stime.labels(self.name).inc(usage.ru_stime) - block_db_txn_count.labels(self.name).inc(usage.db_txn_count) - block_db_txn_duration.labels(self.name).inc(usage.db_txn_duration_sec) - block_db_sched_duration.labels(self.name).inc(usage.db_sched_duration_sec) - except ValueError: - logger.warning("Failed to save metrics! Usage: %s", usage) + labels = {"block_name": self.name, SERVER_NAME_LABEL: self.server_name} + block_counter.labels(**labels).inc() + block_timer.labels(**labels).inc(duration) + block_ru_utime.labels(**labels).inc(usage.ru_utime) + block_ru_stime.labels(**labels).inc(usage.ru_stime) + block_db_txn_count.labels(**labels).inc(usage.db_txn_count) + block_db_txn_duration.labels(**labels).inc(usage.db_txn_duration_sec) + block_db_sched_duration.labels(**labels).inc(usage.db_sched_duration_sec) + except ValueError as exc: + logger.warning("Failed to save metrics! Usage: %s Error: %s", usage, exc) def get_resource_usage(self) -> ContextResourceUsage: """Get the resources used within this Measure block @@ -215,7 +268,7 @@ class Measure: """ return self._logging_context.get_resource_usage() - def _update_in_flight(self, metrics: _InFlightMetric) -> None: + def _update_in_flight(self, metrics: _BlockInFlightMetric) -> None: """Gets called when processing in flight metrics""" assert self.start is not None duration = self.clock.time() - self.start diff --git a/tests/handlers/test_typing.py b/tests/handlers/test_typing.py index 9d8960315f..394315d2b0 100644 --- a/tests/handlers/test_typing.py +++ b/tests/handlers/test_typing.py @@ -86,6 +86,7 @@ class TypingNotificationsTestCase(unittest.HomeserverTestCase): self.mock_federation_client = AsyncMock(spec=["put_json"]) self.mock_federation_client.put_json.return_value = (200, "OK") self.mock_federation_client.agent = MatrixFederationAgent( + "OUR_STUB_HOMESERVER_NAME", reactor, tls_client_options_factory=None, user_agent=b"SynapseInTrialTest/0.0.0", diff --git a/tests/http/federation/test_matrix_federation_agent.py b/tests/http/federation/test_matrix_federation_agent.py index 6b25e53c28..eb859ca47a 100644 --- a/tests/http/federation/test_matrix_federation_agent.py +++ b/tests/http/federation/test_matrix_federation_agent.py @@ -91,6 +91,7 @@ class MatrixFederationAgentTests(unittest.TestCase): "test_cache", timer=self.reactor.seconds ) self.well_known_resolver = WellKnownResolver( + "OUR_STUB_HOMESERVER_NAME", self.reactor, Agent(self.reactor, contextFactory=self.tls_factory), b"test-agent", @@ -269,6 +270,7 @@ class MatrixFederationAgentTests(unittest.TestCase): because it is created too early during setUp """ return MatrixFederationAgent( + "OUR_STUB_HOMESERVER_NAME", reactor=cast(ISynapseReactor, self.reactor), tls_client_options_factory=self.tls_factory, user_agent=b"test-agent", # Note that this is unused since _well_known_resolver is provided. @@ -1011,6 +1013,7 @@ class MatrixFederationAgentTests(unittest.TestCase): # Build a new agent and WellKnownResolver with a different tls factory tls_factory = FederationPolicyForHTTPS(config) agent = MatrixFederationAgent( + "OUR_STUB_HOMESERVER_NAME", reactor=self.reactor, tls_client_options_factory=tls_factory, user_agent=b"test-agent", # This is unused since _well_known_resolver is passed below. @@ -1018,6 +1021,7 @@ class MatrixFederationAgentTests(unittest.TestCase): ip_blocklist=IPSet(), _srv_resolver=self.mock_resolver, _well_known_resolver=WellKnownResolver( + "OUR_STUB_HOMESERVER_NAME", cast(ISynapseReactor, self.reactor), Agent(self.reactor, contextFactory=tls_factory), b"test-agent", diff --git a/tests/replication/test_federation_sender_shard.py b/tests/replication/test_federation_sender_shard.py index 58a7a9dc72..6c4145f2c2 100644 --- a/tests/replication/test_federation_sender_shard.py +++ b/tests/replication/test_federation_sender_shard.py @@ -68,6 +68,7 @@ class FederationSenderTestCase(BaseMultiWorkerStreamTestCase): reactor, _ = get_clock() self.matrix_federation_agent = MatrixFederationAgent( + "OUR_STUB_HOMESERVER_NAME", reactor, tls_client_options_factory=None, user_agent=b"SynapseInTrialTest/0.0.0", From 88785dbaeb30379a97c0f6ebfc51bff675605e54 Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Wed, 16 Jul 2025 16:04:57 -0500 Subject: [PATCH 011/185] Refactor cache metrics to be homeserver-scoped (#18604) (add `server_name` label to cache metrics). Part of https://github.com/element-hq/synapse/issues/18592 --- changelog.d/18604.misc | 1 + synapse/api/auth/base.py | 2 +- synapse/api/auth/msc3861_delegated.py | 6 +- synapse/appservice/__init__.py | 8 +- synapse/appservice/api.py | 6 +- synapse/appservice/scheduler.py | 2 +- synapse/config/appservice.py | 3 +- synapse/federation/federation_client.py | 6 +- synapse/federation/federation_server.py | 17 +++- synapse/handlers/appservice.py | 2 +- synapse/handlers/device.py | 2 + synapse/handlers/directory.py | 2 +- synapse/handlers/initial_sync.py | 7 +- synapse/handlers/message.py | 5 +- synapse/handlers/profile.py | 1 + synapse/handlers/room.py | 6 +- synapse/handlers/room_list.py | 14 ++- synapse/handlers/room_summary.py | 6 +- synapse/handlers/sync.py | 12 +-- synapse/handlers/typing.py | 5 +- .../federation/matrix_federation_agent.py | 11 ++- .../http/federation/well_known_resolver.py | 17 ++-- synapse/http/matrixfederationclient.py | 12 +-- synapse/media/url_previewer.py | 1 + synapse/push/bulk_push_rule_evaluator.py | 6 +- synapse/replication/http/_base.py | 7 +- synapse/rest/client/login.py | 4 +- synapse/rest/client/sync.py | 2 + .../server_notices/server_notices_manager.py | 10 +-- synapse/state/__init__.py | 1 + synapse/storage/_base.py | 2 +- synapse/storage/controllers/state.py | 2 +- .../storage/databases/main/account_data.py | 4 +- synapse/storage/databases/main/appservice.py | 2 +- synapse/storage/databases/main/client_ips.py | 5 +- synapse/storage/databases/main/deviceinbox.py | 11 ++- synapse/storage/databases/main/devices.py | 27 ++++-- .../databases/main/event_federation.py | 5 +- .../storage/databases/main/events_worker.py | 6 +- synapse/storage/databases/main/presence.py | 5 +- synapse/storage/databases/main/push_rule.py | 5 +- synapse/storage/databases/main/receipts.py | 5 +- synapse/storage/databases/main/stream.py | 9 +- synapse/storage/databases/state/store.py | 11 ++- synapse/util/caches/__init__.py | 87 +++++++++++++------ synapse/util/caches/deferred_cache.py | 5 ++ synapse/util/caches/descriptors.py | 24 ++++- synapse/util/caches/dictionary_cache.py | 11 ++- synapse/util/caches/expiringcache.py | 11 ++- synapse/util/caches/lrucache.py | 52 +++++++++-- synapse/util/caches/response_cache.py | 19 +++- synapse/util/caches/stream_change_cache.py | 18 +++- synapse/util/caches/ttlcache.py | 24 ++++- synmark/suites/lrucache.py | 2 +- synmark/suites/lrucache_evict.py | 2 +- tests/api/test_auth.py | 47 ++++++---- tests/api/test_ratelimiting.py | 6 +- tests/appservice/test_api.py | 4 +- tests/appservice/test_appservice.py | 7 +- tests/config/test_cache.py | 14 +-- tests/handlers/test_appservice.py | 11 +-- tests/handlers/test_device.py | 4 +- tests/handlers/test_e2e_keys.py | 6 +- tests/handlers/test_oauth_delegation.py | 2 +- tests/handlers/test_typing.py | 4 +- tests/handlers/test_user_directory.py | 14 +-- .../test_matrix_federation_agent.py | 28 +++--- tests/metrics/test_metrics.py | 4 +- tests/push/test_push_rule_evaluator.py | 6 +- tests/replication/tcp/streams/test_typing.py | 4 +- .../test_federation_sender_shard.py | 4 +- .../test_module_cache_invalidation.py | 1 + tests/rest/client/test_account.py | 4 +- tests/rest/client/test_devices.py | 4 +- tests/rest/client/test_directory.py | 4 +- tests/rest/client/test_login.py | 11 ++- tests/rest/client/test_register.py | 8 +- tests/rest/client/test_rooms.py | 2 +- tests/storage/test_user_directory.py | 5 +- tests/test_mau.py | 8 +- tests/util/caches/test_deferred_cache.py | 45 +++++++--- tests/util/caches/test_descriptors.py | 46 ++++++++++ tests/util/caches/test_response_cache.py | 4 +- tests/util/caches/test_ttlcache.py | 4 +- tests/util/test_dict_cache.py | 2 +- tests/util/test_expiring_cache.py | 21 ++++- tests/util/test_lrucache.py | 52 ++++++----- tests/util/test_stream_change_cache.py | 38 ++++++-- 88 files changed, 694 insertions(+), 268 deletions(-) create mode 100644 changelog.d/18604.misc diff --git a/changelog.d/18604.misc b/changelog.d/18604.misc new file mode 100644 index 0000000000..c06fb23af5 --- /dev/null +++ b/changelog.d/18604.misc @@ -0,0 +1 @@ +Refactor cache metrics to be homeserver-scoped. diff --git a/synapse/api/auth/base.py b/synapse/api/auth/base.py index 3126bc9f27..f97a71caf7 100644 --- a/synapse/api/auth/base.py +++ b/synapse/api/auth/base.py @@ -172,7 +172,7 @@ class BaseAuth: """ # It's ok if the app service is trying to use the sender from their registration - if app_service.sender == user_id: + if app_service.sender.to_string() == user_id: pass # Check to make sure the app service is allowed to control the user elif not app_service.is_interested_in_user(user_id): diff --git a/synapse/api/auth/msc3861_delegated.py b/synapse/api/auth/msc3861_delegated.py index 0cfdf15d60..ad8f4e04f6 100644 --- a/synapse/api/auth/msc3861_delegated.py +++ b/synapse/api/auth/msc3861_delegated.py @@ -176,6 +176,7 @@ class MSC3861DelegatedAuth(BaseAuth): assert self._config.client_id, "No client_id provided" assert auth_method is not None, "Invalid client_auth_method provided" + self.server_name = hs.hostname self._clock = hs.get_clock() self._http_client = hs.get_proxied_http_client() self._hostname = hs.hostname @@ -206,8 +207,9 @@ class MSC3861DelegatedAuth(BaseAuth): # In this case, the device still exists and it's not the end of the world for # the old access token to continue working for a short time. self._introspection_cache: ResponseCache[str] = ResponseCache( - self._clock, - "token_introspection", + clock=self._clock, + name="token_introspection", + server_name=self.server_name, timeout_ms=120_000, # don't log because the keys are access tokens enable_logging=False, diff --git a/synapse/appservice/__init__.py b/synapse/appservice/__init__.py index 6ee5240c4e..2d8d382e68 100644 --- a/synapse/appservice/__init__.py +++ b/synapse/appservice/__init__.py @@ -78,7 +78,7 @@ class ApplicationService: self, token: str, id: str, - sender: str, + sender: UserID, url: Optional[str] = None, namespaces: Optional[JsonDict] = None, hs_token: Optional[str] = None, @@ -96,6 +96,8 @@ class ApplicationService: self.hs_token = hs_token # The full Matrix ID for this application service's sender. self.sender = sender + # The application service user should be part of the server's domain. + self.server_name = sender.domain # nb must be called this for @cached self.namespaces = self._check_namespaces(namespaces) self.id = id self.ip_range_whitelist = ip_range_whitelist @@ -223,7 +225,7 @@ class ApplicationService: """ return ( # User is the appservice's configured sender_localpart user - user_id == self.sender + user_id == self.sender.to_string() # User is in the appservice's user namespace or self.is_user_in_namespace(user_id) ) @@ -347,7 +349,7 @@ class ApplicationService: def is_exclusive_user(self, user_id: str) -> bool: return ( self._is_exclusive(ApplicationService.NS_USERS, user_id) - or user_id == self.sender + or user_id == self.sender.to_string() ) def is_interested_in_protocol(self, protocol: str) -> bool: diff --git a/synapse/appservice/api.py b/synapse/appservice/api.py index 45371d6f3b..8c21e0951a 100644 --- a/synapse/appservice/api.py +++ b/synapse/appservice/api.py @@ -126,11 +126,15 @@ class ApplicationServiceApi(SimpleHttpClient): def __init__(self, hs: "HomeServer"): super().__init__(hs) + self.server_name = hs.hostname self.clock = hs.get_clock() self.config = hs.config.appservice self.protocol_meta_cache: ResponseCache[Tuple[str, str]] = ResponseCache( - hs.get_clock(), "as_protocol_meta", timeout_ms=HOUR_IN_MS + clock=hs.get_clock(), + name="as_protocol_meta", + server_name=self.server_name, + timeout_ms=HOUR_IN_MS, ) def _get_headers(self, service: "ApplicationService") -> Dict[bytes, List[bytes]]: diff --git a/synapse/appservice/scheduler.py b/synapse/appservice/scheduler.py index ab3f4e15fe..9d7fc0995a 100644 --- a/synapse/appservice/scheduler.py +++ b/synapse/appservice/scheduler.py @@ -319,7 +319,7 @@ class _ServiceQueuer: users: Set[str] = set() # The sender is always included - users.add(service.sender) + users.add(service.sender.to_string()) # All AS users that would receive the PDUs or EDUs sent to these rooms # are classed as 'interesting'. diff --git a/synapse/config/appservice.py b/synapse/config/appservice.py index dda6bcd1b7..81dbd330cc 100644 --- a/synapse/config/appservice.py +++ b/synapse/config/appservice.py @@ -122,8 +122,7 @@ def _load_appservice( localpart = as_info["sender_localpart"] if urlparse.quote(localpart) != localpart: raise ValueError("sender_localpart needs characters which are not URL encoded.") - user = UserID(localpart, hostname) - user_id = user.to_string() + user_id = UserID(localpart, hostname) # Rate limiting for users of this AS is on by default (excludes sender) rate_limited = as_info.get("rate_limited") diff --git a/synapse/federation/federation_client.py b/synapse/federation/federation_client.py index 2d1da70793..35c5ac6311 100644 --- a/synapse/federation/federation_client.py +++ b/synapse/federation/federation_client.py @@ -137,13 +137,14 @@ class FederationClient(FederationBase): self.state = hs.get_state_handler() self.transport_layer = hs.get_federation_transport_client() - self.hostname = hs.hostname + self.server_name = hs.hostname self.signing_key = hs.signing_key # Cache mapping `event_id` to a tuple of the event itself and the `pull_origin` # (which server we pulled the event from) self._get_pdu_cache: ExpiringCache[str, Tuple[EventBase, str]] = ExpiringCache( cache_name="get_pdu_cache", + server_name=self.server_name, clock=self._clock, max_len=1000, expiry_ms=120 * 1000, @@ -162,6 +163,7 @@ class FederationClient(FederationBase): Tuple[JsonDict, Sequence[JsonDict], Sequence[JsonDict], Sequence[str]], ] = ExpiringCache( cache_name="get_room_hierarchy_cache", + server_name=self.server_name, clock=self._clock, max_len=1000, expiry_ms=5 * 60 * 1000, @@ -1068,7 +1070,7 @@ class FederationClient(FederationBase): # there's some we never care about ev = builder.create_local_event_from_event_dict( self._clock, - self.hostname, + self.server_name, self.signing_key, room_version=room_version, event_dict=pdu_dict, diff --git a/synapse/federation/federation_server.py b/synapse/federation/federation_server.py index 2a7f5b2c4d..3e6b8b8493 100644 --- a/synapse/federation/federation_server.py +++ b/synapse/federation/federation_server.py @@ -159,7 +159,10 @@ class FederationServer(FederationBase): # We cache results for transaction with the same ID self._transaction_resp_cache: ResponseCache[Tuple[str, str]] = ResponseCache( - hs.get_clock(), "fed_txn_handler", timeout_ms=30000 + clock=hs.get_clock(), + name="fed_txn_handler", + server_name=self.server_name, + timeout_ms=30000, ) self.transaction_actions = TransactionActions(self.store) @@ -169,10 +172,18 @@ class FederationServer(FederationBase): # We cache responses to state queries, as they take a while and often # come in waves. self._state_resp_cache: ResponseCache[Tuple[str, Optional[str]]] = ( - ResponseCache(hs.get_clock(), "state_resp", timeout_ms=30000) + ResponseCache( + clock=hs.get_clock(), + name="state_resp", + server_name=self.server_name, + timeout_ms=30000, + ) ) self._state_ids_resp_cache: ResponseCache[Tuple[str, str]] = ResponseCache( - hs.get_clock(), "state_ids_resp", timeout_ms=30000 + clock=hs.get_clock(), + name="state_ids_resp", + server_name=self.server_name, + timeout_ms=30000, ) self._federation_metrics_domains = ( diff --git a/synapse/handlers/appservice.py b/synapse/handlers/appservice.py index 5aefc73aba..8c5308b522 100644 --- a/synapse/handlers/appservice.py +++ b/synapse/handlers/appservice.py @@ -846,7 +846,7 @@ class ApplicationServicesHandler: # user not found; could be the AS though, so check. services = self.store.get_app_services() - service_list = [s for s in services if s.sender == user_id] + service_list = [s for s in services if s.sender.to_string() == user_id] return len(service_list) == 0 async def _check_user_exists(self, user_id: str) -> bool: diff --git a/synapse/handlers/device.py b/synapse/handlers/device.py index e825626558..65fbb48768 100644 --- a/synapse/handlers/device.py +++ b/synapse/handlers/device.py @@ -1215,6 +1215,7 @@ class DeviceListUpdater(DeviceListWorkerUpdater): "Handles incoming device list updates from federation and updates the DB" def __init__(self, hs: "HomeServer", device_handler: DeviceHandler): + self.server_name = hs.hostname self.store = hs.get_datastores().main self.federation = hs.get_federation_client() self.server_name = hs.hostname # nb must be called this for @measure_func @@ -1235,6 +1236,7 @@ class DeviceListUpdater(DeviceListWorkerUpdater): # resyncs. self._seen_updates: ExpiringCache[str, Set[str]] = ExpiringCache( cache_name="device_update_edu", + server_name=self.server_name, clock=self.clock, max_len=10000, expiry_ms=30 * 60 * 1000, diff --git a/synapse/handlers/directory.py b/synapse/handlers/directory.py index 74c697960f..11284ccd0b 100644 --- a/synapse/handlers/directory.py +++ b/synapse/handlers/directory.py @@ -406,7 +406,7 @@ class DirectoryHandler: ] for service in interested_services: - if user_id == service.sender: + if user_id == service.sender.to_string(): # this user IS the app service so they can do whatever they like return True elif service.is_exclusive_alias(alias.to_string()): diff --git a/synapse/handlers/initial_sync.py b/synapse/handlers/initial_sync.py index bd3c87f5f4..75d64d2d50 100644 --- a/synapse/handlers/initial_sync.py +++ b/synapse/handlers/initial_sync.py @@ -60,6 +60,7 @@ logger = logging.getLogger(__name__) class InitialSyncHandler: def __init__(self, hs: "HomeServer"): + self.server_name = hs.hostname self.store = hs.get_datastores().main self.auth = hs.get_auth() self.state_handler = hs.get_state_handler() @@ -77,7 +78,11 @@ class InitialSyncHandler: bool, bool, ] - ] = ResponseCache(hs.get_clock(), "initial_sync_cache") + ] = ResponseCache( + clock=hs.get_clock(), + name="initial_sync_cache", + server_name=self.server_name, + ) self._event_serializer = hs.get_event_client_serializer() self._storage_controllers = hs.get_storage_controllers() self._state_storage_controller = self._storage_controllers.state diff --git a/synapse/handlers/message.py b/synapse/handlers/message.py index 7c76c187bc..aa295fb6c8 100644 --- a/synapse/handlers/message.py +++ b/synapse/handlers/message.py @@ -558,8 +558,9 @@ class EventCreationHandler: self._external_cache_joined_hosts_updates: Optional[ExpiringCache] = None if self._external_cache.is_enabled(): self._external_cache_joined_hosts_updates = ExpiringCache( - "_external_cache_joined_hosts_updates", - self.clock, + cache_name="_external_cache_joined_hosts_updates", + server_name=self.server_name, + clock=self.clock, expiry_ms=30 * 60 * 1000, ) diff --git a/synapse/handlers/profile.py b/synapse/handlers/profile.py index 76aa90e11b..4958ab5e75 100644 --- a/synapse/handlers/profile.py +++ b/synapse/handlers/profile.py @@ -55,6 +55,7 @@ class ProfileHandler: """ def __init__(self, hs: "HomeServer"): + self.server_name = hs.hostname # nb must be called this for @cached self.store = hs.get_datastores().main self.clock = hs.get_clock() self.hs = hs diff --git a/synapse/handlers/room.py b/synapse/handlers/room.py index b063e301e6..d8c4d0c20e 100644 --- a/synapse/handlers/room.py +++ b/synapse/handlers/room.py @@ -119,6 +119,7 @@ class EventContext: class RoomCreationHandler: def __init__(self, hs: "HomeServer"): + self.server_name = hs.hostname self.store = hs.get_datastores().main self._storage_controllers = hs.get_storage_controllers() self.auth = hs.get_auth() @@ -175,7 +176,10 @@ class RoomCreationHandler: # succession, only process the first attempt and return its result to # subsequent requests self._upgrade_response_cache: ResponseCache[Tuple[str, str]] = ResponseCache( - hs.get_clock(), "room_upgrade", timeout_ms=FIVE_MINUTES_IN_MS + clock=hs.get_clock(), + name="room_upgrade", + server_name=self.server_name, + timeout_ms=FIVE_MINUTES_IN_MS, ) self._server_notices_mxid = hs.config.servernotices.server_notices_mxid diff --git a/synapse/handlers/room_list.py b/synapse/handlers/room_list.py index 07eac71e2a..9d4307fb07 100644 --- a/synapse/handlers/room_list.py +++ b/synapse/handlers/room_list.py @@ -61,16 +61,26 @@ MAX_PUBLIC_ROOMS_IN_RESPONSE = 100 class RoomListHandler: def __init__(self, hs: "HomeServer"): + self.server_name = hs.hostname # nb must be called this for @cached self.store = hs.get_datastores().main self._storage_controllers = hs.get_storage_controllers() self.hs = hs self.enable_room_list_search = hs.config.roomdirectory.enable_room_list_search self.response_cache: ResponseCache[ Tuple[Optional[int], Optional[str], Optional[ThirdPartyInstanceID]] - ] = ResponseCache(hs.get_clock(), "room_list") + ] = ResponseCache( + clock=hs.get_clock(), + name="room_list", + server_name=self.server_name, + ) self.remote_response_cache: ResponseCache[ Tuple[str, Optional[int], Optional[str], bool, Optional[str]] - ] = ResponseCache(hs.get_clock(), "remote_room_list", timeout_ms=30 * 1000) + ] = ResponseCache( + clock=hs.get_clock(), + name="remote_room_list", + server_name=self.server_name, + timeout_ms=30 * 1000, + ) async def get_local_public_room_list( self, diff --git a/synapse/handlers/room_summary.py b/synapse/handlers/room_summary.py index 1f322ac263..838fee6a30 100644 --- a/synapse/handlers/room_summary.py +++ b/synapse/handlers/room_summary.py @@ -96,6 +96,7 @@ class RoomSummaryHandler: _PAGINATION_SESSION_VALIDITY_PERIOD_MS = 5 * 60 * 1000 def __init__(self, hs: "HomeServer"): + self.server_name = hs.hostname self._event_auth_handler = hs.get_event_auth_handler() self._store = hs.get_datastores().main self._storage_controllers = hs.get_storage_controllers() @@ -121,8 +122,9 @@ class RoomSummaryHandler: Optional[Tuple[str, ...]], ] ] = ResponseCache( - hs.get_clock(), - "get_room_hierarchy", + clock=hs.get_clock(), + name="get_room_hierarchy", + server_name=self.server_name, ) self._msc3266_enabled = hs.config.experimental.msc3266_enabled diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py index 7b99defac1..69064e751a 100644 --- a/synapse/handlers/sync.py +++ b/synapse/handlers/sync.py @@ -353,8 +353,9 @@ class SyncHandler: # cached result any more, and we could flush the entry from the cache to save # memory. self.response_cache: ResponseCache[SyncRequestKey] = ResponseCache( - hs.get_clock(), - "sync", + clock=hs.get_clock(), + name="sync", + server_name=self.server_name, timeout_ms=hs.config.caches.sync_response_cache_duration, ) @@ -362,8 +363,9 @@ class SyncHandler: self.lazy_loaded_members_cache: ExpiringCache[ Tuple[str, Optional[str]], LruCache[str, str] ] = ExpiringCache( - "lazy_loaded_members_cache", - self.clock, + cache_name="lazy_loaded_members_cache", + server_name=self.server_name, + clock=self.clock, max_len=0, expiry_ms=LAZY_LOADED_MEMBERS_CACHE_MAX_AGE, ) @@ -1134,7 +1136,7 @@ class SyncHandler: ) if cache is None: logger.debug("creating LruCache for %r", cache_key) - cache = LruCache(LAZY_LOADED_MEMBERS_CACHE_MAX_SIZE) + cache = LruCache(max_size=LAZY_LOADED_MEMBERS_CACHE_MAX_SIZE) self.lazy_loaded_members_cache[cache_key] = cache else: logger.debug("found LruCache for %r", cache_key) diff --git a/synapse/handlers/typing.py b/synapse/handlers/typing.py index bbef3a59a5..3c49655598 100644 --- a/synapse/handlers/typing.py +++ b/synapse/handlers/typing.py @@ -263,6 +263,7 @@ class TypingWriterHandler(FollowerTypingHandler): assert hs.get_instance_name() in hs.config.worker.writers.typing + self.server_name = hs.hostname self.auth = hs.get_auth() self.notifier = hs.get_notifier() self.event_auth_handler = hs.get_event_auth_handler() @@ -280,7 +281,9 @@ class TypingWriterHandler(FollowerTypingHandler): # caches which room_ids changed at which serials self._typing_stream_change_cache = StreamChangeCache( - "TypingStreamChangeCache", self._latest_room_serial + name="TypingStreamChangeCache", + server_name=self.server_name, + current_stream_pos=self._latest_room_serial, ) def _handle_timeout_for_member(self, now: int, member: RoomMember) -> None: diff --git a/synapse/http/federation/matrix_federation_agent.py b/synapse/http/federation/matrix_federation_agent.py index 4a47665abd..15609a799f 100644 --- a/synapse/http/federation/matrix_federation_agent.py +++ b/synapse/http/federation/matrix_federation_agent.py @@ -104,6 +104,13 @@ class MatrixFederationAgent: """ Args: server_name: Our homeserver name (used to label metrics) (`hs.hostname`). + reactor + tls_client_options_factory + user_agent + ip_allowlist + ip_blocklist + _srv_resolver + _well_known_resolver """ # proxy_reactor is not blocklisting reactor @@ -133,8 +140,8 @@ class MatrixFederationAgent: if _well_known_resolver is None: _well_known_resolver = WellKnownResolver( - server_name, - reactor, + server_name=server_name, + reactor=reactor, agent=BlocklistingAgentWrapper( ProxyAgent( reactor, diff --git a/synapse/http/federation/well_known_resolver.py b/synapse/http/federation/well_known_resolver.py index 911cbac7ea..70242ad0ae 100644 --- a/synapse/http/federation/well_known_resolver.py +++ b/synapse/http/federation/well_known_resolver.py @@ -77,10 +77,6 @@ WELL_KNOWN_RETRY_ATTEMPTS = 3 logger = logging.getLogger(__name__) -_well_known_cache: TTLCache[bytes, Optional[bytes]] = TTLCache("well-known") -_had_valid_well_known_cache: TTLCache[bytes, bool] = TTLCache("had-valid-well-known") - - @attr.s(slots=True, frozen=True, auto_attribs=True) class WellKnownLookupResult: delegated_server: Optional[bytes] @@ -101,6 +97,11 @@ class WellKnownResolver: """ Args: server_name: Our homeserver name (used to label metrics) (`hs.hostname`). + reactor + agent + user_agent + well_known_cache + had_well_known_cache """ self.server_name = server_name @@ -108,10 +109,14 @@ class WellKnownResolver: self._clock = Clock(reactor) if well_known_cache is None: - well_known_cache = _well_known_cache + well_known_cache = TTLCache( + cache_name="well-known", server_name=server_name + ) if had_well_known_cache is None: - had_well_known_cache = _had_valid_well_known_cache + had_well_known_cache = TTLCache( + cache_name="had-valid-well-known", server_name=server_name + ) self._well_known_cache = well_known_cache self._had_valid_well_known_cache = had_well_known_cache diff --git a/synapse/http/matrixfederationclient.py b/synapse/http/matrixfederationclient.py index 67ea9cdb81..0013b97723 100644 --- a/synapse/http/matrixfederationclient.py +++ b/synapse/http/matrixfederationclient.py @@ -417,12 +417,12 @@ class MatrixFederationHttpClient: if hs.get_instance_name() in outbound_federation_restricted_to: # Talk to federation directly federation_agent: IAgent = MatrixFederationAgent( - self.server_name, - self.reactor, - tls_client_options_factory, - user_agent.encode("ascii"), - hs.config.server.federation_ip_range_allowlist, - hs.config.server.federation_ip_range_blocklist, + server_name=self.server_name, + reactor=self.reactor, + tls_client_options_factory=tls_client_options_factory, + user_agent=user_agent.encode("ascii"), + ip_allowlist=hs.config.server.federation_ip_range_allowlist, + ip_blocklist=hs.config.server.federation_ip_range_blocklist, ) else: proxy_authorization_secret = hs.config.worker.worker_replication_secret diff --git a/synapse/media/url_previewer.py b/synapse/media/url_previewer.py index 0c665e1942..eb0104e543 100644 --- a/synapse/media/url_previewer.py +++ b/synapse/media/url_previewer.py @@ -200,6 +200,7 @@ class UrlPreviewer: # JSON-encoded OG metadata self._cache: ExpiringCache[str, ObservableDeferred] = ExpiringCache( cache_name="url_previews", + server_name=self.server_name, clock=self.clock, # don't spider URLs more often than once an hour expiry_ms=ONE_HOUR, diff --git a/synapse/push/bulk_push_rule_evaluator.py b/synapse/push/bulk_push_rule_evaluator.py index f20b98f73f..fed9931930 100644 --- a/synapse/push/bulk_push_rule_evaluator.py +++ b/synapse/push/bulk_push_rule_evaluator.py @@ -128,6 +128,7 @@ class BulkPushRuleEvaluator: def __init__(self, hs: "HomeServer"): self.hs = hs + self.server_name = hs.hostname self.store = hs.get_datastores().main self.server_name = hs.hostname # nb must be called this for @measure_func self.clock = hs.get_clock() # nb must be called this for @measure_func @@ -137,10 +138,11 @@ class BulkPushRuleEvaluator: self._related_event_match_enabled = self.hs.config.experimental.msc3664_enabled self.room_push_rule_cache_metrics = register_cache( - "cache", - "room_push_rule_cache", + cache_type="cache", + cache_name="room_push_rule_cache", cache=[], # Meaningless size, as this isn't a cache that stores values, resizable=False, + server_name=self.server_name, ) async def _get_rules_for_event( diff --git a/synapse/replication/http/_base.py b/synapse/replication/http/_base.py index 0002538680..31204a8384 100644 --- a/synapse/replication/http/_base.py +++ b/synapse/replication/http/_base.py @@ -121,9 +121,14 @@ class ReplicationEndpoint(metaclass=abc.ABCMeta): WAIT_FOR_STREAMS: ClassVar[bool] = True def __init__(self, hs: "HomeServer"): + self.server_name = hs.hostname + if self.CACHE: self.response_cache: ResponseCache[str] = ResponseCache( - hs.get_clock(), "repl." + self.NAME, timeout_ms=30 * 60 * 1000 + clock=hs.get_clock(), + name="repl." + self.NAME, + server_name=self.server_name, + timeout_ms=30 * 60 * 1000, ) # We reserve `instance_name` as a parameter to sending requests, so we diff --git a/synapse/rest/client/login.py b/synapse/rest/client/login.py index 8a781f759c..aa0aa36cd9 100644 --- a/synapse/rest/client/login.py +++ b/synapse/rest/client/login.py @@ -314,7 +314,9 @@ class LoginRestServlet(RestServlet): should_issue_refresh_token=should_issue_refresh_token, # The user represented by an appservice's configured sender_localpart # is not actually created in Synapse. - should_check_deactivated_or_locked=qualified_user_id != appservice.sender, + should_check_deactivated_or_locked=( + qualified_user_id != appservice.sender.to_string() + ), request_info=request_info, ) diff --git a/synapse/rest/client/sync.py b/synapse/rest/client/sync.py index bac02122d0..c9fb9dc4d3 100644 --- a/synapse/rest/client/sync.py +++ b/synapse/rest/client/sync.py @@ -111,6 +111,7 @@ class SyncRestServlet(RestServlet): def __init__(self, hs: "HomeServer"): super().__init__() self.hs = hs + self.server_name = hs.hostname self.auth = hs.get_auth() self.store = hs.get_datastores().main self.sync_handler = hs.get_sync_handler() @@ -125,6 +126,7 @@ class SyncRestServlet(RestServlet): self._json_filter_cache: LruCache[str, bool] = LruCache( max_size=1000, cache_name="sync_valid_filter", + server_name=self.server_name, ) # Ratelimiter for presence updates, keyed by requester. diff --git a/synapse/server_notices/server_notices_manager.py b/synapse/server_notices/server_notices_manager.py index 001a290e87..19f86b5a56 100644 --- a/synapse/server_notices/server_notices_manager.py +++ b/synapse/server_notices/server_notices_manager.py @@ -35,6 +35,7 @@ SERVER_NOTICE_ROOM_TAG = "m.server_notice" class ServerNoticesManager: def __init__(self, hs: "HomeServer"): + self.server_name = hs.hostname # nb must be called this for @cached self._store = hs.get_datastores().main self._config = hs.config self._account_data_handler = hs.get_account_data_handler() @@ -44,7 +45,6 @@ class ServerNoticesManager: self._message_handler = hs.get_message_handler() self._storage_controllers = hs.get_storage_controllers() self._is_mine_id = hs.is_mine_id - self._server_name = hs.hostname self._notifier = hs.get_notifier() self.server_notices_mxid = self._config.servernotices.server_notices_mxid @@ -77,7 +77,7 @@ class ServerNoticesManager: assert self.server_notices_mxid is not None requester = create_requester( - self.server_notices_mxid, authenticated_entity=self._server_name + self.server_notices_mxid, authenticated_entity=self.server_name ) logger.info("Sending server notice to %s", user_id) @@ -151,7 +151,7 @@ class ServerNoticesManager: assert self._is_mine_id(user_id), "Cannot send server notices to remote users" requester = create_requester( - self.server_notices_mxid, authenticated_entity=self._server_name + self.server_notices_mxid, authenticated_entity=self.server_name ) room_id = await self.maybe_get_notice_room_for_user(user_id) @@ -256,7 +256,7 @@ class ServerNoticesManager: """ assert self.server_notices_mxid is not None requester = create_requester( - self.server_notices_mxid, authenticated_entity=self._server_name + self.server_notices_mxid, authenticated_entity=self.server_name ) # Check whether the user has already joined or been invited to this room. If @@ -279,7 +279,7 @@ class ServerNoticesManager: if self._config.servernotices.server_notices_auto_join: user_requester = create_requester( - user_id, authenticated_entity=self._server_name + user_id, authenticated_entity=self.server_name ) await self._room_member_handler.update_membership( requester=user_requester, diff --git a/synapse/state/__init__.py b/synapse/state/__init__.py index 9c24525845..976a98a58b 100644 --- a/synapse/state/__init__.py +++ b/synapse/state/__init__.py @@ -641,6 +641,7 @@ class StateResolutionHandler: self._state_cache: ExpiringCache[FrozenSet[int], _StateCacheEntry] = ( ExpiringCache( cache_name="state_cache", + server_name=self.server_name, clock=self.clock, max_len=100000, expiry_ms=EVICTION_TIMEOUT_SECONDS * 1000, diff --git a/synapse/storage/_base.py b/synapse/storage/_base.py index 548e7df930..d55c9e18ed 100644 --- a/synapse/storage/_base.py +++ b/synapse/storage/_base.py @@ -55,7 +55,7 @@ class SQLBaseStore(metaclass=ABCMeta): hs: "HomeServer", ): self.hs = hs - self.server_name = hs.hostname + self.server_name = hs.hostname # nb must be called this for @cached self._clock = hs.get_clock() self.database_engine = database.engine self.db_pool = database diff --git a/synapse/storage/controllers/state.py b/synapse/storage/controllers/state.py index d79791fed4..8997f4526f 100644 --- a/synapse/storage/controllers/state.py +++ b/synapse/storage/controllers/state.py @@ -68,7 +68,7 @@ class StateStorageController: """ def __init__(self, hs: "HomeServer", stores: "Databases"): - self.server_name = hs.hostname + self.server_name = hs.hostname # nb must be called this for @cached self._is_mine_id = hs.is_mine_id self._clock = hs.get_clock() self.stores = stores diff --git a/synapse/storage/databases/main/account_data.py b/synapse/storage/databases/main/account_data.py index be5c494ed6..883ab93f7c 100644 --- a/synapse/storage/databases/main/account_data.py +++ b/synapse/storage/databases/main/account_data.py @@ -90,7 +90,9 @@ class AccountDataWorkerStore(PushRulesWorkerStore, CacheInvalidationWorkerStore) account_max = self.get_max_account_data_stream_id() self._account_data_stream_cache = StreamChangeCache( - "AccountDataAndTagsChangeCache", account_max + name="AccountDataAndTagsChangeCache", + server_name=self.server_name, + current_stream_pos=account_max, ) self.db_pool.updates.register_background_index_update( diff --git a/synapse/storage/databases/main/appservice.py b/synapse/storage/databases/main/appservice.py index 766c94fc14..9862e574fd 100644 --- a/synapse/storage/databases/main/appservice.py +++ b/synapse/storage/databases/main/appservice.py @@ -126,7 +126,7 @@ class ApplicationServiceWorkerStore(RoomMemberWorkerStore): The application service or None. """ for service in self.services_cache: - if service.sender == user_id: + if service.sender.to_string() == user_id: return service return None diff --git a/synapse/storage/databases/main/client_ips.py b/synapse/storage/databases/main/client_ips.py index 69008804bd..cf7bc4ac69 100644 --- a/synapse/storage/databases/main/client_ips.py +++ b/synapse/storage/databases/main/client_ips.py @@ -421,6 +421,7 @@ class ClientIpWorkerStore(ClientIpBackgroundUpdateStore, MonthlyActiveUsersWorke hs: "HomeServer", ): super().__init__(database, db_conn, hs) + self.server_name = hs.hostname if hs.config.redis.redis_enabled: # If we're using Redis, we can shift this update process off to @@ -434,7 +435,9 @@ class ClientIpWorkerStore(ClientIpBackgroundUpdateStore, MonthlyActiveUsersWorke # (user_id, access_token, ip,) -> last_seen self.client_ip_last_seen = LruCache[Tuple[str, str, str], int]( - cache_name="client_ip_last_seen", max_size=50000 + cache_name="client_ip_last_seen", + server_name=self.server_name, + max_size=50000, ) if hs.config.worker.run_background_tasks and self.user_ips_max_age: diff --git a/synapse/storage/databases/main/deviceinbox.py b/synapse/storage/databases/main/deviceinbox.py index a22eab2474..da10afbebe 100644 --- a/synapse/storage/databases/main/deviceinbox.py +++ b/synapse/storage/databases/main/deviceinbox.py @@ -94,6 +94,7 @@ class DeviceInboxWorkerStore(SQLBaseStore): Tuple[str, Optional[str]], int ] = ExpiringCache( cache_name="last_device_delete_cache", + server_name=self.server_name, clock=self._clock, max_len=10000, expiry_ms=30 * 60 * 1000, @@ -127,8 +128,9 @@ class DeviceInboxWorkerStore(SQLBaseStore): limit=1000, ) self._device_inbox_stream_cache = StreamChangeCache( - "DeviceInboxStreamChangeCache", - min_device_inbox_id, + name="DeviceInboxStreamChangeCache", + server_name=self.server_name, + current_stream_pos=min_device_inbox_id, prefilled_cache=device_inbox_prefill, ) @@ -143,8 +145,9 @@ class DeviceInboxWorkerStore(SQLBaseStore): limit=1000, ) self._device_federation_outbox_stream_cache = StreamChangeCache( - "DeviceFederationOutboxStreamChangeCache", - min_device_outbox_id, + name="DeviceFederationOutboxStreamChangeCache", + server_name=self.server_name, + current_stream_pos=min_device_outbox_id, prefilled_cache=device_outbox_prefill, ) diff --git a/synapse/storage/databases/main/devices.py b/synapse/storage/databases/main/devices.py index 941d278e6c..f054c66102 100644 --- a/synapse/storage/databases/main/devices.py +++ b/synapse/storage/databases/main/devices.py @@ -128,8 +128,9 @@ class DeviceWorkerStore(RoomMemberWorkerStore, EndToEndKeyWorkerStore): limit=10000, ) self._device_list_stream_cache = StreamChangeCache( - "DeviceListStreamChangeCache", - min_device_list_id, + name="DeviceListStreamChangeCache", + server_name=self.server_name, + current_stream_pos=min_device_list_id, prefilled_cache=device_list_prefill, ) @@ -142,8 +143,9 @@ class DeviceWorkerStore(RoomMemberWorkerStore, EndToEndKeyWorkerStore): limit=10000, ) self._device_list_room_stream_cache = StreamChangeCache( - "DeviceListRoomStreamChangeCache", - min_device_list_room_id, + name="DeviceListRoomStreamChangeCache", + server_name=self.server_name, + current_stream_pos=min_device_list_room_id, prefilled_cache=device_list_room_prefill, ) @@ -159,8 +161,9 @@ class DeviceWorkerStore(RoomMemberWorkerStore, EndToEndKeyWorkerStore): limit=1000, ) self._user_signature_stream_cache = StreamChangeCache( - "UserSignatureStreamChangeCache", - user_signature_stream_list_id, + name="UserSignatureStreamChangeCache", + server_name=self.server_name, + current_stream_pos=user_signature_stream_list_id, prefilled_cache=user_signature_stream_prefill, ) @@ -178,8 +181,9 @@ class DeviceWorkerStore(RoomMemberWorkerStore, EndToEndKeyWorkerStore): limit=10000, ) self._device_list_federation_stream_cache = StreamChangeCache( - "DeviceListFederationStreamChangeCache", - device_list_federation_list_id, + name="DeviceListFederationStreamChangeCache", + server_name=self.server_name, + current_stream_pos=device_list_federation_list_id, prefilled_cache=device_list_federation_prefill, ) @@ -1769,11 +1773,16 @@ class DeviceStore(DeviceWorkerStore, DeviceBackgroundUpdateStore): hs: "HomeServer", ): super().__init__(database, db_conn, hs) + self.server_name = hs.hostname # Map of (user_id, device_id) -> bool. If there is an entry that implies # the device exists. self.device_id_exists_cache: LruCache[Tuple[str, str], Literal[True]] = ( - LruCache(cache_name="device_id_exists", max_size=10000) + LruCache( + cache_name="device_id_exists", + server_name=self.server_name, + max_size=10000, + ) ) async def store_device( diff --git a/synapse/storage/databases/main/event_federation.py b/synapse/storage/databases/main/event_federation.py index dfc25d8935..8e623bf061 100644 --- a/synapse/storage/databases/main/event_federation.py +++ b/synapse/storage/databases/main/event_federation.py @@ -148,7 +148,10 @@ class EventFederationWorkerStore( # Cache of event ID to list of auth event IDs and their depths. self._event_auth_cache: LruCache[str, List[Tuple[str, int]]] = LruCache( - 500000, "_event_auth_cache", size_callback=len + max_size=500000, + server_name=self.server_name, + cache_name="_event_auth_cache", + size_callback=len, ) # Flag used by unit tests to disable fallback when there is no chain cover diff --git a/synapse/storage/databases/main/events_worker.py b/synapse/storage/databases/main/events_worker.py index 2929b1d57a..d9ef93f826 100644 --- a/synapse/storage/databases/main/events_worker.py +++ b/synapse/storage/databases/main/events_worker.py @@ -269,8 +269,9 @@ class EventsWorkerStore(SQLBaseStore): limit=1000, ) self._curr_state_delta_stream_cache: StreamChangeCache = StreamChangeCache( - "_curr_state_delta_stream_cache", - min_curr_state_delta_id, + name="_curr_state_delta_stream_cache", + server_name=self.server_name, + current_stream_pos=min_curr_state_delta_id, prefilled_cache=curr_state_delta_prefill, ) @@ -283,6 +284,7 @@ class EventsWorkerStore(SQLBaseStore): self._get_event_cache: AsyncLruCache[Tuple[str], EventCacheEntry] = ( AsyncLruCache( + server_name=self.server_name, cache_name="*getEvent*", max_size=hs.config.caches.event_cache_size, # `extra_index_cb` Returns a tuple as that is the key type diff --git a/synapse/storage/databases/main/presence.py b/synapse/storage/databases/main/presence.py index 065c885603..12cff1d352 100644 --- a/synapse/storage/databases/main/presence.py +++ b/synapse/storage/databases/main/presence.py @@ -108,8 +108,9 @@ class PresenceStore(PresenceBackgroundUpdateStore, CacheInvalidationWorkerStore) max_value=self._presence_id_gen.get_current_token(), ) self.presence_stream_cache = StreamChangeCache( - "PresenceStreamChangeCache", - min_presence_val, + name="PresenceStreamChangeCache", + server_name=self.server_name, + current_stream_pos=min_presence_val, prefilled_cache=presence_cache_prefill, ) diff --git a/synapse/storage/databases/main/push_rule.py b/synapse/storage/databases/main/push_rule.py index 86c87f78bf..3bc977d497 100644 --- a/synapse/storage/databases/main/push_rule.py +++ b/synapse/storage/databases/main/push_rule.py @@ -163,8 +163,9 @@ class PushRulesWorkerStore( ) self.push_rules_stream_cache = StreamChangeCache( - "PushRulesStreamChangeCache", - push_rules_id, + name="PushRulesStreamChangeCache", + server_name=self.server_name, + current_stream_pos=push_rules_id, prefilled_cache=push_rules_prefill, ) diff --git a/synapse/storage/databases/main/receipts.py b/synapse/storage/databases/main/receipts.py index 9964331510..81f50467e7 100644 --- a/synapse/storage/databases/main/receipts.py +++ b/synapse/storage/databases/main/receipts.py @@ -158,8 +158,9 @@ class ReceiptsWorkerStore(SQLBaseStore): limit=10000, ) self._receipts_stream_cache = StreamChangeCache( - "ReceiptsRoomChangeCache", - min_receipts_stream_id, + name="ReceiptsRoomChangeCache", + server_name=self.server_name, + current_stream_pos=min_receipts_stream_id, prefilled_cache=receipts_stream_prefill, ) diff --git a/synapse/storage/databases/main/stream.py b/synapse/storage/databases/main/stream.py index 3fda49f31f..b6c6b69b22 100644 --- a/synapse/storage/databases/main/stream.py +++ b/synapse/storage/databases/main/stream.py @@ -617,12 +617,15 @@ class StreamWorkerStore(EventsWorkerStore, SQLBaseStore): max_value=events_max, ) self._events_stream_cache = StreamChangeCache( - "EventsRoomStreamChangeCache", - min_event_val, + name="EventsRoomStreamChangeCache", + server_name=self.server_name, + current_stream_pos=min_event_val, prefilled_cache=event_cache_prefill, ) self._membership_stream_cache = StreamChangeCache( - "MembershipStreamChangeCache", events_max + name="MembershipStreamChangeCache", + server_name=self.server_name, + current_stream_pos=events_max, ) self._stream_order_on_start = self.get_room_max_stream_ordering() diff --git a/synapse/storage/databases/state/store.py b/synapse/storage/databases/state/store.py index c1a66dcba0..9b3b7e086f 100644 --- a/synapse/storage/databases/state/store.py +++ b/synapse/storage/databases/state/store.py @@ -92,6 +92,7 @@ class StateGroupDataStore(StateBackgroundUpdateStore, SQLBaseStore): ): super().__init__(database, db_conn, hs) self._state_deletion_store = state_deletion_store + self.server_name = hs.hostname # Originally the state store used a single DictionaryCache to cache the # event IDs for the state types in a given state group to avoid hammering @@ -123,14 +124,16 @@ class StateGroupDataStore(StateBackgroundUpdateStore, SQLBaseStore): # vast majority of state in Matrix (today) is member events. self._state_group_cache: DictionaryCache[int, StateKey, str] = DictionaryCache( - "*stateGroupCache*", + name="*stateGroupCache*", + server_name=self.server_name, # TODO: this hasn't been tuned yet - 50000, + max_entries=50000, ) self._state_group_members_cache: DictionaryCache[int, StateKey, str] = ( DictionaryCache( - "*stateGroupMembersCache*", - 500000, + name="*stateGroupMembersCache*", + server_name=self.server_name, + max_entries=500000, ) ) diff --git a/synapse/util/caches/__init__.py b/synapse/util/caches/__init__.py index 76e6e139c6..3087ad6adc 100644 --- a/synapse/util/caches/__init__.py +++ b/synapse/util/caches/__init__.py @@ -31,6 +31,7 @@ from prometheus_client import REGISTRY from prometheus_client.core import Gauge from synapse.config.cache import add_resizable_cache +from synapse.metrics import SERVER_NAME_LABEL from synapse.util.metrics import DynamicCollectorRegistry logger = logging.getLogger(__name__) @@ -46,50 +47,65 @@ CACHE_METRIC_REGISTRY = DynamicCollectorRegistry() caches_by_name: Dict[str, Sized] = {} cache_size = Gauge( - "synapse_util_caches_cache_size", "", ["name"], registry=CACHE_METRIC_REGISTRY + "synapse_util_caches_cache_size", + "", + labelnames=["name", SERVER_NAME_LABEL], + registry=CACHE_METRIC_REGISTRY, ) cache_hits = Gauge( - "synapse_util_caches_cache_hits", "", ["name"], registry=CACHE_METRIC_REGISTRY + "synapse_util_caches_cache_hits", + "", + labelnames=["name", SERVER_NAME_LABEL], + registry=CACHE_METRIC_REGISTRY, ) cache_evicted = Gauge( "synapse_util_caches_cache_evicted_size", "", - ["name", "reason"], + labelnames=["name", "reason", SERVER_NAME_LABEL], registry=CACHE_METRIC_REGISTRY, ) cache_total = Gauge( - "synapse_util_caches_cache", "", ["name"], registry=CACHE_METRIC_REGISTRY + "synapse_util_caches_cache", + "", + labelnames=["name", SERVER_NAME_LABEL], + registry=CACHE_METRIC_REGISTRY, ) cache_max_size = Gauge( - "synapse_util_caches_cache_max_size", "", ["name"], registry=CACHE_METRIC_REGISTRY + "synapse_util_caches_cache_max_size", + "", + labelnames=["name", SERVER_NAME_LABEL], + registry=CACHE_METRIC_REGISTRY, ) cache_memory_usage = Gauge( "synapse_util_caches_cache_size_bytes", "Estimated memory usage of the caches", - ["name"], + labelnames=["name", SERVER_NAME_LABEL], registry=CACHE_METRIC_REGISTRY, ) response_cache_size = Gauge( "synapse_util_caches_response_cache_size", "", - ["name"], + labelnames=["name", SERVER_NAME_LABEL], registry=CACHE_METRIC_REGISTRY, ) response_cache_hits = Gauge( "synapse_util_caches_response_cache_hits", "", - ["name"], + labelnames=["name", SERVER_NAME_LABEL], registry=CACHE_METRIC_REGISTRY, ) response_cache_evicted = Gauge( "synapse_util_caches_response_cache_evicted_size", "", - ["name", "reason"], + labelnames=["name", "reason", SERVER_NAME_LABEL], registry=CACHE_METRIC_REGISTRY, ) response_cache_total = Gauge( - "synapse_util_caches_response_cache", "", ["name"], registry=CACHE_METRIC_REGISTRY + "synapse_util_caches_response_cache", + "", + labelnames=["name", SERVER_NAME_LABEL], + registry=CACHE_METRIC_REGISTRY, ) @@ -103,12 +119,17 @@ class EvictionReason(Enum): invalidation = auto() -@attr.s(slots=True, auto_attribs=True) +@attr.s(slots=True, auto_attribs=True, kw_only=True) class CacheMetric: + """ + Used to track cache metrics + """ + _cache: Sized _cache_type: str _cache_name: str _collect_callback: Optional[Callable] + _server_name: str hits: int = 0 misses: int = 0 @@ -145,34 +166,34 @@ class CacheMetric: def collect(self) -> None: try: + labels_base = { + "name": self._cache_name, + SERVER_NAME_LABEL: self._server_name, + } if self._cache_type == "response_cache": - response_cache_size.labels(self._cache_name).set(len(self._cache)) - response_cache_hits.labels(self._cache_name).set(self.hits) + response_cache_size.labels(**labels_base).set(len(self._cache)) + response_cache_hits.labels(**labels_base).set(self.hits) for reason in EvictionReason: - response_cache_evicted.labels(self._cache_name, reason.name).set( - self.eviction_size_by_reason[reason] - ) - response_cache_total.labels(self._cache_name).set( - self.hits + self.misses - ) + response_cache_evicted.labels( + **{**labels_base, "reason": reason.name} + ).set(self.eviction_size_by_reason[reason]) + response_cache_total.labels(**labels_base).set(self.hits + self.misses) else: - cache_size.labels(self._cache_name).set(len(self._cache)) - cache_hits.labels(self._cache_name).set(self.hits) + cache_size.labels(**labels_base).set(len(self._cache)) + cache_hits.labels(**labels_base).set(self.hits) for reason in EvictionReason: - cache_evicted.labels(self._cache_name, reason.name).set( + cache_evicted.labels(**{**labels_base, "reason": reason.name}).set( self.eviction_size_by_reason[reason] ) - cache_total.labels(self._cache_name).set(self.hits + self.misses) + cache_total.labels(**labels_base).set(self.hits + self.misses) max_size = getattr(self._cache, "max_size", None) if max_size: - cache_max_size.labels(self._cache_name).set(max_size) + cache_max_size.labels(**labels_base).set(max_size) if TRACK_MEMORY_USAGE: # self.memory_usage can be None if nothing has been inserted # into the cache yet. - cache_memory_usage.labels(self._cache_name).set( - self.memory_usage or 0 - ) + cache_memory_usage.labels(**labels_base).set(self.memory_usage or 0) if self._collect_callback: self._collect_callback() except Exception as e: @@ -181,9 +202,11 @@ class CacheMetric: def register_cache( + *, cache_type: str, cache_name: str, cache: Sized, + server_name: str, collect_callback: Optional[Callable] = None, resizable: bool = True, resize_callback: Optional[Callable] = None, @@ -196,6 +219,8 @@ def register_cache( cache_name: name of the cache cache: cache itself, which must implement __len__(), and may optionally implement a max_size property + server_name: The homeserver name that this cache is associated with + (used to label the metric) (`hs.hostname`). collect_callback: If given, a function which is called during metric collection to update additional metrics. resizable: Whether this cache supports being resized, in which case either @@ -210,7 +235,13 @@ def register_cache( resize_callback = cache.set_cache_factor # type: ignore add_resizable_cache(cache_name, resize_callback) - metric = CacheMetric(cache, cache_type, cache_name, collect_callback) + metric = CacheMetric( + cache=cache, + cache_type=cache_type, + cache_name=cache_name, + server_name=server_name, + collect_callback=collect_callback, + ) metric_name = "cache_%s_%s" % (cache_type, cache_name) caches_by_name[cache_name] = cache CACHE_METRIC_REGISTRY.register_hook(metric_name, metric.collect) diff --git a/synapse/util/caches/deferred_cache.py b/synapse/util/caches/deferred_cache.py index 14868fa4d3..0c6c912918 100644 --- a/synapse/util/caches/deferred_cache.py +++ b/synapse/util/caches/deferred_cache.py @@ -79,7 +79,9 @@ class DeferredCache(Generic[KT, VT]): def __init__( self, + *, name: str, + server_name: str, max_entries: int = 1000, tree: bool = False, iterable: bool = False, @@ -89,6 +91,8 @@ class DeferredCache(Generic[KT, VT]): """ Args: name: The name of the cache + server_name: server_name: The homeserver name that this cache is associated with + (used to label the metric) (`hs.hostname`). max_entries: Maximum amount of entries that the cache will hold tree: Use a TreeCache instead of a dict as the underlying cache type iterable: If True, count each item in the cached object as an entry, @@ -113,6 +117,7 @@ class DeferredCache(Generic[KT, VT]): # a Deferred. self.cache: LruCache[KT, VT] = LruCache( max_size=max_entries, + server_name=server_name, cache_name=name, cache_type=cache_type, size_callback=( diff --git a/synapse/util/caches/descriptors.py b/synapse/util/caches/descriptors.py index 29a9586710..9630cd6d26 100644 --- a/synapse/util/caches/descriptors.py +++ b/synapse/util/caches/descriptors.py @@ -33,6 +33,7 @@ from typing import ( List, Mapping, Optional, + Protocol, Sequence, Tuple, Type, @@ -153,6 +154,14 @@ class _CacheDescriptorBase: ) +class HasServerName(Protocol): + server_name: str + """ + The homeserver name that this cache is associated with (used to label the metric) + (`hs.hostname`). + """ + + class DeferredCacheDescriptor(_CacheDescriptorBase): """A method decorator that applies a memoizing cache around the function. @@ -200,6 +209,7 @@ class DeferredCacheDescriptor(_CacheDescriptorBase): def __init__( self, + *, orig: Callable[..., Any], max_entries: int = 1000, num_args: Optional[int] = None, @@ -229,10 +239,20 @@ class DeferredCacheDescriptor(_CacheDescriptorBase): self.prune_unread_entries = prune_unread_entries def __get__( - self, obj: Optional[Any], owner: Optional[Type] + self, obj: Optional[HasServerName], owner: Optional[Type] ) -> Callable[..., "defer.Deferred[Any]"]: + # We need access to instance-level `obj.server_name` attribute + assert obj is not None, ( + "Cannot call cached method from class (❌ `MyClass.cached_method()`) " + "and must be called from an instance (✅ `MyClass().cached_method()`). " + ) + assert obj.server_name is not None, ( + "The `server_name` attribute must be set on the object where `@cached` decorator is used." + ) + cache: DeferredCache[CacheKey, Any] = DeferredCache( name=self.name, + server_name=obj.server_name, max_entries=self.max_entries, tree=self.tree, iterable=self.iterable, @@ -490,7 +510,7 @@ class _CachedFunctionDescriptor: def __call__(self, orig: F) -> CachedFunction[F]: d = DeferredCacheDescriptor( - orig, + orig=orig, max_entries=self.max_entries, num_args=self.num_args, uncached_args=self.uncached_args, diff --git a/synapse/util/caches/dictionary_cache.py b/synapse/util/caches/dictionary_cache.py index 14bd3ba3b0..168ddc51cd 100644 --- a/synapse/util/caches/dictionary_cache.py +++ b/synapse/util/caches/dictionary_cache.py @@ -127,7 +127,15 @@ class DictionaryCache(Generic[KT, DKT, DV]): for the '2' dict key. """ - def __init__(self, name: str, max_entries: int = 1000): + def __init__(self, *, name: str, server_name: str, max_entries: int = 1000): + """ + Args: + name + server_name: The homeserver name that this cache is associated with + (used to label the metric) (`hs.hostname`). + max_entries + """ + # We use a single LruCache to store two different types of entries: # 1. Map from (key, dict_key) -> dict value (or sentinel, indicating # the key doesn't exist in the dict); and @@ -152,6 +160,7 @@ class DictionaryCache(Generic[KT, DKT, DV]): Union[_PerKeyValue, Dict[DKT, DV]], ] = LruCache( max_size=max_entries, + server_name=server_name, cache_name=name, cache_type=TreeCache, size_callback=len, diff --git a/synapse/util/caches/expiringcache.py b/synapse/util/caches/expiringcache.py index 3198fdd2ed..4be4c6f01b 100644 --- a/synapse/util/caches/expiringcache.py +++ b/synapse/util/caches/expiringcache.py @@ -46,7 +46,9 @@ VT = TypeVar("VT") class ExpiringCache(Generic[KT, VT]): def __init__( self, + *, cache_name: str, + server_name: str, clock: Clock, max_len: int = 0, expiry_ms: int = 0, @@ -56,6 +58,8 @@ class ExpiringCache(Generic[KT, VT]): """ Args: cache_name: Name of this cache, used for logging. + server_name: The homeserver name that this cache is associated + with (used to label the metric) (`hs.hostname`). clock max_len: Max size of dict. If the dict grows larger than this then the oldest items get automatically evicted. Default is 0, @@ -83,7 +87,12 @@ class ExpiringCache(Generic[KT, VT]): self.iterable = iterable - self.metrics = register_cache("expiring", cache_name, self) + self.metrics = register_cache( + cache_type="expiring", + cache_name=cache_name, + cache=self, + server_name=server_name, + ) if not self._expiry_ms: # Don't bother starting the loop if things never expire diff --git a/synapse/util/caches/lrucache.py b/synapse/util/caches/lrucache.py index 2e5efa3a52..466362e79c 100644 --- a/synapse/util/caches/lrucache.py +++ b/synapse/util/caches/lrucache.py @@ -376,9 +376,43 @@ class LruCache(Generic[KT, VT]): If cache_type=TreeCache, all keys must be tuples. """ + @overload def __init__( self, + *, max_size: int, + server_name: str, + cache_name: str, + cache_type: Type[Union[dict, TreeCache]] = dict, + size_callback: Optional[Callable[[VT], int]] = None, + metrics_collection_callback: Optional[Callable[[], None]] = None, + apply_cache_factor_from_config: bool = True, + clock: Optional[Clock] = None, + prune_unread_entries: bool = True, + extra_index_cb: Optional[Callable[[KT, VT], KT]] = None, + ): ... + + @overload + def __init__( + self, + *, + max_size: int, + server_name: Literal[None] = None, + cache_name: Literal[None] = None, + cache_type: Type[Union[dict, TreeCache]] = dict, + size_callback: Optional[Callable[[VT], int]] = None, + metrics_collection_callback: Optional[Callable[[], None]] = None, + apply_cache_factor_from_config: bool = True, + clock: Optional[Clock] = None, + prune_unread_entries: bool = True, + extra_index_cb: Optional[Callable[[KT, VT], KT]] = None, + ): ... + + def __init__( + self, + *, + max_size: int, + server_name: Optional[str] = None, cache_name: Optional[str] = None, cache_type: Type[Union[dict, TreeCache]] = dict, size_callback: Optional[Callable[[VT], int]] = None, @@ -392,8 +426,13 @@ class LruCache(Generic[KT, VT]): Args: max_size: The maximum amount of entries the cache can hold - cache_name: The name of this cache, for the prometheus metrics. If unset, - no metrics will be reported on this cache. + server_name: The homeserver name that this cache is associated with + (used to label the metric) (`hs.hostname`). Must be set if `cache_name` is + set. If unset, no metrics will be reported on this cache. + + cache_name: The name of this cache, for the prometheus metrics. Must be set + if `server_name` is set. If unset, no metrics will be reported on this + cache. cache_type: type of underlying cache to be used. Typically one of dict @@ -457,11 +496,12 @@ class LruCache(Generic[KT, VT]): # do yet when we get resized. self._on_resize: Optional[Callable[[], None]] = None - if cache_name is not None: + if cache_name is not None and server_name is not None: metrics: Optional[CacheMetric] = register_cache( - "lru_cache", - cache_name, - self, + cache_type="lru_cache", + cache_name=cache_name, + cache=self, + server_name=server_name, collect_callback=metrics_collection_callback, ) else: diff --git a/synapse/util/caches/response_cache.py b/synapse/util/caches/response_cache.py index 54b99134b9..49a9151916 100644 --- a/synapse/util/caches/response_cache.py +++ b/synapse/util/caches/response_cache.py @@ -103,18 +103,35 @@ class ResponseCache(Generic[KV]): def __init__( self, + *, clock: Clock, name: str, + server_name: str, timeout_ms: float = 0, enable_logging: bool = True, ): + """ + Args: + clock + name + server_name: The homeserver name that this cache is associated + with (used to label the metric) (`hs.hostname`). + timeout_ms + enable_logging + """ self._result_cache: Dict[KV, ResponseCacheEntry] = {} self.clock = clock self.timeout_sec = timeout_ms / 1000.0 self._name = name - self._metrics = register_cache("response_cache", name, self, resizable=False) + self._metrics = register_cache( + cache_type="response_cache", + cache_name=name, + cache=self, + server_name=server_name, + resizable=False, + ) self._enable_logging = enable_logging def size(self) -> int: diff --git a/synapse/util/caches/stream_change_cache.py b/synapse/util/caches/stream_change_cache.py index 5ac8643eef..2cffd352d8 100644 --- a/synapse/util/caches/stream_change_cache.py +++ b/synapse/util/caches/stream_change_cache.py @@ -73,11 +73,23 @@ class StreamChangeCache: def __init__( self, + *, name: str, + server_name: str, current_stream_pos: int, max_size: int = 10000, prefilled_cache: Optional[Mapping[EntityType, int]] = None, ) -> None: + """ + Args: + name + server_name: The homeserver name that this cache is associated with + (used to label the metric) (`hs.hostname`). + current_stream_pos + max_size + prefilled_cache + """ + self._original_max_size: int = max_size self._max_size = math.floor(max_size) @@ -96,7 +108,11 @@ class StreamChangeCache: self.name = name self.metrics = caches.register_cache( - "cache", self.name, self._cache, resize_callback=self.set_cache_factor + cache_type="cache", + cache_name=self.name, + server_name=server_name, + cache=self._cache, + resize_callback=self.set_cache_factor, ) if prefilled_cache: diff --git a/synapse/util/caches/ttlcache.py b/synapse/util/caches/ttlcache.py index 26a088603a..18c3a1e51c 100644 --- a/synapse/util/caches/ttlcache.py +++ b/synapse/util/caches/ttlcache.py @@ -40,7 +40,21 @@ VT = TypeVar("VT") class TTLCache(Generic[KT, VT]): """A key/value cache implementation where each entry has its own TTL""" - def __init__(self, cache_name: str, timer: Callable[[], float] = time.time): + def __init__( + self, + *, + cache_name: str, + server_name: str, + timer: Callable[[], float] = time.time, + ): + """ + Args: + cache_name + server_name: The homeserver name that this cache is associated with + (used to label the metric) (`hs.hostname`). + timer: Function used to get the current time in seconds since the epoch. + """ + # map from key to _CacheEntry self._data: Dict[KT, _CacheEntry[KT, VT]] = {} @@ -49,7 +63,13 @@ class TTLCache(Generic[KT, VT]): self._timer = timer - self._metrics = register_cache("ttl", cache_name, self, resizable=False) + self._metrics = register_cache( + cache_type="ttl", + cache_name=cache_name, + cache=self, + server_name=server_name, + resizable=False, + ) def set(self, key: KT, value: VT, ttl: float) -> None: """Add/update an entry in the cache diff --git a/synmark/suites/lrucache.py b/synmark/suites/lrucache.py index 49d200c43b..d109441e55 100644 --- a/synmark/suites/lrucache.py +++ b/synmark/suites/lrucache.py @@ -29,7 +29,7 @@ async def main(reactor: ISynapseReactor, loops: int) -> float: """ Benchmark `loops` number of insertions into LruCache without eviction. """ - cache: LruCache[int, bool] = LruCache(loops) + cache: LruCache[int, bool] = LruCache(max_size=loops) start = perf_counter() diff --git a/synmark/suites/lrucache_evict.py b/synmark/suites/lrucache_evict.py index 77061625a9..00cfdd0447 100644 --- a/synmark/suites/lrucache_evict.py +++ b/synmark/suites/lrucache_evict.py @@ -30,7 +30,7 @@ async def main(reactor: ISynapseReactor, loops: int) -> float: Benchmark `loops` number of insertions into LruCache where half of them are evicted. """ - cache: LruCache[int, bool] = LruCache(loops // 2) + cache: LruCache[int, bool] = LruCache(max_size=loops // 2) start = perf_counter() diff --git a/tests/api/test_auth.py b/tests/api/test_auth.py index bd229cf7e9..95a4683d03 100644 --- a/tests/api/test_auth.py +++ b/tests/api/test_auth.py @@ -60,7 +60,7 @@ class AuthTestCase(unittest.HomeserverTestCase): # modify its config instead of the hs' self.auth_blocking = AuthBlocking(hs) - self.test_user = "@foo:bar" + self.test_user_id = UserID.from_string("@foo:bar") self.test_token = b"_test_token_" # this is overridden for the appservice tests @@ -71,7 +71,7 @@ class AuthTestCase(unittest.HomeserverTestCase): def test_get_user_by_req_user_valid_token(self) -> None: user_info = TokenLookupResult( - user_id=self.test_user, token_id=5, device_id="device" + user_id=self.test_user_id.to_string(), token_id=5, device_id="device" ) self.store.get_user_by_access_token = AsyncMock(return_value=user_info) self.store.mark_access_token_as_used = AsyncMock(return_value=None) @@ -81,7 +81,7 @@ class AuthTestCase(unittest.HomeserverTestCase): request.args[b"access_token"] = [self.test_token] request.requestHeaders.getRawHeaders = mock_getRawHeaders() requester = self.get_success(self.auth.get_user_by_req(request)) - self.assertEqual(requester.user.to_string(), self.test_user) + self.assertEqual(requester.user, self.test_user_id) def test_get_user_by_req_user_bad_token(self) -> None: self.store.get_user_by_access_token = AsyncMock(return_value=None) @@ -96,7 +96,7 @@ class AuthTestCase(unittest.HomeserverTestCase): self.assertEqual(f.errcode, "M_UNKNOWN_TOKEN") def test_get_user_by_req_user_missing_token(self) -> None: - user_info = TokenLookupResult(user_id=self.test_user, token_id=5) + user_info = TokenLookupResult(user_id=self.test_user_id.to_string(), token_id=5) self.store.get_user_by_access_token = AsyncMock(return_value=user_info) request = Mock(args={}) @@ -109,7 +109,10 @@ class AuthTestCase(unittest.HomeserverTestCase): def test_get_user_by_req_appservice_valid_token(self) -> None: app_service = Mock( - token="foobar", url="a_url", sender=self.test_user, ip_range_whitelist=None + token="foobar", + url="a_url", + sender=self.test_user_id, + ip_range_whitelist=None, ) self.store.get_app_service_by_token = Mock(return_value=app_service) self.store.get_user_by_access_token = AsyncMock(return_value=None) @@ -119,7 +122,7 @@ class AuthTestCase(unittest.HomeserverTestCase): request.args[b"access_token"] = [self.test_token] request.requestHeaders.getRawHeaders = mock_getRawHeaders() requester = self.get_success(self.auth.get_user_by_req(request)) - self.assertEqual(requester.user.to_string(), self.test_user) + self.assertEqual(requester.user, self.test_user_id) def test_get_user_by_req_appservice_valid_token_good_ip(self) -> None: from netaddr import IPSet @@ -127,7 +130,7 @@ class AuthTestCase(unittest.HomeserverTestCase): app_service = Mock( token="foobar", url="a_url", - sender=self.test_user, + sender=self.test_user_id.to_string(), ip_range_whitelist=IPSet(["192.168.0.0/16"]), ) self.store.get_app_service_by_token = Mock(return_value=app_service) @@ -138,7 +141,7 @@ class AuthTestCase(unittest.HomeserverTestCase): request.args[b"access_token"] = [self.test_token] request.requestHeaders.getRawHeaders = mock_getRawHeaders() requester = self.get_success(self.auth.get_user_by_req(request)) - self.assertEqual(requester.user.to_string(), self.test_user) + self.assertEqual(requester.user, self.test_user_id) def test_get_user_by_req_appservice_valid_token_bad_ip(self) -> None: from netaddr import IPSet @@ -146,7 +149,7 @@ class AuthTestCase(unittest.HomeserverTestCase): app_service = Mock( token="foobar", url="a_url", - sender=self.test_user, + sender=self.test_user_id, ip_range_whitelist=IPSet(["192.168.0.0/16"]), ) self.store.get_app_service_by_token = Mock(return_value=app_service) @@ -176,7 +179,7 @@ class AuthTestCase(unittest.HomeserverTestCase): self.assertEqual(f.errcode, "M_UNKNOWN_TOKEN") def test_get_user_by_req_appservice_missing_token(self) -> None: - app_service = Mock(token="foobar", url="a_url", sender=self.test_user) + app_service = Mock(token="foobar", url="a_url", sender=self.test_user_id) self.store.get_app_service_by_token = Mock(return_value=app_service) self.store.get_user_by_access_token = AsyncMock(return_value=None) @@ -191,7 +194,10 @@ class AuthTestCase(unittest.HomeserverTestCase): def test_get_user_by_req_appservice_valid_token_valid_user_id(self) -> None: masquerading_user_id = b"@doppelganger:matrix.org" app_service = Mock( - token="foobar", url="a_url", sender=self.test_user, ip_range_whitelist=None + token="foobar", + url="a_url", + sender=self.test_user_id, + ip_range_whitelist=None, ) app_service.is_interested_in_user = Mock(return_value=True) self.store.get_app_service_by_token = Mock(return_value=app_service) @@ -215,7 +221,10 @@ class AuthTestCase(unittest.HomeserverTestCase): def test_get_user_by_req_appservice_valid_token_bad_user_id(self) -> None: masquerading_user_id = b"@doppelganger:matrix.org" app_service = Mock( - token="foobar", url="a_url", sender=self.test_user, ip_range_whitelist=None + token="foobar", + url="a_url", + sender=self.test_user_id, + ip_range_whitelist=None, ) app_service.is_interested_in_user = Mock(return_value=False) self.store.get_app_service_by_token = Mock(return_value=app_service) @@ -238,7 +247,10 @@ class AuthTestCase(unittest.HomeserverTestCase): masquerading_user_id = b"@doppelganger:matrix.org" masquerading_device_id = b"DOPPELDEVICE" app_service = Mock( - token="foobar", url="a_url", sender=self.test_user, ip_range_whitelist=None + token="foobar", + url="a_url", + sender=self.test_user_id, + ip_range_whitelist=None, ) app_service.is_interested_in_user = Mock(return_value=True) self.store.get_app_service_by_token = Mock(return_value=app_service) @@ -270,7 +282,10 @@ class AuthTestCase(unittest.HomeserverTestCase): masquerading_user_id = b"@doppelganger:matrix.org" masquerading_device_id = b"NOT_A_REAL_DEVICE_ID" app_service = Mock( - token="foobar", url="a_url", sender=self.test_user, ip_range_whitelist=None + token="foobar", + url="a_url", + sender=self.test_user_id, + ip_range_whitelist=None, ) app_service.is_interested_in_user = Mock(return_value=True) self.store.get_app_service_by_token = Mock(return_value=app_service) @@ -436,7 +451,7 @@ class AuthTestCase(unittest.HomeserverTestCase): namespaces={ "users": [{"regex": "@_appservice.*:sender", "exclusive": True}] }, - sender="@appservice:sender", + sender=UserID.from_string("@appservice:server"), ) requester = Requester( user=UserID.from_string("@appservice:server"), @@ -467,7 +482,7 @@ class AuthTestCase(unittest.HomeserverTestCase): namespaces={ "users": [{"regex": "@_appservice.*:sender", "exclusive": True}] }, - sender="@appservice:sender", + sender=UserID.from_string("@appservice:server"), ) requester = Requester( user=UserID.from_string("@appservice:server"), diff --git a/tests/api/test_ratelimiting.py b/tests/api/test_ratelimiting.py index 93f4f98916..2e45d4e4d2 100644 --- a/tests/api/test_ratelimiting.py +++ b/tests/api/test_ratelimiting.py @@ -5,7 +5,7 @@ from synapse.appservice import ApplicationService from synapse.config.ratelimiting import RatelimitSettings from synapse.module_api import RatelimitOverride from synapse.module_api.callbacks.ratelimit_callbacks import RatelimitModuleApiCallbacks -from synapse.types import create_requester +from synapse.types import UserID, create_requester from tests import unittest @@ -40,7 +40,7 @@ class TestRatelimiter(unittest.HomeserverTestCase): token="fake_token", id="foo", rate_limited=True, - sender="@as:example.com", + sender=UserID.from_string("@as:example.com"), ) as_requester = create_requester("@user:example.com", app_service=appservice) @@ -76,7 +76,7 @@ class TestRatelimiter(unittest.HomeserverTestCase): token="fake_token", id="foo", rate_limited=False, - sender="@as:example.com", + sender=UserID.from_string("@as:example.com"), ) as_requester = create_requester("@user:example.com", app_service=appservice) diff --git a/tests/appservice/test_api.py b/tests/appservice/test_api.py index 0f19736540..8fcd928d31 100644 --- a/tests/appservice/test_api.py +++ b/tests/appservice/test_api.py @@ -25,7 +25,7 @@ from twisted.test.proto_helpers import MemoryReactor from synapse.appservice import ApplicationService from synapse.server import HomeServer -from synapse.types import JsonDict +from synapse.types import JsonDict, UserID from synapse.util import Clock from tests import unittest @@ -41,7 +41,7 @@ class ApplicationServiceApiTestCase(unittest.HomeserverTestCase): self.api = hs.get_application_service_api() self.service = ApplicationService( id="unique_identifier", - sender="@as:test", + sender=UserID.from_string("@as:test"), url=URL, token="unused", hs_token=TOKEN, diff --git a/tests/appservice/test_appservice.py b/tests/appservice/test_appservice.py index 3fa4426638..620c2b907b 100644 --- a/tests/appservice/test_appservice.py +++ b/tests/appservice/test_appservice.py @@ -25,6 +25,7 @@ from unittest.mock import AsyncMock, Mock from twisted.internet import defer from synapse.appservice import ApplicationService, Namespace +from synapse.types import UserID from tests import unittest @@ -37,7 +38,7 @@ class ApplicationServiceTestCase(unittest.TestCase): def setUp(self) -> None: self.service = ApplicationService( id="unique_identifier", - sender="@as:test", + sender=UserID.from_string("@as:test"), url="some_url", token="some_token", ) @@ -226,11 +227,11 @@ class ApplicationServiceTestCase(unittest.TestCase): @defer.inlineCallbacks def test_interested_in_self(self) -> Generator["defer.Deferred[Any]", object, None]: # make sure invites get through - self.service.sender = "@appservice:name" + self.service.sender = UserID.from_string("@appservice:name") self.service.namespaces[ApplicationService.NS_USERS].append(_regex("@irc_.*")) self.event.type = "m.room.member" self.event.content = {"membership": "invite"} - self.event.state_key = self.service.sender + self.event.state_key = self.service.sender.to_string() self.assertTrue( ( yield self.service.is_interested_in_event( diff --git a/tests/config/test_cache.py b/tests/config/test_cache.py index aead73e059..deb6bade46 100644 --- a/tests/config/test_cache.py +++ b/tests/config/test_cache.py @@ -75,7 +75,7 @@ class CacheConfigTests(TestCase): the default cache size in the interim, and then resized once the config is loaded. """ - cache: LruCache = LruCache(100) + cache: LruCache = LruCache(max_size=100) add_resizable_cache("foo", cache_resize_callback=cache.set_cache_factor) self.assertEqual(cache.max_size, 50) @@ -96,7 +96,7 @@ class CacheConfigTests(TestCase): self.config.read_config(config, config_dir_path="", data_dir_path="") self.config.resize_all_caches() - cache: LruCache = LruCache(100) + cache: LruCache = LruCache(max_size=100) add_resizable_cache("foo", cache_resize_callback=cache.set_cache_factor) self.assertEqual(cache.max_size, 200) @@ -106,7 +106,7 @@ class CacheConfigTests(TestCase): the default cache size in the interim, and then resized to the new default cache size once the config is loaded. """ - cache: LruCache = LruCache(100) + cache: LruCache = LruCache(max_size=100) add_resizable_cache("foo", cache_resize_callback=cache.set_cache_factor) self.assertEqual(cache.max_size, 50) @@ -126,7 +126,7 @@ class CacheConfigTests(TestCase): self.config.read_config(config, config_dir_path="", data_dir_path="") self.config.resize_all_caches() - cache: LruCache = LruCache(100) + cache: LruCache = LruCache(max_size=100) add_resizable_cache("foo", cache_resize_callback=cache.set_cache_factor) self.assertEqual(cache.max_size, 150) @@ -145,15 +145,15 @@ class CacheConfigTests(TestCase): self.config.read_config(config, config_dir_path="", data_dir_path="") self.config.resize_all_caches() - cache_a: LruCache = LruCache(100) + cache_a: LruCache = LruCache(max_size=100) add_resizable_cache("*cache_a*", cache_resize_callback=cache_a.set_cache_factor) self.assertEqual(cache_a.max_size, 200) - cache_b: LruCache = LruCache(100) + cache_b: LruCache = LruCache(max_size=100) add_resizable_cache("*Cache_b*", cache_resize_callback=cache_b.set_cache_factor) self.assertEqual(cache_b.max_size, 300) - cache_c: LruCache = LruCache(100) + cache_c: LruCache = LruCache(max_size=100) add_resizable_cache("*cache_c*", cache_resize_callback=cache_c.set_cache_factor) self.assertEqual(cache_c.max_size, 200) diff --git a/tests/handlers/test_appservice.py b/tests/handlers/test_appservice.py index 1db630e9e4..25cf5269b8 100644 --- a/tests/handlers/test_appservice.py +++ b/tests/handlers/test_appservice.py @@ -43,6 +43,7 @@ from synapse.types import ( MultiWriterStreamToken, RoomStreamToken, StreamKeyType, + UserID, ) from synapse.util import Clock from synapse.util.stringutils import random_string @@ -1009,7 +1010,7 @@ class ApplicationServicesHandlerSendEventsTestCase(unittest.HomeserverTestCase): appservice = ApplicationService( token=random_string(10), id=random_string(10), - sender="@as:example.com", + sender=UserID.from_string("@as:example.com"), rate_limited=False, namespaces=namespaces, supports_ephemeral=True, @@ -1087,7 +1088,7 @@ class ApplicationServicesHandlerDeviceListsTestCase(unittest.HomeserverTestCase) appservice = ApplicationService( token=random_string(10), id=random_string(10), - sender="@as:example.com", + sender=UserID.from_string("@as:example.com"), rate_limited=False, namespaces={ ApplicationService.NS_USERS: [ @@ -1151,9 +1152,9 @@ class ApplicationServicesHandlerOtkCountsTestCase(unittest.HomeserverTestCase): # Define an application service for the tests self._service_token = "VERYSECRET" self._service = ApplicationService( - self._service_token, - "as1", - "@as.sender:test", + token=self._service_token, + id="as1", + sender=UserID.from_string("@as.sender:test"), namespaces={ "users": [ {"regex": "@_as_.*:test", "exclusive": True}, diff --git a/tests/handlers/test_device.py b/tests/handlers/test_device.py index 080e6a7028..99a0c50211 100644 --- a/tests/handlers/test_device.py +++ b/tests/handlers/test_device.py @@ -34,7 +34,7 @@ from synapse.rest import admin from synapse.rest.client import devices, login, register from synapse.server import HomeServer from synapse.storage.databases.main.appservice import _make_exclusive_regex -from synapse.types import JsonDict, create_requester +from synapse.types import JsonDict, UserID, create_requester from synapse.util import Clock from synapse.util.task_scheduler import TaskScheduler @@ -419,7 +419,7 @@ class DeviceTestCase(unittest.HomeserverTestCase): id="1234", namespaces={"users": [{"regex": r"@boris:.+", "exclusive": True}]}, # Note: this user does not have to match the regex above - sender="@as_main:test", + sender=UserID.from_string("@as_main:test"), ) self.hs.get_datastores().main.services_cache = [appservice] self.hs.get_datastores().main.exclusive_user_regex = _make_exclusive_regex( diff --git a/tests/handlers/test_e2e_keys.py b/tests/handlers/test_e2e_keys.py index 70fc4263e7..323950c2f4 100644 --- a/tests/handlers/test_e2e_keys.py +++ b/tests/handlers/test_e2e_keys.py @@ -1457,7 +1457,7 @@ class E2eKeysHandlerTestCase(unittest.HomeserverTestCase): id="1234", namespaces={"users": [{"regex": r"@boris:.+", "exclusive": True}]}, # Note: this user does not have to match the regex above - sender="@as_main:test", + sender=UserID.from_string("@as_main:test"), ) self.hs.get_datastores().main.services_cache = [appservice] self.hs.get_datastores().main.exclusive_user_regex = _make_exclusive_regex( @@ -1525,7 +1525,7 @@ class E2eKeysHandlerTestCase(unittest.HomeserverTestCase): id="1234", namespaces={"users": [{"regex": r"@boris:.+", "exclusive": True}]}, # Note: this user does not have to match the regex above - sender="@as_main:test", + sender=UserID.from_string("@as_main:test"), ) self.hs.get_datastores().main.services_cache = [appservice] self.hs.get_datastores().main.exclusive_user_regex = _make_exclusive_regex( @@ -1751,7 +1751,7 @@ class E2eKeysHandlerTestCase(unittest.HomeserverTestCase): id="1234", namespaces={"users": [{"regex": r"@boris:.+", "exclusive": True}]}, # Note: this user does not have to match the regex above - sender="@as_main:test", + sender=UserID.from_string("@as_main:test"), ) self.hs.get_datastores().main.services_cache = [appservice] self.hs.get_datastores().main.exclusive_user_regex = _make_exclusive_regex( diff --git a/tests/handlers/test_oauth_delegation.py b/tests/handlers/test_oauth_delegation.py index fefa2f1135..20f2306d4c 100644 --- a/tests/handlers/test_oauth_delegation.py +++ b/tests/handlers/test_oauth_delegation.py @@ -726,7 +726,7 @@ class MSC3861OAuthDelegation(HomeserverTestCase): token="i_am_an_app_service", id="1234", namespaces={"users": [{"regex": r"@alice:.+", "exclusive": True}]}, - sender="@as_main:test", + sender=UserID.from_string("@as_main:test"), ) self.hs.get_datastores().main.services_cache = [appservice] diff --git a/tests/handlers/test_typing.py b/tests/handlers/test_typing.py index 394315d2b0..1126b6f183 100644 --- a/tests/handlers/test_typing.py +++ b/tests/handlers/test_typing.py @@ -86,8 +86,8 @@ class TypingNotificationsTestCase(unittest.HomeserverTestCase): self.mock_federation_client = AsyncMock(spec=["put_json"]) self.mock_federation_client.put_json.return_value = (200, "OK") self.mock_federation_client.agent = MatrixFederationAgent( - "OUR_STUB_HOMESERVER_NAME", - reactor, + server_name="OUR_STUB_HOMESERVER_NAME", + reactor=reactor, tls_client_options_factory=None, user_agent=b"SynapseInTrialTest/0.0.0", ip_allowlist=None, diff --git a/tests/handlers/test_user_directory.py b/tests/handlers/test_user_directory.py index b12ffc3665..0da423142c 100644 --- a/tests/handlers/test_user_directory.py +++ b/tests/handlers/test_user_directory.py @@ -31,7 +31,7 @@ from synapse.appservice import ApplicationService from synapse.rest.client import login, register, room, user_directory from synapse.server import HomeServer from synapse.storage.roommember import ProfileInfo -from synapse.types import JsonDict, UserProfile, create_requester +from synapse.types import JsonDict, UserID, UserProfile, create_requester from synapse.util import Clock from tests import unittest @@ -78,7 +78,7 @@ class UserDirectoryTestCase(unittest.HomeserverTestCase): namespaces={"users": [{"regex": r"@as_user.*", "exclusive": True}]}, # Note: this user does not match the regex above, so that tests # can distinguish the sender from the AS user. - sender="@as_main:test", + sender=UserID.from_string("@as_main:test"), ) mock_load_appservices = Mock(return_value=[self.appservice]) @@ -196,7 +196,9 @@ class UserDirectoryTestCase(unittest.HomeserverTestCase): user = self.register_user("user", "pass") token = self.login(user, "pass") room = self.helper.create_room_as(user, is_public=True, tok=token) - self.helper.join(room, self.appservice.sender, tok=self.appservice.token) + self.helper.join( + room, self.appservice.sender.to_string(), tok=self.appservice.token + ) self._check_only_one_user_in_directory(user, room) def test_search_term_with_colon_in_it_does_not_raise(self) -> None: @@ -433,7 +435,7 @@ class UserDirectoryTestCase(unittest.HomeserverTestCase): def test_handle_local_profile_change_with_appservice_sender(self) -> None: # profile is not in directory profile = self.get_success( - self.store._get_user_in_directory(self.appservice.sender) + self.store._get_user_in_directory(self.appservice.sender.to_string()) ) self.assertIsNone(profile) @@ -441,13 +443,13 @@ class UserDirectoryTestCase(unittest.HomeserverTestCase): profile_info = ProfileInfo(avatar_url="avatar_url", display_name="4L1c3") self.get_success( self.handler.handle_local_profile_change( - self.appservice.sender, profile_info + self.appservice.sender.to_string(), profile_info ) ) # profile is still not in directory profile = self.get_success( - self.store._get_user_in_directory(self.appservice.sender) + self.store._get_user_in_directory(self.appservice.sender.to_string()) ) self.assertIsNone(profile) diff --git a/tests/http/federation/test_matrix_federation_agent.py b/tests/http/federation/test_matrix_federation_agent.py index eb859ca47a..a1243b053d 100644 --- a/tests/http/federation/test_matrix_federation_agent.py +++ b/tests/http/federation/test_matrix_federation_agent.py @@ -85,16 +85,20 @@ class MatrixFederationAgentTests(unittest.TestCase): self.tls_factory = FederationPolicyForHTTPS(config) self.well_known_cache: TTLCache[bytes, Optional[bytes]] = TTLCache( - "test_cache", timer=self.reactor.seconds + cache_name="test_cache", + server_name="test_server", + timer=self.reactor.seconds, ) self.had_well_known_cache: TTLCache[bytes, bool] = TTLCache( - "test_cache", timer=self.reactor.seconds + cache_name="test_cache", + server_name="test_server", + timer=self.reactor.seconds, ) self.well_known_resolver = WellKnownResolver( - "OUR_STUB_HOMESERVER_NAME", - self.reactor, - Agent(self.reactor, contextFactory=self.tls_factory), - b"test-agent", + server_name="OUR_STUB_HOMESERVER_NAME", + reactor=self.reactor, + agent=Agent(self.reactor, contextFactory=self.tls_factory), + user_agent=b"test-agent", well_known_cache=self.well_known_cache, had_well_known_cache=self.had_well_known_cache, ) @@ -270,7 +274,7 @@ class MatrixFederationAgentTests(unittest.TestCase): because it is created too early during setUp """ return MatrixFederationAgent( - "OUR_STUB_HOMESERVER_NAME", + server_name="OUR_STUB_HOMESERVER_NAME", reactor=cast(ISynapseReactor, self.reactor), tls_client_options_factory=self.tls_factory, user_agent=b"test-agent", # Note that this is unused since _well_known_resolver is provided. @@ -1013,7 +1017,7 @@ class MatrixFederationAgentTests(unittest.TestCase): # Build a new agent and WellKnownResolver with a different tls factory tls_factory = FederationPolicyForHTTPS(config) agent = MatrixFederationAgent( - "OUR_STUB_HOMESERVER_NAME", + server_name="OUR_STUB_HOMESERVER_NAME", reactor=self.reactor, tls_client_options_factory=tls_factory, user_agent=b"test-agent", # This is unused since _well_known_resolver is passed below. @@ -1021,10 +1025,10 @@ class MatrixFederationAgentTests(unittest.TestCase): ip_blocklist=IPSet(), _srv_resolver=self.mock_resolver, _well_known_resolver=WellKnownResolver( - "OUR_STUB_HOMESERVER_NAME", - cast(ISynapseReactor, self.reactor), - Agent(self.reactor, contextFactory=tls_factory), - b"test-agent", + server_name="OUR_STUB_HOMESERVER_NAME", + reactor=cast(ISynapseReactor, self.reactor), + agent=Agent(self.reactor, contextFactory=tls_factory), + user_agent=b"test-agent", well_known_cache=self.well_known_cache, had_well_known_cache=self.had_well_known_cache, ), diff --git a/tests/metrics/test_metrics.py b/tests/metrics/test_metrics.py index 2e7004df3a..dca8dd79b1 100644 --- a/tests/metrics/test_metrics.py +++ b/tests/metrics/test_metrics.py @@ -159,7 +159,9 @@ class CacheMetricsTests(unittest.HomeserverTestCase): Caches produce metrics reflecting their state when scraped. """ CACHE_NAME = "cache_metrics_test_fgjkbdfg" - cache: DeferredCache[str, str] = DeferredCache(CACHE_NAME, max_entries=777) + cache: DeferredCache[str, str] = DeferredCache( + name=CACHE_NAME, server_name=self.hs.hostname, max_entries=777 + ) items = { x.split(b"{")[0].decode("ascii"): x.split(b" ")[1].decode("ascii") diff --git a/tests/push/test_push_rule_evaluator.py b/tests/push/test_push_rule_evaluator.py index 3898532acf..98a3a22154 100644 --- a/tests/push/test_push_rule_evaluator.py +++ b/tests/push/test_push_rule_evaluator.py @@ -823,9 +823,9 @@ class TestBulkPushRuleEvaluator(unittest.HomeserverTestCase): # Define an application service so that we can register appservice users self._service_token = "some_token" self._service = ApplicationService( - self._service_token, - "as1", - "@as.sender:test", + token=self._service_token, + id="as1", + sender=UserID.from_string("@as.sender:test"), namespaces={ "users": [ {"regex": "@_as_.*:test", "exclusive": True}, diff --git a/tests/replication/tcp/streams/test_typing.py b/tests/replication/tcp/streams/test_typing.py index b1c2f5b03b..c8958189f8 100644 --- a/tests/replication/tcp/streams/test_typing.py +++ b/tests/replication/tcp/streams/test_typing.py @@ -139,7 +139,9 @@ class TypingStreamTestCase(BaseStreamTestCase): self.hs.get_replication_command_handler()._streams["typing"].last_token = 0 typing._latest_room_serial = 0 typing._typing_stream_change_cache = StreamChangeCache( - "TypingStreamChangeCache", typing._latest_room_serial + name="TypingStreamChangeCache", + server_name=self.hs.hostname, + current_stream_pos=typing._latest_room_serial, ) typing._reset() diff --git a/tests/replication/test_federation_sender_shard.py b/tests/replication/test_federation_sender_shard.py index 6c4145f2c2..5b7ed95a23 100644 --- a/tests/replication/test_federation_sender_shard.py +++ b/tests/replication/test_federation_sender_shard.py @@ -68,8 +68,8 @@ class FederationSenderTestCase(BaseMultiWorkerStreamTestCase): reactor, _ = get_clock() self.matrix_federation_agent = MatrixFederationAgent( - "OUR_STUB_HOMESERVER_NAME", - reactor, + server_name="OUR_STUB_HOMESERVER_NAME", + reactor=reactor, tls_client_options_factory=None, user_agent=b"SynapseInTrialTest/0.0.0", ip_allowlist=None, diff --git a/tests/replication/test_module_cache_invalidation.py b/tests/replication/test_module_cache_invalidation.py index 1e7183edaa..8d5d0cce9a 100644 --- a/tests/replication/test_module_cache_invalidation.py +++ b/tests/replication/test_module_cache_invalidation.py @@ -35,6 +35,7 @@ KEY = "mykey" class TestCache: current_value = FIRST_VALUE + server_name = "test_server" # nb must be called this for @cached @cached() async def cached_function(self, user_id: str) -> str: diff --git a/tests/rest/client/test_account.py b/tests/rest/client/test_account.py index a85ea994de..5343b10e92 100644 --- a/tests/rest/client/test_account.py +++ b/tests/rest/client/test_account.py @@ -764,10 +764,10 @@ class WhoamiTestCase(unittest.HomeserverTestCase): as_token = "i_am_an_app_service" appservice = ApplicationService( - as_token, + token=as_token, id="1234", namespaces={"users": [{"regex": user_id, "exclusive": True}]}, - sender=user_id, + sender=UserID.from_string(user_id), ) self.hs.get_datastores().main.services_cache.append(appservice) diff --git a/tests/rest/client/test_devices.py b/tests/rest/client/test_devices.py index dd3abdebac..b7230488e4 100644 --- a/tests/rest/client/test_devices.py +++ b/tests/rest/client/test_devices.py @@ -472,7 +472,7 @@ class MSC4190AppserviceDevicesTestCase(unittest.HomeserverTestCase): id="msc4190", token="some_token", hs_token="some_token", - sender="@as:example.com", + sender=UserID.from_string("@as:example.com"), namespaces={ ApplicationService.NS_USERS: [{"regex": "@.*", "exclusive": False}] }, @@ -483,7 +483,7 @@ class MSC4190AppserviceDevicesTestCase(unittest.HomeserverTestCase): id="regular", token="other_token", hs_token="other_token", - sender="@as2:example.com", + sender=UserID.from_string("@as2:example.com"), namespaces={ ApplicationService.NS_USERS: [{"regex": "@.*", "exclusive": False}] }, diff --git a/tests/rest/client/test_directory.py b/tests/rest/client/test_directory.py index ecf38493c3..6e499093cf 100644 --- a/tests/rest/client/test_directory.py +++ b/tests/rest/client/test_directory.py @@ -25,7 +25,7 @@ from synapse.appservice import ApplicationService from synapse.rest import admin from synapse.rest.client import directory, login, room from synapse.server import HomeServer -from synapse.types import RoomAlias +from synapse.types import RoomAlias, UserID from synapse.util import Clock from synapse.util.stringutils import random_string @@ -140,7 +140,7 @@ class DirectoryTestCase(unittest.HomeserverTestCase): as_token, id="1234", namespaces={"aliases": [{"regex": "#asns-*", "exclusive": True}]}, - sender=user_id, + sender=UserID.from_string(user_id), ) self.hs.get_datastores().main.services_cache.append(appservice) diff --git a/tests/rest/client/test_login.py b/tests/rest/client/test_login.py index c5c6604667..b8bcc235e9 100644 --- a/tests/rest/client/test_login.py +++ b/tests/rest/client/test_login.py @@ -51,7 +51,7 @@ from synapse.rest.client import account, devices, login, logout, profile, regist from synapse.rest.client.account import WhoamiRestServlet from synapse.rest.synapse.client import build_synapse_client_resource_tree from synapse.server import HomeServer -from synapse.types import JsonDict, create_requester +from synapse.types import JsonDict, UserID, create_requester from synapse.util import Clock from tests import unittest @@ -1484,7 +1484,7 @@ class AppserviceLoginRestServletTestCase(unittest.HomeserverTestCase): self.service = ApplicationService( id="unique_identifier", token="some_token", - sender="@asbot:example.com", + sender=UserID.from_string("@asbot:example.com"), namespaces={ ApplicationService.NS_USERS: [ {"regex": r"@as_user.*", "exclusive": False} @@ -1496,7 +1496,7 @@ class AppserviceLoginRestServletTestCase(unittest.HomeserverTestCase): self.another_service = ApplicationService( id="another__identifier", token="another_token", - sender="@as2bot:example.com", + sender=UserID.from_string("@as2bot:example.com"), namespaces={ ApplicationService.NS_USERS: [ {"regex": r"@as2_user.*", "exclusive": False} @@ -1530,7 +1530,10 @@ class AppserviceLoginRestServletTestCase(unittest.HomeserverTestCase): params = { "type": login.LoginRestServlet.APPSERVICE_TYPE, - "identifier": {"type": "m.id.user", "user": self.service.sender}, + "identifier": { + "type": "m.id.user", + "user": self.service.sender.to_string(), + }, } channel = self.make_request( b"POST", LOGIN_URL, params, access_token=self.service.token diff --git a/tests/rest/client/test_register.py b/tests/rest/client/test_register.py index b697bf6f67..638fbf0062 100644 --- a/tests/rest/client/test_register.py +++ b/tests/rest/client/test_register.py @@ -39,7 +39,7 @@ from synapse.appservice import ApplicationService from synapse.rest.client import account, account_validity, login, logout, register, sync from synapse.server import HomeServer from synapse.storage._base import db_to_json -from synapse.types import JsonDict +from synapse.types import JsonDict, UserID from synapse.util import Clock from tests import unittest @@ -75,7 +75,7 @@ class RegisterRestServletTestCase(unittest.HomeserverTestCase): as_token, id="1234", namespaces={"users": [{"regex": r"@as_user.*", "exclusive": True}]}, - sender="@as:test", + sender=UserID.from_string("@as:test"), ) self.hs.get_datastores().main.services_cache.append(appservice) @@ -99,7 +99,7 @@ class RegisterRestServletTestCase(unittest.HomeserverTestCase): as_token, id="1234", namespaces={"users": [{"regex": r"@as_user.*", "exclusive": True}]}, - sender="@as:test", + sender=UserID.from_string("@as:test"), ) self.hs.get_datastores().main.services_cache.append(appservice) @@ -129,7 +129,7 @@ class RegisterRestServletTestCase(unittest.HomeserverTestCase): as_token, id="1234", namespaces={"users": [{"regex": r"@as_user.*", "exclusive": True}]}, - sender="@as:test", + sender=UserID.from_string("@as:test"), msc4190_device_management=True, ) diff --git a/tests/rest/client/test_rooms.py b/tests/rest/client/test_rooms.py index 3ba7584c65..8a6e6f118a 100644 --- a/tests/rest/client/test_rooms.py +++ b/tests/rest/client/test_rooms.py @@ -1479,7 +1479,7 @@ class RoomAppserviceTsParamTestCase(unittest.HomeserverTestCase): id="1234", namespaces={"users": [{"regex": r"@as_user.*", "exclusive": True}]}, # Note: this user does not have to match the regex above - sender="@as_main:test", + sender=UserID.from_string("@as_main:test"), ) mock_load_appservices = Mock(return_value=[self.appservice]) diff --git a/tests/storage/test_user_directory.py b/tests/storage/test_user_directory.py index f97ca12d84..80f491aff9 100644 --- a/tests/storage/test_user_directory.py +++ b/tests/storage/test_user_directory.py @@ -36,6 +36,7 @@ from synapse.storage.databases.main.user_directory import ( _parse_words_with_icu, ) from synapse.storage.roommember import ProfileInfo +from synapse.types import UserID from synapse.util import Clock from tests.server import ThreadedMemoryReactorClock @@ -153,7 +154,7 @@ class UserDirectoryInitialPopulationTestcase(HomeserverTestCase): token="i_am_an_app_service", id="1234", namespaces={"users": [{"regex": r"@as_user.*", "exclusive": True}]}, - sender="@as:test", + sender=UserID.from_string("@as:test"), ) mock_load_appservices = Mock(return_value=[self.appservice]) @@ -378,7 +379,7 @@ class UserDirectoryInitialPopulationTestcase(HomeserverTestCase): # Join the AS sender to rooms owned by the normal user. public, private = self._create_rooms_and_inject_memberships( - user, token, self.appservice.sender + user, token, self.appservice.sender.to_string() ) # Rebuild the directory. diff --git a/tests/test_mau.py b/tests/test_mau.py index 714854cdf2..472965e022 100644 --- a/tests/test_mau.py +++ b/tests/test_mau.py @@ -29,7 +29,7 @@ from synapse.api.errors import Codes, HttpResponseException, SynapseError from synapse.appservice import ApplicationService from synapse.rest.client import register, sync from synapse.server import HomeServer -from synapse.types import JsonDict +from synapse.types import JsonDict, UserID from synapse.util import Clock from tests import unittest @@ -118,7 +118,7 @@ class TestMauLimit(unittest.HomeserverTestCase): ApplicationService( token=as_token, id="SomeASID", - sender="@as_sender:test", + sender=UserID.from_string("@as_sender:test"), namespaces={"users": [{"regex": "@as_*", "exclusive": True}]}, ) ) @@ -263,7 +263,7 @@ class TestMauLimit(unittest.HomeserverTestCase): ApplicationService( token=as_token_1, id="SomeASID", - sender="@as_sender_1:test", + sender=UserID.from_string("@as_sender_1:test"), namespaces={"users": [{"regex": "@as_1.*", "exclusive": True}]}, ) ) @@ -273,7 +273,7 @@ class TestMauLimit(unittest.HomeserverTestCase): ApplicationService( token=as_token_2, id="AnotherASID", - sender="@as_sender_2:test", + sender=UserID.from_string("@as_sender_2:test"), namespaces={"users": [{"regex": "@as_2.*", "exclusive": True}]}, ) ) diff --git a/tests/util/caches/test_deferred_cache.py b/tests/util/caches/test_deferred_cache.py index f99f99237e..7017d6d70a 100644 --- a/tests/util/caches/test_deferred_cache.py +++ b/tests/util/caches/test_deferred_cache.py @@ -31,18 +31,24 @@ from tests.unittest import TestCase class DeferredCacheTestCase(TestCase): def test_empty(self) -> None: - cache: DeferredCache[str, int] = DeferredCache("test") + cache: DeferredCache[str, int] = DeferredCache( + name="test", server_name="test_server" + ) with self.assertRaises(KeyError): cache.get("foo") def test_hit(self) -> None: - cache: DeferredCache[str, int] = DeferredCache("test") + cache: DeferredCache[str, int] = DeferredCache( + name="test", server_name="test_server" + ) cache.prefill("foo", 123) self.assertEqual(self.successResultOf(cache.get("foo")), 123) def test_hit_deferred(self) -> None: - cache: DeferredCache[str, int] = DeferredCache("test") + cache: DeferredCache[str, int] = DeferredCache( + name="test", server_name="test_server" + ) origin_d: "defer.Deferred[int]" = defer.Deferred() set_d = cache.set("k1", origin_d) @@ -65,7 +71,9 @@ class DeferredCacheTestCase(TestCase): def test_callbacks(self) -> None: """Invalidation callbacks are called at the right time""" - cache: DeferredCache[str, int] = DeferredCache("test") + cache: DeferredCache[str, int] = DeferredCache( + name="test", server_name="test_server" + ) callbacks = set() # start with an entry, with a callback @@ -98,7 +106,9 @@ class DeferredCacheTestCase(TestCase): self.assertEqual(callbacks, {"set", "get"}) def test_set_fail(self) -> None: - cache: DeferredCache[str, int] = DeferredCache("test") + cache: DeferredCache[str, int] = DeferredCache( + name="test", server_name="test_server" + ) callbacks = set() # start with an entry, with a callback @@ -135,7 +145,9 @@ class DeferredCacheTestCase(TestCase): self.assertEqual(callbacks, {"prefill", "get2"}) def test_get_immediate(self) -> None: - cache: DeferredCache[str, int] = DeferredCache("test") + cache: DeferredCache[str, int] = DeferredCache( + name="test", server_name="test_server" + ) d1: "defer.Deferred[int]" = defer.Deferred() cache.set("key1", d1) @@ -151,7 +163,9 @@ class DeferredCacheTestCase(TestCase): self.assertEqual(v, 2) def test_invalidate(self) -> None: - cache: DeferredCache[Tuple[str], int] = DeferredCache("test") + cache: DeferredCache[Tuple[str], int] = DeferredCache( + name="test", server_name="test_server" + ) cache.prefill(("foo",), 123) cache.invalidate(("foo",)) @@ -159,7 +173,9 @@ class DeferredCacheTestCase(TestCase): cache.get(("foo",)) def test_invalidate_all(self) -> None: - cache: DeferredCache[str, str] = DeferredCache("testcache") + cache: DeferredCache[str, str] = DeferredCache( + name="testcache", server_name="test_server" + ) callback_record = [False, False] @@ -203,7 +219,10 @@ class DeferredCacheTestCase(TestCase): def test_eviction(self) -> None: cache: DeferredCache[int, str] = DeferredCache( - "test", max_entries=2, apply_cache_factor_from_config=False + name="test", + server_name="test_server", + max_entries=2, + apply_cache_factor_from_config=False, ) cache.prefill(1, "one") @@ -218,7 +237,10 @@ class DeferredCacheTestCase(TestCase): def test_eviction_lru(self) -> None: cache: DeferredCache[int, str] = DeferredCache( - "test", max_entries=2, apply_cache_factor_from_config=False + name="test", + server_name="test_server", + max_entries=2, + apply_cache_factor_from_config=False, ) cache.prefill(1, "one") @@ -237,7 +259,8 @@ class DeferredCacheTestCase(TestCase): def test_eviction_iterable(self) -> None: cache: DeferredCache[int, List[str]] = DeferredCache( - "test", + name="test", + server_name="test_server", max_entries=3, apply_cache_factor_from_config=False, iterable=True, diff --git a/tests/util/caches/test_descriptors.py b/tests/util/caches/test_descriptors.py index 6af9dfaf56..7865a67709 100644 --- a/tests/util/caches/test_descriptors.py +++ b/tests/util/caches/test_descriptors.py @@ -66,6 +66,7 @@ class DescriptorTestCase(unittest.TestCase): class Cls: def __init__(self) -> None: self.mock = mock.Mock() + self.server_name = "test_server" @descriptors.cached() def fn(self, arg1: int, arg2: int) -> str: @@ -100,6 +101,7 @@ class DescriptorTestCase(unittest.TestCase): class Cls: def __init__(self) -> None: self.mock = mock.Mock() + self.server_name = "test_server" @descriptors.cached(num_args=1) def fn(self, arg1: int, arg2: int) -> str: @@ -145,6 +147,7 @@ class DescriptorTestCase(unittest.TestCase): def __init__(self) -> None: self.mock = mock.Mock() + self.server_name = "test_server" obj = Cls() obj.mock.return_value = "fish" @@ -175,6 +178,7 @@ class DescriptorTestCase(unittest.TestCase): class Cls: def __init__(self) -> None: self.mock = mock.Mock() + self.server_name = "test_server" @descriptors.cached() def fn(self, arg1: int, kwarg1: int = 2) -> str: @@ -209,6 +213,8 @@ class DescriptorTestCase(unittest.TestCase): """If the wrapped function throws synchronously, things should continue to work""" class Cls: + server_name = "test_server" # nb must be called this for @cached + @cached() def fn(self, arg1: int) -> NoReturn: raise SynapseError(100, "mai spoon iz too big!!1") @@ -232,6 +238,7 @@ class DescriptorTestCase(unittest.TestCase): class Cls: result: Optional[Deferred] = None call_count = 0 + server_name = "test_server" # nb must be called this for @cached @cached() def fn(self, arg1: int) -> Deferred: @@ -285,6 +292,8 @@ class DescriptorTestCase(unittest.TestCase): complete_lookup: Deferred = Deferred() class Cls: + server_name = "test_server" + @descriptors.cached() def fn(self, arg1: int) -> "Deferred[int]": @defer.inlineCallbacks @@ -327,6 +336,8 @@ class DescriptorTestCase(unittest.TestCase): the lookup function throws an exception""" class Cls: + server_name = "test_server" + @descriptors.cached() def fn(self, arg1: int) -> Deferred: @defer.inlineCallbacks @@ -369,6 +380,7 @@ class DescriptorTestCase(unittest.TestCase): class Cls: def __init__(self) -> None: self.mock = mock.Mock() + self.server_name = "test_server" @descriptors.cached() def fn(self, arg1: int, arg2: int = 2, arg3: int = 3) -> str: @@ -406,6 +418,7 @@ class DescriptorTestCase(unittest.TestCase): class Cls: def __init__(self) -> None: self.mock = mock.Mock() + self.server_name = "test_server" @descriptors.cached(iterable=True) def fn(self, arg1: int, arg2: int) -> Tuple[str, ...]: @@ -439,6 +452,8 @@ class DescriptorTestCase(unittest.TestCase): """If the wrapped function throws synchronously, things should continue to work""" class Cls: + server_name = "test_server" + @descriptors.cached(iterable=True) def fn(self, arg1: int) -> NoReturn: raise SynapseError(100, "mai spoon iz too big!!1") @@ -460,6 +475,8 @@ class DescriptorTestCase(unittest.TestCase): """Invalidations should cascade up through cache contexts""" class Cls: + server_name = "test_server" # nb must be called this for @cached + @cached(cache_context=True) async def func1(self, key: str, cache_context: _CacheContext) -> int: return await self.func2(key, on_invalidate=cache_context.invalidate) @@ -486,6 +503,8 @@ class DescriptorTestCase(unittest.TestCase): complete_lookup: "Deferred[None]" = Deferred() class Cls: + server_name = "test_server" + @cached() async def fn(self, arg1: int) -> str: await complete_lookup @@ -517,6 +536,7 @@ class DescriptorTestCase(unittest.TestCase): class Cls: inner_context_was_finished = False + server_name = "test_server" # nb must be called this for @cached @cached() async def fn(self, arg1: int) -> str: @@ -562,6 +582,8 @@ class CacheDecoratorTestCase(unittest.HomeserverTestCase): @defer.inlineCallbacks def test_passthrough(self) -> Generator["Deferred[Any]", object, None]: class A: + server_name = "test_server" # nb must be called this for @cached + @cached() def func(self, key: str) -> str: return key @@ -576,6 +598,8 @@ class CacheDecoratorTestCase(unittest.HomeserverTestCase): callcount = [0] class A: + server_name = "test_server" # nb must be called this for @cached + @cached() def func(self, key: str) -> str: callcount[0] += 1 @@ -594,6 +618,8 @@ class CacheDecoratorTestCase(unittest.HomeserverTestCase): callcount = [0] class A: + server_name = "test_server" # nb must be called this for @cached + @cached() def func(self, key: str) -> str: callcount[0] += 1 @@ -612,6 +638,8 @@ class CacheDecoratorTestCase(unittest.HomeserverTestCase): def test_invalidate_missing(self) -> None: class A: + server_name = "test_server" # nb must be called this for @cached + @cached() def func(self, key: str) -> str: return key @@ -623,6 +651,8 @@ class CacheDecoratorTestCase(unittest.HomeserverTestCase): callcount = [0] class A: + server_name = "test_server" # nb must be called this for @cached + @cached(max_entries=10) def func(self, key: int) -> int: callcount[0] += 1 @@ -650,6 +680,8 @@ class CacheDecoratorTestCase(unittest.HomeserverTestCase): d = defer.succeed(123) class A: + server_name = "test_server" # nb must be called this for @cached + @cached() def func(self, key: str) -> "Deferred[int]": callcount[0] += 1 @@ -668,6 +700,8 @@ class CacheDecoratorTestCase(unittest.HomeserverTestCase): callcount2 = [0] class A: + server_name = "test_server" # nb must be called this for @cached + @cached() def func(self, key: str) -> str: callcount[0] += 1 @@ -701,6 +735,8 @@ class CacheDecoratorTestCase(unittest.HomeserverTestCase): callcount2 = [0] class A: + server_name = "test_server" # nb must be called this for @cached + @cached(max_entries=2) def func(self, key: str) -> str: callcount[0] += 1 @@ -738,6 +774,8 @@ class CacheDecoratorTestCase(unittest.HomeserverTestCase): callcount2 = [0] class A: + server_name = "test_server" # nb must be called this for @cached + @cached() def func(self, key: str) -> str: callcount[0] += 1 @@ -785,6 +823,7 @@ class CachedListDescriptorTestCase(unittest.TestCase): class Cls: def __init__(self) -> None: self.mock = mock.Mock() + self.server_name = "test_server" @descriptors.cached() def fn(self, arg1: int, arg2: int) -> None: @@ -850,6 +889,7 @@ class CachedListDescriptorTestCase(unittest.TestCase): class Cls: def __init__(self) -> None: self.mock = mock.Mock() + self.server_name = "test_server" @descriptors.cached() def fn(self, arg1: int) -> None: @@ -893,6 +933,7 @@ class CachedListDescriptorTestCase(unittest.TestCase): class Cls: def __init__(self) -> None: self.mock = mock.Mock() + self.server_name = "test_server" @descriptors.cached() def fn(self, arg1: int, arg2: int) -> None: @@ -933,6 +974,8 @@ class CachedListDescriptorTestCase(unittest.TestCase): complete_lookup: "Deferred[None]" = Deferred() class Cls: + server_name = "test_server" # nb must be called this for @cached + @cached() def fn(self, arg1: int) -> None: pass @@ -967,6 +1010,7 @@ class CachedListDescriptorTestCase(unittest.TestCase): class Cls: inner_context_was_finished = False + server_name = "test_server" # nb must be called this for @cached @cached() def fn(self, arg1: int) -> None: @@ -1010,6 +1054,8 @@ class CachedListDescriptorTestCase(unittest.TestCase): """ class Cls: + server_name = "test_server" + @descriptors.cached(tree=True) def fn(self, room_id: str, event_id: str) -> None: pass diff --git a/tests/util/caches/test_response_cache.py b/tests/util/caches/test_response_cache.py index e350967bba..30cd6ef0e4 100644 --- a/tests/util/caches/test_response_cache.py +++ b/tests/util/caches/test_response_cache.py @@ -46,7 +46,9 @@ class ResponseCacheTestCase(TestCase): self.reactor, self.clock = get_clock() def with_cache(self, name: str, ms: int = 0) -> ResponseCache: - return ResponseCache(self.clock, name, timeout_ms=ms) + return ResponseCache( + clock=self.clock, name=name, server_name="test_server", timeout_ms=ms + ) @staticmethod async def instant_return(o: str) -> str: diff --git a/tests/util/caches/test_ttlcache.py b/tests/util/caches/test_ttlcache.py index ae73df1841..a7b55ffecf 100644 --- a/tests/util/caches/test_ttlcache.py +++ b/tests/util/caches/test_ttlcache.py @@ -28,7 +28,9 @@ from tests import unittest class CacheTestCase(unittest.TestCase): def setUp(self) -> None: self.mock_timer = Mock(side_effect=lambda: 100.0) - self.cache: TTLCache[str, str] = TTLCache("test_cache", self.mock_timer) + self.cache: TTLCache[str, str] = TTLCache( + cache_name="test_cache", server_name="test_server", timer=self.mock_timer + ) def test_get(self) -> None: """simple set/get tests""" diff --git a/tests/util/test_dict_cache.py b/tests/util/test_dict_cache.py index 5055e4aead..246e18fd15 100644 --- a/tests/util/test_dict_cache.py +++ b/tests/util/test_dict_cache.py @@ -28,7 +28,7 @@ from tests import unittest class DictCacheTestCase(unittest.TestCase): def setUp(self) -> None: self.cache: DictionaryCache[str, str, str] = DictionaryCache( - "foobar", max_entries=10 + name="foobar", server_name="test_server", max_entries=10 ) def test_simple_cache_hit_full(self) -> None: diff --git a/tests/util/test_expiring_cache.py b/tests/util/test_expiring_cache.py index e97e5cf77d..75bf50e644 100644 --- a/tests/util/test_expiring_cache.py +++ b/tests/util/test_expiring_cache.py @@ -33,7 +33,10 @@ class ExpiringCacheTestCase(unittest.HomeserverTestCase): def test_get_set(self) -> None: clock = MockClock() cache: ExpiringCache[str, str] = ExpiringCache( - "test", cast(Clock, clock), max_len=1 + cache_name="test", + server_name="testserver", + clock=cast(Clock, clock), + max_len=1, ) cache["key"] = "value" @@ -43,7 +46,10 @@ class ExpiringCacheTestCase(unittest.HomeserverTestCase): def test_eviction(self) -> None: clock = MockClock() cache: ExpiringCache[str, str] = ExpiringCache( - "test", cast(Clock, clock), max_len=2 + cache_name="test", + server_name="testserver", + clock=cast(Clock, clock), + max_len=2, ) cache["key"] = "value" @@ -59,7 +65,11 @@ class ExpiringCacheTestCase(unittest.HomeserverTestCase): def test_iterable_eviction(self) -> None: clock = MockClock() cache: ExpiringCache[str, List[int]] = ExpiringCache( - "test", cast(Clock, clock), max_len=5, iterable=True + cache_name="test", + server_name="testserver", + clock=cast(Clock, clock), + max_len=5, + iterable=True, ) cache["key"] = [1] @@ -79,7 +89,10 @@ class ExpiringCacheTestCase(unittest.HomeserverTestCase): def test_time_eviction(self) -> None: clock = MockClock() cache: ExpiringCache[str, int] = ExpiringCache( - "test", cast(Clock, clock), expiry_ms=1000 + cache_name="test", + server_name="testserver", + clock=cast(Clock, clock), + expiry_ms=1000, ) cache["key"] = 1 diff --git a/tests/util/test_lrucache.py b/tests/util/test_lrucache.py index 3f0d8139f8..b7acf58690 100644 --- a/tests/util/test_lrucache.py +++ b/tests/util/test_lrucache.py @@ -34,13 +34,13 @@ from tests.unittest import override_config class LruCacheTestCase(unittest.HomeserverTestCase): def test_get_set(self) -> None: - cache: LruCache[str, str] = LruCache(1) + cache: LruCache[str, str] = LruCache(max_size=1) cache["key"] = "value" self.assertEqual(cache.get("key"), "value") self.assertEqual(cache["key"], "value") def test_eviction(self) -> None: - cache: LruCache[int, int] = LruCache(2) + cache: LruCache[int, int] = LruCache(max_size=2) cache[1] = 1 cache[2] = 2 @@ -54,7 +54,7 @@ class LruCacheTestCase(unittest.HomeserverTestCase): self.assertEqual(cache.get(3), 3) def test_setdefault(self) -> None: - cache: LruCache[str, int] = LruCache(1) + cache: LruCache[str, int] = LruCache(max_size=1) self.assertEqual(cache.setdefault("key", 1), 1) self.assertEqual(cache.get("key"), 1) self.assertEqual(cache.setdefault("key", 2), 1) @@ -63,14 +63,16 @@ class LruCacheTestCase(unittest.HomeserverTestCase): self.assertEqual(cache.get("key"), 2) def test_pop(self) -> None: - cache: LruCache[str, int] = LruCache(1) + cache: LruCache[str, int] = LruCache(max_size=1) cache["key"] = 1 self.assertEqual(cache.pop("key"), 1) self.assertEqual(cache.pop("key"), None) def test_del_multi(self) -> None: # The type here isn't quite correct as they don't handle TreeCache well. - cache: LruCache[Tuple[str, str], str] = LruCache(4, cache_type=TreeCache) + cache: LruCache[Tuple[str, str], str] = LruCache( + max_size=4, cache_type=TreeCache + ) cache[("animal", "cat")] = "mew" cache[("animal", "dog")] = "woof" cache[("vehicles", "car")] = "vroom" @@ -89,21 +91,23 @@ class LruCacheTestCase(unittest.HomeserverTestCase): # Man from del_multi say "Yes". def test_clear(self) -> None: - cache: LruCache[str, int] = LruCache(1) + cache: LruCache[str, int] = LruCache(max_size=1) cache["key"] = 1 cache.clear() self.assertEqual(len(cache), 0) @override_config({"caches": {"per_cache_factors": {"mycache": 10}}}) def test_special_size(self) -> None: - cache: LruCache = LruCache(10, "mycache") + cache: LruCache = LruCache( + max_size=10, server_name="test_server", cache_name="mycache" + ) self.assertEqual(cache.max_size, 100) class LruCacheCallbacksTestCase(unittest.HomeserverTestCase): def test_get(self) -> None: m = Mock() - cache: LruCache[str, str] = LruCache(1) + cache: LruCache[str, str] = LruCache(max_size=1) cache.set("key", "value") self.assertFalse(m.called) @@ -122,7 +126,7 @@ class LruCacheCallbacksTestCase(unittest.HomeserverTestCase): def test_multi_get(self) -> None: m = Mock() - cache: LruCache[str, str] = LruCache(1) + cache: LruCache[str, str] = LruCache(max_size=1) cache.set("key", "value") self.assertFalse(m.called) @@ -141,7 +145,7 @@ class LruCacheCallbacksTestCase(unittest.HomeserverTestCase): def test_set(self) -> None: m = Mock() - cache: LruCache[str, str] = LruCache(1) + cache: LruCache[str, str] = LruCache(max_size=1) cache.set("key", "value", callbacks=[m]) self.assertFalse(m.called) @@ -157,7 +161,7 @@ class LruCacheCallbacksTestCase(unittest.HomeserverTestCase): def test_pop(self) -> None: m = Mock() - cache: LruCache[str, str] = LruCache(1) + cache: LruCache[str, str] = LruCache(max_size=1) cache.set("key", "value", callbacks=[m]) self.assertFalse(m.called) @@ -177,7 +181,9 @@ class LruCacheCallbacksTestCase(unittest.HomeserverTestCase): m3 = Mock() m4 = Mock() # The type here isn't quite correct as they don't handle TreeCache well. - cache: LruCache[Tuple[str, str], str] = LruCache(4, cache_type=TreeCache) + cache: LruCache[Tuple[str, str], str] = LruCache( + max_size=4, cache_type=TreeCache + ) cache.set(("a", "1"), "value", callbacks=[m1]) cache.set(("a", "2"), "value", callbacks=[m2]) @@ -199,7 +205,7 @@ class LruCacheCallbacksTestCase(unittest.HomeserverTestCase): def test_clear(self) -> None: m1 = Mock() m2 = Mock() - cache: LruCache[str, str] = LruCache(5) + cache: LruCache[str, str] = LruCache(max_size=5) cache.set("key1", "value", callbacks=[m1]) cache.set("key2", "value", callbacks=[m2]) @@ -216,7 +222,7 @@ class LruCacheCallbacksTestCase(unittest.HomeserverTestCase): m1 = Mock(name="m1") m2 = Mock(name="m2") m3 = Mock(name="m3") - cache: LruCache[str, str] = LruCache(2) + cache: LruCache[str, str] = LruCache(max_size=2) cache.set("key1", "value", callbacks=[m1]) cache.set("key2", "value", callbacks=[m2]) @@ -252,7 +258,7 @@ class LruCacheCallbacksTestCase(unittest.HomeserverTestCase): class LruCacheSizedTestCase(unittest.HomeserverTestCase): def test_evict(self) -> None: - cache: LruCache[str, List[int]] = LruCache(5, size_callback=len) + cache: LruCache[str, List[int]] = LruCache(max_size=5, size_callback=len) cache["key1"] = [0] cache["key2"] = [1, 2] cache["key3"] = [3] @@ -275,7 +281,9 @@ class LruCacheSizedTestCase(unittest.HomeserverTestCase): def test_zero_size_drop_from_cache(self) -> None: """Test that `drop_from_cache` works correctly with 0-sized entries.""" - cache: LruCache[str, List[int]] = LruCache(5, size_callback=lambda x: 0) + cache: LruCache[str, List[int]] = LruCache( + max_size=5, size_callback=lambda x: 0 + ) cache["key1"] = [] self.assertEqual(len(cache), 0) @@ -299,7 +307,7 @@ class TimeEvictionTestCase(unittest.HomeserverTestCase): def test_evict(self) -> None: setup_expire_lru_cache_entries(self.hs) - cache: LruCache[str, int] = LruCache(5, clock=self.hs.get_clock()) + cache: LruCache[str, int] = LruCache(max_size=5, clock=self.hs.get_clock()) # Check that we evict entries we haven't accessed for 30 minutes. cache["key1"] = 1 @@ -351,7 +359,7 @@ class MemoryEvictionTestCase(unittest.HomeserverTestCase): mock_jemalloc_class.get_stat.return_value = 924288000 setup_expire_lru_cache_entries(self.hs) - cache: LruCache[str, int] = LruCache(4, clock=self.hs.get_clock()) + cache: LruCache[str, int] = LruCache(max_size=4, clock=self.hs.get_clock()) cache["key1"] = 1 cache["key2"] = 2 @@ -387,7 +395,9 @@ class MemoryEvictionTestCase(unittest.HomeserverTestCase): class ExtraIndexLruCacheTestCase(unittest.HomeserverTestCase): def test_invalidate_simple(self) -> None: - cache: LruCache[str, int] = LruCache(10, extra_index_cb=lambda k, v: str(v)) + cache: LruCache[str, int] = LruCache( + max_size=10, extra_index_cb=lambda k, v: str(v) + ) cache["key1"] = 1 cache["key2"] = 2 @@ -400,7 +410,9 @@ class ExtraIndexLruCacheTestCase(unittest.HomeserverTestCase): self.assertEqual(cache.get("key2"), 2) def test_invalidate_multi(self) -> None: - cache: LruCache[str, int] = LruCache(10, extra_index_cb=lambda k, v: str(v)) + cache: LruCache[str, int] = LruCache( + max_size=10, extra_index_cb=lambda k, v: str(v) + ) cache["key1"] = 1 cache["key2"] = 1 cache["key3"] = 2 diff --git a/tests/util/test_stream_change_cache.py b/tests/util/test_stream_change_cache.py index 9254bff79b..69a072cd36 100644 --- a/tests/util/test_stream_change_cache.py +++ b/tests/util/test_stream_change_cache.py @@ -15,7 +15,12 @@ class StreamChangeCacheTests(unittest.HomeserverTestCase): Providing a prefilled cache to StreamChangeCache will result in a cache with the prefilled-cache entered in. """ - cache = StreamChangeCache("#test", 1, prefilled_cache={"user@foo.com": 2}) + cache = StreamChangeCache( + name="#test", + server_name=self.hs.hostname, + current_stream_pos=1, + prefilled_cache={"user@foo.com": 2}, + ) self.assertTrue(cache.has_entity_changed("user@foo.com", 1)) def test_has_entity_changed(self) -> None: @@ -23,7 +28,9 @@ class StreamChangeCacheTests(unittest.HomeserverTestCase): StreamChangeCache.entity_has_changed will mark entities as changed, and has_entity_changed will observe the changed entities. """ - cache = StreamChangeCache("#test", 3) + cache = StreamChangeCache( + name="#test", server_name=self.hs.hostname, current_stream_pos=3 + ) cache.entity_has_changed("user@foo.com", 6) cache.entity_has_changed("bar@baz.net", 7) @@ -61,7 +68,9 @@ class StreamChangeCacheTests(unittest.HomeserverTestCase): StreamChangeCache.entity_has_changed will respect the max size and purge the oldest items upon reaching that max size. """ - cache = StreamChangeCache("#test", 1, max_size=2) + cache = StreamChangeCache( + name="#test", server_name=self.hs.hostname, current_stream_pos=1, max_size=2 + ) cache.entity_has_changed("user@foo.com", 2) cache.entity_has_changed("bar@baz.net", 3) @@ -100,7 +109,9 @@ class StreamChangeCacheTests(unittest.HomeserverTestCase): entities since the given position. If the position is before the start of the known stream, it returns None instead. """ - cache = StreamChangeCache("#test", 1) + cache = StreamChangeCache( + name="#test", server_name=self.hs.hostname, current_stream_pos=1 + ) cache.entity_has_changed("user@foo.com", 2) cache.entity_has_changed("bar@baz.net", 3) @@ -148,7 +159,9 @@ class StreamChangeCacheTests(unittest.HomeserverTestCase): stream position is before it, it will return True, otherwise False if the cache has no entries. """ - cache = StreamChangeCache("#test", 1) + cache = StreamChangeCache( + name="#test", server_name=self.hs.hostname, current_stream_pos=1 + ) # With no entities, it returns True for the past, present, and False for # the future. @@ -175,7 +188,9 @@ class StreamChangeCacheTests(unittest.HomeserverTestCase): stream position is earlier than the earliest known position, it will return all of the entities queried for. """ - cache = StreamChangeCache("#test", 1) + cache = StreamChangeCache( + name="#test", server_name=self.hs.hostname, current_stream_pos=1 + ) cache.entity_has_changed("user@foo.com", 2) cache.entity_has_changed("bar@baz.net", 3) @@ -242,7 +257,9 @@ class StreamChangeCacheTests(unittest.HomeserverTestCase): recent point where the entity could have changed. If the entity is not known, the stream start is provided instead. """ - cache = StreamChangeCache("#test", 1) + cache = StreamChangeCache( + name="#test", server_name=self.hs.hostname, current_stream_pos=1 + ) cache.entity_has_changed("user@foo.com", 2) cache.entity_has_changed("bar@baz.net", 3) @@ -260,7 +277,12 @@ class StreamChangeCacheTests(unittest.HomeserverTestCase): """ `StreamChangeCache.all_entities_changed(...)` will mark all entites as changed. """ - cache = StreamChangeCache("#test", 1, max_size=10) + cache = StreamChangeCache( + name="#test", + server_name=self.hs.hostname, + current_stream_pos=1, + max_size=10, + ) cache.entity_has_changed("user@foo.com", 2) cache.entity_has_changed("bar@baz.net", 3) From 42297bfcebeafae5c2117a94f4b8821dd8d1caf6 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 17 Jul 2025 11:06:54 +0100 Subject: [PATCH 012/185] Bump ruff from 0.12.2 to 0.12.3 (#18683) --- poetry.lock | 40 ++++++++++++++++++++-------------------- pyproject.toml | 2 +- 2 files changed, 21 insertions(+), 21 deletions(-) diff --git a/poetry.lock b/poetry.lock index ed5028c017..b106f20bfb 100644 --- a/poetry.lock +++ b/poetry.lock @@ -2447,30 +2447,30 @@ files = [ [[package]] name = "ruff" -version = "0.12.2" +version = "0.12.3" description = "An extremely fast Python linter and code formatter, written in Rust." optional = false python-versions = ">=3.7" groups = ["dev"] files = [ - {file = "ruff-0.12.2-py3-none-linux_armv6l.whl", hash = "sha256:093ea2b221df1d2b8e7ad92fc6ffdca40a2cb10d8564477a987b44fd4008a7be"}, - {file = "ruff-0.12.2-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:09e4cf27cc10f96b1708100fa851e0daf21767e9709e1649175355280e0d950e"}, - {file = "ruff-0.12.2-py3-none-macosx_11_0_arm64.whl", hash = "sha256:8ae64755b22f4ff85e9c52d1f82644abd0b6b6b6deedceb74bd71f35c24044cc"}, - {file = "ruff-0.12.2-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3eb3a6b2db4d6e2c77e682f0b988d4d61aff06860158fdb413118ca133d57922"}, - {file = "ruff-0.12.2-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:73448de992d05517170fc37169cbca857dfeaeaa8c2b9be494d7bcb0d36c8f4b"}, - {file = "ruff-0.12.2-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3b8b94317cbc2ae4a2771af641739f933934b03555e51515e6e021c64441532d"}, - {file = "ruff-0.12.2-py3-none-manylinux_2_17_ppc64.manylinux2014_ppc64.whl", hash = "sha256:45fc42c3bf1d30d2008023a0a9a0cfb06bf9835b147f11fe0679f21ae86d34b1"}, - {file = "ruff-0.12.2-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ce48f675c394c37e958bf229fb5c1e843e20945a6d962cf3ea20b7a107dcd9f4"}, - {file = "ruff-0.12.2-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:793d8859445ea47591272021a81391350205a4af65a9392401f418a95dfb75c9"}, - {file = "ruff-0.12.2-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6932323db80484dda89153da3d8e58164d01d6da86857c79f1961934354992da"}, - {file = "ruff-0.12.2-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:6aa7e623a3a11538108f61e859ebf016c4f14a7e6e4eba1980190cacb57714ce"}, - {file = "ruff-0.12.2-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:2a4a20aeed74671b2def096bdf2eac610c7d8ffcbf4fb0e627c06947a1d7078d"}, - {file = "ruff-0.12.2-py3-none-musllinux_1_2_i686.whl", hash = "sha256:71a4c550195612f486c9d1f2b045a600aeba851b298c667807ae933478fcef04"}, - {file = "ruff-0.12.2-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:4987b8f4ceadf597c927beee65a5eaf994c6e2b631df963f86d8ad1bdea99342"}, - {file = "ruff-0.12.2-py3-none-win32.whl", hash = "sha256:369ffb69b70cd55b6c3fc453b9492d98aed98062db9fec828cdfd069555f5f1a"}, - {file = "ruff-0.12.2-py3-none-win_amd64.whl", hash = "sha256:dca8a3b6d6dc9810ed8f328d406516bf4d660c00caeaef36eb831cf4871b0639"}, - {file = "ruff-0.12.2-py3-none-win_arm64.whl", hash = "sha256:48d6c6bfb4761df68bc05ae630e24f506755e702d4fb08f08460be778c7ccb12"}, - {file = "ruff-0.12.2.tar.gz", hash = "sha256:d7b4f55cd6f325cb7621244f19c873c565a08aff5a4ba9c69aa7355f3f7afd3e"}, + {file = "ruff-0.12.3-py3-none-linux_armv6l.whl", hash = "sha256:47552138f7206454eaf0c4fe827e546e9ddac62c2a3d2585ca54d29a890137a2"}, + {file = "ruff-0.12.3-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:0a9153b000c6fe169bb307f5bd1b691221c4286c133407b8827c406a55282041"}, + {file = "ruff-0.12.3-py3-none-macosx_11_0_arm64.whl", hash = "sha256:fa6b24600cf3b750e48ddb6057e901dd5b9aa426e316addb2a1af185a7509882"}, + {file = "ruff-0.12.3-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e2506961bf6ead54887ba3562604d69cb430f59b42133d36976421bc8bd45901"}, + {file = "ruff-0.12.3-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:c4faaff1f90cea9d3033cbbcdf1acf5d7fb11d8180758feb31337391691f3df0"}, + {file = "ruff-0.12.3-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:40dced4a79d7c264389de1c59467d5d5cefd79e7e06d1dfa2c75497b5269a5a6"}, + {file = "ruff-0.12.3-py3-none-manylinux_2_17_ppc64.manylinux2014_ppc64.whl", hash = "sha256:0262d50ba2767ed0fe212aa7e62112a1dcbfd46b858c5bf7bbd11f326998bafc"}, + {file = "ruff-0.12.3-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:12371aec33e1a3758597c5c631bae9a5286f3c963bdfb4d17acdd2d395406687"}, + {file = "ruff-0.12.3-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:560f13b6baa49785665276c963edc363f8ad4b4fc910a883e2625bdb14a83a9e"}, + {file = "ruff-0.12.3-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:023040a3499f6f974ae9091bcdd0385dd9e9eb4942f231c23c57708147b06311"}, + {file = "ruff-0.12.3-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:883d844967bffff5ab28bba1a4d246c1a1b2933f48cb9840f3fdc5111c603b07"}, + {file = "ruff-0.12.3-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:2120d3aa855ff385e0e562fdee14d564c9675edbe41625c87eeab744a7830d12"}, + {file = "ruff-0.12.3-py3-none-musllinux_1_2_i686.whl", hash = "sha256:6b16647cbb470eaf4750d27dddc6ebf7758b918887b56d39e9c22cce2049082b"}, + {file = "ruff-0.12.3-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:e1417051edb436230023575b149e8ff843a324557fe0a265863b7602df86722f"}, + {file = "ruff-0.12.3-py3-none-win32.whl", hash = "sha256:dfd45e6e926deb6409d0616078a666ebce93e55e07f0fb0228d4b2608b2c248d"}, + {file = "ruff-0.12.3-py3-none-win_amd64.whl", hash = "sha256:a946cf1e7ba3209bdef039eb97647f1c77f6f540e5845ec9c114d3af8df873e7"}, + {file = "ruff-0.12.3-py3-none-win_arm64.whl", hash = "sha256:5f9c7c9c8f84c2d7f27e93674d27136fbf489720251544c4da7fb3d742e011b1"}, + {file = "ruff-0.12.3.tar.gz", hash = "sha256:f1b5a4b6668fd7b7ea3697d8d98857390b40c1320a63a178eee6be0899ea2d77"}, ] [[package]] @@ -3391,4 +3391,4 @@ url-preview = ["lxml"] [metadata] lock-version = "2.1" python-versions = "^3.9.0" -content-hash = "1629969968eaaf20a8f49ee3cc0075b25e5dee7e81503fff66042ece72793103" +content-hash = "c140ab1db9e5d89d251e76c68f62cfcf5c477a923ba79f0fda186278f12af901" diff --git a/pyproject.toml b/pyproject.toml index 1f42c26817..89cd57a931 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -317,7 +317,7 @@ all = [ # failing on new releases. Keeping lower bounds loose here means that dependabot # can bump versions without having to update the content-hash in the lockfile. # This helps prevents merge conflicts when running a batch of dependabot updates. -ruff = "0.12.2" +ruff = "0.12.3" # Type checking only works with the pydantic.v1 compat module from pydantic v2 pydantic = "^2" From b07dc6a27d61aa568daa2dd3ce442ad3916b0e63 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 17 Jul 2025 11:07:15 +0100 Subject: [PATCH 013/185] Bump types-jsonschema from 4.24.0.20250528 to 4.24.0.20250708 (#18682) --- poetry.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/poetry.lock b/poetry.lock index b106f20bfb..02d3c8edb2 100644 --- a/poetry.lock +++ b/poetry.lock @@ -2970,14 +2970,14 @@ files = [ [[package]] name = "types-jsonschema" -version = "4.24.0.20250528" +version = "4.24.0.20250708" description = "Typing stubs for jsonschema" optional = false python-versions = ">=3.9" groups = ["dev"] files = [ - {file = "types_jsonschema-4.24.0.20250528-py3-none-any.whl", hash = "sha256:6a906b5ff73ac11c8d1e0b6c30a9693e1e4e1ab56c56c932b3a7e081b86d187b"}, - {file = "types_jsonschema-4.24.0.20250528.tar.gz", hash = "sha256:7e28c64e0ae7980eeb158105b20663fc6a6b8f81d5f86ea6614aa0014417bd1e"}, + {file = "types_jsonschema-4.24.0.20250708-py3-none-any.whl", hash = "sha256:d574aa3421d178a8435cc898cf4cf5e5e8c8f37b949c8e3ceeff06da433a18bf"}, + {file = "types_jsonschema-4.24.0.20250708.tar.gz", hash = "sha256:a910e4944681cbb1b18a93ffb502e09910db788314312fc763df08d8ac2aadb7"}, ] [package.dependencies] From a07e26a93603420fc66353dfee2d383729c9e994 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=A4rt?= Date: Thu, 17 Jul 2025 13:13:36 +0300 Subject: [PATCH 014/185] Fix sequence migration for autoincrement tables in synapse_port_db (#18677) Closes https://github.com/element-hq/synapse/issues/18053 - the sliding sync tables will now migrate properly. --- changelog.d/18677.bugfix | 1 + synapse/_scripts/synapse_port_db.py | 89 ++++++++++++++++++++++++++--- 2 files changed, 82 insertions(+), 8 deletions(-) create mode 100644 changelog.d/18677.bugfix diff --git a/changelog.d/18677.bugfix b/changelog.d/18677.bugfix new file mode 100644 index 0000000000..3b443d2055 --- /dev/null +++ b/changelog.d/18677.bugfix @@ -0,0 +1 @@ +Fix sliding_sync_connections related errors when porting from SQLite to Postgres. diff --git a/synapse/_scripts/synapse_port_db.py b/synapse/_scripts/synapse_port_db.py index 573c70696e..6c3e380355 100755 --- a/synapse/_scripts/synapse_port_db.py +++ b/synapse/_scripts/synapse_port_db.py @@ -190,13 +190,18 @@ APPEND_ONLY_TABLES = [ "users", ] +# These tables declare their id column with "PRIMARY KEY AUTOINCREMENT" on sqlite side +# and with "PRIMARY KEY GENERATED ALWAYS AS IDENTITY" on postgres side. This creates an +# implicit sequence that needs its value to be migrated separately. Additionally, +# inserting on postgres side needs to use the "OVERRIDING SYSTEM VALUE" modifier. +AUTOINCREMENT_TABLES = { + "sliding_sync_connections", + "sliding_sync_connection_positions", + "sliding_sync_connection_required_state", + "state_groups_pending_deletion", +} IGNORED_TABLES = { - # Porting the auto generated sequence in this table is non-trivial. - # None of the entries in this list are mandatory for Synapse to keep working. - # If state group disk space is an issue after the port, the - # `mark_unreferenced_state_groups_for_deletion_bg_update` background task can be run again. - "state_groups_pending_deletion", # We don't port these tables, as they're a faff and we can regenerate # them anyway. "user_directory", @@ -284,11 +289,17 @@ class Store( return self.db_pool.runInteraction("execute_sql", r) def insert_many_txn( - self, txn: LoggingTransaction, table: str, headers: List[str], rows: List[Tuple] + self, + txn: LoggingTransaction, + table: str, + headers: List[str], + rows: List[Tuple], + override_system_value: bool = False, ) -> None: - sql = "INSERT INTO %s (%s) VALUES (%s)" % ( + sql = "INSERT INTO %s (%s) %s VALUES (%s)" % ( table, ", ".join(k for k in headers), + "OVERRIDING SYSTEM VALUE" if override_system_value else "", ", ".join("%s" for _ in headers), ) @@ -532,7 +543,13 @@ class Porter: def insert(txn: LoggingTransaction) -> None: assert headers is not None - self.postgres_store.insert_many_txn(txn, table, headers[1:], rows) + self.postgres_store.insert_many_txn( + txn, + table, + headers[1:], + rows, + override_system_value=table in AUTOINCREMENT_TABLES, + ) self.postgres_store.db_pool.simple_update_one_txn( txn, @@ -884,6 +901,19 @@ class Porter: ], ) + await self._setup_autoincrement_sequence( + "sliding_sync_connection_positions", "connection_position" + ) + await self._setup_autoincrement_sequence( + "sliding_sync_connection_required_state", "required_state_id" + ) + await self._setup_autoincrement_sequence( + "sliding_sync_connections", "connection_key" + ) + await self._setup_autoincrement_sequence( + "state_groups_pending_deletion", "sequence_number" + ) + # Step 3. Get tables. self.progress.set_state("Fetching tables") sqlite_tables = await self.sqlite_store.db_pool.simple_select_onecol( @@ -1216,6 +1246,49 @@ class Porter: "_setup_%s" % (sequence_name,), r ) + async def _setup_autoincrement_sequence( + self, + sqlite_table_name: str, + sqlite_id_column_name: str, + ) -> None: + """Set a sequence to the correct value. Use where id column was declared with PRIMARY KEY AUTOINCREMENT.""" + seq_name = await self._pg_get_serial_sequence( + sqlite_table_name, sqlite_id_column_name + ) + if seq_name is None: + raise Exception( + "implicit sequence not found for table " + sqlite_table_name + ) + + seq_value = await self.sqlite_store.db_pool.simple_select_one_onecol( + table="sqlite_sequence", + keyvalues={"name": sqlite_table_name}, + retcol="seq", + allow_none=True, + ) + if seq_value is None: + return + + def r(txn: LoggingTransaction) -> None: + sql = "ALTER SEQUENCE %s RESTART WITH" % (seq_name,) + txn.execute(sql + " %s", (seq_value + 1,)) + + await self.postgres_store.db_pool.runInteraction("_setup_%s" % (seq_name,), r) + + async def _pg_get_serial_sequence(self, table: str, column: str) -> Optional[str]: + """Returns the name of the postgres sequence associated with a column, or NULL.""" + + def r(txn: LoggingTransaction) -> Optional[str]: + txn.execute("SELECT pg_get_serial_sequence('%s', '%s')" % (table, column)) + result = txn.fetchone() + if not result: + return None + return result[0] + + return await self.postgres_store.db_pool.runInteraction( + "_pg_get_serial_sequence", r + ) + async def _setup_auth_chain_sequence(self) -> None: curr_chain_id: Optional[ int From 56c166cbf0cdcbe62420b9f842260f737e4db618 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Thu, 17 Jul 2025 12:28:17 +0200 Subject: [PATCH 015/185] Include `event_id` when getting state with `?format=event` (#18675) --- changelog.d/18675.feature | 1 + synapse/rest/client/room.py | 16 ++++++++++-- tests/rest/client/test_rooms.py | 45 +++++++++++++++++++++++++++++++++ 3 files changed, 60 insertions(+), 2 deletions(-) create mode 100644 changelog.d/18675.feature diff --git a/changelog.d/18675.feature b/changelog.d/18675.feature new file mode 100644 index 0000000000..7ff2445807 --- /dev/null +++ b/changelog.d/18675.feature @@ -0,0 +1 @@ +Include `event_id` when getting state with `?format=event`. Contributed by @tulir @ Beeper. diff --git a/synapse/rest/client/room.py b/synapse/rest/client/room.py index 4600a87778..6b0deda0df 100644 --- a/synapse/rest/client/room.py +++ b/synapse/rest/client/room.py @@ -44,7 +44,11 @@ from synapse.api.errors import ( UnredactedContentDeletedError, ) from synapse.api.filtering import Filter -from synapse.events.utils import SerializeEventConfig, format_event_for_client_v2 +from synapse.events.utils import ( + SerializeEventConfig, + format_event_for_client_v2, + serialize_event, +) from synapse.http.server import HttpServer from synapse.http.servlet import ( ResolveRoomIdMixin, @@ -198,6 +202,7 @@ class RoomStateEventRestServlet(RestServlet): self.message_handler = hs.get_message_handler() self.delayed_events_handler = hs.get_delayed_events_handler() self.auth = hs.get_auth() + self.clock = hs.get_clock() self._max_event_delay_ms = hs.config.server.max_event_delay_ms self._spam_checker_module_callbacks = hs.get_module_api_callbacks().spam_checker @@ -268,7 +273,14 @@ class RoomStateEventRestServlet(RestServlet): raise SynapseError(404, "Event not found.", errcode=Codes.NOT_FOUND) if format == "event": - event = format_event_for_client_v2(data.get_dict()) + event = serialize_event( + data, + self.clock.time_msec(), + config=SerializeEventConfig( + event_format=format_event_for_client_v2, + requester=requester, + ), + ) return 200, event elif format == "content": return 200, data.get_dict()["content"] diff --git a/tests/rest/client/test_rooms.py b/tests/rest/client/test_rooms.py index 8a6e6f118a..48d33b8e17 100644 --- a/tests/rest/client/test_rooms.py +++ b/tests/rest/client/test_rooms.py @@ -552,6 +552,51 @@ class RoomStateTestCase(RoomBase): self.assertEqual(HTTPStatus.OK, channel.code, msg=channel.result["body"]) self.assertEqual(channel.json_body, {"membership": "join"}) + def test_get_state_format_content(self) -> None: + """Test response of a `/rooms/$room_id/state/$event_type?format=content` request.""" + room_id = self.helper.create_room_as(self.user_id) + channel1 = self.make_request( + "GET", + "/rooms/%s/state/m.room.member/%s?format=content" + % ( + room_id, + self.user_id, + ), + ) + self.assertEqual(channel1.code, HTTPStatus.OK, channel1.json_body) + self.assertEqual(channel1.json_body, {"membership": "join"}) + channel2 = self.make_request( + "GET", + "/rooms/%s/state/m.room.member/%s" + % ( + room_id, + self.user_id, + ), + ) + self.assertEqual(channel2.code, HTTPStatus.OK, channel2.json_body) + # "content" is the default format. + self.assertEqual(channel1.json_body, channel2.json_body) + + def test_get_state_format_event(self) -> None: + """Test response of a `/rooms/$room_id/state/$event_type?format=event` request.""" + room_id = self.helper.create_room_as(self.user_id) + channel = self.make_request( + "GET", + "/rooms/%s/state/m.room.member/%s?format=event" + % ( + room_id, + self.user_id, + ), + ) + self.assertEqual(channel.code, HTTPStatus.OK, channel.json_body) + self.assertEqual(channel.json_body["content"], {"membership": "join"}) + self.assertEqual(channel.json_body["room_id"], room_id) + self.assertRegex(channel.json_body["event_id"], r"\$.+") + self.assertEqual(channel.json_body["type"], "m.room.member") + self.assertEqual(channel.json_body["sender"], self.user_id) + self.assertEqual(channel.json_body["state_key"], self.user_id) + self.assertTrue(type(channel.json_body["origin_server_ts"]) is int) + class RoomsMemberListTestCase(RoomBase): """Tests /rooms/$room_id/members/list REST events.""" From 84991317d0ab2270a04cdea710fb45f5ec89f996 Mon Sep 17 00:00:00 2001 From: Patrick Cloke Date: Thu, 17 Jul 2025 09:15:07 -0400 Subject: [PATCH 016/185] Stabilize support for custom profile fields. (#18635) --- changelog.d/18635.feature | 1 + synapse/handlers/profile.py | 24 ++-- synapse/rest/client/capabilities.py | 30 ++-- synapse/rest/client/profile.py | 190 ++++--------------------- tests/rest/client/test_capabilities.py | 12 ++ tests/rest/client/test_profile.py | 57 +++----- 6 files changed, 88 insertions(+), 226 deletions(-) create mode 100644 changelog.d/18635.feature diff --git a/changelog.d/18635.feature b/changelog.d/18635.feature new file mode 100644 index 0000000000..af536f64d3 --- /dev/null +++ b/changelog.d/18635.feature @@ -0,0 +1 @@ +Support arbitrary profile fields. diff --git a/synapse/handlers/profile.py b/synapse/handlers/profile.py index 4958ab5e75..da392e115f 100644 --- a/synapse/handlers/profile.py +++ b/synapse/handlers/profile.py @@ -93,9 +93,7 @@ class ProfileHandler: if self.hs.is_mine(target_user): profileinfo = await self.store.get_profileinfo(target_user) - extra_fields = {} - if self.hs.config.experimental.msc4133_enabled: - extra_fields = await self.store.get_profile_fields(target_user) + extra_fields = await self.store.get_profile_fields(target_user) if ( profileinfo.display_name is None @@ -551,16 +549,16 @@ class ProfileHandler: # since then we send a null in the JSON response if avatar_url is not None: response["avatar_url"] = avatar_url - if self.hs.config.experimental.msc4133_enabled: - if just_field is None: - response.update(await self.store.get_profile_fields(user)) - elif just_field not in ( - ProfileFields.DISPLAYNAME, - ProfileFields.AVATAR_URL, - ): - response[just_field] = await self.store.get_profile_field( - user, just_field - ) + + if just_field is None: + response.update(await self.store.get_profile_fields(user)) + elif just_field not in ( + ProfileFields.DISPLAYNAME, + ProfileFields.AVATAR_URL, + ): + response[just_field] = await self.store.get_profile_field( + user, just_field + ) except StoreError as e: if e.code == 404: raise SynapseError(404, "Profile was not found", Codes.NOT_FOUND) diff --git a/synapse/rest/client/capabilities.py b/synapse/rest/client/capabilities.py index 8f3193fb47..a279db1cc5 100644 --- a/synapse/rest/client/capabilities.py +++ b/synapse/rest/client/capabilities.py @@ -92,22 +92,22 @@ class CapabilitiesRestServlet(RestServlet): "enabled": self.config.experimental.msc3664_enabled, } - if self.config.experimental.msc4133_enabled: - response["capabilities"]["uk.tcpip.msc4133.profile_fields"] = { - "enabled": True, - } + disallowed_profile_fields = [] + response["capabilities"]["m.profile_fields"] = {"enabled": True} + if not self.config.registration.enable_set_displayname: + disallowed_profile_fields.append("displayname") + if not self.config.registration.enable_set_avatar_url: + disallowed_profile_fields.append("avatar_url") + if disallowed_profile_fields: + response["capabilities"]["m.profile_fields"]["disallowed"] = ( + disallowed_profile_fields + ) - # Ensure this is consistent with the legacy m.set_displayname and - # m.set_avatar_url. - disallowed = [] - if not self.config.registration.enable_set_displayname: - disallowed.append("displayname") - if not self.config.registration.enable_set_avatar_url: - disallowed.append("avatar_url") - if disallowed: - response["capabilities"]["uk.tcpip.msc4133.profile_fields"][ - "disallowed" - ] = disallowed + # For transition from unstable to stable identifiers. + if self.config.experimental.msc4133_enabled: + response["capabilities"]["uk.tcpip.msc4133.profile_fields"] = response[ + "capabilities" + ]["m.profile_fields"] if self.config.experimental.msc4267_enabled: response["capabilities"]["org.matrix.msc4267.forget_forced_upon_leave"] = { diff --git a/synapse/rest/client/profile.py b/synapse/rest/client/profile.py index 8326d8017c..243245f739 100644 --- a/synapse/rest/client/profile.py +++ b/synapse/rest/client/profile.py @@ -57,161 +57,6 @@ def _read_propagate(hs: "HomeServer", request: SynapseRequest) -> bool: return propagate -class ProfileDisplaynameRestServlet(RestServlet): - PATTERNS = client_patterns("/profile/(?P[^/]*)/displayname", v1=True) - CATEGORY = "Event sending requests" - - def __init__(self, hs: "HomeServer"): - super().__init__() - self.hs = hs - self.profile_handler = hs.get_profile_handler() - self.auth = hs.get_auth() - - async def on_GET( - self, request: SynapseRequest, user_id: str - ) -> Tuple[int, JsonDict]: - requester_user = None - - if self.hs.config.server.require_auth_for_profile_requests: - requester = await self.auth.get_user_by_req(request) - requester_user = requester.user - - if not UserID.is_valid(user_id): - raise SynapseError( - HTTPStatus.BAD_REQUEST, "Invalid user id", Codes.INVALID_PARAM - ) - - user = UserID.from_string(user_id) - await self.profile_handler.check_profile_query_allowed(user, requester_user) - - displayname = await self.profile_handler.get_displayname(user) - - ret = {} - if displayname is not None: - ret["displayname"] = displayname - - return 200, ret - - async def on_PUT( - self, request: SynapseRequest, user_id: str - ) -> Tuple[int, JsonDict]: - if not UserID.is_valid(user_id): - raise SynapseError( - HTTPStatus.BAD_REQUEST, "Invalid user id", Codes.INVALID_PARAM - ) - - requester = await self.auth.get_user_by_req(request, allow_guest=True) - user = UserID.from_string(user_id) - is_admin = await self.auth.is_server_admin(requester) - - content = parse_json_object_from_request(request) - - try: - new_name = content["displayname"] - except Exception: - raise SynapseError( - 400, "Missing key 'displayname'", errcode=Codes.MISSING_PARAM - ) - - propagate = _read_propagate(self.hs, request) - - requester_suspended = ( - await self.hs.get_datastores().main.get_user_suspended_status( - requester.user.to_string() - ) - ) - - if requester_suspended: - raise SynapseError( - 403, - "Updating displayname while account is suspended is not allowed.", - Codes.USER_ACCOUNT_SUSPENDED, - ) - - await self.profile_handler.set_displayname( - user, requester, new_name, is_admin, propagate=propagate - ) - - return 200, {} - - -class ProfileAvatarURLRestServlet(RestServlet): - PATTERNS = client_patterns("/profile/(?P[^/]*)/avatar_url", v1=True) - CATEGORY = "Event sending requests" - - def __init__(self, hs: "HomeServer"): - super().__init__() - self.hs = hs - self.profile_handler = hs.get_profile_handler() - self.auth = hs.get_auth() - - async def on_GET( - self, request: SynapseRequest, user_id: str - ) -> Tuple[int, JsonDict]: - requester_user = None - - if self.hs.config.server.require_auth_for_profile_requests: - requester = await self.auth.get_user_by_req(request) - requester_user = requester.user - - if not UserID.is_valid(user_id): - raise SynapseError( - HTTPStatus.BAD_REQUEST, "Invalid user id", Codes.INVALID_PARAM - ) - - user = UserID.from_string(user_id) - await self.profile_handler.check_profile_query_allowed(user, requester_user) - - avatar_url = await self.profile_handler.get_avatar_url(user) - - ret = {} - if avatar_url is not None: - ret["avatar_url"] = avatar_url - - return 200, ret - - async def on_PUT( - self, request: SynapseRequest, user_id: str - ) -> Tuple[int, JsonDict]: - if not UserID.is_valid(user_id): - raise SynapseError( - HTTPStatus.BAD_REQUEST, "Invalid user id", Codes.INVALID_PARAM - ) - - requester = await self.auth.get_user_by_req(request) - user = UserID.from_string(user_id) - is_admin = await self.auth.is_server_admin(requester) - - content = parse_json_object_from_request(request) - try: - new_avatar_url = content["avatar_url"] - except KeyError: - raise SynapseError( - 400, "Missing key 'avatar_url'", errcode=Codes.MISSING_PARAM - ) - - propagate = _read_propagate(self.hs, request) - - requester_suspended = ( - await self.hs.get_datastores().main.get_user_suspended_status( - requester.user.to_string() - ) - ) - - if requester_suspended: - raise SynapseError( - 403, - "Updating avatar URL while account is suspended is not allowed.", - Codes.USER_ACCOUNT_SUSPENDED, - ) - - await self.profile_handler.set_avatar_url( - user, requester, new_avatar_url, is_admin, propagate=propagate - ) - - return 200, {} - - class ProfileRestServlet(RestServlet): PATTERNS = client_patterns("/profile/(?P[^/]*)", v1=True) CATEGORY = "Event sending requests" @@ -244,12 +89,19 @@ class ProfileRestServlet(RestServlet): return 200, ret -class UnstableProfileFieldRestServlet(RestServlet): +class ProfileFieldRestServlet(RestServlet): PATTERNS = [ + *client_patterns( + "/profile/(?P[^/]*)/(?Pdisplayname)", v1=True + ), + *client_patterns( + "/profile/(?P[^/]*)/(?Pavatar_url)", v1=True + ), re.compile( - r"^/_matrix/client/unstable/uk\.tcpip\.msc4133/profile/(?P[^/]*)/(?P[^/]*)" - ) + r"^/_matrix/client/v3/profile/(?P[^/]*)/(?P[^/]*)" + ), ] + CATEGORY = "Event sending requests" def __init__(self, hs: "HomeServer"): @@ -304,7 +156,10 @@ class UnstableProfileFieldRestServlet(RestServlet): HTTPStatus.BAD_REQUEST, "Invalid user id", Codes.INVALID_PARAM ) - requester = await self.auth.get_user_by_req(request) + # Guest users are able to set their own displayname. + requester = await self.auth.get_user_by_req( + request, allow_guest=field_name == ProfileFields.DISPLAYNAME + ) user = UserID.from_string(user_id) is_admin = await self.auth.is_server_admin(requester) @@ -366,7 +221,10 @@ class UnstableProfileFieldRestServlet(RestServlet): HTTPStatus.BAD_REQUEST, "Invalid user id", Codes.INVALID_PARAM ) - requester = await self.auth.get_user_by_req(request) + # Guest users are able to set their own displayname. + requester = await self.auth.get_user_by_req( + request, allow_guest=field_name == ProfileFields.DISPLAYNAME + ) user = UserID.from_string(user_id) is_admin = await self.auth.is_server_admin(requester) @@ -413,11 +271,15 @@ class UnstableProfileFieldRestServlet(RestServlet): return 200, {} +class UnstableProfileFieldRestServlet(ProfileFieldRestServlet): + re.compile( + r"^/_matrix/client/unstable/uk\.tcpip\.msc4133/profile/(?P[^/]*)/(?P[^/]*)" + ) + + def register_servlets(hs: "HomeServer", http_server: HttpServer) -> None: - # The specific displayname / avatar URL / custom field endpoints *must* appear - # before their corresponding generic profile endpoint. - ProfileDisplaynameRestServlet(hs).register(http_server) - ProfileAvatarURLRestServlet(hs).register(http_server) + # The specific field endpoint *must* appear before the generic profile endpoint. + ProfileFieldRestServlet(hs).register(http_server) ProfileRestServlet(hs).register(http_server) if hs.config.experimental.msc4133_enabled: UnstableProfileFieldRestServlet(hs).register(http_server) diff --git a/tests/rest/client/test_capabilities.py b/tests/rest/client/test_capabilities.py index 13831462e8..cdf31155fd 100644 --- a/tests/rest/client/test_capabilities.py +++ b/tests/rest/client/test_capabilities.py @@ -130,6 +130,10 @@ class CapabilitiesTestCase(unittest.HomeserverTestCase): self.assertEqual(channel.code, HTTPStatus.OK) self.assertFalse(capabilities["m.set_displayname"]["enabled"]) + self.assertTrue(capabilities["m.profile_fields"]["enabled"]) + self.assertEqual( + capabilities["m.profile_fields"]["disallowed"], ["displayname"] + ) @override_config({"enable_set_avatar_url": False}) def test_get_set_avatar_url_capabilities_avatar_url_disabled(self) -> None: @@ -141,6 +145,8 @@ class CapabilitiesTestCase(unittest.HomeserverTestCase): self.assertEqual(channel.code, HTTPStatus.OK) self.assertFalse(capabilities["m.set_avatar_url"]["enabled"]) + self.assertTrue(capabilities["m.profile_fields"]["enabled"]) + self.assertEqual(capabilities["m.profile_fields"]["disallowed"], ["avatar_url"]) @override_config( { @@ -159,6 +165,10 @@ class CapabilitiesTestCase(unittest.HomeserverTestCase): self.assertEqual(channel.code, HTTPStatus.OK) self.assertFalse(capabilities["m.set_displayname"]["enabled"]) + self.assertTrue(capabilities["m.profile_fields"]["enabled"]) + self.assertEqual( + capabilities["m.profile_fields"]["disallowed"], ["displayname"] + ) self.assertTrue(capabilities["uk.tcpip.msc4133.profile_fields"]["enabled"]) self.assertEqual( capabilities["uk.tcpip.msc4133.profile_fields"]["disallowed"], @@ -180,6 +190,8 @@ class CapabilitiesTestCase(unittest.HomeserverTestCase): self.assertEqual(channel.code, HTTPStatus.OK) self.assertFalse(capabilities["m.set_avatar_url"]["enabled"]) + self.assertTrue(capabilities["m.profile_fields"]["enabled"]) + self.assertEqual(capabilities["m.profile_fields"]["disallowed"], ["avatar_url"]) self.assertTrue(capabilities["uk.tcpip.msc4133.profile_fields"]["enabled"]) self.assertEqual( capabilities["uk.tcpip.msc4133.profile_fields"]["disallowed"], diff --git a/tests/rest/client/test_profile.py b/tests/rest/client/test_profile.py index 708402b792..49776d8e8c 100644 --- a/tests/rest/client/test_profile.py +++ b/tests/rest/client/test_profile.py @@ -484,38 +484,34 @@ class ProfileTestCase(unittest.HomeserverTestCase): # The client requested ?propagate=true, so it should have happened. self.assertEqual(channel.json_body.get(prop), "http://my.server/pic.gif") - @unittest.override_config({"experimental_features": {"msc4133_enabled": True}}) def test_get_missing_custom_field(self) -> None: channel = self.make_request( "GET", - f"/_matrix/client/unstable/uk.tcpip.msc4133/profile/{self.owner}/custom_field", + f"/_matrix/client/v3/profile/{self.owner}/custom_field", ) self.assertEqual(channel.code, HTTPStatus.NOT_FOUND, channel.result) self.assertEqual(channel.json_body["errcode"], Codes.NOT_FOUND) - @unittest.override_config({"experimental_features": {"msc4133_enabled": True}}) def test_get_missing_custom_field_invalid_field_name(self) -> None: channel = self.make_request( "GET", - f"/_matrix/client/unstable/uk.tcpip.msc4133/profile/{self.owner}/[custom_field]", + f"/_matrix/client/v3/profile/{self.owner}/[custom_field]", ) self.assertEqual(channel.code, HTTPStatus.BAD_REQUEST, channel.result) self.assertEqual(channel.json_body["errcode"], Codes.INVALID_PARAM) - @unittest.override_config({"experimental_features": {"msc4133_enabled": True}}) def test_get_custom_field_rejects_bad_username(self) -> None: channel = self.make_request( "GET", - f"/_matrix/client/unstable/uk.tcpip.msc4133/profile/{urllib.parse.quote('@alice:')}/custom_field", + f"/_matrix/client/v3/profile/{urllib.parse.quote('@alice:')}/custom_field", ) self.assertEqual(channel.code, HTTPStatus.BAD_REQUEST, channel.result) self.assertEqual(channel.json_body["errcode"], Codes.INVALID_PARAM) - @unittest.override_config({"experimental_features": {"msc4133_enabled": True}}) def test_set_custom_field(self) -> None: channel = self.make_request( "PUT", - f"/_matrix/client/unstable/uk.tcpip.msc4133/profile/{self.owner}/custom_field", + f"/_matrix/client/v3/profile/{self.owner}/custom_field", content={"custom_field": "test"}, access_token=self.owner_tok, ) @@ -523,7 +519,7 @@ class ProfileTestCase(unittest.HomeserverTestCase): channel = self.make_request( "GET", - f"/_matrix/client/unstable/uk.tcpip.msc4133/profile/{self.owner}/custom_field", + f"/_matrix/client/v3/profile/{self.owner}/custom_field", ) self.assertEqual(channel.code, HTTPStatus.OK, channel.result) self.assertEqual(channel.json_body, {"custom_field": "test"}) @@ -531,7 +527,7 @@ class ProfileTestCase(unittest.HomeserverTestCase): # Overwriting the field should work. channel = self.make_request( "PUT", - f"/_matrix/client/unstable/uk.tcpip.msc4133/profile/{self.owner}/custom_field", + f"/_matrix/client/v3/profile/{self.owner}/custom_field", content={"custom_field": "new_Value"}, access_token=self.owner_tok, ) @@ -539,7 +535,7 @@ class ProfileTestCase(unittest.HomeserverTestCase): channel = self.make_request( "GET", - f"/_matrix/client/unstable/uk.tcpip.msc4133/profile/{self.owner}/custom_field", + f"/_matrix/client/v3/profile/{self.owner}/custom_field", ) self.assertEqual(channel.code, HTTPStatus.OK, channel.result) self.assertEqual(channel.json_body, {"custom_field": "new_Value"}) @@ -547,7 +543,7 @@ class ProfileTestCase(unittest.HomeserverTestCase): # Deleting the field should work. channel = self.make_request( "DELETE", - f"/_matrix/client/unstable/uk.tcpip.msc4133/profile/{self.owner}/custom_field", + f"/_matrix/client/v3/profile/{self.owner}/custom_field", content={}, access_token=self.owner_tok, ) @@ -555,12 +551,11 @@ class ProfileTestCase(unittest.HomeserverTestCase): channel = self.make_request( "GET", - f"/_matrix/client/unstable/uk.tcpip.msc4133/profile/{self.owner}/custom_field", + f"/_matrix/client/v3/profile/{self.owner}/custom_field", ) self.assertEqual(channel.code, HTTPStatus.NOT_FOUND, channel.result) self.assertEqual(channel.json_body["errcode"], Codes.NOT_FOUND) - @unittest.override_config({"experimental_features": {"msc4133_enabled": True}}) def test_non_string(self) -> None: """Non-string fields are supported for custom fields.""" fields = { @@ -574,7 +569,7 @@ class ProfileTestCase(unittest.HomeserverTestCase): for key, value in fields.items(): channel = self.make_request( "PUT", - f"/_matrix/client/unstable/uk.tcpip.msc4133/profile/{self.owner}/{key}", + f"/_matrix/client/v3/profile/{self.owner}/{key}", content={key: value}, access_token=self.owner_tok, ) @@ -591,22 +586,20 @@ class ProfileTestCase(unittest.HomeserverTestCase): for key, value in fields.items(): channel = self.make_request( "GET", - f"/_matrix/client/unstable/uk.tcpip.msc4133/profile/{self.owner}/{key}", + f"/_matrix/client/v3/profile/{self.owner}/{key}", ) self.assertEqual(channel.code, HTTPStatus.OK, channel.result) self.assertEqual(channel.json_body, {key: value}) - @unittest.override_config({"experimental_features": {"msc4133_enabled": True}}) def test_set_custom_field_noauth(self) -> None: channel = self.make_request( "PUT", - f"/_matrix/client/unstable/uk.tcpip.msc4133/profile/{self.owner}/custom_field", + f"/_matrix/client/v3/profile/{self.owner}/custom_field", content={"custom_field": "test"}, ) self.assertEqual(channel.code, 401, channel.result) self.assertEqual(channel.json_body["errcode"], Codes.MISSING_TOKEN) - @unittest.override_config({"experimental_features": {"msc4133_enabled": True}}) def test_set_custom_field_size(self) -> None: """ Attempts to set a custom field name that is too long should get a 400 error. @@ -614,7 +607,7 @@ class ProfileTestCase(unittest.HomeserverTestCase): # Key is missing. channel = self.make_request( "PUT", - f"/_matrix/client/unstable/uk.tcpip.msc4133/profile/{self.owner}/", + f"/_matrix/client/v3/profile/{self.owner}/", content={"": "test"}, access_token=self.owner_tok, ) @@ -625,7 +618,7 @@ class ProfileTestCase(unittest.HomeserverTestCase): key = "c" * 500 channel = self.make_request( "PUT", - f"/_matrix/client/unstable/uk.tcpip.msc4133/profile/{self.owner}/{key}", + f"/_matrix/client/v3/profile/{self.owner}/{key}", content={key: "test"}, access_token=self.owner_tok, ) @@ -634,7 +627,7 @@ class ProfileTestCase(unittest.HomeserverTestCase): channel = self.make_request( "DELETE", - f"/_matrix/client/unstable/uk.tcpip.msc4133/profile/{self.owner}/{key}", + f"/_matrix/client/v3/profile/{self.owner}/{key}", content={key: "test"}, access_token=self.owner_tok, ) @@ -644,14 +637,13 @@ class ProfileTestCase(unittest.HomeserverTestCase): # Key doesn't match body. channel = self.make_request( "PUT", - f"/_matrix/client/unstable/uk.tcpip.msc4133/profile/{self.owner}/custom_field", + f"/_matrix/client/v3/profile/{self.owner}/custom_field", content={"diff_key": "test"}, access_token=self.owner_tok, ) self.assertEqual(channel.code, 400, channel.result) self.assertEqual(channel.json_body["errcode"], Codes.MISSING_PARAM) - @unittest.override_config({"experimental_features": {"msc4133_enabled": True}}) def test_set_custom_field_profile_too_long(self) -> None: """ Attempts to set a custom field that would push the overall profile too large. @@ -664,7 +656,7 @@ class ProfileTestCase(unittest.HomeserverTestCase): key = "a" channel = self.make_request( "PUT", - f"/_matrix/client/unstable/uk.tcpip.msc4133/profile/{self.owner}/{key}", + f"/_matrix/client/v3/profile/{self.owner}/{key}", content={key: "a" * 65498}, access_token=self.owner_tok, ) @@ -692,7 +684,7 @@ class ProfileTestCase(unittest.HomeserverTestCase): key = "b" channel = self.make_request( "PUT", - f"/_matrix/client/unstable/uk.tcpip.msc4133/profile/{self.owner}/{key}", + f"/_matrix/client/v3/profile/{self.owner}/{key}", content={key: "1" + "a" * ADDITIONAL_CHARS}, access_token=self.owner_tok, ) @@ -722,7 +714,7 @@ class ProfileTestCase(unittest.HomeserverTestCase): key = "b" channel = self.make_request( "PUT", - f"/_matrix/client/unstable/uk.tcpip.msc4133/profile/{self.owner}/{key}", + f"/_matrix/client/v3/profile/{self.owner}/{key}", content={key: "" + "a" * ADDITIONAL_CHARS}, access_token=self.owner_tok, ) @@ -732,17 +724,16 @@ class ProfileTestCase(unittest.HomeserverTestCase): key = "a" channel = self.make_request( "PUT", - f"/_matrix/client/unstable/uk.tcpip.msc4133/profile/{self.owner}/{key}", + f"/_matrix/client/v3/profile/{self.owner}/{key}", content={key: ""}, access_token=self.owner_tok, ) self.assertEqual(channel.code, 200, channel.result) - @unittest.override_config({"experimental_features": {"msc4133_enabled": True}}) def test_set_custom_field_displayname(self) -> None: channel = self.make_request( "PUT", - f"/_matrix/client/unstable/uk.tcpip.msc4133/profile/{self.owner}/displayname", + f"/_matrix/client/v3/profile/{self.owner}/displayname", content={"displayname": "test"}, access_token=self.owner_tok, ) @@ -751,11 +742,10 @@ class ProfileTestCase(unittest.HomeserverTestCase): displayname = self._get_displayname() self.assertEqual(displayname, "test") - @unittest.override_config({"experimental_features": {"msc4133_enabled": True}}) def test_set_custom_field_avatar_url(self) -> None: channel = self.make_request( "PUT", - f"/_matrix/client/unstable/uk.tcpip.msc4133/profile/{self.owner}/avatar_url", + f"/_matrix/client/v3/profile/{self.owner}/avatar_url", content={"avatar_url": "mxc://test/good"}, access_token=self.owner_tok, ) @@ -764,12 +754,11 @@ class ProfileTestCase(unittest.HomeserverTestCase): avatar_url = self._get_avatar_url() self.assertEqual(avatar_url, "mxc://test/good") - @unittest.override_config({"experimental_features": {"msc4133_enabled": True}}) def test_set_custom_field_other(self) -> None: """Setting someone else's profile field should fail""" channel = self.make_request( "PUT", - f"/_matrix/client/unstable/uk.tcpip.msc4133/profile/{self.other}/custom_field", + f"/_matrix/client/v3/profile/{self.other}/custom_field", content={"custom_field": "test"}, access_token=self.owner_tok, ) From a0d6469069d0bd2b1aed2b0c78606d02ebb15ae9 Mon Sep 17 00:00:00 2001 From: Kim Brose <2803622+HarHarLinks@users.noreply.github.com> Date: Thu, 17 Jul 2025 14:54:33 +0000 Subject: [PATCH 017/185] fix schema and docs of `rc_delayed_event_mgmt` (#18692) Signed-off-by: Kim Brose --- changelog.d/18692.doc | 1 + docs/usage/configuration/config_documentation.md | 5 ++--- schema/synapse-config.schema.yaml | 5 ++--- 3 files changed, 5 insertions(+), 6 deletions(-) create mode 100644 changelog.d/18692.doc diff --git a/changelog.d/18692.doc b/changelog.d/18692.doc new file mode 100644 index 0000000000..1514ef6fb8 --- /dev/null +++ b/changelog.d/18692.doc @@ -0,0 +1 @@ +Update `rc_delayed_event_mgmt` docs to the actual nesting level. Contributed by @HarHarLinks. diff --git a/docs/usage/configuration/config_documentation.md b/docs/usage/configuration/config_documentation.md index 6918559dea..e47a32f510 100644 --- a/docs/usage/configuration/config_documentation.md +++ b/docs/usage/configuration/config_documentation.md @@ -1925,9 +1925,8 @@ This setting has the following sub-options: Default configuration: ```yaml rc_delayed_event_mgmt: - per_user: - per_second: 1.0 - burst_count: 5.0 + per_second: 1.0 + burst_count: 5.0 ``` Example configuration: diff --git a/schema/synapse-config.schema.yaml b/schema/synapse-config.schema.yaml index d9a5a98496..9a244856e6 100644 --- a/schema/synapse-config.schema.yaml +++ b/schema/synapse-config.schema.yaml @@ -2179,9 +2179,8 @@ properties: with a short timeout, or restarting several different delayed events all at once) without the risk of being ratelimited. default: - per_user: - per_second: 1.0 - burst_count: 5.0 + per_second: 1.0 + burst_count: 5.0 examples: - per_second: 2.0 burst_count: 20.0 From f031105eee6a1b0f9d7231712ebb68c32f900d59 Mon Sep 17 00:00:00 2001 From: Quentin Gliech Date: Thu, 17 Jul 2025 17:15:11 +0200 Subject: [PATCH 018/185] Start and stop the Tokio runtime with the Twisted reactor (#18691) Fixes https://github.com/element-hq/synapse/issues/18659 This changes the Tokio runtime to be attached to the Twisted reactor. This way, the Tokio runtime starts when the Twisted reactor starts, and *not* when the module gets loaded. This is important as starting the runtime on module load meant that it broke when Synapse was started with `daemonize`/`synctl`, as forks only retain the calling threads, breaking the Tokio runtime. This also changes so that the HttpClient gets the Twisted reactor explicitly as parameter instead of loading it from `twisted.internet.reactor` --- Cargo.lock | 1 + changelog.d/18691.bugfix | 1 + rust/Cargo.toml | 1 + rust/src/http_client.rs | 227 ++++++++++++++++++-------- synapse/api/auth/msc3861_delegated.py | 3 +- synapse/synapse_rust/http_client.pyi | 4 +- 6 files changed, 164 insertions(+), 73 deletions(-) create mode 100644 changelog.d/18691.bugfix diff --git a/Cargo.lock b/Cargo.lock index abb3c6bed3..026be1b6e8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1470,6 +1470,7 @@ dependencies = [ "lazy_static", "log", "mime", + "once_cell", "pyo3", "pyo3-log", "pythonize", diff --git a/changelog.d/18691.bugfix b/changelog.d/18691.bugfix new file mode 100644 index 0000000000..27bc09e4fd --- /dev/null +++ b/changelog.d/18691.bugfix @@ -0,0 +1 @@ +Fix the MAS integration not working when Synapse is started with `--daemonize` or using `synctl`. diff --git a/rust/Cargo.toml b/rust/Cargo.toml index dab32c8952..4f5ebb68b7 100644 --- a/rust/Cargo.toml +++ b/rust/Cargo.toml @@ -52,6 +52,7 @@ reqwest = { version = "0.12.15", default-features = false, features = [ http-body-util = "0.1.3" futures = "0.3.31" tokio = { version = "1.44.2", features = ["rt", "rt-multi-thread"] } +once_cell = "1.18.0" [features] extension-module = ["pyo3/extension-module"] diff --git a/rust/src/http_client.rs b/rust/src/http_client.rs index eda0197c74..b6cdf98f55 100644 --- a/rust/src/http_client.rs +++ b/rust/src/http_client.rs @@ -12,58 +12,149 @@ * . */ -use std::{collections::HashMap, future::Future, panic::AssertUnwindSafe, sync::LazyLock}; +use std::{collections::HashMap, future::Future}; use anyhow::Context; -use futures::{FutureExt, TryStreamExt}; -use pyo3::{exceptions::PyException, prelude::*, types::PyString}; +use futures::TryStreamExt; +use once_cell::sync::OnceCell; +use pyo3::{create_exception, exceptions::PyException, prelude::*}; use reqwest::RequestBuilder; use tokio::runtime::Runtime; use crate::errors::HttpResponseException; -/// The tokio runtime that we're using to run async Rust libs. -static RUNTIME: LazyLock = LazyLock::new(|| { - tokio::runtime::Builder::new_multi_thread() - .worker_threads(4) - .enable_all() - .build() - .unwrap() -}); +create_exception!( + synapse.synapse_rust.http_client, + RustPanicError, + PyException, + "A panic which happened in a Rust future" +); -/// A reference to the `Deferred` python class. -static DEFERRED_CLASS: LazyLock = LazyLock::new(|| { - Python::with_gil(|py| { - py.import("twisted.internet.defer") - .expect("module 'twisted.internet.defer' should be importable") - .getattr("Deferred") - .expect("module 'twisted.internet.defer' should have a 'Deferred' class") - .unbind() - }) -}); +impl RustPanicError { + fn from_panic(panic_err: &(dyn std::any::Any + Send + 'static)) -> PyErr { + // Apparently this is how you extract the panic message from a panic + let panic_message = if let Some(str_slice) = panic_err.downcast_ref::<&str>() { + str_slice + } else if let Some(string) = panic_err.downcast_ref::() { + string + } else { + "unknown error" + }; + Self::new_err(panic_message.to_owned()) + } +} -/// A reference to the twisted `reactor`. -static TWISTED_REACTOR: LazyLock> = LazyLock::new(|| { - Python::with_gil(|py| { - py.import("twisted.internet.reactor") - .expect("module 'twisted.internet.reactor' should be importable") - .unbind() - }) -}); +/// This is the name of the attribute where we store the runtime on the reactor +static TOKIO_RUNTIME_ATTR: &str = "__synapse_rust_tokio_runtime"; + +/// A Python wrapper around a Tokio runtime. +/// +/// This allows us to 'store' the runtime on the reactor instance, starting it +/// when the reactor starts, and stopping it when the reactor shuts down. +#[pyclass] +struct PyTokioRuntime { + runtime: Option, +} + +#[pymethods] +impl PyTokioRuntime { + fn start(&mut self) -> PyResult<()> { + // TODO: allow customization of the runtime like the number of threads + let runtime = tokio::runtime::Builder::new_multi_thread() + .worker_threads(4) + .enable_all() + .build()?; + + self.runtime = Some(runtime); + + Ok(()) + } + + fn shutdown(&mut self) -> PyResult<()> { + let runtime = self + .runtime + .take() + .context("Runtime was already shutdown")?; + + // Dropping the runtime will shut it down + drop(runtime); + + Ok(()) + } +} + +impl PyTokioRuntime { + /// Get the handle to the Tokio runtime, if it is running. + fn handle(&self) -> PyResult<&tokio::runtime::Handle> { + let handle = self + .runtime + .as_ref() + .context("Tokio runtime is not running")? + .handle(); + + Ok(handle) + } +} + +/// Get a handle to the Tokio runtime stored on the reactor instance, or create +/// a new one. +fn runtime<'a>(reactor: &Bound<'a, PyAny>) -> PyResult> { + if !reactor.hasattr(TOKIO_RUNTIME_ATTR)? { + install_runtime(reactor)?; + } + + get_runtime(reactor) +} + +/// Install a new Tokio runtime on the reactor instance. +fn install_runtime(reactor: &Bound) -> PyResult<()> { + let py = reactor.py(); + let runtime = PyTokioRuntime { runtime: None }; + let runtime = runtime.into_pyobject(py)?; + + // Attach the runtime to the reactor, starting it when the reactor is + // running, stopping it when the reactor is shutting down + reactor.call_method1("callWhenRunning", (runtime.getattr("start")?,))?; + reactor.call_method1( + "addSystemEventTrigger", + ("after", "shutdown", runtime.getattr("shutdown")?), + )?; + reactor.setattr(TOKIO_RUNTIME_ATTR, runtime)?; + + Ok(()) +} + +/// Get a reference to a Tokio runtime handle stored on the reactor instance. +fn get_runtime<'a>(reactor: &Bound<'a, PyAny>) -> PyResult> { + // This will raise if `TOKIO_RUNTIME_ATTR` is not set or if it is + // not a `Runtime`. Careful that this could happen if the user sets it + // manually, or if multiple versions of `pyo3-twisted` are used! + let runtime: Bound = reactor.getattr(TOKIO_RUNTIME_ATTR)?.extract()?; + Ok(runtime.borrow()) +} + +/// A reference to the `twisted.internet.defer` module. +static DEFER: OnceCell = OnceCell::new(); + +/// Access to the `twisted.internet.defer` module. +fn defer(py: Python<'_>) -> PyResult<&Bound> { + Ok(DEFER + .get_or_try_init(|| py.import("twisted.internet.defer").map(Into::into))? + .bind(py)) +} /// Called when registering modules with python. pub fn register_module(py: Python<'_>, m: &Bound<'_, PyModule>) -> PyResult<()> { let child_module: Bound<'_, PyModule> = PyModule::new(py, "http_client")?; child_module.add_class::()?; - // Make sure we fail early if we can't build the lazy statics. - LazyLock::force(&RUNTIME); - LazyLock::force(&DEFERRED_CLASS); + // Make sure we fail early if we can't load some modules + defer(py)?; m.add_submodule(&child_module)?; // We need to manually add the module to sys.modules to make `from - // synapse.synapse_rust import acl` work. + // synapse.synapse_rust import http_client` work. py.import("sys")? .getattr("modules")? .set_item("synapse.synapse_rust.http_client", child_module)?; @@ -72,26 +163,24 @@ pub fn register_module(py: Python<'_>, m: &Bound<'_, PyModule>) -> PyResult<()> } #[pyclass] -#[derive(Clone)] struct HttpClient { client: reqwest::Client, + reactor: PyObject, } #[pymethods] impl HttpClient { #[new] - pub fn py_new(user_agent: &str) -> PyResult { - // The twisted reactor can only be imported after Synapse has been - // imported, to allow Synapse to change the twisted reactor. If we try - // and import the reactor too early twisted installs a default reactor, - // which can't be replaced. - LazyLock::force(&TWISTED_REACTOR); + pub fn py_new(reactor: Bound, user_agent: &str) -> PyResult { + // Make sure the runtime gets installed + let _ = runtime(&reactor)?; Ok(HttpClient { client: reqwest::Client::builder() .user_agent(user_agent) .build() .context("building reqwest client")?, + reactor: reactor.unbind(), }) } @@ -129,7 +218,7 @@ impl HttpClient { builder: RequestBuilder, response_limit: usize, ) -> PyResult> { - create_deferred(py, async move { + create_deferred(py, self.reactor.bind(py), async move { let response = builder.send().await.context("sending request")?; let status = response.status(); @@ -159,43 +248,51 @@ impl HttpClient { /// tokio runtime. /// /// Does not handle deferred cancellation or contextvars. -fn create_deferred(py: Python, fut: F) -> PyResult> +fn create_deferred<'py, F, O>( + py: Python<'py>, + reactor: &Bound<'py, PyAny>, + fut: F, +) -> PyResult> where F: Future> + Send + 'static, - for<'a> O: IntoPyObject<'a>, + for<'a> O: IntoPyObject<'a> + Send + 'static, { - let deferred = DEFERRED_CLASS.bind(py).call0()?; + let deferred = defer(py)?.call_method0("Deferred")?; let deferred_callback = deferred.getattr("callback")?.unbind(); let deferred_errback = deferred.getattr("errback")?.unbind(); - RUNTIME.spawn(async move { - // TODO: Is it safe to assert unwind safety here? I think so, as we - // don't use anything that could be tainted by the panic afterwards. - // Note that `.spawn(..)` asserts unwind safety on the future too. - let res = AssertUnwindSafe(fut).catch_unwind().await; + let rt = runtime(reactor)?; + let handle = rt.handle()?; + let task = handle.spawn(fut); + + // Unbind the reactor so that we can pass it to the task + let reactor = reactor.clone().unbind(); + handle.spawn(async move { + let res = task.await; Python::with_gil(move |py| { // Flatten the panic into standard python error let res = match res { Ok(r) => r, - Err(panic_err) => { - let panic_message = get_panic_message(&panic_err); - Err(PyException::new_err( - PyString::new(py, panic_message).unbind(), - )) - } + Err(join_err) => match join_err.try_into_panic() { + Ok(panic_err) => Err(RustPanicError::from_panic(&panic_err)), + Err(err) => Err(PyException::new_err(format!("Task cancelled: {err}"))), + }, }; + // Re-bind the reactor + let reactor = reactor.bind(py); + // Send the result to the deferred, via `.callback(..)` or `.errback(..)` match res { Ok(obj) => { - TWISTED_REACTOR - .call_method(py, "callFromThread", (deferred_callback, obj), None) + reactor + .call_method("callFromThread", (deferred_callback, obj), None) .expect("callFromThread should not fail"); // There's nothing we can really do with errors here } Err(err) => { - TWISTED_REACTOR - .call_method(py, "callFromThread", (deferred_errback, err), None) + reactor + .call_method("callFromThread", (deferred_errback, err), None) .expect("callFromThread should not fail"); // There's nothing we can really do with errors here } } @@ -204,15 +301,3 @@ where Ok(deferred) } - -/// Try and get the panic message out of the panic -fn get_panic_message<'a>(panic_err: &'a (dyn std::any::Any + Send + 'static)) -> &'a str { - // Apparently this is how you extract the panic message from a panic - if let Some(str_slice) = panic_err.downcast_ref::<&str>() { - str_slice - } else if let Some(string) = panic_err.downcast_ref::() { - string - } else { - "unknown error" - } -} diff --git a/synapse/api/auth/msc3861_delegated.py b/synapse/api/auth/msc3861_delegated.py index ad8f4e04f6..581c9c1e74 100644 --- a/synapse/api/auth/msc3861_delegated.py +++ b/synapse/api/auth/msc3861_delegated.py @@ -184,7 +184,8 @@ class MSC3861DelegatedAuth(BaseAuth): self._force_tracing_for_users = hs.config.tracing.force_tracing_for_users self._rust_http_client = HttpClient( - user_agent=self._http_client.user_agent.decode("utf8") + reactor=hs.get_reactor(), + user_agent=self._http_client.user_agent.decode("utf8"), ) # # Token Introspection Cache diff --git a/synapse/synapse_rust/http_client.pyi b/synapse/synapse_rust/http_client.pyi index 5fa6226fd5..cdc501e606 100644 --- a/synapse/synapse_rust/http_client.pyi +++ b/synapse/synapse_rust/http_client.pyi @@ -12,8 +12,10 @@ from typing import Awaitable, Mapping +from synapse.types import ISynapseReactor + class HttpClient: - def __init__(self, user_agent: str) -> None: ... + def __init__(self, reactor: ISynapseReactor, user_agent: str) -> None: ... def get(self, url: str, response_limit: int) -> Awaitable[bytes]: ... def post( self, From f0f9a82ca4f3d6a9f4ae4514a9b085ccfd7f01d5 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 17 Jul 2025 17:24:57 +0200 Subject: [PATCH 019/185] Bump lxml from 5.4.0 to 6.0.0 (#18631) Bumps [lxml](https://github.com/lxml/lxml) from 5.4.0 to 6.0.0.
Changelog

Sourced from lxml's changelog.

6.0.0 (2025-06-26)

Features added

  • GH#463: lxml.html.diff is faster and provides structurally better diffs. Original patch by Steven Fernandez.

  • GH#405: The factories Element and ElementTree can now be used in type hints.

  • GH#448: Parsing from memoryview and other buffers is supported to allow zero-copy parsing.

  • GH#437: lxml.html.builder was missing several HTML5 tag names. Patch by Nick Tarleton.

  • GH#458: CDATA can now be written into the incremental xmlfile() writer. Original patch by Lane Shaw.

  • A new parser option decompress=False was added that controls the automatic input decompression when using libxml2 2.15.0 or later. Disabling this option by default will effectively prevent decompression bombs when handling untrusted input. Code that depends on automatic decompression must enable this option. Note that libxml2 2.15.0 was not released yet, so this option currently has no effect but can already be used.

  • The set of compile time / runtime supported libxml2 feature names is available as etree.LIBXML_COMPILED_FEATURES and etree.LIBXML_FEATURES. This currently includes catalog, ftp, html, http, iconv, icu, lzma, regexp, schematron, xmlschema, xpath, zlib.

Bugs fixed

  • GH#353: Predicates in .find*() could mishandle tag indices if a default namespace is provided. Original patch by Luise K.

  • GH#272: The head and body properties of lxml.html elements failed if no such element was found. They now return None instead. Original patch by FVolral.

  • Tag names provided by code (API, not data) that are longer than INT_MAX could be truncated or mishandled in other ways.

  • .text_content() on lxml.html elements accidentally returned a "smart string" without additional information. It now returns a plain string.

  • LP#2109931: When building lxml with coverage reporting, it now disables the sys.monitoring support due to the lack of support in nedbat/coveragepy#1790

... (truncated)

Commits
  • 2a67034 Prepare release of 6.0.0.
  • e0b4e02 Update changelog.
  • d3f4dcf Build: Upgrade libxml2 to latest 2.14.4.
  • 014e51c Build: Add Windows arm64 wheel builds (GH-465)
  • d3914dc Only use "xmlCtxtIsStopped()" from libxml2 2.15.0 on since it fails to cover ...
  • 6e41390 Avoid reading the deprecated "disableSAX" attribute of "xmlParserCtxt".
  • f85da81 Use newer "language_level=3" in ElementPath module.
  • 787315e Build: bump pypa/cibuildwheel in the github-actions group (#464)
  • fb3adb1 Readme: Add project income report for 2024.
  • 8e61a75 Fit cached tuple more nicely into cachelines.
  • Additional commits viewable in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=lxml&package-manager=pip&previous-version=5.4.0&new-version=6.0.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- poetry.lock | 231 ++++++++++++++++++++++------------------------------ 1 file changed, 96 insertions(+), 135 deletions(-) diff --git a/poetry.lock b/poetry.lock index 02d3c8edb2..ebdb8e3896 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1014,145 +1014,107 @@ pyasn1 = ">=0.4.6" [[package]] name = "lxml" -version = "5.4.0" +version = "6.0.0" description = "Powerful and Pythonic XML processing library combining libxml2/libxslt with the ElementTree API." optional = true -python-versions = ">=3.6" +python-versions = ">=3.8" groups = ["main"] markers = "extra == \"all\" or extra == \"url-preview\"" files = [ - {file = "lxml-5.4.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:e7bc6df34d42322c5289e37e9971d6ed114e3776b45fa879f734bded9d1fea9c"}, - {file = "lxml-5.4.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:6854f8bd8a1536f8a1d9a3655e6354faa6406621cf857dc27b681b69860645c7"}, - {file = "lxml-5.4.0-cp310-cp310-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:696ea9e87442467819ac22394ca36cb3d01848dad1be6fac3fb612d3bd5a12cf"}, - {file = "lxml-5.4.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6ef80aeac414f33c24b3815ecd560cee272786c3adfa5f31316d8b349bfade28"}, - {file = "lxml-5.4.0-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3b9c2754cef6963f3408ab381ea55f47dabc6f78f4b8ebb0f0b25cf1ac1f7609"}, - {file = "lxml-5.4.0-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:7a62cc23d754bb449d63ff35334acc9f5c02e6dae830d78dab4dd12b78a524f4"}, - {file = "lxml-5.4.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8f82125bc7203c5ae8633a7d5d20bcfdff0ba33e436e4ab0abc026a53a8960b7"}, - {file = "lxml-5.4.0-cp310-cp310-manylinux_2_28_aarch64.whl", hash = "sha256:b67319b4aef1a6c56576ff544b67a2a6fbd7eaee485b241cabf53115e8908b8f"}, - {file = "lxml-5.4.0-cp310-cp310-manylinux_2_28_ppc64le.whl", hash = "sha256:a8ef956fce64c8551221f395ba21d0724fed6b9b6242ca4f2f7beb4ce2f41997"}, - {file = "lxml-5.4.0-cp310-cp310-manylinux_2_28_s390x.whl", hash = "sha256:0a01ce7d8479dce84fc03324e3b0c9c90b1ece9a9bb6a1b6c9025e7e4520e78c"}, - {file = "lxml-5.4.0-cp310-cp310-manylinux_2_28_x86_64.whl", hash = "sha256:91505d3ddebf268bb1588eb0f63821f738d20e1e7f05d3c647a5ca900288760b"}, - {file = "lxml-5.4.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:a3bcdde35d82ff385f4ede021df801b5c4a5bcdfb61ea87caabcebfc4945dc1b"}, - {file = "lxml-5.4.0-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:aea7c06667b987787c7d1f5e1dfcd70419b711cdb47d6b4bb4ad4b76777a0563"}, - {file = "lxml-5.4.0-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:a7fb111eef4d05909b82152721a59c1b14d0f365e2be4c742a473c5d7372f4f5"}, - {file = "lxml-5.4.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:43d549b876ce64aa18b2328faff70f5877f8c6dede415f80a2f799d31644d776"}, - {file = "lxml-5.4.0-cp310-cp310-win32.whl", hash = "sha256:75133890e40d229d6c5837b0312abbe5bac1c342452cf0e12523477cd3aa21e7"}, - {file = "lxml-5.4.0-cp310-cp310-win_amd64.whl", hash = "sha256:de5b4e1088523e2b6f730d0509a9a813355b7f5659d70eb4f319c76beea2e250"}, - {file = "lxml-5.4.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:98a3912194c079ef37e716ed228ae0dcb960992100461b704aea4e93af6b0bb9"}, - {file = "lxml-5.4.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:0ea0252b51d296a75f6118ed0d8696888e7403408ad42345d7dfd0d1e93309a7"}, - {file = "lxml-5.4.0-cp311-cp311-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b92b69441d1bd39f4940f9eadfa417a25862242ca2c396b406f9272ef09cdcaa"}, - {file = "lxml-5.4.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:20e16c08254b9b6466526bc1828d9370ee6c0d60a4b64836bc3ac2917d1e16df"}, - {file = "lxml-5.4.0-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:7605c1c32c3d6e8c990dd28a0970a3cbbf1429d5b92279e37fda05fb0c92190e"}, - {file = "lxml-5.4.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:ecf4c4b83f1ab3d5a7ace10bafcb6f11df6156857a3c418244cef41ca9fa3e44"}, - {file = "lxml-5.4.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0cef4feae82709eed352cd7e97ae062ef6ae9c7b5dbe3663f104cd2c0e8d94ba"}, - {file = "lxml-5.4.0-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:df53330a3bff250f10472ce96a9af28628ff1f4efc51ccba351a8820bca2a8ba"}, - {file = "lxml-5.4.0-cp311-cp311-manylinux_2_28_ppc64le.whl", hash = "sha256:aefe1a7cb852fa61150fcb21a8c8fcea7b58c4cb11fbe59c97a0a4b31cae3c8c"}, - {file = "lxml-5.4.0-cp311-cp311-manylinux_2_28_s390x.whl", hash = "sha256:ef5a7178fcc73b7d8c07229e89f8eb45b2908a9238eb90dcfc46571ccf0383b8"}, - {file = "lxml-5.4.0-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:d2ed1b3cb9ff1c10e6e8b00941bb2e5bb568b307bfc6b17dffbbe8be5eecba86"}, - {file = "lxml-5.4.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:72ac9762a9f8ce74c9eed4a4e74306f2f18613a6b71fa065495a67ac227b3056"}, - {file = "lxml-5.4.0-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:f5cb182f6396706dc6cc1896dd02b1c889d644c081b0cdec38747573db88a7d7"}, - {file = "lxml-5.4.0-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:3a3178b4873df8ef9457a4875703488eb1622632a9cee6d76464b60e90adbfcd"}, - {file = "lxml-5.4.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:e094ec83694b59d263802ed03a8384594fcce477ce484b0cbcd0008a211ca751"}, - {file = "lxml-5.4.0-cp311-cp311-win32.whl", hash = "sha256:4329422de653cdb2b72afa39b0aa04252fca9071550044904b2e7036d9d97fe4"}, - {file = "lxml-5.4.0-cp311-cp311-win_amd64.whl", hash = "sha256:fd3be6481ef54b8cfd0e1e953323b7aa9d9789b94842d0e5b142ef4bb7999539"}, - {file = "lxml-5.4.0-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:b5aff6f3e818e6bdbbb38e5967520f174b18f539c2b9de867b1e7fde6f8d95a4"}, - {file = "lxml-5.4.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:942a5d73f739ad7c452bf739a62a0f83e2578afd6b8e5406308731f4ce78b16d"}, - {file = "lxml-5.4.0-cp312-cp312-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:460508a4b07364d6abf53acaa0a90b6d370fafde5693ef37602566613a9b0779"}, - {file = "lxml-5.4.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:529024ab3a505fed78fe3cc5ddc079464e709f6c892733e3f5842007cec8ac6e"}, - {file = "lxml-5.4.0-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:7ca56ebc2c474e8f3d5761debfd9283b8b18c76c4fc0967b74aeafba1f5647f9"}, - {file = "lxml-5.4.0-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a81e1196f0a5b4167a8dafe3a66aa67c4addac1b22dc47947abd5d5c7a3f24b5"}, - {file = "lxml-5.4.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:00b8686694423ddae324cf614e1b9659c2edb754de617703c3d29ff568448df5"}, - {file = "lxml-5.4.0-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:c5681160758d3f6ac5b4fea370495c48aac0989d6a0f01bb9a72ad8ef5ab75c4"}, - {file = "lxml-5.4.0-cp312-cp312-manylinux_2_28_ppc64le.whl", hash = "sha256:2dc191e60425ad70e75a68c9fd90ab284df64d9cd410ba8d2b641c0c45bc006e"}, - {file = "lxml-5.4.0-cp312-cp312-manylinux_2_28_s390x.whl", hash = "sha256:67f779374c6b9753ae0a0195a892a1c234ce8416e4448fe1e9f34746482070a7"}, - {file = "lxml-5.4.0-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:79d5bfa9c1b455336f52343130b2067164040604e41f6dc4d8313867ed540079"}, - {file = "lxml-5.4.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:3d3c30ba1c9b48c68489dc1829a6eede9873f52edca1dda900066542528d6b20"}, - {file = "lxml-5.4.0-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:1af80c6316ae68aded77e91cd9d80648f7dd40406cef73df841aa3c36f6907c8"}, - {file = "lxml-5.4.0-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:4d885698f5019abe0de3d352caf9466d5de2baded00a06ef3f1216c1a58ae78f"}, - {file = "lxml-5.4.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:aea53d51859b6c64e7c51d522c03cc2c48b9b5d6172126854cc7f01aa11f52bc"}, - {file = "lxml-5.4.0-cp312-cp312-win32.whl", hash = "sha256:d90b729fd2732df28130c064aac9bb8aff14ba20baa4aee7bd0795ff1187545f"}, - {file = "lxml-5.4.0-cp312-cp312-win_amd64.whl", hash = "sha256:1dc4ca99e89c335a7ed47d38964abcb36c5910790f9bd106f2a8fa2ee0b909d2"}, - {file = "lxml-5.4.0-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:773e27b62920199c6197130632c18fb7ead3257fce1ffb7d286912e56ddb79e0"}, - {file = "lxml-5.4.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:ce9c671845de9699904b1e9df95acfe8dfc183f2310f163cdaa91a3535af95de"}, - {file = "lxml-5.4.0-cp313-cp313-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:9454b8d8200ec99a224df8854786262b1bd6461f4280064c807303c642c05e76"}, - {file = "lxml-5.4.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cccd007d5c95279e529c146d095f1d39ac05139de26c098166c4beb9374b0f4d"}, - {file = "lxml-5.4.0-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:0fce1294a0497edb034cb416ad3e77ecc89b313cff7adbee5334e4dc0d11f422"}, - {file = "lxml-5.4.0-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:24974f774f3a78ac12b95e3a20ef0931795ff04dbb16db81a90c37f589819551"}, - {file = "lxml-5.4.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:497cab4d8254c2a90bf988f162ace2ddbfdd806fce3bda3f581b9d24c852e03c"}, - {file = "lxml-5.4.0-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:e794f698ae4c5084414efea0f5cc9f4ac562ec02d66e1484ff822ef97c2cadff"}, - {file = "lxml-5.4.0-cp313-cp313-manylinux_2_28_ppc64le.whl", hash = "sha256:2c62891b1ea3094bb12097822b3d44b93fc6c325f2043c4d2736a8ff09e65f60"}, - {file = "lxml-5.4.0-cp313-cp313-manylinux_2_28_s390x.whl", hash = "sha256:142accb3e4d1edae4b392bd165a9abdee8a3c432a2cca193df995bc3886249c8"}, - {file = "lxml-5.4.0-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:1a42b3a19346e5601d1b8296ff6ef3d76038058f311902edd574461e9c036982"}, - {file = "lxml-5.4.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:4291d3c409a17febf817259cb37bc62cb7eb398bcc95c1356947e2871911ae61"}, - {file = "lxml-5.4.0-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:4f5322cf38fe0e21c2d73901abf68e6329dc02a4994e483adbcf92b568a09a54"}, - {file = "lxml-5.4.0-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:0be91891bdb06ebe65122aa6bf3fc94489960cf7e03033c6f83a90863b23c58b"}, - {file = "lxml-5.4.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:15a665ad90054a3d4f397bc40f73948d48e36e4c09f9bcffc7d90c87410e478a"}, - {file = "lxml-5.4.0-cp313-cp313-win32.whl", hash = "sha256:d5663bc1b471c79f5c833cffbc9b87d7bf13f87e055a5c86c363ccd2348d7e82"}, - {file = "lxml-5.4.0-cp313-cp313-win_amd64.whl", hash = "sha256:bcb7a1096b4b6b24ce1ac24d4942ad98f983cd3810f9711bcd0293f43a9d8b9f"}, - {file = "lxml-5.4.0-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:7be701c24e7f843e6788353c055d806e8bd8466b52907bafe5d13ec6a6dbaecd"}, - {file = "lxml-5.4.0-cp36-cp36m-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:fb54f7c6bafaa808f27166569b1511fc42701a7713858dddc08afdde9746849e"}, - {file = "lxml-5.4.0-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:97dac543661e84a284502e0cf8a67b5c711b0ad5fb661d1bd505c02f8cf716d7"}, - {file = "lxml-5.4.0-cp36-cp36m-manylinux_2_28_x86_64.whl", hash = "sha256:c70e93fba207106cb16bf852e421c37bbded92acd5964390aad07cb50d60f5cf"}, - {file = "lxml-5.4.0-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:9c886b481aefdf818ad44846145f6eaf373a20d200b5ce1a5c8e1bc2d8745410"}, - {file = "lxml-5.4.0-cp36-cp36m-musllinux_1_2_x86_64.whl", hash = "sha256:fa0e294046de09acd6146be0ed6727d1f42ded4ce3ea1e9a19c11b6774eea27c"}, - {file = "lxml-5.4.0-cp36-cp36m-win32.whl", hash = "sha256:61c7bbf432f09ee44b1ccaa24896d21075e533cd01477966a5ff5a71d88b2f56"}, - {file = "lxml-5.4.0-cp36-cp36m-win_amd64.whl", hash = "sha256:7ce1a171ec325192c6a636b64c94418e71a1964f56d002cc28122fceff0b6121"}, - {file = "lxml-5.4.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:795f61bcaf8770e1b37eec24edf9771b307df3af74d1d6f27d812e15a9ff3872"}, - {file = "lxml-5.4.0-cp37-cp37m-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:29f451a4b614a7b5b6c2e043d7b64a15bd8304d7e767055e8ab68387a8cacf4e"}, - {file = "lxml-5.4.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:891f7f991a68d20c75cb13c5c9142b2a3f9eb161f1f12a9489c82172d1f133c0"}, - {file = "lxml-5.4.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4aa412a82e460571fad592d0f93ce9935a20090029ba08eca05c614f99b0cc92"}, - {file = "lxml-5.4.0-cp37-cp37m-manylinux_2_28_aarch64.whl", hash = "sha256:ac7ba71f9561cd7d7b55e1ea5511543c0282e2b6450f122672a2694621d63b7e"}, - {file = "lxml-5.4.0-cp37-cp37m-manylinux_2_28_x86_64.whl", hash = "sha256:c5d32f5284012deaccd37da1e2cd42f081feaa76981f0eaa474351b68df813c5"}, - {file = "lxml-5.4.0-cp37-cp37m-musllinux_1_2_aarch64.whl", hash = "sha256:ce31158630a6ac85bddd6b830cffd46085ff90498b397bd0a259f59d27a12188"}, - {file = "lxml-5.4.0-cp37-cp37m-musllinux_1_2_x86_64.whl", hash = "sha256:31e63621e073e04697c1b2d23fcb89991790eef370ec37ce4d5d469f40924ed6"}, - {file = "lxml-5.4.0-cp37-cp37m-win32.whl", hash = "sha256:be2ba4c3c5b7900246a8f866580700ef0d538f2ca32535e991027bdaba944063"}, - {file = "lxml-5.4.0-cp37-cp37m-win_amd64.whl", hash = "sha256:09846782b1ef650b321484ad429217f5154da4d6e786636c38e434fa32e94e49"}, - {file = "lxml-5.4.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:eaf24066ad0b30917186420d51e2e3edf4b0e2ea68d8cd885b14dc8afdcf6556"}, - {file = "lxml-5.4.0-cp38-cp38-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2b31a3a77501d86d8ade128abb01082724c0dfd9524f542f2f07d693c9f1175f"}, - {file = "lxml-5.4.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0e108352e203c7afd0eb91d782582f00a0b16a948d204d4dec8565024fafeea5"}, - {file = "lxml-5.4.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a11a96c3b3f7551c8a8109aa65e8594e551d5a84c76bf950da33d0fb6dfafab7"}, - {file = "lxml-5.4.0-cp38-cp38-manylinux_2_28_aarch64.whl", hash = "sha256:ca755eebf0d9e62d6cb013f1261e510317a41bf4650f22963474a663fdfe02aa"}, - {file = "lxml-5.4.0-cp38-cp38-manylinux_2_28_x86_64.whl", hash = "sha256:4cd915c0fb1bed47b5e6d6edd424ac25856252f09120e3e8ba5154b6b921860e"}, - {file = "lxml-5.4.0-cp38-cp38-musllinux_1_2_aarch64.whl", hash = "sha256:226046e386556a45ebc787871d6d2467b32c37ce76c2680f5c608e25823ffc84"}, - {file = "lxml-5.4.0-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:b108134b9667bcd71236c5a02aad5ddd073e372fb5d48ea74853e009fe38acb6"}, - {file = "lxml-5.4.0-cp38-cp38-win32.whl", hash = "sha256:1320091caa89805df7dcb9e908add28166113dcd062590668514dbd510798c88"}, - {file = "lxml-5.4.0-cp38-cp38-win_amd64.whl", hash = "sha256:073eb6dcdf1f587d9b88c8c93528b57eccda40209cf9be549d469b942b41d70b"}, - {file = "lxml-5.4.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:bda3ea44c39eb74e2488297bb39d47186ed01342f0022c8ff407c250ac3f498e"}, - {file = "lxml-5.4.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9ceaf423b50ecfc23ca00b7f50b64baba85fb3fb91c53e2c9d00bc86150c7e40"}, - {file = "lxml-5.4.0-cp39-cp39-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:664cdc733bc87449fe781dbb1f309090966c11cc0c0cd7b84af956a02a8a4729"}, - {file = "lxml-5.4.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:67ed8a40665b84d161bae3181aa2763beea3747f748bca5874b4af4d75998f87"}, - {file = "lxml-5.4.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9b4a3bd174cc9cdaa1afbc4620c049038b441d6ba07629d89a83b408e54c35cd"}, - {file = "lxml-5.4.0-cp39-cp39-manylinux_2_28_aarch64.whl", hash = "sha256:b0989737a3ba6cf2a16efb857fb0dfa20bc5c542737fddb6d893fde48be45433"}, - {file = "lxml-5.4.0-cp39-cp39-manylinux_2_28_x86_64.whl", hash = "sha256:dc0af80267edc68adf85f2a5d9be1cdf062f973db6790c1d065e45025fa26140"}, - {file = "lxml-5.4.0-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:639978bccb04c42677db43c79bdaa23785dc7f9b83bfd87570da8207872f1ce5"}, - {file = "lxml-5.4.0-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:5a99d86351f9c15e4a901fc56404b485b1462039db59288b203f8c629260a142"}, - {file = "lxml-5.4.0-cp39-cp39-win32.whl", hash = "sha256:3e6d5557989cdc3ebb5302bbdc42b439733a841891762ded9514e74f60319ad6"}, - {file = "lxml-5.4.0-cp39-cp39-win_amd64.whl", hash = "sha256:a8c9b7f16b63e65bbba889acb436a1034a82d34fa09752d754f88d708eca80e1"}, - {file = "lxml-5.4.0-pp310-pypy310_pp73-macosx_10_15_x86_64.whl", hash = "sha256:1b717b00a71b901b4667226bba282dd462c42ccf618ade12f9ba3674e1fabc55"}, - {file = "lxml-5.4.0-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:27a9ded0f0b52098ff89dd4c418325b987feed2ea5cc86e8860b0f844285d740"}, - {file = "lxml-5.4.0-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4b7ce10634113651d6f383aa712a194179dcd496bd8c41e191cec2099fa09de5"}, - {file = "lxml-5.4.0-pp310-pypy310_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:53370c26500d22b45182f98847243efb518d268374a9570409d2e2276232fd37"}, - {file = "lxml-5.4.0-pp310-pypy310_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:c6364038c519dffdbe07e3cf42e6a7f8b90c275d4d1617a69bb59734c1a2d571"}, - {file = "lxml-5.4.0-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:b12cb6527599808ada9eb2cd6e0e7d3d8f13fe7bbb01c6311255a15ded4c7ab4"}, - {file = "lxml-5.4.0-pp37-pypy37_pp73-macosx_10_9_x86_64.whl", hash = "sha256:5f11a1526ebd0dee85e7b1e39e39a0cc0d9d03fb527f56d8457f6df48a10dc0c"}, - {file = "lxml-5.4.0-pp37-pypy37_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:48b4afaf38bf79109bb060d9016fad014a9a48fb244e11b94f74ae366a64d252"}, - {file = "lxml-5.4.0-pp37-pypy37_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:de6f6bb8a7840c7bf216fb83eec4e2f79f7325eca8858167b68708b929ab2172"}, - {file = "lxml-5.4.0-pp37-pypy37_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:5cca36a194a4eb4e2ed6be36923d3cffd03dcdf477515dea687185506583d4c9"}, - {file = "lxml-5.4.0-pp37-pypy37_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:b7c86884ad23d61b025989d99bfdd92a7351de956e01c61307cb87035960bcb1"}, - {file = "lxml-5.4.0-pp37-pypy37_pp73-win_amd64.whl", hash = "sha256:53d9469ab5460402c19553b56c3648746774ecd0681b1b27ea74d5d8a3ef5590"}, - {file = "lxml-5.4.0-pp38-pypy38_pp73-macosx_10_9_x86_64.whl", hash = "sha256:56dbdbab0551532bb26c19c914848d7251d73edb507c3079d6805fa8bba5b706"}, - {file = "lxml-5.4.0-pp38-pypy38_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:14479c2ad1cb08b62bb941ba8e0e05938524ee3c3114644df905d2331c76cd57"}, - {file = "lxml-5.4.0-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:32697d2ea994e0db19c1df9e40275ffe84973e4232b5c274f47e7c1ec9763cdd"}, - {file = "lxml-5.4.0-pp38-pypy38_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:24f6df5f24fc3385f622c0c9d63fe34604893bc1a5bdbb2dbf5870f85f9a404a"}, - {file = "lxml-5.4.0-pp38-pypy38_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:151d6c40bc9db11e960619d2bf2ec5829f0aaffb10b41dcf6ad2ce0f3c0b2325"}, - {file = "lxml-5.4.0-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:4025bf2884ac4370a3243c5aa8d66d3cb9e15d3ddd0af2d796eccc5f0244390e"}, - {file = "lxml-5.4.0-pp39-pypy39_pp73-macosx_10_15_x86_64.whl", hash = "sha256:9459e6892f59ecea2e2584ee1058f5d8f629446eab52ba2305ae13a32a059530"}, - {file = "lxml-5.4.0-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:47fb24cc0f052f0576ea382872b3fc7e1f7e3028e53299ea751839418ade92a6"}, - {file = "lxml-5.4.0-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:50441c9de951a153c698b9b99992e806b71c1f36d14b154592580ff4a9d0d877"}, - {file = "lxml-5.4.0-pp39-pypy39_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:ab339536aa798b1e17750733663d272038bf28069761d5be57cb4a9b0137b4f8"}, - {file = "lxml-5.4.0-pp39-pypy39_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:9776af1aad5a4b4a1317242ee2bea51da54b2a7b7b48674be736d463c999f37d"}, - {file = "lxml-5.4.0-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:63e7968ff83da2eb6fdda967483a7a023aa497d85ad8f05c3ad9b1f2e8c84987"}, - {file = "lxml-5.4.0.tar.gz", hash = "sha256:d12832e1dbea4be280b22fd0ea7c9b87f0d8fc51ba06e92dc62d52f804f78ebd"}, + {file = "lxml-6.0.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:35bc626eec405f745199200ccb5c6b36f202675d204aa29bb52e27ba2b71dea8"}, + {file = "lxml-6.0.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:246b40f8a4aec341cbbf52617cad8ab7c888d944bfe12a6abd2b1f6cfb6f6082"}, + {file = "lxml-6.0.0-cp310-cp310-manylinux2010_i686.manylinux2014_i686.manylinux_2_12_i686.manylinux_2_17_i686.whl", hash = "sha256:2793a627e95d119e9f1e19720730472f5543a6d84c50ea33313ce328d870f2dd"}, + {file = "lxml-6.0.0-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:46b9ed911f36bfeb6338e0b482e7fe7c27d362c52fde29f221fddbc9ee2227e7"}, + {file = "lxml-6.0.0-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:2b4790b558bee331a933e08883c423f65bbcd07e278f91b2272489e31ab1e2b4"}, + {file = "lxml-6.0.0-cp310-cp310-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e2030956cf4886b10be9a0285c6802e078ec2391e1dd7ff3eb509c2c95a69b76"}, + {file = "lxml-6.0.0-cp310-cp310-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:4d23854ecf381ab1facc8f353dcd9adeddef3652268ee75297c1164c987c11dc"}, + {file = "lxml-6.0.0-cp310-cp310-manylinux_2_31_armv7l.whl", hash = "sha256:43fe5af2d590bf4691531b1d9a2495d7aab2090547eaacd224a3afec95706d76"}, + {file = "lxml-6.0.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:74e748012f8c19b47f7d6321ac929a9a94ee92ef12bc4298c47e8b7219b26541"}, + {file = "lxml-6.0.0-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:43cfbb7db02b30ad3926e8fceaef260ba2fb7df787e38fa2df890c1ca7966c3b"}, + {file = "lxml-6.0.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:34190a1ec4f1e84af256495436b2d196529c3f2094f0af80202947567fdbf2e7"}, + {file = "lxml-6.0.0-cp310-cp310-win32.whl", hash = "sha256:5967fe415b1920a3877a4195e9a2b779249630ee49ece22021c690320ff07452"}, + {file = "lxml-6.0.0-cp310-cp310-win_amd64.whl", hash = "sha256:f3389924581d9a770c6caa4df4e74b606180869043b9073e2cec324bad6e306e"}, + {file = "lxml-6.0.0-cp310-cp310-win_arm64.whl", hash = "sha256:522fe7abb41309e9543b0d9b8b434f2b630c5fdaf6482bee642b34c8c70079c8"}, + {file = "lxml-6.0.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:4ee56288d0df919e4aac43b539dd0e34bb55d6a12a6562038e8d6f3ed07f9e36"}, + {file = "lxml-6.0.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:b8dd6dd0e9c1992613ccda2bcb74fc9d49159dbe0f0ca4753f37527749885c25"}, + {file = "lxml-6.0.0-cp311-cp311-manylinux2010_i686.manylinux2014_i686.manylinux_2_12_i686.manylinux_2_17_i686.whl", hash = "sha256:d7ae472f74afcc47320238b5dbfd363aba111a525943c8a34a1b657c6be934c3"}, + {file = "lxml-6.0.0-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:5592401cdf3dc682194727c1ddaa8aa0f3ddc57ca64fd03226a430b955eab6f6"}, + {file = "lxml-6.0.0-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:58ffd35bd5425c3c3b9692d078bf7ab851441434531a7e517c4984d5634cd65b"}, + {file = "lxml-6.0.0-cp311-cp311-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f720a14aa102a38907c6d5030e3d66b3b680c3e6f6bc95473931ea3c00c59967"}, + {file = "lxml-6.0.0-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c2a5e8d207311a0170aca0eb6b160af91adc29ec121832e4ac151a57743a1e1e"}, + {file = "lxml-6.0.0-cp311-cp311-manylinux_2_31_armv7l.whl", hash = "sha256:2dd1cc3ea7e60bfb31ff32cafe07e24839df573a5e7c2d33304082a5019bcd58"}, + {file = "lxml-6.0.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:2cfcf84f1defed7e5798ef4f88aa25fcc52d279be731ce904789aa7ccfb7e8d2"}, + {file = "lxml-6.0.0-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:a52a4704811e2623b0324a18d41ad4b9fabf43ce5ff99b14e40a520e2190c851"}, + {file = "lxml-6.0.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:c16304bba98f48a28ae10e32a8e75c349dd742c45156f297e16eeb1ba9287a1f"}, + {file = "lxml-6.0.0-cp311-cp311-win32.whl", hash = "sha256:f8d19565ae3eb956d84da3ef367aa7def14a2735d05bd275cd54c0301f0d0d6c"}, + {file = "lxml-6.0.0-cp311-cp311-win_amd64.whl", hash = "sha256:b2d71cdefda9424adff9a3607ba5bbfc60ee972d73c21c7e3c19e71037574816"}, + {file = "lxml-6.0.0-cp311-cp311-win_arm64.whl", hash = "sha256:8a2e76efbf8772add72d002d67a4c3d0958638696f541734304c7f28217a9cab"}, + {file = "lxml-6.0.0-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:78718d8454a6e928470d511bf8ac93f469283a45c354995f7d19e77292f26108"}, + {file = "lxml-6.0.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:84ef591495ffd3f9dcabffd6391db7bb70d7230b5c35ef5148354a134f56f2be"}, + {file = "lxml-6.0.0-cp312-cp312-manylinux2010_i686.manylinux2014_i686.manylinux_2_12_i686.manylinux_2_17_i686.whl", hash = "sha256:2930aa001a3776c3e2601cb8e0a15d21b8270528d89cc308be4843ade546b9ab"}, + {file = "lxml-6.0.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:219e0431ea8006e15005767f0351e3f7f9143e793e58519dc97fe9e07fae5563"}, + {file = "lxml-6.0.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:bd5913b4972681ffc9718bc2d4c53cde39ef81415e1671ff93e9aa30b46595e7"}, + {file = "lxml-6.0.0-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:390240baeb9f415a82eefc2e13285016f9c8b5ad71ec80574ae8fa9605093cd7"}, + {file = "lxml-6.0.0-cp312-cp312-manylinux_2_27_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:d6e200909a119626744dd81bae409fc44134389e03fbf1d68ed2a55a2fb10991"}, + {file = "lxml-6.0.0-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ca50bd612438258a91b5b3788c6621c1f05c8c478e7951899f492be42defc0da"}, + {file = "lxml-6.0.0-cp312-cp312-manylinux_2_31_armv7l.whl", hash = "sha256:c24b8efd9c0f62bad0439283c2c795ef916c5a6b75f03c17799775c7ae3c0c9e"}, + {file = "lxml-6.0.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:afd27d8629ae94c5d863e32ab0e1d5590371d296b87dae0a751fb22bf3685741"}, + {file = "lxml-6.0.0-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:54c4855eabd9fc29707d30141be99e5cd1102e7d2258d2892314cf4c110726c3"}, + {file = "lxml-6.0.0-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:c907516d49f77f6cd8ead1322198bdfd902003c3c330c77a1c5f3cc32a0e4d16"}, + {file = "lxml-6.0.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:36531f81c8214e293097cd2b7873f178997dae33d3667caaae8bdfb9666b76c0"}, + {file = "lxml-6.0.0-cp312-cp312-win32.whl", hash = "sha256:690b20e3388a7ec98e899fd54c924e50ba6693874aa65ef9cb53de7f7de9d64a"}, + {file = "lxml-6.0.0-cp312-cp312-win_amd64.whl", hash = "sha256:310b719b695b3dd442cdfbbe64936b2f2e231bb91d998e99e6f0daf991a3eba3"}, + {file = "lxml-6.0.0-cp312-cp312-win_arm64.whl", hash = "sha256:8cb26f51c82d77483cdcd2b4a53cda55bbee29b3c2f3ddeb47182a2a9064e4eb"}, + {file = "lxml-6.0.0-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:6da7cd4f405fd7db56e51e96bff0865b9853ae70df0e6720624049da76bde2da"}, + {file = "lxml-6.0.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:b34339898bb556a2351a1830f88f751679f343eabf9cf05841c95b165152c9e7"}, + {file = "lxml-6.0.0-cp313-cp313-manylinux2010_i686.manylinux2014_i686.manylinux_2_12_i686.manylinux_2_17_i686.whl", hash = "sha256:51a5e4c61a4541bd1cd3ba74766d0c9b6c12d6a1a4964ef60026832aac8e79b3"}, + {file = "lxml-6.0.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:d18a25b19ca7307045581b18b3ec9ead2b1db5ccd8719c291f0cd0a5cec6cb81"}, + {file = "lxml-6.0.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:d4f0c66df4386b75d2ab1e20a489f30dc7fd9a06a896d64980541506086be1f1"}, + {file = "lxml-6.0.0-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9f4b481b6cc3a897adb4279216695150bbe7a44c03daba3c894f49d2037e0a24"}, + {file = "lxml-6.0.0-cp313-cp313-manylinux_2_27_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:8a78d6c9168f5bcb20971bf3329c2b83078611fbe1f807baadc64afc70523b3a"}, + {file = "lxml-6.0.0-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:2ae06fbab4f1bb7db4f7c8ca9897dc8db4447d1a2b9bee78474ad403437bcc29"}, + {file = "lxml-6.0.0-cp313-cp313-manylinux_2_31_armv7l.whl", hash = "sha256:1fa377b827ca2023244a06554c6e7dc6828a10aaf74ca41965c5d8a4925aebb4"}, + {file = "lxml-6.0.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:1676b56d48048a62ef77a250428d1f31f610763636e0784ba67a9740823988ca"}, + {file = "lxml-6.0.0-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:0e32698462aacc5c1cf6bdfebc9c781821b7e74c79f13e5ffc8bfe27c42b1abf"}, + {file = "lxml-6.0.0-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:4d6036c3a296707357efb375cfc24bb64cd955b9ec731abf11ebb1e40063949f"}, + {file = "lxml-6.0.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:7488a43033c958637b1a08cddc9188eb06d3ad36582cebc7d4815980b47e27ef"}, + {file = "lxml-6.0.0-cp313-cp313-win32.whl", hash = "sha256:5fcd7d3b1d8ecb91445bd71b9c88bdbeae528fefee4f379895becfc72298d181"}, + {file = "lxml-6.0.0-cp313-cp313-win_amd64.whl", hash = "sha256:2f34687222b78fff795feeb799a7d44eca2477c3d9d3a46ce17d51a4f383e32e"}, + {file = "lxml-6.0.0-cp313-cp313-win_arm64.whl", hash = "sha256:21db1ec5525780fd07251636eb5f7acb84003e9382c72c18c542a87c416ade03"}, + {file = "lxml-6.0.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:4eb114a0754fd00075c12648d991ec7a4357f9cb873042cc9a77bf3a7e30c9db"}, + {file = "lxml-6.0.0-cp38-cp38-manylinux2010_i686.manylinux2014_i686.manylinux_2_12_i686.manylinux_2_17_i686.whl", hash = "sha256:7da298e1659e45d151b4028ad5c7974917e108afb48731f4ed785d02b6818994"}, + {file = "lxml-6.0.0-cp38-cp38-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:7bf61bc4345c1895221357af8f3e89f8c103d93156ef326532d35c707e2fb19d"}, + {file = "lxml-6.0.0-cp38-cp38-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:63b634facdfbad421d4b61c90735688465d4ab3a8853ac22c76ccac2baf98d97"}, + {file = "lxml-6.0.0-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:e380e85b93f148ad28ac15f8117e2fd8e5437aa7732d65e260134f83ce67911b"}, + {file = "lxml-6.0.0-cp38-cp38-win32.whl", hash = "sha256:185efc2fed89cdd97552585c624d3c908f0464090f4b91f7d92f8ed2f3b18f54"}, + {file = "lxml-6.0.0-cp38-cp38-win_amd64.whl", hash = "sha256:f97487996a39cb18278ca33f7be98198f278d0bc3c5d0fd4d7b3d63646ca3c8a"}, + {file = "lxml-6.0.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:85b14a4689d5cff426c12eefe750738648706ea2753b20c2f973b2a000d3d261"}, + {file = "lxml-6.0.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:f64ccf593916e93b8d36ed55401bb7fe9c7d5de3180ce2e10b08f82a8f397316"}, + {file = "lxml-6.0.0-cp39-cp39-manylinux2010_i686.manylinux2014_i686.manylinux_2_12_i686.manylinux_2_17_i686.whl", hash = "sha256:b372d10d17a701b0945f67be58fae4664fd056b85e0ff0fbc1e6c951cdbc0512"}, + {file = "lxml-6.0.0-cp39-cp39-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:a674c0948789e9136d69065cc28009c1b1874c6ea340253db58be7622ce6398f"}, + {file = "lxml-6.0.0-cp39-cp39-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:edf6e4c8fe14dfe316939711e3ece3f9a20760aabf686051b537a7562f4da91a"}, + {file = "lxml-6.0.0-cp39-cp39-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:048a930eb4572829604982e39a0c7289ab5dc8abc7fc9f5aabd6fbc08c154e93"}, + {file = "lxml-6.0.0-cp39-cp39-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c0b5fa5eda84057a4f1bbb4bb77a8c28ff20ae7ce211588d698ae453e13c6281"}, + {file = "lxml-6.0.0-cp39-cp39-manylinux_2_31_armv7l.whl", hash = "sha256:c352fc8f36f7e9727db17adbf93f82499457b3d7e5511368569b4c5bd155a922"}, + {file = "lxml-6.0.0-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:8db5dc617cb937ae17ff3403c3a70a7de9df4852a046f93e71edaec678f721d0"}, + {file = "lxml-6.0.0-cp39-cp39-musllinux_1_2_armv7l.whl", hash = "sha256:2181e4b1d07dde53986023482673c0f1fba5178ef800f9ab95ad791e8bdded6a"}, + {file = "lxml-6.0.0-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:b3c98d5b24c6095e89e03d65d5c574705be3d49c0d8ca10c17a8a4b5201b72f5"}, + {file = "lxml-6.0.0-cp39-cp39-win32.whl", hash = "sha256:04d67ceee6db4bcb92987ccb16e53bef6b42ced872509f333c04fb58a3315256"}, + {file = "lxml-6.0.0-cp39-cp39-win_amd64.whl", hash = "sha256:e0b1520ef900e9ef62e392dd3d7ae4f5fa224d1dd62897a792cf353eb20b6cae"}, + {file = "lxml-6.0.0-cp39-cp39-win_arm64.whl", hash = "sha256:e35e8aaaf3981489f42884b59726693de32dabfc438ac10ef4eb3409961fd402"}, + {file = "lxml-6.0.0-pp310-pypy310_pp73-macosx_10_15_x86_64.whl", hash = "sha256:dbdd7679a6f4f08152818043dbb39491d1af3332128b3752c3ec5cebc0011a72"}, + {file = "lxml-6.0.0-pp310-pypy310_pp73-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:40442e2a4456e9910875ac12951476d36c0870dcb38a68719f8c4686609897c4"}, + {file = "lxml-6.0.0-pp310-pypy310_pp73-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:db0efd6bae1c4730b9c863fc4f5f3c0fa3e8f05cae2c44ae141cb9dfc7d091dc"}, + {file = "lxml-6.0.0-pp310-pypy310_pp73-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9ab542c91f5a47aaa58abdd8ea84b498e8e49fe4b883d67800017757a3eb78e8"}, + {file = "lxml-6.0.0-pp310-pypy310_pp73-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:013090383863b72c62a702d07678b658fa2567aa58d373d963cca245b017e065"}, + {file = "lxml-6.0.0-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:c86df1c9af35d903d2b52d22ea3e66db8058d21dc0f59842ca5deb0595921141"}, + {file = "lxml-6.0.0-pp39-pypy39_pp73-macosx_10_15_x86_64.whl", hash = "sha256:4337e4aec93b7c011f7ee2e357b0d30562edd1955620fdd4aeab6aacd90d43c5"}, + {file = "lxml-6.0.0-pp39-pypy39_pp73-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:ae74f7c762270196d2dda56f8dd7309411f08a4084ff2dfcc0b095a218df2e06"}, + {file = "lxml-6.0.0-pp39-pypy39_pp73-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:059c4cbf3973a621b62ea3132934ae737da2c132a788e6cfb9b08d63a0ef73f9"}, + {file = "lxml-6.0.0-pp39-pypy39_pp73-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:17f090a9bc0ce8da51a5632092f98a7e7f84bca26f33d161a98b57f7fb0004ca"}, + {file = "lxml-6.0.0-pp39-pypy39_pp73-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9da022c14baeec36edfcc8daf0e281e2f55b950249a455776f0d1adeeada4734"}, + {file = "lxml-6.0.0-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:a55da151d0b0c6ab176b4e761670ac0e2667817a1e0dadd04a01d0561a219349"}, + {file = "lxml-6.0.0.tar.gz", hash = "sha256:032e65120339d44cdc3efc326c9f660f5f7205f3a535c1fdbf898b29ea01fb72"}, ] [package.extras] @@ -1160,7 +1122,6 @@ cssselect = ["cssselect (>=0.7)"] html-clean = ["lxml_html_clean"] html5 = ["html5lib"] htmlsoup = ["BeautifulSoup4"] -source = ["Cython (>=3.0.11,<3.1.0)"] [[package]] name = "lxml-stubs" From cda922830eb99850b0a06fdf22721cbe318c6ca3 Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Thu, 17 Jul 2025 11:57:19 -0500 Subject: [PATCH 020/185] Clean up `MetricsResource` and Prometheus hacks (#18687) Clean up `MetricsResource`, Prometheus hacks (`_set_prometheus_client_use_created_metrics`), and better document why we care about having a separate `metrics` listener type. These clean-up changes have been split out from https://github.com/element-hq/synapse/pull/18584 since that PR was closed. --- changelog.d/18687.misc | 1 + poetry.lock | 48 ++++++++-------- pyproject.toml | 4 +- synapse/app/_base.py | 33 ++++------- synapse/metrics/__init__.py | 78 +++++++++++++++++++++++++- synapse/metrics/_twisted_exposition.py | 45 --------------- tests/metrics/test_metrics.py | 31 ---------- tests/storage/test_event_metrics.py | 3 +- 8 files changed, 115 insertions(+), 128 deletions(-) create mode 100644 changelog.d/18687.misc delete mode 100644 synapse/metrics/_twisted_exposition.py diff --git a/changelog.d/18687.misc b/changelog.d/18687.misc new file mode 100644 index 0000000000..2d1ba2c790 --- /dev/null +++ b/changelog.d/18687.misc @@ -0,0 +1 @@ +Clean up `MetricsResource` and Prometheus hacks. diff --git a/poetry.lock b/poetry.lock index ebdb8e3896..4912432dbe 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 2.1.1 and should not be changed by hand. +# This file is automatically @generated by Poetry 2.1.2 and should not be changed by hand. [[package]] name = "annotated-types" @@ -39,7 +39,7 @@ description = "The ultimate Python library in building OAuth and OpenID Connect optional = true python-versions = ">=3.9" groups = ["main"] -markers = "extra == \"all\" or extra == \"jwt\" or extra == \"oidc\"" +markers = "extra == \"oidc\" or extra == \"jwt\" or extra == \"all\"" files = [ {file = "authlib-1.6.0-py2.py3-none-any.whl", hash = "sha256:91685589498f79e8655e8a8947431ad6288831d643f11c55c2143ffcc738048d"}, {file = "authlib-1.6.0.tar.gz", hash = "sha256:4367d32031b7af175ad3a323d571dc7257b7099d55978087ceae4a0d88cd3210"}, @@ -435,7 +435,7 @@ description = "XML bomb protection for Python stdlib modules" optional = true python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" groups = ["main"] -markers = "extra == \"all\" or extra == \"saml2\"" +markers = "extra == \"saml2\" or extra == \"all\"" files = [ {file = "defusedxml-0.7.1-py2.py3-none-any.whl", hash = "sha256:a352e7e428770286cc899e2542b6cdaedb2b4953ff269a210103ec58f6198a61"}, {file = "defusedxml-0.7.1.tar.gz", hash = "sha256:1bb3032db185915b62d7c6209c5a8792be6a32ab2fedacc84e01b52c51aa3e69"}, @@ -478,7 +478,7 @@ description = "XPath 1.0/2.0/3.0/3.1 parsers and selectors for ElementTree and l optional = true python-versions = ">=3.7" groups = ["main"] -markers = "extra == \"all\" or extra == \"saml2\"" +markers = "extra == \"saml2\" or extra == \"all\"" files = [ {file = "elementpath-4.1.5-py3-none-any.whl", hash = "sha256:2ac1a2fb31eb22bbbf817f8cf6752f844513216263f0e3892c8e79782fe4bb55"}, {file = "elementpath-4.1.5.tar.gz", hash = "sha256:c2d6dc524b29ef751ecfc416b0627668119d8812441c555d7471da41d4bacb8d"}, @@ -528,7 +528,7 @@ description = "Python wrapper for hiredis" optional = true python-versions = ">=3.8" groups = ["main"] -markers = "extra == \"all\" or extra == \"redis\"" +markers = "extra == \"redis\" or extra == \"all\"" files = [ {file = "hiredis-3.2.1-cp310-cp310-macosx_10_15_universal2.whl", hash = "sha256:add17efcbae46c5a6a13b244ff0b4a8fa079602ceb62290095c941b42e9d5dec"}, {file = "hiredis-3.2.1-cp310-cp310-macosx_10_15_x86_64.whl", hash = "sha256:5fe955cc4f66c57df1ae8e5caf4de2925d43b5efab4e40859662311d1bcc5f54"}, @@ -865,7 +865,7 @@ description = "Jaeger Python OpenTracing Tracer implementation" optional = true python-versions = ">=3.7" groups = ["main"] -markers = "extra == \"all\" or extra == \"opentracing\"" +markers = "extra == \"opentracing\" or extra == \"all\"" files = [ {file = "jaeger-client-4.8.0.tar.gz", hash = "sha256:3157836edab8e2c209bd2d6ae61113db36f7ee399e66b1dcbb715d87ab49bfe0"}, ] @@ -1003,7 +1003,7 @@ description = "A strictly RFC 4510 conforming LDAP V3 pure Python client library optional = true python-versions = "*" groups = ["main"] -markers = "extra == \"all\" or extra == \"matrix-synapse-ldap3\"" +markers = "extra == \"matrix-synapse-ldap3\" or extra == \"all\"" files = [ {file = "ldap3-2.9.1-py2.py3-none-any.whl", hash = "sha256:5869596fc4948797020d3f03b7939da938778a0f9e2009f7a072ccf92b8e8d70"}, {file = "ldap3-2.9.1.tar.gz", hash = "sha256:f3e7fc4718e3f09dda568b57100095e0ce58633bcabbed8667ce3f8fbaa4229f"}, @@ -1019,7 +1019,7 @@ description = "Powerful and Pythonic XML processing library combining libxml2/li optional = true python-versions = ">=3.8" groups = ["main"] -markers = "extra == \"all\" or extra == \"url-preview\"" +markers = "extra == \"url-preview\" or extra == \"all\"" files = [ {file = "lxml-6.0.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:35bc626eec405f745199200ccb5c6b36f202675d204aa29bb52e27ba2b71dea8"}, {file = "lxml-6.0.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:246b40f8a4aec341cbbf52617cad8ab7c888d944bfe12a6abd2b1f6cfb6f6082"}, @@ -1260,7 +1260,7 @@ description = "An LDAP3 auth provider for Synapse" optional = true python-versions = ">=3.7" groups = ["main"] -markers = "extra == \"all\" or extra == \"matrix-synapse-ldap3\"" +markers = "extra == \"matrix-synapse-ldap3\" or extra == \"all\"" files = [ {file = "matrix-synapse-ldap3-0.3.0.tar.gz", hash = "sha256:8bb6517173164d4b9cc44f49de411d8cebdb2e705d5dd1ea1f38733c4a009e1d"}, {file = "matrix_synapse_ldap3-0.3.0-py3-none-any.whl", hash = "sha256:8b4d701f8702551e98cc1d8c20dbed532de5613584c08d0df22de376ba99159d"}, @@ -1493,7 +1493,7 @@ description = "OpenTracing API for Python. See documentation at http://opentraci optional = true python-versions = "*" groups = ["main"] -markers = "extra == \"all\" or extra == \"opentracing\"" +markers = "extra == \"opentracing\" or extra == \"all\"" files = [ {file = "opentracing-2.4.0.tar.gz", hash = "sha256:a173117e6ef580d55874734d1fa7ecb6f3655160b8b8974a2a1e98e5ec9c840d"}, ] @@ -1699,7 +1699,7 @@ description = "psycopg2 - Python-PostgreSQL Database Adapter" optional = true python-versions = ">=3.8" groups = ["main"] -markers = "extra == \"all\" or extra == \"postgres\"" +markers = "extra == \"postgres\" or extra == \"all\"" files = [ {file = "psycopg2-2.9.10-cp310-cp310-win32.whl", hash = "sha256:5df2b672140f95adb453af93a7d669d7a7bf0a56bcd26f1502329166f4a61716"}, {file = "psycopg2-2.9.10-cp310-cp310-win_amd64.whl", hash = "sha256:c6f7b8561225f9e711a9c47087388a97fdc948211c10a4bccbf0ba68ab7b3b5a"}, @@ -1720,7 +1720,7 @@ description = ".. image:: https://travis-ci.org/chtd/psycopg2cffi.svg?branch=mas optional = true python-versions = "*" groups = ["main"] -markers = "platform_python_implementation == \"PyPy\" and (extra == \"all\" or extra == \"postgres\")" +markers = "platform_python_implementation == \"PyPy\" and (extra == \"postgres\" or extra == \"all\")" files = [ {file = "psycopg2cffi-2.9.0.tar.gz", hash = "sha256:7e272edcd837de3a1d12b62185eb85c45a19feda9e62fa1b120c54f9e8d35c52"}, ] @@ -1736,7 +1736,7 @@ description = "A Simple library to enable psycopg2 compatability" optional = true python-versions = "*" groups = ["main"] -markers = "platform_python_implementation == \"PyPy\" and (extra == \"all\" or extra == \"postgres\")" +markers = "platform_python_implementation == \"PyPy\" and (extra == \"postgres\" or extra == \"all\")" files = [ {file = "psycopg2cffi-compat-1.1.tar.gz", hash = "sha256:d25e921748475522b33d13420aad5c2831c743227dc1f1f2585e0fdb5c914e05"}, ] @@ -1996,7 +1996,7 @@ description = "A development tool to measure, monitor and analyze the memory beh optional = true python-versions = ">=3.6" groups = ["main"] -markers = "extra == \"all\" or extra == \"cache-memory\"" +markers = "extra == \"cache-memory\" or extra == \"all\"" files = [ {file = "Pympler-1.0.1-py3-none-any.whl", hash = "sha256:d260dda9ae781e1eab6ea15bacb84015849833ba5555f141d2d9b7b7473b307d"}, {file = "Pympler-1.0.1.tar.gz", hash = "sha256:993f1a3599ca3f4fcd7160c7545ad06310c9e12f70174ae7ae8d4e25f6c5d3fa"}, @@ -2056,7 +2056,7 @@ description = "Python implementation of SAML Version 2 Standard" optional = true python-versions = ">=3.9,<4.0" groups = ["main"] -markers = "extra == \"all\" or extra == \"saml2\"" +markers = "extra == \"saml2\" or extra == \"all\"" files = [ {file = "pysaml2-7.5.0-py3-none-any.whl", hash = "sha256:bc6627cc344476a83c757f440a73fda1369f13b6fda1b4e16bca63ffbabb5318"}, {file = "pysaml2-7.5.0.tar.gz", hash = "sha256:f36871d4e5ee857c6b85532e942550d2cf90ea4ee943d75eb681044bbc4f54f7"}, @@ -2081,7 +2081,7 @@ description = "Extensions to the standard Python datetime module" optional = true python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" groups = ["main"] -markers = "extra == \"all\" or extra == \"saml2\"" +markers = "extra == \"saml2\" or extra == \"all\"" files = [ {file = "python-dateutil-2.8.2.tar.gz", hash = "sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86"}, {file = "python_dateutil-2.8.2-py2.py3-none-any.whl", hash = "sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9"}, @@ -2109,7 +2109,7 @@ description = "World timezone definitions, modern and historical" optional = true python-versions = "*" groups = ["main"] -markers = "extra == \"all\" or extra == \"saml2\"" +markers = "extra == \"saml2\" or extra == \"all\"" files = [ {file = "pytz-2022.7.1-py2.py3-none-any.whl", hash = "sha256:78f4f37d8198e0627c5f1143240bb0206b8691d8d7ac6d78fee88b78733f8c4a"}, {file = "pytz-2022.7.1.tar.gz", hash = "sha256:01a0681c4b9684a28304615eba55d1ab31ae00bf68ec157ec3708a8182dbbcd0"}, @@ -2474,7 +2474,7 @@ description = "Python client for Sentry (https://sentry.io)" optional = true python-versions = ">=3.6" groups = ["main"] -markers = "extra == \"all\" or extra == \"sentry\"" +markers = "extra == \"sentry\" or extra == \"all\"" files = [ {file = "sentry_sdk-2.32.0-py2.py3-none-any.whl", hash = "sha256:6cf51521b099562d7ce3606da928c473643abe99b00ce4cb5626ea735f4ec345"}, {file = "sentry_sdk-2.32.0.tar.gz", hash = "sha256:9016c75d9316b0f6921ac14c8cd4fb938f26002430ac5be9945ab280f78bec6b"}, @@ -2662,7 +2662,7 @@ description = "Tornado IOLoop Backed Concurrent Futures" optional = true python-versions = "*" groups = ["main"] -markers = "extra == \"all\" or extra == \"opentracing\"" +markers = "extra == \"opentracing\" or extra == \"all\"" files = [ {file = "threadloop-1.0.2-py2-none-any.whl", hash = "sha256:5c90dbefab6ffbdba26afb4829d2a9df8275d13ac7dc58dccb0e279992679599"}, {file = "threadloop-1.0.2.tar.gz", hash = "sha256:8b180aac31013de13c2ad5c834819771992d350267bddb854613ae77ef571944"}, @@ -2678,7 +2678,7 @@ description = "Python bindings for the Apache Thrift RPC system" optional = true python-versions = "*" groups = ["main"] -markers = "extra == \"all\" or extra == \"opentracing\"" +markers = "extra == \"opentracing\" or extra == \"all\"" files = [ {file = "thrift-0.16.0.tar.gz", hash = "sha256:2b5b6488fcded21f9d312aa23c9ff6a0195d0f6ae26ddbd5ad9e3e25dfc14408"}, ] @@ -2740,7 +2740,7 @@ description = "Tornado is a Python web framework and asynchronous networking lib optional = true python-versions = ">=3.9" groups = ["main"] -markers = "extra == \"all\" or extra == \"opentracing\"" +markers = "extra == \"opentracing\" or extra == \"all\"" files = [ {file = "tornado-6.5-cp39-abi3-macosx_10_9_universal2.whl", hash = "sha256:f81067dad2e4443b015368b24e802d0083fecada4f0a4572fdb72fc06e54a9a6"}, {file = "tornado-6.5-cp39-abi3-macosx_10_9_x86_64.whl", hash = "sha256:9ac1cbe1db860b3cbb251e795c701c41d343f06a96049d6274e7c77559117e41"}, @@ -2877,7 +2877,7 @@ description = "non-blocking redis client for python" optional = true python-versions = "*" groups = ["main"] -markers = "extra == \"all\" or extra == \"redis\"" +markers = "extra == \"redis\" or extra == \"all\"" files = [ {file = "txredisapi-1.4.11-py3-none-any.whl", hash = "sha256:ac64d7a9342b58edca13ef267d4fa7637c1aa63f8595e066801c1e8b56b22d0b"}, {file = "txredisapi-1.4.11.tar.gz", hash = "sha256:3eb1af99aefdefb59eb877b1dd08861efad60915e30ad5bf3d5bf6c5cedcdbc6"}, @@ -3208,7 +3208,7 @@ description = "An XML Schema validator and decoder" optional = true python-versions = ">=3.7" groups = ["main"] -markers = "extra == \"all\" or extra == \"saml2\"" +markers = "extra == \"saml2\" or extra == \"all\"" files = [ {file = "xmlschema-2.4.0-py3-none-any.whl", hash = "sha256:dc87be0caaa61f42649899189aab2fd8e0d567f2cf548433ba7b79278d231a4a"}, {file = "xmlschema-2.4.0.tar.gz", hash = "sha256:d74cd0c10866ac609e1ef94a5a69b018ad16e39077bc6393408b40c6babee793"}, @@ -3352,4 +3352,4 @@ url-preview = ["lxml"] [metadata] lock-version = "2.1" python-versions = "^3.9.0" -content-hash = "c140ab1db9e5d89d251e76c68f62cfcf5c477a923ba79f0fda186278f12af901" +content-hash = "a6965a294ca751ec2b5b0b92a050acc9afd4efb3e58550845dd32c60b74a70d1" diff --git a/pyproject.toml b/pyproject.toml index 89cd57a931..bd820bcb89 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -195,7 +195,9 @@ pymacaroons = ">=0.13.0" msgpack = ">=0.5.2" phonenumbers = ">=8.2.0" # we use GaugeHistogramMetric, which was added in prom-client 0.4.0. -prometheus-client = ">=0.4.0" +# `prometheus_client.metrics` was added in 0.5.0, so we require that too. +# We chose 0.6.0 as that is the current version in Debian Buster (oldstable). +prometheus-client = ">=0.6.0" # we use `order`, which arrived in attrs 19.2.0. # Note: 21.1.0 broke `/sync`, see https://github.com/matrix-org/synapse/issues/9936 attrs = ">=19.2.0,!=21.1.0" diff --git a/synapse/app/_base.py b/synapse/app/_base.py index 49ab5d680d..16aab93cd6 100644 --- a/synapse/app/_base.py +++ b/synapse/app/_base.py @@ -286,6 +286,16 @@ def register_start( def listen_metrics(bind_addresses: StrCollection, port: int) -> None: """ Start Prometheus metrics server. + + This method runs the metrics server on a different port, in a different thread to + Synapse. This can make it more resilient to heavy load in Synapse causing metric + requests to be slow or timeout. + + Even though `start_http_server_prometheus(...)` uses `threading.Thread` behind the + scenes (where all threads share the GIL and only one thread can execute Python + bytecode at a time), this still works because the metrics thread can preempt the + Twisted reactor thread between bytecode boundaries and the metrics thread gets + scheduled with roughly equal priority to the Twisted reactor thread. """ from prometheus_client import start_http_server as start_http_server_prometheus @@ -293,32 +303,9 @@ def listen_metrics(bind_addresses: StrCollection, port: int) -> None: for host in bind_addresses: logger.info("Starting metrics listener on %s:%d", host, port) - _set_prometheus_client_use_created_metrics(False) start_http_server_prometheus(port, addr=host, registry=RegistryProxy) -def _set_prometheus_client_use_created_metrics(new_value: bool) -> None: - """ - Sets whether prometheus_client should expose `_created`-suffixed metrics for - all gauges, histograms and summaries. - There is no programmatic way to disable this without poking at internals; - the proper way is to use an environment variable which prometheus_client - loads at import time. - - The motivation for disabling these `_created` metrics is that they're - a waste of space as they're not useful but they take up space in Prometheus. - """ - - import prometheus_client.metrics - - if hasattr(prometheus_client.metrics, "_use_created"): - prometheus_client.metrics._use_created = new_value - else: - logger.error( - "Can't disable `_created` metrics in prometheus_client (brittle hack broken?)" - ) - - def listen_manhole( bind_addresses: StrCollection, port: int, diff --git a/synapse/metrics/__init__.py b/synapse/metrics/__init__.py index 7e508dba05..de750a5de2 100644 --- a/synapse/metrics/__init__.py +++ b/synapse/metrics/__init__.py @@ -25,6 +25,7 @@ import logging import os import platform import threading +from importlib import metadata from typing import ( Callable, Dict, @@ -41,7 +42,15 @@ from typing import ( ) import attr -from prometheus_client import CollectorRegistry, Counter, Gauge, Histogram, Metric +from pkg_resources import parse_version +from prometheus_client import ( + CollectorRegistry, + Counter, + Gauge, + Histogram, + Metric, + generate_latest, +) from prometheus_client.core import ( REGISTRY, GaugeHistogramMetricFamily, @@ -49,11 +58,12 @@ from prometheus_client.core import ( ) from twisted.python.threadpool import ThreadPool +from twisted.web.resource import Resource +from twisted.web.server import Request # This module is imported for its side effects; flake8 needn't warn that it's unused. import synapse.metrics._reactor_metrics # noqa: F401 from synapse.metrics._gc import MIN_TIME_BETWEEN_GCS, install_gc_manager -from synapse.metrics._twisted_exposition import MetricsResource, generate_latest from synapse.metrics._types import Collector from synapse.types import StrSequence from synapse.util import SYNAPSE_VERSION @@ -81,6 +91,53 @@ terms, an endpoint you can scrape is called an *instance*, usually corresponding single process." (source: https://prometheus.io/docs/concepts/jobs_instances/) """ +CONTENT_TYPE_LATEST = "text/plain; version=0.0.4; charset=utf-8" +""" +Content type of the latest text format for Prometheus metrics. + +Pulled directly from the prometheus_client library. +""" + + +def _set_prometheus_client_use_created_metrics(new_value: bool) -> None: + """ + Sets whether prometheus_client should expose `_created`-suffixed metrics for + all gauges, histograms and summaries. + + There is no programmatic way in the old versions of `prometheus_client` to disable + this without poking at internals; the proper way in the old `prometheus_client` + versions (> `0.14.0` < `0.18.0`) is to use an environment variable which + prometheus_client loads at import time. For versions > `0.18.0`, we can use the + dedicated `disable_created_metrics()`/`enable_created_metrics()`. + + The motivation for disabling these `_created` metrics is that they're a waste of + space as they're not useful but they take up space in Prometheus. It's not the end + of the world if this doesn't work. + """ + import prometheus_client.metrics + + if hasattr(prometheus_client.metrics, "_use_created"): + prometheus_client.metrics._use_created = new_value + # Just log an error for old versions that don't support disabling the unecessary + # metrics. It's not the end of the world if this doesn't work as it just means extra + # wasted space taken up in Prometheus but things keep working. + elif parse_version(metadata.version("prometheus_client")) < parse_version("0.14.0"): + logger.error( + "Can't disable `_created` metrics in prometheus_client (unsupported `prometheus_client` version, too old)" + ) + # If the attribute doesn't exist on a newer version, this is a sign that the brittle + # hack is broken. We should consider updating the minimum version of + # `prometheus_client` to a version (> `0.18.0`) where we can use dedicated + # `disable_created_metrics()`/`enable_created_metrics()` functions. + else: + raise Exception( + "Can't disable `_created` metrics in prometheus_client (brittle hack broken?)" + ) + + +# Set this globally so it applies wherever we generate/collect metrics +_set_prometheus_client_use_created_metrics(False) + class _RegistryProxy: @staticmethod @@ -508,6 +565,23 @@ def register_threadpool(name: str, threadpool: ThreadPool) -> None: ) +class MetricsResource(Resource): + """ + Twisted ``Resource`` that serves prometheus metrics. + """ + + isLeaf = True + + def __init__(self, registry: CollectorRegistry = REGISTRY): + self.registry = registry + + def render_GET(self, request: Request) -> bytes: + request.setHeader(b"Content-Type", CONTENT_TYPE_LATEST.encode("ascii")) + response = generate_latest(self.registry) + request.setHeader(b"Content-Length", str(len(response))) + return response + + __all__ = [ "Collector", "MetricsResource", diff --git a/synapse/metrics/_twisted_exposition.py b/synapse/metrics/_twisted_exposition.py deleted file mode 100644 index 9652ca83fb..0000000000 --- a/synapse/metrics/_twisted_exposition.py +++ /dev/null @@ -1,45 +0,0 @@ -# -# This file is licensed under the Affero General Public License (AGPL) version 3. -# -# Copyright 2019 Matrix.org Foundation C.I.C. -# Copyright 2015-2019 Prometheus Python Client Developers -# Copyright (C) 2023 New Vector, Ltd -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as -# published by the Free Software Foundation, either version 3 of the -# License, or (at your option) any later version. -# -# See the GNU Affero General Public License for more details: -# . -# -# Originally licensed under the Apache License, Version 2.0: -# . -# -# [This file includes modifications made by New Vector Limited] -# -# - -from prometheus_client import REGISTRY, CollectorRegistry, generate_latest - -from twisted.web.resource import Resource -from twisted.web.server import Request - -CONTENT_TYPE_LATEST = "text/plain; version=0.0.4; charset=utf-8" - - -class MetricsResource(Resource): - """ - Twisted ``Resource`` that serves prometheus metrics. - """ - - isLeaf = True - - def __init__(self, registry: CollectorRegistry = REGISTRY): - self.registry = registry - - def render_GET(self, request: Request) -> bytes: - request.setHeader(b"Content-Type", CONTENT_TYPE_LATEST.encode("ascii")) - response = generate_latest(self.registry) - request.setHeader(b"Content-Length", str(len(response))) - return response diff --git a/tests/metrics/test_metrics.py b/tests/metrics/test_metrics.py index dca8dd79b1..e92d5f6dfa 100644 --- a/tests/metrics/test_metrics.py +++ b/tests/metrics/test_metrics.py @@ -18,14 +18,10 @@ # [This file includes modifications made by New Vector Limited] # # -from importlib import metadata from typing import Dict, Protocol, Tuple -from unittest.mock import patch -from pkg_resources import parse_version from prometheus_client.core import Sample -from synapse.app._base import _set_prometheus_client_use_created_metrics from synapse.metrics import REGISTRY, InFlightGauge, generate_latest from synapse.util.caches.deferred_cache import DeferredCache @@ -186,30 +182,3 @@ class CacheMetricsTests(unittest.HomeserverTestCase): self.assertEqual(items["synapse_util_caches_cache_size"], "1.0") self.assertEqual(items["synapse_util_caches_cache_max_size"], "777.0") - - -class PrometheusMetricsHackTestCase(unittest.HomeserverTestCase): - if parse_version(metadata.version("prometheus_client")) < parse_version("0.14.0"): - skip = "prometheus-client too old" - - def test_created_metrics_disabled(self) -> None: - """ - Tests that a brittle hack, to disable `_created` metrics, works. - This involves poking at the internals of prometheus-client. - It's not the end of the world if this doesn't work. - - This test gives us a way to notice if prometheus-client changes - their internals. - """ - import prometheus_client.metrics - - PRIVATE_FLAG_NAME = "_use_created" - - # By default, the pesky `_created` metrics are enabled. - # Check this assumption is still valid. - self.assertTrue(getattr(prometheus_client.metrics, PRIVATE_FLAG_NAME)) - - with patch("prometheus_client.metrics") as mock: - setattr(mock, PRIVATE_FLAG_NAME, True) - _set_prometheus_client_use_created_metrics(False) - self.assertFalse(getattr(mock, PRIVATE_FLAG_NAME, False)) diff --git a/tests/storage/test_event_metrics.py b/tests/storage/test_event_metrics.py index 3f7ee86498..fc6e02545f 100644 --- a/tests/storage/test_event_metrics.py +++ b/tests/storage/test_event_metrics.py @@ -18,9 +18,8 @@ # [This file includes modifications made by New Vector Limited] # # -from prometheus_client import generate_latest -from synapse.metrics import REGISTRY +from synapse.metrics import REGISTRY, generate_latest from synapse.types import UserID, create_requester from tests.unittest import HomeserverTestCase From 66504d1144c9620db378a43e1a982d807a0e6e63 Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Thu, 17 Jul 2025 15:20:42 -0500 Subject: [PATCH 021/185] Fix dirty `Cargo.lock` changes appearing after install (`base64`) (#18689) Normal install results in `Cargo.lock` changes constantly popping up for me as I navigate my branches. This was probably caused by some Depdendabot PR updating the `Cargo.toml` without `Cargo.lock` or something. ``` poetry install --extras all ``` In another PR, I've also added CI to ensure we don't leave `Cargo.lock` changes behind to avoid this annoyance in the future -> https://github.com/element-hq/synapse/pull/18693 --- changelog.d/18689.misc | 1 + rust/Cargo.toml | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) create mode 100644 changelog.d/18689.misc diff --git a/changelog.d/18689.misc b/changelog.d/18689.misc new file mode 100644 index 0000000000..51f0586e19 --- /dev/null +++ b/changelog.d/18689.misc @@ -0,0 +1 @@ +Fix dirty `Cargo.lock` changes appearing after install (`base64`). diff --git a/rust/Cargo.toml b/rust/Cargo.toml index 4f5ebb68b7..ab87de33ab 100644 --- a/rust/Cargo.toml +++ b/rust/Cargo.toml @@ -23,7 +23,7 @@ name = "synapse.synapse_rust" [dependencies] anyhow = "1.0.63" -base64 = "0.21.7" +base64 = "0.22.1" bytes = "1.6.0" headers = "0.4.0" http = "1.1.0" From 5ea2cf2484903a005f6a8af9fe8a0617e3cff373 Mon Sep 17 00:00:00 2001 From: Quentin Gliech Date: Fri, 18 Jul 2025 09:06:14 +0200 Subject: [PATCH 022/185] Move device changes off the main process (#18581) The main goal of this PR is to handle device list changes onto multiple writers, off the main process, so that we can have logins happening whilst Synapse is rolling-restarting. This is quite an intrusive change, so I would advise to review this commit by commit; I tried to keep the history as clean as possible. There are a few things to consider: - the `device_list_key` in stream tokens becomes a `MultiWriterStreamToken`, which has a few implications in sync and on the storage layer - we had a split between `DeviceHandler` and `DeviceWorkerHandler` for master vs. worker process. I've kept this split, but making it rather writer vs. non-writer worker, using method overrides for doing replication calls when needed - there are a few operations that need to happen on a single worker at a time. Instead of using cross-worker locks, for now I made them run on the first writer on the list --------- Co-authored-by: Eric Eastwood --- changelog.d/18581.feature | 1 + .../complement/conf/start_for_complement.sh | 2 +- docker/configure_workers_and_start.py | 54 +- .../configuration/config_documentation.md | 2 + docs/workers.md | 21 +- schema/synapse-config.schema.yaml | 3 + synapse/config/workers.py | 46 +- synapse/handlers/appservice.py | 3 +- synapse/handlers/deactivate_account.py | 5 - synapse/handlers/device.py | 1053 ++++++++++------- synapse/handlers/devicemessage.py | 22 +- synapse/handlers/e2e_keys.py | 36 +- synapse/handlers/federation.py | 11 +- synapse/handlers/federation_event.py | 17 +- synapse/handlers/register.py | 4 - synapse/handlers/set_password.py | 16 +- synapse/handlers/sso.py | 8 - synapse/module_api/__init__.py | 7 - synapse/replication/http/__init__.py | 2 +- synapse/replication/http/devices.py | 185 ++- synapse/replication/tcp/client.py | 6 +- synapse/replication/tcp/handler.py | 7 + synapse/rest/admin/__init__.py | 4 +- synapse/rest/admin/devices.py | 26 +- synapse/rest/client/devices.py | 40 +- synapse/rest/client/keys.py | 5 +- synapse/rest/client/logout.py | 9 +- synapse/server.py | 12 +- synapse/storage/databases/main/devices.py | 935 ++++++++------- .../storage/databases/main/end_to_end_keys.py | 96 +- synapse/storage/databases/main/receipts.py | 21 +- .../storage/databases/main/registration.py | 408 ++++--- synapse/storage/databases/main/relations.py | 2 +- synapse/storage/databases/main/room.py | 5 + synapse/storage/databases/main/stream.py | 19 +- synapse/streams/events.py | 2 +- synapse/types/__init__.py | 54 +- tests/federation/test_federation_sender.py | 6 +- tests/handlers/test_device.py | 8 +- tests/handlers/test_e2e_keys.py | 4 +- tests/module_api/test_api.py | 4 +- tests/rest/admin/test_device.py | 4 +- 42 files changed, 1753 insertions(+), 1422 deletions(-) create mode 100644 changelog.d/18581.feature diff --git a/changelog.d/18581.feature b/changelog.d/18581.feature new file mode 100644 index 0000000000..c7b7017b0c --- /dev/null +++ b/changelog.d/18581.feature @@ -0,0 +1 @@ +Enable workers to write directly to the device lists stream and handle device list updates, reducing load on the main process. diff --git a/docker/complement/conf/start_for_complement.sh b/docker/complement/conf/start_for_complement.sh index a5e06396e2..da1b26a283 100755 --- a/docker/complement/conf/start_for_complement.sh +++ b/docker/complement/conf/start_for_complement.sh @@ -54,7 +54,6 @@ if [[ -n "$SYNAPSE_COMPLEMENT_USE_WORKERS" ]]; then export SYNAPSE_WORKER_TYPES="\ event_persister:2, \ background_worker, \ - frontend_proxy, \ event_creator, \ user_dir, \ media_repository, \ @@ -65,6 +64,7 @@ if [[ -n "$SYNAPSE_COMPLEMENT_USE_WORKERS" ]]; then client_reader, \ appservice, \ pusher, \ + device_lists:2, \ stream_writers=account_data+presence+receipts+to_device+typing" fi diff --git a/docker/configure_workers_and_start.py b/docker/configure_workers_and_start.py index 102a88fad1..7909b9d932 100755 --- a/docker/configure_workers_and_start.py +++ b/docker/configure_workers_and_start.py @@ -178,6 +178,8 @@ WORKERS_CONFIG: Dict[str, Dict[str, Any]] = { "^/_matrix/client/(api/v1|r0|v3|unstable)/login$", "^/_matrix/client/(api/v1|r0|v3|unstable)/account/3pid$", "^/_matrix/client/(api/v1|r0|v3|unstable)/account/whoami$", + "^/_matrix/client/(api/v1|r0|v3|unstable)/devices(/|$)", + "^/_matrix/client/(r0|v3)/delete_devices$", "^/_matrix/client/versions$", "^/_matrix/client/(api/v1|r0|v3|unstable)/voip/turnServer$", "^/_matrix/client/(r0|v3|unstable)/register$", @@ -194,6 +196,9 @@ WORKERS_CONFIG: Dict[str, Dict[str, Any]] = { "^/_matrix/client/(api/v1|r0|v3|unstable)/directory/room/.*$", "^/_matrix/client/(r0|v3|unstable)/capabilities$", "^/_matrix/client/(r0|v3|unstable)/notifications$", + "^/_matrix/client/(api/v1|r0|v3|unstable)/keys/upload", + "^/_matrix/client/(api/v1|r0|v3|unstable)/keys/device_signing/upload$", + "^/_matrix/client/(api/v1|r0|v3|unstable)/keys/signatures/upload$", ], "shared_extra_conf": {}, "worker_extra_conf": "", @@ -265,13 +270,6 @@ WORKERS_CONFIG: Dict[str, Dict[str, Any]] = { "shared_extra_conf": {}, "worker_extra_conf": "", }, - "frontend_proxy": { - "app": "synapse.app.generic_worker", - "listener_resources": ["client", "replication"], - "endpoint_patterns": ["^/_matrix/client/(api/v1|r0|v3|unstable)/keys/upload"], - "shared_extra_conf": {}, - "worker_extra_conf": "", - }, "account_data": { "app": "synapse.app.generic_worker", "listener_resources": ["client", "replication"], @@ -306,6 +304,13 @@ WORKERS_CONFIG: Dict[str, Dict[str, Any]] = { "shared_extra_conf": {}, "worker_extra_conf": "", }, + "device_lists": { + "app": "synapse.app.generic_worker", + "listener_resources": ["client", "replication"], + "endpoint_patterns": [], + "shared_extra_conf": {}, + "worker_extra_conf": "", + }, "typing": { "app": "synapse.app.generic_worker", "listener_resources": ["client", "replication"], @@ -412,16 +417,17 @@ def add_worker_roles_to_shared_config( # streams instance_map = shared_config.setdefault("instance_map", {}) - # This is a list of the stream_writers that there can be only one of. Events can be - # sharded, and therefore doesn't belong here. - singular_stream_writers = [ + # This is a list of the stream_writers. + stream_writers = { "account_data", + "events", + "device_lists", "presence", "receipts", "to_device", "typing", "push_rules", - ] + } # Worker-type specific sharding config. Now a single worker can fulfill multiple # roles, check each. @@ -431,28 +437,11 @@ def add_worker_roles_to_shared_config( if "federation_sender" in worker_types_set: shared_config.setdefault("federation_sender_instances", []).append(worker_name) - if "event_persister" in worker_types_set: - # Event persisters write to the events stream, so we need to update - # the list of event stream writers - shared_config.setdefault("stream_writers", {}).setdefault("events", []).append( - worker_name - ) - - # Map of stream writer instance names to host/ports combos - if os.environ.get("SYNAPSE_USE_UNIX_SOCKET", False): - instance_map[worker_name] = { - "path": f"/run/worker.{worker_port}", - } - else: - instance_map[worker_name] = { - "host": "localhost", - "port": worker_port, - } # Update the list of stream writers. It's convenient that the name of the worker # type is the same as the stream to write. Iterate over the whole list in case there # is more than one. for worker in worker_types_set: - if worker in singular_stream_writers: + if worker in stream_writers: shared_config.setdefault("stream_writers", {}).setdefault( worker, [] ).append(worker_name) @@ -876,6 +865,13 @@ def generate_worker_files( else: healthcheck_urls.append("http://localhost:%d/health" % (worker_port,)) + # Special case for event_persister: those are just workers that write to + # the `events` stream. For other workers, the worker name is the same + # name of the stream they write to, but for some reason it is not the + # case for event_persister. + if "event_persister" in worker_types_set: + worker_types_set.add("events") + # Update the shared config with sharding-related options if necessary add_worker_roles_to_shared_config( shared_config, worker_types_set, worker_name, worker_port diff --git a/docs/usage/configuration/config_documentation.md b/docs/usage/configuration/config_documentation.md index e47a32f510..fc838a1f0e 100644 --- a/docs/usage/configuration/config_documentation.md +++ b/docs/usage/configuration/config_documentation.md @@ -4341,6 +4341,8 @@ This setting has the following sub-options: * `push_rules` (string): Name of a worker assigned to the `push_rules` stream. +* `device_lists` (string): Name of a worker assigned to the `device_lists` stream. + Example configuration: ```yaml stream_writers: diff --git a/docs/workers.md b/docs/workers.md index 45a00696f3..7881aeebbe 100644 --- a/docs/workers.md +++ b/docs/workers.md @@ -238,7 +238,8 @@ information. ^/_matrix/client/unstable/im.nheko.summary/summary/.*$ ^/_matrix/client/(r0|v3|unstable)/account/3pid$ ^/_matrix/client/(r0|v3|unstable)/account/whoami$ - ^/_matrix/client/(r0|v3|unstable)/devices$ + ^/_matrix/client/(r0|v3)/delete_devices$ + ^/_matrix/client/(api/v1|r0|v3|unstable)/devices(/|$) ^/_matrix/client/versions$ ^/_matrix/client/(api/v1|r0|v3|unstable)/voip/turnServer$ ^/_matrix/client/(api/v1|r0|v3|unstable)/rooms/.*/event/ @@ -257,7 +258,9 @@ information. ^/_matrix/client/(r0|v3|unstable)/keys/changes$ ^/_matrix/client/(r0|v3|unstable)/keys/claim$ ^/_matrix/client/(r0|v3|unstable)/room_keys/ - ^/_matrix/client/(r0|v3|unstable)/keys/upload$ + ^/_matrix/client/(r0|v3|unstable)/keys/upload + ^/_matrix/client/(api/v1|r0|v3|unstable/keys/device_signing/upload$ + ^/_matrix/client/(api/v1|r0|v3|unstable)/keys/signatures/upload$ # Registration/login requests ^/_matrix/client/(api/v1|r0|v3|unstable)/login$ @@ -282,7 +285,6 @@ Additionally, the following REST endpoints can be handled for GET requests: ^/_matrix/client/(api/v1|r0|v3|unstable)/pushrules/ ^/_matrix/client/unstable/org.matrix.msc4140/delayed_events - ^/_matrix/client/(api/v1|r0|v3|unstable)/devices/ # Account data requests ^/_matrix/client/(r0|v3|unstable)/.*/tags @@ -329,7 +331,6 @@ set to `true`), the following endpoints can be handled by the worker: ^/_synapse/admin/v2/users/[^/]+$ ^/_synapse/admin/v1/username_available$ ^/_synapse/admin/v1/users/[^/]+/_allow_cross_signing_replacement_without_uia$ - # Only the GET method: ^/_synapse/admin/v1/users/[^/]+/devices$ Note that a [HTTP listener](usage/configuration/config_documentation.md#listeners) @@ -550,6 +551,18 @@ the stream writer for the `push_rules` stream: ^/_matrix/client/(api/v1|r0|v3|unstable)/pushrules/ +##### The `device_lists` stream + +The `device_lists` stream supports multiple writers. The following endpoints +can be handled by any worker, but should be routed directly one of the workers +configured as stream writer for the `device_lists` stream: + + ^/_matrix/client/(r0|v3)/delete_devices$ + ^/_matrix/client/(api/v1|r0|v3|unstable)/devices/ + ^/_matrix/client/(r0|v3|unstable)/keys/upload + ^/_matrix/client/(api/v1|r0|v3|unstable/keys/device_signing/upload$ + ^/_matrix/client/(api/v1|r0|v3|unstable)/keys/signatures/upload$ + #### Restrict outbound federation traffic to a specific set of workers The diff --git a/schema/synapse-config.schema.yaml b/schema/synapse-config.schema.yaml index 9a244856e6..72b80253f8 100644 --- a/schema/synapse-config.schema.yaml +++ b/schema/synapse-config.schema.yaml @@ -5383,6 +5383,9 @@ properties: push_rules: type: string description: Name of a worker assigned to the `push_rules` stream. + device_lists: + type: string + description: Name of a worker assigned to the `device_lists` stream. default: {} examples: - events: worker1 diff --git a/synapse/config/workers.py b/synapse/config/workers.py index 69036f9b52..c0c8a13861 100644 --- a/synapse/config/workers.py +++ b/synapse/config/workers.py @@ -134,40 +134,44 @@ class WriterLocations: can only be a single instance. account_data: The instances that write to the account data streams. Currently can only be a single instance. - receipts: The instances that write to the receipts stream. Currently - can only be a single instance. + receipts: The instances that write to the receipts stream. presence: The instances that write to the presence stream. Currently can only be a single instance. push_rules: The instances that write to the push stream. Currently can only be a single instance. + device_lists: The instances that write to the device list stream. """ events: List[str] = attr.ib( - default=["master"], + default=[MAIN_PROCESS_INSTANCE_NAME], converter=_instance_to_list_converter, ) typing: List[str] = attr.ib( - default=["master"], + default=[MAIN_PROCESS_INSTANCE_NAME], converter=_instance_to_list_converter, ) to_device: List[str] = attr.ib( - default=["master"], + default=[MAIN_PROCESS_INSTANCE_NAME], converter=_instance_to_list_converter, ) account_data: List[str] = attr.ib( - default=["master"], + default=[MAIN_PROCESS_INSTANCE_NAME], converter=_instance_to_list_converter, ) receipts: List[str] = attr.ib( - default=["master"], + default=[MAIN_PROCESS_INSTANCE_NAME], converter=_instance_to_list_converter, ) presence: List[str] = attr.ib( - default=["master"], + default=[MAIN_PROCESS_INSTANCE_NAME], converter=_instance_to_list_converter, ) push_rules: List[str] = attr.ib( - default=["master"], + default=[MAIN_PROCESS_INSTANCE_NAME], + converter=_instance_to_list_converter, + ) + device_lists: List[str] = attr.ib( + default=[MAIN_PROCESS_INSTANCE_NAME], converter=_instance_to_list_converter, ) @@ -358,7 +362,10 @@ class WorkerConfig(Config): ): instances = _instance_to_list_converter(getattr(self.writers, stream)) for instance in instances: - if instance != "master" and instance not in self.instance_map: + if ( + instance != MAIN_PROCESS_INSTANCE_NAME + and instance not in self.instance_map + ): raise ConfigError( "Instance %r is configured to write %s but does not appear in `instance_map` config." % (instance, stream) @@ -397,6 +404,11 @@ class WorkerConfig(Config): "Must only specify one instance to handle `push` messages." ) + if len(self.writers.device_lists) == 0: + raise ConfigError( + "Must specify at least one instance to handle `device_lists` messages." + ) + self.events_shard_config = RoutableShardedWorkerHandlingConfig( self.writers.events ) @@ -419,9 +431,12 @@ class WorkerConfig(Config): # # No effort is made to ensure only a single instance of these tasks is # running. - background_tasks_instance = config.get("run_background_tasks_on") or "master" + background_tasks_instance = ( + config.get("run_background_tasks_on") or MAIN_PROCESS_INSTANCE_NAME + ) self.run_background_tasks = ( - self.worker_name is None and background_tasks_instance == "master" + self.worker_name is None + and background_tasks_instance == MAIN_PROCESS_INSTANCE_NAME ) or self.worker_name == background_tasks_instance self.should_notify_appservices = self._should_this_worker_perform_duty( @@ -493,9 +508,10 @@ class WorkerConfig(Config): # 'don't run here'. new_option_should_run_here = None if new_option_name in config: - designated_worker = config[new_option_name] or "master" + designated_worker = config[new_option_name] or MAIN_PROCESS_INSTANCE_NAME new_option_should_run_here = ( - designated_worker == "master" and self.worker_name is None + designated_worker == MAIN_PROCESS_INSTANCE_NAME + and self.worker_name is None ) or designated_worker == self.worker_name legacy_option_should_run_here = None @@ -592,7 +608,7 @@ class WorkerConfig(Config): # If no worker instances are set we check if the legacy option # is set, which means use the main process. if legacy_option: - worker_instances = ["master"] + worker_instances = [MAIN_PROCESS_INSTANCE_NAME] if self.worker_app == legacy_app_name: if legacy_option: diff --git a/synapse/handlers/appservice.py b/synapse/handlers/appservice.py index 8c5308b522..93224d0c1b 100644 --- a/synapse/handlers/appservice.py +++ b/synapse/handlers/appservice.py @@ -638,7 +638,8 @@ class ApplicationServicesHandler: # Fetch the users who have modified their device list since then. users_with_changed_device_lists = await self.store.get_all_devices_changed( - from_key, to_key=new_key + MultiWriterStreamToken(stream=from_key), + to_key=MultiWriterStreamToken(stream=new_key), ) # Filter out any users the application service is not interested in diff --git a/synapse/handlers/deactivate_account.py b/synapse/handlers/deactivate_account.py index 42e53d920a..8d4d84bed1 100644 --- a/synapse/handlers/deactivate_account.py +++ b/synapse/handlers/deactivate_account.py @@ -24,7 +24,6 @@ from typing import TYPE_CHECKING, Optional from synapse.api.constants import Membership from synapse.api.errors import SynapseError -from synapse.handlers.device import DeviceHandler from synapse.metrics.background_process_metrics import run_as_background_process from synapse.types import Codes, Requester, UserID, create_requester @@ -84,10 +83,6 @@ class DeactivateAccountHandler: Returns: True if identity server supports removing threepids, otherwise False. """ - - # This can only be called on the main process. - assert isinstance(self._device_handler, DeviceHandler) - # Check if this user can be deactivated if not await self._third_party_rules.check_can_deactivate_user( user_id, by_admin diff --git a/synapse/handlers/device.py b/synapse/handlers/device.py index 65fbb48768..80d49fc18d 100644 --- a/synapse/handlers/device.py +++ b/synapse/handlers/device.py @@ -20,6 +20,7 @@ # # import logging +import random from threading import Lock from typing import ( TYPE_CHECKING, @@ -31,6 +32,7 @@ from typing import ( Optional, Set, Tuple, + cast, ) from synapse.api import errors @@ -48,6 +50,13 @@ from synapse.metrics.background_process_metrics import ( run_as_background_process, wrap_as_background_process, ) +from synapse.replication.http.devices import ( + ReplicationDeviceHandleRoomUnPartialStated, + ReplicationHandleNewDeviceUpdateRestServlet, + ReplicationMultiUserDevicesResyncRestServlet, + ReplicationNotifyDeviceUpdateRestServlet, + ReplicationNotifyUserSignatureUpdateRestServlet, +) from synapse.storage.databases.main.client_ips import DeviceLastConnectionInfo from synapse.storage.databases.main.roommember import EventIdMembership from synapse.storage.databases.main.state_deltas import StateDelta @@ -75,6 +84,7 @@ from synapse.util.retryutils import ( ) if TYPE_CHECKING: + from synapse.app.generic_worker import GenericWorkerStore from synapse.server import HomeServer logger = logging.getLogger(__name__) @@ -84,32 +94,319 @@ MAX_DEVICE_DISPLAY_NAME_LEN = 100 DELETE_STALE_DEVICES_INTERVAL_MS = 24 * 60 * 60 * 1000 -class DeviceWorkerHandler: +def _check_device_name_length(name: Optional[str]) -> None: + """ + Checks whether a device name is longer than the maximum allowed length. + + Args: + name: The name of the device. + + Raises: + SynapseError: if the device name is too long. + """ + if name and len(name) > MAX_DEVICE_DISPLAY_NAME_LEN: + raise SynapseError( + 400, + "Device display name is too long (max %i)" % (MAX_DEVICE_DISPLAY_NAME_LEN,), + errcode=Codes.TOO_LARGE, + ) + + +class DeviceHandler: + """ + Handles most things related to devices. This doesn't do any writing to the + device list stream on its own, and will call to device list writers through + replication when necessary (see DeviceWriterHandler). + """ + device_list_updater: "DeviceListWorkerUpdater" + store: "GenericWorkerStore" def __init__(self, hs: "HomeServer"): - self.clock = hs.get_clock() + self.server_name = hs.hostname # nb must be called this for @measure_func + self.clock = hs.get_clock() # nb must be called this for @measure_func self.hs = hs - self.store = hs.get_datastores().main + self.store = cast("GenericWorkerStore", hs.get_datastores().main) self.notifier = hs.get_notifier() self.state = hs.get_state_handler() self._appservice_handler = hs.get_application_service_handler() self._state_storage = hs.get_storage_controllers().state self._auth_handler = hs.get_auth_handler() + self._account_data_handler = hs.get_account_data_handler() self._event_sources = hs.get_event_sources() - self.server_name = hs.hostname self._msc3852_enabled = hs.config.experimental.msc3852_enabled self._query_appservices_for_keys = ( hs.config.experimental.msc3984_appservice_key_query ) self._task_scheduler = hs.get_task_scheduler() + self._dont_notify_new_devices_for = ( + hs.config.registration.dont_notify_new_devices_for + ) + self.device_list_updater = DeviceListWorkerUpdater(hs) self._task_scheduler.register_action( self._delete_device_messages, DELETE_DEVICE_MSGS_TASK_NAME ) + self._device_list_writers = hs.config.worker.writers.device_lists + + # Ensure a few operations are only running on the first device list writer + # + # This is needed because of a few linearizers in the DeviceListUpdater, + # and avoid using cross-worker locks. + # + # The main logic update is that the DeviceListUpdater is now only + # instantiated on the first device list writer, and a few methods that + # were safe to move to any worker were moved to the DeviceListWorkerUpdater + # This must be kept in sync with DeviceListWorkerUpdater + self._main_device_list_writer = hs.config.worker.writers.device_lists[0] + + self._notify_device_update_client = ( + ReplicationNotifyDeviceUpdateRestServlet.make_client(hs) + ) + self._notify_user_signature_update_client = ( + ReplicationNotifyUserSignatureUpdateRestServlet.make_client(hs) + ) + self._handle_new_device_update_client = ( + ReplicationHandleNewDeviceUpdateRestServlet.make_client(hs) + ) + self._handle_room_un_partial_stated_client = ( + ReplicationDeviceHandleRoomUnPartialStated.make_client(hs) + ) + + # The EDUs are handled on a single writer, as it needs to acquire a + # per-user lock, for which it is cheaper to use in-memory linearizers + # than cross-worker locks. + hs.get_federation_registry().register_instances_for_edu( + EduTypes.DEVICE_LIST_UPDATE, + [self._main_device_list_writer], + ) + + self._delete_stale_devices_after = hs.config.server.delete_stale_devices_after + + if ( + hs.config.worker.run_background_tasks + and self._delete_stale_devices_after is not None + ): + self.clock.looping_call( + run_as_background_process, + DELETE_STALE_DEVICES_INTERVAL_MS, + "delete_stale_devices", + self._delete_stale_devices, + ) + + async def _delete_stale_devices(self) -> None: + """Background task that deletes devices which haven't been accessed for more than + a configured time period. + """ + # We should only be running this job if the config option is defined. + assert self._delete_stale_devices_after is not None + now_ms = self.clock.time_msec() + since_ms = now_ms - self._delete_stale_devices_after + devices = await self.store.get_local_devices_not_accessed_since(since_ms) + + for user_id, user_devices in devices.items(): + await self.delete_devices(user_id, user_devices) + + async def check_device_registered( + self, + user_id: str, + device_id: Optional[str], + initial_device_display_name: Optional[str] = None, + auth_provider_id: Optional[str] = None, + auth_provider_session_id: Optional[str] = None, + ) -> str: + """ + If the given device has not been registered, register it with the + supplied display name. + + If no device_id is supplied, we make one up. + + Args: + user_id: @user:id + device_id: device id supplied by client + initial_device_display_name: device display name from client + auth_provider_id: The SSO IdP the user used, if any. + auth_provider_session_id: The session ID (sid) got from the SSO IdP. + Returns: + device id (generated if none was supplied) + """ + + _check_device_name_length(initial_device_display_name) + + # Check if we should send out device lists updates for this new device. + notify = user_id not in self._dont_notify_new_devices_for + + if device_id is not None: + new_device = await self.store.store_device( + user_id=user_id, + device_id=device_id, + initial_device_display_name=initial_device_display_name, + auth_provider_id=auth_provider_id, + auth_provider_session_id=auth_provider_session_id, + ) + if new_device: + if notify: + await self.notify_device_update(user_id, [device_id]) + return device_id + + # if the device id is not specified, we'll autogen one, but loop a few + # times in case of a clash. + attempts = 0 + while attempts < 5: + new_device_id = stringutils.random_string(10).upper() + new_device = await self.store.store_device( + user_id=user_id, + device_id=new_device_id, + initial_device_display_name=initial_device_display_name, + auth_provider_id=auth_provider_id, + auth_provider_session_id=auth_provider_session_id, + ) + if new_device: + if notify: + await self.notify_device_update(user_id, [new_device_id]) + return new_device_id + attempts += 1 + + raise errors.StoreError(500, "Couldn't generate a device ID.") + + @trace + async def delete_all_devices_for_user( + self, user_id: str, except_device_id: Optional[str] = None + ) -> None: + """Delete all of the user's devices + + Args: + user_id: The user to remove all devices from + except_device_id: optional device id which should not be deleted + """ + device_map = await self.store.get_devices_by_user(user_id) + if except_device_id is not None: + device_map.pop(except_device_id, None) + user_device_ids = device_map.keys() + await self.delete_devices(user_id, user_device_ids) + + async def delete_devices(self, user_id: str, device_ids: StrCollection) -> None: + """Delete several devices + + Args: + user_id: The user to delete devices from. + device_ids: The list of device IDs to delete + """ + to_device_stream_id = self._event_sources.get_current_token().to_device_key + + try: + await self.store.delete_devices(user_id, device_ids) + except errors.StoreError as e: + if e.code == 404: + # no match + set_tag("error", True) + set_tag("reason", "User doesn't have that device id.") + else: + raise + + # Delete data specific to each device. Not optimised as its an + # experimental MSC. + if self.hs.config.experimental.msc3890_enabled: + for device_id in device_ids: + # Remove any local notification settings for this device in accordance + # with MSC3890. + await self._account_data_handler.remove_account_data_for_user( + user_id, + f"org.matrix.msc3890.local_notification_settings.{device_id}", + ) + + # If we're deleting a lot of devices, a bunch of them may not have any + # to-device messages queued up. We filter those out to avoid scheduling + # unnecessary tasks. + devices_with_messages = await self.store.get_devices_with_messages( + user_id, device_ids + ) + for device_id in devices_with_messages: + # Delete device messages asynchronously and in batches using the task scheduler + # We specify an upper stream id to avoid deleting non delivered messages + # if an user re-uses a device ID. + await self._task_scheduler.schedule_task( + DELETE_DEVICE_MSGS_TASK_NAME, + resource_id=device_id, + params={ + "user_id": user_id, + "device_id": device_id, + "up_to_stream_id": to_device_stream_id, + }, + ) + + await self._auth_handler.delete_access_tokens_for_devices( + user_id, device_ids=device_ids + ) + + # Pushers are deleted after `delete_access_tokens_for_user` is called so that + # modules using `on_logged_out` hook can use them if needed. + await self.hs.get_pusherpool().remove_pushers_by_devices(user_id, device_ids) + + await self.notify_device_update(user_id, device_ids) + + async def upsert_device( + self, user_id: str, device_id: str, display_name: Optional[str] = None + ) -> bool: + """Create or update a device + + Args: + user_id: The user to update devices of. + device_id: The device to update. + display_name: The new display name for this device. + + Returns: + True if the device was created, False if it was updated. + + """ + + # Reject a new displayname which is too long. + _check_device_name_length(display_name) + + created = await self.store.store_device( + user_id, + device_id, + initial_device_display_name=display_name, + ) + + if not created: + await self.store.update_device( + user_id, + device_id, + new_display_name=display_name, + ) + + await self.notify_device_update(user_id, [device_id]) + return created + + async def update_device(self, user_id: str, device_id: str, content: dict) -> None: + """Update the given device + + Args: + user_id: The user to update devices of. + device_id: The device to update. + content: body of update request + """ + + # Reject a new displayname which is too long. + new_display_name = content.get("display_name") + + _check_device_name_length(new_display_name) + + try: + await self.store.update_device( + user_id, device_id, new_display_name=new_display_name + ) + await self.notify_device_update(user_id, [device_id]) + except errors.StoreError as e: + if e.code == 404: + raise errors.NotFoundError() + else: + raise + @trace async def get_devices_by_user(self, user_id: str) -> List[JsonDict]: """ @@ -146,6 +443,98 @@ class DeviceWorkerHandler: """ return await self.store.get_dehydrated_device(user_id) + async def store_dehydrated_device( + self, + user_id: str, + device_id: Optional[str], + device_data: JsonDict, + initial_device_display_name: Optional[str] = None, + keys_for_device: Optional[JsonDict] = None, + ) -> str: + """Store a dehydrated device for a user, optionally storing the keys associated with + it as well. If the user had a previous dehydrated device, it is removed. + + Args: + user_id: the user that we are storing the device for + device_id: device id supplied by client + device_data: the dehydrated device information + initial_device_display_name: The display name to use for the device + keys_for_device: keys for the dehydrated device + Returns: + device id of the dehydrated device + """ + device_id = await self.check_device_registered( + user_id, + device_id, + initial_device_display_name, + ) + + time_now = self.clock.time_msec() + + old_device_id = await self.store.store_dehydrated_device( + user_id, device_id, device_data, time_now, keys_for_device + ) + + if old_device_id is not None: + await self.delete_devices(user_id, [old_device_id]) + + return device_id + + async def rehydrate_device( + self, user_id: str, access_token: str, device_id: str + ) -> dict: + """Process a rehydration request from the user. + + Args: + user_id: the user who is rehydrating the device + access_token: the access token used for the request + device_id: the ID of the device that will be rehydrated + Returns: + a dict containing {"success": True} + """ + success = await self.store.remove_dehydrated_device(user_id, device_id) + + if not success: + raise errors.NotFoundError() + + # If the dehydrated device was successfully deleted (the device ID + # matched the stored dehydrated device), then modify the access + # token and refresh token to use the dehydrated device's ID and + # copy the old device display name to the dehydrated device, + # and destroy the old device ID + old_device_id = await self.store.set_device_for_access_token( + access_token, device_id + ) + await self.store.set_device_for_refresh_token(user_id, old_device_id, device_id) + old_device = await self.store.get_device(user_id, old_device_id) + if old_device is None: + raise errors.NotFoundError() + await self.store.update_device(user_id, device_id, old_device["display_name"]) + # can't call self.delete_device because that will clobber the + # access token so call the storage layer directly + await self.store.delete_devices(user_id, [old_device_id]) + + # tell everyone that the old device is gone and that the dehydrated + # device has a new display name + await self.notify_device_update(user_id, [old_device_id, device_id]) + + return {"success": True} + + async def delete_dehydrated_device(self, user_id: str, device_id: str) -> None: + """ + Delete a stored dehydrated device. + + Args: + user_id: the user_id to delete the device from + device_id: id of the dehydrated device to delete + """ + success = await self.store.remove_dehydrated_device(user_id, device_id) + + if not success: + raise errors.NotFoundError() + + await self.delete_devices(user_id, [device_id]) + @trace async def get_device(self, user_id: str, device_id: str) -> JsonDict: """Retrieve the given device @@ -484,10 +873,53 @@ class DeviceWorkerHandler: gone from partial to full state. """ - # TODO(faster_joins): worker mode support - # https://github.com/matrix-org/synapse/issues/12994 - logger.error( - "Trying handling device list state for partial join: not supported on workers." + await self._handle_room_un_partial_stated_client( + instance_name=random.choice(self._device_list_writers), + room_id=room_id, + ) + + @trace + @measure_func("notify_device_update") + async def notify_device_update( + self, user_id: str, device_ids: StrCollection + ) -> None: + """Notify that a user's device(s) has changed. Pokes the notifier, and + remote servers if the user is local. + + Args: + user_id: The Matrix ID of the user who's device list has been updated. + device_ids: The device IDs that have changed. + """ + await self._notify_device_update_client( + instance_name=random.choice(self._device_list_writers), + user_id=user_id, + device_ids=list(device_ids), + ) + + async def notify_user_signature_update( + self, + from_user_id: str, + user_ids: List[str], + ) -> None: + """Notify a device writer that a user have made new signatures of other users. + + Args: + from_user_id: The Matrix ID of the user who's signatures have been updated. + user_ids: The Matrix IDs of the users that have changed. + """ + await self._notify_user_signature_update_client( + instance_name=random.choice(self._device_list_writers), + from_user_id=from_user_id, + user_ids=user_ids, + ) + + async def handle_new_device_update(self) -> None: + """Wake up a device writer to send local device list changes as federation outbound pokes.""" + # This is only sent to the first device writer to avoid cross-worker + # locks in _handle_new_device_update_async, as it makes assumptions + # about being the only instance running. + await self._handle_new_device_update_client( + instance_name=self._device_list_writers[0], ) DEVICE_MSGS_DELETE_BATCH_LIMIT = 1000 @@ -511,39 +943,43 @@ class DeviceWorkerHandler: device_id=device_id, from_stream_id=from_stream_id, to_stream_id=up_to_stream_id, - limit=DeviceHandler.DEVICE_MSGS_DELETE_BATCH_LIMIT, + limit=DeviceWriterHandler.DEVICE_MSGS_DELETE_BATCH_LIMIT, ) if from_stream_id is None: return TaskStatus.COMPLETE, None, None - await self.clock.sleep(DeviceHandler.DEVICE_MSGS_DELETE_SLEEP_MS / 1000.0) + await self.clock.sleep( + DeviceWriterHandler.DEVICE_MSGS_DELETE_SLEEP_MS / 1000.0 + ) -class DeviceHandler(DeviceWorkerHandler): - device_list_updater: "DeviceListUpdater" +class DeviceWriterHandler(DeviceHandler): + """ + Superclass of the DeviceHandler which gets instantiated on workers that can + write to the device list stream. + """ def __init__(self, hs: "HomeServer"): super().__init__(hs) - self.server_name = hs.hostname # nb must be called this for @measure_func - self.clock = hs.get_clock() # nb must be called this for @measure_func - self.federation_sender = hs.get_federation_sender() - self._account_data_handler = hs.get_account_data_handler() + # We only need to poke the federation sender explicitly if its on the + # same instance. Other federation sender instances will get notified by + # `synapse.app.generic_worker.FederationSenderHandler` when it sees it + # in the device lists stream. + self.federation_sender = None + if hs.should_send_federation(): + self.federation_sender = hs.get_federation_sender() + self._storage_controllers = hs.get_storage_controllers() - self.db_pool = hs.get_datastores().main.db_pool - self._dont_notify_new_devices_for = ( - hs.config.registration.dont_notify_new_devices_for - ) - - self.device_list_updater = DeviceListUpdater(hs, self) - - federation_registry = hs.get_federation_registry() - - federation_registry.register_edu_handler( - EduTypes.DEVICE_LIST_UPDATE, - self.device_list_updater.incoming_device_list_update, + # There are a few things that are only handled on the main device list + # writer to avoid cross-worker locks + # + # This mainly concerns the `DeviceListUpdater` class, which is only + # instantiated on the first device list writer. + self._is_main_device_list_writer = ( + hs.get_instance_name() == self._main_device_list_writer ) # Whether `_handle_new_device_update_async` is currently processing. @@ -553,250 +989,22 @@ class DeviceHandler(DeviceWorkerHandler): # processing. self._handle_new_device_update_new_data = False - # On start up check if there are any updates pending. - hs.get_reactor().callWhenRunning(self._handle_new_device_update_async) - - self._delete_stale_devices_after = hs.config.server.delete_stale_devices_after - - # Ideally we would run this on a worker and condition this on the - # "run_background_tasks_on" setting, but this would mean making the notification - # of device list changes over federation work on workers, which is nontrivial. - if self._delete_stale_devices_after is not None: - self.clock.looping_call( - run_as_background_process, - DELETE_STALE_DEVICES_INTERVAL_MS, - "delete_stale_devices", - self._delete_stale_devices, + # Only the main device list writer handles device list EDUs and converts + # device list updates to outbound federation pokes. This allows us to + # use in-memory per-user locks instead of cross-worker locks, and + # simplifies the logic for converting outbound pokes. This makes the + # device_list writers a little bit unbalanced in terms of load, but + # still unlocks local device changes (and therefore login/logouts) when + # rolling-restarting Synapse. + if self._is_main_device_list_writer: + # On start up check if there are any updates pending. + hs.get_reactor().callWhenRunning(self._handle_new_device_update_async) + self.device_list_updater = DeviceListUpdater(hs, self) + hs.get_federation_registry().register_edu_handler( + EduTypes.DEVICE_LIST_UPDATE, + self.device_list_updater.incoming_device_list_update, ) - def _check_device_name_length(self, name: Optional[str]) -> None: - """ - Checks whether a device name is longer than the maximum allowed length. - - Args: - name: The name of the device. - - Raises: - SynapseError: if the device name is too long. - """ - if name and len(name) > MAX_DEVICE_DISPLAY_NAME_LEN: - raise SynapseError( - 400, - "Device display name is too long (max %i)" - % (MAX_DEVICE_DISPLAY_NAME_LEN,), - errcode=Codes.TOO_LARGE, - ) - - async def check_device_registered( - self, - user_id: str, - device_id: Optional[str], - initial_device_display_name: Optional[str] = None, - auth_provider_id: Optional[str] = None, - auth_provider_session_id: Optional[str] = None, - ) -> str: - """ - If the given device has not been registered, register it with the - supplied display name. - - If no device_id is supplied, we make one up. - - Args: - user_id: @user:id - device_id: device id supplied by client - initial_device_display_name: device display name from client - auth_provider_id: The SSO IdP the user used, if any. - auth_provider_session_id: The session ID (sid) got from the SSO IdP. - Returns: - device id (generated if none was supplied) - """ - - self._check_device_name_length(initial_device_display_name) - - # Check if we should send out device lists updates for this new device. - notify = user_id not in self._dont_notify_new_devices_for - - if device_id is not None: - new_device = await self.store.store_device( - user_id=user_id, - device_id=device_id, - initial_device_display_name=initial_device_display_name, - auth_provider_id=auth_provider_id, - auth_provider_session_id=auth_provider_session_id, - ) - if new_device: - if notify: - await self.notify_device_update(user_id, [device_id]) - return device_id - - # if the device id is not specified, we'll autogen one, but loop a few - # times in case of a clash. - attempts = 0 - while attempts < 5: - new_device_id = stringutils.random_string(10).upper() - new_device = await self.store.store_device( - user_id=user_id, - device_id=new_device_id, - initial_device_display_name=initial_device_display_name, - auth_provider_id=auth_provider_id, - auth_provider_session_id=auth_provider_session_id, - ) - if new_device: - if notify: - await self.notify_device_update(user_id, [new_device_id]) - return new_device_id - attempts += 1 - - raise errors.StoreError(500, "Couldn't generate a device ID.") - - async def _delete_stale_devices(self) -> None: - """Background task that deletes devices which haven't been accessed for more than - a configured time period. - """ - # We should only be running this job if the config option is defined. - assert self._delete_stale_devices_after is not None - now_ms = self.clock.time_msec() - since_ms = now_ms - self._delete_stale_devices_after - devices = await self.store.get_local_devices_not_accessed_since(since_ms) - - for user_id, user_devices in devices.items(): - await self.delete_devices(user_id, user_devices) - - @trace - async def delete_all_devices_for_user( - self, user_id: str, except_device_id: Optional[str] = None - ) -> None: - """Delete all of the user's devices - - Args: - user_id: The user to remove all devices from - except_device_id: optional device id which should not be deleted - """ - device_map = await self.store.get_devices_by_user(user_id) - if except_device_id is not None: - device_map.pop(except_device_id, None) - user_device_ids = device_map.keys() - await self.delete_devices(user_id, user_device_ids) - - async def delete_devices(self, user_id: str, device_ids: StrCollection) -> None: - """Delete several devices - - Args: - user_id: The user to delete devices from. - device_ids: The list of device IDs to delete - """ - to_device_stream_id = self._event_sources.get_current_token().to_device_key - - try: - await self.store.delete_devices(user_id, device_ids) - except errors.StoreError as e: - if e.code == 404: - # no match - set_tag("error", True) - set_tag("reason", "User doesn't have that device id.") - else: - raise - - # Delete data specific to each device. Not optimised as its an - # experimental MSC. - if self.hs.config.experimental.msc3890_enabled: - for device_id in device_ids: - # Remove any local notification settings for this device in accordance - # with MSC3890. - await self._account_data_handler.remove_account_data_for_user( - user_id, - f"org.matrix.msc3890.local_notification_settings.{device_id}", - ) - - # If we're deleting a lot of devices, a bunch of them may not have any - # to-device messages queued up. We filter those out to avoid scheduling - # unnecessary tasks. - devices_with_messages = await self.store.get_devices_with_messages( - user_id, device_ids - ) - for device_id in devices_with_messages: - # Delete device messages asynchronously and in batches using the task scheduler - # We specify an upper stream id to avoid deleting non delivered messages - # if an user re-uses a device ID. - await self._task_scheduler.schedule_task( - DELETE_DEVICE_MSGS_TASK_NAME, - resource_id=device_id, - params={ - "user_id": user_id, - "device_id": device_id, - "up_to_stream_id": to_device_stream_id, - }, - ) - - await self._auth_handler.delete_access_tokens_for_devices( - user_id, device_ids=device_ids - ) - - # Pushers are deleted after `delete_access_tokens_for_user` is called so that - # modules using `on_logged_out` hook can use them if needed. - await self.hs.get_pusherpool().remove_pushers_by_devices(user_id, device_ids) - - await self.notify_device_update(user_id, device_ids) - - async def upsert_device( - self, user_id: str, device_id: str, display_name: Optional[str] = None - ) -> bool: - """Create or update a device - - Args: - user_id: The user to update devices of. - device_id: The device to update. - display_name: The new display name for this device. - - Returns: - True if the device was created, False if it was updated. - - """ - - # Reject a new displayname which is too long. - self._check_device_name_length(display_name) - - created = await self.store.store_device( - user_id, - device_id, - initial_device_display_name=display_name, - ) - - if not created: - await self.store.update_device( - user_id, - device_id, - new_display_name=display_name, - ) - - await self.notify_device_update(user_id, [device_id]) - return created - - async def update_device(self, user_id: str, device_id: str, content: dict) -> None: - """Update the given device - - Args: - user_id: The user to update devices of. - device_id: The device to update. - content: body of update request - """ - - # Reject a new displayname which is too long. - new_display_name = content.get("display_name") - - self._check_device_name_length(new_display_name) - - try: - await self.store.update_device( - user_id, device_id, new_display_name=new_display_name - ) - await self.notify_device_update(user_id, [device_id]) - except errors.StoreError as e: - if e.code == 404: - raise errors.NotFoundError() - else: - raise - @trace @measure_func("notify_device_update") async def notify_device_update( @@ -839,7 +1047,7 @@ class DeviceHandler(DeviceWorkerHandler): # We may need to do some processing asynchronously for local user IDs. if self.hs.is_mine_id(user_id): - self._handle_new_device_update_async() + await self.handle_new_device_update() async def notify_user_signature_update( self, from_user_id: str, user_ids: List[str] @@ -859,97 +1067,16 @@ class DeviceHandler(DeviceWorkerHandler): StreamKeyType.DEVICE_LIST, position, users=[from_user_id] ) - async def store_dehydrated_device( - self, - user_id: str, - device_id: Optional[str], - device_data: JsonDict, - initial_device_display_name: Optional[str] = None, - keys_for_device: Optional[JsonDict] = None, - ) -> str: - """Store a dehydrated device for a user, optionally storing the keys associated with - it as well. If the user had a previous dehydrated device, it is removed. + async def handle_new_device_update(self) -> None: + # _handle_new_device_update_async is only called on the first device + # writer, as it makes assumptions about only having one instance running + # at a time. If this is not the first device writer, we defer to the + # superclass, which will make the call go through replication. + if not self._is_main_device_list_writer: + return await super().handle_new_device_update() - Args: - user_id: the user that we are storing the device for - device_id: device id supplied by client - device_data: the dehydrated device information - initial_device_display_name: The display name to use for the device - keys_for_device: keys for the dehydrated device - Returns: - device id of the dehydrated device - """ - device_id = await self.check_device_registered( - user_id, - device_id, - initial_device_display_name, - ) - - time_now = self.clock.time_msec() - - old_device_id = await self.store.store_dehydrated_device( - user_id, device_id, device_data, time_now, keys_for_device - ) - - if old_device_id is not None: - await self.delete_devices(user_id, [old_device_id]) - - return device_id - - async def rehydrate_device( - self, user_id: str, access_token: str, device_id: str - ) -> dict: - """Process a rehydration request from the user. - - Args: - user_id: the user who is rehydrating the device - access_token: the access token used for the request - device_id: the ID of the device that will be rehydrated - Returns: - a dict containing {"success": True} - """ - success = await self.store.remove_dehydrated_device(user_id, device_id) - - if not success: - raise errors.NotFoundError() - - # If the dehydrated device was successfully deleted (the device ID - # matched the stored dehydrated device), then modify the access - # token and refresh token to use the dehydrated device's ID and - # copy the old device display name to the dehydrated device, - # and destroy the old device ID - old_device_id = await self.store.set_device_for_access_token( - access_token, device_id - ) - await self.store.set_device_for_refresh_token(user_id, old_device_id, device_id) - old_device = await self.store.get_device(user_id, old_device_id) - if old_device is None: - raise errors.NotFoundError() - await self.store.update_device(user_id, device_id, old_device["display_name"]) - # can't call self.delete_device because that will clobber the - # access token so call the storage layer directly - await self.store.delete_devices(user_id, [old_device_id]) - - # tell everyone that the old device is gone and that the dehydrated - # device has a new display name - await self.notify_device_update(user_id, [old_device_id, device_id]) - - return {"success": True} - - async def delete_dehydrated_device(self, user_id: str, device_id: str) -> None: - """ - Delete a stored dehydrated device. - - Args: - user_id: the user_id to delete the device from - device_id: id of the dehydrated device to delete - """ - success = await self.store.remove_dehydrated_device(user_id, device_id) - - if not success: - raise errors.NotFoundError() - - await self.delete_devices(user_id, [device_id]) + self._handle_new_device_update_async() + return @wrap_as_background_process("_handle_new_device_update_async") async def _handle_new_device_update_async(self) -> None: @@ -959,12 +1086,27 @@ class DeviceHandler(DeviceWorkerHandler): This happens in the background so as not to block the original request that generated the device update. """ + # This should only ever be called on the main device list writer, as it + # expects to only have a single instance of this loop running at a time. + # See `handle_new_device_update`. + assert self._is_main_device_list_writer + if self._handle_new_device_update_is_processing: self._handle_new_device_update_new_data = True return self._handle_new_device_update_is_processing = True + # Note that this logic only deals with the minimum stream ID, and not + # the full stream token. This means that oubound pokes are only sent + # once every writer on the device_lists stream has caught up. This is + # fine, it may only introduces a bit of lag on the outbound pokes. + # To fix this, 'device_lists_changes_converted_stream_position' would + # need to include the full stream token instead of just a stream ID. + # We could also consider have each writer converting their own device + # list updates, but that can quickly become complex to handle changes in + # the list of device writers. + # The stream ID we processed previous iteration (if any), and the set of # hosts we've already poked about for this update. This is so that we # don't poke the same remote server about the same update repeatedly. @@ -976,7 +1118,7 @@ class DeviceHandler(DeviceWorkerHandler): while True: self._handle_new_device_update_new_data = False - max_stream_id = self.store.get_device_stream_token() + max_stream_id = self.store.get_device_stream_token().stream rows = await self.store.get_uncoverted_outbound_room_pokes( stream_id, room_id ) @@ -1041,7 +1183,7 @@ class DeviceHandler(DeviceWorkerHandler): # Notify replication that we've updated the device list stream. self.notifier.notify_replication() - if hosts: + if hosts and self.federation_sender: logger.info( "Sending device list update notif for %r to: %r", user_id, @@ -1161,9 +1303,10 @@ class DeviceHandler(DeviceWorkerHandler): # Notify things that device lists need to be sent out. self.notifier.notify_replication() - await self.federation_sender.send_device_messages( - potentially_changed_hosts, immediate=False - ) + if self.federation_sender: + await self.federation_sender.send_device_messages( + potentially_changed_hosts, immediate=False + ) def _update_device_from_client_ips( @@ -1180,19 +1323,21 @@ def _update_device_from_client_ips( class DeviceListWorkerUpdater: - "Handles incoming device list updates from federation and contacts the main process over replication" + "Handles incoming device list updates from federation and contacts the main device list writer over replication" def __init__(self, hs: "HomeServer"): - from synapse.replication.http.devices import ( - ReplicationMultiUserDevicesResyncRestServlet, - ) - + self.store = hs.get_datastores().main + self._notifier = hs.get_notifier() + # On which instance the DeviceListUpdater is running + # Must be kept in sync with DeviceHandler + self._main_device_list_writer = hs.config.worker.writers.device_lists[0] self._multi_user_device_resync_client = ( ReplicationMultiUserDevicesResyncRestServlet.make_client(hs) ) async def multi_user_device_resync( - self, user_ids: List[str], mark_failed_as_stale: bool = True + self, + user_ids: List[str], ) -> Dict[str, Optional[JsonMapping]]: """ Like `user_device_resync` but operates on multiple users **from the same origin** @@ -1201,27 +1346,104 @@ class DeviceListWorkerUpdater: Returns: Dict from User ID to the same Dict as `user_device_resync`. """ - # mark_failed_as_stale is not sent. Ensure this doesn't break expectations. - assert mark_failed_as_stale if not user_ids: # Shortcut empty requests return {} - return await self._multi_user_device_resync_client(user_ids=user_ids) + # This uses a per-user-id lock; to avoid using cross-worker locks, we + # forward the request to the main device list writer. + # See DeviceListUpdater + return await self._multi_user_device_resync_client( + instance_name=self._main_device_list_writer, + user_ids=user_ids, + ) + + async def process_cross_signing_key_update( + self, + user_id: str, + master_key: Optional[JsonDict], + self_signing_key: Optional[JsonDict], + ) -> List[str]: + """Process the given new master and self-signing key for the given remote user. + + Args: + user_id: The ID of the user these keys are for. + master_key: The dict of the cross-signing master key as returned by the + remote server. + self_signing_key: The dict of the cross-signing self-signing key as returned + by the remote server. + + Return: + The device IDs for the given keys. + """ + device_ids = [] + + current_keys_map = await self.store.get_e2e_cross_signing_keys_bulk([user_id]) + current_keys = current_keys_map.get(user_id) or {} + + if master_key and master_key != current_keys.get("master"): + await self.store.set_e2e_cross_signing_key(user_id, "master", master_key) + _, verify_key = get_verify_key_from_cross_signing_key(master_key) + # verify_key is a VerifyKey from signedjson, which uses + # .version to denote the portion of the key ID after the + # algorithm and colon, which is the device ID + device_ids.append(verify_key.version) + if self_signing_key and self_signing_key != current_keys.get("self_signing"): + await self.store.set_e2e_cross_signing_key( + user_id, "self_signing", self_signing_key + ) + _, verify_key = get_verify_key_from_cross_signing_key(self_signing_key) + device_ids.append(verify_key.version) + + return device_ids + + async def handle_room_un_partial_stated(self, room_id: str) -> None: + """Handles sending appropriate device list updates in a room that has + gone from partial to full state. + """ + + pending_updates = ( + await self.store.get_pending_remote_device_list_updates_for_room(room_id) + ) + + for user_id, device_id in pending_updates: + logger.info( + "Got pending device list update in room %s: %s / %s", + room_id, + user_id, + device_id, + ) + position = await self.store.add_device_change_to_streams( + user_id, + [device_id], + room_ids=[room_id], + ) + + if not position: + # This should only happen if there are no updates, which + # shouldn't happen when we've passed in a non-empty set of + # device IDs. + continue + + self._notifier.on_new_event( + StreamKeyType.DEVICE_LIST, position, rooms=[room_id] + ) class DeviceListUpdater(DeviceListWorkerUpdater): - "Handles incoming device list updates from federation and updates the DB" + """Handles incoming device list updates from federation and updates the DB. + + This is only instanciated on the first device list writer, as it uses + in-process linearizers for some operations.""" + + def __init__(self, hs: "HomeServer", device_handler: DeviceWriterHandler): + super().__init__(hs) - def __init__(self, hs: "HomeServer", device_handler: DeviceHandler): - self.server_name = hs.hostname - self.store = hs.get_datastores().main self.federation = hs.get_federation_client() self.server_name = hs.hostname # nb must be called this for @measure_func self.clock = hs.get_clock() # nb must be called this for @measure_func self.device_handler = device_handler - self._notifier = hs.get_notifier() self._remote_edu_linearizer = Linearizer(name="remote_device_list") self._resync_linearizer = Linearizer(name="remote_device_resync") @@ -1646,74 +1868,3 @@ class DeviceListUpdater(DeviceListWorkerUpdater): self._seen_updates[user_id] = {stream_id} return result, False - - async def process_cross_signing_key_update( - self, - user_id: str, - master_key: Optional[JsonDict], - self_signing_key: Optional[JsonDict], - ) -> List[str]: - """Process the given new master and self-signing key for the given remote user. - - Args: - user_id: The ID of the user these keys are for. - master_key: The dict of the cross-signing master key as returned by the - remote server. - self_signing_key: The dict of the cross-signing self-signing key as returned - by the remote server. - - Return: - The device IDs for the given keys. - """ - device_ids = [] - - current_keys_map = await self.store.get_e2e_cross_signing_keys_bulk([user_id]) - current_keys = current_keys_map.get(user_id) or {} - - if master_key and master_key != current_keys.get("master"): - await self.store.set_e2e_cross_signing_key(user_id, "master", master_key) - _, verify_key = get_verify_key_from_cross_signing_key(master_key) - # verify_key is a VerifyKey from signedjson, which uses - # .version to denote the portion of the key ID after the - # algorithm and colon, which is the device ID - device_ids.append(verify_key.version) - if self_signing_key and self_signing_key != current_keys.get("self_signing"): - await self.store.set_e2e_cross_signing_key( - user_id, "self_signing", self_signing_key - ) - _, verify_key = get_verify_key_from_cross_signing_key(self_signing_key) - device_ids.append(verify_key.version) - - return device_ids - - async def handle_room_un_partial_stated(self, room_id: str) -> None: - """Handles sending appropriate device list updates in a room that has - gone from partial to full state. - """ - - pending_updates = ( - await self.store.get_pending_remote_device_list_updates_for_room(room_id) - ) - - for user_id, device_id in pending_updates: - logger.info( - "Got pending device list update in room %s: %s / %s", - room_id, - user_id, - device_id, - ) - position = await self.store.add_device_change_to_streams( - user_id, - [device_id], - room_ids=[room_id], - ) - - if not position: - # This should only happen if there are no updates, which - # shouldn't happen when we've passed in a non-empty set of - # device IDs. - continue - - self.device_handler.notifier.on_new_event( - StreamKeyType.DEVICE_LIST, position, rooms=[room_id] - ) diff --git a/synapse/handlers/devicemessage.py b/synapse/handlers/devicemessage.py index e56bdb4072..b43cbd9c15 100644 --- a/synapse/handlers/devicemessage.py +++ b/synapse/handlers/devicemessage.py @@ -33,9 +33,6 @@ from synapse.logging.opentracing import ( log_kv, set_tag, ) -from synapse.replication.http.devices import ( - ReplicationMultiUserDevicesResyncRestServlet, -) from synapse.types import JsonDict, Requester, StreamKeyType, UserID, get_domain_from_id from synapse.util import json_encoder from synapse.util.stringutils import random_string @@ -56,9 +53,9 @@ class DeviceMessageHandler: self.store = hs.get_datastores().main self.notifier = hs.get_notifier() self.is_mine = hs.is_mine + self.device_handler = hs.get_device_handler() if hs.config.experimental.msc3814_enabled: self.event_sources = hs.get_event_sources() - self.device_handler = hs.get_device_handler() # We only need to poke the federation sender explicitly if its on the # same instance. Other federation sender instances will get notified by @@ -80,18 +77,6 @@ class DeviceMessageHandler: hs.config.worker.writers.to_device, ) - # The handler to call when we think a user's device list might be out of - # sync. We do all device list resyncing on the master instance, so if - # we're on a worker we hit the device resync replication API. - if hs.config.worker.worker_app is None: - self._multi_user_device_resync = ( - hs.get_device_handler().device_list_updater.multi_user_device_resync - ) - else: - self._multi_user_device_resync = ( - ReplicationMultiUserDevicesResyncRestServlet.make_client(hs) - ) - # a rate limiter for room key requests. The keys are # (sending_user_id, sending_device_id). self._ratelimiter = Ratelimiter( @@ -213,7 +198,10 @@ class DeviceMessageHandler: await self.store.mark_remote_users_device_caches_as_stale((sender_user_id,)) # Immediately attempt a resync in the background - run_in_background(self._multi_user_device_resync, user_ids=[sender_user_id]) + run_in_background( + self.device_handler.device_list_updater.multi_user_device_resync, + user_ids=[sender_user_id], + ) async def send_device_message( self, diff --git a/synapse/handlers/e2e_keys.py b/synapse/handlers/e2e_keys.py index 6171aaf29f..b9abad2188 100644 --- a/synapse/handlers/e2e_keys.py +++ b/synapse/handlers/e2e_keys.py @@ -32,10 +32,9 @@ from twisted.internet import defer from synapse.api.constants import EduTypes from synapse.api.errors import CodeMessageException, Codes, NotFoundError, SynapseError -from synapse.handlers.device import DeviceHandler +from synapse.handlers.device import DeviceWriterHandler from synapse.logging.context import make_deferred_yieldable, run_in_background from synapse.logging.opentracing import log_kv, set_tag, tag_args, trace -from synapse.replication.http.devices import ReplicationUploadKeysForUserRestServlet from synapse.types import ( JsonDict, JsonMapping, @@ -76,8 +75,10 @@ class E2eKeysHandler: federation_registry = hs.get_federation_registry() - is_master = hs.config.worker.worker_app is None - if is_master: + # Only the first writer in the list should handle EDUs for signing key + # updates, so that we can use an in-memory linearizer instead of worker locks. + edu_writer = hs.config.worker.writers.device_lists[0] + if hs.get_instance_name() == edu_writer: edu_updater = SigningKeyEduUpdater(hs) # Only register this edu handler on master as it requires writing @@ -92,11 +93,14 @@ class E2eKeysHandler: EduTypes.UNSTABLE_SIGNING_KEY_UPDATE, edu_updater.incoming_signing_key_update, ) - - self.device_key_uploader = self.upload_device_keys_for_user else: - self.device_key_uploader = ( - ReplicationUploadKeysForUserRestServlet.make_client(hs) + federation_registry.register_instances_for_edu( + EduTypes.SIGNING_KEY_UPDATE, + [edu_writer], + ) + federation_registry.register_instances_for_edu( + EduTypes.UNSTABLE_SIGNING_KEY_UPDATE, + [edu_writer], ) # doesn't really work as part of the generic query API, because the @@ -847,7 +851,7 @@ class E2eKeysHandler: # TODO: Validate the JSON to make sure it has the right keys. device_keys = keys.get("device_keys", None) if device_keys: - await self.device_key_uploader( + await self.upload_device_keys_for_user( user_id=user_id, device_id=device_id, keys={"device_keys": device_keys}, @@ -904,9 +908,6 @@ class E2eKeysHandler: device_keys: the `device_keys` of an /keys/upload request. """ - # This can only be called from the main process. - assert isinstance(self.device_handler, DeviceHandler) - time_now = self.clock.time_msec() device_keys = keys["device_keys"] @@ -998,9 +999,6 @@ class E2eKeysHandler: user_id: the user uploading the keys keys: the signing keys """ - # This can only be called from the main process. - assert isinstance(self.device_handler, DeviceHandler) - # if a master key is uploaded, then check it. Otherwise, load the # stored master key, to check signatures on other keys if "master_key" in keys: @@ -1091,9 +1089,6 @@ class E2eKeysHandler: Raises: SynapseError: if the signatures dict is not valid. """ - # This can only be called from the main process. - assert isinstance(self.device_handler, DeviceHandler) - failures = {} # signatures to be stored. Each item will be a SignatureListItem @@ -1467,9 +1462,6 @@ class E2eKeysHandler: A tuple of the retrieved key content, the key's ID and the matching VerifyKey. If the key cannot be retrieved, all values in the tuple will instead be None. """ - # This can only be called from the main process. - assert isinstance(self.device_handler, DeviceHandler) - try: remote_result = await self.federation.query_user_devices( user.domain, user.to_string() @@ -1770,7 +1762,7 @@ class SigningKeyEduUpdater: self.clock = hs.get_clock() device_handler = hs.get_device_handler() - assert isinstance(device_handler, DeviceHandler) + assert isinstance(device_handler, DeviceWriterHandler) self._device_handler = device_handler self._remote_edu_linearizer = Linearizer(name="remote_signing_key") diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index f7806e67a9..c709ed2c63 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -698,10 +698,19 @@ class FederationHandler: # We may want to reset the partial state info if it's from an # old, failed partial state join. # https://github.com/matrix-org/synapse/issues/13000 + + # FIXME: Ideally, we would store the full stream token here + # not just the minimum stream ID, so that we can compute an + # accurate list of device changes when un-partial-ing the + # room. The only side effect of this is that we may send + # extra unecessary device list outbound pokes through + # federation, which is harmless. + device_lists_stream_id = self.store.get_device_stream_token().stream + await self.store.store_partial_state_room( room_id=room_id, servers=ret.servers_in_room, - device_lists_stream_id=self.store.get_device_stream_token(), + device_lists_stream_id=device_lists_stream_id, joined_via=origin, ) diff --git a/synapse/handlers/federation_event.py b/synapse/handlers/federation_event.py index 1e738f484f..5cec2b01e5 100644 --- a/synapse/handlers/federation_event.py +++ b/synapse/handlers/federation_event.py @@ -77,9 +77,6 @@ from synapse.logging.opentracing import ( trace, ) from synapse.metrics.background_process_metrics import run_as_background_process -from synapse.replication.http.devices import ( - ReplicationMultiUserDevicesResyncRestServlet, -) from synapse.replication.http.federation import ( ReplicationFederationSendEventsRestServlet, ) @@ -180,12 +177,7 @@ class FederationEventHandler: self._ephemeral_messages_enabled = hs.config.server.enable_ephemeral_messages self._send_events = ReplicationFederationSendEventsRestServlet.make_client(hs) - if hs.config.worker.worker_app: - self._multi_user_device_resync = ( - ReplicationMultiUserDevicesResyncRestServlet.make_client(hs) - ) - else: - self._device_list_updater = hs.get_device_handler().device_list_updater + self._device_list_updater = hs.get_device_handler().device_list_updater # When joining a room we need to queue any events for that room up. # For each room, a list of (pdu, origin) tuples. @@ -1544,12 +1536,7 @@ class FederationEventHandler: await self._store.mark_remote_users_device_caches_as_stale((sender,)) # Immediately attempt a resync in the background - if self._config.worker.worker_app: - await self._multi_user_device_resync(user_ids=[sender]) - else: - await self._device_list_updater.multi_user_device_resync( - user_ids=[sender] - ) + await self._device_list_updater.multi_user_device_resync(user_ids=[sender]) except Exception: logger.exception("Failed to resync device for %s", sender) diff --git a/synapse/handlers/register.py b/synapse/handlers/register.py index 1e1f0c79c8..6322d980d4 100644 --- a/synapse/handlers/register.py +++ b/synapse/handlers/register.py @@ -44,7 +44,6 @@ from synapse.api.errors import ( ) from synapse.appservice import ApplicationService from synapse.config.server import is_threepid_reserved -from synapse.handlers.device import DeviceHandler from synapse.http.servlet import assert_params_in_dict from synapse.replication.http.login import RegisterDeviceReplicationServlet from synapse.replication.http.register import ( @@ -840,9 +839,6 @@ class RegistrationHandler: refresh_token = None refresh_token_id = None - # This can only run on the main process. - assert isinstance(self.device_handler, DeviceHandler) - registered_device_id = await self.device_handler.check_device_registered( user_id, device_id, diff --git a/synapse/handlers/set_password.py b/synapse/handlers/set_password.py index 94301add9e..54116a9b72 100644 --- a/synapse/handlers/set_password.py +++ b/synapse/handlers/set_password.py @@ -21,7 +21,6 @@ import logging from typing import TYPE_CHECKING, Optional from synapse.api.errors import Codes, StoreError, SynapseError -from synapse.handlers.device import DeviceHandler from synapse.types import Requester if TYPE_CHECKING: @@ -36,17 +35,7 @@ class SetPasswordHandler: def __init__(self, hs: "HomeServer"): self.store = hs.get_datastores().main self._auth_handler = hs.get_auth_handler() - - # We don't need the device handler if password changing is disabled. - # This allows us to instantiate the SetPasswordHandler on the workers - # that have admin APIs for MAS - if self._auth_handler.can_change_password(): - # This can only be instantiated on the main process. - device_handler = hs.get_device_handler() - assert isinstance(device_handler, DeviceHandler) - self._device_handler: Optional[DeviceHandler] = device_handler - else: - self._device_handler = None + self._device_handler = hs.get_device_handler() async def set_password( self, @@ -58,9 +47,6 @@ class SetPasswordHandler: if not self._auth_handler.can_change_password(): raise SynapseError(403, "Password change disabled", errcode=Codes.FORBIDDEN) - # We should have this available only if password changing is enabled. - assert self._device_handler is not None - try: await self.store.user_set_password_hash(user_id, password_hash) except StoreError as e: diff --git a/synapse/handlers/sso.py b/synapse/handlers/sso.py index 0dd64618e0..48f7ba094e 100644 --- a/synapse/handlers/sso.py +++ b/synapse/handlers/sso.py @@ -46,7 +46,6 @@ from twisted.web.server import Request from synapse.api.constants import LoginType, ProfileFields from synapse.api.errors import Codes, NotFoundError, RedirectException, SynapseError from synapse.config.sso import SsoAttributeRequirement -from synapse.handlers.device import DeviceHandler from synapse.handlers.register import init_counters_for_auth_provider from synapse.handlers.ui_auth import UIAuthSessionDataConstants from synapse.http import get_request_user_agent @@ -1181,8 +1180,6 @@ class SsoHandler: ) -> None: """Revoke any devices and in-flight logins tied to a provider session. - Can only be called from the main process. - Args: auth_provider_id: A unique identifier for this SSO provider, e.g. "oidc" or "saml". @@ -1191,11 +1188,6 @@ class SsoHandler: sessions belonging to other users and log an error. """ - # It is expected that this is the main process. - assert isinstance(self._device_handler, DeviceHandler), ( - "revoking SSO sessions can only be called on the main process" - ) - # Invalidate any running user-mapping sessions to_delete = [] for session_id, session in self._username_mapping_sessions.items(): diff --git a/synapse/module_api/__init__.py b/synapse/module_api/__init__.py index dd58d3aedc..f039cd54c3 100644 --- a/synapse/module_api/__init__.py +++ b/synapse/module_api/__init__.py @@ -66,7 +66,6 @@ from synapse.handlers.auth import ( ON_LOGGED_OUT_CALLBACK, AuthHandler, ) -from synapse.handlers.device import DeviceHandler from synapse.handlers.push_rules import RuleSpec, check_actions from synapse.http.client import SimpleHttpClient from synapse.http.server import ( @@ -925,8 +924,6 @@ class ModuleApi: ) -> Generator["defer.Deferred[Any]", Any, None]: """Invalidate an access token for a user - Can only be called from the main process. - Added in Synapse v0.25.0. Args: @@ -939,10 +936,6 @@ class ModuleApi: Raises: synapse.api.errors.AuthError: the access token is invalid """ - assert isinstance(self._device_handler, DeviceHandler), ( - "invalidate_access_token can only be called on the main process" - ) - # see if the access token corresponds to a device user_info = yield defer.ensureDeferred( self._auth.get_user_by_access_token(access_token) diff --git a/synapse/replication/http/__init__.py b/synapse/replication/http/__init__.py index d500051714..555444fa3d 100644 --- a/synapse/replication/http/__init__.py +++ b/synapse/replication/http/__init__.py @@ -59,10 +59,10 @@ class ReplicationRestResource(JsonResource): account_data.register_servlets(hs, self) push.register_servlets(hs, self) state.register_servlets(hs, self) + devices.register_servlets(hs, self) # The following can't currently be instantiated on workers. if hs.config.worker.worker_app is None: login.register_servlets(hs, self) register.register_servlets(hs, self) - devices.register_servlets(hs, self) delayed_events.register_servlets(hs, self) diff --git a/synapse/replication/http/devices.py b/synapse/replication/http/devices.py index 08cf9eff97..974d83bb8b 100644 --- a/synapse/replication/http/devices.py +++ b/synapse/replication/http/devices.py @@ -34,6 +34,92 @@ if TYPE_CHECKING: logger = logging.getLogger(__name__) +class ReplicationNotifyDeviceUpdateRestServlet(ReplicationEndpoint): + """Notify a device writer that a user's device list has changed. + + Request format: + + POST /_synapse/replication/notify_device_update/:user_id + + { + "device_ids": ["JLAFKJWSCS", "JLAFKJWSCS"] + } + """ + + NAME = "notify_device_update" + PATH_ARGS = ("user_id",) + CACHE = False + + def __init__(self, hs: "HomeServer"): + super().__init__(hs) + + self.device_handler = hs.get_device_handler() + self.store = hs.get_datastores().main + self.clock = hs.get_clock() + + @staticmethod + async def _serialize_payload( # type: ignore[override] + user_id: str, device_ids: List[str] + ) -> JsonDict: + return {"device_ids": device_ids} + + async def _handle_request( # type: ignore[override] + self, request: Request, content: JsonDict, user_id: str + ) -> Tuple[int, JsonDict]: + device_ids = content["device_ids"] + + span = active_span() + if span: + span.set_tag("user_id", user_id) + span.set_tag("device_ids", f"{device_ids!r}") + + await self.device_handler.notify_device_update(user_id, device_ids) + + return 200, {} + + +class ReplicationNotifyUserSignatureUpdateRestServlet(ReplicationEndpoint): + """Notify a device writer that a user have made new signatures of other users. + + Request format: + + POST /_synapse/replication/notify_user_signature_update/:from_user_id + + { + "user_ids": ["@alice:example.org", "@bob:example.org", ...] + } + """ + + NAME = "notify_user_signature_update" + PATH_ARGS = ("from_user_id",) + CACHE = False + + def __init__(self, hs: "HomeServer"): + super().__init__(hs) + + self.device_handler = hs.get_device_handler() + self.store = hs.get_datastores().main + self.clock = hs.get_clock() + + @staticmethod + async def _serialize_payload(from_user_id: str, user_ids: List[str]) -> JsonDict: # type: ignore[override] + return {"user_ids": user_ids} + + async def _handle_request( # type: ignore[override] + self, request: Request, content: JsonDict, from_user_id: str + ) -> Tuple[int, JsonDict]: + user_ids = content["user_ids"] + + span = active_span() + if span: + span.set_tag("from_user_id", from_user_id) + span.set_tag("user_ids", f"{user_ids!r}") + + await self.device_handler.notify_user_signature_update(from_user_id, user_ids) + + return 200, {} + + class ReplicationMultiUserDevicesResyncRestServlet(ReplicationEndpoint): """Ask master to resync the device list for multiple users from the same remote server by contacting their server. @@ -73,11 +159,7 @@ class ReplicationMultiUserDevicesResyncRestServlet(ReplicationEndpoint): def __init__(self, hs: "HomeServer"): super().__init__(hs) - from synapse.handlers.device import DeviceHandler - - handler = hs.get_device_handler() - assert isinstance(handler, DeviceHandler) - self.device_list_updater = handler.device_list_updater + self.device_list_updater = hs.get_device_handler().device_list_updater self.store = hs.get_datastores().main self.clock = hs.get_clock() @@ -103,32 +185,10 @@ class ReplicationMultiUserDevicesResyncRestServlet(ReplicationEndpoint): return 200, multi_user_devices +# FIXME(2025-07-22): Remove this on the next release, this will only get used +# during rollout to Synapse 1.135 and can be removed after that release. class ReplicationUploadKeysForUserRestServlet(ReplicationEndpoint): - """Ask master to upload keys for the user and send them out over federation to - update other servers. - - For now, only the master is permitted to handle key upload requests; - any worker can handle key query requests (since they're read-only). - - Calls to e2e_keys_handler.upload_keys_for_user(user_id, device_id, keys) on - the main process to accomplish this. - - Request format for this endpoint (borrowed and expanded from KeyUploadServlet): - - POST /_synapse/replication/upload_keys_for_user - - { - "user_id": "", - "device_id": "", - "keys": { - ....this part can be found in KeyUploadServlet in rest/client/keys.py.... - or as defined in https://spec.matrix.org/v1.4/client-server-api/#post_matrixclientv3keysupload - } - } - - Response is equivalent to ` /_matrix/client/v3/keys/upload` found in KeyUploadServlet - - """ + """Unused endpoint, kept for backwards compatibility during rollout.""" NAME = "upload_keys_for_user" PATH_ARGS = () @@ -165,6 +225,71 @@ class ReplicationUploadKeysForUserRestServlet(ReplicationEndpoint): return 200, results +class ReplicationHandleNewDeviceUpdateRestServlet(ReplicationEndpoint): + """Wake up a device writer to send local device list changes as federation outbound pokes. + + Request format: + + POST /_synapse/replication/handle_new_device_update + + {} + """ + + NAME = "handle_new_device_update" + PATH_ARGS = () + CACHE = False + + def __init__(self, hs: "HomeServer"): + super().__init__(hs) + + self.device_handler = hs.get_device_handler() + + @staticmethod + async def _serialize_payload() -> JsonDict: # type: ignore[override] + return {} + + async def _handle_request( # type: ignore[override] + self, request: Request, content: JsonDict + ) -> Tuple[int, JsonDict]: + await self.device_handler.handle_new_device_update() + return 200, {} + + +class ReplicationDeviceHandleRoomUnPartialStated(ReplicationEndpoint): + """Handles sending appropriate device list updates in a room that has + gone from partial to full state. + + Request format: + + POST /_synapse/replication/device_handle_room_un_partial_stated/:room_id + + {} + """ + + NAME = "device_handle_room_un_partial_stated" + PATH_ARGS = ("room_id",) + CACHE = True + + def __init__(self, hs: "HomeServer"): + super().__init__(hs) + + self.device_handler = hs.get_device_handler() + + @staticmethod + async def _serialize_payload(room_id: str) -> JsonDict: # type: ignore[override] + return {} + + async def _handle_request( # type: ignore[override] + self, request: Request, content: JsonDict, room_id: str + ) -> Tuple[int, JsonDict]: + await self.device_handler.handle_room_un_partial_stated(room_id) + return 200, {} + + def register_servlets(hs: "HomeServer", http_server: HttpServer) -> None: + ReplicationNotifyDeviceUpdateRestServlet(hs).register(http_server) + ReplicationNotifyUserSignatureUpdateRestServlet(hs).register(http_server) ReplicationMultiUserDevicesResyncRestServlet(hs).register(http_server) + ReplicationHandleNewDeviceUpdateRestServlet(hs).register(http_server) ReplicationUploadKeysForUserRestServlet(hs).register(http_server) + ReplicationDeviceHandleRoomUnPartialStated(hs).register(http_server) diff --git a/synapse/replication/tcp/client.py b/synapse/replication/tcp/client.py index e71588f3de..b99f11f7c6 100644 --- a/synapse/replication/tcp/client.py +++ b/synapse/replication/tcp/client.py @@ -116,7 +116,11 @@ class ReplicationDataHandler: all_room_ids: Set[str] = set() if stream_name == DeviceListsStream.NAME: if any(not row.is_signature and not row.hosts_calculated for row in rows): - prev_token = self.store.get_device_stream_token() + # This only uses the minimum stream position on the device lists + # stream, which means that we may process a device list change + # twice in case of concurrent writes. This is fine, as this only + # triggers cache invalidation, which is harmless if done twice. + prev_token = self.store.get_device_stream_token().stream all_room_ids = await self.store.get_all_device_list_changes( prev_token, token ) diff --git a/synapse/replication/tcp/handler.py b/synapse/replication/tcp/handler.py index 1fafbb48c3..e434bed3e5 100644 --- a/synapse/replication/tcp/handler.py +++ b/synapse/replication/tcp/handler.py @@ -72,6 +72,7 @@ from synapse.replication.tcp.streams import ( ToDeviceStream, TypingStream, ) +from synapse.replication.tcp.streams._base import DeviceListsStream if TYPE_CHECKING: from synapse.server import HomeServer @@ -185,6 +186,12 @@ class ReplicationCommandHandler: continue + if isinstance(stream, DeviceListsStream): + if hs.get_instance_name() in hs.config.worker.writers.device_lists: + self._streams_to_replicate.append(stream) + + continue + # Only add any other streams if we're on master. if hs.config.worker.worker_app is not None: continue diff --git a/synapse/rest/admin/__init__.py b/synapse/rest/admin/__init__.py index e55cdc0470..32df4b244c 100644 --- a/synapse/rest/admin/__init__.py +++ b/synapse/rest/admin/__init__.py @@ -51,7 +51,6 @@ from synapse.rest.admin.background_updates import ( from synapse.rest.admin.devices import ( DeleteDevicesRestServlet, DeviceRestServlet, - DevicesGetRestServlet, DevicesRestServlet, ) from synapse.rest.admin.event_reports import ( @@ -375,4 +374,5 @@ def register_servlets_for_msc3861_delegation( UserRestServletV2(hs).register(http_server) UsernameAvailableRestServlet(hs).register(http_server) UserReplaceMasterCrossSigningKeyRestServlet(hs).register(http_server) - DevicesGetRestServlet(hs).register(http_server) + DeviceRestServlet(hs).register(http_server) + DevicesRestServlet(hs).register(http_server) diff --git a/synapse/rest/admin/devices.py b/synapse/rest/admin/devices.py index 09baf8ce21..c488bce58e 100644 --- a/synapse/rest/admin/devices.py +++ b/synapse/rest/admin/devices.py @@ -23,7 +23,6 @@ from http import HTTPStatus from typing import TYPE_CHECKING, Tuple from synapse.api.errors import NotFoundError, SynapseError -from synapse.handlers.device import DeviceHandler from synapse.http.servlet import ( RestServlet, assert_params_in_dict, @@ -51,9 +50,7 @@ class DeviceRestServlet(RestServlet): def __init__(self, hs: "HomeServer"): super().__init__() self.auth = hs.get_auth() - handler = hs.get_device_handler() - assert isinstance(handler, DeviceHandler) - self.device_handler = handler + self.device_handler = hs.get_device_handler() self.store = hs.get_datastores().main self.is_mine = hs.is_mine @@ -113,7 +110,7 @@ class DeviceRestServlet(RestServlet): return HTTPStatus.OK, {} -class DevicesGetRestServlet(RestServlet): +class DevicesRestServlet(RestServlet): """ Retrieve the given user's devices @@ -158,19 +155,6 @@ class DevicesGetRestServlet(RestServlet): return HTTPStatus.OK, {"devices": devices, "total": len(devices)} - -class DevicesRestServlet(DevicesGetRestServlet): - """ - Retrieve the given user's devices - """ - - PATTERNS = admin_patterns("/users/(?P[^/]*)/devices$", "v2") - - def __init__(self, hs: "HomeServer"): - super().__init__(hs) - assert isinstance(self.device_worker_handler, DeviceHandler) - self.device_handler = self.device_worker_handler - async def on_POST( self, request: SynapseRequest, user_id: str ) -> Tuple[int, JsonDict]: @@ -194,7 +178,7 @@ class DevicesRestServlet(DevicesGetRestServlet): if not isinstance(device_id, str): raise SynapseError(HTTPStatus.BAD_REQUEST, "device_id must be a string") - await self.device_handler.check_device_registered( + await self.device_worker_handler.check_device_registered( user_id=user_id, device_id=device_id ) @@ -211,9 +195,7 @@ class DeleteDevicesRestServlet(RestServlet): def __init__(self, hs: "HomeServer"): self.auth = hs.get_auth() - handler = hs.get_device_handler() - assert isinstance(handler, DeviceHandler) - self.device_handler = handler + self.device_handler = hs.get_device_handler() self.store = hs.get_datastores().main self.is_mine = hs.is_mine diff --git a/synapse/rest/client/devices.py b/synapse/rest/client/devices.py index 0b075cc2f2..5667af20d4 100644 --- a/synapse/rest/client/devices.py +++ b/synapse/rest/client/devices.py @@ -27,7 +27,6 @@ from typing import TYPE_CHECKING, List, Optional, Tuple from synapse._pydantic_compat import Extra, StrictStr from synapse.api import errors from synapse.api.errors import NotFoundError, SynapseError, UnrecognizedRequestError -from synapse.handlers.device import DeviceHandler from synapse.http.server import HttpServer from synapse.http.servlet import ( RestServlet, @@ -91,7 +90,6 @@ class DeleteDevicesRestServlet(RestServlet): self.hs = hs self.auth = hs.get_auth() handler = hs.get_device_handler() - assert isinstance(handler, DeviceHandler) self.device_handler = handler self.auth_handler = hs.get_auth_handler() @@ -147,7 +145,6 @@ class DeviceRestServlet(RestServlet): self.auth_handler = hs.get_auth_handler() self._msc3852_enabled = hs.config.experimental.msc3852_enabled self._msc3861_oauth_delegation_enabled = hs.config.experimental.msc3861.enabled - self._is_main_process = hs.config.worker.worker_app is None async def on_GET( self, request: SynapseRequest, device_id: str @@ -179,14 +176,6 @@ class DeviceRestServlet(RestServlet): async def on_DELETE( self, request: SynapseRequest, device_id: str ) -> Tuple[int, JsonDict]: - # Can only be run on main process, as changes to device lists must - # happen on main. - if not self._is_main_process: - error_message = "DELETE on /devices/ must be routed to main process" - logger.error(error_message) - raise SynapseError(500, error_message) - assert isinstance(self.device_handler, DeviceHandler) - requester = await self.auth.get_user_by_req(request) try: @@ -231,14 +220,6 @@ class DeviceRestServlet(RestServlet): async def on_PUT( self, request: SynapseRequest, device_id: str ) -> Tuple[int, JsonDict]: - # Can only be run on main process, as changes to device lists must - # happen on main. - if not self._is_main_process: - error_message = "PUT on /devices/ must be routed to main process" - logger.error(error_message) - raise SynapseError(500, error_message) - assert isinstance(self.device_handler, DeviceHandler) - requester = await self.auth.get_user_by_req(request, allow_guest=True) body = parse_and_validate_json_object_from_request(request, self.PutBody) @@ -317,7 +298,6 @@ class DehydratedDeviceServlet(RestServlet): self.hs = hs self.auth = hs.get_auth() handler = hs.get_device_handler() - assert isinstance(handler, DeviceHandler) self.device_handler = handler async def on_GET(self, request: SynapseRequest) -> Tuple[int, JsonDict]: @@ -377,7 +357,6 @@ class ClaimDehydratedDeviceServlet(RestServlet): self.hs = hs self.auth = hs.get_auth() handler = hs.get_device_handler() - assert isinstance(handler, DeviceHandler) self.device_handler = handler class PostBody(RequestBodyModel): @@ -517,7 +496,6 @@ class DehydratedDeviceV2Servlet(RestServlet): self.hs = hs self.auth = hs.get_auth() handler = hs.get_device_handler() - assert isinstance(handler, DeviceHandler) self.e2e_keys_handler = hs.get_e2e_keys_handler() self.device_handler = handler @@ -595,18 +573,14 @@ class DehydratedDeviceV2Servlet(RestServlet): def register_servlets(hs: "HomeServer", http_server: HttpServer) -> None: - if ( - hs.config.worker.worker_app is None - and not hs.config.experimental.msc3861.enabled - ): + if not hs.config.experimental.msc3861.enabled: DeleteDevicesRestServlet(hs).register(http_server) DevicesRestServlet(hs).register(http_server) DeviceRestServlet(hs).register(http_server) - if hs.config.worker.worker_app is None: - if hs.config.experimental.msc2697_enabled: - DehydratedDeviceServlet(hs).register(http_server) - ClaimDehydratedDeviceServlet(hs).register(http_server) - if hs.config.experimental.msc3814_enabled: - DehydratedDeviceV2Servlet(hs).register(http_server) - DehydratedDeviceEventsServlet(hs).register(http_server) + if hs.config.experimental.msc2697_enabled: + DehydratedDeviceServlet(hs).register(http_server) + ClaimDehydratedDeviceServlet(hs).register(http_server) + if hs.config.experimental.msc3814_enabled: + DehydratedDeviceV2Servlet(hs).register(http_server) + DehydratedDeviceEventsServlet(hs).register(http_server) diff --git a/synapse/rest/client/keys.py b/synapse/rest/client/keys.py index 7025662fdc..09749b840f 100644 --- a/synapse/rest/client/keys.py +++ b/synapse/rest/client/keys.py @@ -504,6 +504,5 @@ def register_servlets(hs: "HomeServer", http_server: HttpServer) -> None: OneTimeKeyServlet(hs).register(http_server) if hs.config.experimental.msc3983_appservice_otk_claims: UnstableOneTimeKeyServlet(hs).register(http_server) - if hs.config.worker.worker_app is None: - SigningKeyUploadServlet(hs).register(http_server) - SignaturesUploadServlet(hs).register(http_server) + SigningKeyUploadServlet(hs).register(http_server) + SignaturesUploadServlet(hs).register(http_server) diff --git a/synapse/rest/client/logout.py b/synapse/rest/client/logout.py index e6b4a34d51..206865e989 100644 --- a/synapse/rest/client/logout.py +++ b/synapse/rest/client/logout.py @@ -22,7 +22,6 @@ import logging from typing import TYPE_CHECKING, Tuple -from synapse.handlers.device import DeviceHandler from synapse.http.server import HttpServer from synapse.http.servlet import RestServlet from synapse.http.site import SynapseRequest @@ -42,9 +41,7 @@ class LogoutRestServlet(RestServlet): super().__init__() self.auth = hs.get_auth() self._auth_handler = hs.get_auth_handler() - handler = hs.get_device_handler() - assert isinstance(handler, DeviceHandler) - self._device_handler = handler + self._device_handler = hs.get_device_handler() async def on_POST(self, request: SynapseRequest) -> Tuple[int, JsonDict]: requester = await self.auth.get_user_by_req( @@ -71,9 +68,7 @@ class LogoutAllRestServlet(RestServlet): super().__init__() self.auth = hs.get_auth() self._auth_handler = hs.get_auth_handler() - handler = hs.get_device_handler() - assert isinstance(handler, DeviceHandler) - self._device_handler = handler + self._device_handler = hs.get_device_handler() async def on_POST(self, request: SynapseRequest) -> Tuple[int, JsonDict]: requester = await self.auth.get_user_by_req( diff --git a/synapse/server.py b/synapse/server.py index fd16abb9ea..5270f7792d 100644 --- a/synapse/server.py +++ b/synapse/server.py @@ -69,7 +69,7 @@ from synapse.handlers.auth import AuthHandler, PasswordAuthProvider from synapse.handlers.cas import CasHandler from synapse.handlers.deactivate_account import DeactivateAccountHandler from synapse.handlers.delayed_events import DelayedEventsHandler -from synapse.handlers.device import DeviceHandler, DeviceWorkerHandler +from synapse.handlers.device import DeviceHandler, DeviceWriterHandler from synapse.handlers.devicemessage import DeviceMessageHandler from synapse.handlers.directory import DirectoryHandler from synapse.handlers.e2e_keys import E2eKeysHandler @@ -586,11 +586,11 @@ class HomeServer(metaclass=abc.ABCMeta): ) @cache_in_self - def get_device_handler(self) -> DeviceWorkerHandler: - if self.config.worker.worker_app: - return DeviceWorkerHandler(self) - else: - return DeviceHandler(self) + def get_device_handler(self) -> DeviceHandler: + if self.get_instance_name() in self.config.worker.writers.device_lists: + return DeviceWriterHandler(self) + + return DeviceHandler(self) @cache_in_self def get_device_message_handler(self) -> DeviceMessageHandler: diff --git a/synapse/storage/databases/main/devices.py b/synapse/storage/databases/main/devices.py index f054c66102..6ed9f85800 100644 --- a/synapse/storage/databases/main/devices.py +++ b/synapse/storage/databases/main/devices.py @@ -27,7 +27,6 @@ from typing import ( Dict, Iterable, List, - Literal, Mapping, Optional, Set, @@ -61,12 +60,12 @@ from synapse.storage.util.id_generators import MultiWriterIdGenerator from synapse.types import ( JsonDict, JsonMapping, + MultiWriterStreamToken, StrCollection, get_verify_key_from_cross_signing_key, ) from synapse.util import json_decoder, json_encoder from synapse.util.caches.descriptors import cached, cachedList -from synapse.util.caches.lrucache import LruCache from synapse.util.caches.stream_change_cache import StreamChangeCache from synapse.util.cancellation import cancellable from synapse.util.iterutils import batch_iter @@ -86,6 +85,9 @@ BG_UPDATE_REMOVE_DUP_OUTBOUND_POKES = "remove_dup_outbound_pokes" class DeviceWorkerStore(RoomMemberWorkerStore, EndToEndKeyWorkerStore): + _device_list_id_gen: MultiWriterIdGenerator + _instance_name: str + def __init__( self, database: DatabasePool, @@ -115,7 +117,11 @@ class DeviceWorkerStore(RoomMemberWorkerStore, EndToEndKeyWorkerStore): ), ], sequence_name="device_lists_sequence", - writers=["master"], + writers=hs.config.worker.writers.device_lists, + ) + + self._is_device_list_writer = ( + self._instance_name in hs.config.worker.writers.device_lists ) device_list_max = self._device_list_id_gen.get_current_token() @@ -244,8 +250,8 @@ class DeviceWorkerStore(RoomMemberWorkerStore, EndToEndKeyWorkerStore): for room_id in room_ids: self._device_list_room_stream_cache.entity_has_changed(room_id, token) - def get_device_stream_token(self) -> int: - return self._device_list_id_gen.get_current_token() + def get_device_stream_token(self) -> MultiWriterStreamToken: + return MultiWriterStreamToken.from_generator(self._device_list_id_gen) def get_device_stream_id_generator(self) -> MultiWriterIdGenerator: return self._device_list_id_gen @@ -286,6 +292,183 @@ class DeviceWorkerStore(RoomMemberWorkerStore, EndToEndKeyWorkerStore): "count_devices_by_users", count_devices_by_users_txn, user_ids ) + async def store_device( + self, + user_id: str, + device_id: str, + initial_device_display_name: Optional[str], + auth_provider_id: Optional[str] = None, + auth_provider_session_id: Optional[str] = None, + ) -> bool: + """Ensure the given device is known; add it to the store if not + + Args: + user_id: id of user associated with the device + device_id: id of device + initial_device_display_name: initial displayname of the device. + Ignored if device exists. + auth_provider_id: The SSO IdP the user used, if any. + auth_provider_session_id: The session ID (sid) got from a OIDC login. + + Returns: + Whether the device was inserted or an existing device existed with that ID. + + Raises: + StoreError: if the device is already in use + """ + try: + inserted = await self.db_pool.simple_upsert( + "devices", + keyvalues={ + "user_id": user_id, + "device_id": device_id, + }, + values={}, + insertion_values={ + "display_name": initial_device_display_name, + "hidden": False, + }, + desc="store_device", + ) + await self.invalidate_cache_and_stream("get_device", (user_id, device_id)) + + if not inserted: + # if the device already exists, check if it's a real device, or + # if the device ID is reserved by something else + hidden = await self.db_pool.simple_select_one_onecol( + "devices", + keyvalues={"user_id": user_id, "device_id": device_id}, + retcol="hidden", + ) + if hidden: + raise StoreError(400, "The device ID is in use", Codes.FORBIDDEN) + + if auth_provider_id and auth_provider_session_id: + await self.db_pool.simple_insert( + "device_auth_providers", + values={ + "user_id": user_id, + "device_id": device_id, + "auth_provider_id": auth_provider_id, + "auth_provider_session_id": auth_provider_session_id, + }, + desc="store_device_auth_provider", + ) + + return inserted + except StoreError: + raise + except Exception as e: + logger.error( + "store_device with device_id=%s(%r) user_id=%s(%r)" + " display_name=%s(%r) failed: %s", + type(device_id).__name__, + device_id, + type(user_id).__name__, + user_id, + type(initial_device_display_name).__name__, + initial_device_display_name, + e, + ) + raise StoreError(500, "Problem storing device.") + + async def delete_devices(self, user_id: str, device_ids: StrCollection) -> None: + """Deletes several devices. + + Args: + user_id: The ID of the user which owns the devices + device_ids: The IDs of the devices to delete + """ + + def _delete_devices_txn(txn: LoggingTransaction, device_ids: List[str]) -> None: + self.db_pool.simple_delete_many_txn( + txn, + table="devices", + column="device_id", + values=device_ids, + keyvalues={"user_id": user_id, "hidden": False}, + ) + + self.db_pool.simple_delete_many_txn( + txn, + table="device_auth_providers", + column="device_id", + values=device_ids, + keyvalues={"user_id": user_id}, + ) + + # Also delete associated e2e keys. + self.db_pool.simple_delete_many_txn( + txn, + table="e2e_device_keys_json", + keyvalues={"user_id": user_id}, + column="device_id", + values=device_ids, + ) + self.db_pool.simple_delete_many_txn( + txn, + table="e2e_one_time_keys_json", + keyvalues={"user_id": user_id}, + column="device_id", + values=device_ids, + ) + self.db_pool.simple_delete_many_txn( + txn, + table="dehydrated_devices", + keyvalues={"user_id": user_id}, + column="device_id", + values=device_ids, + ) + self.db_pool.simple_delete_many_txn( + txn, + table="e2e_fallback_keys_json", + keyvalues={"user_id": user_id}, + column="device_id", + values=device_ids, + ) + + # We're bulk deleting potentially many devices at once, so + # let's not invalidate the cache for each device individually. + # Instead, we will invalidate the cache for the user as a whole. + self._invalidate_cache_and_stream(txn, self.get_device, (user_id,)) + self._invalidate_cache_and_stream( + txn, self.count_e2e_one_time_keys, (user_id,) + ) + self._invalidate_cache_and_stream( + txn, self.get_e2e_unused_fallback_key_types, (user_id,) + ) + + for batch in batch_iter(device_ids, 1000): + await self.db_pool.runInteraction( + "delete_devices", _delete_devices_txn, batch + ) + + async def update_device( + self, user_id: str, device_id: str, new_display_name: Optional[str] = None + ) -> None: + """Update a device. Only updates the device if it is not marked as + hidden. + + Args: + user_id: The ID of the user which owns the device + device_id: The ID of the device to update + new_display_name: new displayname for device; None to leave unchanged + Raises: + StoreError: if the device is not found + """ + updates = {} + if new_display_name is not None: + updates["display_name"] = new_display_name + if not updates: + return None + await self.db_pool.simple_update_one( + table="devices", + keyvalues={"user_id": user_id, "device_id": device_id, "hidden": False}, + updatevalues=updates, + desc="update_device", + ) + await self.invalidate_cache_and_stream("get_device", (user_id, device_id)) + @cached(tree=True) async def get_device( self, user_id: str, device_id: str @@ -379,7 +562,11 @@ class DeviceWorkerStore(RoomMemberWorkerStore, EndToEndKeyWorkerStore): - The list of updates, where each update is a pair of EDU type and EDU contents. """ - now_stream_id = self.get_device_stream_token() + # Here, we don't use the individual instances positions, as we only + # record the last stream position we've sent to a destination. This + # means we have to wait for all the writers to catch up before sending + # device list updates, which is fine. + now_stream_id = self.get_device_stream_token().stream if from_stream_id == now_stream_id: return now_stream_id, [] @@ -756,6 +943,9 @@ class DeviceWorkerStore(RoomMemberWorkerStore, EndToEndKeyWorkerStore): Returns: The new stream ID. """ + # This generates new stream IDs and therefore must be called on a writer. + if not self._is_device_list_writer: + raise Exception("Can only be called on device list writers") async with self._device_list_id_gen.get_next() as stream_id: await self.db_pool.runInteraction( @@ -878,8 +1068,8 @@ class DeviceWorkerStore(RoomMemberWorkerStore, EndToEndKeyWorkerStore): @cancellable async def get_all_devices_changed( self, - from_key: int, - to_key: int, + from_key: MultiWriterStreamToken, + to_key: MultiWriterStreamToken, ) -> Set[str]: """Get all users whose devices have changed in the given range. @@ -894,7 +1084,9 @@ class DeviceWorkerStore(RoomMemberWorkerStore, EndToEndKeyWorkerStore): (exclusive) until `to_key` (inclusive). """ - result = self._device_list_stream_cache.get_all_entities_changed(from_key) + result = self._device_list_stream_cache.get_all_entities_changed( + from_key.stream + ) if result.hit: # We know which users might have changed devices. @@ -910,24 +1102,34 @@ class DeviceWorkerStore(RoomMemberWorkerStore, EndToEndKeyWorkerStore): # If the cache didn't tell us anything, we just need to query the full # range. sql = """ - SELECT DISTINCT user_id FROM device_lists_stream + SELECT user_id, stream_id, instance_name + FROM device_lists_stream WHERE ? < stream_id AND stream_id <= ? """ rows = await self.db_pool.execute( "get_all_devices_changed", sql, - from_key, - to_key, + from_key.stream, + to_key.get_max_stream_pos(), ) - return {u for (u,) in rows} + return { + user_id + for (user_id, stream_id, instance_name) in rows + if MultiWriterStreamToken.is_stream_position_in_range( + low=from_key, + high=to_key, + instance_name=instance_name, + pos=stream_id, + ) + } @cancellable async def get_users_whose_devices_changed( self, - from_key: int, + from_key: MultiWriterStreamToken, user_ids: Collection[str], - to_key: Optional[int] = None, + to_key: Optional[MultiWriterStreamToken] = None, ) -> Set[str]: """Get set of users whose devices have changed since `from_key` that are in the given list of user_ids. @@ -947,7 +1149,7 @@ class DeviceWorkerStore(RoomMemberWorkerStore, EndToEndKeyWorkerStore): # Get set of users who *may* have changed. Users not in the returned # list have definitely not changed. user_ids_to_check = self._device_list_stream_cache.get_entities_changed( - user_ids, from_key + user_ids, from_key.stream ) # If an empty set was returned, there's nothing to do. @@ -955,11 +1157,16 @@ class DeviceWorkerStore(RoomMemberWorkerStore, EndToEndKeyWorkerStore): return set() if to_key is None: - to_key = self._device_list_id_gen.get_current_token() + to_key = self.get_device_stream_token() - def _get_users_whose_devices_changed_txn(txn: LoggingTransaction) -> Set[str]: + def _get_users_whose_devices_changed_txn( + txn: LoggingTransaction, + from_key: MultiWriterStreamToken, + to_key: MultiWriterStreamToken, + ) -> Set[str]: sql = """ - SELECT DISTINCT user_id FROM device_lists_stream + SELECT user_id, stream_id, instance_name + FROM device_lists_stream WHERE ? < stream_id AND stream_id <= ? AND %s """ @@ -970,17 +1177,32 @@ class DeviceWorkerStore(RoomMemberWorkerStore, EndToEndKeyWorkerStore): clause, args = make_in_list_sql_clause( txn.database_engine, "user_id", chunk ) - txn.execute(sql % (clause,), [from_key, to_key] + args) - changes.update(user_id for (user_id,) in txn) + txn.execute( + sql % (clause,), + [from_key.stream, to_key.get_max_stream_pos()] + args, + ) + changes.update( + user_id + for (user_id, stream_id, instance_name) in txn + if MultiWriterStreamToken.is_stream_position_in_range( + low=from_key, + high=to_key, + instance_name=instance_name, + pos=stream_id, + ) + ) return changes return await self.db_pool.runInteraction( - "get_users_whose_devices_changed", _get_users_whose_devices_changed_txn + "get_users_whose_devices_changed", + _get_users_whose_devices_changed_txn, + from_key, + to_key, ) async def get_users_whose_signatures_changed( - self, user_id: str, from_key: int + self, user_id: str, from_key: MultiWriterStreamToken ) -> Set[str]: """Get the users who have new cross-signing signatures made by `user_id` since `from_key`. @@ -993,18 +1215,31 @@ class DeviceWorkerStore(RoomMemberWorkerStore, EndToEndKeyWorkerStore): A set of user IDs with updated signatures. """ - if self._user_signature_stream_cache.has_entity_changed(user_id, from_key): - sql = """ - SELECT DISTINCT user_ids FROM user_signature_stream - WHERE from_user_id = ? AND stream_id > ? - """ - rows = await self.db_pool.execute( - "get_users_whose_signatures_changed", sql, user_id, from_key - ) - return {user for row in rows for user in db_to_json(row[0])} - else: + if not self._user_signature_stream_cache.has_entity_changed( + user_id, from_key.stream + ): return set() + sql = """ + SELECT user_ids, stream_id, instance_name + FROM user_signature_stream + WHERE from_user_id = ? AND stream_id > ? + """ + rows = await self.db_pool.execute( + "get_users_whose_signatures_changed", sql, user_id, from_key.stream + ) + return { + user + for (user_ids, stream_id, instance_name) in rows + if MultiWriterStreamToken.is_stream_position_in_range( + low=from_key, + high=None, + instance_name=instance_name, + pos=stream_id, + ) + for user in db_to_json(user_ids) + } + async def get_all_device_list_changes_for_remotes( self, instance_name: str, last_id: int, current_id: int, limit: int ) -> Tuple[List[Tuple[int, tuple]], int, bool]: @@ -1258,9 +1493,7 @@ class DeviceWorkerStore(RoomMemberWorkerStore, EndToEndKeyWorkerStore): if keys: device_keys = keys.get("device_keys", None) if device_keys: - # Type ignore - this function is defined on EndToEndKeyStore which we do - # have access to due to hs.get_datastore() "magic" - self._set_e2e_device_keys_txn( # type: ignore[attr-defined] + self._set_e2e_device_keys_txn( txn, user_id, device_id, time, device_keys ) @@ -1490,7 +1723,10 @@ class DeviceWorkerStore(RoomMemberWorkerStore, EndToEndKeyWorkerStore): @cancellable async def get_device_list_changes_in_rooms( - self, room_ids: Collection[str], from_id: int, to_id: int + self, + room_ids: Collection[str], + from_token: MultiWriterStreamToken, + to_token: MultiWriterStreamToken, ) -> Optional[Set[str]]: """Return the set of users whose devices have changed in the given rooms since the given stream ID. @@ -1503,41 +1739,50 @@ class DeviceWorkerStore(RoomMemberWorkerStore, EndToEndKeyWorkerStore): min_stream_id = await self._get_min_device_lists_changes_in_room() - if min_stream_id > from_id: + # Return early if there are no rows to process in device_lists_changes_in_room + if min_stream_id > from_token.stream: return None changed_room_ids = self._device_list_room_stream_cache.get_entities_changed( - room_ids, from_id + room_ids, from_token.stream ) if not changed_room_ids: return set() sql = """ - SELECT DISTINCT user_id FROM device_lists_changes_in_room + SELECT user_id, stream_id, instance_name + FROM device_lists_changes_in_room WHERE {clause} AND stream_id > ? AND stream_id <= ? """ def _get_device_list_changes_in_rooms_txn( txn: LoggingTransaction, - clause: str, - args: List[Any], + chunk: list[str], ) -> Set[str]: - txn.execute(sql.format(clause=clause), args) - return {user_id for (user_id,) in txn} - - changes = set() - for chunk in batch_iter(changed_room_ids, 1000): clause, args = make_in_list_sql_clause( self.database_engine, "room_id", chunk ) - args.append(from_id) - args.append(to_id) + args.append(from_token.stream) + args.append(to_token.get_max_stream_pos()) + txn.execute(sql.format(clause=clause), args) + return { + user_id + for (user_id, stream_id, instance_name) in txn + if MultiWriterStreamToken.is_stream_position_in_range( + low=from_token, + high=to_token, + instance_name=instance_name, + pos=stream_id, + ) + } + + changes = set() + for chunk in batch_iter(changed_room_ids, 1000): changes |= await self.db_pool.runInteraction( "get_device_list_changes_in_rooms", _get_device_list_changes_in_rooms_txn, - clause, - args, + chunk, ) return changes @@ -1605,371 +1850,6 @@ class DeviceWorkerStore(RoomMemberWorkerStore, EndToEndKeyWorkerStore): desc="get_destinations_for_device", ) - -class DeviceBackgroundUpdateStore(SQLBaseStore): - def __init__( - self, - database: DatabasePool, - db_conn: LoggingDatabaseConnection, - hs: "HomeServer", - ): - super().__init__(database, db_conn, hs) - - self._instance_name = hs.get_instance_name() - - self.db_pool.updates.register_background_index_update( - "device_lists_stream_idx", - index_name="device_lists_stream_user_id", - table="device_lists_stream", - columns=["user_id", "device_id"], - ) - - # create a unique index on device_lists_remote_cache - self.db_pool.updates.register_background_index_update( - "device_lists_remote_cache_unique_idx", - index_name="device_lists_remote_cache_unique_id", - table="device_lists_remote_cache", - columns=["user_id", "device_id"], - unique=True, - ) - - # And one on device_lists_remote_extremeties - self.db_pool.updates.register_background_index_update( - "device_lists_remote_extremeties_unique_idx", - index_name="device_lists_remote_extremeties_unique_idx", - table="device_lists_remote_extremeties", - columns=["user_id"], - unique=True, - ) - - # once they complete, we can remove the old non-unique indexes. - self.db_pool.updates.register_background_update_handler( - DROP_DEVICE_LIST_STREAMS_NON_UNIQUE_INDEXES, - self._drop_device_list_streams_non_unique_indexes, - ) - - # clear out duplicate device list outbound pokes - self.db_pool.updates.register_background_update_handler( - BG_UPDATE_REMOVE_DUP_OUTBOUND_POKES, - self._remove_duplicate_outbound_pokes, - ) - - self.db_pool.updates.register_background_index_update( - "device_lists_changes_in_room_by_room_index", - index_name="device_lists_changes_in_room_by_room_idx", - table="device_lists_changes_in_room", - columns=["room_id", "stream_id"], - ) - - async def _drop_device_list_streams_non_unique_indexes( - self, progress: JsonDict, batch_size: int - ) -> int: - def f(conn: LoggingDatabaseConnection) -> None: - txn = conn.cursor() - txn.execute("DROP INDEX IF EXISTS device_lists_remote_cache_id") - txn.execute("DROP INDEX IF EXISTS device_lists_remote_extremeties_id") - txn.close() - - await self.db_pool.runWithConnection(f) - await self.db_pool.updates._end_background_update( - DROP_DEVICE_LIST_STREAMS_NON_UNIQUE_INDEXES - ) - return 1 - - async def _remove_duplicate_outbound_pokes( - self, progress: JsonDict, batch_size: int - ) -> int: - # for some reason, we have accumulated duplicate entries in - # device_lists_outbound_pokes, which makes prune_outbound_device_list_pokes less - # efficient. - # - # For each duplicate, we delete all the existing rows and put one back. - - last_row = progress.get( - "last_row", - {"stream_id": 0, "destination": "", "user_id": "", "device_id": ""}, - ) - - def _txn(txn: LoggingTransaction) -> int: - clause, args = make_tuple_comparison_clause( - [ - ("stream_id", last_row["stream_id"]), - ("destination", last_row["destination"]), - ("user_id", last_row["user_id"]), - ("device_id", last_row["device_id"]), - ] - ) - sql = f""" - SELECT stream_id, destination, user_id, device_id, MAX(ts) AS ts - FROM device_lists_outbound_pokes - WHERE {clause} - GROUP BY stream_id, destination, user_id, device_id - HAVING count(*) > 1 - ORDER BY stream_id, destination, user_id, device_id - LIMIT ? - """ - txn.execute(sql, args + [batch_size]) - rows = txn.fetchall() - - stream_id, destination, user_id, device_id = None, None, None, None - for stream_id, destination, user_id, device_id, _ in rows: - self.db_pool.simple_delete_txn( - txn, - "device_lists_outbound_pokes", - { - "stream_id": stream_id, - "destination": destination, - "user_id": user_id, - "device_id": device_id, - }, - ) - - self.db_pool.simple_insert_txn( - txn, - "device_lists_outbound_pokes", - { - "stream_id": stream_id, - "instance_name": self._instance_name, - "destination": destination, - "user_id": user_id, - "device_id": device_id, - "sent": False, - }, - ) - - if rows: - self.db_pool.updates._background_update_progress_txn( - txn, - BG_UPDATE_REMOVE_DUP_OUTBOUND_POKES, - { - "last_row": { - "stream_id": stream_id, - "destination": destination, - "user_id": user_id, - "device_id": device_id, - } - }, - ) - - return len(rows) - - rows = await self.db_pool.runInteraction( - BG_UPDATE_REMOVE_DUP_OUTBOUND_POKES, _txn - ) - - if not rows: - await self.db_pool.updates._end_background_update( - BG_UPDATE_REMOVE_DUP_OUTBOUND_POKES - ) - - return rows - - -class DeviceStore(DeviceWorkerStore, DeviceBackgroundUpdateStore): - def __init__( - self, - database: DatabasePool, - db_conn: LoggingDatabaseConnection, - hs: "HomeServer", - ): - super().__init__(database, db_conn, hs) - self.server_name = hs.hostname - - # Map of (user_id, device_id) -> bool. If there is an entry that implies - # the device exists. - self.device_id_exists_cache: LruCache[Tuple[str, str], Literal[True]] = ( - LruCache( - cache_name="device_id_exists", - server_name=self.server_name, - max_size=10000, - ) - ) - - async def store_device( - self, - user_id: str, - device_id: str, - initial_device_display_name: Optional[str], - auth_provider_id: Optional[str] = None, - auth_provider_session_id: Optional[str] = None, - ) -> bool: - """Ensure the given device is known; add it to the store if not - - Args: - user_id: id of user associated with the device - device_id: id of device - initial_device_display_name: initial displayname of the device. - Ignored if device exists. - auth_provider_id: The SSO IdP the user used, if any. - auth_provider_session_id: The session ID (sid) got from a OIDC login. - - Returns: - Whether the device was inserted or an existing device existed with that ID. - - Raises: - StoreError: if the device is already in use - """ - key = (user_id, device_id) - if self.device_id_exists_cache.get(key, None): - return False - - try: - inserted = await self.db_pool.simple_upsert( - "devices", - keyvalues={ - "user_id": user_id, - "device_id": device_id, - }, - values={}, - insertion_values={ - "display_name": initial_device_display_name, - "hidden": False, - }, - desc="store_device", - ) - await self.invalidate_cache_and_stream("get_device", (user_id, device_id)) - - if not inserted: - # if the device already exists, check if it's a real device, or - # if the device ID is reserved by something else - hidden = await self.db_pool.simple_select_one_onecol( - "devices", - keyvalues={"user_id": user_id, "device_id": device_id}, - retcol="hidden", - ) - if hidden: - raise StoreError(400, "The device ID is in use", Codes.FORBIDDEN) - - if auth_provider_id and auth_provider_session_id: - await self.db_pool.simple_insert( - "device_auth_providers", - values={ - "user_id": user_id, - "device_id": device_id, - "auth_provider_id": auth_provider_id, - "auth_provider_session_id": auth_provider_session_id, - }, - desc="store_device_auth_provider", - ) - - self.device_id_exists_cache.set(key, True) - return inserted - except StoreError: - raise - except Exception as e: - logger.error( - "store_device with device_id=%s(%r) user_id=%s(%r)" - " display_name=%s(%r) failed: %s", - type(device_id).__name__, - device_id, - type(user_id).__name__, - user_id, - type(initial_device_display_name).__name__, - initial_device_display_name, - e, - ) - raise StoreError(500, "Problem storing device.") - - async def delete_devices(self, user_id: str, device_ids: StrCollection) -> None: - """Deletes several devices. - - Args: - user_id: The ID of the user which owns the devices - device_ids: The IDs of the devices to delete - """ - - def _delete_devices_txn(txn: LoggingTransaction, device_ids: List[str]) -> None: - self.db_pool.simple_delete_many_txn( - txn, - table="devices", - column="device_id", - values=device_ids, - keyvalues={"user_id": user_id, "hidden": False}, - ) - - self.db_pool.simple_delete_many_txn( - txn, - table="device_auth_providers", - column="device_id", - values=device_ids, - keyvalues={"user_id": user_id}, - ) - - # Also delete associated e2e keys. - self.db_pool.simple_delete_many_txn( - txn, - table="e2e_device_keys_json", - keyvalues={"user_id": user_id}, - column="device_id", - values=device_ids, - ) - self.db_pool.simple_delete_many_txn( - txn, - table="e2e_one_time_keys_json", - keyvalues={"user_id": user_id}, - column="device_id", - values=device_ids, - ) - self.db_pool.simple_delete_many_txn( - txn, - table="dehydrated_devices", - keyvalues={"user_id": user_id}, - column="device_id", - values=device_ids, - ) - self.db_pool.simple_delete_many_txn( - txn, - table="e2e_fallback_keys_json", - keyvalues={"user_id": user_id}, - column="device_id", - values=device_ids, - ) - - # We're bulk deleting potentially many devices at once, so - # let's not invalidate the cache for each device individually. - # Instead, we will invalidate the cache for the user as a whole. - self._invalidate_cache_and_stream(txn, self.get_device, (user_id,)) - self._invalidate_cache_and_stream( - txn, self.count_e2e_one_time_keys, (user_id,) - ) - self._invalidate_cache_and_stream( - txn, self.get_e2e_unused_fallback_key_types, (user_id,) - ) - - for batch in batch_iter(device_ids, 1000): - await self.db_pool.runInteraction( - "delete_devices", _delete_devices_txn, batch - ) - - for device_id in device_ids: - self.device_id_exists_cache.invalidate((user_id, device_id)) - - async def update_device( - self, user_id: str, device_id: str, new_display_name: Optional[str] = None - ) -> None: - """Update a device. Only updates the device if it is not marked as - hidden. - - Args: - user_id: The ID of the user which owns the device - device_id: The ID of the device to update - new_display_name: new displayname for device; None to leave unchanged - Raises: - StoreError: if the device is not found - """ - updates = {} - if new_display_name is not None: - updates["display_name"] = new_display_name - if not updates: - return None - await self.db_pool.simple_update_one( - table="devices", - keyvalues={"user_id": user_id, "device_id": device_id, "hidden": False}, - updatevalues=updates, - desc="update_device", - ) - await self.invalidate_cache_and_stream("get_device", (user_id, device_id)) - async def update_remote_device_list_cache_entry( self, user_id: str, device_id: str, content: JsonDict, stream_id: str ) -> None: @@ -2008,8 +1888,6 @@ class DeviceStore(DeviceWorkerStore, DeviceBackgroundUpdateStore): table="device_lists_remote_cache", keyvalues={"user_id": user_id, "device_id": device_id}, ) - - txn.call_after(self.device_id_exists_cache.invalidate, (user_id, device_id)) else: self.db_pool.simple_upsert_txn( txn, @@ -2102,6 +1980,10 @@ class DeviceStore(DeviceWorkerStore, DeviceBackgroundUpdateStore): The maximum stream ID of device list updates that were added to the database, or None if no updates were added. """ + # This generates new stream IDs and therefore must be called on a writer. + if not self._is_device_list_writer: + raise Exception("Can only be called on device list writers") + if not device_ids: return None @@ -2110,8 +1992,11 @@ class DeviceStore(DeviceWorkerStore, DeviceBackgroundUpdateStore): def add_device_changes_txn( txn: LoggingTransaction, batch_device_ids: StrCollection, - stream_ids: List[int], - ) -> None: + ) -> int: + stream_ids = self._device_list_id_gen.get_next_mult_txn( + txn, len(device_ids) + ) + self._add_device_change_to_stream_txn( txn, user_id, @@ -2128,18 +2013,17 @@ class DeviceStore(DeviceWorkerStore, DeviceBackgroundUpdateStore): context, ) - for batch_device_ids in batch_iter(device_ids, 1000): - async with self._device_list_id_gen.get_next_mult( - len(device_ids) - ) as stream_ids: - await self.db_pool.runInteraction( - "add_device_change_to_stream", - add_device_changes_txn, - batch_device_ids, - stream_ids, - ) + return stream_ids[-1] - return stream_ids[-1] + last_stream_id: Optional[int] = None + for batch_device_ids in batch_iter(device_ids, 1000): + last_stream_id = await self.db_pool.runInteraction( + "add_device_change_to_stream", + add_device_changes_txn, + batch_device_ids, + ) + + return last_stream_id def _add_device_change_to_stream_txn( self, @@ -2338,7 +2222,6 @@ class DeviceStore(DeviceWorkerStore, DeviceBackgroundUpdateStore): A list of user ID, device ID, room ID, stream ID and optional opentracing context, in order of ascending (stream ID, room ID). """ - sql = """ SELECT user_id, device_id, room_id, stream_id, opentracing_context FROM device_lists_changes_in_room @@ -2391,6 +2274,10 @@ class DeviceStore(DeviceWorkerStore, DeviceBackgroundUpdateStore): """Queue the device update to be sent to the given set of hosts, calculated from the room ID. """ + # This generates new stream IDs and therefore must be called on a writer. + if not self._is_device_list_writer: + raise Exception("Can only be called on device list writers") + if not hosts: return @@ -2419,6 +2306,9 @@ class DeviceStore(DeviceWorkerStore, DeviceBackgroundUpdateStore): """Add a device list update to the table tracking remote device list updates during partial joins. """ + # This generates new stream IDs and therefore must be called on a writer. + if not self._is_device_list_writer: + raise Exception("Can only be called on device list writers") async with self._device_list_id_gen.get_next() as stream_id: await self.db_pool.simple_upsert( @@ -2441,6 +2331,14 @@ class DeviceStore(DeviceWorkerStore, DeviceBackgroundUpdateStore): the room. """ + # The device list stream is a multi-writer stream, but when we partially + # join a room, we only record the minimum stream ID. This means that we + # may be returning a device update that was already sent through + # federation here in case of concurrent writes. This is absolutely fine, + # sending a device update multiple times through federation is safe + + # FIXME: record the full multi-writer stream token with individual + # writer positions at the time of the join to avoid this min_device_stream_id = await self.db_pool.simple_select_one_onecol( table="partial_state_rooms", keyvalues={ @@ -2511,3 +2409,176 @@ class DeviceStore(DeviceWorkerStore, DeviceBackgroundUpdateStore): }, desc="set_device_change_last_converted_pos", ) + + +class DeviceBackgroundUpdateStore(SQLBaseStore): + _instance_name: str + + def __init__( + self, + database: DatabasePool, + db_conn: LoggingDatabaseConnection, + hs: "HomeServer", + ): + super().__init__(database, db_conn, hs) + + self._instance_name = hs.get_instance_name() + + self.db_pool.updates.register_background_index_update( + "device_lists_stream_idx", + index_name="device_lists_stream_user_id", + table="device_lists_stream", + columns=["user_id", "device_id"], + ) + + # create a unique index on device_lists_remote_cache + self.db_pool.updates.register_background_index_update( + "device_lists_remote_cache_unique_idx", + index_name="device_lists_remote_cache_unique_id", + table="device_lists_remote_cache", + columns=["user_id", "device_id"], + unique=True, + ) + + # And one on device_lists_remote_extremeties + self.db_pool.updates.register_background_index_update( + "device_lists_remote_extremeties_unique_idx", + index_name="device_lists_remote_extremeties_unique_idx", + table="device_lists_remote_extremeties", + columns=["user_id"], + unique=True, + ) + + # once they complete, we can remove the old non-unique indexes. + self.db_pool.updates.register_background_update_handler( + DROP_DEVICE_LIST_STREAMS_NON_UNIQUE_INDEXES, + self._drop_device_list_streams_non_unique_indexes, + ) + + # clear out duplicate device list outbound pokes + self.db_pool.updates.register_background_update_handler( + BG_UPDATE_REMOVE_DUP_OUTBOUND_POKES, + self._remove_duplicate_outbound_pokes, + ) + + self.db_pool.updates.register_background_index_update( + "device_lists_changes_in_room_by_room_index", + index_name="device_lists_changes_in_room_by_room_idx", + table="device_lists_changes_in_room", + columns=["room_id", "stream_id"], + ) + + async def _drop_device_list_streams_non_unique_indexes( + self, progress: JsonDict, batch_size: int + ) -> int: + def f(conn: LoggingDatabaseConnection) -> None: + txn = conn.cursor() + txn.execute("DROP INDEX IF EXISTS device_lists_remote_cache_id") + txn.execute("DROP INDEX IF EXISTS device_lists_remote_extremeties_id") + txn.close() + + await self.db_pool.runWithConnection(f) + await self.db_pool.updates._end_background_update( + DROP_DEVICE_LIST_STREAMS_NON_UNIQUE_INDEXES + ) + return 1 + + async def _remove_duplicate_outbound_pokes( + self, progress: JsonDict, batch_size: int + ) -> int: + # for some reason, we have accumulated duplicate entries in + # device_lists_outbound_pokes, which makes prune_outbound_device_list_pokes less + # efficient. + # + # For each duplicate, we delete all the existing rows and put one back. + + last_row = progress.get( + "last_row", + {"stream_id": 0, "destination": "", "user_id": "", "device_id": ""}, + ) + + def _txn(txn: LoggingTransaction) -> int: + clause, args = make_tuple_comparison_clause( + [ + ("stream_id", last_row["stream_id"]), + ("destination", last_row["destination"]), + ("user_id", last_row["user_id"]), + ("device_id", last_row["device_id"]), + ] + ) + sql = f""" + SELECT stream_id, destination, user_id, device_id, MAX(ts) AS ts + FROM device_lists_outbound_pokes + WHERE {clause} + GROUP BY stream_id, destination, user_id, device_id + HAVING count(*) > 1 + ORDER BY stream_id, destination, user_id, device_id + LIMIT ? + """ + txn.execute(sql, args + [batch_size]) + rows = txn.fetchall() + + stream_id, destination, user_id, device_id = None, None, None, None + for stream_id, destination, user_id, device_id, _ in rows: + self.db_pool.simple_delete_txn( + txn, + "device_lists_outbound_pokes", + { + "stream_id": stream_id, + "destination": destination, + "user_id": user_id, + "device_id": device_id, + }, + ) + + self.db_pool.simple_insert_txn( + txn, + "device_lists_outbound_pokes", + { + "stream_id": stream_id, + "instance_name": self._instance_name, + "destination": destination, + "user_id": user_id, + "device_id": device_id, + "sent": False, + }, + ) + + if rows: + self.db_pool.updates._background_update_progress_txn( + txn, + BG_UPDATE_REMOVE_DUP_OUTBOUND_POKES, + { + "last_row": { + "stream_id": stream_id, + "destination": destination, + "user_id": user_id, + "device_id": device_id, + } + }, + ) + + return len(rows) + + rows = await self.db_pool.runInteraction( + BG_UPDATE_REMOVE_DUP_OUTBOUND_POKES, _txn + ) + + if not rows: + await self.db_pool.updates._end_background_update( + BG_UPDATE_REMOVE_DUP_OUTBOUND_POKES + ) + + return rows + + +class DeviceStore(DeviceWorkerStore, DeviceBackgroundUpdateStore): + _instance_name: str + + def __init__( + self, + database: DatabasePool, + db_conn: LoggingDatabaseConnection, + hs: "HomeServer", + ): + super().__init__(database, db_conn, hs) diff --git a/synapse/storage/databases/main/end_to_end_keys.py b/synapse/storage/databases/main/end_to_end_keys.py index 0700b0087b..a4a8aafa0c 100644 --- a/synapse/storage/databases/main/end_to_end_keys.py +++ b/synapse/storage/databases/main/end_to_end_keys.py @@ -59,7 +59,7 @@ from synapse.storage.database import ( from synapse.storage.databases.main.cache import CacheInvalidationWorkerStore from synapse.storage.engines import PostgresEngine from synapse.storage.util.id_generators import MultiWriterIdGenerator -from synapse.types import JsonDict, JsonMapping +from synapse.types import JsonDict, JsonMapping, MultiWriterStreamToken from synapse.util import json_decoder, json_encoder from synapse.util.caches.descriptors import cached, cachedList from synapse.util.cancellation import cancellable @@ -120,6 +120,20 @@ class EndToEndKeyWorkerStore(EndToEndKeyBackgroundStore, CacheInvalidationWorker self.hs.config.federation.allow_device_name_lookup_over_federation ) + self._cross_signing_id_gen = MultiWriterIdGenerator( + db_conn=db_conn, + db=database, + notifier=hs.get_replication_notifier(), + stream_name="e2e_cross_signing_keys", + instance_name=self._instance_name, + tables=[ + ("e2e_cross_signing_keys", "instance_name", "stream_id"), + ], + sequence_name="e2e_cross_signing_keys_sequence", + # No one reads the stream positions, so we're allowed to have an empty list of writers + writers=[], + ) + def process_replication_rows( self, stream_name: str, @@ -145,7 +159,12 @@ class EndToEndKeyWorkerStore(EndToEndKeyBackgroundStore, CacheInvalidationWorker Returns: (stream_id, devices) """ - now_stream_id = self.get_device_stream_token() + # Here, we don't use the individual instances positions, as we *need* to + # give out the stream_id as an integer in the federation API. + # This means that we'll potentially return the same data twice with a + # different stream_id, and invalidate cache more often than necessary, + # which is fine overall. + now_stream_id = self.get_device_stream_token().stream # We need to be careful with the caching here, as we need to always # return *all* persisted devices, however there may be a lag between a @@ -164,8 +183,10 @@ class EndToEndKeyWorkerStore(EndToEndKeyBackgroundStore, CacheInvalidationWorker # have to check for potential invalidations after the # `now_stream_id`. sql = """ - SELECT user_id FROM device_lists_stream + SELECT 1 + FROM device_lists_stream WHERE stream_id >= ? AND user_id = ? + LIMIT 1 """ rows = await self.db_pool.execute( "get_e2e_device_keys_for_federation_query_check", @@ -1117,7 +1138,7 @@ class EndToEndKeyWorkerStore(EndToEndKeyBackgroundStore, CacheInvalidationWorker ) @abc.abstractmethod - def get_device_stream_token(self) -> int: + def get_device_stream_token(self) -> MultiWriterStreamToken: """Get the current stream id from the _device_list_id_gen""" ... @@ -1540,27 +1561,44 @@ class EndToEndKeyWorkerStore(EndToEndKeyBackgroundStore, CacheInvalidationWorker impl, ) + async def delete_e2e_keys_by_device(self, user_id: str, device_id: str) -> None: + def delete_e2e_keys_by_device_txn(txn: LoggingTransaction) -> None: + log_kv( + { + "message": "Deleting keys for device", + "device_id": device_id, + "user_id": user_id, + } + ) + self.db_pool.simple_delete_txn( + txn, + table="e2e_device_keys_json", + keyvalues={"user_id": user_id, "device_id": device_id}, + ) + self.db_pool.simple_delete_txn( + txn, + table="e2e_one_time_keys_json", + keyvalues={"user_id": user_id, "device_id": device_id}, + ) + self._invalidate_cache_and_stream( + txn, self.count_e2e_one_time_keys, (user_id, device_id) + ) + self.db_pool.simple_delete_txn( + txn, + table="dehydrated_devices", + keyvalues={"user_id": user_id, "device_id": device_id}, + ) + self.db_pool.simple_delete_txn( + txn, + table="e2e_fallback_keys_json", + keyvalues={"user_id": user_id, "device_id": device_id}, + ) + self._invalidate_cache_and_stream( + txn, self.get_e2e_unused_fallback_key_types, (user_id, device_id) + ) -class EndToEndKeyStore(EndToEndKeyWorkerStore, SQLBaseStore): - def __init__( - self, - database: DatabasePool, - db_conn: LoggingDatabaseConnection, - hs: "HomeServer", - ): - super().__init__(database, db_conn, hs) - - self._cross_signing_id_gen = MultiWriterIdGenerator( - db_conn=db_conn, - db=database, - notifier=hs.get_replication_notifier(), - stream_name="e2e_cross_signing_keys", - instance_name=self._instance_name, - tables=[ - ("e2e_cross_signing_keys", "instance_name", "stream_id"), - ], - sequence_name="e2e_cross_signing_keys_sequence", - writers=["master"], + await self.db_pool.runInteraction( + "delete_e2e_keys_by_device", delete_e2e_keys_by_device_txn ) async def set_e2e_device_keys( @@ -1754,3 +1792,13 @@ class EndToEndKeyStore(EndToEndKeyWorkerStore, SQLBaseStore): ], desc="add_e2e_signing_key", ) + + +class EndToEndKeyStore(EndToEndKeyWorkerStore, SQLBaseStore): + def __init__( + self, + database: DatabasePool, + db_conn: LoggingDatabaseConnection, + hs: "HomeServer", + ): + super().__init__(database, db_conn, hs) diff --git a/synapse/storage/databases/main/receipts.py b/synapse/storage/databases/main/receipts.py index 81f50467e7..16af68108d 100644 --- a/synapse/storage/databases/main/receipts.py +++ b/synapse/storage/databases/main/receipts.py @@ -36,7 +36,6 @@ from typing import ( ) import attr -from immutabledict import immutabledict from synapse.api.constants import EduTypes from synapse.replication.tcp.streams import ReceiptsStream @@ -167,25 +166,7 @@ class ReceiptsWorkerStore(SQLBaseStore): def get_max_receipt_stream_id(self) -> MultiWriterStreamToken: """Get the current max stream ID for receipts stream""" - min_pos = self._receipts_id_gen.get_current_token() - - positions = {} - if isinstance(self._receipts_id_gen, MultiWriterIdGenerator): - # The `min_pos` is the minimum position that we know all instances - # have finished persisting to, so we only care about instances whose - # positions are ahead of that. (Instance positions can be behind the - # min position as there are times we can work out that the minimum - # position is ahead of the naive minimum across all current - # positions. See MultiWriterIdGenerator for details) - positions = { - i: p - for i, p in self._receipts_id_gen.get_positions().items() - if p > min_pos - } - - return MultiWriterStreamToken( - stream=min_pos, instance_map=immutabledict(positions) - ) + return MultiWriterStreamToken.from_generator(self._receipts_id_gen) def get_receipt_stream_id_for_instance(self, instance_name: str) -> int: return self._receipts_id_gen.get_current_token_for_writer(instance_name) diff --git a/synapse/storage/databases/main/registration.py b/synapse/storage/databases/main/registration.py index 38d1e650ad..320d29e474 100644 --- a/synapse/storage/databases/main/registration.py +++ b/synapse/storage/databases/main/registration.py @@ -2093,6 +2093,58 @@ class RegistrationWorkerStore(StatsStore, CacheInvalidationWorkerStore): "replace_refresh_token", _replace_refresh_token_txn ) + async def set_device_for_refresh_token( + self, user_id: str, old_device_id: str, device_id: str + ) -> None: + """Moves refresh tokens from old device to current device + + Args: + user_id: The user of the devices. + old_device_id: The old device. + device_id: The new device ID. + Returns: + None + """ + + await self.db_pool.simple_update( + "refresh_tokens", + keyvalues={"user_id": user_id, "device_id": old_device_id}, + updatevalues={"device_id": device_id}, + desc="set_device_for_refresh_token", + ) + + def _set_device_for_access_token_txn( + self, txn: LoggingTransaction, token: str, device_id: str + ) -> str: + old_device_id = self.db_pool.simple_select_one_onecol_txn( + txn, "access_tokens", {"token": token}, "device_id" + ) + + self.db_pool.simple_update_txn( + txn, "access_tokens", {"token": token}, {"device_id": device_id} + ) + + self._invalidate_cache_and_stream(txn, self.get_user_by_access_token, (token,)) + + return old_device_id + + async def set_device_for_access_token(self, token: str, device_id: str) -> str: + """Sets the device ID associated with an access token. + + Args: + token: The access token to modify. + device_id: The new device ID. + Returns: + The old device ID associated with the access token. + """ + + return await self.db_pool.runInteraction( + "set_device_for_access_token", + self._set_device_for_access_token_txn, + token, + device_id, + ) + async def add_login_token_to_user( self, user_id: str, @@ -2396,6 +2448,154 @@ class RegistrationWorkerStore(StatsStore, CacheInvalidationWorkerStore): self._invalidate_cache_and_stream(txn, self.get_user_by_id, (user_id,)) self._invalidate_cache_and_stream(txn, self.is_user_approved, (user_id,)) + async def user_delete_access_tokens( + self, + user_id: str, + except_token_id: Optional[int] = None, + device_id: Optional[str] = None, + ) -> List[Tuple[str, int, Optional[str]]]: + """ + Invalidate access and refresh tokens belonging to a user + + Args: + user_id: ID of user the tokens belong to + except_token_id: access_tokens ID which should *not* be deleted + device_id: ID of device the tokens are associated with. + If None, tokens associated with any device (or no device) will + be deleted + Returns: + A tuple of (token, token id, device id) for each of the deleted tokens + """ + + def f(txn: LoggingTransaction) -> List[Tuple[str, int, Optional[str]]]: + keyvalues = {"user_id": user_id} + if device_id is not None: + keyvalues["device_id"] = device_id + + items = keyvalues.items() + where_clause = " AND ".join(k + " = ?" for k, _ in items) + values: List[Union[str, int]] = [v for _, v in items] + # Conveniently, refresh_tokens and access_tokens both use the user_id and device_id fields. Only caveat + # is the `except_token_id` param that is tricky to get right, so for now we're just using the same where + # clause and values before we handle that. This seems to be only used in the "set password" handler. + refresh_where_clause = where_clause + refresh_values = values.copy() + if except_token_id: + # TODO: support that for refresh tokens + where_clause += " AND id != ?" + values.append(except_token_id) + + txn.execute( + "SELECT token, id, device_id FROM access_tokens WHERE %s" + % where_clause, + values, + ) + tokens_and_devices = [(r[0], r[1], r[2]) for r in txn] + + self._invalidate_cache_and_stream_bulk( + txn, + self.get_user_by_access_token, + [(token,) for token, _, _ in tokens_and_devices], + ) + + txn.execute("DELETE FROM access_tokens WHERE %s" % where_clause, values) + + txn.execute( + "DELETE FROM refresh_tokens WHERE %s" % refresh_where_clause, + refresh_values, + ) + + return tokens_and_devices + + return await self.db_pool.runInteraction("user_delete_access_tokens", f) + + async def user_delete_access_tokens_for_devices( + self, + user_id: str, + device_ids: StrCollection, + ) -> List[Tuple[str, int, Optional[str]]]: + """ + Invalidate access and refresh tokens belonging to a user + + Args: + user_id: ID of user the tokens belong to + device_ids: The devices to delete tokens for. + Returns: + A tuple of (token, token id, device id) for each of the deleted tokens + """ + + def user_delete_access_tokens_for_devices_txn( + txn: LoggingTransaction, batch_device_ids: StrCollection + ) -> List[Tuple[str, int, Optional[str]]]: + self.db_pool.simple_delete_many_txn( + txn, + table="refresh_tokens", + keyvalues={"user_id": user_id}, + column="device_id", + values=batch_device_ids, + ) + + clause, args = make_in_list_sql_clause( + txn.database_engine, "device_id", batch_device_ids + ) + args.append(user_id) + + if self.database_engine.supports_returning: + sql = f""" + DELETE FROM access_tokens + WHERE {clause} AND user_id = ? + RETURNING token, id, device_id + """ + txn.execute(sql, args) + tokens_and_devices = txn.fetchall() + else: + tokens_and_devices = self.db_pool.simple_select_many_txn( + txn, + table="access_tokens", + column="device_id", + iterable=batch_device_ids, + keyvalues={"user_id": user_id}, + retcols=("token", "id", "device_id"), + ) + + self.db_pool.simple_delete_many_txn( + txn, + table="access_tokens", + keyvalues={"user_id": user_id}, + column="device_id", + values=batch_device_ids, + ) + + self._invalidate_cache_and_stream_bulk( + txn, + self.get_user_by_access_token, + [(t[0],) for t in tokens_and_devices], + ) + return tokens_and_devices + + results = [] + for batch_device_ids in batch_iter(device_ids, 1000): + tokens_and_devices = await self.db_pool.runInteraction( + "user_delete_access_tokens_for_devices", + user_delete_access_tokens_for_devices_txn, + batch_device_ids, + ) + results.extend(tokens_and_devices) + + return results + + async def delete_access_token(self, access_token: str) -> None: + def f(txn: LoggingTransaction) -> None: + self.db_pool.simple_delete_one_txn( + txn, table="access_tokens", keyvalues={"token": access_token} + ) + + self._invalidate_cache_and_stream( + txn, self.get_user_by_access_token, (access_token,) + ) + + await self.db_pool.runInteraction("delete_access_token", f) + class RegistrationBackgroundUpdateStore(RegistrationWorkerStore): def __init__( @@ -2620,58 +2820,6 @@ class RegistrationStore(RegistrationBackgroundUpdateStore): return next_id - async def set_device_for_refresh_token( - self, user_id: str, old_device_id: str, device_id: str - ) -> None: - """Moves refresh tokens from old device to current device - - Args: - user_id: The user of the devices. - old_device_id: The old device. - device_id: The new device ID. - Returns: - None - """ - - await self.db_pool.simple_update( - "refresh_tokens", - keyvalues={"user_id": user_id, "device_id": old_device_id}, - updatevalues={"device_id": device_id}, - desc="set_device_for_refresh_token", - ) - - def _set_device_for_access_token_txn( - self, txn: LoggingTransaction, token: str, device_id: str - ) -> str: - old_device_id = self.db_pool.simple_select_one_onecol_txn( - txn, "access_tokens", {"token": token}, "device_id" - ) - - self.db_pool.simple_update_txn( - txn, "access_tokens", {"token": token}, {"device_id": device_id} - ) - - self._invalidate_cache_and_stream(txn, self.get_user_by_access_token, (token,)) - - return old_device_id - - async def set_device_for_access_token(self, token: str, device_id: str) -> str: - """Sets the device ID associated with an access token. - - Args: - token: The access token to modify. - device_id: The new device ID. - Returns: - The old device ID associated with the access token. - """ - - return await self.db_pool.runInteraction( - "set_device_for_access_token", - self._set_device_for_access_token_txn, - token, - device_id, - ) - async def user_set_password_hash( self, user_id: str, password_hash: Optional[str] ) -> None: @@ -2743,162 +2891,6 @@ class RegistrationStore(RegistrationBackgroundUpdateStore): await self.db_pool.runInteraction("user_set_consent_server_notice_sent", f) - async def user_delete_access_tokens( - self, - user_id: str, - except_token_id: Optional[int] = None, - device_id: Optional[str] = None, - ) -> List[Tuple[str, int, Optional[str]]]: - """ - Invalidate access and refresh tokens belonging to a user - - Args: - user_id: ID of user the tokens belong to - except_token_id: access_tokens ID which should *not* be deleted - device_id: ID of device the tokens are associated with. - If None, tokens associated with any device (or no device) will - be deleted - Returns: - A tuple of (token, token id, device id) for each of the deleted tokens - """ - - def f(txn: LoggingTransaction) -> List[Tuple[str, int, Optional[str]]]: - keyvalues = {"user_id": user_id} - if device_id is not None: - keyvalues["device_id"] = device_id - - items = keyvalues.items() - where_clause = " AND ".join(k + " = ?" for k, _ in items) - values: List[Union[str, int]] = [v for _, v in items] - # Conveniently, refresh_tokens and access_tokens both use the user_id and device_id fields. Only caveat - # is the `except_token_id` param that is tricky to get right, so for now we're just using the same where - # clause and values before we handle that. This seems to be only used in the "set password" handler. - refresh_where_clause = where_clause - refresh_values = values.copy() - if except_token_id: - # TODO: support that for refresh tokens - where_clause += " AND id != ?" - values.append(except_token_id) - - txn.execute( - "SELECT token, id, device_id FROM access_tokens WHERE %s" - % where_clause, - values, - ) - tokens_and_devices = [(r[0], r[1], r[2]) for r in txn] - - self._invalidate_cache_and_stream_bulk( - txn, - self.get_user_by_access_token, - [(token,) for token, _, _ in tokens_and_devices], - ) - - txn.execute("DELETE FROM access_tokens WHERE %s" % where_clause, values) - - txn.execute( - "DELETE FROM refresh_tokens WHERE %s" % refresh_where_clause, - refresh_values, - ) - - return tokens_and_devices - - return await self.db_pool.runInteraction("user_delete_access_tokens", f) - - async def user_delete_access_tokens_for_devices( - self, - user_id: str, - device_ids: StrCollection, - ) -> List[Tuple[str, int, Optional[str]]]: - """ - Invalidate access and refresh tokens belonging to a user - - Args: - user_id: ID of user the tokens belong to - device_ids: The devices to delete tokens for. - Returns: - A tuple of (token, token id, device id) for each of the deleted tokens - """ - - def user_delete_access_tokens_for_devices_txn( - txn: LoggingTransaction, batch_device_ids: StrCollection - ) -> List[Tuple[str, int, Optional[str]]]: - self.db_pool.simple_delete_many_txn( - txn, - table="refresh_tokens", - keyvalues={"user_id": user_id}, - column="device_id", - values=batch_device_ids, - ) - - clause, args = make_in_list_sql_clause( - txn.database_engine, "device_id", batch_device_ids - ) - args.append(user_id) - - if self.database_engine.supports_returning: - sql = f""" - DELETE FROM access_tokens - WHERE {clause} AND user_id = ? - RETURNING token, id, device_id - """ - txn.execute(sql, args) - tokens_and_devices = txn.fetchall() - else: - tokens_and_devices = self.db_pool.simple_select_many_txn( - txn, - table="access_tokens", - column="device_id", - iterable=batch_device_ids, - keyvalues={"user_id": user_id}, - retcols=("token", "id", "device_id"), - ) - - self.db_pool.simple_delete_many_txn( - txn, - table="access_tokens", - keyvalues={"user_id": user_id}, - column="device_id", - values=batch_device_ids, - ) - - self._invalidate_cache_and_stream_bulk( - txn, - self.get_user_by_access_token, - [(t[0],) for t in tokens_and_devices], - ) - return tokens_and_devices - - results = [] - for batch_device_ids in batch_iter(device_ids, 1000): - tokens_and_devices = await self.db_pool.runInteraction( - "user_delete_access_tokens_for_devices", - user_delete_access_tokens_for_devices_txn, - batch_device_ids, - ) - results.extend(tokens_and_devices) - - return results - - async def delete_access_token(self, access_token: str) -> None: - def f(txn: LoggingTransaction) -> None: - self.db_pool.simple_delete_one_txn( - txn, table="access_tokens", keyvalues={"token": access_token} - ) - - self._invalidate_cache_and_stream( - txn, self.get_user_by_access_token, (access_token,) - ) - - await self.db_pool.runInteraction("delete_access_token", f) - - async def delete_refresh_token(self, refresh_token: str) -> None: - def f(txn: LoggingTransaction) -> None: - self.db_pool.simple_delete_one_txn( - txn, table="refresh_tokens", keyvalues={"token": refresh_token} - ) - - await self.db_pool.runInteraction("delete_refresh_token", f) - async def add_user_pending_deactivation(self, user_id: str) -> None: """ Adds a user to the table of users who need to be parted from all the rooms they're diff --git a/synapse/storage/databases/main/relations.py b/synapse/storage/databases/main/relations.py index 29a001ff92..5edac56ec3 100644 --- a/synapse/storage/databases/main/relations.py +++ b/synapse/storage/databases/main/relations.py @@ -324,7 +324,7 @@ class RelationsWorkerStore(SQLBaseStore): account_data_key=0, push_rules_key=0, to_device_key=0, - device_list_key=0, + device_list_key=MultiWriterStreamToken(stream=0), groups_key=0, un_partial_stated_rooms_key=0, ) diff --git a/synapse/storage/databases/main/room.py b/synapse/storage/databases/main/room.py index 153bdb3510..604365badf 100644 --- a/synapse/storage/databases/main/room.py +++ b/synapse/storage/databases/main/room.py @@ -1574,6 +1574,11 @@ class RoomWorkerStore(CacheInvalidationWorkerStore): """Get the event ID of the initial join that started the partial join, and the device list stream ID at the point we started the partial join. + + This only returns the minimum device list stream ID at the time of + joining, not the full device list stream token. The only impact of this + is that we may be sending again device list updates that we've already + sent to some destinations, which is harmless. """ return cast( diff --git a/synapse/storage/databases/main/stream.py b/synapse/storage/databases/main/stream.py index b6c6b69b22..66280f2f9a 100644 --- a/synapse/storage/databases/main/stream.py +++ b/synapse/storage/databases/main/stream.py @@ -61,7 +61,6 @@ from typing import ( ) import attr -from immutabledict import immutabledict from typing_extensions import assert_never from twisted.internet import defer @@ -657,23 +656,7 @@ class StreamWorkerStore(EventsWorkerStore, SQLBaseStore): component. """ - min_pos = self._stream_id_gen.get_current_token() - - positions = {} - if isinstance(self._stream_id_gen, MultiWriterIdGenerator): - # The `min_pos` is the minimum position that we know all instances - # have finished persisting to, so we only care about instances whose - # positions are ahead of that. (Instance positions can be behind the - # min position as there are times we can work out that the minimum - # position is ahead of the naive minimum across all current - # positions. See MultiWriterIdGenerator for details) - positions = { - i: p - for i, p in self._stream_id_gen.get_positions().items() - if p > min_pos - } - - return RoomStreamToken(stream=min_pos, instance_map=immutabledict(positions)) + return RoomStreamToken.from_generator(self._stream_id_gen) def get_events_stream_id_generator(self) -> MultiWriterIdGenerator: return self._stream_id_gen diff --git a/synapse/streams/events.py b/synapse/streams/events.py index 856f646795..4534068e7c 100644 --- a/synapse/streams/events.py +++ b/synapse/streams/events.py @@ -203,7 +203,7 @@ class EventSources: account_data_key=0, push_rules_key=0, to_device_key=0, - device_list_key=0, + device_list_key=MultiWriterStreamToken(stream=0), groups_key=0, un_partial_stated_rooms_key=0, ) diff --git a/synapse/types/__init__.py b/synapse/types/__init__.py index 5549f3c9f8..d09fd30e81 100644 --- a/synapse/types/__init__.py +++ b/synapse/types/__init__.py @@ -75,6 +75,7 @@ if TYPE_CHECKING: from synapse.appservice.api import ApplicationService from synapse.storage.databases.main import DataStore, PurgeEventsStore from synapse.storage.databases.main.appservice import ApplicationServiceWorkerStore + from synapse.storage.util.id_generators import MultiWriterIdGenerator logger = logging.getLogger(__name__) @@ -570,6 +571,25 @@ class AbstractMultiWriterStreamToken(metaclass=abc.ABCMeta): ), ) + @classmethod + def from_generator(cls, generator: "MultiWriterIdGenerator") -> Self: + """Get the current token out of a MultiWriterIdGenerator""" + + # The `min_pos` is the minimum position that we know all instances + # have finished persisting to, so we only care about instances whose + # positions are ahead of that. (Instance positions can be behind the + # min position as there are times we can work out that the minimum + # position is ahead of the naive minimum across all current + # positions. See MultiWriterIdGenerator for details) + min_pos = generator.get_current_token() + positions = { + instance: position + for instance, position in generator.get_positions().items() + if position > min_pos + } + + return cls(stream=min_pos, instance_map=immutabledict(positions)) + @attr.s(frozen=True, slots=True, order=False) class RoomStreamToken(AbstractMultiWriterStreamToken): @@ -980,7 +1000,9 @@ class StreamToken: account_data_key: int push_rules_key: int to_device_key: int - device_list_key: int + device_list_key: MultiWriterStreamToken = attr.ib( + validator=attr.validators.instance_of(MultiWriterStreamToken) + ) # Note that the groups key is no longer used and may have bogus values. groups_key: int un_partial_stated_rooms_key: int @@ -1021,7 +1043,9 @@ class StreamToken: account_data_key=int(account_data_key), push_rules_key=int(push_rules_key), to_device_key=int(to_device_key), - device_list_key=int(device_list_key), + device_list_key=await MultiWriterStreamToken.parse( + store, device_list_key + ), groups_key=int(groups_key), un_partial_stated_rooms_key=int(un_partial_stated_rooms_key), ) @@ -1040,7 +1064,7 @@ class StreamToken: str(self.account_data_key), str(self.push_rules_key), str(self.to_device_key), - str(self.device_list_key), + await self.device_list_key.to_string(store), # Note that the groups key is no longer used, but it is still # serialized so that there will not be confusion in the future # if additional tokens are added. @@ -1069,6 +1093,12 @@ class StreamToken: StreamKeyType.RECEIPT, self.receipt_key.copy_and_advance(new_value) ) return new_token + elif key == StreamKeyType.DEVICE_LIST: + new_token = self.copy_and_replace( + StreamKeyType.DEVICE_LIST, + self.device_list_key.copy_and_advance(new_value), + ) + return new_token new_token = self.copy_and_replace(key, new_value) new_id = new_token.get_field(key) @@ -1087,7 +1117,11 @@ class StreamToken: @overload def get_field( - self, key: Literal[StreamKeyType.RECEIPT] + self, + key: Literal[ + StreamKeyType.RECEIPT, + StreamKeyType.DEVICE_LIST, + ], ) -> MultiWriterStreamToken: ... @overload @@ -1095,7 +1129,6 @@ class StreamToken: self, key: Literal[ StreamKeyType.ACCOUNT_DATA, - StreamKeyType.DEVICE_LIST, StreamKeyType.PRESENCE, StreamKeyType.PUSH_RULES, StreamKeyType.TO_DEVICE, @@ -1161,7 +1194,16 @@ class StreamToken: StreamToken.START = StreamToken( - RoomStreamToken(stream=0), 0, 0, MultiWriterStreamToken(stream=0), 0, 0, 0, 0, 0, 0 + room_key=RoomStreamToken(stream=0), + presence_key=0, + typing_key=0, + receipt_key=MultiWriterStreamToken(stream=0), + account_data_key=0, + push_rules_key=0, + to_device_key=0, + device_list_key=MultiWriterStreamToken(stream=0), + groups_key=0, + un_partial_stated_rooms_key=0, ) diff --git a/tests/federation/test_federation_sender.py b/tests/federation/test_federation_sender.py index 64e8c12817..267ea0b06e 100644 --- a/tests/federation/test_federation_sender.py +++ b/tests/federation/test_federation_sender.py @@ -30,7 +30,7 @@ from synapse.api.constants import EduTypes, RoomEncryptionAlgorithms from synapse.api.presence import UserPresenceState from synapse.federation.sender.per_destination_queue import MAX_PRESENCE_STATES_PER_EDU from synapse.federation.units import Transaction -from synapse.handlers.device import DeviceHandler +from synapse.handlers.device import DeviceListUpdater, DeviceWriterHandler from synapse.rest import admin from synapse.rest.client import login from synapse.server import HomeServer @@ -500,7 +500,7 @@ class FederationSenderDevicesTestCases(HomeserverTestCase): hs.get_datastores().main.get_current_hosts_in_room = get_current_hosts_in_room # type: ignore[assignment] device_handler = hs.get_device_handler() - assert isinstance(device_handler, DeviceHandler) + assert isinstance(device_handler, DeviceWriterHandler) self.device_handler = device_handler # whenever send_transaction is called, record the edu data @@ -554,6 +554,8 @@ class FederationSenderDevicesTestCases(HomeserverTestCase): "devices": [{"device_id": "D1"}], } + assert isinstance(self.device_handler.device_list_updater, DeviceListUpdater) + self.get_success( self.device_handler.device_list_updater.incoming_device_list_update( "host2", diff --git a/tests/handlers/test_device.py b/tests/handlers/test_device.py index 99a0c50211..1e989ca528 100644 --- a/tests/handlers/test_device.py +++ b/tests/handlers/test_device.py @@ -29,7 +29,7 @@ from twisted.test.proto_helpers import MemoryReactor from synapse.api.constants import RoomEncryptionAlgorithms from synapse.api.errors import NotFoundError, SynapseError from synapse.appservice import ApplicationService -from synapse.handlers.device import MAX_DEVICE_DISPLAY_NAME_LEN, DeviceHandler +from synapse.handlers.device import MAX_DEVICE_DISPLAY_NAME_LEN, DeviceWriterHandler from synapse.rest import admin from synapse.rest.client import devices, login, register from synapse.server import HomeServer @@ -53,7 +53,7 @@ class DeviceTestCase(unittest.HomeserverTestCase): application_service_api=self.appservice_api, ) handler = hs.get_device_handler() - assert isinstance(handler, DeviceHandler) + assert isinstance(handler, DeviceWriterHandler) self.handler = handler self.store = hs.get_datastores().main self.device_message_handler = hs.get_device_message_handler() @@ -229,7 +229,7 @@ class DeviceTestCase(unittest.HomeserverTestCase): # queue a bunch of messages in the inbox requester = create_requester(sender, device_id=DEVICE_ID) - for i in range(DeviceHandler.DEVICE_MSGS_DELETE_BATCH_LIMIT + 10): + for i in range(DeviceWriterHandler.DEVICE_MSGS_DELETE_BATCH_LIMIT + 10): self.get_success( self.device_message_handler.send_device_message( requester, "message_type", {receiver: {"*": {"val": i}}} @@ -462,7 +462,7 @@ class DehydrationTestCase(unittest.HomeserverTestCase): def make_homeserver(self, reactor: MemoryReactor, clock: Clock) -> HomeServer: hs = self.setup_test_homeserver("server") handler = hs.get_device_handler() - assert isinstance(handler, DeviceHandler) + assert isinstance(handler, DeviceWriterHandler) self.handler = handler self.message_handler = hs.get_device_message_handler() self.registration = hs.get_registration_handler() diff --git a/tests/handlers/test_e2e_keys.py b/tests/handlers/test_e2e_keys.py index 323950c2f4..182f9dab5d 100644 --- a/tests/handlers/test_e2e_keys.py +++ b/tests/handlers/test_e2e_keys.py @@ -31,7 +31,7 @@ from twisted.test.proto_helpers import MemoryReactor from synapse.api.constants import RoomEncryptionAlgorithms from synapse.api.errors import Codes, SynapseError from synapse.appservice import ApplicationService -from synapse.handlers.device import DeviceHandler +from synapse.handlers.device import DeviceWriterHandler from synapse.server import HomeServer from synapse.storage.databases.main.appservice import _make_exclusive_regex from synapse.types import JsonDict, UserID @@ -856,7 +856,7 @@ class E2eKeysHandlerTestCase(unittest.HomeserverTestCase): self.get_success(self.handler.upload_signing_keys_for_user(local_user, keys1)) device_handler = self.hs.get_device_handler() - assert isinstance(device_handler, DeviceHandler) + assert isinstance(device_handler, DeviceWriterHandler) e = self.get_failure( device_handler.check_device_registered( user_id=local_user, diff --git a/tests/module_api/test_api.py b/tests/module_api/test_api.py index b6ba472d7d..9972af3aa3 100644 --- a/tests/module_api/test_api.py +++ b/tests/module_api/test_api.py @@ -28,7 +28,7 @@ from synapse.api.constants import EduTypes, EventTypes from synapse.api.errors import NotFoundError from synapse.events import EventBase from synapse.federation.units import Transaction -from synapse.handlers.device import DeviceHandler +from synapse.handlers.device import DeviceWriterHandler from synapse.handlers.presence import UserPresenceState from synapse.handlers.push_rules import InvalidRuleException from synapse.module_api import ModuleApi @@ -819,7 +819,7 @@ class ModuleApiTestCase(BaseModuleApiTestCase): # Delete the device. device_handler = self.hs.get_device_handler() - assert isinstance(device_handler, DeviceHandler) + assert isinstance(device_handler, DeviceWriterHandler) self.get_success(device_handler.delete_devices(user_id, [device_id])) # Check that the callback was called and the pushers still existed. diff --git a/tests/rest/admin/test_device.py b/tests/rest/admin/test_device.py index 531162a6e9..660fa465bd 100644 --- a/tests/rest/admin/test_device.py +++ b/tests/rest/admin/test_device.py @@ -26,7 +26,7 @@ from twisted.test.proto_helpers import MemoryReactor import synapse.rest.admin from synapse.api.errors import Codes -from synapse.handlers.device import DeviceHandler +from synapse.handlers.device import DeviceWriterHandler from synapse.rest.client import devices, login from synapse.server import HomeServer from synapse.util import Clock @@ -42,7 +42,7 @@ class DeviceRestTestCase(unittest.HomeserverTestCase): def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None: handler = hs.get_device_handler() - assert isinstance(handler, DeviceHandler) + assert isinstance(handler, DeviceWriterHandler) self.handler = handler self.admin_user = self.register_user("admin", "pass", admin=True) From 6127aa0d5022c662e81d7d6d5d823c600774d505 Mon Sep 17 00:00:00 2001 From: Alex Durham Date: Fri, 18 Jul 2025 15:23:28 +0200 Subject: [PATCH 023/185] Don't allow tagnames longer than 255 bytes (#18660) --- changelog.d/18660.bugfix | 1 + synapse/rest/client/tags.py | 15 +++++++- tests/rest/client/test_tags.py | 66 ++++++++++++++++++++++++++++++++++ 3 files changed, 81 insertions(+), 1 deletion(-) create mode 100644 changelog.d/18660.bugfix diff --git a/changelog.d/18660.bugfix b/changelog.d/18660.bugfix new file mode 100644 index 0000000000..fd65b43a1b --- /dev/null +++ b/changelog.d/18660.bugfix @@ -0,0 +1 @@ +Don't allow creation of tags with names longer than 255 bytes, as per the spec. \ No newline at end of file diff --git a/synapse/rest/client/tags.py b/synapse/rest/client/tags.py index b6648f3499..fb59efb11f 100644 --- a/synapse/rest/client/tags.py +++ b/synapse/rest/client/tags.py @@ -20,9 +20,10 @@ # import logging +from http import HTTPStatus from typing import TYPE_CHECKING, Tuple -from synapse.api.errors import AuthError +from synapse.api.errors import AuthError, Codes, SynapseError from synapse.http.server import HttpServer from synapse.http.servlet import RestServlet, parse_json_object_from_request from synapse.http.site import SynapseRequest @@ -35,6 +36,8 @@ if TYPE_CHECKING: logger = logging.getLogger(__name__) +MAX_TAG_LENGTH = 255 + class TagListServlet(RestServlet): """ @@ -86,6 +89,16 @@ class TagServlet(RestServlet): requester = await self.auth.get_user_by_req(request) if user_id != requester.user.to_string(): raise AuthError(403, "Cannot add tags for other users.") + + # check if the tag exceeds the length allowed by the matrix-specification + # as defined in: https://spec.matrix.org/v1.15/client-server-api/#events-14 + if len(tag.encode("utf-8")) > MAX_TAG_LENGTH: + raise SynapseError( + HTTPStatus.BAD_REQUEST, + "tag parameter's length is over 255 bytes", + errcode=Codes.INVALID_PARAM, + ) + # Check if the user has any membership in the room and raise error if not. # Although it's not harmful for users to tag random rooms, it's just superfluous # data we don't need to track or allow. diff --git a/tests/rest/client/test_tags.py b/tests/rest/client/test_tags.py index 5d596409e1..aee2f6affb 100644 --- a/tests/rest/client/test_tags.py +++ b/tests/rest/client/test_tags.py @@ -15,6 +15,7 @@ """Tests REST events for /tags paths.""" from http import HTTPStatus +from urllib import parse as urlparse import synapse.rest.admin from synapse.rest.client import login, room, tags @@ -93,3 +94,68 @@ class RoomTaggingTestCase(unittest.HomeserverTestCase): ) # Check that the request failed with the correct error self.assertEqual(channel.code, HTTPStatus.FORBIDDEN, channel.result) + + def test_put_tag_fails_if_tag_is_too_long(self) -> None: + """ + Test that a user cannot add a tag to a room that is longer than the 255 bytes + allowed by the matrix specification. + """ + user1_id = self.register_user("user1", "pass") + user1_tok = self.login(user1_id, "pass") + room_id = self.helper.create_room_as(user1_id, tok=user1_tok) + # create a string which is larger than 255 bytes + tag = "X" * 300 + + # Make the request + channel = self.make_request( + "PUT", + f"/user/{user1_id}/rooms/{room_id}/tags/{tag}", + content={"order": 0.5}, + access_token=user1_tok, + ) + # Check that the request failed + self.assertEqual(channel.code, HTTPStatus.BAD_REQUEST, channel.result) + + def test_put_tag_fails_if_tag_is_too_long_with_graphemes(self) -> None: + """ + Test that a user cannot add a tag to a room that contains graphemes which are in total + longer than the 255 bytes allowed by the matrix specification. + """ + user1_id = self.register_user("user1", "pass") + user1_tok = self.login(user1_id, "pass") + room_id = self.helper.create_room_as(user1_id, tok=user1_tok) + # create a string which is larger than 255 bytes (275) + tag = "👩‍🚒" * 25 + + # Make the request + channel = self.make_request( + "PUT", + f"/user/{user1_id}/rooms/{room_id}/tags/" + + urlparse.quote(tag.encode("utf-8")), + content={"order": 0.5}, + access_token=user1_tok, + ) + # Check that the request failed + self.assertEqual(channel.code, HTTPStatus.BAD_REQUEST, channel.result) + + def test_put_tag_succeeds_with_graphemes(self) -> None: + """ + Test that a user can add a tag to a room that contains graphemes which are in total + less than the 255 bytes allowed by the matrix specification. + """ + user1_id = self.register_user("user1", "pass") + user1_tok = self.login(user1_id, "pass") + room_id = self.helper.create_room_as(user1_id, tok=user1_tok) + # create a string of acceptable length (220 bytes) + tag = "👩‍🚒" * 20 + + # Make the request + channel = self.make_request( + "PUT", + f"/user/{user1_id}/rooms/{room_id}/tags/" + + urlparse.quote(tag.encode("utf-8")), + content={"order": 0.5}, + access_token=user1_tok, + ) + # Check that the request succeeded + self.assertEqual(channel.code, HTTPStatus.OK, channel.result) From c58d7ade38ef5070bfacfd2911a43a92b08f1ac7 Mon Sep 17 00:00:00 2001 From: Strac Consulting Engineers Pty Ltd Date: Sat, 19 Jul 2025 00:37:17 +1000 Subject: [PATCH 024/185] Update msc3861_delegated.py spelling correction (#18697) --- synapse/api/auth/msc3861_delegated.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/api/auth/msc3861_delegated.py b/synapse/api/auth/msc3861_delegated.py index 581c9c1e74..a584ef9ab3 100644 --- a/synapse/api/auth/msc3861_delegated.py +++ b/synapse/api/auth/msc3861_delegated.py @@ -480,7 +480,7 @@ class MSC3861DelegatedAuth(BaseAuth): # XXX: This is a temporary solution so that the admin API can be called by # the OIDC provider. This will be removed once we have OIDC client # credentials grant support in matrix-authentication-service. - logger.info("Admin toked used") + logger.info("Admin token used") # XXX: that user doesn't exist and won't be provisioned. # This is mostly fine for admin calls, but we should also think about doing # requesters without a user_id. From 797fa5728d137e29c034731c11d80305c39dad04 Mon Sep 17 00:00:00 2001 From: Strac Consulting Engineers Pty Ltd Date: Sat, 19 Jul 2025 00:37:34 +1000 Subject: [PATCH 025/185] 18697.misc (#18698) --- changelog.d/18697.misc | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog.d/18697.misc diff --git a/changelog.d/18697.misc b/changelog.d/18697.misc new file mode 100644 index 0000000000..1289e56fd5 --- /dev/null +++ b/changelog.d/18697.misc @@ -0,0 +1 @@ +Correct spelling of 'Admin token used' log line. From 56f5097d1c8de968b845eda9e2abd5207ee76825 Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Fri, 18 Jul 2025 10:28:10 -0500 Subject: [PATCH 026/185] Prevent dirty `Cargo.lock` changes from install (#18693) Spawning from https://github.com/element-hq/synapse/pull/18689 Example CI failure that will stop people from leaving stray `Cargo.lock` changes behind, ``` Error: Cargo.lock has uncommitted changes after install. Please run 'poetry install --extras all' and commit the Cargo.lock changes. ``` --- .github/workflows/tests.yml | 42 ++++++++++++++++++++++++++++++++++++- .rustfmt.toml | 5 +++++ changelog.d/18693.misc | 1 + 3 files changed, 47 insertions(+), 1 deletion(-) create mode 100644 changelog.d/18693.misc diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index f6250c8ed2..4e32349432 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -263,6 +263,43 @@ jobs: - run: cargo clippy --all-features -- -D warnings + lint-rust: + runs-on: ubuntu-latest + needs: changes + if: ${{ needs.changes.outputs.rust == 'true' }} + + steps: + - name: Checkout repository + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + + - name: Install Rust + uses: dtolnay/rust-toolchain@b3b07ba8b418998c39fb20f53e8b695cdcc8de1b # master + with: + toolchain: ${{ env.RUST_VERSION }} + - uses: Swatinem/rust-cache@98c8021b550208e191a6a3145459bfc9fb29c4c0 # v2.8.0 + + - name: Setup Poetry + uses: matrix-org/setup-python-poetry@5bbf6603c5c930615ec8a29f1b5d7d258d905aa4 # v2.0.0 + with: + # Install like a normal project from source with all optional dependencies + extras: all + install-project: "true" + poetry-version: "2.1.1" + + - name: Ensure `Cargo.lock` is up to date (no stray changes after install) + # The `::error::` syntax is using GitHub Actions' error annotations, see + # https://docs.github.com/en/actions/reference/workflow-commands-for-github-actions + run: | + if git diff --quiet Cargo.lock; then + echo "Cargo.lock is up to date" + else + echo "::error::Cargo.lock has uncommitted changes after install. Please run 'poetry install --extras all' and commit the Cargo.lock changes." + git diff --exit-code Cargo.lock + exit 1 + fi + + # This job is split from `lint-rust` because it requires a nightly Rust toolchain + # for some of the unstable options we use in `.rustfmt.toml`. lint-rustfmt: runs-on: ubuntu-latest needs: changes @@ -274,7 +311,8 @@ jobs: - name: Install Rust uses: dtolnay/rust-toolchain@b3b07ba8b418998c39fb20f53e8b695cdcc8de1b # master with: - # We use nightly so that it correctly groups together imports + # We use nightly so that we can use some unstable options that we use in + # `.rustfmt.toml`. toolchain: nightly-2025-04-23 components: rustfmt - uses: Swatinem/rust-cache@98c8021b550208e191a6a3145459bfc9fb29c4c0 # v2.8.0 @@ -309,6 +347,7 @@ jobs: - check-lockfile - lint-clippy - lint-clippy-nightly + - lint-rust - lint-rustfmt - lint-readme runs-on: ubuntu-latest @@ -327,6 +366,7 @@ jobs: lint-pydantic lint-clippy lint-clippy-nightly + lint-rust lint-rustfmt lint-readme diff --git a/.rustfmt.toml b/.rustfmt.toml index bf96e7743d..29e67033cc 100644 --- a/.rustfmt.toml +++ b/.rustfmt.toml @@ -1 +1,6 @@ +# Unstable options are only available on a nightly toolchain and must be opted into +unstable_features = true + +# `group_imports` is an unstable option that requires nightly Rust toolchain. Tracked by +# https://github.com/rust-lang/rustfmt/issues/5083 group_imports = "StdExternalCrate" diff --git a/changelog.d/18693.misc b/changelog.d/18693.misc new file mode 100644 index 0000000000..e2fdaed6b2 --- /dev/null +++ b/changelog.d/18693.misc @@ -0,0 +1 @@ +Prevent dirty `Cargo.lock` changes from install. From 875269eb5359b4e5d8e6b78533757009bc4329ad Mon Sep 17 00:00:00 2001 From: reivilibre Date: Mon, 21 Jul 2025 14:54:28 +0000 Subject: [PATCH 027/185] Add experimental and incomplete support for MSC4306: Thread Subscriptions. (#18674) Implements: [MSC4306](https://github.com/matrix-org/matrix-spec-proposals/blob/rei/msc_thread_subscriptions/proposals/4306-thread-subscriptions.md) (partially) What's missing: - Changes to push rules Signed-off-by: Olivier 'reivilibre --- changelog.d/18674.feature | 1 + docker/configure_workers_and_start.py | 10 + synapse/_scripts/synapse_port_db.py | 1 + synapse/app/generic_worker.py | 4 + synapse/config/experimental.py | 4 + synapse/config/workers.py | 4 + synapse/handlers/deactivate_account.py | 3 + synapse/handlers/thread_subscriptions.py | 126 ++++++ synapse/replication/tcp/handler.py | 14 +- synapse/replication/tcp/streams/__init__.py | 3 + synapse/replication/tcp/streams/_base.py | 44 ++ synapse/rest/client/thread_subscriptions.py | 98 +++++ synapse/server.py | 5 + synapse/storage/databases/main/__init__.py | 5 +- synapse/storage/databases/main/events.py | 4 + .../databases/main/thread_subscriptions.py | 382 ++++++++++++++++++ .../main/delta/92/04_thread_subscriptions.sql | 59 +++ .../04_thread_subscriptions_seq.sql.postgres | 19 + ...thread_subscriptions_comments.sql.postgres | 18 + ...sent_stream_ordering_comments.sql.postgres | 24 ++ synapse/storage/util/id_generators.py | 9 + synapse/types/__init__.py | 3 +- .../tcp/streams/test_thread_subscriptions.py | 157 +++++++ .../rest/client/test_thread_subscriptions.py | 256 ++++++++++++ tests/storage/test_thread_subscriptions.py | 272 +++++++++++++ 25 files changed, 1522 insertions(+), 3 deletions(-) create mode 100644 changelog.d/18674.feature create mode 100644 synapse/handlers/thread_subscriptions.py create mode 100644 synapse/rest/client/thread_subscriptions.py create mode 100644 synapse/storage/databases/main/thread_subscriptions.py create mode 100644 synapse/storage/schema/main/delta/92/04_thread_subscriptions.sql create mode 100644 synapse/storage/schema/main/delta/92/04_thread_subscriptions_seq.sql.postgres create mode 100644 synapse/storage/schema/main/delta/92/05_thread_subscriptions_comments.sql.postgres create mode 100644 synapse/storage/schema/main/delta/92/06_threads_last_sent_stream_ordering_comments.sql.postgres create mode 100644 tests/replication/tcp/streams/test_thread_subscriptions.py create mode 100644 tests/rest/client/test_thread_subscriptions.py create mode 100644 tests/storage/test_thread_subscriptions.py diff --git a/changelog.d/18674.feature b/changelog.d/18674.feature new file mode 100644 index 0000000000..b1a1aa11f1 --- /dev/null +++ b/changelog.d/18674.feature @@ -0,0 +1 @@ +Add experimental and incomplete support for [MSC4306: Thread Subscriptions](https://github.com/matrix-org/matrix-spec-proposals/blob/rei/msc_thread_subscriptions/proposals/4306-thread-subscriptions.md). \ No newline at end of file diff --git a/docker/configure_workers_and_start.py b/docker/configure_workers_and_start.py index 7909b9d932..6212a94042 100755 --- a/docker/configure_workers_and_start.py +++ b/docker/configure_workers_and_start.py @@ -327,6 +327,15 @@ WORKERS_CONFIG: Dict[str, Dict[str, Any]] = { "shared_extra_conf": {}, "worker_extra_conf": "", }, + "thread_subscriptions": { + "app": "synapse.app.generic_worker", + "listener_resources": ["client", "replication"], + "endpoint_patterns": [ + "^/_matrix/client/unstable/io.element.msc4306/.*", + ], + "shared_extra_conf": {}, + "worker_extra_conf": "", + }, } # Templates for sections that may be inserted multiple times in config files @@ -427,6 +436,7 @@ def add_worker_roles_to_shared_config( "to_device", "typing", "push_rules", + "thread_subscriptions", } # Worker-type specific sharding config. Now a single worker can fulfill multiple diff --git a/synapse/_scripts/synapse_port_db.py b/synapse/_scripts/synapse_port_db.py index 6c3e380355..9a0b459e65 100755 --- a/synapse/_scripts/synapse_port_db.py +++ b/synapse/_scripts/synapse_port_db.py @@ -136,6 +136,7 @@ BOOLEAN_COLUMNS = { "has_known_state", "is_encrypted", ], + "thread_subscriptions": ["subscribed", "automatic"], "users": ["shadow_banned", "approved", "locked", "suspended"], "un_partial_stated_event_stream": ["rejection_status_changed"], "users_who_share_rooms": ["share_private"], diff --git a/synapse/app/generic_worker.py b/synapse/app/generic_worker.py index d0924c413b..4f5bea6bd6 100644 --- a/synapse/app/generic_worker.py +++ b/synapse/app/generic_worker.py @@ -104,6 +104,9 @@ from synapse.storage.databases.main.stats import StatsStore from synapse.storage.databases.main.stream import StreamWorkerStore from synapse.storage.databases.main.tags import TagsWorkerStore from synapse.storage.databases.main.task_scheduler import TaskSchedulerWorkerStore +from synapse.storage.databases.main.thread_subscriptions import ( + ThreadSubscriptionsWorkerStore, +) from synapse.storage.databases.main.transactions import TransactionWorkerStore from synapse.storage.databases.main.ui_auth import UIAuthWorkerStore from synapse.storage.databases.main.user_directory import UserDirectoryStore @@ -132,6 +135,7 @@ class GenericWorkerStore( KeyStore, RoomWorkerStore, DirectoryWorkerStore, + ThreadSubscriptionsWorkerStore, PushRulesWorkerStore, ApplicationServiceTransactionWorkerStore, ApplicationServiceWorkerStore, diff --git a/synapse/config/experimental.py b/synapse/config/experimental.py index b14bc97ae7..1b7474034f 100644 --- a/synapse/config/experimental.py +++ b/synapse/config/experimental.py @@ -581,3 +581,7 @@ class ExperimentalConfig(Config): # MSC4155: Invite filtering self.msc4155_enabled: bool = experimental.get("msc4155_enabled", False) + + # MSC4306: Thread Subscriptions + # (and MSC4308: sliding sync extension for thread subscriptions) + self.msc4306_enabled: bool = experimental.get("msc4306_enabled", False) diff --git a/synapse/config/workers.py b/synapse/config/workers.py index c0c8a13861..825ba78482 100644 --- a/synapse/config/workers.py +++ b/synapse/config/workers.py @@ -174,6 +174,10 @@ class WriterLocations: default=[MAIN_PROCESS_INSTANCE_NAME], converter=_instance_to_list_converter, ) + thread_subscriptions: List[str] = attr.ib( + default=["master"], + converter=_instance_to_list_converter, + ) @attr.s(auto_attribs=True) diff --git a/synapse/handlers/deactivate_account.py b/synapse/handlers/deactivate_account.py index 8d4d84bed1..305363892f 100644 --- a/synapse/handlers/deactivate_account.py +++ b/synapse/handlers/deactivate_account.py @@ -187,6 +187,9 @@ class DeactivateAccountHandler: # Remove account data (including ignored users and push rules). await self.store.purge_account_data_for_user(user_id) + # Remove thread subscriptions for the user + await self.store.purge_thread_subscription_settings_for_user(user_id) + # Delete any server-side backup keys await self.store.bulk_delete_backup_keys_and_versions_for_user(user_id) diff --git a/synapse/handlers/thread_subscriptions.py b/synapse/handlers/thread_subscriptions.py new file mode 100644 index 0000000000..79e4d6040d --- /dev/null +++ b/synapse/handlers/thread_subscriptions.py @@ -0,0 +1,126 @@ +import logging +from typing import TYPE_CHECKING, Optional + +from synapse.api.errors import AuthError, NotFoundError +from synapse.storage.databases.main.thread_subscriptions import ThreadSubscription +from synapse.types import UserID + +if TYPE_CHECKING: + from synapse.server import HomeServer + +logger = logging.getLogger(__name__) + + +class ThreadSubscriptionsHandler: + def __init__(self, hs: "HomeServer"): + self.store = hs.get_datastores().main + self.event_handler = hs.get_event_handler() + self.auth = hs.get_auth() + + async def get_thread_subscription_settings( + self, + user_id: UserID, + room_id: str, + thread_root_event_id: str, + ) -> Optional[ThreadSubscription]: + """Get thread subscription settings for a specific thread and user. + Checks that the thread root is both a real event and also that it is visible + to the user. + + Args: + user_id: The ID of the user + thread_root_event_id: The event ID of the thread root + + Returns: + A `ThreadSubscription` containing the active subscription settings or None if not set + """ + # First check that the user can access the thread root event + # and that it exists + try: + event = await self.event_handler.get_event( + user_id, room_id, thread_root_event_id + ) + if event is None: + raise NotFoundError("No such thread root") + except AuthError: + raise NotFoundError("No such thread root") + + return await self.store.get_subscription_for_thread( + user_id.to_string(), event.room_id, thread_root_event_id + ) + + async def subscribe_user_to_thread( + self, + user_id: UserID, + room_id: str, + thread_root_event_id: str, + *, + automatic: bool, + ) -> Optional[int]: + """Sets or updates a user's subscription settings for a specific thread root. + + Args: + requester_user_id: The ID of the user whose settings are being updated. + thread_root_event_id: The event ID of the thread root. + automatic: whether the user was subscribed by an automatic decision by + their client. + + Returns: + The stream ID for this update, if the update isn't no-opped. + + Raises: + NotFoundError if the user cannot access the thread root event, or it isn't + known to this homeserver. + """ + # First check that the user can access the thread root event + # and that it exists + try: + event = await self.event_handler.get_event( + user_id, room_id, thread_root_event_id + ) + if event is None: + raise NotFoundError("No such thread root") + except AuthError: + logger.info("rejecting thread subscriptions change (thread not accessible)") + raise NotFoundError("No such thread root") + + return await self.store.subscribe_user_to_thread( + user_id.to_string(), + event.room_id, + thread_root_event_id, + automatic=automatic, + ) + + async def unsubscribe_user_from_thread( + self, user_id: UserID, room_id: str, thread_root_event_id: str + ) -> Optional[int]: + """Clears a user's subscription settings for a specific thread root. + + Args: + requester_user_id: The ID of the user whose settings are being updated. + thread_root_event_id: The event ID of the thread root. + + Returns: + The stream ID for this update, if the update isn't no-opped. + + Raises: + NotFoundError if the user cannot access the thread root event, or it isn't + known to this homeserver. + """ + # First check that the user can access the thread root event + # and that it exists + try: + event = await self.event_handler.get_event( + user_id, room_id, thread_root_event_id + ) + if event is None: + raise NotFoundError("No such thread root") + except AuthError: + logger.info("rejecting thread subscriptions change (thread not accessible)") + raise NotFoundError("No such thread root") + + return await self.store.unsubscribe_user_from_thread( + user_id.to_string(), + event.room_id, + thread_root_event_id, + ) diff --git a/synapse/replication/tcp/handler.py b/synapse/replication/tcp/handler.py index e434bed3e5..3611c678c2 100644 --- a/synapse/replication/tcp/handler.py +++ b/synapse/replication/tcp/handler.py @@ -72,7 +72,10 @@ from synapse.replication.tcp.streams import ( ToDeviceStream, TypingStream, ) -from synapse.replication.tcp.streams._base import DeviceListsStream +from synapse.replication.tcp.streams._base import ( + DeviceListsStream, + ThreadSubscriptionsStream, +) if TYPE_CHECKING: from synapse.server import HomeServer @@ -186,6 +189,15 @@ class ReplicationCommandHandler: continue + if isinstance(stream, ThreadSubscriptionsStream): + if ( + hs.get_instance_name() + in hs.config.worker.writers.thread_subscriptions + ): + self._streams_to_replicate.append(stream) + + continue + if isinstance(stream, DeviceListsStream): if hs.get_instance_name() in hs.config.worker.writers.device_lists: self._streams_to_replicate.append(stream) diff --git a/synapse/replication/tcp/streams/__init__.py b/synapse/replication/tcp/streams/__init__.py index 677dcb7b40..25c15e5d48 100644 --- a/synapse/replication/tcp/streams/__init__.py +++ b/synapse/replication/tcp/streams/__init__.py @@ -41,6 +41,7 @@ from synapse.replication.tcp.streams._base import ( PushRulesStream, ReceiptsStream, Stream, + ThreadSubscriptionsStream, ToDeviceStream, TypingStream, ) @@ -67,6 +68,7 @@ STREAMS_MAP = { ToDeviceStream, FederationStream, AccountDataStream, + ThreadSubscriptionsStream, UnPartialStatedRoomStream, UnPartialStatedEventStream, ) @@ -86,6 +88,7 @@ __all__ = [ "DeviceListsStream", "ToDeviceStream", "AccountDataStream", + "ThreadSubscriptionsStream", "UnPartialStatedRoomStream", "UnPartialStatedEventStream", ] diff --git a/synapse/replication/tcp/streams/_base.py b/synapse/replication/tcp/streams/_base.py index ebf5964d29..3ef86486e6 100644 --- a/synapse/replication/tcp/streams/_base.py +++ b/synapse/replication/tcp/streams/_base.py @@ -723,3 +723,47 @@ class AccountDataStream(_StreamFromIdGen): heapq.merge(room_rows, global_rows, tag_rows, key=lambda row: row[0]) ) return updates, to_token, limited + + +class ThreadSubscriptionsStream(_StreamFromIdGen): + """A thread subscription was changed.""" + + @attr.s(slots=True, auto_attribs=True) + class ThreadSubscriptionsStreamRow: + """Stream to inform workers about changes to thread subscriptions.""" + + user_id: str + room_id: str + event_id: str # The event ID of the thread root + + NAME = "thread_subscriptions" + ROW_TYPE = ThreadSubscriptionsStreamRow + + def __init__(self, hs: Any): + self.store = hs.get_datastores().main + super().__init__( + hs.get_instance_name(), + self._update_function, + self.store._thread_subscriptions_id_gen, + ) + + async def _update_function( + self, instance_name: str, from_token: int, to_token: int, limit: int + ) -> StreamUpdateResult: + updates = await self.store.get_updated_thread_subscriptions( + from_token, to_token, limit + ) + rows = [ + ( + stream_id, + # These are the args to `ThreadSubscriptionsStreamRow` + (user_id, room_id, event_id), + ) + for stream_id, user_id, room_id, event_id in updates + ] + + logger.error("TS %d->%d %r", from_token, to_token, rows) + if not rows: + return [], to_token, False + + return rows, rows[-1][0], len(updates) == limit diff --git a/synapse/rest/client/thread_subscriptions.py b/synapse/rest/client/thread_subscriptions.py new file mode 100644 index 0000000000..5307132ec3 --- /dev/null +++ b/synapse/rest/client/thread_subscriptions.py @@ -0,0 +1,98 @@ +from http import HTTPStatus +from typing import Tuple + +from synapse._pydantic_compat import StrictBool +from synapse.api.errors import Codes, NotFoundError, SynapseError +from synapse.http.server import HttpServer +from synapse.http.servlet import ( + RestServlet, + parse_and_validate_json_object_from_request, +) +from synapse.http.site import SynapseRequest +from synapse.rest.client._base import client_patterns +from synapse.server import HomeServer +from synapse.types import JsonDict, RoomID +from synapse.types.rest import RequestBodyModel + + +class ThreadSubscriptionsRestServlet(RestServlet): + PATTERNS = client_patterns( + "/io.element.msc4306/rooms/(?P[^/]*)/thread/(?P[^/]*)/subscription$", + unstable=True, + releases=(), + ) + CATEGORY = "Thread Subscriptions requests (unstable)" + + def __init__(self, hs: "HomeServer"): + self.auth = hs.get_auth() + self.is_mine = hs.is_mine + self.store = hs.get_datastores().main + self.handler = hs.get_thread_subscriptions_handler() + + class PutBody(RequestBodyModel): + automatic: StrictBool + + async def on_GET( + self, request: SynapseRequest, room_id: str, thread_root_id: str + ) -> Tuple[int, JsonDict]: + RoomID.from_string(room_id) + if not thread_root_id.startswith("$"): + raise SynapseError( + HTTPStatus.BAD_REQUEST, "Invalid event ID", errcode=Codes.INVALID_PARAM + ) + requester = await self.auth.get_user_by_req(request) + + subscription = await self.handler.get_thread_subscription_settings( + requester.user, + room_id, + thread_root_id, + ) + + if subscription is None: + raise NotFoundError("Not subscribed.") + + return HTTPStatus.OK, {"automatic": subscription.automatic} + + async def on_PUT( + self, request: SynapseRequest, room_id: str, thread_root_id: str + ) -> Tuple[int, JsonDict]: + RoomID.from_string(room_id) + if not thread_root_id.startswith("$"): + raise SynapseError( + HTTPStatus.BAD_REQUEST, "Invalid event ID", errcode=Codes.INVALID_PARAM + ) + requester = await self.auth.get_user_by_req(request) + + body = parse_and_validate_json_object_from_request(request, self.PutBody) + + await self.handler.subscribe_user_to_thread( + requester.user, + room_id, + thread_root_id, + automatic=body.automatic, + ) + + return HTTPStatus.OK, {} + + async def on_DELETE( + self, request: SynapseRequest, room_id: str, thread_root_id: str + ) -> Tuple[int, JsonDict]: + RoomID.from_string(room_id) + if not thread_root_id.startswith("$"): + raise SynapseError( + HTTPStatus.BAD_REQUEST, "Invalid event ID", errcode=Codes.INVALID_PARAM + ) + requester = await self.auth.get_user_by_req(request) + + await self.handler.unsubscribe_user_from_thread( + requester.user, + room_id, + thread_root_id, + ) + + return HTTPStatus.OK, {} + + +def register_servlets(hs: "HomeServer", http_server: HttpServer) -> None: + if hs.config.experimental.msc4306_enabled: + ThreadSubscriptionsRestServlet(hs).register(http_server) diff --git a/synapse/server.py b/synapse/server.py index 5270f7792d..231bd14907 100644 --- a/synapse/server.py +++ b/synapse/server.py @@ -117,6 +117,7 @@ from synapse.handlers.sliding_sync import SlidingSyncHandler from synapse.handlers.sso import SsoHandler from synapse.handlers.stats import StatsHandler from synapse.handlers.sync import SyncHandler +from synapse.handlers.thread_subscriptions import ThreadSubscriptionsHandler from synapse.handlers.typing import FollowerTypingHandler, TypingWriterHandler from synapse.handlers.user_directory import UserDirectoryHandler from synapse.handlers.worker_lock import WorkerLocksHandler @@ -789,6 +790,10 @@ class HomeServer(metaclass=abc.ABCMeta): def get_timestamp_lookup_handler(self) -> TimestampLookupHandler: return TimestampLookupHandler(self) + @cache_in_self + def get_thread_subscriptions_handler(self) -> ThreadSubscriptionsHandler: + return ThreadSubscriptionsHandler(self) + @cache_in_self def get_registration_handler(self) -> RegistrationHandler: return RegistrationHandler(self) diff --git a/synapse/storage/databases/main/__init__.py b/synapse/storage/databases/main/__init__.py index 86431f6e40..de55c452ae 100644 --- a/synapse/storage/databases/main/__init__.py +++ b/synapse/storage/databases/main/__init__.py @@ -19,7 +19,6 @@ # [This file includes modifications made by New Vector Limited] # # - import logging from typing import TYPE_CHECKING, List, Optional, Tuple, Union, cast @@ -35,6 +34,9 @@ from synapse.storage.database import ( ) from synapse.storage.databases.main.sliding_sync import SlidingSyncStore from synapse.storage.databases.main.stats import UserSortOrder +from synapse.storage.databases.main.thread_subscriptions import ( + ThreadSubscriptionsWorkerStore, +) from synapse.storage.engines import BaseDatabaseEngine from synapse.storage.types import Cursor from synapse.types import get_domain_from_id @@ -141,6 +143,7 @@ class DataStore( SearchStore, TagsStore, AccountDataStore, + ThreadSubscriptionsWorkerStore, PushRulesWorkerStore, StreamWorkerStore, OpenIdStore, diff --git a/synapse/storage/databases/main/events.py b/synapse/storage/databases/main/events.py index b7fbfdc0ca..741146417f 100644 --- a/synapse/storage/databases/main/events.py +++ b/synapse/storage/databases/main/events.py @@ -2986,6 +2986,10 @@ class PersistEventsStore: # Upsert into the threads table, but only overwrite the value if the # new event is of a later topological order OR if the topological # ordering is equal, but the stream ordering is later. + # (Note by definition that the stream ordering will always be later + # unless this is a backfilled event [= negative stream ordering] + # because we are only persisting this event now and stream_orderings + # are strictly monotonically increasing) sql = """ INSERT INTO threads (room_id, thread_id, latest_event_id, topological_ordering, stream_ordering) VALUES (?, ?, ?, ?, ?) diff --git a/synapse/storage/databases/main/thread_subscriptions.py b/synapse/storage/databases/main/thread_subscriptions.py new file mode 100644 index 0000000000..e04e692e6a --- /dev/null +++ b/synapse/storage/databases/main/thread_subscriptions.py @@ -0,0 +1,382 @@ +# +# This file is licensed under the Affero General Public License (AGPL) version 3. +# +# Copyright (C) 2025 New Vector, Ltd +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# See the GNU Affero General Public License for more details: +# . +import logging +from typing import ( + TYPE_CHECKING, + Any, + Dict, + Iterable, + List, + Optional, + Tuple, + Union, + cast, +) + +import attr + +from synapse.replication.tcp.streams._base import ThreadSubscriptionsStream +from synapse.storage.database import ( + DatabasePool, + LoggingDatabaseConnection, + LoggingTransaction, +) +from synapse.storage.databases.main.cache import CacheInvalidationWorkerStore +from synapse.storage.util.id_generators import MultiWriterIdGenerator +from synapse.util.caches.descriptors import cached + +if TYPE_CHECKING: + from synapse.server import HomeServer + +logger = logging.getLogger(__name__) + + +@attr.s(slots=True, frozen=True, auto_attribs=True) +class ThreadSubscription: + automatic: bool + """ + whether the subscription was made automatically (as opposed to by manual + action from the user) + """ + + +class ThreadSubscriptionsWorkerStore(CacheInvalidationWorkerStore): + def __init__( + self, + database: DatabasePool, + db_conn: LoggingDatabaseConnection, + hs: "HomeServer", + ): + super().__init__(database, db_conn, hs) + + self._can_write_to_thread_subscriptions = ( + self._instance_name in hs.config.worker.writers.thread_subscriptions + ) + + self._thread_subscriptions_id_gen: MultiWriterIdGenerator = ( + MultiWriterIdGenerator( + db_conn=db_conn, + db=database, + notifier=hs.get_replication_notifier(), + stream_name="thread_subscriptions", + instance_name=self._instance_name, + tables=[ + ("thread_subscriptions", "instance_name", "stream_id"), + ], + sequence_name="thread_subscriptions_sequence", + writers=hs.config.worker.writers.thread_subscriptions, + ) + ) + + def process_replication_rows( + self, + stream_name: str, + instance_name: str, + token: int, + rows: Iterable[Any], + ) -> None: + if stream_name == ThreadSubscriptionsStream.NAME: + for row in rows: + self.get_subscription_for_thread.invalidate( + (row.user_id, row.room_id, row.event_id) + ) + + super().process_replication_rows(stream_name, instance_name, token, rows) + + def process_replication_position( + self, stream_name: str, instance_name: str, token: int + ) -> None: + if stream_name == ThreadSubscriptionsStream.NAME: + self._thread_subscriptions_id_gen.advance(instance_name, token) + super().process_replication_position(stream_name, instance_name, token) + + async def subscribe_user_to_thread( + self, user_id: str, room_id: str, thread_root_event_id: str, *, automatic: bool + ) -> Optional[int]: + """Updates a user's subscription settings for a specific thread root. + + If no change would be made to the subscription, does not produce any database change. + + Args: + user_id: The ID of the user whose settings are being updated. + room_id: The ID of the room the thread root belongs to. + thread_root_event_id: The event ID of the thread root. + automatic: Whether the subscription was performed automatically by the user's client. + Only `False` will overwrite an existing value of automatic for a subscription row. + + Returns: + The stream ID for this update, if the update isn't no-opped. + """ + assert self._can_write_to_thread_subscriptions + + def _subscribe_user_to_thread_txn(txn: LoggingTransaction) -> Optional[int]: + already_automatic = self.db_pool.simple_select_one_onecol_txn( + txn, + table="thread_subscriptions", + keyvalues={ + "user_id": user_id, + "event_id": thread_root_event_id, + "room_id": room_id, + "subscribed": True, + }, + retcol="automatic", + allow_none=True, + ) + + if already_automatic is None: + already_subscribed = False + already_automatic = True + else: + already_subscribed = True + # convert int (SQLite bool) to Python bool + already_automatic = bool(already_automatic) + + if already_subscribed and already_automatic == automatic: + # there is nothing we need to do here + return None + + stream_id = self._thread_subscriptions_id_gen.get_next_txn(txn) + + values: Dict[str, Optional[Union[bool, int, str]]] = { + "subscribed": True, + "stream_id": stream_id, + "instance_name": self._instance_name, + "automatic": already_automatic and automatic, + } + + self.db_pool.simple_upsert_txn( + txn, + table="thread_subscriptions", + keyvalues={ + "user_id": user_id, + "event_id": thread_root_event_id, + "room_id": room_id, + }, + values=values, + ) + + txn.call_after( + self.get_subscription_for_thread.invalidate, + (user_id, room_id, thread_root_event_id), + ) + + return stream_id + + return await self.db_pool.runInteraction( + "subscribe_user_to_thread", _subscribe_user_to_thread_txn + ) + + async def unsubscribe_user_from_thread( + self, user_id: str, room_id: str, thread_root_event_id: str + ) -> Optional[int]: + """Unsubscribes a user from a thread. + + If no change would be made to the subscription, does not produce any database change. + + Args: + user_id: The ID of the user whose settings are being updated. + room_id: The ID of the room the thread root belongs to. + thread_root_event_id: The event ID of the thread root. + + Returns: + The stream ID for this update, if the update isn't no-opped. + """ + + assert self._can_write_to_thread_subscriptions + + def _unsubscribe_user_from_thread_txn(txn: LoggingTransaction) -> Optional[int]: + already_subscribed = self.db_pool.simple_select_one_onecol_txn( + txn, + table="thread_subscriptions", + keyvalues={ + "user_id": user_id, + "event_id": thread_root_event_id, + "room_id": room_id, + }, + retcol="subscribed", + allow_none=True, + ) + + if already_subscribed is None or already_subscribed is False: + # there is nothing we need to do here + return None + + stream_id = self._thread_subscriptions_id_gen.get_next_txn(txn) + + self.db_pool.simple_update_txn( + txn, + table="thread_subscriptions", + keyvalues={ + "user_id": user_id, + "event_id": thread_root_event_id, + "room_id": room_id, + "subscribed": True, + }, + updatevalues={ + "subscribed": False, + "stream_id": stream_id, + "instance_name": self._instance_name, + }, + ) + + txn.call_after( + self.get_subscription_for_thread.invalidate, + (user_id, room_id, thread_root_event_id), + ) + + return stream_id + + return await self.db_pool.runInteraction( + "unsubscribe_user_from_thread", _unsubscribe_user_from_thread_txn + ) + + async def purge_thread_subscription_settings_for_user(self, user_id: str) -> None: + """ + Purge all subscriptions for the user. + The fact that subscriptions have been purged will not be streamed; + all stream rows for the user will in fact be removed. + This is intended only for dealing with user deactivation. + """ + + def _purge_thread_subscription_settings_for_user_txn( + txn: LoggingTransaction, + ) -> None: + self.db_pool.simple_delete_txn( + txn, + table="thread_subscriptions", + keyvalues={"user_id": user_id}, + ) + self._invalidate_cache_and_stream( + txn, self.get_subscription_for_thread, (user_id,) + ) + + await self.db_pool.runInteraction( + desc="purge_thread_subscription_settings_for_user", + func=_purge_thread_subscription_settings_for_user_txn, + ) + + @cached(tree=True) + async def get_subscription_for_thread( + self, user_id: str, room_id: str, thread_root_event_id: str + ) -> Optional[ThreadSubscription]: + """Get the thread subscription for a specific thread and user. + + Args: + user_id: The ID of the user + room_id: The ID of the room + thread_root_event_id: The event ID of the thread root + + Returns: + A `ThreadSubscription` dataclass if there is a subscription, + or `None` if there is no subscription. + + If there is a row in the table but `subscribed` is `False`, + behaves the same as if there was no row at all and returns `None`. + """ + row = await self.db_pool.simple_select_one( + table="thread_subscriptions", + keyvalues={ + "user_id": user_id, + "room_id": room_id, + "event_id": thread_root_event_id, + "subscribed": True, + }, + retcols=("automatic",), + allow_none=True, + desc="get_subscription_for_thread", + ) + + if row is None: + return None + + (automatic_rawbool,) = row + + # convert SQLite integer booleans into real booleans + automatic = bool(automatic_rawbool) + + return ThreadSubscription(automatic=automatic) + + def get_max_thread_subscriptions_stream_id(self) -> int: + """Get the current maximum stream_id for thread subscriptions. + + Returns: + The maximum stream_id + """ + return self._thread_subscriptions_id_gen.get_current_token() + + async def get_updated_thread_subscriptions( + self, from_id: int, to_id: int, limit: int + ) -> List[Tuple[int, str, str, str]]: + """Get updates to thread subscriptions between two stream IDs. + + Args: + from_id: The starting stream ID (exclusive) + to_id: The ending stream ID (inclusive) + limit: The maximum number of rows to return + + Returns: + list of (stream_id, user_id, room_id, thread_root_id) tuples + """ + + def get_updated_thread_subscriptions_txn( + txn: LoggingTransaction, + ) -> List[Tuple[int, str, str, str]]: + sql = """ + SELECT stream_id, user_id, room_id, event_id + FROM thread_subscriptions + WHERE ? < stream_id AND stream_id <= ? + ORDER BY stream_id ASC + LIMIT ? + """ + + txn.execute(sql, (from_id, to_id, limit)) + return cast(List[Tuple[int, str, str, str]], txn.fetchall()) + + return await self.db_pool.runInteraction( + "get_updated_thread_subscriptions", + get_updated_thread_subscriptions_txn, + ) + + async def get_updated_thread_subscriptions_for_user( + self, user_id: str, from_id: int, to_id: int, limit: int + ) -> List[Tuple[int, str, str]]: + """Get updates to thread subscriptions for a specific user. + + Args: + user_id: The ID of the user + from_id: The starting stream ID (exclusive) + to_id: The ending stream ID (inclusive) + limit: The maximum number of rows to return + + Returns: + A list of (stream_id, room_id, thread_root_event_id) tuples. + """ + + def get_updated_thread_subscriptions_for_user_txn( + txn: LoggingTransaction, + ) -> List[Tuple[int, str, str]]: + sql = """ + SELECT stream_id, room_id, event_id + FROM thread_subscriptions + WHERE user_id = ? AND ? < stream_id AND stream_id <= ? + ORDER BY stream_id ASC + LIMIT ? + """ + + txn.execute(sql, (user_id, from_id, to_id, limit)) + return [(row[0], row[1], row[2]) for row in txn] + + return await self.db_pool.runInteraction( + "get_updated_thread_subscriptions_for_user", + get_updated_thread_subscriptions_for_user_txn, + ) diff --git a/synapse/storage/schema/main/delta/92/04_thread_subscriptions.sql b/synapse/storage/schema/main/delta/92/04_thread_subscriptions.sql new file mode 100644 index 0000000000..d19dd7a46d --- /dev/null +++ b/synapse/storage/schema/main/delta/92/04_thread_subscriptions.sql @@ -0,0 +1,59 @@ +-- +-- This file is licensed under the Affero General Public License (AGPL) version 3. +-- +-- Copyright (C) 2025 New Vector, Ltd +-- +-- This program is free software: you can redistribute it and/or modify +-- it under the terms of the GNU Affero General Public License as +-- published by the Free Software Foundation, either version 3 of the +-- License, or (at your option) any later version. +-- +-- See the GNU Affero General Public License for more details: +-- . + +-- Introduce a table for tracking users' subscriptions to threads. +CREATE TABLE thread_subscriptions ( + stream_id INTEGER NOT NULL PRIMARY KEY, + instance_name TEXT NOT NULL, + + room_id TEXT NOT NULL, + event_id TEXT NOT NULL, + user_id TEXT NOT NULL, + + subscribed BOOLEAN NOT NULL, + automatic BOOLEAN NOT NULL, + + CONSTRAINT thread_subscriptions_fk_users + FOREIGN KEY (user_id) + REFERENCES users(name), + + CONSTRAINT thread_subscriptions_fk_rooms + FOREIGN KEY (room_id) + -- When we delete a room, we should already have deleted all the events in that room + -- and so there shouldn't be any subscriptions left in that room. + -- So the `ON DELETE CASCADE` should be optional, but included anyway for good measure. + REFERENCES rooms(room_id) ON DELETE CASCADE, + + CONSTRAINT thread_subscriptions_fk_events + FOREIGN KEY (event_id) + REFERENCES events(event_id) ON DELETE CASCADE, + + -- This order provides a useful index for: + -- 1. foreign key constraint on (room_id) + -- 2. foreign key constraint on (room_id, event_id) + -- 3. finding the user's settings for a specific thread (as well as enforcing uniqueness) + UNIQUE (room_id, event_id, user_id) +); + +-- this provides a useful index for finding a user's own rules, +-- potentially scoped to a single room +CREATE INDEX thread_subscriptions_user_room ON thread_subscriptions (user_id, room_id); + +-- this provides a useful way for clients to efficiently find new changes to +-- their subscriptions. +-- (This is necessary to sync subscriptions between multiple devices.) +CREATE INDEX thread_subscriptions_by_user ON thread_subscriptions (user_id, stream_id); + +-- this provides a useful index for deleting the subscriptions when the underlying +-- events are removed. This also covers the foreign key constraint on `events`. +CREATE INDEX thread_subscriptions_by_event ON thread_subscriptions (event_id); diff --git a/synapse/storage/schema/main/delta/92/04_thread_subscriptions_seq.sql.postgres b/synapse/storage/schema/main/delta/92/04_thread_subscriptions_seq.sql.postgres new file mode 100644 index 0000000000..8d53691747 --- /dev/null +++ b/synapse/storage/schema/main/delta/92/04_thread_subscriptions_seq.sql.postgres @@ -0,0 +1,19 @@ +-- +-- This file is licensed under the Affero General Public License (AGPL) version 3. +-- +-- Copyright (C) 2025 New Vector, Ltd +-- +-- This program is free software: you can redistribute it and/or modify +-- it under the terms of the GNU Affero General Public License as +-- published by the Free Software Foundation, either version 3 of the +-- License, or (at your option) any later version. +-- +-- See the GNU Affero General Public License for more details: +-- . + +CREATE SEQUENCE thread_subscriptions_sequence + -- Synapse streams start at 2, because the default position is 1 + -- so any item inserted at position 1 is ignored. + -- This is also what existing streams do, except they use `setval(..., 1)` + -- which is semantically the same except less obvious. + START WITH 2; diff --git a/synapse/storage/schema/main/delta/92/05_thread_subscriptions_comments.sql.postgres b/synapse/storage/schema/main/delta/92/05_thread_subscriptions_comments.sql.postgres new file mode 100644 index 0000000000..b0729894c0 --- /dev/null +++ b/synapse/storage/schema/main/delta/92/05_thread_subscriptions_comments.sql.postgres @@ -0,0 +1,18 @@ +-- +-- This file is licensed under the Affero General Public License (AGPL) version 3. +-- +-- Copyright (C) 2025 New Vector, Ltd +-- +-- This program is free software: you can redistribute it and/or modify +-- it under the terms of the GNU Affero General Public License as +-- published by the Free Software Foundation, either version 3 of the +-- License, or (at your option) any later version. +-- +-- See the GNU Affero General Public License for more details: +-- . + +COMMENT ON TABLE thread_subscriptions IS 'Tracks local users that subscribe to threads'; + +COMMENT ON COLUMN thread_subscriptions.subscribed IS 'Whether the user is subscribed to the thread or not. We track unsubscribed threads because we need to stream the subscription change to the client.'; + +COMMENT ON COLUMN thread_subscriptions.automatic IS 'True if the user was subscribed to the thread automatically by their client, or false if the client manually requested the subscription.'; diff --git a/synapse/storage/schema/main/delta/92/06_threads_last_sent_stream_ordering_comments.sql.postgres b/synapse/storage/schema/main/delta/92/06_threads_last_sent_stream_ordering_comments.sql.postgres new file mode 100644 index 0000000000..3fc7e4b11e --- /dev/null +++ b/synapse/storage/schema/main/delta/92/06_threads_last_sent_stream_ordering_comments.sql.postgres @@ -0,0 +1,24 @@ +-- +-- This file is licensed under the Affero General Public License (AGPL) version 3. +-- +-- Copyright (C) 2025 New Vector, Ltd +-- +-- This program is free software: you can redistribute it and/or modify +-- it under the terms of the GNU Affero General Public License as +-- published by the Free Software Foundation, either version 3 of the +-- License, or (at your option) any later version. +-- +-- See the GNU Affero General Public License for more details: +-- . + +COMMENT ON COLUMN threads.latest_event_id IS + 'the ID of the event that is latest, ordered by (topological_ordering, stream_ordering)'; + +COMMENT ON COLUMN threads.topological_ordering IS + $$the topological ordering of the thread''s LATEST event. +Used as the primary way of ordering threads by recency in a room.$$; + +COMMENT ON COLUMN threads.stream_ordering IS + $$the stream ordering of the thread's LATEST event. +Used as a tie-breaker for ordering threads by recency in a room, when the topological order is a tie. +Also used for recency ordering in sliding sync.$$; diff --git a/synapse/storage/util/id_generators.py b/synapse/storage/util/id_generators.py index fe6b6579e6..026a0517d2 100644 --- a/synapse/storage/util/id_generators.py +++ b/synapse/storage/util/id_generators.py @@ -184,6 +184,12 @@ class MultiWriterIdGenerator(AbstractStreamIdGenerator): Note: Only works with Postgres. + Warning: Streams using this generator start at ID 2, because ID 1 is always assumed + to have been 'seen as persisted'. + Unclear if this extant behaviour is desirable for some reason. + When creating a new sequence for a new stream, + it will be necessary to use `START WITH 2`. + Args: db_conn db @@ -269,6 +275,9 @@ class MultiWriterIdGenerator(AbstractStreamIdGenerator): self._known_persisted_positions: List[int] = [] # The maximum stream ID that we have seen been allocated across any writer. + # Since this defaults to 1, this means that ID 1 is assumed to have already + # been 'seen'. In other words, multi-writer streams start at 2. + # Unclear if this is desirable behaviour. self._max_seen_allocated_stream_id = 1 # The maximum position of the local instance. This can be higher than diff --git a/synapse/types/__init__.py b/synapse/types/__init__.py index d09fd30e81..3b516fce3d 100644 --- a/synapse/types/__init__.py +++ b/synapse/types/__init__.py @@ -362,7 +362,8 @@ class RoomID(DomainSpecificString): @attr.s(slots=True, frozen=True, repr=False) class EventID(DomainSpecificString): - """Structure representing an event id.""" + """Structure representing an event ID which is namespaced to a homeserver. + Room versions 3 and above are not supported by this grammar.""" SIGIL = "$" diff --git a/tests/replication/tcp/streams/test_thread_subscriptions.py b/tests/replication/tcp/streams/test_thread_subscriptions.py new file mode 100644 index 0000000000..30c3415ad4 --- /dev/null +++ b/tests/replication/tcp/streams/test_thread_subscriptions.py @@ -0,0 +1,157 @@ +# +# This file is licensed under the Affero General Public License (AGPL) version 3. +# +# Copyright (C) 2025 New Vector, Ltd +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# See the GNU Affero General Public License for more details: +# . +# + +from twisted.test.proto_helpers import MemoryReactor + +from synapse.replication.tcp.streams._base import ( + _STREAM_UPDATE_TARGET_ROW_COUNT, + ThreadSubscriptionsStream, +) +from synapse.server import HomeServer +from synapse.storage.database import LoggingTransaction +from synapse.util import Clock + +from tests.replication._base import BaseStreamTestCase + + +class ThreadSubscriptionsStreamTestCase(BaseStreamTestCase): + def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None: + super().prepare(reactor, clock, hs) + + # Postgres + def f(txn: LoggingTransaction) -> None: + txn.execute( + """ + ALTER TABLE thread_subscriptions + DROP CONSTRAINT thread_subscriptions_fk_users, + DROP CONSTRAINT thread_subscriptions_fk_rooms, + DROP CONSTRAINT thread_subscriptions_fk_events; + """, + ) + + self.get_success( + self.hs.get_datastores().main.db_pool.runInteraction( + "disable_foreign_keys", f + ) + ) + + def test_thread_subscription_updates(self) -> None: + """Test replication with thread subscription updates""" + store = self.hs.get_datastores().main + + # Create thread subscription updates + updates = [] + room_id = "!test_room:example.com" + + # Generate several thread subscription updates + for i in range(_STREAM_UPDATE_TARGET_ROW_COUNT + 5): + thread_root_id = f"$thread_{i}:example.com" + self.get_success( + store.subscribe_user_to_thread( + "@test_user:example.org", + room_id, + thread_root_id, + automatic=True, + ) + ) + updates.append(thread_root_id) + + # Also add one in a different room + other_room_id = "!other_room:example.com" + other_thread_root_id = "$other_thread:example.com" + self.get_success( + store.subscribe_user_to_thread( + "@test_user:example.org", + other_room_id, + other_thread_root_id, + automatic=False, + ) + ) + + # Not yet connected: no rows should yet have been received + self.assertEqual([], self.test_handler.received_rdata_rows) + + # Now reconnect to pull the updates + self.reconnect() + self.replicate() + + # We should have received all the expected rows in the right order + # Filter the updates to only include thread subscription changes + received_rows = [ + upd + for upd in self.test_handler.received_rdata_rows + if upd[0] == ThreadSubscriptionsStream.NAME + ] + + # Verify all the thread subscription updates + for thread_id in updates: + (stream_name, token, row) = received_rows.pop(0) + self.assertEqual(stream_name, ThreadSubscriptionsStream.NAME) + self.assertIsInstance(row, ThreadSubscriptionsStream.ROW_TYPE) + self.assertEqual(row.user_id, "@test_user:example.org") + self.assertEqual(row.room_id, room_id) + self.assertEqual(row.event_id, thread_id) + + # Verify the last update in the different room + (stream_name, token, row) = received_rows.pop(0) + self.assertEqual(stream_name, ThreadSubscriptionsStream.NAME) + self.assertIsInstance(row, ThreadSubscriptionsStream.ROW_TYPE) + self.assertEqual(row.user_id, "@test_user:example.org") + self.assertEqual(row.room_id, other_room_id) + self.assertEqual(row.event_id, other_thread_root_id) + + self.assertEqual([], received_rows) + + def test_multiple_users_thread_subscription_updates(self) -> None: + """Test replication with thread subscription updates for multiple users""" + store = self.hs.get_datastores().main + room_id = "!test_room:example.com" + thread_root_id = "$thread_root:example.com" + + # Create updates for multiple users + users = ["@user1:example.com", "@user2:example.com", "@user3:example.com"] + for user_id in users: + self.get_success( + store.subscribe_user_to_thread( + user_id, room_id, thread_root_id, automatic=True + ) + ) + + # Check no rows have been received yet + self.replicate() + self.assertEqual([], self.test_handler.received_rdata_rows) + + # Not yet connected: no rows should yet have been received + self.reconnect() + self.replicate() + + # We should have received all the expected rows + # Filter the updates to only include thread subscription changes + received_rows = [ + upd + for upd in self.test_handler.received_rdata_rows + if upd[0] == ThreadSubscriptionsStream.NAME + ] + + # Should have one update per user + self.assertEqual(len(received_rows), len(users)) + + # Verify all updates + for i, user_id in enumerate(users): + (stream_name, token, row) = received_rows[i] + self.assertEqual(stream_name, ThreadSubscriptionsStream.NAME) + self.assertIsInstance(row, ThreadSubscriptionsStream.ROW_TYPE) + self.assertEqual(row.user_id, user_id) + self.assertEqual(row.room_id, room_id) + self.assertEqual(row.event_id, thread_root_id) diff --git a/tests/rest/client/test_thread_subscriptions.py b/tests/rest/client/test_thread_subscriptions.py new file mode 100644 index 0000000000..a5c38753cb --- /dev/null +++ b/tests/rest/client/test_thread_subscriptions.py @@ -0,0 +1,256 @@ +# +# This file is licensed under the Affero General Public License (AGPL) version 3. +# +# Copyright (C) 2025 New Vector, Ltd +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# See the GNU Affero General Public License for more details: +# . + +from http import HTTPStatus + +from twisted.test.proto_helpers import MemoryReactor + +from synapse.rest import admin +from synapse.rest.client import login, profile, room, thread_subscriptions +from synapse.server import HomeServer +from synapse.types import JsonDict +from synapse.util import Clock + +from tests import unittest + +PREFIX = "/_matrix/client/unstable/io.element.msc4306/rooms" + + +class ThreadSubscriptionsTestCase(unittest.HomeserverTestCase): + servlets = [ + admin.register_servlets_for_client_rest_resource, + login.register_servlets, + profile.register_servlets, + room.register_servlets, + thread_subscriptions.register_servlets, + ] + + def default_config(self) -> JsonDict: + config = super().default_config() + config["experimental_features"] = {"msc4306_enabled": True} + return config + + def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None: + self.user_id = self.register_user("user", "password") + self.token = self.login("user", "password") + self.other_user_id = self.register_user("other_user", "password") + self.other_token = self.login("other_user", "password") + + # Create a room and send a message to use as a thread root + self.room_id = self.helper.create_room_as(self.user_id, tok=self.token) + self.helper.join(self.room_id, self.other_user_id, tok=self.other_token) + response = self.helper.send(self.room_id, body="Root message", tok=self.token) + self.root_event_id = response["event_id"] + + # Send a message in the thread + self.helper.send_event( + room_id=self.room_id, + type="m.room.message", + content={ + "body": "Thread message", + "msgtype": "m.text", + "m.relates_to": { + "rel_type": "m.thread", + "event_id": self.root_event_id, + }, + }, + tok=self.token, + ) + + def test_get_thread_subscription_unsubscribed(self) -> None: + """Test retrieving thread subscription when not subscribed.""" + channel = self.make_request( + "GET", + f"{PREFIX}/{self.room_id}/thread/{self.root_event_id}/subscription", + access_token=self.token, + ) + self.assertEqual(channel.code, HTTPStatus.NOT_FOUND) + self.assertEqual(channel.json_body["errcode"], "M_NOT_FOUND") + + def test_get_thread_subscription_nonexistent_thread(self) -> None: + """Test retrieving subscription settings for a nonexistent thread.""" + channel = self.make_request( + "GET", + f"{PREFIX}/{self.room_id}/thread/$nonexistent:example.org/subscription", + access_token=self.token, + ) + self.assertEqual(channel.code, HTTPStatus.NOT_FOUND) + self.assertEqual(channel.json_body["errcode"], "M_NOT_FOUND") + + def test_get_thread_subscription_no_access(self) -> None: + """Test that a user can't get thread subscription for a thread they can't access.""" + self.register_user("no_access", "password") + no_access_token = self.login("no_access", "password") + + channel = self.make_request( + "GET", + f"{PREFIX}/{self.room_id}/thread/{self.root_event_id}/subscription", + access_token=no_access_token, + ) + self.assertEqual(channel.code, HTTPStatus.NOT_FOUND) + self.assertEqual(channel.json_body["errcode"], "M_NOT_FOUND") + + def test_subscribe_manual_then_automatic(self) -> None: + """Test subscribing to a thread, first a manual subscription then an automatic subscription. + The manual subscription wins over the automatic one.""" + channel = self.make_request( + "PUT", + f"{PREFIX}/{self.room_id}/thread/{self.root_event_id}/subscription", + { + "automatic": False, + }, + access_token=self.token, + ) + self.assertEqual(channel.code, HTTPStatus.OK) + + # Assert the subscription was saved + channel = self.make_request( + "GET", + f"{PREFIX}/{self.room_id}/thread/{self.root_event_id}/subscription", + access_token=self.token, + ) + self.assertEqual(channel.code, HTTPStatus.OK) + self.assertEqual(channel.json_body, {"automatic": False}) + + # Now also register an automatic subscription; it should not + # override the manual subscription + channel = self.make_request( + "PUT", + f"{PREFIX}/{self.room_id}/thread/{self.root_event_id}/subscription", + {"automatic": True}, + access_token=self.token, + ) + self.assertEqual(channel.code, HTTPStatus.OK) + + # Assert the manual subscription was not overridden + channel = self.make_request( + "GET", + f"{PREFIX}/{self.room_id}/thread/{self.root_event_id}/subscription", + access_token=self.token, + ) + self.assertEqual(channel.code, HTTPStatus.OK) + self.assertEqual(channel.json_body, {"automatic": False}) + + def test_subscribe_automatic_then_manual(self) -> None: + """Test subscribing to a thread, first an automatic subscription then a manual subscription. + The manual subscription wins over the automatic one.""" + channel = self.make_request( + "PUT", + f"{PREFIX}/{self.room_id}/thread/{self.root_event_id}/subscription", + { + "automatic": True, + }, + access_token=self.token, + ) + self.assertEqual(channel.code, HTTPStatus.OK) + + # Assert the subscription was saved + channel = self.make_request( + "GET", + f"{PREFIX}/{self.room_id}/thread/{self.root_event_id}/subscription", + access_token=self.token, + ) + self.assertEqual(channel.code, HTTPStatus.OK) + self.assertEqual(channel.json_body, {"automatic": True}) + + # Now also register a manual subscription + channel = self.make_request( + "PUT", + f"{PREFIX}/{self.room_id}/thread/{self.root_event_id}/subscription", + {"automatic": False}, + access_token=self.token, + ) + self.assertEqual(channel.code, HTTPStatus.OK) + + # Assert the manual subscription was not overridden + channel = self.make_request( + "GET", + f"{PREFIX}/{self.room_id}/thread/{self.root_event_id}/subscription", + access_token=self.token, + ) + self.assertEqual(channel.code, HTTPStatus.OK) + self.assertEqual(channel.json_body, {"automatic": False}) + + def test_unsubscribe(self) -> None: + """Test subscribing to a thread, then unsubscribing.""" + channel = self.make_request( + "PUT", + f"{PREFIX}/{self.room_id}/thread/{self.root_event_id}/subscription", + { + "automatic": True, + }, + access_token=self.token, + ) + self.assertEqual(channel.code, HTTPStatus.OK) + + # Assert the subscription was saved + channel = self.make_request( + "GET", + f"{PREFIX}/{self.room_id}/thread/{self.root_event_id}/subscription", + access_token=self.token, + ) + self.assertEqual(channel.code, HTTPStatus.OK) + self.assertEqual(channel.json_body, {"automatic": True}) + + # Now also register a manual subscription + channel = self.make_request( + "DELETE", + f"{PREFIX}/{self.room_id}/thread/{self.root_event_id}/subscription", + access_token=self.token, + ) + self.assertEqual(channel.code, HTTPStatus.OK) + + # Assert the manual subscription was not overridden + channel = self.make_request( + "GET", + f"{PREFIX}/{self.room_id}/thread/{self.root_event_id}/subscription", + access_token=self.token, + ) + self.assertEqual(channel.code, HTTPStatus.NOT_FOUND) + self.assertEqual(channel.json_body["errcode"], "M_NOT_FOUND") + + def test_set_thread_subscription_nonexistent_thread(self) -> None: + """Test setting subscription settings for a nonexistent thread.""" + channel = self.make_request( + "PUT", + f"{PREFIX}/{self.room_id}/thread/$nonexistent:example.org/subscription", + {"automatic": True}, + access_token=self.token, + ) + self.assertEqual(channel.code, HTTPStatus.NOT_FOUND) + self.assertEqual(channel.json_body["errcode"], "M_NOT_FOUND") + + def test_set_thread_subscription_no_access(self) -> None: + """Test that a user can't set thread subscription for a thread they can't access.""" + self.register_user("no_access2", "password") + no_access_token = self.login("no_access2", "password") + + channel = self.make_request( + "PUT", + f"{PREFIX}/{self.room_id}/thread/{self.root_event_id}/subscription", + {"automatic": True}, + access_token=no_access_token, + ) + self.assertEqual(channel.code, HTTPStatus.NOT_FOUND) + self.assertEqual(channel.json_body["errcode"], "M_NOT_FOUND") + + def test_invalid_body(self) -> None: + """Test that sending invalid subscription settings is rejected.""" + channel = self.make_request( + "PUT", + f"{PREFIX}/{self.room_id}/thread/{self.root_event_id}/subscription", + # non-boolean `automatic` + {"automatic": "true"}, + access_token=self.token, + ) + self.assertEqual(channel.code, HTTPStatus.BAD_REQUEST) diff --git a/tests/storage/test_thread_subscriptions.py b/tests/storage/test_thread_subscriptions.py new file mode 100644 index 0000000000..dd0b804f1f --- /dev/null +++ b/tests/storage/test_thread_subscriptions.py @@ -0,0 +1,272 @@ +# +# This file is licensed under the Affero General Public License (AGPL) version 3. +# +# Copyright (C) 2025 New Vector, Ltd +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# See the GNU Affero General Public License for more details: +# . +# + +from typing import Optional + +from twisted.test.proto_helpers import MemoryReactor + +from synapse.server import HomeServer +from synapse.storage.database import LoggingTransaction +from synapse.storage.engines.sqlite import Sqlite3Engine +from synapse.util import Clock + +from tests import unittest + + +class ThreadSubscriptionsTestCase(unittest.HomeserverTestCase): + def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None: + self.store = self.hs.get_datastores().main + self.user_id = "@user:test" + self.room_id = "!room:test" + self.thread_root_id = "$thread_root:test" + self.other_thread_root_id = "$other_thread_root:test" + + # Disable foreign key checks for testing + # This allows us to insert test data without having to create actual events + db_pool = self.store.db_pool + if isinstance(db_pool.engine, Sqlite3Engine): + self.get_success( + db_pool.execute("disable_foreign_keys", "PRAGMA foreign_keys = OFF;") + ) + else: + # Postgres + def f(txn: LoggingTransaction) -> None: + txn.execute( + """ + ALTER TABLE thread_subscriptions + DROP CONSTRAINT thread_subscriptions_fk_users, + DROP CONSTRAINT thread_subscriptions_fk_rooms, + DROP CONSTRAINT thread_subscriptions_fk_events; + """, + ) + + self.get_success(db_pool.runInteraction("disable_foreign_keys", f)) + + # Create rooms and events in the db to satisfy foreign key constraints + self.get_success(db_pool.simple_insert("rooms", {"room_id": self.room_id})) + + self.get_success( + db_pool.simple_insert( + "events", + { + "event_id": self.thread_root_id, + "room_id": self.room_id, + "topological_ordering": 1, + "stream_ordering": 1, + "type": "m.room.message", + "depth": 1, + "processed": True, + "outlier": False, + }, + ) + ) + + self.get_success( + db_pool.simple_insert( + "events", + { + "event_id": self.other_thread_root_id, + "room_id": self.room_id, + "topological_ordering": 2, + "stream_ordering": 2, + "type": "m.room.message", + "depth": 2, + "processed": True, + "outlier": False, + }, + ) + ) + + # Create the user + self.get_success( + db_pool.simple_insert("users", {"name": self.user_id, "is_guest": 0}) + ) + + def _subscribe( + self, + thread_root_id: str, + *, + automatic: bool, + room_id: Optional[str] = None, + user_id: Optional[str] = None, + ) -> Optional[int]: + if user_id is None: + user_id = self.user_id + + if room_id is None: + room_id = self.room_id + + return self.get_success( + self.store.subscribe_user_to_thread( + user_id, + room_id, + thread_root_id, + automatic=automatic, + ) + ) + + def _unsubscribe( + self, + thread_root_id: str, + room_id: Optional[str] = None, + user_id: Optional[str] = None, + ) -> Optional[int]: + if user_id is None: + user_id = self.user_id + + if room_id is None: + room_id = self.room_id + + return self.get_success( + self.store.unsubscribe_user_from_thread( + user_id, + room_id, + thread_root_id, + ) + ) + + def test_set_and_get_thread_subscription(self) -> None: + """Test basic setting and getting of thread subscriptions.""" + # Initial state: no subscription + subscription = self.get_success( + self.store.get_subscription_for_thread( + self.user_id, self.room_id, self.thread_root_id + ) + ) + self.assertIsNone(subscription) + + # Subscribe + self._subscribe( + self.thread_root_id, + automatic=True, + ) + + # Assert subscription went through + subscription = self.get_success( + self.store.get_subscription_for_thread( + self.user_id, self.room_id, self.thread_root_id + ) + ) + self.assertIsNotNone(subscription) + self.assertTrue(subscription.automatic) # type: ignore + + # Now make it a manual subscription + self._subscribe( + self.thread_root_id, + automatic=False, + ) + + # Assert the manual subscription overrode the automatic one + subscription = self.get_success( + self.store.get_subscription_for_thread( + self.user_id, self.room_id, self.thread_root_id + ) + ) + self.assertFalse(subscription.automatic) # type: ignore + + def test_purge_thread_subscriptions_for_user(self) -> None: + """Test purging all thread subscription settings for a user.""" + # Set subscription settings for multiple threads + self._subscribe(self.thread_root_id, automatic=True) + self._subscribe(self.other_thread_root_id, automatic=False) + + subscriptions = self.get_success( + self.store.get_updated_thread_subscriptions_for_user( + self.user_id, + from_id=0, + to_id=50, + limit=50, + ) + ) + min_id = min(id for (id, _, _) in subscriptions) + self.assertEqual( + subscriptions, + [ + (min_id, self.room_id, self.thread_root_id), + (min_id + 1, self.room_id, self.other_thread_root_id), + ], + ) + + # Purge all settings for the user + self.get_success( + self.store.purge_thread_subscription_settings_for_user(self.user_id) + ) + + # Check user has no subscriptions + subscriptions = self.get_success( + self.store.get_updated_thread_subscriptions_for_user( + self.user_id, + from_id=0, + to_id=50, + limit=50, + ) + ) + self.assertEqual(subscriptions, []) + + def test_get_updated_thread_subscriptions(self) -> None: + """Test getting updated thread subscriptions since a stream ID.""" + + stream_id1 = self._subscribe(self.thread_root_id, automatic=False) + stream_id2 = self._subscribe(self.other_thread_root_id, automatic=True) + assert stream_id1 is not None + assert stream_id2 is not None + + # Get updates since initial ID (should include both changes) + updates = self.get_success( + self.store.get_updated_thread_subscriptions(0, stream_id2, 10) + ) + self.assertEqual(len(updates), 2) + + # Get updates since first change (should include only the second change) + updates = self.get_success( + self.store.get_updated_thread_subscriptions(stream_id1, stream_id2, 10) + ) + self.assertEqual( + updates, + [(stream_id2, self.user_id, self.room_id, self.other_thread_root_id)], + ) + + def test_get_updated_thread_subscriptions_for_user(self) -> None: + """Test getting updated thread subscriptions for a specific user.""" + other_user_id = "@other_user:test" + + # Set thread subscription for main user + stream_id1 = self._subscribe(self.thread_root_id, automatic=True) + assert stream_id1 is not None + + # Set thread subscription for other user + stream_id2 = self._subscribe( + self.other_thread_root_id, + automatic=True, + user_id=other_user_id, + ) + assert stream_id2 is not None + + # Get updates for main user + updates = self.get_success( + self.store.get_updated_thread_subscriptions_for_user( + self.user_id, 0, stream_id2, 10 + ) + ) + self.assertEqual(updates, [(stream_id1, self.room_id, self.thread_root_id)]) + + # Get updates for other user + updates = self.get_success( + self.store.get_updated_thread_subscriptions_for_user( + other_user_id, 0, max(stream_id1, stream_id2), 10 + ) + ) + self.assertEqual( + updates, [(stream_id2, self.room_id, self.other_thread_root_id)] + ) From 8a4e2e826de553669b10245871524f40a04adf08 Mon Sep 17 00:00:00 2001 From: Quentin Gliech Date: Mon, 21 Jul 2025 18:17:43 +0200 Subject: [PATCH 028/185] Dedicated MAS API (#18520) This introduces a dedicated API for MAS to consume. Companion PR on the MAS side: element-hq/matrix-authentication-service#4801 This has a few advantages over the previous admin API: - it works on workers (this will be documented once we stabilise MSC3861 as a whole) - it is more efficient because more focused - it propagates trace contexts from MAS - it is only accessible to MAS (through the shared secret) and will let us remove the weird hack that made this token 'admin' with a ghost '@__oidc_admin:' user The next MAS version should support it, but will be opt-in. The version after that should use this new API by default --------- Co-authored-by: Eric Eastwood --- changelog.d/18520.misc | 1 + synapse/_pydantic_compat.py | 3 + synapse/api/auth/msc3861_delegated.py | 24 +- synapse/rest/synapse/client/__init__.py | 2 + synapse/rest/synapse/mas/__init__.py | 71 ++ synapse/rest/synapse/mas/_base.py | 47 + synapse/rest/synapse/mas/devices.py | 238 ++++ synapse/rest/synapse/mas/users.py | 467 ++++++++ tests/rest/synapse/mas/__init__.py | 12 + tests/rest/synapse/mas/_base.py | 43 + tests/rest/synapse/mas/test_devices.py | 693 +++++++++++ tests/rest/synapse/mas/test_users.py | 1399 +++++++++++++++++++++++ 12 files changed, 2997 insertions(+), 3 deletions(-) create mode 100644 changelog.d/18520.misc create mode 100644 synapse/rest/synapse/mas/__init__.py create mode 100644 synapse/rest/synapse/mas/_base.py create mode 100644 synapse/rest/synapse/mas/devices.py create mode 100644 synapse/rest/synapse/mas/users.py create mode 100644 tests/rest/synapse/mas/__init__.py create mode 100644 tests/rest/synapse/mas/_base.py create mode 100644 tests/rest/synapse/mas/test_devices.py create mode 100644 tests/rest/synapse/mas/test_users.py diff --git a/changelog.d/18520.misc b/changelog.d/18520.misc new file mode 100644 index 0000000000..d005d3b7f7 --- /dev/null +++ b/changelog.d/18520.misc @@ -0,0 +1 @@ +Dedicated internal API for Matrix Authentication Service to Synapse communication. diff --git a/synapse/_pydantic_compat.py b/synapse/_pydantic_compat.py index f0eedf5c6d..e9b43aebe3 100644 --- a/synapse/_pydantic_compat.py +++ b/synapse/_pydantic_compat.py @@ -48,6 +48,7 @@ if TYPE_CHECKING or HAS_PYDANTIC_V2: conint, constr, parse_obj_as, + root_validator, validator, ) from pydantic.v1.error_wrappers import ErrorWrapper @@ -68,6 +69,7 @@ else: conint, constr, parse_obj_as, + root_validator, validator, ) from pydantic.error_wrappers import ErrorWrapper @@ -92,4 +94,5 @@ __all__ = ( "StrictStr", "ValidationError", "validator", + "root_validator", ) diff --git a/synapse/api/auth/msc3861_delegated.py b/synapse/api/auth/msc3861_delegated.py index a584ef9ab3..567f2e834c 100644 --- a/synapse/api/auth/msc3861_delegated.py +++ b/synapse/api/auth/msc3861_delegated.py @@ -369,6 +369,12 @@ class MSC3861DelegatedAuth(BaseAuth): async def is_server_admin(self, requester: Requester) -> bool: return "urn:synapse:admin:*" in requester.scope + def _is_access_token_the_admin_token(self, token: str) -> bool: + admin_token = self._admin_token() + if admin_token is None: + return False + return token == admin_token + async def get_user_by_req( self, request: SynapseRequest, @@ -434,7 +440,7 @@ class MSC3861DelegatedAuth(BaseAuth): requester = await self.get_user_by_access_token(access_token, allow_expired) # Do not record requests from MAS using the virtual `__oidc_admin` user. - if access_token != self._admin_token(): + if not self._is_access_token_the_admin_token(access_token): await self._record_request(request, requester) if not allow_guest and requester.is_guest: @@ -470,13 +476,25 @@ class MSC3861DelegatedAuth(BaseAuth): raise UnrecognizedRequestError(code=404) + def is_request_using_the_admin_token(self, request: SynapseRequest) -> bool: + """ + Check if the request is using the admin token. + + Args: + request: The request to check. + + Returns: + True if the request is using the admin token, False otherwise. + """ + access_token = self.get_access_token_from_request(request) + return self._is_access_token_the_admin_token(access_token) + async def get_user_by_access_token( self, token: str, allow_expired: bool = False, ) -> Requester: - admin_token = self._admin_token() - if admin_token is not None and token == admin_token: + if self._is_access_token_the_admin_token(token): # XXX: This is a temporary solution so that the admin API can be called by # the OIDC provider. This will be removed once we have OIDC client # credentials grant support in matrix-authentication-service. diff --git a/synapse/rest/synapse/client/__init__.py b/synapse/rest/synapse/client/__init__.py index 7b5bfc0421..043c508379 100644 --- a/synapse/rest/synapse/client/__init__.py +++ b/synapse/rest/synapse/client/__init__.py @@ -30,6 +30,7 @@ from synapse.rest.synapse.client.pick_username import pick_username_resource from synapse.rest.synapse.client.rendezvous import MSC4108RendezvousSessionResource from synapse.rest.synapse.client.sso_register import SsoRegisterResource from synapse.rest.synapse.client.unsubscribe import UnsubscribeResource +from synapse.rest.synapse.mas import MasResource if TYPE_CHECKING: from synapse.server import HomeServer @@ -60,6 +61,7 @@ def build_synapse_client_resource_tree(hs: "HomeServer") -> Mapping[str, Resourc from synapse.rest.synapse.client.jwks import JwksResource resources["/_synapse/jwks"] = JwksResource(hs) + resources["/_synapse/mas"] = MasResource(hs) # provider-specific SSO bits. Only load these if they are enabled, since they # rely on optional dependencies. diff --git a/synapse/rest/synapse/mas/__init__.py b/synapse/rest/synapse/mas/__init__.py new file mode 100644 index 0000000000..8115c563d2 --- /dev/null +++ b/synapse/rest/synapse/mas/__init__.py @@ -0,0 +1,71 @@ +# +# This file is licensed under the Affero General Public License (AGPL) version 3. +# +# Copyright (C) 2025 New Vector, Ltd +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# See the GNU Affero General Public License for more details: +# . +# +# + + +import logging +from typing import TYPE_CHECKING + +from twisted.web.resource import Resource + +from synapse.rest.synapse.mas.devices import ( + MasDeleteDeviceResource, + MasSyncDevicesResource, + MasUpdateDeviceDisplayNameResource, + MasUpsertDeviceResource, +) +from synapse.rest.synapse.mas.users import ( + MasAllowCrossSigningResetResource, + MasDeleteUserResource, + MasIsLocalpartAvailableResource, + MasProvisionUserResource, + MasQueryUserResource, + MasReactivateUserResource, + MasSetDisplayNameResource, + MasUnsetDisplayNameResource, +) + +if TYPE_CHECKING: + from synapse.server import HomeServer + + +logger = logging.getLogger(__name__) + + +class MasResource(Resource): + """ + Provides endpoints for MAS to manage user accounts and devices. + + All endpoints are mounted under the path `/_synapse/mas/` and only work + using the MAS admin token. + """ + + def __init__(self, hs: "HomeServer"): + Resource.__init__(self) + self.putChild(b"query_user", MasQueryUserResource(hs)) + self.putChild(b"provision_user", MasProvisionUserResource(hs)) + self.putChild(b"is_localpart_available", MasIsLocalpartAvailableResource(hs)) + self.putChild(b"delete_user", MasDeleteUserResource(hs)) + self.putChild(b"upsert_device", MasUpsertDeviceResource(hs)) + self.putChild(b"delete_device", MasDeleteDeviceResource(hs)) + self.putChild( + b"update_device_display_name", MasUpdateDeviceDisplayNameResource(hs) + ) + self.putChild(b"sync_devices", MasSyncDevicesResource(hs)) + self.putChild(b"reactivate_user", MasReactivateUserResource(hs)) + self.putChild(b"set_displayname", MasSetDisplayNameResource(hs)) + self.putChild(b"unset_displayname", MasUnsetDisplayNameResource(hs)) + self.putChild( + b"allow_cross_signing_reset", MasAllowCrossSigningResetResource(hs) + ) diff --git a/synapse/rest/synapse/mas/_base.py b/synapse/rest/synapse/mas/_base.py new file mode 100644 index 0000000000..caf392fc3a --- /dev/null +++ b/synapse/rest/synapse/mas/_base.py @@ -0,0 +1,47 @@ +# +# This file is licensed under the Affero General Public License (AGPL) version 3. +# +# Copyright (C) 2025 New Vector, Ltd +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# See the GNU Affero General Public License for more details: +# . +# +# + + +from typing import TYPE_CHECKING, cast + +from synapse.api.errors import SynapseError +from synapse.http.server import DirectServeJsonResource + +if TYPE_CHECKING: + from synapse.app.generic_worker import GenericWorkerStore + from synapse.http.site import SynapseRequest + from synapse.server import HomeServer + + +class MasBaseResource(DirectServeJsonResource): + def __init__(self, hs: "HomeServer"): + # Importing this module requires authlib, which is an optional + # dependency but required if msc3861 is enabled + from synapse.api.auth.msc3861_delegated import MSC3861DelegatedAuth + + DirectServeJsonResource.__init__(self, extract_context=True) + auth = hs.get_auth() + assert isinstance(auth, MSC3861DelegatedAuth) + self.msc3861_auth = auth + self.store = cast("GenericWorkerStore", hs.get_datastores().main) + self.hostname = hs.hostname + + def assert_request_is_from_mas(self, request: "SynapseRequest") -> None: + """Assert that the request is coming from MAS itself, not a regular user. + + Throws a 403 if the request is not coming from MAS. + """ + if not self.msc3861_auth.is_request_using_the_admin_token(request): + raise SynapseError(403, "This endpoint must only be called by MAS") diff --git a/synapse/rest/synapse/mas/devices.py b/synapse/rest/synapse/mas/devices.py new file mode 100644 index 0000000000..6cc1153590 --- /dev/null +++ b/synapse/rest/synapse/mas/devices.py @@ -0,0 +1,238 @@ +# +# This file is licensed under the Affero General Public License (AGPL) version 3. +# +# Copyright (C) 2025 New Vector, Ltd +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# See the GNU Affero General Public License for more details: +# . +# +# + +import logging +from http import HTTPStatus +from typing import TYPE_CHECKING, Optional, Tuple + +from synapse._pydantic_compat import StrictStr +from synapse.api.errors import NotFoundError +from synapse.http.servlet import parse_and_validate_json_object_from_request +from synapse.types import JsonDict, UserID +from synapse.types.rest import RequestBodyModel + +if TYPE_CHECKING: + from synapse.http.site import SynapseRequest + from synapse.server import HomeServer + + +from ._base import MasBaseResource + +logger = logging.getLogger(__name__) + + +class MasUpsertDeviceResource(MasBaseResource): + """ + Endpoint for MAS to create or update user devices. + + Takes a localpart, device ID, and optional display name to create new devices + or update existing ones. + + POST /_synapse/mas/upsert_device + {"localpart": "alice", "device_id": "DEVICE123", "display_name": "Alice's Phone"} + """ + + def __init__(self, hs: "HomeServer"): + MasBaseResource.__init__(self, hs) + + self.device_handler = hs.get_device_handler() + + class PostBody(RequestBodyModel): + localpart: StrictStr + device_id: StrictStr + display_name: Optional[StrictStr] + + async def _async_render_POST( + self, request: "SynapseRequest" + ) -> Tuple[int, JsonDict]: + self.assert_request_is_from_mas(request) + + body = parse_and_validate_json_object_from_request(request, self.PostBody) + user_id = UserID(body.localpart, self.hostname) + + # Check the user exists + user = await self.store.get_user_by_id(user_id=str(user_id)) + if user is None: + raise NotFoundError("User not found") + + inserted = await self.device_handler.upsert_device( + user_id=str(user_id), + device_id=body.device_id, + display_name=body.display_name, + ) + + return HTTPStatus.CREATED if inserted else HTTPStatus.OK, {} + + +class MasDeleteDeviceResource(MasBaseResource): + """ + Endpoint for MAS to delete user devices. + + Takes a localpart and device ID to remove the specified device from the user's account. + + POST /_synapse/mas/delete_device + {"localpart": "alice", "device_id": "DEVICE123"} + """ + + def __init__(self, hs: "HomeServer"): + MasBaseResource.__init__(self, hs) + + self.device_handler = hs.get_device_handler() + + class PostBody(RequestBodyModel): + localpart: StrictStr + device_id: StrictStr + + async def _async_render_POST( + self, request: "SynapseRequest" + ) -> Tuple[int, JsonDict]: + self.assert_request_is_from_mas(request) + + body = parse_and_validate_json_object_from_request(request, self.PostBody) + user_id = UserID(body.localpart, self.hostname) + + # Check the user exists + user = await self.store.get_user_by_id(user_id=str(user_id)) + if user is None: + raise NotFoundError("User not found") + + await self.device_handler.delete_devices( + user_id=str(user_id), + device_ids=[body.device_id], + ) + + return HTTPStatus.NO_CONTENT, {} + + +class MasUpdateDeviceDisplayNameResource(MasBaseResource): + """ + Endpoint for MAS to update a device's display name. + + Takes a localpart, device ID, and new display name to update the device's name. + + POST /_synapse/mas/update_device_display_name + {"localpart": "alice", "device_id": "DEVICE123", "display_name": "Alice's New Phone"} + """ + + def __init__(self, hs: "HomeServer"): + MasBaseResource.__init__(self, hs) + + self.device_handler = hs.get_device_handler() + + class PostBody(RequestBodyModel): + localpart: StrictStr + device_id: StrictStr + display_name: StrictStr + + async def _async_render_POST( + self, request: "SynapseRequest" + ) -> Tuple[int, JsonDict]: + self.assert_request_is_from_mas(request) + + body = parse_and_validate_json_object_from_request(request, self.PostBody) + user_id = UserID(body.localpart, self.hostname) + + # Check the user exists + user = await self.store.get_user_by_id(user_id=str(user_id)) + if user is None: + raise NotFoundError("User not found") + + await self.device_handler.update_device( + user_id=str(user_id), + device_id=body.device_id, + content={"display_name": body.display_name}, + ) + + return HTTPStatus.OK, {} + + +class MasSyncDevicesResource(MasBaseResource): + """ + Endpoint for MAS to synchronize a user's complete device list. + + Takes a localpart and a set of device IDs to ensure the user's device list + matches the provided set by adding missing devices and removing extra ones. + + POST /_synapse/mas/sync_devices + {"localpart": "alice", "devices": ["DEVICE123", "DEVICE456"]} + """ + + def __init__(self, hs: "HomeServer"): + MasBaseResource.__init__(self, hs) + + self.device_handler = hs.get_device_handler() + + class PostBody(RequestBodyModel): + localpart: StrictStr + devices: set[StrictStr] + + async def _async_render_POST( + self, request: "SynapseRequest" + ) -> Tuple[int, JsonDict]: + self.assert_request_is_from_mas(request) + + body = parse_and_validate_json_object_from_request(request, self.PostBody) + user_id = UserID(body.localpart, self.hostname) + + # Check the user exists + user = await self.store.get_user_by_id(user_id=str(user_id)) + if user is None: + raise NotFoundError("User not found") + + current_devices = await self.store.get_devices_by_user(user_id=str(user_id)) + current_devices_list = set(current_devices.keys()) + target_device_list = set(body.devices) + + to_add = target_device_list - current_devices_list + to_delete = current_devices_list - target_device_list + + # Log what we're about to do to make it easier to debug if it stops + # mid-way, as this can be a long operation if there are a lot of devices + # to delete or to add. + if to_add and to_delete: + logger.info( + "Syncing %d devices for user %s will add %d devices and delete %d devices", + len(target_device_list), + user_id, + len(to_add), + len(to_delete), + ) + elif to_add: + logger.info( + "Syncing %d devices for user %s will add %d devices", + len(target_device_list), + user_id, + len(to_add), + ) + elif to_delete: + logger.info( + "Syncing %d devices for user %s will delete %d devices", + len(target_device_list), + user_id, + len(to_delete), + ) + + if to_delete: + await self.device_handler.delete_devices( + user_id=str(user_id), device_ids=to_delete + ) + + for device_id in to_add: + await self.device_handler.upsert_device( + user_id=str(user_id), + device_id=device_id, + ) + + return 200, {} diff --git a/synapse/rest/synapse/mas/users.py b/synapse/rest/synapse/mas/users.py new file mode 100644 index 0000000000..09aa13bebb --- /dev/null +++ b/synapse/rest/synapse/mas/users.py @@ -0,0 +1,467 @@ +# +# This file is licensed under the Affero General Public License (AGPL) version 3. +# +# Copyright (C) 2025 New Vector, Ltd +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# See the GNU Affero General Public License for more details: +# . +# +# + +import logging +from http import HTTPStatus +from typing import TYPE_CHECKING, Any, Optional, Tuple, TypedDict + +from synapse._pydantic_compat import StrictBool, StrictStr, root_validator +from synapse.api.errors import NotFoundError, SynapseError +from synapse.http.servlet import ( + parse_and_validate_json_object_from_request, + parse_string, +) +from synapse.types import JsonDict, UserID, UserInfo, create_requester +from synapse.types.rest import RequestBodyModel + +if TYPE_CHECKING: + from synapse.http.site import SynapseRequest + from synapse.server import HomeServer + + +from ._base import MasBaseResource + +logger = logging.getLogger(__name__) + + +class MasQueryUserResource(MasBaseResource): + """ + Endpoint for MAS to query user information by localpart. + + Takes a localpart parameter and returns user profile data including display name, + avatar URL, and account status (suspended/deactivated). + + GET /_synapse/mas/query_user?localpart=alice + """ + + def __init__(self, hs: "HomeServer"): + MasBaseResource.__init__(self, hs) + + class Response(TypedDict): + user_id: str + display_name: Optional[str] + avatar_url: Optional[str] + is_suspended: bool + is_deactivated: bool + + async def _async_render_GET( + self, request: "SynapseRequest" + ) -> Tuple[int, Response]: + self.assert_request_is_from_mas(request) + + localpart = parse_string(request, "localpart", required=True) + user_id = UserID(localpart, self.hostname) + + user: Optional[UserInfo] = await self.store.get_user_by_id(user_id=str(user_id)) + if user is None: + raise NotFoundError("User not found") + + profile = await self.store.get_profileinfo(user_id=user_id) + + return HTTPStatus.OK, self.Response( + user_id=user_id.to_string(), + display_name=profile.display_name, + avatar_url=profile.avatar_url, + is_suspended=user.suspended, + is_deactivated=user.is_deactivated, + ) + + +class MasProvisionUserResource(MasBaseResource): + """ + Endpoint for MAS to create or update user accounts and their profile data. + + Takes a localpart and optional profile fields (display name, avatar URL, email addresses). + Can create new users or update existing ones by setting or unsetting profile fields. + + POST /_synapse/mas/provision_user + {"localpart": "alice", "set_displayname": "Alice", "set_emails": ["alice@example.com"]} + """ + + def __init__(self, hs: "HomeServer"): + MasBaseResource.__init__(self, hs) + self.registration_handler = hs.get_registration_handler() + self.identity_handler = hs.get_identity_handler() + self.auth_handler = hs.get_auth_handler() + self.profile_handler = hs.get_profile_handler() + self.clock = hs.get_clock() + self.auth = hs.get_auth() + + class PostBody(RequestBodyModel): + localpart: StrictStr + + unset_displayname: StrictBool = False + set_displayname: Optional[StrictStr] = None + + unset_avatar_url: StrictBool = False + set_avatar_url: Optional[StrictStr] = None + + unset_emails: StrictBool = False + set_emails: Optional[list[StrictStr]] = None + + @root_validator(pre=True) + def validate_exclusive(cls, values: Any) -> Any: + if "unset_displayname" in values and "set_displayname" in values: + raise ValueError( + "Cannot specify both unset_displayname and set_displayname" + ) + if "unset_avatar_url" in values and "set_avatar_url" in values: + raise ValueError( + "Cannot specify both unset_avatar_url and set_avatar_url" + ) + if "unset_emails" in values and "set_emails" in values: + raise ValueError("Cannot specify both unset_emails and set_emails") + + return values + + async def _async_render_POST( + self, request: "SynapseRequest" + ) -> Tuple[int, JsonDict]: + self.assert_request_is_from_mas(request) + + body = parse_and_validate_json_object_from_request(request, self.PostBody) + + localpart = body.localpart + user_id = UserID(localpart, self.hostname) + + requester = create_requester(user_id=user_id) + existing_user = await self.store.get_user_by_id(user_id=str(user_id)) + if existing_user is None: + created = True + await self.registration_handler.register_user( + localpart=localpart, + default_display_name=body.set_displayname, + bind_emails=body.set_emails, + by_admin=True, + ) + else: + created = False + if body.unset_displayname: + await self.profile_handler.set_displayname( + target_user=user_id, + requester=requester, + new_displayname="", + by_admin=True, + ) + elif body.set_displayname is not None: + await self.profile_handler.set_displayname( + target_user=user_id, + requester=requester, + new_displayname=body.set_displayname, + by_admin=True, + ) + + new_email_list: Optional[set[str]] = None + if body.unset_emails: + new_email_list = set() + elif body.set_emails is not None: + new_email_list = set(body.set_emails) + + if new_email_list is not None: + medium = "email" + current_threepid_list = await self.store.user_get_threepids( + user_id=user_id.to_string() + ) + current_email_list = { + t.address for t in current_threepid_list if t.medium == medium + } + + to_delete = current_email_list - new_email_list + to_add = new_email_list - current_email_list + + for address in to_delete: + await self.identity_handler.try_unbind_threepid( + mxid=user_id.to_string(), + medium=medium, + address=address, + id_server=None, + ) + + await self.auth_handler.delete_local_threepid( + user_id=user_id.to_string(), + medium=medium, + address=address, + ) + + current_time = self.clock.time_msec() + for address in to_add: + await self.auth_handler.add_threepid( + user_id=user_id.to_string(), + medium=medium, + address=address, + validated_at=current_time, + ) + + if body.unset_avatar_url: + await self.profile_handler.set_avatar_url( + target_user=user_id, + requester=requester, + new_avatar_url="", + by_admin=True, + ) + elif body.set_avatar_url is not None: + await self.profile_handler.set_avatar_url( + target_user=user_id, + requester=requester, + new_avatar_url=body.set_avatar_url, + by_admin=True, + ) + + return HTTPStatus.CREATED if created else HTTPStatus.OK, {} + + +class MasIsLocalpartAvailableResource(MasBaseResource): + """ + Endpoint for MAS to check if a localpart is available for user registration. + + Takes a localpart parameter and validates its format and availability, + checking for conflicts with existing users or application service namespaces. + + GET /_synapse/mas/is_localpart_available?localpart=alice + """ + + def __init__(self, hs: "HomeServer"): + super().__init__(hs) + + self.registration_handler = hs.get_registration_handler() + + async def _async_render_GET( + self, request: "SynapseRequest" + ) -> Tuple[int, JsonDict]: + self.assert_request_is_from_mas(request) + localpart = parse_string(request, "localpart") + if localpart is None: + raise SynapseError(400, "Missing localpart") + + await self.registration_handler.check_username(localpart) + + return HTTPStatus.OK, {} + + +class MasDeleteUserResource(MasBaseResource): + """ + Endpoint for MAS to delete/deactivate user accounts. + + Takes a localpart and an erase flag to determine whether to deactivate + the account and optionally erase user data for compliance purposes. + + POST /_synapse/mas/delete_user + {"localpart": "alice", "erase": true} + """ + + def __init__(self, hs: "HomeServer"): + super().__init__(hs) + + self.deactivate_account_handler = hs.get_deactivate_account_handler() + + class PostBody(RequestBodyModel): + localpart: StrictStr + erase: StrictBool + + async def _async_render_POST( + self, request: "SynapseRequest" + ) -> Tuple[int, JsonDict]: + self.assert_request_is_from_mas(request) + + body = parse_and_validate_json_object_from_request(request, self.PostBody) + user_id = UserID(body.localpart, self.hostname) + + # Check the user exists + user = await self.store.get_user_by_id(user_id=str(user_id)) + if user is None: + raise NotFoundError("User not found") + + await self.deactivate_account_handler.deactivate_account( + user_id=user_id.to_string(), + erase_data=body.erase, + requester=create_requester(user_id=user_id), + ) + + return HTTPStatus.OK, {} + + +class MasReactivateUserResource(MasBaseResource): + """ + Endpoint for MAS to reactivate previously deactivated user accounts. + + Takes a localpart parameter to restore access to deactivated accounts. + + POST /_synapse/mas/reactivate_user + {"localpart": "alice"} + """ + + def __init__(self, hs: "HomeServer"): + MasBaseResource.__init__(self, hs) + + self.deactivate_account_handler = hs.get_deactivate_account_handler() + + class PostBody(RequestBodyModel): + localpart: StrictStr + + async def _async_render_POST( + self, request: "SynapseRequest" + ) -> Tuple[int, JsonDict]: + self.assert_request_is_from_mas(request) + + body = parse_and_validate_json_object_from_request(request, self.PostBody) + user_id = UserID(body.localpart, self.hostname) + + # Check the user exists + user = await self.store.get_user_by_id(user_id=str(user_id)) + if user is None: + raise NotFoundError("User not found") + + await self.deactivate_account_handler.activate_account(user_id=str(user_id)) + + return HTTPStatus.OK, {} + + +class MasSetDisplayNameResource(MasBaseResource): + """ + Endpoint for MAS to set a user's display name. + + Takes a localpart and display name to update the user's profile. + + POST /_synapse/mas/set_displayname + {"localpart": "alice", "displayname": "Alice"} + """ + + def __init__(self, hs: "HomeServer"): + MasBaseResource.__init__(self, hs) + + self.profile_handler = hs.get_profile_handler() + self.auth_handler = hs.get_auth_handler() + + class PostBody(RequestBodyModel): + localpart: StrictStr + displayname: StrictStr + + async def _async_render_POST( + self, request: "SynapseRequest" + ) -> Tuple[int, JsonDict]: + self.assert_request_is_from_mas(request) + + body = parse_and_validate_json_object_from_request(request, self.PostBody) + user_id = UserID(body.localpart, self.hostname) + + # Check the user exists + user = await self.store.get_user_by_id(user_id=str(user_id)) + if user is None: + raise NotFoundError("User not found") + + requester = create_requester(user_id=user_id) + + await self.profile_handler.set_displayname( + target_user=requester.user, + requester=requester, + new_displayname=body.displayname, + by_admin=True, + ) + + return HTTPStatus.OK, {} + + +class MasUnsetDisplayNameResource(MasBaseResource): + """ + Endpoint for MAS to clear a user's display name. + + Takes a localpart parameter to remove the display name for the specified user. + + POST /_synapse/mas/unset_displayname + {"localpart": "alice"} + """ + + def __init__(self, hs: "HomeServer"): + MasBaseResource.__init__(self, hs) + + self.profile_handler = hs.get_profile_handler() + self.auth_handler = hs.get_auth_handler() + + class PostBody(RequestBodyModel): + localpart: StrictStr + + async def _async_render_POST( + self, request: "SynapseRequest" + ) -> Tuple[int, JsonDict]: + self.assert_request_is_from_mas(request) + + body = parse_and_validate_json_object_from_request(request, self.PostBody) + user_id = UserID(body.localpart, self.hostname) + + # Check the user exists + user = await self.store.get_user_by_id(user_id=str(user_id)) + if user is None: + raise NotFoundError("User not found") + + requester = create_requester(user_id=user_id) + + await self.profile_handler.set_displayname( + target_user=requester.user, + requester=requester, + new_displayname="", + by_admin=True, + ) + + return HTTPStatus.OK, {} + + +class MasAllowCrossSigningResetResource(MasBaseResource): + """ + Endpoint for MAS to allow cross-signing key reset without user interaction. + + Takes a localpart parameter to temporarily allow cross-signing key replacement + without requiring User-Interactive Authentication (UIA). + + POST /_synapse/mas/allow_cross_signing_reset + {"localpart": "alice"} + """ + + REPLACEMENT_PERIOD_MS = 10 * 60 * 1000 # 10 minutes + + def __init__(self, hs: "HomeServer"): + MasBaseResource.__init__(self, hs) + + self.auth_handler = hs.get_auth_handler() + + class PostBody(RequestBodyModel): + localpart: StrictStr + + async def _async_render_POST( + self, request: "SynapseRequest" + ) -> Tuple[int, JsonDict]: + self.assert_request_is_from_mas(request) + + body = parse_and_validate_json_object_from_request(request, self.PostBody) + user_id = UserID(body.localpart, self.hostname) + + # Check the user exists + user = await self.store.get_user_by_id(user_id=str(user_id)) + if user is None: + raise NotFoundError("User not found") + + timestamp = ( + await self.store.allow_master_cross_signing_key_replacement_without_uia( + user_id=str(user_id), + duration_ms=self.REPLACEMENT_PERIOD_MS, + ) + ) + + if timestamp is None: + # If there are no cross-signing keys, this is a no-op, but we should log + logger.warning( + "User %s has no master cross-signing key", user_id.to_string() + ) + + return HTTPStatus.OK, {} diff --git a/tests/rest/synapse/mas/__init__.py b/tests/rest/synapse/mas/__init__.py new file mode 100644 index 0000000000..db2cfe109f --- /dev/null +++ b/tests/rest/synapse/mas/__init__.py @@ -0,0 +1,12 @@ +# +# This file is licensed under the Affero General Public License (AGPL) version 3. +# +# Copyright (C) 2025 New Vector, Ltd +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# See the GNU Affero General Public License for more details: +# . diff --git a/tests/rest/synapse/mas/_base.py b/tests/rest/synapse/mas/_base.py new file mode 100644 index 0000000000..19d33807a6 --- /dev/null +++ b/tests/rest/synapse/mas/_base.py @@ -0,0 +1,43 @@ +# +# This file is licensed under the Affero General Public License (AGPL) version 3. +# +# Copyright (C) 2025 New Vector, Ltd +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# See the GNU Affero General Public License for more details: +# . + +from twisted.web.resource import Resource + +from synapse.rest.synapse.client import build_synapse_client_resource_tree +from synapse.types import JsonDict + +from tests import unittest + + +class BaseTestCase(unittest.HomeserverTestCase): + SHARED_SECRET = "shared_secret" + + def default_config(self) -> JsonDict: + config = super().default_config() + config["enable_registration"] = False + config["experimental_features"] = { + "msc3861": { + "enabled": True, + "issuer": "https://example.com", + "client_id": "dummy", + "client_auth_method": "client_secret_basic", + "client_secret": "dummy", + "admin_token": self.SHARED_SECRET, + } + } + return config + + def create_resource_dict(self) -> dict[str, Resource]: + base = super().create_resource_dict() + base.update(build_synapse_client_resource_tree(self.hs)) + return base diff --git a/tests/rest/synapse/mas/test_devices.py b/tests/rest/synapse/mas/test_devices.py new file mode 100644 index 0000000000..a7cd58d8ff --- /dev/null +++ b/tests/rest/synapse/mas/test_devices.py @@ -0,0 +1,693 @@ +# +# This file is licensed under the Affero General Public License (AGPL) version 3. +# +# Copyright (C) 2025 New Vector, Ltd +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# See the GNU Affero General Public License for more details: +# . + +from twisted.test.proto_helpers import MemoryReactor + +from synapse.server import HomeServer +from synapse.types import UserID +from synapse.util import Clock + +from tests.unittest import skip_unless +from tests.utils import HAS_AUTHLIB + +from ._base import BaseTestCase + + +@skip_unless(HAS_AUTHLIB, "requires authlib") +class MasUpsertDeviceResource(BaseTestCase): + def prepare( + self, reactor: MemoryReactor, clock: Clock, homeserver: HomeServer + ) -> None: + # Create a user for testing + self.alice_user_id = UserID("alice", "test") + self.get_success( + homeserver.get_registration_handler().register_user( + localpart=self.alice_user_id.localpart, + ) + ) + + def test_other_token(self) -> None: + channel = self.make_request( + "POST", + "/_synapse/mas/upsert_device", + shorthand=False, + access_token="other_token", + content={ + "localpart": "alice", + "device_id": "DEVICE1", + }, + ) + + self.assertEqual(channel.code, 403, channel.json_body) + self.assertEqual( + channel.json_body["error"], "This endpoint must only be called by MAS" + ) + + def test_upsert_device(self) -> None: + store = self.hs.get_datastores().main + + channel = self.make_request( + "POST", + "/_synapse/mas/upsert_device", + shorthand=False, + access_token=self.SHARED_SECRET, + content={ + "localpart": "alice", + "device_id": "DEVICE1", + }, + ) + + # This created a new device, hence the 201 status code + self.assertEqual(channel.code, 201, channel.json_body) + self.assertEqual(channel.json_body, {}) + + # Verify the device exists + device = self.get_success(store.get_device(str(self.alice_user_id), "DEVICE1")) + assert device is not None + self.assertEqual(device["device_id"], "DEVICE1") + self.assertIsNone(device["display_name"]) + + def test_update_existing_device(self) -> None: + store = self.hs.get_datastores().main + device_handler = self.hs.get_device_handler() + + # Create an initial device + self.get_success( + device_handler.upsert_device( + user_id=str(self.alice_user_id), + device_id="DEVICE1", + display_name="Old Name", + ) + ) + + channel = self.make_request( + "POST", + "/_synapse/mas/upsert_device", + shorthand=False, + access_token=self.SHARED_SECRET, + content={ + "localpart": "alice", + "device_id": "DEVICE1", + "display_name": "New Name", + }, + ) + + # This updated an existing device, hence the 200 status code + self.assertEqual(channel.code, 200, channel.json_body) + self.assertEqual(channel.json_body, {}) + + # Verify the device was updated + device = self.get_success(store.get_device(str(self.alice_user_id), "DEVICE1")) + assert device is not None + self.assertEqual(device["display_name"], "New Name") + + def test_upsert_device_with_display_name(self) -> None: + store = self.hs.get_datastores().main + + channel = self.make_request( + "POST", + "/_synapse/mas/upsert_device", + shorthand=False, + access_token=self.SHARED_SECRET, + content={ + "localpart": "alice", + "device_id": "DEVICE1", + "display_name": "Alice's Phone", + }, + ) + + self.assertEqual(channel.code, 201, channel.json_body) + self.assertEqual(channel.json_body, {}) + + # Verify the device exists with correct display name + device = self.get_success(store.get_device(str(self.alice_user_id), "DEVICE1")) + assert device is not None + self.assertEqual(device["display_name"], "Alice's Phone") + + def test_upsert_device_missing_localpart(self) -> None: + channel = self.make_request( + "POST", + "/_synapse/mas/upsert_device", + shorthand=False, + access_token=self.SHARED_SECRET, + content={ + "device_id": "DEVICE1", + }, + ) + + self.assertEqual(channel.code, 400, channel.json_body) + + def test_upsert_device_missing_device_id(self) -> None: + channel = self.make_request( + "POST", + "/_synapse/mas/upsert_device", + shorthand=False, + access_token=self.SHARED_SECRET, + content={ + "localpart": "alice", + }, + ) + + self.assertEqual(channel.code, 400, channel.json_body) + + def test_upsert_device_nonexistent_user(self) -> None: + channel = self.make_request( + "POST", + "/_synapse/mas/upsert_device", + shorthand=False, + access_token=self.SHARED_SECRET, + content={ + "localpart": "nonexistent", + "device_id": "DEVICE1", + }, + ) + + # We get a 404 here as the user doesn't exist + self.assertEqual(channel.code, 404, channel.json_body) + + +@skip_unless(HAS_AUTHLIB, "requires authlib") +class MasDeleteDeviceResource(BaseTestCase): + def prepare( + self, reactor: MemoryReactor, clock: Clock, homeserver: HomeServer + ) -> None: + # Create a user and device for testing + self.alice_user_id = UserID("alice", "test") + self.get_success( + homeserver.get_registration_handler().register_user( + localpart=self.alice_user_id.localpart, + ) + ) + + # Create a device + device_handler = homeserver.get_device_handler() + self.get_success( + device_handler.upsert_device( + user_id=str(self.alice_user_id), + device_id="DEVICE1", + display_name="Test Device", + ) + ) + + def test_other_token(self) -> None: + channel = self.make_request( + "POST", + "/_synapse/mas/delete_device", + shorthand=False, + access_token="other_token", + content={ + "localpart": "alice", + "device_id": "DEVICE1", + }, + ) + + self.assertEqual(channel.code, 403, channel.json_body) + self.assertEqual( + channel.json_body["error"], "This endpoint must only be called by MAS" + ) + + def test_delete_device(self) -> None: + store = self.hs.get_datastores().main + + # Verify device exists before deletion + device = self.get_success(store.get_device(str(self.alice_user_id), "DEVICE1")) + assert device is not None + + channel = self.make_request( + "POST", + "/_synapse/mas/delete_device", + shorthand=False, + access_token=self.SHARED_SECRET, + content={ + "localpart": "alice", + "device_id": "DEVICE1", + }, + ) + + self.assertEqual(channel.code, 204) + + # Verify the device no longer exists + device = self.get_success(store.get_device(str(self.alice_user_id), "DEVICE1")) + self.assertIsNone(device) + + def test_delete_nonexistent_device(self) -> None: + # Deleting a non-existent device should be idempotent + channel = self.make_request( + "POST", + "/_synapse/mas/delete_device", + shorthand=False, + access_token=self.SHARED_SECRET, + content={ + "localpart": "alice", + "device_id": "NONEXISTENT", + }, + ) + + self.assertEqual(channel.code, 204) + + def test_delete_device_missing_localpart(self) -> None: + channel = self.make_request( + "POST", + "/_synapse/mas/delete_device", + shorthand=False, + access_token=self.SHARED_SECRET, + content={ + "device_id": "DEVICE1", + }, + ) + + self.assertEqual(channel.code, 400, channel.json_body) + + def test_delete_device_missing_device_id(self) -> None: + channel = self.make_request( + "POST", + "/_synapse/mas/delete_device", + shorthand=False, + access_token=self.SHARED_SECRET, + content={ + "localpart": "alice", + }, + ) + + self.assertEqual(channel.code, 400, channel.json_body) + + def test_delete_device_nonexistent_user(self) -> None: + channel = self.make_request( + "POST", + "/_synapse/mas/delete_device", + shorthand=False, + access_token=self.SHARED_SECRET, + content={ + "localpart": "nonexistent", + "device_id": "DEVICE1", + }, + ) + + # Should fail on a non-existent user + self.assertEqual(channel.code, 404, channel.json_body) + + +@skip_unless(HAS_AUTHLIB, "requires authlib") +class MasUpdateDeviceDisplayNameResource(BaseTestCase): + def prepare( + self, reactor: MemoryReactor, clock: Clock, homeserver: HomeServer + ) -> None: + # Create a user and device for testing + self.alice_user_id = UserID("alice", "test") + self.get_success( + homeserver.get_registration_handler().register_user( + localpart=self.alice_user_id.localpart, + ) + ) + + # Create a device + device_handler = homeserver.get_device_handler() + self.get_success( + device_handler.upsert_device( + user_id=str(self.alice_user_id), + device_id="DEVICE1", + display_name="Old Name", + ) + ) + + def test_other_token(self) -> None: + channel = self.make_request( + "POST", + "/_synapse/mas/update_device_display_name", + shorthand=False, + access_token="other_token", + content={ + "localpart": "alice", + "device_id": "DEVICE1", + "display_name": "New Name", + }, + ) + + self.assertEqual(channel.code, 403, channel.json_body) + self.assertEqual( + channel.json_body["error"], "This endpoint must only be called by MAS" + ) + + def test_update_device_display_name(self) -> None: + store = self.hs.get_datastores().main + + # Verify initial display name + device = self.get_success(store.get_device(str(self.alice_user_id), "DEVICE1")) + assert device is not None + self.assertEqual(device["display_name"], "Old Name") + + channel = self.make_request( + "POST", + "/_synapse/mas/update_device_display_name", + shorthand=False, + access_token=self.SHARED_SECRET, + content={ + "localpart": "alice", + "device_id": "DEVICE1", + "display_name": "Updated Name", + }, + ) + + self.assertEqual(channel.code, 200, channel.json_body) + self.assertEqual(channel.json_body, {}) + + # Verify the display name was updated + device = self.get_success(store.get_device(str(self.alice_user_id), "DEVICE1")) + assert device is not None + self.assertEqual(device["display_name"], "Updated Name") + + def test_update_nonexistent_device(self) -> None: + # Updating a non-existent device should fail + channel = self.make_request( + "POST", + "/_synapse/mas/update_device_display_name", + shorthand=False, + access_token=self.SHARED_SECRET, + content={ + "localpart": "alice", + "device_id": "NONEXISTENT", + "display_name": "New Name", + }, + ) + + self.assertEqual(channel.code, 404, channel.json_body) + + def test_update_device_display_name_missing_localpart(self) -> None: + channel = self.make_request( + "POST", + "/_synapse/mas/update_device_display_name", + shorthand=False, + access_token=self.SHARED_SECRET, + content={ + "device_id": "DEVICE1", + "display_name": "New Name", + }, + ) + + self.assertEqual(channel.code, 400, channel.json_body) + + def test_update_device_display_name_missing_device_id(self) -> None: + channel = self.make_request( + "POST", + "/_synapse/mas/update_device_display_name", + shorthand=False, + access_token=self.SHARED_SECRET, + content={ + "localpart": "alice", + "display_name": "New Name", + }, + ) + + self.assertEqual(channel.code, 400, channel.json_body) + + def test_update_device_display_name_missing_display_name(self) -> None: + channel = self.make_request( + "POST", + "/_synapse/mas/update_device_display_name", + shorthand=False, + access_token=self.SHARED_SECRET, + content={ + "localpart": "alice", + "device_id": "DEVICE1", + }, + ) + + self.assertEqual(channel.code, 400, channel.json_body) + + def test_update_device_display_name_nonexistent_user(self) -> None: + channel = self.make_request( + "POST", + "/_synapse/mas/update_device_display_name", + shorthand=False, + access_token=self.SHARED_SECRET, + content={ + "localpart": "nonexistent", + "device_id": "DEVICE1", + "display_name": "New Name", + }, + ) + + self.assertEqual(channel.code, 404, channel.json_body) + + +@skip_unless(HAS_AUTHLIB, "requires authlib") +class MasSyncDevicesResource(BaseTestCase): + def prepare( + self, reactor: MemoryReactor, clock: Clock, homeserver: HomeServer + ) -> None: + # Create a user for testing + self.alice_user_id = UserID("alice", "test") + self.get_success( + homeserver.get_registration_handler().register_user( + localpart=self.alice_user_id.localpart, + ) + ) + + # Create some initial devices + device_handler = homeserver.get_device_handler() + for device_id in ["DEVICE1", "DEVICE2", "DEVICE3"]: + self.get_success( + device_handler.upsert_device( + user_id=str(self.alice_user_id), + device_id=device_id, + display_name=f"Device {device_id}", + ) + ) + + def test_other_token(self) -> None: + channel = self.make_request( + "POST", + "/_synapse/mas/sync_devices", + shorthand=False, + access_token="other_token", + content={ + "localpart": "alice", + "devices": ["DEVICE1", "DEVICE2"], + }, + ) + + self.assertEqual(channel.code, 403, channel.json_body) + self.assertEqual( + channel.json_body["error"], "This endpoint must only be called by MAS" + ) + + def test_sync_devices_no_changes(self) -> None: + # Sync with the same devices that already exist + channel = self.make_request( + "POST", + "/_synapse/mas/sync_devices", + shorthand=False, + access_token=self.SHARED_SECRET, + content={ + "localpart": "alice", + "devices": ["DEVICE1", "DEVICE2", "DEVICE3"], + }, + ) + + self.assertEqual(channel.code, 200, channel.json_body) + self.assertEqual(channel.json_body, {}) + + # Verify all devices still exist + store = self.hs.get_datastores().main + devices = self.get_success(store.get_devices_by_user(str(self.alice_user_id))) + self.assertEqual(set(devices.keys()), {"DEVICE1", "DEVICE2", "DEVICE3"}) + + def test_sync_devices_add_only(self) -> None: + # Sync with additional devices + channel = self.make_request( + "POST", + "/_synapse/mas/sync_devices", + shorthand=False, + access_token=self.SHARED_SECRET, + content={ + "localpart": "alice", + "devices": ["DEVICE1", "DEVICE2", "DEVICE3", "DEVICE4", "DEVICE5"], + }, + ) + + self.assertEqual(channel.code, 200, channel.json_body) + self.assertEqual(channel.json_body, {}) + + # Verify new devices were added + store = self.hs.get_datastores().main + devices = self.get_success(store.get_devices_by_user(str(self.alice_user_id))) + self.assertEqual( + set(devices.keys()), {"DEVICE1", "DEVICE2", "DEVICE3", "DEVICE4", "DEVICE5"} + ) + + def test_sync_devices_delete_only(self) -> None: + # Sync with fewer devices + channel = self.make_request( + "POST", + "/_synapse/mas/sync_devices", + shorthand=False, + access_token=self.SHARED_SECRET, + content={ + "localpart": "alice", + "devices": ["DEVICE1"], + }, + ) + + self.assertEqual(channel.code, 200, channel.json_body) + self.assertEqual(channel.json_body, {}) + + # Verify devices were deleted + store = self.hs.get_datastores().main + devices = self.get_success(store.get_devices_by_user(str(self.alice_user_id))) + self.assertEqual(set(devices.keys()), {"DEVICE1"}) + + def test_sync_devices_add_and_delete(self) -> None: + # Sync with a mix of additions and deletions + channel = self.make_request( + "POST", + "/_synapse/mas/sync_devices", + shorthand=False, + access_token=self.SHARED_SECRET, + content={ + "localpart": "alice", + "devices": ["DEVICE1", "DEVICE4", "DEVICE5"], + }, + ) + + self.assertEqual(channel.code, 200, channel.json_body) + self.assertEqual(channel.json_body, {}) + + # Verify the correct devices exist + store = self.hs.get_datastores().main + devices = self.get_success(store.get_devices_by_user(str(self.alice_user_id))) + self.assertEqual(set(devices.keys()), {"DEVICE1", "DEVICE4", "DEVICE5"}) + + def test_sync_devices_empty_list(self) -> None: + # Sync with empty device list (delete all devices) + channel = self.make_request( + "POST", + "/_synapse/mas/sync_devices", + shorthand=False, + access_token=self.SHARED_SECRET, + content={ + "localpart": "alice", + "devices": [], + }, + ) + + self.assertEqual(channel.code, 200, channel.json_body) + self.assertEqual(channel.json_body, {}) + + # Verify all devices were deleted + store = self.hs.get_datastores().main + devices = self.get_success(store.get_devices_by_user(str(self.alice_user_id))) + self.assertEqual(devices, {}) + + def test_sync_devices_for_new_user(self) -> None: + # Test syncing devices for a user that doesn't have any devices yet + bob_user_id = UserID("bob", "test") + self.get_success( + self.hs.get_registration_handler().register_user( + localpart=bob_user_id.localpart, + ) + ) + + channel = self.make_request( + "POST", + "/_synapse/mas/sync_devices", + shorthand=False, + access_token=self.SHARED_SECRET, + content={ + "localpart": "bob", + "devices": ["DEVICE1", "DEVICE2"], + }, + ) + + self.assertEqual(channel.code, 200, channel.json_body) + self.assertEqual(channel.json_body, {}) + + # Verify devices were created + store = self.hs.get_datastores().main + devices = self.get_success(store.get_devices_by_user(str(bob_user_id))) + self.assertEqual(set(devices.keys()), {"DEVICE1", "DEVICE2"}) + + def test_sync_devices_missing_localpart(self) -> None: + channel = self.make_request( + "POST", + "/_synapse/mas/sync_devices", + shorthand=False, + access_token=self.SHARED_SECRET, + content={ + "devices": ["DEVICE1", "DEVICE2"], + }, + ) + + self.assertEqual(channel.code, 400, channel.json_body) + + def test_sync_devices_missing_devices(self) -> None: + channel = self.make_request( + "POST", + "/_synapse/mas/sync_devices", + shorthand=False, + access_token=self.SHARED_SECRET, + content={ + "localpart": "alice", + }, + ) + + self.assertEqual(channel.code, 400, channel.json_body) + + def test_sync_devices_invalid_devices_type(self) -> None: + channel = self.make_request( + "POST", + "/_synapse/mas/sync_devices", + shorthand=False, + access_token=self.SHARED_SECRET, + content={ + "localpart": "alice", + "devices": "not_a_list", + }, + ) + + self.assertEqual(channel.code, 400, channel.json_body) + + def test_sync_devices_nonexistent_user(self) -> None: + # Test syncing devices for a user that doesn't exist + channel = self.make_request( + "POST", + "/_synapse/mas/sync_devices", + shorthand=False, + access_token=self.SHARED_SECRET, + content={ + "localpart": "nonexistent", + "devices": ["DEVICE1", "DEVICE2"], + }, + ) + + self.assertEqual(channel.code, 404, channel.json_body) + + def test_sync_devices_duplicate_device_ids(self) -> None: + # Test syncing with duplicate device IDs (sets should handle this) + channel = self.make_request( + "POST", + "/_synapse/mas/sync_devices", + shorthand=False, + access_token=self.SHARED_SECRET, + content={ + "localpart": "alice", + "devices": ["DEVICE1", "DEVICE1", "DEVICE2"], + }, + ) + + self.assertEqual(channel.code, 200, channel.json_body) + self.assertEqual(channel.json_body, {}) + + # Verify the correct devices exist (duplicates should be handled) + store = self.hs.get_datastores().main + devices = self.get_success(store.get_devices_by_user(str(self.alice_user_id))) + self.assertEqual(sorted(devices.keys()), ["DEVICE1", "DEVICE2"]) diff --git a/tests/rest/synapse/mas/test_users.py b/tests/rest/synapse/mas/test_users.py new file mode 100644 index 0000000000..378f29fd4c --- /dev/null +++ b/tests/rest/synapse/mas/test_users.py @@ -0,0 +1,1399 @@ +# +# This file is licensed under the Affero General Public License (AGPL) version 3. +# +# Copyright (C) 2025 New Vector, Ltd +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# See the GNU Affero General Public License for more details: +# . + +from urllib.parse import urlencode + +from twisted.test.proto_helpers import MemoryReactor + +from synapse.appservice import ApplicationService +from synapse.server import HomeServer +from synapse.types import JsonDict, UserID, create_requester +from synapse.util import Clock + +from tests.unittest import skip_unless +from tests.utils import HAS_AUTHLIB + +from ._base import BaseTestCase + + +@skip_unless(HAS_AUTHLIB, "requires authlib") +class MasQueryUserResource(BaseTestCase): + def test_other_token(self) -> None: + channel = self.make_request( + "GET", + "/_synapse/mas/query_user?localpart=alice", + shorthand=False, + access_token="other_token", + ) + + self.assertEqual(channel.code, 403, channel.json_body) + self.assertEqual( + channel.json_body["error"], "This endpoint must only be called by MAS" + ) + + def test_query_user(self) -> None: + alice = UserID("alice", "test") + store = self.hs.get_datastores().main + self.get_success( + self.hs.get_registration_handler().register_user( + localpart=alice.localpart, + default_display_name="Alice", + ) + ) + self.get_success( + store.set_profile_avatar_url( + user_id=alice, + new_avatar_url="mxc://example.com/avatar", + ) + ) + + channel = self.make_request( + "GET", + "/_synapse/mas/query_user?localpart=alice", + shorthand=False, + access_token=self.SHARED_SECRET, + ) + + self.assertEqual(channel.code, 200, channel.json_body) + self.assertEqual( + channel.json_body, + { + "user_id": "@alice:test", + "display_name": "Alice", + "avatar_url": "mxc://example.com/avatar", + "is_suspended": False, + "is_deactivated": False, + }, + ) + + self.get_success( + store.set_user_suspended_status(user_id=str(alice), suspended=True) + ) + + channel = self.make_request( + "GET", + "/_synapse/mas/query_user?localpart=alice", + shorthand=False, + access_token=self.SHARED_SECRET, + ) + + self.assertEqual(channel.code, 200, channel.json_body) + self.assertEqual( + channel.json_body, + { + "user_id": "@alice:test", + "display_name": "Alice", + "avatar_url": "mxc://example.com/avatar", + "is_suspended": True, + "is_deactivated": False, + }, + ) + + # Deactivate the account, it should clear the display name and avatar + # and mark the user as deactivated + self.get_success( + self.hs.get_deactivate_account_handler().deactivate_account( + user_id=str(alice), + erase_data=True, + requester=create_requester(alice), + ) + ) + + channel = self.make_request( + "GET", + "/_synapse/mas/query_user?localpart=alice", + shorthand=False, + access_token=self.SHARED_SECRET, + ) + + self.assertEqual(channel.code, 200, channel.json_body) + self.assertEqual( + channel.json_body, + { + "user_id": "@alice:test", + "display_name": None, + "avatar_url": None, + "is_suspended": True, + "is_deactivated": True, + }, + ) + + def test_query_unknown_user(self) -> None: + channel = self.make_request( + "GET", + "/_synapse/mas/query_user?localpart=alice", + shorthand=False, + access_token=self.SHARED_SECRET, + ) + + self.assertEqual(channel.code, 404, channel.json_body) + + def test_query_user_missing_localpart(self) -> None: + channel = self.make_request( + "GET", + "/_synapse/mas/query_user", + shorthand=False, + access_token=self.SHARED_SECRET, + ) + + self.assertEqual(channel.code, 400, channel.json_body) + + +@skip_unless(HAS_AUTHLIB, "requires authlib") +class MasProvisionUserResource(BaseTestCase): + def test_other_token(self) -> None: + channel = self.make_request( + "POST", + "/_synapse/mas/provision_user", + shorthand=False, + access_token="other_token", + content={"localpart": "alice"}, + ) + + self.assertEqual(channel.code, 403, channel.json_body) + self.assertEqual( + channel.json_body["error"], "This endpoint must only be called by MAS" + ) + + def test_provision_user(self) -> None: + store = self.hs.get_datastores().main + + channel = self.make_request( + "POST", + "/_synapse/mas/provision_user", + shorthand=False, + access_token=self.SHARED_SECRET, + content={ + "localpart": "alice", + "set_displayname": "Alice", + "set_emails": ["alice@example.com"], + "set_avatar_url": "mxc://example.com/avatar", + }, + ) + + # This created the user, hence the 201 status code + self.assertEqual(channel.code, 201, channel.json_body) + self.assertEqual(channel.json_body, {}) + + alice = UserID("alice", "test") + profile = self.get_success(store.get_profileinfo(alice)) + self.assertEqual(profile.display_name, "Alice") + self.assertEqual(profile.avatar_url, "mxc://example.com/avatar") + threepids = self.get_success(store.user_get_threepids(str(alice))) + self.assertEqual(len(threepids), 1) + self.assertEqual(threepids[0].medium, "email") + self.assertEqual(threepids[0].address, "alice@example.com") + + channel = self.make_request( + "POST", + "/_synapse/mas/provision_user", + shorthand=False, + access_token=self.SHARED_SECRET, + content={ + "localpart": "alice", + "unset_displayname": True, + "unset_avatar_url": True, + "unset_emails": True, + }, + ) + + # This updated the user, hence the 200 status code + self.assertEqual(channel.code, 200, channel.json_body) + self.assertEqual(channel.json_body, {}) + + # Check that the profile and threepids were deleted + profile = self.get_success(store.get_profileinfo(alice)) + self.assertEqual(profile.display_name, None) + self.assertEqual(profile.avatar_url, None) + threepids = self.get_success(store.user_get_threepids(str(alice))) + self.assertEqual(threepids, []) + + def test_provision_user_missing_localpart(self) -> None: + channel = self.make_request( + "POST", + "/_synapse/mas/provision_user", + shorthand=False, + access_token=self.SHARED_SECRET, + content={ + "set_displayname": "Alice", + }, + ) + + self.assertEqual(channel.code, 400, channel.json_body) + + def test_provision_user_empty_localpart(self) -> None: + channel = self.make_request( + "POST", + "/_synapse/mas/provision_user", + shorthand=False, + access_token=self.SHARED_SECRET, + content={ + "localpart": "", + "set_displayname": "Alice", + }, + ) + + self.assertEqual(channel.code, 400, channel.json_body) + + def test_provision_user_invalid_localpart(self) -> None: + # Test with characters that are invalid in localparts + invalid_localparts = [ + "@alice:test", # That's a MXID + "alice@domain.com", + "alice:test", + "alice space", + "alice#hash", + "a" * 1000, # Very long localpart + ] + + for localpart in invalid_localparts: + channel = self.make_request( + "POST", + "/_synapse/mas/provision_user", + shorthand=False, + access_token=self.SHARED_SECRET, + content={ + "localpart": localpart, + "set_displayname": "Alice", + }, + ) + # Should be a validation error + self.assertEqual( + channel.code, 400, f"Should fail for localpart: {localpart}" + ) + + def test_provision_user_multiple_emails(self) -> None: + store = self.hs.get_datastores().main + + channel = self.make_request( + "POST", + "/_synapse/mas/provision_user", + shorthand=False, + access_token=self.SHARED_SECRET, + content={ + "localpart": "alice", + "set_emails": ["alice@example.com", "alice.alt@example.com"], + }, + ) + + self.assertEqual(channel.code, 201, channel.json_body) + + alice = UserID("alice", "test") + threepids = self.get_success(store.user_get_threepids(str(alice))) + self.assertEqual(len(threepids), 2) + email_addresses = {tp.address for tp in threepids} + self.assertEqual( + email_addresses, {"alice@example.com", "alice.alt@example.com"} + ) + + def test_provision_user_duplicate_emails(self) -> None: + store = self.hs.get_datastores().main + + channel = self.make_request( + "POST", + "/_synapse/mas/provision_user", + shorthand=False, + access_token=self.SHARED_SECRET, + content={ + "localpart": "alice", + "set_emails": ["alice@example.com", "alice@example.com"], + }, + ) + + self.assertEqual(channel.code, 201, channel.json_body) + + alice = UserID("alice", "test") + threepids = self.get_success(store.user_get_threepids(str(alice))) + # Should deduplicate + self.assertEqual(len(threepids), 1) + self.assertEqual(threepids[0].address, "alice@example.com") + + def test_provision_user_conflicting_operations(self) -> None: + # Test setting and unsetting the same field + channel = self.make_request( + "POST", + "/_synapse/mas/provision_user", + shorthand=False, + access_token=self.SHARED_SECRET, + content={ + "localpart": "alice", + "set_displayname": "Alice", + "unset_displayname": True, + }, + ) + + self.assertEqual(channel.code, 400, channel.json_body) + + def test_provision_user_invalid_json_types(self) -> None: + # Test with wrong data types + invalid_contents: list[JsonDict] = [ + {"localpart": "alice", "set_displayname": 123}, # Number instead of string + { + "localpart": "alice", + "set_emails": "not-an-array", + }, # String instead of array + { + "localpart": "alice", + "unset_displayname": "not-a-bool", + }, # String instead of bool + {"localpart": 123}, # Number instead of string for localpart + ] + + for content in invalid_contents: + channel = self.make_request( + "POST", + "/_synapse/mas/provision_user", + shorthand=False, + access_token=self.SHARED_SECRET, + content=content, + ) + self.assertEqual(channel.code, 400, f"Should fail for content: {content}") + + +@skip_unless(HAS_AUTHLIB, "requires authlib") +class MasIsLocalpartAvailableResource(BaseTestCase): + def prepare( + self, reactor: MemoryReactor, clock: Clock, homeserver: HomeServer + ) -> None: + # Provision a user + store = homeserver.get_datastores().main + self.get_success(store.register_user("@alice:test")) + + def test_other_token(self) -> None: + channel = self.make_request( + "GET", + "/_synapse/mas/is_localpart_available?localpart=alice", + shorthand=False, + access_token="other_token", + ) + + self.assertEqual(channel.code, 403, channel.json_body) + self.assertEqual( + channel.json_body["error"], "This endpoint must only be called by MAS" + ) + + def test_is_localpart_available(self) -> None: + # "alice" is not available + channel = self.make_request( + "GET", + "/_synapse/mas/is_localpart_available?localpart=alice", + shorthand=False, + access_token=self.SHARED_SECRET, + ) + + self.assertEqual(channel.code, 400, channel.json_body) + self.assertEqual(channel.json_body["errcode"], "M_USER_IN_USE") + + # "bob" is available + channel = self.make_request( + "GET", + "/_synapse/mas/is_localpart_available?localpart=bob", + shorthand=False, + access_token=self.SHARED_SECRET, + ) + + self.assertEqual(channel.code, 200, channel.json_body) + self.assertEqual(channel.json_body, {}) + + def test_is_localpart_available_invalid_localparts(self) -> None: + # Numeric-only localparts are not allowed + channel = self.make_request( + "GET", + "/_synapse/mas/is_localpart_available?localpart=0", + shorthand=False, + access_token=self.SHARED_SECRET, + ) + + self.assertEqual(channel.code, 400, channel.json_body) + self.assertEqual(channel.json_body["errcode"], "M_INVALID_USERNAME") + + # A super-long MXID is not allowed by the spec + super_long = "a" * 1000 + channel = self.make_request( + "GET", + f"/_synapse/mas/is_localpart_available?localpart={super_long}", + shorthand=False, + access_token=self.SHARED_SECRET, + ) + + self.assertEqual(channel.code, 400, channel.json_body) + self.assertEqual(channel.json_body["errcode"], "M_INVALID_USERNAME") + + def test_is_localpart_available_appservice_exclusive(self) -> None: + # Insert an appservice which has exclusive namespaces + appservice = ApplicationService( + token="i_am_an_app_service", + id="1234", + namespaces={"users": [{"regex": r"@as_user_.*:.+", "exclusive": True}]}, + sender=UserID.from_string("@as_main:test"), + ) + self.hs.get_datastores().main.services_cache = [appservice] + + channel = self.make_request( + "GET", + "/_synapse/mas/is_localpart_available?localpart=as_main", + shorthand=False, + access_token=self.SHARED_SECRET, + ) + + self.assertEqual(channel.code, 400, channel.json_body) + self.assertEqual(channel.json_body["errcode"], "M_EXCLUSIVE") + + channel = self.make_request( + "GET", + "/_synapse/mas/is_localpart_available?localpart=as_user_alice", + shorthand=False, + access_token=self.SHARED_SECRET, + ) + + self.assertEqual(channel.code, 400, channel.json_body) + self.assertEqual(channel.json_body["errcode"], "M_EXCLUSIVE") + + # Sanity-check that "bob" is available + channel = self.make_request( + "GET", + "/_synapse/mas/is_localpart_available?localpart=bob", + shorthand=False, + access_token=self.SHARED_SECRET, + ) + + self.assertEqual(channel.code, 200, channel.json_body) + self.assertEqual(channel.json_body, {}) + + def test_is_localpart_available_missing_localpart(self) -> None: + channel = self.make_request( + "GET", + "/_synapse/mas/is_localpart_available", + shorthand=False, + access_token=self.SHARED_SECRET, + ) + + self.assertEqual(channel.code, 400, channel.json_body) + + def test_is_localpart_available_empty_localpart(self) -> None: + channel = self.make_request( + "GET", + "/_synapse/mas/is_localpart_available?localpart=", + shorthand=False, + access_token=self.SHARED_SECRET, + ) + + self.assertEqual(channel.code, 400, channel.json_body) + + def test_is_localpart_available_invalid_characters(self) -> None: + # Test with characters that are invalid in localparts + invalid_localparts = [ + "alice@domain.com", # Contains @ + "alice:test", # Contains : + "alice space", # Contains space + "alice\\backslash", # Contains backslash + "alice#hash", # Contains hash + "alice$dollar", # Contains $ + "alice%percent", # Contains % + "alice&", # Contains & + "alice?question", # Contains ? + "alice[bracket", # Contains [ + "alice]bracket", # Contains ] + "alice{brace", # Contains { + "alice}brace", # Contains } + "alice|pipe", # Contains | + 'alice"quote', # Contains " + "alice'apostrophe", # Contains ' + "alicegreater", # Contains > + "alice\ttab", # Contains tab + "alice\nnewline", # Contains newline + ] + + for localpart in invalid_localparts: + channel = self.make_request( + "GET", + f"/_synapse/mas/is_localpart_available?{urlencode({'localpart': localpart})}", + shorthand=False, + access_token=self.SHARED_SECRET, + ) + # Should return 400 for invalid characters + self.assertEqual( + channel.code, + 400, + f"Should reject localpart with invalid chars: {localpart}", + ) + self.assertEqual( + channel.json_body["errcode"], "M_INVALID_USERNAME", localpart + ) + + def test_is_localpart_available_case_sensitivity(self) -> None: + # Register a user with an uppercase localpart + self.get_success(self.hs.get_datastores().main.register_user("@BOB:test")) + + # It should report as not available, the search should be case-insensitive + channel = self.make_request( + "GET", + "/_synapse/mas/is_localpart_available?localpart=bob", + shorthand=False, + access_token=self.SHARED_SECRET, + ) + + self.assertEqual(channel.code, 400, channel.json_body) + self.assertEqual(channel.json_body["errcode"], "M_USER_IN_USE") + + +@skip_unless(HAS_AUTHLIB, "requires authlib") +class MasDeleteUserResource(BaseTestCase): + def prepare( + self, reactor: MemoryReactor, clock: Clock, homeserver: HomeServer + ) -> None: + # Provision a user with a display name + self.get_success( + homeserver.get_registration_handler().register_user( + localpart="alice", + default_display_name="Alice", + ) + ) + + def test_other_token(self) -> None: + channel = self.make_request( + "POST", + "/_synapse/mas/delete_user", + shorthand=False, + access_token="other_token", + content={"localpart": "alice", "erase": False}, + ) + + self.assertEqual(channel.code, 403, channel.json_body) + self.assertEqual( + channel.json_body["error"], "This endpoint must only be called by MAS" + ) + + def test_delete_user_no_erase(self) -> None: + alice = UserID("alice", "test") + store = self.hs.get_datastores().main + + # Delete the user + channel = self.make_request( + "POST", + "/_synapse/mas/delete_user", + shorthand=False, + access_token=self.SHARED_SECRET, + content={"localpart": "alice", "erase": False}, + ) + + self.assertEqual(channel.code, 200, channel.json_body) + self.assertEqual(channel.json_body, {}) + + # Check that the user was deleted + self.assertTrue( + self.get_success(store.get_user_deactivated_status(user_id=str(alice))) + ) + # But not erased + self.assertFalse(self.get_success(store.is_user_erased(user_id=str(alice)))) + + def test_delete_user_erase(self) -> None: + alice = UserID("alice", "test") + store = self.hs.get_datastores().main + + # Delete the user + channel = self.make_request( + "POST", + "/_synapse/mas/delete_user", + shorthand=False, + access_token=self.SHARED_SECRET, + content={"localpart": "alice", "erase": True}, + ) + + self.assertEqual(channel.code, 200, channel.json_body) + self.assertEqual(channel.json_body, {}) + + # Check that the user was deleted + self.assertTrue( + self.get_success(store.get_user_deactivated_status(user_id=str(alice))) + ) + # And erased + self.assertTrue(self.get_success(store.is_user_erased(user_id=str(alice)))) + + def test_delete_user_missing_localpart(self) -> None: + channel = self.make_request( + "POST", + "/_synapse/mas/delete_user", + shorthand=False, + access_token=self.SHARED_SECRET, + content={"erase": False}, + ) + + self.assertEqual(channel.code, 400, channel.json_body) + + def test_delete_user_missing_erase(self) -> None: + channel = self.make_request( + "POST", + "/_synapse/mas/delete_user", + shorthand=False, + access_token=self.SHARED_SECRET, + content={"localpart": "alice"}, + ) + + self.assertEqual(channel.code, 400, channel.json_body) + + def test_delete_user_invalid_erase_type(self) -> None: + invalid_erase_values = [ + "true", # String instead of bool + 1, # Number instead of bool + "false", # String instead of bool + 0, # Number instead of bool + {}, # Object instead of bool + [], # Array instead of bool + ] + + for erase_value in invalid_erase_values: + channel = self.make_request( + "POST", + "/_synapse/mas/delete_user", + shorthand=False, + access_token=self.SHARED_SECRET, + content={"localpart": "alice", "erase": erase_value}, + ) + self.assertEqual( + channel.code, 400, f"Should fail for erase value: {erase_value}" + ) + + def test_delete_nonexistent_user(self) -> None: + channel = self.make_request( + "POST", + "/_synapse/mas/delete_user", + shorthand=False, + access_token=self.SHARED_SECRET, + content={"localpart": "nonexistent", "erase": False}, + ) + + self.assertEqual(channel.code, 404) + + def test_delete_already_deleted_user(self) -> None: + # First deletion + channel = self.make_request( + "POST", + "/_synapse/mas/delete_user", + shorthand=False, + access_token=self.SHARED_SECRET, + content={"localpart": "alice", "erase": False}, + ) + self.assertEqual(channel.code, 200) + + # Second deletion should be idempotent + channel = self.make_request( + "POST", + "/_synapse/mas/delete_user", + shorthand=False, + access_token=self.SHARED_SECRET, + content={"localpart": "alice", "erase": False}, + ) + self.assertEqual(channel.code, 200) + + def test_delete_user_erase_already_deleted_user(self) -> None: + alice = UserID("alice", "test") + store = self.hs.get_datastores().main + + # First delete without erase + channel = self.make_request( + "POST", + "/_synapse/mas/delete_user", + shorthand=False, + access_token=self.SHARED_SECRET, + content={"localpart": "alice", "erase": False}, + ) + self.assertEqual(channel.code, 200) + + # Verify not erased initially + self.assertFalse(self.get_success(store.is_user_erased(user_id=str(alice)))) + + # Now delete with erase + channel = self.make_request( + "POST", + "/_synapse/mas/delete_user", + shorthand=False, + access_token=self.SHARED_SECRET, + content={"localpart": "alice", "erase": True}, + ) + self.assertEqual(channel.code, 200) + + # Should now be erased + self.assertTrue(self.get_success(store.is_user_erased(user_id=str(alice)))) + + def test_delete_user_empty_json(self) -> None: + channel = self.make_request( + "POST", + "/_synapse/mas/delete_user", + shorthand=False, + access_token=self.SHARED_SECRET, + content={}, + ) + + self.assertEqual(channel.code, 400, channel.json_body) + + def test_delete_user_extra_fields(self) -> None: + channel = self.make_request( + "POST", + "/_synapse/mas/delete_user", + shorthand=False, + access_token=self.SHARED_SECRET, + content={ + "localpart": "alice", + "erase": False, + "extra_field": "should_be_ignored", + "another_field": 123, + }, + ) + + # Should succeed and ignore extra fields + self.assertEqual(channel.code, 200, channel.json_body) + + +@skip_unless(HAS_AUTHLIB, "requires authlib") +class MasReactivateUserResource(BaseTestCase): + def test_other_token(self) -> None: + channel = self.make_request( + "POST", + "/_synapse/mas/reactivate_user", + shorthand=False, + access_token="other_token", + content={"localpart": "alice"}, + ) + + self.assertEqual(channel.code, 403, channel.json_body) + self.assertEqual( + channel.json_body["error"], "This endpoint must only be called by MAS" + ) + + def test_reactivate_user(self) -> None: + alice = UserID("alice", "test") + store = self.hs.get_datastores().main + self.get_success( + self.hs.get_registration_handler().register_user( + localpart=alice.localpart, + default_display_name="Alice", + ) + ) + self.get_success( + self.hs.get_deactivate_account_handler().deactivate_account( + user_id=str(alice), + erase_data=True, + requester=create_requester(alice), + ) + ) + + channel = self.make_request( + "POST", + "/_synapse/mas/reactivate_user", + shorthand=False, + access_token=self.SHARED_SECRET, + content={"localpart": "alice"}, + ) + + self.assertEqual(channel.code, 200, channel.json_body) + self.assertEqual(channel.json_body, {}) + + # Check that the user was reactivated + self.assertFalse( + self.get_success(store.get_user_deactivated_status(user_id=str(alice))) + ) + + def test_reactivate_user_missing_localpart(self) -> None: + channel = self.make_request( + "POST", + "/_synapse/mas/reactivate_user", + shorthand=False, + access_token=self.SHARED_SECRET, + content={}, + ) + + self.assertEqual(channel.code, 400, channel.json_body) + + def test_reactivate_nonexistent_user(self) -> None: + channel = self.make_request( + "POST", + "/_synapse/mas/reactivate_user", + shorthand=False, + access_token=self.SHARED_SECRET, + content={"localpart": "nonexistent"}, + ) + + self.assertEqual(channel.code, 404, channel.json_body) + + def test_reactivate_active_user(self) -> None: + # Create an active user + alice = UserID("alice", "test") + self.get_success( + self.hs.get_registration_handler().register_user( + localpart=alice.localpart, + default_display_name="Alice", + ) + ) + + channel = self.make_request( + "POST", + "/_synapse/mas/reactivate_user", + shorthand=False, + access_token=self.SHARED_SECRET, + content={"localpart": "alice"}, + ) + + # Should be idempotent + self.assertEqual(channel.code, 200, channel.json_body) + + def test_reactivate_erased_user(self) -> None: + alice = UserID("alice", "test") + store = self.hs.get_datastores().main + self.get_success( + self.hs.get_registration_handler().register_user( + localpart=alice.localpart, + default_display_name="Alice", + ) + ) + + # Deactivate with erase + self.get_success( + self.hs.get_deactivate_account_handler().deactivate_account( + user_id=str(alice), + erase_data=True, + requester=create_requester(alice), + ) + ) + + # Verify user is erased + self.assertTrue(self.get_success(store.is_user_erased(user_id=str(alice)))) + + channel = self.make_request( + "POST", + "/_synapse/mas/reactivate_user", + shorthand=False, + access_token=self.SHARED_SECRET, + content={"localpart": "alice"}, + ) + + # Should succeed even for erased users + self.assertEqual(channel.code, 200, channel.json_body) + # Shouldn't be erased anymore + self.assertFalse(self.get_success(store.is_user_erased(user_id=str(alice)))) + + def test_reactivate_user_extra_fields(self) -> None: + alice = UserID("alice", "test") + self.get_success( + self.hs.get_registration_handler().register_user( + localpart=alice.localpart, + ) + ) + self.get_success( + self.hs.get_deactivate_account_handler().deactivate_account( + user_id=str(alice), + erase_data=False, + requester=create_requester(alice), + ) + ) + + channel = self.make_request( + "POST", + "/_synapse/mas/reactivate_user", + shorthand=False, + access_token=self.SHARED_SECRET, + content={ + "localpart": "alice", + "extra_field": "should_be_ignored", + "another_field": 123, + }, + ) + + # Should succeed and ignore extra fields + self.assertEqual(channel.code, 200, channel.json_body) + + +@skip_unless(HAS_AUTHLIB, "requires authlib") +class MasSetDisplayNameResource(BaseTestCase): + def test_other_token(self) -> None: + channel = self.make_request( + "POST", + "/_synapse/mas/set_displayname", + shorthand=False, + access_token="other_token", + content={"localpart": "alice", "displayname": "Bob"}, + ) + + self.assertEqual(channel.code, 403, channel.json_body) + self.assertEqual( + channel.json_body["error"], "This endpoint must only be called by MAS" + ) + + def test_set_display_name(self) -> None: + alice = UserID("alice", "test") + store = self.hs.get_datastores().main + self.get_success( + self.hs.get_registration_handler().register_user( + localpart=alice.localpart, + default_display_name="Alice", + ) + ) + profile = self.get_success(store.get_profileinfo(alice)) + self.assertEqual(profile.display_name, "Alice") + + channel = self.make_request( + "POST", + "/_synapse/mas/set_displayname", + shorthand=False, + access_token=self.SHARED_SECRET, + content={"localpart": "alice", "displayname": "Bob"}, + ) + + self.assertEqual(channel.code, 200, channel.json_body) + self.assertEqual(channel.json_body, {}) + + # Check that the profile was updated + profile = self.get_success(store.get_profileinfo(alice)) + self.assertEqual(profile.display_name, "Bob") + + def test_set_display_name_missing_localpart(self) -> None: + channel = self.make_request( + "POST", + "/_synapse/mas/set_displayname", + shorthand=False, + access_token=self.SHARED_SECRET, + content={"displayname": "Bob"}, + ) + + self.assertEqual(channel.code, 400, channel.json_body) + + def test_set_display_name_missing_displayname(self) -> None: + channel = self.make_request( + "POST", + "/_synapse/mas/set_displayname", + shorthand=False, + access_token=self.SHARED_SECRET, + content={"localpart": "alice"}, + ) + + self.assertEqual(channel.code, 400, channel.json_body) + + def test_set_display_name_very_long(self) -> None: + alice = UserID("alice", "test") + self.get_success( + self.hs.get_registration_handler().register_user( + localpart=alice.localpart, + ) + ) + + long_name = "A" * 1000 + channel = self.make_request( + "POST", + "/_synapse/mas/set_displayname", + shorthand=False, + access_token=self.SHARED_SECRET, + content={"localpart": "alice", "displayname": long_name}, + ) + + self.assertEqual(channel.code, 400, channel.json_body) + + def test_set_display_name_special_characters(self) -> None: + alice = UserID("alice", "test") + self.get_success( + self.hs.get_registration_handler().register_user( + localpart=alice.localpart, + ) + ) + + special_names = [ + "Alice 👋", # Emoji + "Alice & Bob", # HTML entities + "Alice\nNewline", # Newline + "Alice\tTab", # Tab + 'Alice"Quote', # Quote + "Alice'Apostrophe", # Apostrophe + "Alice