From 0bf180cbb43f6e3d489ddaa5d66226b6daeea027 Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Wed, 24 Aug 2022 18:59:27 -0500 Subject: [PATCH 001/107] Comment about a better future where we can get the state diff between two events (#13586) Split off from https://github.com/matrix-org/synapse/pull/13561 Part of https://github.com/matrix-org/synapse/issues/13356 Mentioned in [internal doc](https://docs.google.com/document/d/1lvUoVfYUiy6UaHB6Rb4HicjaJAU40-APue9Q4vzuW3c/edit#bookmark=id.2tvwz3yhcafh) --- changelog.d/13586.misc | 1 + synapse/handlers/federation_event.py | 8 ++++++++ 2 files changed, 9 insertions(+) create mode 100644 changelog.d/13586.misc diff --git a/changelog.d/13586.misc b/changelog.d/13586.misc new file mode 100644 index 0000000000..daa71470f6 --- /dev/null +++ b/changelog.d/13586.misc @@ -0,0 +1 @@ +Comment about a better future where we can get the state diff between two events. diff --git a/synapse/handlers/federation_event.py b/synapse/handlers/federation_event.py index 048c4111f6..ace7adcffb 100644 --- a/synapse/handlers/federation_event.py +++ b/synapse/handlers/federation_event.py @@ -1041,6 +1041,14 @@ class FederationEventHandler: InvalidResponseError: if the remote homeserver's response contains fields of the wrong type. """ + + # It would be better if we could query the difference from our known + # state to the given `event_id` so the sending server doesn't have to + # send as much and we don't have to process as many events. For example + # in a room like #matrix:matrix.org, we get 200k events (77k state_events, 122k + # auth_events) from this call. + # + # Tracked by https://github.com/matrix-org/synapse/issues/13618 ( state_event_ids, auth_event_ids, From a2824465027c9036b98e56a52b53c80ca2aaa71b Mon Sep 17 00:00:00 2001 From: Kat Gerasimova Date: Thu, 25 Aug 2022 12:09:23 +0100 Subject: [PATCH 002/107] Update automation for incoming issues (#13629) GitHub appears to be deprecating addProjectNextItem by not allowing it to be used alongside projectV2 to get the project ID, so switching to using addProjectV2ItemById instead. --- .github/workflows/triage-incoming.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/triage-incoming.yml b/.github/workflows/triage-incoming.yml index 8467970128..f926bcb759 100644 --- a/.github/workflows/triage-incoming.yml +++ b/.github/workflows/triage-incoming.yml @@ -14,13 +14,13 @@ jobs: with: headers: '{"GraphQL-Features": "projects_next_graphql"}' query: | - mutation add_to_project($projectid:ID!,$contentid:ID!) { - addProjectNextItem(input:{projectId:$projectid contentId:$contentid}) { - projectNextItem { + mutation add_to_project($projectid:ID!,$contentid:ID!) { + addProjectV2ItemById(input: {projectId: $projectid contentId: $contentid}) { + item { id + } } } - } projectid: ${{ env.PROJECT_ID }} contentid: ${{ github.event.issue.node_id }} env: From a2ce6144479c9f0400517d505c7288f2ab785a95 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff <1389908+richvdh@users.noreply.github.com> Date: Thu, 25 Aug 2022 15:29:08 +0100 Subject: [PATCH 003/107] register_new_matrix_user: read server url from config (#13616) Fixes https://github.com/matrix-org/synapse/issues/3672: `https://localhost:8448` is virtually never right. --- changelog.d/13616.bugfix | 1 + synapse/_scripts/register_new_matrix_user.py | 57 +++++++++++++++++--- 2 files changed, 52 insertions(+), 6 deletions(-) create mode 100644 changelog.d/13616.bugfix diff --git a/changelog.d/13616.bugfix b/changelog.d/13616.bugfix new file mode 100644 index 0000000000..f2c48d1d8d --- /dev/null +++ b/changelog.d/13616.bugfix @@ -0,0 +1 @@ +Fix a longstanding bug in `register_new_matrix_user` which meant it was always necessary to explicitly give a server URL. diff --git a/synapse/_scripts/register_new_matrix_user.py b/synapse/_scripts/register_new_matrix_user.py index 092601f530..42ae0dbea3 100644 --- a/synapse/_scripts/register_new_matrix_user.py +++ b/synapse/_scripts/register_new_matrix_user.py @@ -20,11 +20,13 @@ import hashlib import hmac import logging import sys -from typing import Callable, Optional +from typing import Any, Callable, Dict, Optional import requests import yaml +_DEFAULT_SERVER_URL = "http://localhost:8008" + def request_registration( user: str, @@ -203,31 +205,74 @@ def main() -> None: parser.add_argument( "server_url", - default="https://localhost:8448", nargs="?", - help="URL to use to talk to the homeserver. Defaults to " - " 'https://localhost:8448'.", + help="URL to use to talk to the homeserver. By default, tries to find a " + "suitable URL from the configuration file. Otherwise, defaults to " + f"'{_DEFAULT_SERVER_URL}'.", ) args = parser.parse_args() if "config" in args and args.config: config = yaml.safe_load(args.config) + + if args.shared_secret: + secret = args.shared_secret + else: + # argparse should check that we have either config or shared secret + assert config + secret = config.get("registration_shared_secret", None) if not secret: print("No 'registration_shared_secret' defined in config.") sys.exit(1) + + if args.server_url: + server_url = args.server_url + elif config: + server_url = _find_client_listener(config) + if not server_url: + server_url = _DEFAULT_SERVER_URL + print( + "Unable to find a suitable HTTP listener in the configuration file. " + f"Trying {server_url} as a last resort.", + file=sys.stderr, + ) else: - secret = args.shared_secret + server_url = _DEFAULT_SERVER_URL + print( + f"No server url or configuration file given. Defaulting to {server_url}.", + file=sys.stderr, + ) admin = None if args.admin or args.no_admin: admin = args.admin register_new_user( - args.user, args.password, args.server_url, secret, admin, args.user_type + args.user, args.password, server_url, secret, admin, args.user_type ) +def _find_client_listener(config: Dict[str, Any]) -> Optional[str]: + # try to find a listener in the config. Returns a host:port pair + for listener in config.get("listeners", []): + if listener.get("type") != "http" or listener.get("tls", False): + continue + + if not any( + name == "client" + for resource in listener.get("resources", []) + for name in resource.get("names", []) + ): + continue + + # TODO: consider bind_addresses + return f"http://localhost:{listener['port']}" + + # no suitable listeners? + return None + + if __name__ == "__main__": main() From d092e6f32a1a3d79337774746720a73762a35e8e Mon Sep 17 00:00:00 2001 From: Richard van der Hoff <1389908+richvdh@users.noreply.github.com> Date: Thu, 25 Aug 2022 17:27:46 +0100 Subject: [PATCH 004/107] Support `registration_shared_secret` in a file (#13614) A new `registration_shared_secret_path` option. This is kinda handy for k8s deployments and things. --- changelog.d/13614.feature | 1 + .../configuration/config_documentation.md | 18 ++++++++ synapse/_scripts/register_new_matrix_user.py | 45 +++++++++++++++++-- synapse/config/registration.py | 33 +++++++++++++- 4 files changed, 92 insertions(+), 5 deletions(-) create mode 100644 changelog.d/13614.feature diff --git a/changelog.d/13614.feature b/changelog.d/13614.feature new file mode 100644 index 0000000000..fa177ead09 --- /dev/null +++ b/changelog.d/13614.feature @@ -0,0 +1 @@ +Support setting the registration shared secret in a file, via a new `registration_shared_secret_path` configuration option. diff --git a/docs/usage/configuration/config_documentation.md b/docs/usage/configuration/config_documentation.md index 8ae018e628..e9ab58854e 100644 --- a/docs/usage/configuration/config_documentation.md +++ b/docs/usage/configuration/config_documentation.md @@ -2124,10 +2124,28 @@ registration_requires_token: true If set, allows registration of standard or admin accounts by anyone who has the shared secret, even if registration is otherwise disabled. +See also [`registration_shared_secret_path`](#registration_shared_secret_path). + Example configuration: ```yaml registration_shared_secret: ``` + +--- +### `registration_shared_secret_path` + +An alternative to [`registration_shared_secret`](#registration_shared_secret): +allows the shared secret to be specified in an external file. + +The file should be a plain text file, containing only the shared secret. + +Example configuration: +```yaml +registration_shared_secret_file: /path/to/secrets/file +``` + +_Added in Synapse 1.67.0._ + --- ### `bcrypt_rounds` diff --git a/synapse/_scripts/register_new_matrix_user.py b/synapse/_scripts/register_new_matrix_user.py index 42ae0dbea3..0c4504d5d8 100644 --- a/synapse/_scripts/register_new_matrix_user.py +++ b/synapse/_scripts/register_new_matrix_user.py @@ -1,6 +1,6 @@ # Copyright 2015, 2016 OpenMarket Ltd # Copyright 2018 New Vector -# Copyright 2021 The Matrix.org Foundation C.I.C. +# Copyright 2021-22 The Matrix.org Foundation C.I.C. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -25,6 +25,15 @@ from typing import Any, Callable, Dict, Optional import requests import yaml +_CONFLICTING_SHARED_SECRET_OPTS_ERROR = """\ +Conflicting options 'registration_shared_secret' and 'registration_shared_secret_path' +are both defined in config file. +""" + +_NO_SHARED_SECRET_OPTS_ERROR = """\ +No 'registration_shared_secret' or 'registration_shared_secret_path' defined in config. +""" + _DEFAULT_SERVER_URL = "http://localhost:8008" @@ -222,9 +231,15 @@ def main() -> None: # argparse should check that we have either config or shared secret assert config - secret = config.get("registration_shared_secret", None) + secret = config.get("registration_shared_secret") + secret_file = config.get("registration_shared_secret_path") + if secret_file: + if secret: + print(_CONFLICTING_SHARED_SECRET_OPTS_ERROR, file=sys.stderr) + sys.exit(1) + secret = _read_file(secret_file, "registration_shared_secret_path").strip() if not secret: - print("No 'registration_shared_secret' defined in config.") + print(_NO_SHARED_SECRET_OPTS_ERROR, file=sys.stderr) sys.exit(1) if args.server_url: @@ -254,6 +269,30 @@ def main() -> None: ) +def _read_file(file_path: Any, config_path: str) -> str: + """Check the given file exists, and read it into a string + + If it does not, exit with an error indicating the problem + + Args: + file_path: the file to be read + config_path: where in the configuration file_path came from, so that a useful + error can be emitted if it does not exist. + Returns: + content of the file. + """ + if not isinstance(file_path, str): + print(f"{config_path} setting is not a string", file=sys.stderr) + sys.exit(1) + + try: + with open(file_path) as file_stream: + return file_stream.read() + except OSError as e: + print(f"Error accessing file {file_path}: {e}", file=sys.stderr) + sys.exit(1) + + def _find_client_listener(config: Dict[str, Any]) -> Optional[str]: # try to find a listener in the config. Returns a host:port pair for listener in config.get("listeners", []): diff --git a/synapse/config/registration.py b/synapse/config/registration.py index a888d976f2..df1d83dfaa 100644 --- a/synapse/config/registration.py +++ b/synapse/config/registration.py @@ -13,10 +13,10 @@ # See the License for the specific language governing permissions and # limitations under the License. import argparse -from typing import Any, Optional +from typing import Any, Dict, Optional from synapse.api.constants import RoomCreationPreset -from synapse.config._base import Config, ConfigError +from synapse.config._base import Config, ConfigError, read_file from synapse.types import JsonDict, RoomAlias, UserID from synapse.util.stringutils import random_string_with_symbols, strtobool @@ -27,6 +27,11 @@ password resets, configure Synapse with an SMTP server via the `email` setting, remove `account_threepid_delegates.email`. """ +CONFLICTING_SHARED_SECRET_OPTS_ERROR = """\ +You have configured both `registration_shared_secret` and +`registration_shared_secret_path`. These are mutually incompatible. +""" + class RegistrationConfig(Config): section = "registration" @@ -53,7 +58,16 @@ class RegistrationConfig(Config): self.enable_registration_token_3pid_bypass = config.get( "enable_registration_token_3pid_bypass", False ) + + # read the shared secret, either inline or from an external file self.registration_shared_secret = config.get("registration_shared_secret") + registration_shared_secret_path = config.get("registration_shared_secret_path") + if registration_shared_secret_path: + if self.registration_shared_secret: + raise ConfigError(CONFLICTING_SHARED_SECRET_OPTS_ERROR) + self.registration_shared_secret = read_file( + registration_shared_secret_path, ("registration_shared_secret_path",) + ).strip() self.bcrypt_rounds = config.get("bcrypt_rounds", 12) @@ -218,6 +232,21 @@ class RegistrationConfig(Config): else: return "" + def generate_files(self, config: Dict[str, Any], config_dir_path: str) -> None: + # if 'registration_shared_secret_path' is specified, and the target file + # does not exist, generate it. + registration_shared_secret_path = config.get("registration_shared_secret_path") + if registration_shared_secret_path and not self.path_exists( + registration_shared_secret_path + ): + print( + "Generating registration shared secret file " + + registration_shared_secret_path + ) + secret = random_string_with_symbols(50) + with open(registration_shared_secret_path, "w") as f: + f.write(f"{secret}\n") + @staticmethod def add_arguments(parser: argparse.ArgumentParser) -> None: reg_group = parser.add_argument_group("registration") From 978666a088aacf467d3d265ff3b7b09c0d19733d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Behrmann?= Date: Thu, 25 Aug 2022 18:56:55 +0200 Subject: [PATCH 005/107] Debian packaging: explicitly allocate a group for the system user (#13593) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Otherwise the files of the synapse user are readable by the nobody user, which is unsafe. Signed-off-by: Jörg Behrmann --- debian/changelog | 6 ++++++ debian/matrix-synapse-py3.postinst | 4 ++-- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/debian/changelog b/debian/changelog index c3974261a9..605e660928 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,9 @@ +matrix-synapse-py3 (1.66.0~rc1+nmu1) UNRELEASED; urgency=medium + + * Allocate a group for the system user. + + -- Jörg Behrmann Tue, 23 Aug 2022 17:17:00 +0100 + matrix-synapse-py3 (1.66.0~rc1) stable; urgency=medium * New Synapse release 1.66.0rc1. diff --git a/debian/matrix-synapse-py3.postinst b/debian/matrix-synapse-py3.postinst index 029b9e0243..3c72b69bb7 100644 --- a/debian/matrix-synapse-py3.postinst +++ b/debian/matrix-synapse-py3.postinst @@ -40,12 +40,12 @@ EOF /opt/venvs/matrix-synapse/lib/manage_debconf.pl update if ! getent passwd $USER >/dev/null; then - adduser --quiet --system --no-create-home --home /var/lib/matrix-synapse $USER + adduser --quiet --system --group --no-create-home --home /var/lib/matrix-synapse $USER fi for DIR in /var/lib/matrix-synapse /var/log/matrix-synapse /etc/matrix-synapse; do if ! dpkg-statoverride --list --quiet $DIR >/dev/null; then - dpkg-statoverride --force --quiet --update --add $USER nogroup 0755 $DIR + dpkg-statoverride --force --quiet --update --add $USER "$(id -gn $USER)" 0755 $DIR fi done From 967d7bad6c1ea97fb234c3e6d0cbc87b73bb7d0a Mon Sep 17 00:00:00 2001 From: Brad Murray Date: Fri, 26 Aug 2022 03:38:10 -0400 Subject: [PATCH 006/107] Move the execution of the retention purge_jobs to the main worker (#13632) Fixes #9927 Signed-off-by: Brad Murray brad@beeper.com --- changelog.d/13632.bugfix | 1 + synapse/handlers/pagination.py | 6 ++---- 2 files changed, 3 insertions(+), 4 deletions(-) create mode 100644 changelog.d/13632.bugfix diff --git a/changelog.d/13632.bugfix b/changelog.d/13632.bugfix new file mode 100644 index 0000000000..e4b7b403cd --- /dev/null +++ b/changelog.d/13632.bugfix @@ -0,0 +1 @@ +Fix the running of MSC1763 retention purge_jobs in deployments with background jobs running on a worker by forcing them back onto the main worker. Contributed by Brad @ Beeper. diff --git a/synapse/handlers/pagination.py b/synapse/handlers/pagination.py index 74e944bce7..a0c39778ab 100644 --- a/synapse/handlers/pagination.py +++ b/synapse/handlers/pagination.py @@ -159,11 +159,9 @@ class PaginationHandler: self._retention_allowed_lifetime_max = ( hs.config.retention.retention_allowed_lifetime_max ) + self._is_master = hs.config.worker.worker_app is None - if ( - hs.config.worker.run_background_tasks - and hs.config.retention.retention_enabled - ): + if hs.config.retention.retention_enabled and self._is_master: # Run the purge jobs described in the configuration file. for job in hs.config.retention.retention_purge_jobs: logger.info("Setting up purge job with config: %s", job) From 998e211836b99cbe3d07e594c3c3d1d4ebea5aae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Behrmann?= Date: Fri, 26 Aug 2022 10:10:54 +0200 Subject: [PATCH 007/107] Update debhelper (#13594) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Update debian packaging to debhelper version 12 Don't call dh_installinit anymore, because it has been deprecated, and use dh_installsystemd instead of dh_systemd_enable for the same reason. Signed-off-by: Jörg Behrmann * Drop preinst script It was used for reasons of interactions of dh_systemd_start and dh_installinit, which have both be deprecated Signed-off-by: Jörg Behrmann * Drop /etc/default file It was no longer being installed. * Remove debian/compat file This is managed by the control file nowadays --- debian/changelog | 2 ++ debian/compat | 1 - debian/control | 2 +- debian/matrix-synapse-py3.preinst | 31 ------------------------------- debian/matrix-synapse.default | 2 -- debian/matrix-synapse.service | 6 +++++- debian/rules | 12 +++++++----- 7 files changed, 15 insertions(+), 41 deletions(-) delete mode 100644 debian/compat delete mode 100644 debian/matrix-synapse-py3.preinst delete mode 100644 debian/matrix-synapse.default diff --git a/debian/changelog b/debian/changelog index 605e660928..3257adef77 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,5 +1,7 @@ matrix-synapse-py3 (1.66.0~rc1+nmu1) UNRELEASED; urgency=medium + * Update debhelper to compatibility level 12. + * Drop the preinst script stopping synapse. * Allocate a group for the system user. -- Jörg Behrmann Tue, 23 Aug 2022 17:17:00 +0100 diff --git a/debian/compat b/debian/compat deleted file mode 100644 index f599e28b8a..0000000000 --- a/debian/compat +++ /dev/null @@ -1 +0,0 @@ -10 diff --git a/debian/control b/debian/control index 412a9e1d4c..86f5a66d02 100644 --- a/debian/control +++ b/debian/control @@ -4,7 +4,7 @@ Priority: extra Maintainer: Synapse Packaging team # keep this list in sync with the build dependencies in docker/Dockerfile-dhvirtualenv. Build-Depends: - debhelper (>= 10), + debhelper-compat (= 12), dh-virtualenv (>= 1.1), libsystemd-dev, libpq-dev, diff --git a/debian/matrix-synapse-py3.preinst b/debian/matrix-synapse-py3.preinst deleted file mode 100644 index 4b5612f050..0000000000 --- a/debian/matrix-synapse-py3.preinst +++ /dev/null @@ -1,31 +0,0 @@ -#!/bin/sh -e - -# Attempt to undo some of the braindamage caused by -# https://github.com/matrix-org/package-synapse-debian/issues/18. -# -# Due to reasons [1], the old python2 matrix-synapse package will not stop the -# service when the package is uninstalled. Our maintainer scripts will do the -# right thing in terms of ensuring the service is enabled and unmasked, but -# then do a `systemctl start matrix-synapse`, which of course does nothing - -# leaving the old (py2) service running. -# -# There should normally be no reason for the service to be running during our -# preinst, so we assume that if it *is* running, it's due to that situation, -# and stop it. -# -# [1] dh_systemd_start doesn't do anything because it sees that there is an -# init.d script with the same name, so leaves it to dh_installinit. -# -# dh_installinit doesn't do anything because somebody gave it a --no-start -# for unknown reasons. - -if [ -x /bin/systemctl ]; then - if /bin/systemctl --quiet is-active -- matrix-synapse; then - echo >&2 "stopping existing matrix-synapse service" - /bin/systemctl stop matrix-synapse || true - fi -fi - -#DEBHELPER# - -exit 0 diff --git a/debian/matrix-synapse.default b/debian/matrix-synapse.default deleted file mode 100644 index f402d73bbf..0000000000 --- a/debian/matrix-synapse.default +++ /dev/null @@ -1,2 +0,0 @@ -# Specify environment variables used when running Synapse -# SYNAPSE_CACHE_FACTOR=0.5 (default) diff --git a/debian/matrix-synapse.service b/debian/matrix-synapse.service index bde1c6cb9f..c3f9660283 100644 --- a/debian/matrix-synapse.service +++ b/debian/matrix-synapse.service @@ -5,7 +5,6 @@ Description=Synapse Matrix homeserver Type=notify User=matrix-synapse WorkingDirectory=/var/lib/matrix-synapse -EnvironmentFile=-/etc/default/matrix-synapse ExecStartPre=/opt/venvs/matrix-synapse/bin/python -m synapse.app.homeserver --config-path=/etc/matrix-synapse/homeserver.yaml --config-path=/etc/matrix-synapse/conf.d/ --generate-keys ExecStart=/opt/venvs/matrix-synapse/bin/python -m synapse.app.homeserver --config-path=/etc/matrix-synapse/homeserver.yaml --config-path=/etc/matrix-synapse/conf.d/ ExecReload=/bin/kill -HUP $MAINPID @@ -13,5 +12,10 @@ Restart=always RestartSec=3 SyslogIdentifier=matrix-synapse +# The environment file is not shipped by default anymore and the below directive +# is for backwards compatibility only. Please use your homeserver.yaml if +# possible. +EnvironmentFile=-/etc/default/matrix-synapse + [Install] WantedBy=multi-user.target diff --git a/debian/rules b/debian/rules index 5baf2475f0..3b79d56074 100755 --- a/debian/rules +++ b/debian/rules @@ -6,15 +6,17 @@ # assume we only have one package PACKAGE_NAME:=`dh_listpackages` -override_dh_systemd_enable: - dh_systemd_enable --name=matrix-synapse - -override_dh_installinit: - dh_installinit --name=matrix-synapse +override_dh_installsystemd: + dh_installsystemd --name=matrix-synapse # we don't really want to strip the symbols from our object files. override_dh_strip: +# many libraries pulled from PyPI have allocatable sections after +# non-allocatable ones on which dwz errors out. For those without the issue the +# gains are only marginal +override_dh_dwz: + # dh_shlibdeps calls dpkg-shlibdeps, which finds all the binary files # (executables and shared libs) in the package, and looks for the shared # libraries that they depend on. It then adds a dependency on the package that From 5e5c8150d798f6929ddedbb39f9f11486558cdbc Mon Sep 17 00:00:00 2001 From: Richard van der Hoff <1389908+richvdh@users.noreply.github.com> Date: Fri, 26 Aug 2022 12:26:06 +0100 Subject: [PATCH 008/107] Generate missing configuration files at startup (#13615) If things like the signing key file are missing, let's just try to generate them on startup. Again, this is useful for k8s-like deployments where we just want to generate keys on the first run. --- changelog.d/13615.feature | 1 + .../configuration/config_documentation.md | 8 ++- synapse/config/_base.py | 63 +++++++++++++++---- 3 files changed, 58 insertions(+), 14 deletions(-) create mode 100644 changelog.d/13615.feature diff --git a/changelog.d/13615.feature b/changelog.d/13615.feature new file mode 100644 index 0000000000..c2c568f1eb --- /dev/null +++ b/changelog.d/13615.feature @@ -0,0 +1 @@ +Change the default startup behaviour so that any missing "additional" configuration files (signing key, etc) are generated automatically. diff --git a/docs/usage/configuration/config_documentation.md b/docs/usage/configuration/config_documentation.md index e9ab58854e..4c59e3dcf2 100644 --- a/docs/usage/configuration/config_documentation.md +++ b/docs/usage/configuration/config_documentation.md @@ -2139,6 +2139,9 @@ allows the shared secret to be specified in an external file. The file should be a plain text file, containing only the shared secret. +If this file does not exist, Synapse will create a new signing +key on startup and store it in this file. + Example configuration: ```yaml registration_shared_secret_file: /path/to/secrets/file @@ -2555,7 +2558,10 @@ Config options relating to signing keys --- ### `signing_key_path` -Path to the signing key to sign messages with. +Path to the signing key to sign events and federation requests with. + +*New in Synapse 1.67*: If this file does not exist, Synapse will create a new signing +key on startup and store it in this file. Example configuration: ```yaml diff --git a/synapse/config/_base.py b/synapse/config/_base.py index 7c9cf403ef..1f6362aedd 100644 --- a/synapse/config/_base.py +++ b/synapse/config/_base.py @@ -20,6 +20,7 @@ import logging import os import re from collections import OrderedDict +from enum import Enum, auto from hashlib import sha256 from textwrap import dedent from typing import ( @@ -603,18 +604,44 @@ class RootConfig: " may specify directories containing *.yaml files.", ) - generate_group = parser.add_argument_group("Config generation") - generate_group.add_argument( - "--generate-config", - action="store_true", - help="Generate a config file, then exit.", + # we nest the mutually-exclusive group inside another group so that the help + # text shows them in their own group. + generate_mode_group = parser.add_argument_group( + "Config generation mode", ) - generate_group.add_argument( + generate_mode_exclusive = generate_mode_group.add_mutually_exclusive_group() + generate_mode_exclusive.add_argument( + # hidden option to make the type and default work + "--generate-mode", + help=argparse.SUPPRESS, + type=_ConfigGenerateMode, + default=_ConfigGenerateMode.GENERATE_MISSING_AND_RUN, + ) + generate_mode_exclusive.add_argument( + "--generate-config", + help="Generate a config file, then exit.", + action="store_const", + const=_ConfigGenerateMode.GENERATE_EVERYTHING_AND_EXIT, + dest="generate_mode", + ) + generate_mode_exclusive.add_argument( "--generate-missing-configs", "--generate-keys", - action="store_true", help="Generate any missing additional config files, then exit.", + action="store_const", + const=_ConfigGenerateMode.GENERATE_MISSING_AND_EXIT, + dest="generate_mode", ) + generate_mode_exclusive.add_argument( + "--generate-missing-and-run", + help="Generate any missing additional config files, then run. This is the " + "default behaviour.", + action="store_const", + const=_ConfigGenerateMode.GENERATE_MISSING_AND_RUN, + dest="generate_mode", + ) + + generate_group = parser.add_argument_group("Details for --generate-config") generate_group.add_argument( "-H", "--server-name", help="The server name to generate a config file for." ) @@ -670,11 +697,12 @@ class RootConfig: config_dir_path = os.path.abspath(config_dir_path) data_dir_path = os.getcwd() - generate_missing_configs = config_args.generate_missing_configs - obj = cls(config_files) - if config_args.generate_config: + if ( + config_args.generate_mode + == _ConfigGenerateMode.GENERATE_EVERYTHING_AND_EXIT + ): if config_args.report_stats is None: parser.error( "Please specify either --report-stats=yes or --report-stats=no\n\n" @@ -732,11 +760,14 @@ class RootConfig: ) % (config_path,) ) - generate_missing_configs = True config_dict = read_config_files(config_files) - if generate_missing_configs: - obj.generate_missing_files(config_dict, config_dir_path) + obj.generate_missing_files(config_dict, config_dir_path) + + if config_args.generate_mode in ( + _ConfigGenerateMode.GENERATE_EVERYTHING_AND_EXIT, + _ConfigGenerateMode.GENERATE_MISSING_AND_EXIT, + ): return None obj.parse_config_dict( @@ -965,6 +996,12 @@ def read_file(file_path: Any, config_path: Iterable[str]) -> str: raise ConfigError("Error accessing file %r" % (file_path,), config_path) from e +class _ConfigGenerateMode(Enum): + GENERATE_MISSING_AND_RUN = auto() + GENERATE_MISSING_AND_EXIT = auto() + GENERATE_EVERYTHING_AND_EXIT = auto() + + __all__ = [ "Config", "RootConfig", From c4e29b6908ac8ae57b5e9a3e7662ad638b61e94a Mon Sep 17 00:00:00 2001 From: Richard van der Hoff <1389908+richvdh@users.noreply.github.com> Date: Fri, 26 Aug 2022 14:29:31 +0100 Subject: [PATCH 009/107] Improve documentation around user registration (#13640) Update a bunch of the documentation for user registration, add some cross links, etc. --- changelog.d/13640.doc | 1 + docs/admin_api/register_api.md | 6 +- docs/setup/installation.md | 23 +- .../configuration/config_documentation.md | 209 ++++++++++-------- 4 files changed, 135 insertions(+), 104 deletions(-) create mode 100644 changelog.d/13640.doc diff --git a/changelog.d/13640.doc b/changelog.d/13640.doc new file mode 100644 index 0000000000..fa049371cf --- /dev/null +++ b/changelog.d/13640.doc @@ -0,0 +1 @@ +Improve documentation around user registration. diff --git a/docs/admin_api/register_api.md b/docs/admin_api/register_api.md index d7b7cf6a76..f6be31b443 100644 --- a/docs/admin_api/register_api.md +++ b/docs/admin_api/register_api.md @@ -5,9 +5,9 @@ non-interactive way. This is generally used for bootstrapping a Synapse instance with administrator accounts. To authenticate yourself to the server, you will need both the shared secret -(`registration_shared_secret` in the homeserver configuration), and a -one-time nonce. If the registration shared secret is not configured, this API -is not enabled. +([`registration_shared_secret`](../configuration/config_documentation.md#registration_shared_secret) +in the homeserver configuration), and a one-time nonce. If the registration +shared secret is not configured, this API is not enabled. To fetch the nonce, you need to request one from the API: diff --git a/docs/setup/installation.md b/docs/setup/installation.md index 260e50577b..bb78b3267a 100644 --- a/docs/setup/installation.md +++ b/docs/setup/installation.md @@ -506,9 +506,13 @@ email will be disabled. ### Registering a user -The easiest way to create a new user is to do so from a client like [Element](https://element.io/). +One way to create a new user is to do so from a client like +[Element](https://element.io/). This requires registration to be enabled via +the +[`enable_registration`](../usage/configuration/config_documentation.md#enable_registration) +setting. -Alternatively, you can do so from the command line. This can be done as follows: +Alternatively, you can create new users from the command line. This can be done as follows: 1. If synapse was installed via pip, activate the virtualenv as follows (if Synapse was installed via a prebuilt package, `register_new_matrix_user` should already be @@ -520,7 +524,7 @@ Alternatively, you can do so from the command line. This can be done as follows: ``` 2. Run the following command: ```sh - register_new_matrix_user -c homeserver.yaml http://localhost:8008 + register_new_matrix_user -c homeserver.yaml ``` This will prompt you to add details for the new user, and will then connect to @@ -533,12 +537,13 @@ Make admin [no]: Success! ``` -This process uses a setting `registration_shared_secret` in -`homeserver.yaml`, which is shared between Synapse itself and the -`register_new_matrix_user` script. It doesn't matter what it is (a random -value is generated by `--generate-config`), but it should be kept secret, as -anyone with knowledge of it can register users, including admin accounts, -on your server even if `enable_registration` is `false`. +This process uses a setting +[`registration_shared_secret`](../usage/configuration/config_documentation.md#registration_shared_secret), +which is shared between Synapse itself and the `register_new_matrix_user` +script. It doesn't matter what it is (a random value is generated by +`--generate-config`), but it should be kept secret, as anyone with knowledge of +it can register users, including admin accounts, on your server even if +`enable_registration` is `false`. ### Setting up a TURN server diff --git a/docs/usage/configuration/config_documentation.md b/docs/usage/configuration/config_documentation.md index 4c59e3dcf2..c0cfe5dcea 100644 --- a/docs/usage/configuration/config_documentation.md +++ b/docs/usage/configuration/config_documentation.md @@ -1873,8 +1873,8 @@ See [here](../../CAPTCHA_SETUP.md) for full details on setting up captcha. --- ### `recaptcha_public_key` -This homeserver's ReCAPTCHA public key. Must be specified if `enable_registration_captcha` is -enabled. +This homeserver's ReCAPTCHA public key. Must be specified if +[`enable_registration_captcha`](#enable_registration_captcha) is enabled. Example configuration: ```yaml @@ -1883,7 +1883,8 @@ recaptcha_public_key: "YOUR_PUBLIC_KEY" --- ### `recaptcha_private_key` -This homeserver's ReCAPTCHA private key. Must be specified if `enable_registration_captcha` is +This homeserver's ReCAPTCHA private key. Must be specified if +[`enable_registration_captcha`](#enable_registration_captcha) is enabled. Example configuration: @@ -1893,9 +1894,11 @@ recaptcha_private_key: "YOUR_PRIVATE_KEY" --- ### `enable_registration_captcha` -Set to true to enable ReCaptcha checks when registering, preventing signup -unless a captcha is answered. Requires a valid ReCaptcha public/private key. -Defaults to false. +Set to `true` to require users to complete a CAPTCHA test when registering an account. +Requires a valid ReCaptcha public/private key. +Defaults to `false`. + +Note that [`enable_registration`](#enable_registration) must also be set to allow account registration. Example configuration: ```yaml @@ -1971,9 +1974,21 @@ Registration can be rate-limited using the parameters in the [Ratelimiting](#rat --- ### `enable_registration` -Enable registration for new users. Defaults to false. It is highly recommended that if you enable registration, -you use either captcha, email, or token-based verification to verify that new users are not bots. In order to enable registration -without any verification, you must also set `enable_registration_without_verification` to true. +Enable registration for new users. Defaults to `false`. + +It is highly recommended that if you enable registration, you set one or more +or the following options, to avoid abuse of your server by "bots": + + * [`enable_registration_captcha`](#enable_registration_captcha) + * [`registrations_require_3pid`](#registrations_require_3pid) + * [`registration_requires_token`](#registration_requires_token) + +(In order to enable registration without any verification, you must also set +[`enable_registration_without_verification`](#enable_registration_without_verification).) + +Note that even if this setting is disabled, new accounts can still be created +via the admin API if +[`registration_shared_secret`](#registration_shared_secret) is set. Example configuration: ```yaml @@ -1981,88 +1996,21 @@ enable_registration: true ``` --- ### `enable_registration_without_verification` + Enable registration without email or captcha verification. Note: this option is *not* recommended, -as registration without verification is a known vector for spam and abuse. Defaults to false. Has no effect -unless `enable_registration` is also enabled. +as registration without verification is a known vector for spam and abuse. Defaults to `false`. Has no effect +unless [`enable_registration`](#enable_registration) is also enabled. Example configuration: ```yaml enable_registration_without_verification: true ``` --- -### `session_lifetime` - -Time that a user's session remains valid for, after they log in. - -Note that this is not currently compatible with guest logins. - -Note also that this is calculated at login time: changes are not applied retrospectively to users who have already -logged in. - -By default, this is infinite. - -Example configuration: -```yaml -session_lifetime: 24h -``` ----- -### `refresh_access_token_lifetime` - -Time that an access token remains valid for, if the session is using refresh tokens. - -For more information about refresh tokens, please see the [manual](user_authentication/refresh_tokens.md). - -Note that this only applies to clients which advertise support for refresh tokens. - -Note also that this is calculated at login time and refresh time: changes are not applied to -existing sessions until they are refreshed. - -By default, this is 5 minutes. - -Example configuration: -```yaml -refreshable_access_token_lifetime: 10m -``` ---- -### `refresh_token_lifetime: 24h` - -Time that a refresh token remains valid for (provided that it is not -exchanged for another one first). -This option can be used to automatically log-out inactive sessions. -Please see the manual for more information. - -Note also that this is calculated at login time and refresh time: -changes are not applied to existing sessions until they are refreshed. - -By default, this is infinite. - -Example configuration: -```yaml -refresh_token_lifetime: 24h -``` ---- -### `nonrefreshable_access_token_lifetime` - -Time that an access token remains valid for, if the session is NOT -using refresh tokens. - -Please note that not all clients support refresh tokens, so setting -this to a short value may be inconvenient for some users who will -then be logged out frequently. - -Note also that this is calculated at login time: changes are not applied -retrospectively to existing sessions for users that have already logged in. - -By default, this is infinite. - -Example configuration: -```yaml -nonrefreshable_access_token_lifetime: 24h -``` ---- ### `registrations_require_3pid` -If this is set, the user must provide all of the specified types of 3PID when registering. +If this is set, users must provide all of the specified types of 3PID when registering an account. + +Note that [`enable_registration`](#enable_registration) must also be set to allow account registration. Example configuration: ```yaml @@ -2110,9 +2058,11 @@ enable_3pid_lookup: false Require users to submit a token during registration. Tokens can be managed using the admin [API](../administration/admin_api/registration_tokens.md). -Note that `enable_registration` must be set to true. Disabling this option will not delete any tokens previously generated. -Defaults to false. Set to true to enable. +Defaults to `false`. Set to `true` to enable. + + +Note that [`enable_registration`](#enable_registration) must also be set to allow account registration. Example configuration: ```yaml @@ -2121,8 +2071,13 @@ registration_requires_token: true --- ### `registration_shared_secret` -If set, allows registration of standard or admin accounts by anyone who -has the shared secret, even if registration is otherwise disabled. +If set, allows registration of standard or admin accounts by anyone who has the +shared secret, even if [`enable_registration`](#enable_registration) is not +set. + +This is primarily intended for use with the `register_new_matrix_user` script +(see [Registering a user](../../setup/installation.md#registering-a-user)); +however, the interface is [documented](../admin_api/register_api.html). See also [`registration_shared_secret_path`](#registration_shared_secret_path). @@ -2379,6 +2334,79 @@ Example configuration: ```yaml inhibit_user_in_use_error: true ``` +--- +## User session management +--- +### `session_lifetime` + +Time that a user's session remains valid for, after they log in. + +Note that this is not currently compatible with guest logins. + +Note also that this is calculated at login time: changes are not applied retrospectively to users who have already +logged in. + +By default, this is infinite. + +Example configuration: +```yaml +session_lifetime: 24h +``` +---- +### `refresh_access_token_lifetime` + +Time that an access token remains valid for, if the session is using refresh tokens. + +For more information about refresh tokens, please see the [manual](user_authentication/refresh_tokens.md). + +Note that this only applies to clients which advertise support for refresh tokens. + +Note also that this is calculated at login time and refresh time: changes are not applied to +existing sessions until they are refreshed. + +By default, this is 5 minutes. + +Example configuration: +```yaml +refreshable_access_token_lifetime: 10m +``` +--- +### `refresh_token_lifetime: 24h` + +Time that a refresh token remains valid for (provided that it is not +exchanged for another one first). +This option can be used to automatically log-out inactive sessions. +Please see the manual for more information. + +Note also that this is calculated at login time and refresh time: +changes are not applied to existing sessions until they are refreshed. + +By default, this is infinite. + +Example configuration: +```yaml +refresh_token_lifetime: 24h +``` +--- +### `nonrefreshable_access_token_lifetime` + +Time that an access token remains valid for, if the session is NOT +using refresh tokens. + +Please note that not all clients support refresh tokens, so setting +this to a short value may be inconvenient for some users who will +then be logged out frequently. + +Note also that this is calculated at login time: changes are not applied +retrospectively to existing sessions for users that have already logged in. + +By default, this is infinite. + +Example configuration: +```yaml +nonrefreshable_access_token_lifetime: 24h +``` + --- ## Metrics ### Config options related to metrics. @@ -2666,13 +2694,10 @@ key_server_signing_keys_path: "key_server_signing_keys.key" The following settings can be used to make Synapse use a single sign-on provider for authentication, instead of its internal password database. -You will probably also want to set the following options to false to +You will probably also want to set the following options to `false` to disable the regular login/registration flows: - * `enable_registration` - * `password_config.enabled` - -You will also want to investigate the settings under the "sso" configuration -section below. + * [`enable_registration`](#enable_registration) + * [`password_config.enabled`](#password_config) --- ### `saml2_config` From 4f6de33f4147f745eb808b717666eb4d35a12716 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff <1389908+richvdh@users.noreply.github.com> Date: Sun, 28 Aug 2022 21:05:30 +0100 Subject: [PATCH 010/107] Print complement failure results last (#13639) Since github always scrolls to the bottom of any test output, let's put the failed tests last and hide any successful packages. --- .ci/complement_package.gotpl | 14 ++++++-------- .ci/scripts/gotestfmt | 21 +++++++++++++++++++++ .github/workflows/latest_deps.yml | 2 +- .github/workflows/tests.yml | 2 +- .github/workflows/twisted_trunk.yml | 2 +- changelog.d/13639.misc | 1 + 6 files changed, 31 insertions(+), 11 deletions(-) create mode 100755 .ci/scripts/gotestfmt create mode 100644 changelog.d/13639.misc diff --git a/.ci/complement_package.gotpl b/.ci/complement_package.gotpl index e1625fd31f..2dd5e5e0e6 100644 --- a/.ci/complement_package.gotpl +++ b/.ci/complement_package.gotpl @@ -27,10 +27,10 @@ which is under the Unlicense licence. {{- . -}}{{- "\n" -}} {{- end -}} {{- with .TestCases -}} - {{- /* Failing tests are first */ -}} + {{- /* Passing tests are first */ -}} {{- range . -}} - {{- if and (ne .Result "PASS") (ne .Result "SKIP") -}} - ::group::{{ "\033" }}[0;31m❌{{ " " }}{{- .Name -}} + {{- if eq .Result "PASS" -}} + ::group::{{ "\033" }}[0;32m✅{{ " " }}{{- .Name -}} {{- "\033" -}}[0;37m ({{if $settings.ShowTestStatus}}{{.Result}}; {{end}}{{ .Duration -}} {{- with .Coverage -}} , coverage: {{ . }}% @@ -47,7 +47,6 @@ which is under the Unlicense licence. {{- end -}} {{- end -}} - {{- /* Then skipped tests are second */ -}} {{- range . -}} {{- if eq .Result "SKIP" -}} @@ -68,11 +67,10 @@ which is under the Unlicense licence. {{- end -}} {{- end -}} - - {{- /* Then passing tests are last */ -}} + {{- /* and failing tests are last */ -}} {{- range . -}} - {{- if eq .Result "PASS" -}} - ::group::{{ "\033" }}[0;32m✅{{ " " }}{{- .Name -}} + {{- if and (ne .Result "PASS") (ne .Result "SKIP") -}} + ::group::{{ "\033" }}[0;31m❌{{ " " }}{{- .Name -}} {{- "\033" -}}[0;37m ({{if $settings.ShowTestStatus}}{{.Result}}; {{end}}{{ .Duration -}} {{- with .Coverage -}} , coverage: {{ . }}% diff --git a/.ci/scripts/gotestfmt b/.ci/scripts/gotestfmt new file mode 100755 index 0000000000..83e0ec6361 --- /dev/null +++ b/.ci/scripts/gotestfmt @@ -0,0 +1,21 @@ +#!/bin/bash +# +# wraps `gotestfmt`, hiding output from successful packages unless +# all tests passed. + +set -o pipefail +set -e + +# tee the test results to a log, whilst also piping them into gotestfmt, +# telling it to hide successful results, so that we can clearly see +# unsuccessful results. +tee complement.log | gotestfmt -hide successful-packages + +# gotestfmt will exit non-zero if there were any failures, so if we got to this +# point, we must have had a successful result. +echo "All tests successful; showing all test results" + +# Pipe the test results back through gotestfmt, showing all results. +# The log file consists of JSON lines giving the test results, interspersed +# with regular stdout lines (including reports of downloaded packages). +grep '^{"Time":' complement.log | gotestfmt diff --git a/.github/workflows/latest_deps.yml b/.github/workflows/latest_deps.yml index f263cf612d..7dac617c4b 100644 --- a/.github/workflows/latest_deps.yml +++ b/.github/workflows/latest_deps.yml @@ -163,7 +163,7 @@ jobs: - run: | set -o pipefail - TEST_ONLY_IGNORE_POETRY_LOCKFILE=1 POSTGRES=${{ (matrix.database == 'Postgres') && 1 || '' }} WORKERS=${{ (matrix.arrangement == 'workers') && 1 || '' }} COMPLEMENT_DIR=`pwd`/complement synapse/scripts-dev/complement.sh -json 2>&1 | gotestfmt + TEST_ONLY_IGNORE_POETRY_LOCKFILE=1 POSTGRES=${{ (matrix.database == 'Postgres') && 1 || '' }} WORKERS=${{ (matrix.arrangement == 'workers') && 1 || '' }} COMPLEMENT_DIR=`pwd`/complement synapse/scripts-dev/complement.sh -json 2>&1 | synapse/.ci/scripts/gotestfmt shell: bash name: Run Complement Tests diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 144cb9ffaa..3e6d718a7c 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -354,7 +354,7 @@ jobs: - run: | set -o pipefail - POSTGRES=${{ (matrix.database == 'Postgres') && 1 || '' }} WORKERS=${{ (matrix.arrangement == 'workers') && 1 || '' }} COMPLEMENT_DIR=`pwd`/complement synapse/scripts-dev/complement.sh -json 2>&1 | gotestfmt + POSTGRES=${{ (matrix.database == 'Postgres') && 1 || '' }} WORKERS=${{ (matrix.arrangement == 'workers') && 1 || '' }} COMPLEMENT_DIR=`pwd`/complement synapse/scripts-dev/complement.sh -json 2>&1 | synapse/.ci/scripts/gotestfmt shell: bash name: Run Complement Tests diff --git a/.github/workflows/twisted_trunk.yml b/.github/workflows/twisted_trunk.yml index dd8e6fbb1c..0906101cc1 100644 --- a/.github/workflows/twisted_trunk.yml +++ b/.github/workflows/twisted_trunk.yml @@ -137,7 +137,7 @@ jobs: - run: | set -o pipefail - TEST_ONLY_SKIP_DEP_HASH_VERIFICATION=1 POSTGRES=${{ (matrix.database == 'Postgres') && 1 || '' }} WORKERS=${{ (matrix.arrangement == 'workers') && 1 || '' }} COMPLEMENT_DIR=`pwd`/complement synapse/scripts-dev/complement.sh -json 2>&1 | gotestfmt + TEST_ONLY_SKIP_DEP_HASH_VERIFICATION=1 POSTGRES=${{ (matrix.database == 'Postgres') && 1 || '' }} WORKERS=${{ (matrix.arrangement == 'workers') && 1 || '' }} COMPLEMENT_DIR=`pwd`/complement synapse/scripts-dev/complement.sh -json 2>&1 | synapse/.ci/scripts/gotestfmt shell: bash name: Run Complement Tests diff --git a/changelog.d/13639.misc b/changelog.d/13639.misc new file mode 100644 index 0000000000..de4e4d1206 --- /dev/null +++ b/changelog.d/13639.misc @@ -0,0 +1 @@ +Improve readability of Complement CI logs by printing failure results last. From 51d732db3b4ab13eb58e937a546abce7968112ef Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Tue, 30 Aug 2022 01:38:14 -0500 Subject: [PATCH 011/107] Optimize how we calculate `likely_domains` during backfill (#13575) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Optimize how we calculate `likely_domains` during backfill because I've seen this take 17s in production just to `get_current_state` which is used to `get_domains_from_state` (see case [*2. Loading tons of events* in the `/messages` investigation issue](https://github.com/matrix-org/synapse/issues/13356)). There are 3 ways we currently calculate hosts that are in the room: 1. `get_current_state` -> `get_domains_from_state` - Used in `backfill` to calculate `likely_domains` and `/timestamp_to_event` because it was cargo-culted from `backfill` - This one is being eliminated in favor of `get_current_hosts_in_room` in this PR 🕳 1. `get_current_hosts_in_room` - Used for other federation things like sending read receipts and typing indicators 1. `get_hosts_in_room_at_events` - Used when pushing out events over federation to other servers in the `_process_event_queue_loop` Fix https://github.com/matrix-org/synapse/issues/13626 Part of https://github.com/matrix-org/synapse/issues/13356 Mentioned in [internal doc](https://docs.google.com/document/d/1lvUoVfYUiy6UaHB6Rb4HicjaJAU40-APue9Q4vzuW3c/edit#bookmark=id.2tvwz3yhcafh) ### Query performance #### Before The query from `get_current_state` sucks just because we have to get all 80k events. And we see almost the exact same performance locally trying to get all of these events (16s vs 17s): ``` synapse=# SELECT type, state_key, event_id FROM current_state_events WHERE room_id = '!OGEhHVWSdvArJzumhm:matrix.org'; Time: 16035.612 ms (00:16.036) synapse=# SELECT type, state_key, event_id FROM current_state_events WHERE room_id = '!OGEhHVWSdvArJzumhm:matrix.org'; Time: 4243.237 ms (00:04.243) ``` But what about `get_current_hosts_in_room`: When there is 8M rows in the `current_state_events` table, the previous query in `get_current_hosts_in_room` took 13s from complete freshness (when the events were first added). But takes 930ms after a Postgres restart or 390ms if running back to back to back. ```sh $ psql synapse synapse=# \timing on synapse=# SELECT COUNT(DISTINCT substring(state_key FROM '@[^:]*:(.*)$')) FROM current_state_events WHERE type = 'm.room.member' AND membership = 'join' AND room_id = '!OGEhHVWSdvArJzumhm:matrix.org'; count ------- 4130 (1 row) Time: 13181.598 ms (00:13.182) synapse=# SELECT COUNT(*) from current_state_events where room_id = '!OGEhHVWSdvArJzumhm:matrix.org'; count ------- 80814 synapse=# SELECT COUNT(*) from current_state_events; count --------- 8162847 synapse=# SELECT pg_size_pretty( pg_total_relation_size('current_state_events') ); pg_size_pretty ---------------- 4702 MB ``` #### After I'm not sure how long it takes from complete freshness as I only really get that opportunity once (maybe restarting computer but that's cumbersome) and it's not really relevant to normal operating times. Maybe you get closer to the fresh times the more access variability there is so that Postgres caches aren't as exact. Update: The longest I've seen this run for is 6.4s and 4.5s after a computer restart. After a Postgres restart, it takes 330ms and running back to back takes 260ms. ```sh $ psql synapse synapse=# \timing on Timing is on. synapse=# SELECT substring(c.state_key FROM '@[^:]*:(.*)$') as host FROM current_state_events c /* Get the depth of the event from the events table */ INNER JOIN events AS e USING (event_id) WHERE c.type = 'm.room.member' AND c.membership = 'join' AND c.room_id = '!OGEhHVWSdvArJzumhm:matrix.org' GROUP BY host ORDER BY min(e.depth) ASC; Time: 333.800 ms ``` #### Going further To improve things further we could add a `limit` parameter to `get_current_hosts_in_room`. Realistically, we don't need 4k domains to choose from because there is no way we're going to query that many before we a) probably get an answer or b) we give up. Another thing we can do is optimize the query to use a index skip scan: - https://wiki.postgresql.org/wiki/Loose_indexscan - Index Skip Scan, https://commitfest.postgresql.org/37/1741/ - https://www.timescale.com/blog/how-we-made-distinct-queries-up-to-8000x-faster-on-postgresql/ --- changelog.d/13575.misc | 1 + synapse/handlers/federation.py | 53 +++--------- synapse/handlers/room.py | 14 ++-- synapse/storage/controllers/state.py | 3 +- synapse/storage/databases/main/roommember.py | 88 ++++++++++++++++---- 5 files changed, 89 insertions(+), 70 deletions(-) create mode 100644 changelog.d/13575.misc diff --git a/changelog.d/13575.misc b/changelog.d/13575.misc new file mode 100644 index 0000000000..3841472617 --- /dev/null +++ b/changelog.d/13575.misc @@ -0,0 +1 @@ +Optimize how Synapse calculates domains to fetch from during backfill. diff --git a/synapse/handlers/federation.py b/synapse/handlers/federation.py index e151962055..dd4b9f66d1 100644 --- a/synapse/handlers/federation.py +++ b/synapse/handlers/federation.py @@ -70,7 +70,7 @@ from synapse.replication.http.federation import ( from synapse.storage.databases.main.events import PartialStateConflictError from synapse.storage.databases.main.events_worker import EventRedactBehaviour from synapse.storage.state import StateFilter -from synapse.types import JsonDict, StateMap, get_domain_from_id +from synapse.types import JsonDict, get_domain_from_id from synapse.util.async_helpers import Linearizer from synapse.util.retryutils import NotRetryingDestination from synapse.visibility import filter_events_for_server @@ -104,37 +104,6 @@ backfill_processing_before_timer = Histogram( ) -def get_domains_from_state(state: StateMap[EventBase]) -> List[Tuple[str, int]]: - """Get joined domains from state - - Args: - state: State map from type/state key to event. - - Returns: - Returns a list of servers with the lowest depth of their joins. - Sorted by lowest depth first. - """ - joined_users = [ - (state_key, int(event.depth)) - for (e_type, state_key), event in state.items() - if e_type == EventTypes.Member and event.membership == Membership.JOIN - ] - - joined_domains: Dict[str, int] = {} - for u, d in joined_users: - try: - dom = get_domain_from_id(u) - old_d = joined_domains.get(dom) - if old_d: - joined_domains[dom] = min(d, old_d) - else: - joined_domains[dom] = d - except Exception: - pass - - return sorted(joined_domains.items(), key=lambda d: d[1]) - - class _BackfillPointType(Enum): # a regular backwards extremity (ie, an event which we don't yet have, but which # is referred to by other events in the DAG) @@ -432,21 +401,19 @@ class FederationHandler: ) # Now we need to decide which hosts to hit first. - - # First we try hosts that are already in the room + # First we try hosts that are already in the room. # TODO: HEURISTIC ALERT. + likely_domains = ( + await self._storage_controllers.state.get_current_hosts_in_room(room_id) + ) - curr_state = await self._storage_controllers.state.get_current_state(room_id) - - curr_domains = get_domains_from_state(curr_state) - - likely_domains = [ - domain for domain, depth in curr_domains if domain != self.server_name - ] - - async def try_backfill(domains: List[str]) -> bool: + async def try_backfill(domains: Collection[str]) -> bool: # TODO: Should we try multiple of these at a time? for dom in domains: + # We don't want to ask our own server for information we don't have + if dom == self.server_name: + continue + try: await self._federation_event_handler.backfill( dom, room_id, limit=100, extremities=extremities_to_request diff --git a/synapse/handlers/room.py b/synapse/handlers/room.py index 2fc8264858..f64a8690a5 100644 --- a/synapse/handlers/room.py +++ b/synapse/handlers/room.py @@ -60,7 +60,6 @@ from synapse.event_auth import validate_event_for_room_version from synapse.events import EventBase from synapse.events.utils import copy_and_fixup_power_levels_contents from synapse.federation.federation_client import InvalidResponseError -from synapse.handlers.federation import get_domains_from_state from synapse.handlers.relations import BundledAggregations from synapse.module_api import NOT_SPAM from synapse.rest.admin._base import assert_user_is_admin @@ -1462,17 +1461,16 @@ class TimestampLookupHandler: timestamp, ) - # Find other homeservers from the given state in the room - curr_state = await self._storage_controllers.state.get_current_state( - room_id + likely_domains = ( + await self._storage_controllers.state.get_current_hosts_in_room(room_id) ) - curr_domains = get_domains_from_state(curr_state) - likely_domains = [ - domain for domain, depth in curr_domains if domain != self.server_name - ] # Loop through each homeserver candidate until we get a succesful response for domain in likely_domains: + # We don't want to ask our own server for information we don't have + if domain == self.server_name: + continue + try: remote_response = await self.federation_client.timestamp_to_event( domain, room_id, timestamp, direction diff --git a/synapse/storage/controllers/state.py b/synapse/storage/controllers/state.py index f9ffd0e29e..ba5380ce3e 100644 --- a/synapse/storage/controllers/state.py +++ b/synapse/storage/controllers/state.py @@ -23,7 +23,6 @@ from typing import ( List, Mapping, Optional, - Set, Tuple, ) @@ -520,7 +519,7 @@ class StateStorageController: ) return state_map.get(key) - async def get_current_hosts_in_room(self, room_id: str) -> Set[str]: + async def get_current_hosts_in_room(self, room_id: str) -> List[str]: """Get current hosts in room based on current state.""" await self._partial_state_room_tracker.await_full_state(room_id) diff --git a/synapse/storage/databases/main/roommember.py b/synapse/storage/databases/main/roommember.py index 9e5034b401..06500457bd 100644 --- a/synapse/storage/databases/main/roommember.py +++ b/synapse/storage/databases/main/roommember.py @@ -187,27 +187,48 @@ class RoomMemberWorkerStore(EventsWorkerStore): @cached(max_entries=100000, iterable=True) async def get_users_in_room(self, room_id: str) -> List[str]: + """ + Returns a list of users in the room sorted by longest in the room first + (aka. with the lowest depth). This is done to match the sort in + `get_current_hosts_in_room()` and so we can re-use the cache but it's + not horrible to have here either. + """ + return await self.db_pool.runInteraction( "get_users_in_room", self.get_users_in_room_txn, room_id ) def get_users_in_room_txn(self, txn: LoggingTransaction, room_id: str) -> List[str]: + """ + Returns a list of users in the room sorted by longest in the room first + (aka. with the lowest depth). This is done to match the sort in + `get_current_hosts_in_room()` and so we can re-use the cache but it's + not horrible to have here either. + """ # If we can assume current_state_events.membership is up to date # then we can avoid a join, which is a Very Good Thing given how # frequently this function gets called. if self._current_state_events_membership_up_to_date: sql = """ - SELECT state_key FROM current_state_events - WHERE type = 'm.room.member' AND room_id = ? AND membership = ? + SELECT c.state_key FROM current_state_events as c + /* Get the depth of the event from the events table */ + INNER JOIN events AS e USING (event_id) + WHERE c.type = 'm.room.member' AND c.room_id = ? AND membership = ? + /* Sorted by lowest depth first */ + ORDER BY e.depth ASC; """ else: sql = """ - SELECT state_key FROM room_memberships as m + SELECT c.state_key FROM room_memberships as m + /* Get the depth of the event from the events table */ + INNER JOIN events AS e USING (event_id) INNER JOIN current_state_events as c ON m.event_id = c.event_id AND m.room_id = c.room_id AND m.user_id = c.state_key WHERE c.type = 'm.room.member' AND c.room_id = ? AND m.membership = ? + /* Sorted by lowest depth first */ + ORDER BY e.depth ASC; """ txn.execute(sql, (room_id, Membership.JOIN)) @@ -1037,37 +1058,70 @@ class RoomMemberWorkerStore(EventsWorkerStore): return True @cached(iterable=True, max_entries=10000) - async def get_current_hosts_in_room(self, room_id: str) -> Set[str]: - """Get current hosts in room based on current state.""" + async def get_current_hosts_in_room(self, room_id: str) -> List[str]: + """ + Get current hosts in room based on current state. + + The heuristic of sorting by servers who have been in the room the + longest is good because they're most likely to have anything we ask + about. + + Returns: + Returns a list of servers sorted by longest in the room first. (aka. + sorted by join with the lowest depth first). + """ # First we check if we already have `get_users_in_room` in the cache, as # we can just calculate result from that users = self.get_users_in_room.cache.get_immediate( (room_id,), None, update_metrics=False ) - if users is not None: - return {get_domain_from_id(u) for u in users} - - if isinstance(self.database_engine, Sqlite3Engine): + if users is None and isinstance(self.database_engine, Sqlite3Engine): # If we're using SQLite then let's just always use # `get_users_in_room` rather than funky SQL. users = await self.get_users_in_room(room_id) - return {get_domain_from_id(u) for u in users} + + if users is not None: + # Because `users` is sorted from lowest -> highest depth, the list + # of domains will also be sorted that way. + domains: List[str] = [] + # We use a `Set` just for fast lookups + domain_set: Set[str] = set() + for u in users: + domain = get_domain_from_id(u) + if domain not in domain_set: + domain_set.add(domain) + domains.append(domain) + return domains # For PostgreSQL we can use a regex to pull out the domains from the # joined users in `current_state_events` via regex. - def get_current_hosts_in_room_txn(txn: LoggingTransaction) -> Set[str]: + def get_current_hosts_in_room_txn(txn: LoggingTransaction) -> List[str]: + # Returns a list of servers currently joined in the room sorted by + # longest in the room first (aka. with the lowest depth). The + # heuristic of sorting by servers who have been in the room the + # longest is good because they're most likely to have anything we + # ask about. sql = """ - SELECT DISTINCT substring(state_key FROM '@[^:]*:(.*)$') - FROM current_state_events + SELECT + /* Match the domain part of the MXID */ + substring(c.state_key FROM '@[^:]*:(.*)$') as server_domain + FROM current_state_events c + /* Get the depth of the event from the events table */ + INNER JOIN events AS e USING (event_id) WHERE - type = 'm.room.member' - AND membership = 'join' - AND room_id = ? + /* Find any join state events in the room */ + c.type = 'm.room.member' + AND c.membership = 'join' + AND c.room_id = ? + /* Group all state events from the same domain into their own buckets (groups) */ + GROUP BY server_domain + /* Sorted by lowest depth first */ + ORDER BY min(e.depth) ASC; """ txn.execute(sql, (room_id,)) - return {d for d, in txn} + return [d for d, in txn] return await self.db_pool.runInteraction( "get_current_hosts_in_room", get_current_hosts_in_room_txn From 682dfcfc0db05d9c99b7615d950997535df4d533 Mon Sep 17 00:00:00 2001 From: Dirk Klimpel <5740567+dklimpel@users.noreply.github.com> Date: Tue, 30 Aug 2022 11:58:38 +0200 Subject: [PATCH 012/107] Fix that user cannot `/forget` rooms after the last member has left (#13546) --- changelog.d/13546.bugfix | 1 + synapse/handlers/room_member.py | 7 ++- tests/handlers/test_room_member.py | 93 +++++++++++++++++++++++++++++- tests/storage/test_roommember.py | 4 +- 4 files changed, 99 insertions(+), 6 deletions(-) create mode 100644 changelog.d/13546.bugfix diff --git a/changelog.d/13546.bugfix b/changelog.d/13546.bugfix new file mode 100644 index 0000000000..83bc3a61d2 --- /dev/null +++ b/changelog.d/13546.bugfix @@ -0,0 +1 @@ +Fix bug that user cannot `/forget` rooms after the last member has left the room. \ No newline at end of file diff --git a/synapse/handlers/room_member.py b/synapse/handlers/room_member.py index 709682622f..e726997d83 100644 --- a/synapse/handlers/room_member.py +++ b/synapse/handlers/room_member.py @@ -1925,8 +1925,11 @@ class RoomMemberMasterHandler(RoomMemberHandler): ]: raise SynapseError(400, "User %s in room %s" % (user_id, room_id)) - if membership: - await self.store.forget(user_id, room_id) + # In normal case this call is only required if `membership` is not `None`. + # But: After the last member had left the room, the background update + # `_background_remove_left_rooms` is deleting rows related to this room from + # the table `current_state_events` and `get_current_state_events` is `None`. + await self.store.forget(user_id, room_id) def get_users_which_can_issue_invite(auth_events: StateMap[EventBase]) -> List[str]: diff --git a/tests/handlers/test_room_member.py b/tests/handlers/test_room_member.py index 1d13ed1e88..6bbfd5dc84 100644 --- a/tests/handlers/test_room_member.py +++ b/tests/handlers/test_room_member.py @@ -6,7 +6,7 @@ import synapse.rest.admin import synapse.rest.client.login import synapse.rest.client.room from synapse.api.constants import EventTypes, Membership -from synapse.api.errors import LimitExceededError +from synapse.api.errors import LimitExceededError, SynapseError from synapse.crypto.event_signing import add_hashes_and_signatures from synapse.events import FrozenEventV3 from synapse.federation.federation_client import SendJoinResult @@ -17,7 +17,11 @@ from synapse.util import Clock from tests.replication._base import BaseMultiWorkerStreamTestCase from tests.server import make_request from tests.test_utils import make_awaitable -from tests.unittest import FederatingHomeserverTestCase, override_config +from tests.unittest import ( + FederatingHomeserverTestCase, + HomeserverTestCase, + override_config, +) class TestJoinsLimitedByPerRoomRateLimiter(FederatingHomeserverTestCase): @@ -287,3 +291,88 @@ class TestReplicatedJoinsLimitedByPerRoomRateLimiter(BaseMultiWorkerStreamTestCa ), LimitExceededError, ) + + +class RoomMemberMasterHandlerTestCase(HomeserverTestCase): + servlets = [ + synapse.rest.admin.register_servlets, + synapse.rest.client.login.register_servlets, + synapse.rest.client.room.register_servlets, + ] + + def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None: + self.handler = hs.get_room_member_handler() + self.store = hs.get_datastores().main + + # Create two users. + self.alice = self.register_user("alice", "pass") + self.alice_ID = UserID.from_string(self.alice) + self.alice_token = self.login("alice", "pass") + self.bob = self.register_user("bob", "pass") + self.bob_ID = UserID.from_string(self.bob) + self.bob_token = self.login("bob", "pass") + + # Create a room on this homeserver. + self.room_id = self.helper.create_room_as(self.alice, tok=self.alice_token) + + def test_leave_and_forget(self) -> None: + """Tests that forget a room is successfully. The test is performed with two users, + as forgetting by the last user respectively after all users had left the + is a special edge case.""" + self.helper.join(self.room_id, user=self.bob, tok=self.bob_token) + + # alice is not the last room member that leaves and forgets the room + self.helper.leave(self.room_id, user=self.alice, tok=self.alice_token) + self.get_success(self.handler.forget(self.alice_ID, self.room_id)) + self.assertTrue( + self.get_success(self.store.did_forget(self.alice, self.room_id)) + ) + + # the server has not forgotten the room + self.assertFalse( + self.get_success(self.store.is_locally_forgotten_room(self.room_id)) + ) + + def test_leave_and_forget_last_user(self) -> None: + """Tests that forget a room is successfully when the last user has left the room.""" + + # alice is the last room member that leaves and forgets the room + self.helper.leave(self.room_id, user=self.alice, tok=self.alice_token) + self.get_success(self.handler.forget(self.alice_ID, self.room_id)) + self.assertTrue( + self.get_success(self.store.did_forget(self.alice, self.room_id)) + ) + + # the server has forgotten the room + self.assertTrue( + self.get_success(self.store.is_locally_forgotten_room(self.room_id)) + ) + + def test_forget_when_not_left(self) -> None: + """Tests that a user cannot not forgets a room that has not left.""" + self.get_failure(self.handler.forget(self.alice_ID, self.room_id), SynapseError) + + def test_rejoin_forgotten_by_user(self) -> None: + """Test that a user that has forgotten a room can do a re-join. + The room was not forgotten from the local server. + One local user is still member of the room.""" + self.helper.join(self.room_id, user=self.bob, tok=self.bob_token) + + self.helper.leave(self.room_id, user=self.alice, tok=self.alice_token) + self.get_success(self.handler.forget(self.alice_ID, self.room_id)) + self.assertTrue( + self.get_success(self.store.did_forget(self.alice, self.room_id)) + ) + + # the server has not forgotten the room + self.assertFalse( + self.get_success(self.store.is_locally_forgotten_room(self.room_id)) + ) + + self.helper.join(self.room_id, user=self.alice, tok=self.alice_token) + # TODO: A join to a room does not invalidate the forgotten cache + # see https://github.com/matrix-org/synapse/issues/13262 + self.store.did_forget.invalidate_all() + self.assertFalse( + self.get_success(self.store.did_forget(self.alice, self.room_id)) + ) diff --git a/tests/storage/test_roommember.py b/tests/storage/test_roommember.py index ceec690285..8794401823 100644 --- a/tests/storage/test_roommember.py +++ b/tests/storage/test_roommember.py @@ -158,7 +158,7 @@ class RoomMemberStoreTestCase(unittest.HomeserverTestCase): # Check that alice's display name is now None self.assertEqual(row[0]["display_name"], None) - def test_room_is_locally_forgotten(self): + def test_room_is_locally_forgotten(self) -> None: """Test that when the last local user has forgotten a room it is known as forgotten.""" # join two local and one remote user self.room = self.helper.create_room_as(self.u_alice, tok=self.t_alice) @@ -199,7 +199,7 @@ class RoomMemberStoreTestCase(unittest.HomeserverTestCase): self.get_success(self.store.is_locally_forgotten_room(self.room)) ) - def test_join_locally_forgotten_room(self): + def test_join_locally_forgotten_room(self) -> None: """Tests if a user joins a forgotten room the room is not forgotten anymore.""" self.room = self.helper.create_room_as(self.u_alice, tok=self.t_alice) self.assertFalse( From 1eea73b4133da3afc2361592f7b2fa4a4125249d Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Tue, 30 Aug 2022 06:08:29 -0500 Subject: [PATCH 013/107] Fix rate limit metrics registering twice and misreporting (#13649) * Fix rate limit metrics registering twice and misreporting Fix https://github.com/matrix-org/synapse/issues/13641 * Fix lints * Add changelog * Document `metrics_name=None`. --- changelog.d/13649.bugfix | 1 + synapse/server.py | 4 +- synapse/util/ratelimitutils.py | 155 ++++++++++++++++++++++++++------- 3 files changed, 128 insertions(+), 32 deletions(-) create mode 100644 changelog.d/13649.bugfix diff --git a/changelog.d/13649.bugfix b/changelog.d/13649.bugfix new file mode 100644 index 0000000000..e6513a585a --- /dev/null +++ b/changelog.d/13649.bugfix @@ -0,0 +1 @@ +Fix rate limit gauge metrics registering twice and misreporting (`synapse_rate_limit_sleep_affected_hosts`, `synapse_rate_limit_reject_affected_hosts`). diff --git a/synapse/server.py b/synapse/server.py index 181984a1a4..c2e55bf0b1 100644 --- a/synapse/server.py +++ b/synapse/server.py @@ -756,7 +756,9 @@ class HomeServer(metaclass=abc.ABCMeta): @cache_in_self def get_federation_ratelimiter(self) -> FederationRateLimiter: return FederationRateLimiter( - self.get_clock(), config=self.config.ratelimiting.rc_federation + self.get_clock(), + config=self.config.ratelimiting.rc_federation, + metrics_name="federation_servlets", ) @cache_in_self diff --git a/synapse/util/ratelimitutils.py b/synapse/util/ratelimitutils.py index f678b52cb4..9f64fed0d7 100644 --- a/synapse/util/ratelimitutils.py +++ b/synapse/util/ratelimitutils.py @@ -15,10 +15,23 @@ import collections import contextlib import logging +import threading import typing -from typing import Any, DefaultDict, Iterator, List, Set +from typing import ( + Any, + Callable, + DefaultDict, + Dict, + Iterator, + List, + Mapping, + Optional, + Set, + Tuple, +) from prometheus_client.core import Counter +from typing_extensions import ContextManager from twisted.internet import defer @@ -40,12 +53,20 @@ logger = logging.getLogger(__name__) # Track how much the ratelimiter is affecting requests -rate_limit_sleep_counter = Counter("synapse_rate_limit_sleep", "") -rate_limit_reject_counter = Counter("synapse_rate_limit_reject", "") +rate_limit_sleep_counter = Counter( + "synapse_rate_limit_sleep", + "Number of requests slept by the rate limiter", + ["rate_limiter_name"], +) +rate_limit_reject_counter = Counter( + "synapse_rate_limit_reject", + "Number of requests rejected by the rate limiter", + ["rate_limiter_name"], +) queue_wait_timer = Histogram( "synapse_rate_limit_queue_wait_time_seconds", - "sec", - [], + "Amount of time spent waiting for the rate limiter to let our request through.", + ["rate_limiter_name"], buckets=( 0.005, 0.01, @@ -65,35 +86,92 @@ queue_wait_timer = Histogram( ) +_rate_limiter_instances: Set["FederationRateLimiter"] = set() +# Protects the _rate_limiter_instances set from concurrent access +_rate_limiter_instances_lock = threading.Lock() + + +def _get_counts_from_rate_limiter_instance( + count_func: Callable[["FederationRateLimiter"], int] +) -> Mapping[Tuple[str, ...], int]: + """Returns a count of something (slept/rejected hosts) by (metrics_name)""" + # Cast to a list to prevent it changing while the Prometheus + # thread is collecting metrics + with _rate_limiter_instances_lock: + rate_limiter_instances = list(_rate_limiter_instances) + + # Map from (metrics_name,) -> int, the number of something like slept hosts + # or rejected hosts. The key type is Tuple[str], but we leave the length + # unspecified for compatability with LaterGauge's annotations. + counts: Dict[Tuple[str, ...], int] = {} + for rate_limiter_instance in rate_limiter_instances: + # Only track metrics if they provided a `metrics_name` to + # differentiate this instance of the rate limiter. + if rate_limiter_instance.metrics_name: + key = (rate_limiter_instance.metrics_name,) + counts[key] = count_func(rate_limiter_instance) + + return counts + + +# We track the number of affected hosts per time-period so we can +# differentiate one really noisy homeserver from a general +# ratelimit tuning problem across the federation. +LaterGauge( + "synapse_rate_limit_sleep_affected_hosts", + "Number of hosts that had requests put to sleep", + ["rate_limiter_name"], + lambda: _get_counts_from_rate_limiter_instance( + lambda rate_limiter_instance: sum( + ratelimiter.should_sleep() + for ratelimiter in rate_limiter_instance.ratelimiters.values() + ) + ), +) +LaterGauge( + "synapse_rate_limit_reject_affected_hosts", + "Number of hosts that had requests rejected", + ["rate_limiter_name"], + lambda: _get_counts_from_rate_limiter_instance( + lambda rate_limiter_instance: sum( + ratelimiter.should_reject() + for ratelimiter in rate_limiter_instance.ratelimiters.values() + ) + ), +) + + class FederationRateLimiter: - def __init__(self, clock: Clock, config: FederationRatelimitSettings): + """Used to rate limit request per-host.""" + + def __init__( + self, + clock: Clock, + config: FederationRatelimitSettings, + metrics_name: Optional[str] = None, + ): + """ + Args: + clock + config + metrics_name: The name of the rate limiter so we can differentiate it + from the rest in the metrics. If `None`, we don't track metrics + for this rate limiter. + + """ + self.metrics_name = metrics_name + def new_limiter() -> "_PerHostRatelimiter": - return _PerHostRatelimiter(clock=clock, config=config) + return _PerHostRatelimiter( + clock=clock, config=config, metrics_name=metrics_name + ) self.ratelimiters: DefaultDict[ str, "_PerHostRatelimiter" ] = collections.defaultdict(new_limiter) - # We track the number of affected hosts per time-period so we can - # differentiate one really noisy homeserver from a general - # ratelimit tuning problem across the federation. - LaterGauge( - "synapse_rate_limit_sleep_affected_hosts", - "Number of hosts that had requests put to sleep", - [], - lambda: sum( - ratelimiter.should_sleep() for ratelimiter in self.ratelimiters.values() - ), - ) - LaterGauge( - "synapse_rate_limit_reject_affected_hosts", - "Number of hosts that had requests rejected", - [], - lambda: sum( - ratelimiter.should_reject() - for ratelimiter in self.ratelimiters.values() - ), - ) + with _rate_limiter_instances_lock: + _rate_limiter_instances.add(self) def ratelimit(self, host: str) -> "_GeneratorContextManager[defer.Deferred[None]]": """Used to ratelimit an incoming request from a given host @@ -114,13 +192,23 @@ class FederationRateLimiter: class _PerHostRatelimiter: - def __init__(self, clock: Clock, config: FederationRatelimitSettings): + def __init__( + self, + clock: Clock, + config: FederationRatelimitSettings, + metrics_name: Optional[str] = None, + ): """ Args: clock config + metrics_name: The name of the rate limiter so we can differentiate it + from the rest in the metrics. If `None`, we don't track metrics + for this rate limiter. + from the rest in the metrics """ self.clock = clock + self.metrics_name = metrics_name self.window_size = config.window_size self.sleep_limit = config.sleep_limit @@ -178,7 +266,10 @@ class _PerHostRatelimiter: return len(self.request_times) > self.sleep_limit async def _on_enter_with_tracing(self, request_id: object) -> None: - with start_active_span("ratelimit wait"), queue_wait_timer.time(): + maybe_metrics_cm: ContextManager = contextlib.nullcontext() + if self.metrics_name: + maybe_metrics_cm = queue_wait_timer.labels(self.metrics_name).time() + with start_active_span("ratelimit wait"), maybe_metrics_cm: await self._on_enter(request_id) def _on_enter(self, request_id: object) -> "defer.Deferred[None]": @@ -193,7 +284,8 @@ class _PerHostRatelimiter: # sleeping or in the ready queue). if self.should_reject(): logger.debug("Ratelimiter(%s): rejecting request", self.host) - rate_limit_reject_counter.inc() + if self.metrics_name: + rate_limit_reject_counter.labels(self.metrics_name).inc() raise LimitExceededError( retry_after_ms=int(self.window_size / self.sleep_limit) ) @@ -228,7 +320,8 @@ class _PerHostRatelimiter: id(request_id), self.sleep_sec, ) - rate_limit_sleep_counter.inc() + if self.metrics_name: + rate_limit_sleep_counter.labels(self.metrics_name).inc() ret_defer = run_in_background(self.clock.sleep, self.sleep_sec) self.sleeping_requests.add(request_id) From 20df96a7a7e3d676b0beae12d0eeb1f1d668247e Mon Sep 17 00:00:00 2001 From: Patrick Cloke Date: Tue, 30 Aug 2022 07:12:48 -0400 Subject: [PATCH 014/107] Speed up inserting `event_push_actions_staging`. (#13634) By using `execute_values` instead of `execute_batch`. --- changelog.d/13634.feature | 1 + .../databases/main/event_push_actions.py | 28 ++++++------------- 2 files changed, 9 insertions(+), 20 deletions(-) create mode 100644 changelog.d/13634.feature diff --git a/changelog.d/13634.feature b/changelog.d/13634.feature new file mode 100644 index 0000000000..0a8827205d --- /dev/null +++ b/changelog.d/13634.feature @@ -0,0 +1 @@ +Improve performance of sending messages in rooms with thousands of local users. diff --git a/synapse/storage/databases/main/event_push_actions.py b/synapse/storage/databases/main/event_push_actions.py index 8dfa545c27..9f410d69de 100644 --- a/synapse/storage/databases/main/event_push_actions.py +++ b/synapse/storage/databases/main/event_push_actions.py @@ -700,26 +700,14 @@ class EventPushActionsWorkerStore(ReceiptsWorkerStore, StreamWorkerStore, SQLBas int(count_as_unread), # unread column ) - def _add_push_actions_to_staging_txn(txn: LoggingTransaction) -> None: - # We don't use simple_insert_many here to avoid the overhead - # of generating lists of dicts. - - sql = """ - INSERT INTO event_push_actions_staging - (event_id, user_id, actions, notif, highlight, unread) - VALUES (?, ?, ?, ?, ?, ?) - """ - - txn.execute_batch( - sql, - ( - _gen_entry(user_id, actions) - for user_id, actions in user_id_actions.items() - ), - ) - - return await self.db_pool.runInteraction( - "add_push_actions_to_staging", _add_push_actions_to_staging_txn + await self.db_pool.simple_insert_many( + "event_push_actions_staging", + keys=("event_id", "user_id", "actions", "notif", "highlight", "unread"), + values=[ + _gen_entry(user_id, actions) + for user_id, actions in user_id_actions.items() + ], + desc="add_push_actions_to_staging", ) async def remove_push_actions_from_staging(self, event_id: str) -> None: From 303b40b988bc372a81fc1a3cf62d3f5d074970ff Mon Sep 17 00:00:00 2001 From: Patrick Cloke Date: Tue, 30 Aug 2022 07:15:54 -0400 Subject: [PATCH 015/107] Do not wait for background updates to complete do expire URL cache. (#13657) Media downloaded as part of a URL preview is normally deleted after two days. However, while a background database migration is running, the process is stopped. A long-running database migration can therefore cause the media store to fill up with old preview files. This logic was added in #2697 to make sure that we didn't try to run the expiry without an index on `local_media_repository.created_ts`; the original logic that needs that index was added in #2478 (in `get_url_cache_media_before`, as amended by 93247a424a5068b088567fa98b6990e47608b7cb), and is still present. Given that the background update was added before Synapse v1.0.0, just drop this check and assume the index exists. --- changelog.d/13657.bugfix | 1 + synapse/rest/media/v1/preview_url_resource.py | 4 ---- 2 files changed, 1 insertion(+), 4 deletions(-) create mode 100644 changelog.d/13657.bugfix diff --git a/changelog.d/13657.bugfix b/changelog.d/13657.bugfix new file mode 100644 index 0000000000..d314d9c52f --- /dev/null +++ b/changelog.d/13657.bugfix @@ -0,0 +1 @@ +Fix a long-standing bug that downloaded media for URL previews was not deleted while database background updates were running. diff --git a/synapse/rest/media/v1/preview_url_resource.py b/synapse/rest/media/v1/preview_url_resource.py index b36c98a08e..a8f6fd6b35 100644 --- a/synapse/rest/media/v1/preview_url_resource.py +++ b/synapse/rest/media/v1/preview_url_resource.py @@ -732,10 +732,6 @@ class PreviewUrlResource(DirectServeJsonResource): logger.debug("Running url preview cache expiry") - if not (await self.store.db_pool.updates.has_completed_background_updates()): - logger.debug("Still running DB updates; skipping url preview cache expiry") - return - def try_remove_parent_dirs(dirs: Iterable[str]) -> None: """Attempt to remove the given chain of parent directories From 1c26acd815a8609314991e539dd99ceb2b9b1b43 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 30 Aug 2022 12:17:48 +0100 Subject: [PATCH 016/107] Fix bug where we wedge media plugins if clients disconnect early (#13660) We incorrectly didn't use the returned `Responder` if the client had disconnected, which meant that the resource used by the Responder wasn't correctly released. In particular, this exhausted the thread pools so that *all* requests timed out. --- changelog.d/13660.bugfix | 1 + synapse/rest/media/v1/_base.py | 40 ++++++++++++++++++---------------- 2 files changed, 22 insertions(+), 19 deletions(-) create mode 100644 changelog.d/13660.bugfix diff --git a/changelog.d/13660.bugfix b/changelog.d/13660.bugfix new file mode 100644 index 0000000000..43859a4d65 --- /dev/null +++ b/changelog.d/13660.bugfix @@ -0,0 +1 @@ +Fix bug where we wedge media plugins if clients disconnect early. Introduced in v1.22.0. diff --git a/synapse/rest/media/v1/_base.py b/synapse/rest/media/v1/_base.py index c35d42fab8..d30878f704 100644 --- a/synapse/rest/media/v1/_base.py +++ b/synapse/rest/media/v1/_base.py @@ -254,30 +254,32 @@ async def respond_with_responder( file_size: Size in bytes of the media. If not known it should be None upload_name: The name of the requested file, if any. """ - if request._disconnected: - logger.warning( - "Not sending response to request %s, already disconnected.", request - ) - return - if not responder: respond_404(request) return - logger.debug("Responding to media request with responder %s", responder) - add_file_headers(request, media_type, file_size, upload_name) - try: - with responder: - await responder.write_to_consumer(request) - except Exception as e: - # The majority of the time this will be due to the client having gone - # away. Unfortunately, Twisted simply throws a generic exception at us - # in that case. - logger.warning("Failed to write to consumer: %s %s", type(e), e) + # If we have a responder we *must* use it as a context manager. + with responder: + if request._disconnected: + logger.warning( + "Not sending response to request %s, already disconnected.", request + ) + return - # Unregister the producer, if it has one, so Twisted doesn't complain - if request.producer: - request.unregisterProducer() + logger.debug("Responding to media request with responder %s", responder) + add_file_headers(request, media_type, file_size, upload_name) + try: + + await responder.write_to_consumer(request) + except Exception as e: + # The majority of the time this will be due to the client having gone + # away. Unfortunately, Twisted simply throws a generic exception at us + # in that case. + logger.warning("Failed to write to consumer: %s %s", type(e), e) + + # Unregister the producer, if it has one, so Twisted doesn't complain + if request.producer: + request.unregisterProducer() finish_request(request) From 8f6aa015a80d1e34a25d100c5fd6ec4dde1fb728 Mon Sep 17 00:00:00 2001 From: David Robertson Date: Tue, 30 Aug 2022 12:25:44 +0100 Subject: [PATCH 017/107] 1.66.0rc2 --- CHANGES.md | 9 +++++++++ changelog.d/13649.bugfix | 1 - debian/changelog | 6 ++++++ pyproject.toml | 2 +- 4 files changed, 16 insertions(+), 2 deletions(-) delete mode 100644 changelog.d/13649.bugfix diff --git a/CHANGES.md b/CHANGES.md index 14fafc260d..8790812f8e 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,3 +1,12 @@ +Synapse 1.66.0rc2 (2022-08-30) +============================== + +Bugfixes +-------- + +- Fix rate limit gauge metrics registering twice and misreporting (`synapse_rate_limit_sleep_affected_hosts`, `synapse_rate_limit_reject_affected_hosts`). ([\#13649](https://github.com/matrix-org/synapse/issues/13649)) + + Synapse 1.66.0rc1 (2022-08-23) ============================== diff --git a/changelog.d/13649.bugfix b/changelog.d/13649.bugfix deleted file mode 100644 index e6513a585a..0000000000 --- a/changelog.d/13649.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix rate limit gauge metrics registering twice and misreporting (`synapse_rate_limit_sleep_affected_hosts`, `synapse_rate_limit_reject_affected_hosts`). diff --git a/debian/changelog b/debian/changelog index c3974261a9..b42c99d81a 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,9 @@ +matrix-synapse-py3 (1.66.0~rc2) stable; urgency=medium + + * New Synapse release 1.66.0rc2. + + -- Synapse Packaging team Tue, 30 Aug 2022 12:25:19 +0100 + matrix-synapse-py3 (1.66.0~rc1) stable; urgency=medium * New Synapse release 1.66.0rc1. diff --git a/pyproject.toml b/pyproject.toml index 745b6067aa..7146897837 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -54,7 +54,7 @@ skip_gitignore = true [tool.poetry] name = "matrix-synapse" -version = "1.66.0rc1" +version = "1.66.0rc2" description = "Homeserver for the Matrix decentralised comms protocol" authors = ["Matrix.org Team and Contributors "] license = "Apache-2.0" From e761e8b475e26341d6d26ecc1499233c5f57c7ec Mon Sep 17 00:00:00 2001 From: Patrick Cloke Date: Tue, 30 Aug 2022 08:21:19 -0400 Subject: [PATCH 018/107] Clarify documentation about replication traffic. (#13656) It can be authenticated with the worker_replication_secret setting, but is always unencrypted. --- changelog.d/13656.doc | 1 + docs/workers.md | 5 ++++- 2 files changed, 5 insertions(+), 1 deletion(-) create mode 100644 changelog.d/13656.doc diff --git a/changelog.d/13656.doc b/changelog.d/13656.doc new file mode 100644 index 0000000000..61013a0daf --- /dev/null +++ b/changelog.d/13656.doc @@ -0,0 +1 @@ +Clarify documentation that HTTP replication traffic can be protected with a shared secret. diff --git a/docs/workers.md b/docs/workers.md index 6969c424d8..dce584972d 100644 --- a/docs/workers.md +++ b/docs/workers.md @@ -120,7 +120,10 @@ redis: See the sample config for the full documentation of each option. Under **no circumstances** should the replication listener be exposed to the -public internet; it has no authentication and is unencrypted. +public internet; replication traffic is: + +* always unencrypted +* unauthenticated, unless `worker_replication_secret` is configured ### Worker configuration From 31f2a3fbc3f46b88cb400f69cd59f3551c1401f5 Mon Sep 17 00:00:00 2001 From: David Robertson Date: Tue, 30 Aug 2022 14:19:52 +0100 Subject: [PATCH 019/107] Update changes --- CHANGES.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index 8790812f8e..a39fe661ce 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -4,7 +4,7 @@ Synapse 1.66.0rc2 (2022-08-30) Bugfixes -------- -- Fix rate limit gauge metrics registering twice and misreporting (`synapse_rate_limit_sleep_affected_hosts`, `synapse_rate_limit_reject_affected_hosts`). ([\#13649](https://github.com/matrix-org/synapse/issues/13649)) +- Fix a bug introduced in Synapse 1.66.0rc1 where the new rate limit metrics were misreported (`synapse_rate_limit_sleep_affected_hosts`, `synapse_rate_limit_reject_affected_hosts`). ([\#13649](https://github.com/matrix-org/synapse/issues/13649)) Synapse 1.66.0rc1 (2022-08-23) From 372136d3a8c6cd0d7506ae75b18128476ba368e4 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff <1389908+richvdh@users.noreply.github.com> Date: Tue, 30 Aug 2022 18:01:51 +0100 Subject: [PATCH 020/107] Remove documentation of legacy `frontend_proxy` worker app (#13645) This has been the same as a generic_worker since #6964, so let's get rid of it. Fixes #3717 --- changelog.d/13645.doc | 1 + .../workers/generic_worker.yaml | 2 + docs/workers.md | 61 ++++++------------- 3 files changed, 23 insertions(+), 41 deletions(-) create mode 100644 changelog.d/13645.doc diff --git a/changelog.d/13645.doc b/changelog.d/13645.doc new file mode 100644 index 0000000000..04c302ec2c --- /dev/null +++ b/changelog.d/13645.doc @@ -0,0 +1 @@ +Remove documentation of legacy `frontend_proxy` worker app. diff --git a/docs/systemd-with-workers/workers/generic_worker.yaml b/docs/systemd-with-workers/workers/generic_worker.yaml index a82f9c161f..6e7b60886e 100644 --- a/docs/systemd-with-workers/workers/generic_worker.yaml +++ b/docs/systemd-with-workers/workers/generic_worker.yaml @@ -5,6 +5,8 @@ worker_name: generic_worker1 worker_replication_host: 127.0.0.1 worker_replication_http_port: 9093 +worker_main_http_uri: http://localhost:8008/ + worker_listeners: - type: http port: 8083 diff --git a/docs/workers.md b/docs/workers.md index dce584972d..176bb1475e 100644 --- a/docs/workers.md +++ b/docs/workers.md @@ -117,7 +117,7 @@ redis: enabled: true ``` -See the sample config for the full documentation of each option. +See the [configuration manual](usage/configuration/config_documentation.html) for the full documentation of each option. Under **no circumstances** should the replication listener be exposed to the public internet; replication traffic is: @@ -128,15 +128,15 @@ public internet; replication traffic is: ### Worker configuration -In the config file for each worker, you must specify the type of worker -application (`worker_app`), and you should specify a unique name for the worker -(`worker_name`). The currently available worker applications are listed below. -You must also specify the HTTP replication endpoint that it should talk to on -the main synapse process. `worker_replication_host` should specify the host of -the main synapse and `worker_replication_http_port` should point to the HTTP -replication port. If the worker will handle HTTP requests then the -`worker_listeners` option should be set with a `http` listener, in the same way -as the `listeners` option in the shared config. +In the config file for each worker, you must specify: + * The type of worker (`worker_app`). The currently available worker applications are listed below. + * A unique name for the worker (`worker_name`). + * The HTTP replication endpoint that it should talk to on the main synapse process + (`worker_replication_host` and `worker_replication_http_port`) + * If handling HTTP requests, a `worker_listeners` option with an `http` + listener, in the same way as the `listeners` option in the shared config. + * If handling the `^/_matrix/client/v3/keys/upload` endpoint, the HTTP URI for + the main process (`worker_main_http_uri`). For example: @@ -220,10 +220,12 @@ information. ^/_matrix/client/(api/v1|r0|v3|unstable)/search$ # Encryption requests + # Note that ^/_matrix/client/(r0|v3|unstable)/keys/upload/ requires `worker_main_http_uri` ^/_matrix/client/(r0|v3|unstable)/keys/query$ ^/_matrix/client/(r0|v3|unstable)/keys/changes$ ^/_matrix/client/(r0|v3|unstable)/keys/claim$ ^/_matrix/client/(r0|v3|unstable)/room_keys/ + ^/_matrix/client/(r0|v3|unstable)/keys/upload/ # Registration/login requests ^/_matrix/client/(api/v1|r0|v3|unstable)/login$ @@ -584,39 +586,16 @@ handle it, and are online. If `update_user_directory` is set to `false`, and this worker is not running, the above endpoint may give outdated results. -### `synapse.app.frontend_proxy` - -Proxies some frequently-requested client endpoints to add caching and remove -load from the main synapse. It can handle REST endpoints matching the following -regular expressions: - - ^/_matrix/client/(r0|v3|unstable)/keys/upload - -If `use_presence` is False in the homeserver config, it can also handle REST -endpoints matching the following regular expressions: - - ^/_matrix/client/(api/v1|r0|v3|unstable)/presence/[^/]+/status - -This "stub" presence handler will pass through `GET` request but make the -`PUT` effectively a no-op. - -It will proxy any requests it cannot handle to the main synapse instance. It -must therefore be configured with the location of the main instance, via -the `worker_main_http_uri` setting in the `frontend_proxy` worker configuration -file. For example: - -```yaml -worker_main_http_uri: http://127.0.0.1:8008 -``` - ### Historical apps -*Note:* Historically there used to be more apps, however they have been -amalgamated into a single `synapse.app.generic_worker` app. The remaining apps -are ones that do specific processing unrelated to requests, e.g. the `pusher` -that handles sending out push notifications for new events. The intention is for -all these to be folded into the `generic_worker` app and to use config to define -which processes handle the various proccessing such as push notifications. +The following used to be separate worker application types, but are now +equivalent to `synapse.app.generic_worker`: + + * `synapse.app.client_reader` + * `synapse.app.event_creator` + * `synapse.app.federation_reader` + * `synapse.app.frontend_proxy` + * `synapse.app.synchrotron` ## Migration from old config From 20c76cecb9eb84dadfa7b2d25b436d3ab9218a1a Mon Sep 17 00:00:00 2001 From: Shay Date: Tue, 30 Aug 2022 10:29:16 -0700 Subject: [PATCH 021/107] Drop unused column `application_services_state.last_txn` (#13627) --- changelog.d/13627.misc | 1 + synapse/storage/schema/__init__.py | 1 + ...ation_services_state_last_txn.sql.postgres | 17 ++++++++ ...ication_services_state_last_txn.sql.sqlite | 40 +++++++++++++++++++ 4 files changed, 59 insertions(+) create mode 100644 changelog.d/13627.misc create mode 100644 synapse/storage/schema/main/delta/72/04drop_column_application_services_state_last_txn.sql.postgres create mode 100644 synapse/storage/schema/main/delta/72/04drop_column_application_services_state_last_txn.sql.sqlite diff --git a/changelog.d/13627.misc b/changelog.d/13627.misc new file mode 100644 index 0000000000..1a15709aff --- /dev/null +++ b/changelog.d/13627.misc @@ -0,0 +1 @@ +Drop unused column `application_services_state.last_txn`. diff --git a/synapse/storage/schema/__init__.py b/synapse/storage/schema/__init__.py index a9a88c8bfd..256f745dc0 100644 --- a/synapse/storage/schema/__init__.py +++ b/synapse/storage/schema/__init__.py @@ -75,6 +75,7 @@ Changes in SCHEMA_VERSION = 71: Changes in SCHEMA_VERSION = 72: - event_edges.(room_id, is_state) are no longer written to. - Tables related to groups are dropped. + - Unused column application_services_state.last_txn is dropped """ diff --git a/synapse/storage/schema/main/delta/72/04drop_column_application_services_state_last_txn.sql.postgres b/synapse/storage/schema/main/delta/72/04drop_column_application_services_state_last_txn.sql.postgres new file mode 100644 index 0000000000..13d47de9e6 --- /dev/null +++ b/synapse/storage/schema/main/delta/72/04drop_column_application_services_state_last_txn.sql.postgres @@ -0,0 +1,17 @@ +/* Copyright 2022 The Matrix.org Foundation C.I.C + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +-- Drop unused column application_services_state.last_txn +ALTER table application_services_state DROP COLUMN last_txn; \ No newline at end of file diff --git a/synapse/storage/schema/main/delta/72/04drop_column_application_services_state_last_txn.sql.sqlite b/synapse/storage/schema/main/delta/72/04drop_column_application_services_state_last_txn.sql.sqlite new file mode 100644 index 0000000000..3be1c88d72 --- /dev/null +++ b/synapse/storage/schema/main/delta/72/04drop_column_application_services_state_last_txn.sql.sqlite @@ -0,0 +1,40 @@ +/* Copyright 2022 The Matrix.org Foundation C.I.C + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +-- Drop unused column application_services_state.last_txn + +CREATE TABLE application_services_state2 ( + as_id TEXT PRIMARY KEY NOT NULL, + state VARCHAR(5), + read_receipt_stream_id BIGINT, + presence_stream_id BIGINT, + to_device_stream_id BIGINT, + device_list_stream_id BIGINT +); + + +INSERT INTO application_services_state2 ( + as_id, + state, + read_receipt_stream_id, + presence_stream_id, + to_device_stream_id, + device_list_stream_id +) +SELECT as_id, state, read_receipt_stream_id, presence_stream_id, to_device_stream_id, device_list_stream_id +FROM application_services_state; + +DROP TABLE application_services_state; +ALTER TABLE application_services_state2 RENAME TO application_services_state; From 92c5817e34cb421dbe2c0df275238e84866932a8 Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Tue, 30 Aug 2022 14:50:06 -0500 Subject: [PATCH 022/107] Give the correct next event when the message timestamps are the same - MSC3030 (#13658) Discovered while working on https://github.com/matrix-org/synapse/pull/13589 and I had all the messages at the same timestamp in the tests. Part of https://github.com/matrix-org/matrix-spec-proposals/pull/3030 Complement tests: https://github.com/matrix-org/complement/pull/457 --- changelog.d/13658.bugfix | 1 + synapse/storage/databases/main/events_worker.py | 12 ++++++++++-- 2 files changed, 11 insertions(+), 2 deletions(-) create mode 100644 changelog.d/13658.bugfix diff --git a/changelog.d/13658.bugfix b/changelog.d/13658.bugfix new file mode 100644 index 0000000000..8740f066bb --- /dev/null +++ b/changelog.d/13658.bugfix @@ -0,0 +1 @@ +Fix MSC3030 `/timestamp_to_event` endpoint to return the correct next event when the events have the same timestamp. diff --git a/synapse/storage/databases/main/events_worker.py b/synapse/storage/databases/main/events_worker.py index 8a7cdb024d..9b997c304d 100644 --- a/synapse/storage/databases/main/events_worker.py +++ b/synapse/storage/databases/main/events_worker.py @@ -2111,7 +2111,14 @@ class EventsWorkerStore(SQLBaseStore): AND room_id = ? /* Make sure event is not rejected */ AND rejections.event_id IS NULL - ORDER BY origin_server_ts %s + /** + * First sort by the message timestamp. If the message timestamps are the + * same, we want the message that logically comes "next" (before/after + * the given timestamp) based on the DAG and its topological order (`depth`). + * Finally, we can tie-break based on when it was received on the server + * (`stream_ordering`). + */ + ORDER BY origin_server_ts %s, depth %s, stream_ordering %s LIMIT 1; """ @@ -2130,7 +2137,8 @@ class EventsWorkerStore(SQLBaseStore): order = "ASC" txn.execute( - sql_template % (comparison_operator, order), (timestamp, room_id) + sql_template % (comparison_operator, order, order, order), + (timestamp, room_id), ) row = txn.fetchone() if row: From 61b37ddd37dbe6c54576a97be7f85ad9735252c0 Mon Sep 17 00:00:00 2001 From: Patrick Cloke Date: Wed, 31 Aug 2022 05:43:00 -0400 Subject: [PATCH 023/107] Remind people that direct TCP replication is disabled. (#13674) --- CHANGES.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/CHANGES.md b/CHANGES.md index a39fe661ce..0f822556ec 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,3 +1,11 @@ +Synapse 1.66.0 +============== + +Deployments with multiple workers should note that the direct TCP replication +configuration was deprecated in Synapse v1.18.0 and will be removed in Synapse +v1.67.0. See [docs/workers.md](https://github.com/matrix-org/synapse/blob/release-v1.18.0/docs/workers.md) +for more details. + Synapse 1.66.0rc2 (2022-08-30) ============================== From b9924df264884634092465b9d2edac3393768f2e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Behrmann?= Date: Wed, 31 Aug 2022 12:15:28 +0200 Subject: [PATCH 024/107] Change dpkg-statoverride to use --force-statoverride-add (#13638) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The --force flag of dpkg-statoverride has been deprecated (apparently starting with the dpkg version in Debian buster). It offers --force-all as q quick fix, but the usage in the Debian postinst script is probably covered by --force-statoverride-add. Fixes: #8391 Signed-off-by: Jörg Behrmann --- debian/changelog | 1 + debian/matrix-synapse-py3.postinst | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/debian/changelog b/debian/changelog index 0709ae24e9..6a6f306019 100644 --- a/debian/changelog +++ b/debian/changelog @@ -3,6 +3,7 @@ matrix-synapse-py3 (1.66.0~rc2+nmu1) UNRELEASED; urgency=medium * Update debhelper to compatibility level 12. * Drop the preinst script stopping synapse. * Allocate a group for the system user. + * Change dpkg-statoverride to --force-statoverride-add. -- Jörg Behrmann Tue, 23 Aug 2022 17:17:00 +0100 diff --git a/debian/matrix-synapse-py3.postinst b/debian/matrix-synapse-py3.postinst index 3c72b69bb7..acab0877ad 100644 --- a/debian/matrix-synapse-py3.postinst +++ b/debian/matrix-synapse-py3.postinst @@ -45,7 +45,7 @@ EOF for DIR in /var/lib/matrix-synapse /var/log/matrix-synapse /etc/matrix-synapse; do if ! dpkg-statoverride --list --quiet $DIR >/dev/null; then - dpkg-statoverride --force --quiet --update --add $USER "$(id -gn $USER)" 0755 $DIR + dpkg-statoverride --force-statoverride-add --quiet --update --add $USER "$(id -gn $USER)" 0755 $DIR fi done From d48b70fd37888346fc6a4c800ecfaa6a2395d087 Mon Sep 17 00:00:00 2001 From: Sean Quah Date: Wed, 31 Aug 2022 11:18:56 +0100 Subject: [PATCH 025/107] Update changelog for v1.62.0 --- CHANGES.md | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/CHANGES.md b/CHANGES.md index 0f822556ec..9ba9498344 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -398,6 +398,20 @@ No significant changes since 1.62.0rc3. Authors of spam-checker plugins should consult the [upgrade notes](https://github.com/matrix-org/synapse/blob/release-v1.62/docs/upgrade.md#upgrading-to-v1620) to learn about the enriched signatures for spam checker callbacks, which are supported with this release of Synapse. +## Security advisory + +The following issue is fixed in 1.62.0. + +* [GHSA-jhjh-776m-4765](https://github.com/matrix-org/synapse/security/advisories/GHSA-jhjh-776m-4765) / [CVE-2022-31152](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2022-31152) + + Synapse instances prior to 1.62.0 did not implement the Matrix [event authorization rules](https://spec.matrix.org/v1.3/rooms/v10/#authorization-rules) correctly. An attacker could craft events which would be accepted by Synapse but not a spec-conformant server, potentially causing divergence in the room state between servers. + + Homeservers with federation disabled via the [`federation_domain_whitelist`](https://matrix-org.github.io/synapse/latest/usage/configuration/config_documentation.html#federation_domain_whitelist) config option are unaffected. + + Administrators of homeservers with federation enabled are advised to upgrade to v1.62.0 or higher. + + Fixed by [#13087](https://github.com/matrix-org/synapse/pull/13087) and [#13088](https://github.com/matrix-org/synapse/pull/13088). + Synapse 1.62.0rc3 (2022-07-04) ============================== From ef88bc0775b7c01dc3abfbfca3e8aaa566b1871d Mon Sep 17 00:00:00 2001 From: Sean Quah Date: Wed, 31 Aug 2022 11:21:09 +0100 Subject: [PATCH 026/107] 1.66.0 --- CHANGES.md | 7 +++++-- debian/changelog | 6 ++++++ pyproject.toml | 2 +- 3 files changed, 12 insertions(+), 3 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 9ba9498344..c890f64d25 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,11 +1,14 @@ -Synapse 1.66.0 -============== +Synapse 1.66.0 (2022-08-31) +=========================== + +No significant changes since 1.66.0rc2. Deployments with multiple workers should note that the direct TCP replication configuration was deprecated in Synapse v1.18.0 and will be removed in Synapse v1.67.0. See [docs/workers.md](https://github.com/matrix-org/synapse/blob/release-v1.18.0/docs/workers.md) for more details. + Synapse 1.66.0rc2 (2022-08-30) ============================== diff --git a/debian/changelog b/debian/changelog index b42c99d81a..7ff1428842 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,9 @@ +matrix-synapse-py3 (1.66.0) stable; urgency=medium + + * New Synapse release 1.66.0. + + -- Synapse Packaging team Wed, 31 Aug 2022 11:20:17 +0100 + matrix-synapse-py3 (1.66.0~rc2) stable; urgency=medium * New Synapse release 1.66.0rc2. diff --git a/pyproject.toml b/pyproject.toml index 7146897837..a41d88ea74 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -54,7 +54,7 @@ skip_gitignore = true [tool.poetry] name = "matrix-synapse" -version = "1.66.0rc2" +version = "1.66.0" description = "Homeserver for the Matrix decentralised comms protocol" authors = ["Matrix.org Team and Contributors "] license = "Apache-2.0" From 5634267d33fd55131dfb0789e5510247a0b6e8f2 Mon Sep 17 00:00:00 2001 From: Sean Quah Date: Wed, 31 Aug 2022 11:31:51 +0100 Subject: [PATCH 027/107] Update changelog to link to the Synapse docs instead of markdown --- CHANGES.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index c890f64d25..f502866f76 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -5,7 +5,7 @@ No significant changes since 1.66.0rc2. Deployments with multiple workers should note that the direct TCP replication configuration was deprecated in Synapse v1.18.0 and will be removed in Synapse -v1.67.0. See [docs/workers.md](https://github.com/matrix-org/synapse/blob/release-v1.18.0/docs/workers.md) +v1.67.0. See the [worker documentation](https://matrix-org.github.io/synapse/v1.66/workers.html) for more details. From a160406d245cb84b48fa67fe3ee73f0cffceb495 Mon Sep 17 00:00:00 2001 From: David Robertson Date: Wed, 31 Aug 2022 11:38:16 +0100 Subject: [PATCH 028/107] Fix admin List Room API return type on sqlite (#13509) --- changelog.d/13509.bugfix | 1 + synapse/storage/databases/main/room.py | 6 ++++-- tests/rest/admin/test_room.py | 19 ++++++++++++++----- 3 files changed, 19 insertions(+), 7 deletions(-) create mode 100644 changelog.d/13509.bugfix diff --git a/changelog.d/13509.bugfix b/changelog.d/13509.bugfix new file mode 100644 index 0000000000..6dcb9741d9 --- /dev/null +++ b/changelog.d/13509.bugfix @@ -0,0 +1 @@ +Fix a bug introduced in Synapse 1.13 where the [List Rooms admin API](https://matrix-org.github.io/synapse/develop/admin_api/rooms.html#list-room-api) would return integers instead of booleans for the `federatable` and `public` fields when using a Sqlite database. diff --git a/synapse/storage/databases/main/room.py b/synapse/storage/databases/main/room.py index b7d4baa6bb..367424b4a8 100644 --- a/synapse/storage/databases/main/room.py +++ b/synapse/storage/databases/main/room.py @@ -641,8 +641,10 @@ class RoomWorkerStore(CacheInvalidationWorkerStore): "version": room[5], "creator": room[6], "encryption": room[7], - "federatable": room[8], - "public": room[9], + # room_stats_state.federatable is an integer on sqlite. + "federatable": bool(room[8]), + # rooms.is_public is an integer on sqlite. + "public": bool(room[9]), "join_rules": room[10], "guest_access": room[11], "history_visibility": room[12], diff --git a/tests/rest/admin/test_room.py b/tests/rest/admin/test_room.py index fd6da557c1..9d71a97524 100644 --- a/tests/rest/admin/test_room.py +++ b/tests/rest/admin/test_room.py @@ -1080,7 +1080,9 @@ class RoomTestCase(unittest.HomeserverTestCase): room_ids = [] for _ in range(total_rooms): room_id = self.helper.create_room_as( - self.admin_user, tok=self.admin_user_tok + self.admin_user, + tok=self.admin_user_tok, + is_public=True, ) room_ids.append(room_id) @@ -1119,8 +1121,8 @@ class RoomTestCase(unittest.HomeserverTestCase): self.assertIn("version", r) self.assertIn("creator", r) self.assertIn("encryption", r) - self.assertIn("federatable", r) - self.assertIn("public", r) + self.assertIs(r["federatable"], True) + self.assertIs(r["public"], True) self.assertIn("join_rules", r) self.assertIn("guest_access", r) self.assertIn("history_visibility", r) @@ -1587,8 +1589,12 @@ class RoomTestCase(unittest.HomeserverTestCase): def test_single_room(self) -> None: """Test that a single room can be requested correctly""" # Create two test rooms - room_id_1 = self.helper.create_room_as(self.admin_user, tok=self.admin_user_tok) - room_id_2 = self.helper.create_room_as(self.admin_user, tok=self.admin_user_tok) + room_id_1 = self.helper.create_room_as( + self.admin_user, tok=self.admin_user_tok, is_public=True + ) + room_id_2 = self.helper.create_room_as( + self.admin_user, tok=self.admin_user_tok, is_public=False + ) room_name_1 = "something" room_name_2 = "else" @@ -1634,7 +1640,10 @@ class RoomTestCase(unittest.HomeserverTestCase): self.assertIn("state_events", channel.json_body) self.assertIn("room_type", channel.json_body) self.assertIn("forgotten", channel.json_body) + self.assertEqual(room_id_1, channel.json_body["room_id"]) + self.assertIs(True, channel.json_body["federatable"]) + self.assertIs(True, channel.json_body["public"]) def test_single_room_devices(self) -> None: """Test that `joined_local_devices` can be requested correctly""" From 90c99fb3aab2d371039cf4aaa61305928e77230d Mon Sep 17 00:00:00 2001 From: Sean Quah Date: Wed, 31 Aug 2022 11:53:30 +0100 Subject: [PATCH 029/107] Fix dead link in 1.18.0 upgrade notes --- docs/upgrade.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/upgrade.md b/docs/upgrade.md index 0ab5bfeaf0..51719f8c73 100644 --- a/docs/upgrade.md +++ b/docs/upgrade.md @@ -1200,7 +1200,7 @@ updated. When setting up worker processes, we now recommend the use of a Redis server for replication. **The old direct TCP connection method is deprecated and will be removed in a future release.** See -[workers](workers.md) for more details. +the [worker documentation](https://matrix-org.github.io/synapse/v1.66/workers.html) for more details. # Upgrading to v1.14.0 From 7bc110a19e6de0572b0c9513726d13298b45ced2 Mon Sep 17 00:00:00 2001 From: reivilibre Date: Wed, 31 Aug 2022 11:16:05 +0000 Subject: [PATCH 030/107] Generalise the `@cancellable` annotation so it can be used on functions other than just servlet methods. (#13662) --- changelog.d/13662.misc | 1 + synapse/federation/transport/server/_base.py | 5 +- synapse/http/server.py | 68 +------------------ synapse/replication/http/_base.py | 7 +- synapse/rest/client/room.py | 3 +- synapse/util/cancellation.py | 56 +++++++++++++++ .../federation/transport/server/test__base.py | 3 +- tests/http/test_servlet.py | 2 +- tests/replication/http/test__base.py | 3 +- tests/test_server.py | 2 +- 10 files changed, 75 insertions(+), 75 deletions(-) create mode 100644 changelog.d/13662.misc create mode 100644 synapse/util/cancellation.py diff --git a/changelog.d/13662.misc b/changelog.d/13662.misc new file mode 100644 index 0000000000..3dea4a1c2c --- /dev/null +++ b/changelog.d/13662.misc @@ -0,0 +1 @@ +Generalise the `@cancellable` annotation so it can be used on functions other than just servlet methods. \ No newline at end of file diff --git a/synapse/federation/transport/server/_base.py b/synapse/federation/transport/server/_base.py index bb0f8d6b7b..1db8009d6c 100644 --- a/synapse/federation/transport/server/_base.py +++ b/synapse/federation/transport/server/_base.py @@ -21,7 +21,7 @@ from typing import TYPE_CHECKING, Any, Awaitable, Callable, Dict, Optional, Tupl from synapse.api.errors import Codes, FederationDeniedError, SynapseError from synapse.api.urls import FEDERATION_V1_PREFIX -from synapse.http.server import HttpServer, ServletCallback, is_method_cancellable +from synapse.http.server import HttpServer, ServletCallback from synapse.http.servlet import parse_json_object_from_request from synapse.http.site import SynapseRequest from synapse.logging.context import run_in_background @@ -34,6 +34,7 @@ from synapse.logging.opentracing import ( whitelisted_homeserver, ) from synapse.types import JsonDict +from synapse.util.cancellation import is_function_cancellable from synapse.util.ratelimitutils import FederationRateLimiter from synapse.util.stringutils import parse_and_validate_server_name @@ -375,7 +376,7 @@ class BaseFederationServlet: if code is None: continue - if is_method_cancellable(code): + if is_function_cancellable(code): # The wrapper added by `self._wrap` will inherit the cancellable flag, # but the wrapper itself does not support cancellation yet. # Once resolved, the cancellation tests in diff --git a/synapse/http/server.py b/synapse/http/server.py index 19f42159b8..6068a94b40 100644 --- a/synapse/http/server.py +++ b/synapse/http/server.py @@ -33,7 +33,6 @@ from typing import ( Optional, Pattern, Tuple, - TypeVar, Union, ) @@ -64,6 +63,7 @@ from synapse.logging.context import defer_to_thread, preserve_fn, run_in_backgro from synapse.logging.opentracing import active_span, start_active_span, trace_servlet from synapse.util import json_encoder from synapse.util.caches import intern_dict +from synapse.util.cancellation import is_function_cancellable from synapse.util.iterutils import chunk_seq if TYPE_CHECKING: @@ -94,68 +94,6 @@ HTML_ERROR_TEMPLATE = """ HTTP_STATUS_REQUEST_CANCELLED = 499 -F = TypeVar("F", bound=Callable[..., Any]) - - -_cancellable_method_names = frozenset( - { - # `RestServlet`, `BaseFederationServlet` and `BaseFederationServerServlet` - # methods - "on_GET", - "on_PUT", - "on_POST", - "on_DELETE", - # `_AsyncResource`, `DirectServeHtmlResource` and `DirectServeJsonResource` - # methods - "_async_render_GET", - "_async_render_PUT", - "_async_render_POST", - "_async_render_DELETE", - "_async_render_OPTIONS", - # `ReplicationEndpoint` methods - "_handle_request", - } -) - - -def cancellable(method: F) -> F: - """Marks a servlet method as cancellable. - - Methods with this decorator will be cancelled if the client disconnects before we - finish processing the request. - - During cancellation, `Deferred.cancel()` will be invoked on the `Deferred` wrapping - the method. The `cancel()` call will propagate down to the `Deferred` that is - currently being waited on. That `Deferred` will raise a `CancelledError`, which will - propagate up, as per normal exception handling. - - Before applying this decorator to a new endpoint, you MUST recursively check - that all `await`s in the function are on `async` functions or `Deferred`s that - handle cancellation cleanly, otherwise a variety of bugs may occur, ranging from - premature logging context closure, to stuck requests, to database corruption. - - Usage: - class SomeServlet(RestServlet): - @cancellable - async def on_GET(self, request: SynapseRequest) -> ...: - ... - """ - if method.__name__ not in _cancellable_method_names and not any( - method.__name__.startswith(prefix) for prefix in _cancellable_method_names - ): - raise ValueError( - "@cancellable decorator can only be applied to servlet methods." - ) - - method.cancellable = True # type: ignore[attr-defined] - return method - - -def is_method_cancellable(method: Callable[..., Any]) -> bool: - """Checks whether a servlet method has the `@cancellable` flag.""" - return getattr(method, "cancellable", False) - - def return_json_error( f: failure.Failure, request: SynapseRequest, config: Optional[HomeServerConfig] ) -> None: @@ -389,7 +327,7 @@ class _AsyncResource(resource.Resource, metaclass=abc.ABCMeta): method_handler = getattr(self, "_async_render_%s" % (request_method,), None) if method_handler: - request.is_render_cancellable = is_method_cancellable(method_handler) + request.is_render_cancellable = is_function_cancellable(method_handler) raw_callback_return = method_handler(request) @@ -551,7 +489,7 @@ class JsonResource(DirectServeJsonResource): async def _async_render(self, request: SynapseRequest) -> Tuple[int, Any]: callback, servlet_classname, group_dict = self._get_handler_for_request(request) - request.is_render_cancellable = is_method_cancellable(callback) + request.is_render_cancellable = is_function_cancellable(callback) # Make sure we have an appropriate name for this handler in prometheus # (rather than the default of JsonResource). diff --git a/synapse/replication/http/_base.py b/synapse/replication/http/_base.py index 561ad5bf04..acb0bd18f7 100644 --- a/synapse/replication/http/_base.py +++ b/synapse/replication/http/_base.py @@ -26,12 +26,13 @@ from twisted.web.server import Request from synapse.api.errors import HttpResponseException, SynapseError from synapse.http import RequestTimedOutError -from synapse.http.server import HttpServer, is_method_cancellable +from synapse.http.server import HttpServer from synapse.http.site import SynapseRequest from synapse.logging import opentracing from synapse.logging.opentracing import trace_with_opname from synapse.types import JsonDict from synapse.util.caches.response_cache import ResponseCache +from synapse.util.cancellation import is_function_cancellable from synapse.util.stringutils import random_string if TYPE_CHECKING: @@ -311,7 +312,7 @@ class ReplicationEndpoint(metaclass=abc.ABCMeta): url_args = list(self.PATH_ARGS) method = self.METHOD - if self.CACHE and is_method_cancellable(self._handle_request): + if self.CACHE and is_function_cancellable(self._handle_request): raise Exception( f"{self.__class__.__name__} has been marked as cancellable, but CACHE " "is set. The cancellable flag would have no effect." @@ -359,6 +360,6 @@ class ReplicationEndpoint(metaclass=abc.ABCMeta): # The `@cancellable` decorator may be applied to `_handle_request`. But we # told `HttpServer.register_paths` that our handler is `_check_auth_and_handle`, # so we have to set up the cancellable flag ourselves. - request.is_render_cancellable = is_method_cancellable(self._handle_request) + request.is_render_cancellable = is_function_cancellable(self._handle_request) return await self._handle_request(request, **kwargs) diff --git a/synapse/rest/client/room.py b/synapse/rest/client/room.py index 3259de4802..0e2834008e 100644 --- a/synapse/rest/client/room.py +++ b/synapse/rest/client/room.py @@ -37,7 +37,7 @@ from synapse.api.errors import ( ) from synapse.api.filtering import Filter from synapse.events.utils import format_event_for_client_v2 -from synapse.http.server import HttpServer, cancellable +from synapse.http.server import HttpServer from synapse.http.servlet import ( ResolveRoomIdMixin, RestServlet, @@ -57,6 +57,7 @@ from synapse.storage.state import StateFilter from synapse.streams.config import PaginationConfig from synapse.types import JsonDict, StreamToken, ThirdPartyInstanceID, UserID from synapse.util import json_decoder +from synapse.util.cancellation import cancellable from synapse.util.stringutils import parse_and_validate_server_name, random_string if TYPE_CHECKING: diff --git a/synapse/util/cancellation.py b/synapse/util/cancellation.py new file mode 100644 index 0000000000..472d2e3aeb --- /dev/null +++ b/synapse/util/cancellation.py @@ -0,0 +1,56 @@ +# Copyright 2022 The Matrix.org Foundation C.I.C. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from typing import Any, Callable, TypeVar + +F = TypeVar("F", bound=Callable[..., Any]) + + +def cancellable(function: F) -> F: + """Marks a function as cancellable. + + Servlet methods with this decorator will be cancelled if the client disconnects before we + finish processing the request. + + Although this annotation is particularly useful for servlet methods, it's also + useful for intermediate functions, where it documents the fact that the function has + been audited for cancellation safety and needs to preserve that. + This then simplifies auditing new functions that call those same intermediate + functions. + + During cancellation, `Deferred.cancel()` will be invoked on the `Deferred` wrapping + the method. The `cancel()` call will propagate down to the `Deferred` that is + currently being waited on. That `Deferred` will raise a `CancelledError`, which will + propagate up, as per normal exception handling. + + Before applying this decorator to a new function, you MUST recursively check + that all `await`s in the function are on `async` functions or `Deferred`s that + handle cancellation cleanly, otherwise a variety of bugs may occur, ranging from + premature logging context closure, to stuck requests, to database corruption. + + See the documentation page on Cancellation for more information. + + Usage: + class SomeServlet(RestServlet): + @cancellable + async def on_GET(self, request: SynapseRequest) -> ...: + ... + """ + + function.cancellable = True # type: ignore[attr-defined] + return function + + +def is_function_cancellable(function: Callable[..., Any]) -> bool: + """Checks whether a servlet method has the `@cancellable` flag.""" + return getattr(function, "cancellable", False) diff --git a/tests/federation/transport/server/test__base.py b/tests/federation/transport/server/test__base.py index d33e86db4c..e88e5d8bb3 100644 --- a/tests/federation/transport/server/test__base.py +++ b/tests/federation/transport/server/test__base.py @@ -18,9 +18,10 @@ from typing import Dict, List, Tuple from synapse.api.errors import Codes from synapse.federation.transport.server import BaseFederationServlet from synapse.federation.transport.server._base import Authenticator, _parse_auth_header -from synapse.http.server import JsonResource, cancellable +from synapse.http.server import JsonResource from synapse.server import HomeServer from synapse.types import JsonDict +from synapse.util.cancellation import cancellable from synapse.util.ratelimitutils import FederationRateLimiter from tests import unittest diff --git a/tests/http/test_servlet.py b/tests/http/test_servlet.py index bb966c80c6..3cbca0f5a3 100644 --- a/tests/http/test_servlet.py +++ b/tests/http/test_servlet.py @@ -18,7 +18,6 @@ from typing import Tuple from unittest.mock import Mock from synapse.api.errors import Codes, SynapseError -from synapse.http.server import cancellable from synapse.http.servlet import ( RestServlet, parse_json_object_from_request, @@ -28,6 +27,7 @@ from synapse.http.site import SynapseRequest from synapse.rest.client._base import client_patterns from synapse.server import HomeServer from synapse.types import JsonDict +from synapse.util.cancellation import cancellable from tests import unittest from tests.http.server._base import test_disconnect diff --git a/tests/replication/http/test__base.py b/tests/replication/http/test__base.py index 822a957c3a..936ab4504a 100644 --- a/tests/replication/http/test__base.py +++ b/tests/replication/http/test__base.py @@ -18,11 +18,12 @@ from typing import Tuple from twisted.web.server import Request from synapse.api.errors import Codes -from synapse.http.server import JsonResource, cancellable +from synapse.http.server import JsonResource from synapse.replication.http import REPLICATION_PREFIX from synapse.replication.http._base import ReplicationEndpoint from synapse.server import HomeServer from synapse.types import JsonDict +from synapse.util.cancellation import cancellable from tests import unittest from tests.http.server._base import test_disconnect diff --git a/tests/test_server.py b/tests/test_server.py index d2b2d8344a..23975d59c3 100644 --- a/tests/test_server.py +++ b/tests/test_server.py @@ -26,12 +26,12 @@ from synapse.http.server import ( DirectServeJsonResource, JsonResource, OptionsResource, - cancellable, ) from synapse.http.site import SynapseRequest, SynapseSite from synapse.logging.context import make_deferred_yieldable from synapse.types import JsonDict from synapse.util import Clock +from synapse.util.cancellation import cancellable from tests import unittest from tests.http.server._base import test_disconnect From 42b11d5565ed026c7d71f433c69e7b7007f45918 Mon Sep 17 00:00:00 2001 From: Nick Mills-Barrett Date: Wed, 31 Aug 2022 12:19:39 +0100 Subject: [PATCH 031/107] Remove cached wrap on `_get_joined_users_from_context` method (#13569) The method doesn't actually do any data fetching and the method that does, `_get_joined_profile_from_event_id`, has its own cache. Signed off by Nick @ Beeper (@Fizzadar). --- changelog.d/13569.removal | 1 + synapse/state/__init__.py | 2 +- synapse/storage/databases/main/roommember.py | 124 ++++++------------- 3 files changed, 41 insertions(+), 86 deletions(-) create mode 100644 changelog.d/13569.removal diff --git a/changelog.d/13569.removal b/changelog.d/13569.removal new file mode 100644 index 0000000000..af9d407671 --- /dev/null +++ b/changelog.d/13569.removal @@ -0,0 +1 @@ +Remove redundant `_get_joined_users_from_context` cache. Contributed by Nick @ Beeper (@fizzadar). diff --git a/synapse/state/__init__.py b/synapse/state/__init__.py index 3047e1b1ad..3787d35b24 100644 --- a/synapse/state/__init__.py +++ b/synapse/state/__init__.py @@ -232,7 +232,7 @@ class StateHandler: logger.debug("calling resolve_state_groups from get_current_user_ids_in_room") entry = await self.resolve_state_groups_for_events(room_id, latest_event_ids) state = await entry.get_state(self._state_storage_controller, StateFilter.all()) - return await self.store.get_joined_user_ids_from_state(room_id, state, entry) + return await self.store.get_joined_user_ids_from_state(room_id, state) async def get_hosts_in_room_at_events( self, room_id: str, event_ids: Collection[str] diff --git a/synapse/storage/databases/main/roommember.py b/synapse/storage/databases/main/roommember.py index 06500457bd..4f0adb136a 100644 --- a/synapse/storage/databases/main/roommember.py +++ b/synapse/storage/databases/main/roommember.py @@ -31,7 +31,6 @@ from typing import ( import attr from synapse.api.constants import EventTypes, Membership -from synapse.events import EventBase from synapse.metrics import LaterGauge from synapse.metrics.background_process_metrics import ( run_as_background_process, @@ -883,96 +882,51 @@ class RoomMemberWorkerStore(EventsWorkerStore): return shared_room_ids or frozenset() async def get_joined_user_ids_from_state( - self, room_id: str, state: StateMap[str], state_entry: "_StateCacheEntry" + self, room_id: str, state: StateMap[str] ) -> Set[str]: - state_group: Union[object, int] = state_entry.state_group - if not state_group: - # If state_group is None it means it has yet to be assigned a - # state group, i.e. we need to make sure that calls with a state_group - # of None don't hit previous cached calls with a None state_group. - # To do this we set the state_group to a new object as object() != object() - state_group = object() + """ + For a given set of state IDs, get a set of user IDs in the room. - assert state_group is not None - with Measure(self._clock, "get_joined_users_from_state"): - return await self._get_joined_user_ids_from_context( - room_id, state_group, state, context=state_entry + This method checks the local event cache, before calling + `_get_user_ids_from_membership_event_ids` for any uncached events. + """ + + with Measure(self._clock, "get_joined_user_ids_from_state"): + users_in_room = set() + member_event_ids = [ + e_id for key, e_id in state.items() if key[0] == EventTypes.Member + ] + + # We check if we have any of the member event ids in the event cache + # before we ask the DB + + # We don't update the event cache hit ratio as it completely throws off + # the hit ratio counts. After all, we don't populate the cache if we + # miss it here + event_map = self._get_events_from_local_cache( + member_event_ids, update_metrics=False ) - @cached(num_args=2, iterable=True, max_entries=100000) - async def _get_joined_user_ids_from_context( - self, - room_id: str, - state_group: Union[object, int], - current_state_ids: StateMap[str], - event: Optional[EventBase] = None, - context: Optional["_StateCacheEntry"] = None, - ) -> Set[str]: - # We don't use `state_group`, it's there so that we can cache based - # on it. However, it's important that it's never None, since two current_states - # with a state_group of None are likely to be different. - assert state_group is not None + missing_member_event_ids = [] + for event_id in member_event_ids: + ev_entry = event_map.get(event_id) + if ev_entry and not ev_entry.event.rejected_reason: + if ev_entry.event.membership == Membership.JOIN: + users_in_room.add(ev_entry.event.state_key) + else: + missing_member_event_ids.append(event_id) - users_in_room = set() - member_event_ids = [ - e_id - for key, e_id in current_state_ids.items() - if key[0] == EventTypes.Member - ] - - if context is not None: - # If we have a context with a delta from a previous state group, - # check if we also have the result from the previous group in cache. - # If we do then we can reuse that result and simply update it with - # any membership changes in `delta_ids` - if context.prev_group and context.delta_ids: - prev_res = self._get_joined_user_ids_from_context.cache.get_immediate( - (room_id, context.prev_group), None + if missing_member_event_ids: + event_to_memberships = ( + await self._get_user_ids_from_membership_event_ids( + missing_member_event_ids + ) + ) + users_in_room.update( + user_id for user_id in event_to_memberships.values() if user_id ) - if prev_res and isinstance(prev_res, set): - users_in_room = prev_res - member_event_ids = [ - e_id - for key, e_id in context.delta_ids.items() - if key[0] == EventTypes.Member - ] - for etype, state_key in context.delta_ids: - if etype == EventTypes.Member: - users_in_room.discard(state_key) - # We check if we have any of the member event ids in the event cache - # before we ask the DB - - # We don't update the event cache hit ratio as it completely throws off - # the hit ratio counts. After all, we don't populate the cache if we - # miss it here - event_map = self._get_events_from_local_cache( - member_event_ids, update_metrics=False - ) - - missing_member_event_ids = [] - for event_id in member_event_ids: - ev_entry = event_map.get(event_id) - if ev_entry and not ev_entry.event.rejected_reason: - if ev_entry.event.membership == Membership.JOIN: - users_in_room.add(ev_entry.event.state_key) - else: - missing_member_event_ids.append(event_id) - - if missing_member_event_ids: - event_to_memberships = await self._get_user_ids_from_membership_event_ids( - missing_member_event_ids - ) - users_in_room.update( - user_id for user_id in event_to_memberships.values() if user_id - ) - - if event is not None and event.type == EventTypes.Member: - if event.membership == Membership.JOIN: - if event.event_id in member_event_ids: - users_in_room.add(event.state_key) - - return users_in_room + return users_in_room @cached( max_entries=10000, @@ -1205,7 +1159,7 @@ class RoomMemberWorkerStore(EventsWorkerStore): # The cache doesn't match the state group or prev state group, # so we calculate the result from first principles. joined_user_ids = await self.get_joined_user_ids_from_state( - room_id, state, state_entry + room_id, state ) cache.hosts_to_joined_users = {} From d1fb46fbc987fc0f2672780e373db83c7dacb6cf Mon Sep 17 00:00:00 2001 From: Sean Quah Date: Wed, 31 Aug 2022 12:19:40 +0100 Subject: [PATCH 032/107] Improve clarity on deprecation of TCP replication Borrows some text from https://github.com/matrix-org/synapse/pull/13647 for the changelog. --- CHANGES.md | 10 +++++++++- docs/usage/configuration/config_documentation.md | 2 +- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index f502866f76..712d3b134a 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -5,7 +5,15 @@ No significant changes since 1.66.0rc2. Deployments with multiple workers should note that the direct TCP replication configuration was deprecated in Synapse v1.18.0 and will be removed in Synapse -v1.67.0. See the [worker documentation](https://matrix-org.github.io/synapse/v1.66/workers.html) +v1.67.0. In particular, the TCP `replication` listener and the +`worker_replication_port` config option are deprecated. + +To migrate to Redis, add the [`redis` config](https://matrix-org.github.io/synapse/v1.66/workers.html#shared-configuration) +and remove the TCP `replication` listener from config of the master and +`worker_replication_port` from worker config. Note that a HTTP listener with a +`replication` resource is still required. + +See the [worker documentation](https://matrix-org.github.io/synapse/v1.66/workers.html) for more details. diff --git a/docs/usage/configuration/config_documentation.md b/docs/usage/configuration/config_documentation.md index 8ae018e628..5dee38d28d 100644 --- a/docs/usage/configuration/config_documentation.md +++ b/docs/usage/configuration/config_documentation.md @@ -431,7 +431,7 @@ Sub-options for each listener include: * `metrics`: (see the docs [here](../../metrics-howto.md)), - * `replication`: (see the docs [here](../../workers.md)). + * `replication`: (deprecated as of Synapse 1.18, see the docs [here](../../workers.md)). * `tls`: set to true to enable TLS for this listener. Will use the TLS key/cert specified in tls_private_key_path / tls_certificate_path. From c01f21d31da83e3ad7844d40b293df941f304663 Mon Sep 17 00:00:00 2001 From: Sean Quah Date: Wed, 31 Aug 2022 12:35:25 +0100 Subject: [PATCH 033/107] Tweak changelog wording --- CHANGES.md | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 712d3b134a..7e9862f61d 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -5,11 +5,12 @@ No significant changes since 1.66.0rc2. Deployments with multiple workers should note that the direct TCP replication configuration was deprecated in Synapse v1.18.0 and will be removed in Synapse -v1.67.0. In particular, the TCP `replication` listener and the -`worker_replication_port` config option are deprecated. +v1.67.0. In particular, the TCP `replication` [listener](https://matrix-org.github.io/synapse/v1.66/usage/configuration/config_documentation.html#listeners) +type (not to be confused with the `replication` resource on the `http` listener +type) and the `worker_replication_port` config option will be removed . -To migrate to Redis, add the [`redis` config](https://matrix-org.github.io/synapse/v1.66/workers.html#shared-configuration) -and remove the TCP `replication` listener from config of the master and +To migrate to Redis, add the [`redis` config](https://matrix-org.github.io/synapse/v1.66/workers.html#shared-configuration), +then remove the TCP `replication` listener from config of the master and `worker_replication_port` from worker config. Note that a HTTP listener with a `replication` resource is still required. From 838d722eba6c3f9c9e3e6433b3771c994f75a2ec Mon Sep 17 00:00:00 2001 From: Sean Quah Date: Wed, 31 Aug 2022 12:40:14 +0100 Subject: [PATCH 034/107] Move notice from 1.66.0rc1 to 1.66.0 section in changelog --- CHANGES.md | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 7e9862f61d..535fd36a0e 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -3,6 +3,12 @@ Synapse 1.66.0 (2022-08-31) No significant changes since 1.66.0rc2. +This release removes the ability for homeservers to delegate email ownership +verification and password reset confirmation to identity servers. This removal +was originally planned for Synapse 1.64, but was later deferred until now. + +See the [upgrade notes](https://matrix-org.github.io/synapse/v1.66/upgrade.html#upgrading-to-v1660) for more details. + Deployments with multiple workers should note that the direct TCP replication configuration was deprecated in Synapse v1.18.0 and will be removed in Synapse v1.67.0. In particular, the TCP `replication` [listener](https://matrix-org.github.io/synapse/v1.66/usage/configuration/config_documentation.html#listeners) @@ -30,12 +36,6 @@ Bugfixes Synapse 1.66.0rc1 (2022-08-23) ============================== -This release removes the ability for homeservers to delegate email ownership -verification and password reset confirmation to identity servers. This removal -was originally planned for Synapse 1.64, but was later deferred until now. - -See the [upgrade notes](https://matrix-org.github.io/synapse/v1.66/upgrade.html#upgrading-to-v1660) for more details. - Features -------- From 6f80fe1e1bbb6cab3ce605b2023e0488e2d80d52 Mon Sep 17 00:00:00 2001 From: Sean Quah Date: Wed, 31 Aug 2022 12:51:57 +0100 Subject: [PATCH 035/107] Tweak changelog formatting --- CHANGES.md | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 535fd36a0e..0b10e90186 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -5,9 +5,8 @@ No significant changes since 1.66.0rc2. This release removes the ability for homeservers to delegate email ownership verification and password reset confirmation to identity servers. This removal -was originally planned for Synapse 1.64, but was later deferred until now. - -See the [upgrade notes](https://matrix-org.github.io/synapse/v1.66/upgrade.html#upgrading-to-v1660) for more details. +was originally planned for Synapse 1.64, but was later deferred until now. See +the [upgrade notes](https://matrix-org.github.io/synapse/v1.66/upgrade.html#upgrading-to-v1660) for more details. Deployments with multiple workers should note that the direct TCP replication configuration was deprecated in Synapse v1.18.0 and will be removed in Synapse @@ -18,9 +17,8 @@ type) and the `worker_replication_port` config option will be removed . To migrate to Redis, add the [`redis` config](https://matrix-org.github.io/synapse/v1.66/workers.html#shared-configuration), then remove the TCP `replication` listener from config of the master and `worker_replication_port` from worker config. Note that a HTTP listener with a -`replication` resource is still required. - -See the [worker documentation](https://matrix-org.github.io/synapse/v1.66/workers.html) +`replication` resource is still required. See the +[worker documentation](https://matrix-org.github.io/synapse/v1.66/workers.html) for more details. From 84ddcd7bbfe4100101741a408a91f283a8f742c7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jacek=20Ku=C5=9Bnierz?= Date: Wed, 31 Aug 2022 14:10:25 +0200 Subject: [PATCH 036/107] Drop support for calling `/_matrix/client/v3/rooms/{roomId}/invite` without an `id_access_token` (#13241) Fixes #13206 Signed-off-by: Jacek Kusnierz jacek.kusnierz@tum.de --- changelog.d/13241.removal | 1 + synapse/handlers/identity.py | 142 ++++------------------ synapse/handlers/room.py | 20 ++- synapse/handlers/room_member.py | 6 +- synapse/rest/client/room.py | 20 +-- synapse/rest/media/v1/media_repository.py | 1 - tests/rest/client/test_identity.py | 3 +- tests/rest/client/test_rooms.py | 18 +++ tests/rest/client/test_shadow_banned.py | 7 +- 9 files changed, 81 insertions(+), 137 deletions(-) create mode 100644 changelog.d/13241.removal diff --git a/changelog.d/13241.removal b/changelog.d/13241.removal new file mode 100644 index 0000000000..60b0e7969c --- /dev/null +++ b/changelog.d/13241.removal @@ -0,0 +1 @@ +Drop support for calling `/_matrix/client/v3/rooms/{roomId}/invite` without an `id_access_token`, which was not permitted by the spec. Contributed by @Vetchu. \ No newline at end of file diff --git a/synapse/handlers/identity.py b/synapse/handlers/identity.py index 9571d461c8..93d09e9939 100644 --- a/synapse/handlers/identity.py +++ b/synapse/handlers/identity.py @@ -538,11 +538,7 @@ class IdentityHandler: raise SynapseError(400, "Error contacting the identity server") async def lookup_3pid( - self, - id_server: str, - medium: str, - address: str, - id_access_token: Optional[str] = None, + self, id_server: str, medium: str, address: str, id_access_token: str ) -> Optional[str]: """Looks up a 3pid in the passed identity server. @@ -557,60 +553,15 @@ class IdentityHandler: Returns: the matrix ID of the 3pid, or None if it is not recognized. """ - if id_access_token is not None: - try: - results = await self._lookup_3pid_v2( - id_server, id_access_token, medium, address - ) - return results - except Exception as e: - # Catch HttpResponseExcept for a non-200 response code - # Check if this identity server does not know about v2 lookups - if isinstance(e, HttpResponseException) and e.code == 404: - # This is an old identity server that does not yet support v2 lookups - logger.warning( - "Attempted v2 lookup on v1 identity server %s. Falling " - "back to v1", - id_server, - ) - else: - logger.warning("Error when looking up hashing details: %s", e) - return None - - return await self._lookup_3pid_v1(id_server, medium, address) - - async def _lookup_3pid_v1( - self, id_server: str, medium: str, address: str - ) -> Optional[str]: - """Looks up a 3pid in the passed identity server using v1 lookup. - - Args: - id_server: The server name (including port, if required) - of the identity server to use. - medium: The type of the third party identifier (e.g. "email"). - address: The third party identifier (e.g. "foo@example.com"). - - Returns: - the matrix ID of the 3pid, or None if it is not recognized. - """ try: - data = await self.blacklisting_http_client.get_json( - "%s%s/_matrix/identity/api/v1/lookup" % (id_server_scheme, id_server), - {"medium": medium, "address": address}, + results = await self._lookup_3pid_v2( + id_server, id_access_token, medium, address ) - - if "mxid" in data: - # note: we used to verify the identity server's signature here, but no longer - # require or validate it. See the following for context: - # https://github.com/matrix-org/synapse/issues/5253#issuecomment-666246950 - return data["mxid"] - except RequestTimedOutError: - raise SynapseError(500, "Timed out contacting identity server") - except OSError as e: - logger.warning("Error from v1 identity server lookup: %s" % (e,)) - - return None + return results + except Exception as e: + logger.warning("Error when looking up hashing details: %s", e) + return None async def _lookup_3pid_v2( self, id_server: str, id_access_token: str, medium: str, address: str @@ -739,7 +690,7 @@ class IdentityHandler: room_type: Optional[str], inviter_display_name: str, inviter_avatar_url: str, - id_access_token: Optional[str] = None, + id_access_token: str, ) -> Tuple[str, List[Dict[str, str]], Dict[str, str], str]: """ Asks an identity server for a third party invite. @@ -760,7 +711,7 @@ class IdentityHandler: inviter_display_name: The current display name of the inviter. inviter_avatar_url: The URL of the inviter's avatar. - id_access_token (str|None): The access token to authenticate to the identity + id_access_token (str): The access token to authenticate to the identity server with Returns: @@ -792,71 +743,24 @@ class IdentityHandler: invite_config["org.matrix.web_client_location"] = self._web_client_location # Add the identity service access token to the JSON body and use the v2 - # Identity Service endpoints if id_access_token is present + # Identity Service endpoints data = None - base_url = "%s%s/_matrix/identity" % (id_server_scheme, id_server) - if id_access_token: - key_validity_url = "%s%s/_matrix/identity/v2/pubkey/isvalid" % ( - id_server_scheme, - id_server, + key_validity_url = "%s%s/_matrix/identity/v2/pubkey/isvalid" % ( + id_server_scheme, + id_server, + ) + + url = "%s%s/_matrix/identity/v2/store-invite" % (id_server_scheme, id_server) + try: + data = await self.blacklisting_http_client.post_json_get_json( + url, + invite_config, + {"Authorization": create_id_access_token_header(id_access_token)}, ) + except RequestTimedOutError: + raise SynapseError(500, "Timed out contacting identity server") - # Attempt a v2 lookup - url = base_url + "/v2/store-invite" - try: - data = await self.blacklisting_http_client.post_json_get_json( - url, - invite_config, - {"Authorization": create_id_access_token_header(id_access_token)}, - ) - except RequestTimedOutError: - raise SynapseError(500, "Timed out contacting identity server") - except HttpResponseException as e: - if e.code != 404: - logger.info("Failed to POST %s with JSON: %s", url, e) - raise e - - if data is None: - key_validity_url = "%s%s/_matrix/identity/api/v1/pubkey/isvalid" % ( - id_server_scheme, - id_server, - ) - url = base_url + "/api/v1/store-invite" - - try: - data = await self.blacklisting_http_client.post_json_get_json( - url, invite_config - ) - except RequestTimedOutError: - raise SynapseError(500, "Timed out contacting identity server") - except HttpResponseException as e: - logger.warning( - "Error trying to call /store-invite on %s%s: %s", - id_server_scheme, - id_server, - e, - ) - - if data is None: - # Some identity servers may only support application/x-www-form-urlencoded - # types. This is especially true with old instances of Sydent, see - # https://github.com/matrix-org/sydent/pull/170 - try: - data = await self.blacklisting_http_client.post_urlencoded_get_json( - url, invite_config - ) - except HttpResponseException as e: - logger.warning( - "Error calling /store-invite on %s%s with fallback " - "encoding: %s", - id_server_scheme, - id_server, - e, - ) - raise e - - # TODO: Check for success token = data["token"] public_keys = data.get("public_keys", []) if "public_key" in data: diff --git a/synapse/handlers/room.py b/synapse/handlers/room.py index f64a8690a5..33e9a87002 100644 --- a/synapse/handlers/room.py +++ b/synapse/handlers/room.py @@ -19,6 +19,7 @@ import math import random import string from collections import OrderedDict +from http import HTTPStatus from typing import ( TYPE_CHECKING, Any, @@ -704,8 +705,8 @@ class RoomCreationHandler: was, requested, `room_alias`. Secondly, the stream_id of the last persisted event. Raises: - SynapseError if the room ID couldn't be stored, or something went - horribly wrong. + SynapseError if the room ID couldn't be stored, 3pid invitation config + validation failed, or something went horribly wrong. ResourceLimitError if server is blocked to some resource being exceeded """ @@ -731,6 +732,19 @@ class RoomCreationHandler: invite_3pid_list = config.get("invite_3pid", []) invite_list = config.get("invite", []) + # validate each entry for correctness + for invite_3pid in invite_3pid_list: + if not all( + key in invite_3pid + for key in ("medium", "address", "id_server", "id_access_token") + ): + raise SynapseError( + HTTPStatus.BAD_REQUEST, + "all of `medium`, `address`, `id_server` and `id_access_token` " + "are required when making a 3pid invite", + Codes.MISSING_PARAM, + ) + if not is_requester_admin: spam_check = await self.spam_checker.user_may_create_room(user_id) if spam_check != NOT_SPAM: @@ -978,7 +992,7 @@ class RoomCreationHandler: for invite_3pid in invite_3pid_list: id_server = invite_3pid["id_server"] - id_access_token = invite_3pid.get("id_access_token") # optional + id_access_token = invite_3pid["id_access_token"] address = invite_3pid["address"] medium = invite_3pid["medium"] # Note that do_3pid_invite can raise a ShadowBanError, but this was diff --git a/synapse/handlers/room_member.py b/synapse/handlers/room_member.py index e726997d83..5d4adf5bfd 100644 --- a/synapse/handlers/room_member.py +++ b/synapse/handlers/room_member.py @@ -1382,7 +1382,7 @@ class RoomMemberHandler(metaclass=abc.ABCMeta): id_server: str, requester: Requester, txn_id: Optional[str], - id_access_token: Optional[str] = None, + id_access_token: str, prev_event_ids: Optional[List[str]] = None, depth: Optional[int] = None, ) -> Tuple[str, int]: @@ -1397,7 +1397,7 @@ class RoomMemberHandler(metaclass=abc.ABCMeta): requester: The user making the request. txn_id: The transaction ID this is part of, or None if this is not part of a transaction. - id_access_token: The optional identity server access token. + id_access_token: Identity server access token. depth: Override the depth used to order the event in the DAG. prev_event_ids: The event IDs to use as the prev events Should normally be set to None, which will cause the depth to be calculated @@ -1494,7 +1494,7 @@ class RoomMemberHandler(metaclass=abc.ABCMeta): room_id: str, user: UserID, txn_id: Optional[str], - id_access_token: Optional[str] = None, + id_access_token: str, prev_event_ids: Optional[List[str]] = None, depth: Optional[int] = None, ) -> Tuple[EventBase, int]: diff --git a/synapse/rest/client/room.py b/synapse/rest/client/room.py index 0e2834008e..0bca012535 100644 --- a/synapse/rest/client/room.py +++ b/synapse/rest/client/room.py @@ -17,6 +17,7 @@ import logging import re from enum import Enum +from http import HTTPStatus from typing import TYPE_CHECKING, Awaitable, Dict, List, Optional, Tuple from urllib import parse as urlparse @@ -947,7 +948,16 @@ class RoomMembershipRestServlet(TransactionRestServlet): # cheekily send invalid bodies. content = {} - if membership_action == "invite" and self._has_3pid_invite_keys(content): + if membership_action == "invite" and all( + key in content for key in ("medium", "address") + ): + if not all(key in content for key in ("id_server", "id_access_token")): + raise SynapseError( + HTTPStatus.BAD_REQUEST, + "`id_server` and `id_access_token` are required when doing 3pid invite", + Codes.MISSING_PARAM, + ) + try: await self.room_member_handler.do_3pid_invite( room_id, @@ -957,7 +967,7 @@ class RoomMembershipRestServlet(TransactionRestServlet): content["id_server"], requester, txn_id, - content.get("id_access_token"), + content["id_access_token"], ) except ShadowBanError: # Pretend the request succeeded. @@ -994,12 +1004,6 @@ class RoomMembershipRestServlet(TransactionRestServlet): return 200, return_value - def _has_3pid_invite_keys(self, content: JsonDict) -> bool: - for key in {"id_server", "medium", "address"}: - if key not in content: - return False - return True - def on_PUT( self, request: SynapseRequest, room_id: str, membership_action: str, txn_id: str ) -> Awaitable[Tuple[int, JsonDict]]: diff --git a/synapse/rest/media/v1/media_repository.py b/synapse/rest/media/v1/media_repository.py index 7435fd9130..9dd3c8d4bb 100644 --- a/synapse/rest/media/v1/media_repository.py +++ b/synapse/rest/media/v1/media_repository.py @@ -64,7 +64,6 @@ if TYPE_CHECKING: logger = logging.getLogger(__name__) - # How often to run the background job to update the "recently accessed" # attribute of local and remote media. UPDATE_RECENTLY_ACCESSED_TS = 60 * 1000 # 1 minute diff --git a/tests/rest/client/test_identity.py b/tests/rest/client/test_identity.py index dc17c9d113..b0c8215744 100644 --- a/tests/rest/client/test_identity.py +++ b/tests/rest/client/test_identity.py @@ -25,7 +25,6 @@ from tests import unittest class IdentityTestCase(unittest.HomeserverTestCase): - servlets = [ synapse.rest.admin.register_servlets_for_client_rest_resource, room.register_servlets, @@ -33,7 +32,6 @@ class IdentityTestCase(unittest.HomeserverTestCase): ] def make_homeserver(self, reactor: MemoryReactor, clock: Clock) -> HomeServer: - config = self.default_config() config["enable_3pid_lookup"] = False self.hs = self.setup_test_homeserver(config=config) @@ -54,6 +52,7 @@ class IdentityTestCase(unittest.HomeserverTestCase): "id_server": "testis", "medium": "email", "address": "test@example.com", + "id_access_token": tok, } request_url = ("/rooms/%s/invite" % (room_id)).encode("ascii") channel = self.make_request( diff --git a/tests/rest/client/test_rooms.py b/tests/rest/client/test_rooms.py index aa2f578441..c7eb88d33f 100644 --- a/tests/rest/client/test_rooms.py +++ b/tests/rest/client/test_rooms.py @@ -3461,3 +3461,21 @@ class ThreepidInviteTestCase(unittest.HomeserverTestCase): # Also check that it stopped before calling _make_and_store_3pid_invite. make_invite_mock.assert_called_once() + + def test_400_missing_param_without_id_access_token(self) -> None: + """ + Test that a 3pid invite request returns 400 M_MISSING_PARAM + if we do not include id_access_token. + """ + channel = self.make_request( + method="POST", + path="/rooms/" + self.room_id + "/invite", + content={ + "id_server": "example.com", + "medium": "email", + "address": "teresa@example.com", + }, + access_token=self.tok, + ) + self.assertEqual(channel.code, 400) + self.assertEqual(channel.json_body["errcode"], "M_MISSING_PARAM") diff --git a/tests/rest/client/test_shadow_banned.py b/tests/rest/client/test_shadow_banned.py index c50f034b34..c807a37bc2 100644 --- a/tests/rest/client/test_shadow_banned.py +++ b/tests/rest/client/test_shadow_banned.py @@ -97,7 +97,12 @@ class RoomTestCase(_ShadowBannedBase): channel = self.make_request( "POST", "/rooms/%s/invite" % (room_id,), - {"id_server": "test", "medium": "email", "address": "test@test.test"}, + { + "id_server": "test", + "medium": "email", + "address": "test@test.test", + "id_access_token": "anytoken", + }, access_token=self.banned_access_token, ) self.assertEqual(200, channel.code, channel.result) From 0e99f07952edcb6396654e34da50ddeb0a211067 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Thu, 1 Sep 2022 14:31:54 +0200 Subject: [PATCH 037/107] Remove support for unstable private read receipts (#13653) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- changelog.d/13653.removal | 1 + synapse/api/constants.py | 1 - synapse/config/experimental.py | 3 -- synapse/handlers/receipts.py | 29 +++-------- synapse/replication/tcp/client.py | 5 +- synapse/rest/client/notifications.py | 1 - synapse/rest/client/read_marker.py | 2 - synapse/rest/client/receipts.py | 2 - synapse/rest/client/versions.py | 1 - .../databases/main/event_push_actions.py | 2 - tests/handlers/test_receipts.py | 48 +++++-------------- tests/rest/client/test_sync.py | 37 +++++--------- tests/storage/test_receipts.py | 34 +++++-------- 13 files changed, 44 insertions(+), 122 deletions(-) create mode 100644 changelog.d/13653.removal diff --git a/changelog.d/13653.removal b/changelog.d/13653.removal new file mode 100644 index 0000000000..eb075d4517 --- /dev/null +++ b/changelog.d/13653.removal @@ -0,0 +1 @@ +Remove support for unstable [private read receipts](https://github.com/matrix-org/matrix-spec-proposals/pull/2285). diff --git a/synapse/api/constants.py b/synapse/api/constants.py index c73aea622a..c178ddf070 100644 --- a/synapse/api/constants.py +++ b/synapse/api/constants.py @@ -258,7 +258,6 @@ class GuestAccess: class ReceiptTypes: READ: Final = "m.read" READ_PRIVATE: Final = "m.read.private" - UNSTABLE_READ_PRIVATE: Final = "org.matrix.msc2285.read.private" FULLY_READ: Final = "m.fully_read" diff --git a/synapse/config/experimental.py b/synapse/config/experimental.py index c1ff417539..260db49cad 100644 --- a/synapse/config/experimental.py +++ b/synapse/config/experimental.py @@ -32,9 +32,6 @@ class ExperimentalConfig(Config): # MSC2716 (importing historical messages) self.msc2716_enabled: bool = experimental.get("msc2716_enabled", False) - # MSC2285 (unstable private read receipts) - self.msc2285_enabled: bool = experimental.get("msc2285_enabled", False) - # MSC3244 (room version capabilities) self.msc3244_enabled: bool = experimental.get("msc3244_enabled", True) diff --git a/synapse/handlers/receipts.py b/synapse/handlers/receipts.py index d4a866b346..d2bdb9c8be 100644 --- a/synapse/handlers/receipts.py +++ b/synapse/handlers/receipts.py @@ -163,10 +163,7 @@ class ReceiptsHandler: if not is_new: return - if self.federation_sender and receipt_type not in ( - ReceiptTypes.READ_PRIVATE, - ReceiptTypes.UNSTABLE_READ_PRIVATE, - ): + if self.federation_sender and receipt_type != ReceiptTypes.READ_PRIVATE: await self.federation_sender.send_read_receipt(receipt) @@ -206,38 +203,24 @@ class ReceiptEventSource(EventSource[int, JsonDict]): for event_id, orig_event_content in room.get("content", {}).items(): event_content = orig_event_content # If there are private read receipts, additional logic is necessary. - if ( - ReceiptTypes.READ_PRIVATE in event_content - or ReceiptTypes.UNSTABLE_READ_PRIVATE in event_content - ): + if ReceiptTypes.READ_PRIVATE in event_content: # Make a copy without private read receipts to avoid leaking # other user's private read receipts.. event_content = { receipt_type: receipt_value for receipt_type, receipt_value in event_content.items() - if receipt_type - not in ( - ReceiptTypes.READ_PRIVATE, - ReceiptTypes.UNSTABLE_READ_PRIVATE, - ) + if receipt_type != ReceiptTypes.READ_PRIVATE } # Copy the current user's private read receipt from the # original content, if it exists. - user_private_read_receipt = orig_event_content.get( - ReceiptTypes.READ_PRIVATE, {} - ).get(user_id, None) + user_private_read_receipt = orig_event_content[ + ReceiptTypes.READ_PRIVATE + ].get(user_id, None) if user_private_read_receipt: event_content[ReceiptTypes.READ_PRIVATE] = { user_id: user_private_read_receipt } - user_unstable_private_read_receipt = orig_event_content.get( - ReceiptTypes.UNSTABLE_READ_PRIVATE, {} - ).get(user_id, None) - if user_unstable_private_read_receipt: - event_content[ReceiptTypes.UNSTABLE_READ_PRIVATE] = { - user_id: user_unstable_private_read_receipt - } # Include the event if there is at least one non-private read # receipt or the current user has a private read receipt. diff --git a/synapse/replication/tcp/client.py b/synapse/replication/tcp/client.py index 1ed7230e32..e4f2201c92 100644 --- a/synapse/replication/tcp/client.py +++ b/synapse/replication/tcp/client.py @@ -416,10 +416,7 @@ class FederationSenderHandler: if not self._is_mine_id(receipt.user_id): continue # Private read receipts never get sent over federation. - if receipt.receipt_type in ( - ReceiptTypes.READ_PRIVATE, - ReceiptTypes.UNSTABLE_READ_PRIVATE, - ): + if receipt.receipt_type == ReceiptTypes.READ_PRIVATE: continue receipt_info = ReadReceipt( receipt.room_id, diff --git a/synapse/rest/client/notifications.py b/synapse/rest/client/notifications.py index a73322a6a4..61268e3af1 100644 --- a/synapse/rest/client/notifications.py +++ b/synapse/rest/client/notifications.py @@ -62,7 +62,6 @@ class NotificationsServlet(RestServlet): [ ReceiptTypes.READ, ReceiptTypes.READ_PRIVATE, - ReceiptTypes.UNSTABLE_READ_PRIVATE, ], ) diff --git a/synapse/rest/client/read_marker.py b/synapse/rest/client/read_marker.py index aaad8b233f..5e53096539 100644 --- a/synapse/rest/client/read_marker.py +++ b/synapse/rest/client/read_marker.py @@ -45,8 +45,6 @@ class ReadMarkerRestServlet(RestServlet): ReceiptTypes.FULLY_READ, ReceiptTypes.READ_PRIVATE, } - if hs.config.experimental.msc2285_enabled: - self._known_receipt_types.add(ReceiptTypes.UNSTABLE_READ_PRIVATE) async def on_POST( self, request: SynapseRequest, room_id: str diff --git a/synapse/rest/client/receipts.py b/synapse/rest/client/receipts.py index c6108fc5eb..5b7fad7402 100644 --- a/synapse/rest/client/receipts.py +++ b/synapse/rest/client/receipts.py @@ -49,8 +49,6 @@ class ReceiptRestServlet(RestServlet): ReceiptTypes.READ_PRIVATE, ReceiptTypes.FULLY_READ, } - if hs.config.experimental.msc2285_enabled: - self._known_receipt_types.add(ReceiptTypes.UNSTABLE_READ_PRIVATE) async def on_POST( self, request: SynapseRequest, room_id: str, receipt_type: str, event_id: str diff --git a/synapse/rest/client/versions.py b/synapse/rest/client/versions.py index c9a830cbac..c516cda95d 100644 --- a/synapse/rest/client/versions.py +++ b/synapse/rest/client/versions.py @@ -95,7 +95,6 @@ class VersionsRestServlet(RestServlet): "org.matrix.msc3026.busy_presence": self.config.experimental.msc3026_enabled, # Supports receiving private read receipts as per MSC2285 "org.matrix.msc2285.stable": True, # TODO: Remove when MSC2285 becomes a part of the spec - "org.matrix.msc2285": self.config.experimental.msc2285_enabled, # Supports filtering of /publicRooms by room type as per MSC3827 "org.matrix.msc3827.stable": True, # Adds support for importing historical messages as per MSC2716 diff --git a/synapse/storage/databases/main/event_push_actions.py b/synapse/storage/databases/main/event_push_actions.py index 9f410d69de..f4a07de2a3 100644 --- a/synapse/storage/databases/main/event_push_actions.py +++ b/synapse/storage/databases/main/event_push_actions.py @@ -274,7 +274,6 @@ class EventPushActionsWorkerStore(ReceiptsWorkerStore, StreamWorkerStore, SQLBas receipt_types=( ReceiptTypes.READ, ReceiptTypes.READ_PRIVATE, - ReceiptTypes.UNSTABLE_READ_PRIVATE, ), ) @@ -468,7 +467,6 @@ class EventPushActionsWorkerStore(ReceiptsWorkerStore, StreamWorkerStore, SQLBas ( ReceiptTypes.READ, ReceiptTypes.READ_PRIVATE, - ReceiptTypes.UNSTABLE_READ_PRIVATE, ), ) diff --git a/tests/handlers/test_receipts.py b/tests/handlers/test_receipts.py index 5f70a2db79..b55238650c 100644 --- a/tests/handlers/test_receipts.py +++ b/tests/handlers/test_receipts.py @@ -15,8 +15,6 @@ from copy import deepcopy from typing import List -from parameterized import parameterized - from synapse.api.constants import EduTypes, ReceiptTypes from synapse.types import JsonDict @@ -27,16 +25,13 @@ class ReceiptsTestCase(unittest.HomeserverTestCase): def prepare(self, reactor, clock, hs): self.event_source = hs.get_event_sources().sources.receipt - @parameterized.expand( - [ReceiptTypes.READ_PRIVATE, ReceiptTypes.UNSTABLE_READ_PRIVATE] - ) - def test_filters_out_private_receipt(self, receipt_type: str) -> None: + def test_filters_out_private_receipt(self) -> None: self._test_filters_private( [ { "content": { "$1435641916114394fHBLK:matrix.org": { - receipt_type: { + ReceiptTypes.READ_PRIVATE: { "@rikj:jki.re": { "ts": 1436451550453, } @@ -50,18 +45,13 @@ class ReceiptsTestCase(unittest.HomeserverTestCase): [], ) - @parameterized.expand( - [ReceiptTypes.READ_PRIVATE, ReceiptTypes.UNSTABLE_READ_PRIVATE] - ) - def test_filters_out_private_receipt_and_ignores_rest( - self, receipt_type: str - ) -> None: + def test_filters_out_private_receipt_and_ignores_rest(self) -> None: self._test_filters_private( [ { "content": { "$1dgdgrd5641916114394fHBLK:matrix.org": { - receipt_type: { + ReceiptTypes.READ_PRIVATE: { "@rikj:jki.re": { "ts": 1436451550453, }, @@ -94,18 +84,15 @@ class ReceiptsTestCase(unittest.HomeserverTestCase): ], ) - @parameterized.expand( - [ReceiptTypes.READ_PRIVATE, ReceiptTypes.UNSTABLE_READ_PRIVATE] - ) def test_filters_out_event_with_only_private_receipts_and_ignores_the_rest( - self, receipt_type: str + self, ) -> None: self._test_filters_private( [ { "content": { "$14356419edgd14394fHBLK:matrix.org": { - receipt_type: { + ReceiptTypes.READ_PRIVATE: { "@rikj:jki.re": { "ts": 1436451550453, }, @@ -175,18 +162,15 @@ class ReceiptsTestCase(unittest.HomeserverTestCase): ], ) - @parameterized.expand( - [ReceiptTypes.READ_PRIVATE, ReceiptTypes.UNSTABLE_READ_PRIVATE] - ) def test_filters_out_receipt_event_with_only_private_receipt_and_ignores_rest( - self, receipt_type: str + self, ) -> None: self._test_filters_private( [ { "content": { "$14356419edgd14394fHBLK:matrix.org": { - receipt_type: { + ReceiptTypes.READ_PRIVATE: { "@rikj:jki.re": { "ts": 1436451550453, }, @@ -262,16 +246,13 @@ class ReceiptsTestCase(unittest.HomeserverTestCase): ], ) - @parameterized.expand( - [ReceiptTypes.READ_PRIVATE, ReceiptTypes.UNSTABLE_READ_PRIVATE] - ) - def test_leaves_our_private_and_their_public(self, receipt_type: str) -> None: + def test_leaves_our_private_and_their_public(self) -> None: self._test_filters_private( [ { "content": { "$1dgdgrd5641916114394fHBLK:matrix.org": { - receipt_type: { + ReceiptTypes.READ_PRIVATE: { "@me:server.org": { "ts": 1436451550453, }, @@ -296,7 +277,7 @@ class ReceiptsTestCase(unittest.HomeserverTestCase): { "content": { "$1dgdgrd5641916114394fHBLK:matrix.org": { - receipt_type: { + ReceiptTypes.READ_PRIVATE: { "@me:server.org": { "ts": 1436451550453, }, @@ -319,16 +300,13 @@ class ReceiptsTestCase(unittest.HomeserverTestCase): ], ) - @parameterized.expand( - [ReceiptTypes.READ_PRIVATE, ReceiptTypes.UNSTABLE_READ_PRIVATE] - ) - def test_we_do_not_mutate(self, receipt_type: str) -> None: + def test_we_do_not_mutate(self) -> None: """Ensure the input values are not modified.""" events = [ { "content": { "$1435641916114394fHBLK:matrix.org": { - receipt_type: { + ReceiptTypes.READ_PRIVATE: { "@rikj:jki.re": { "ts": 1436451550453, } diff --git a/tests/rest/client/test_sync.py b/tests/rest/client/test_sync.py index de0dec8539..0af643ecd9 100644 --- a/tests/rest/client/test_sync.py +++ b/tests/rest/client/test_sync.py @@ -391,7 +391,6 @@ class ReadReceiptsTestCase(unittest.HomeserverTestCase): def make_homeserver(self, reactor: MemoryReactor, clock: Clock) -> HomeServer: config = self.default_config() - config["experimental_features"] = {"msc2285_enabled": True} return self.setup_test_homeserver(config=config) @@ -413,17 +412,14 @@ class ReadReceiptsTestCase(unittest.HomeserverTestCase): # Join the second user self.helper.join(room=self.room_id, user=self.user2, tok=self.tok2) - @parameterized.expand( - [ReceiptTypes.READ_PRIVATE, ReceiptTypes.UNSTABLE_READ_PRIVATE] - ) - def test_private_read_receipts(self, receipt_type: str) -> None: + def test_private_read_receipts(self) -> None: # Send a message as the first user res = self.helper.send(self.room_id, body="hello", tok=self.tok) # Send a private read receipt to tell the server the first user's message was read channel = self.make_request( "POST", - f"/rooms/{self.room_id}/receipt/{receipt_type}/{res['event_id']}", + f"/rooms/{self.room_id}/receipt/{ReceiptTypes.READ_PRIVATE}/{res['event_id']}", {}, access_token=self.tok2, ) @@ -432,10 +428,7 @@ class ReadReceiptsTestCase(unittest.HomeserverTestCase): # Test that the first user can't see the other user's private read receipt self.assertIsNone(self._get_read_receipt()) - @parameterized.expand( - [ReceiptTypes.READ_PRIVATE, ReceiptTypes.UNSTABLE_READ_PRIVATE] - ) - def test_public_receipt_can_override_private(self, receipt_type: str) -> None: + def test_public_receipt_can_override_private(self) -> None: """ Sending a public read receipt to the same event which has a private read receipt should cause that receipt to become public. @@ -446,7 +439,7 @@ class ReadReceiptsTestCase(unittest.HomeserverTestCase): # Send a private read receipt channel = self.make_request( "POST", - f"/rooms/{self.room_id}/receipt/{receipt_type}/{res['event_id']}", + f"/rooms/{self.room_id}/receipt/{ReceiptTypes.READ_PRIVATE}/{res['event_id']}", {}, access_token=self.tok2, ) @@ -465,10 +458,7 @@ class ReadReceiptsTestCase(unittest.HomeserverTestCase): # Test that we did override the private read receipt self.assertNotEqual(self._get_read_receipt(), None) - @parameterized.expand( - [ReceiptTypes.READ_PRIVATE, ReceiptTypes.UNSTABLE_READ_PRIVATE] - ) - def test_private_receipt_cannot_override_public(self, receipt_type: str) -> None: + def test_private_receipt_cannot_override_public(self) -> None: """ Sending a private read receipt to the same event which has a public read receipt should cause no change. @@ -489,7 +479,7 @@ class ReadReceiptsTestCase(unittest.HomeserverTestCase): # Send a private read receipt channel = self.make_request( "POST", - f"/rooms/{self.room_id}/receipt/{receipt_type}/{res['event_id']}", + f"/rooms/{self.room_id}/receipt/{ReceiptTypes.READ_PRIVATE}/{res['event_id']}", {}, access_token=self.tok2, ) @@ -554,7 +544,6 @@ class UnreadMessagesTestCase(unittest.HomeserverTestCase): config = super().default_config() config["experimental_features"] = { "msc2654_enabled": True, - "msc2285_enabled": True, } return config @@ -601,10 +590,7 @@ class UnreadMessagesTestCase(unittest.HomeserverTestCase): tok=self.tok, ) - @parameterized.expand( - [ReceiptTypes.READ_PRIVATE, ReceiptTypes.UNSTABLE_READ_PRIVATE] - ) - def test_unread_counts(self, receipt_type: str) -> None: + def test_unread_counts(self) -> None: """Tests that /sync returns the right value for the unread count (MSC2654).""" # Check that our own messages don't increase the unread count. @@ -638,7 +624,7 @@ class UnreadMessagesTestCase(unittest.HomeserverTestCase): # Send a read receipt to tell the server we've read the latest event. channel = self.make_request( "POST", - f"/rooms/{self.room_id}/receipt/{receipt_type}/{res['event_id']}", + f"/rooms/{self.room_id}/receipt/{ReceiptTypes.READ_PRIVATE}/{res['event_id']}", {}, access_token=self.tok, ) @@ -726,7 +712,7 @@ class UnreadMessagesTestCase(unittest.HomeserverTestCase): channel = self.make_request( "POST", - f"/rooms/{self.room_id}/receipt/{receipt_type}/{res2['event_id']}", + f"/rooms/{self.room_id}/receipt/{ReceiptTypes.READ_PRIVATE}/{res2['event_id']}", {}, access_token=self.tok, ) @@ -738,7 +724,6 @@ class UnreadMessagesTestCase(unittest.HomeserverTestCase): [ ReceiptTypes.READ, ReceiptTypes.READ_PRIVATE, - ReceiptTypes.UNSTABLE_READ_PRIVATE, ] ) def test_read_receipts_only_go_down(self, receipt_type: str) -> None: @@ -752,7 +737,7 @@ class UnreadMessagesTestCase(unittest.HomeserverTestCase): # Read last event channel = self.make_request( "POST", - f"/rooms/{self.room_id}/receipt/{receipt_type}/{res2['event_id']}", + f"/rooms/{self.room_id}/receipt/{ReceiptTypes.READ_PRIVATE}/{res2['event_id']}", {}, access_token=self.tok, ) @@ -763,7 +748,7 @@ class UnreadMessagesTestCase(unittest.HomeserverTestCase): # read receipt go up to an older event channel = self.make_request( "POST", - f"/rooms/{self.room_id}/receipt/{receipt_type}/{res1['event_id']}", + f"/rooms/{self.room_id}/receipt/{ReceiptTypes.READ_PRIVATE}/{res1['event_id']}", {}, access_token=self.tok, ) diff --git a/tests/storage/test_receipts.py b/tests/storage/test_receipts.py index 191c957fb5..c89bfff241 100644 --- a/tests/storage/test_receipts.py +++ b/tests/storage/test_receipts.py @@ -12,7 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -from parameterized import parameterized from synapse.api.constants import ReceiptTypes from synapse.types import UserID, create_requester @@ -92,7 +91,6 @@ class ReceiptTestCase(HomeserverTestCase): [ ReceiptTypes.READ, ReceiptTypes.READ_PRIVATE, - ReceiptTypes.UNSTABLE_READ_PRIVATE, ], ) ) @@ -104,7 +102,6 @@ class ReceiptTestCase(HomeserverTestCase): [ ReceiptTypes.READ, ReceiptTypes.READ_PRIVATE, - ReceiptTypes.UNSTABLE_READ_PRIVATE, ], ) ) @@ -117,16 +114,12 @@ class ReceiptTestCase(HomeserverTestCase): [ ReceiptTypes.READ, ReceiptTypes.READ_PRIVATE, - ReceiptTypes.UNSTABLE_READ_PRIVATE, ], ) ) self.assertEqual(res, None) - @parameterized.expand( - [ReceiptTypes.READ_PRIVATE, ReceiptTypes.UNSTABLE_READ_PRIVATE] - ) - def test_get_receipts_for_user(self, receipt_type: str) -> None: + def test_get_receipts_for_user(self) -> None: # Send some events into the first room event1_1_id = self.create_and_send_event( self.room_id1, UserID.from_string(OTHER_USER_ID) @@ -144,14 +137,14 @@ class ReceiptTestCase(HomeserverTestCase): # Send private read receipt for the second event self.get_success( self.store.insert_receipt( - self.room_id1, receipt_type, OUR_USER_ID, [event1_2_id], {} + self.room_id1, ReceiptTypes.READ_PRIVATE, OUR_USER_ID, [event1_2_id], {} ) ) # Test we get the latest event when we want both private and public receipts res = self.get_success( self.store.get_receipts_for_user( - OUR_USER_ID, [ReceiptTypes.READ, receipt_type] + OUR_USER_ID, [ReceiptTypes.READ, ReceiptTypes.READ_PRIVATE] ) ) self.assertEqual(res, {self.room_id1: event1_2_id}) @@ -164,7 +157,7 @@ class ReceiptTestCase(HomeserverTestCase): # Test we get the latest event when we want only the public receipt res = self.get_success( - self.store.get_receipts_for_user(OUR_USER_ID, [receipt_type]) + self.store.get_receipts_for_user(OUR_USER_ID, [ReceiptTypes.READ_PRIVATE]) ) self.assertEqual(res, {self.room_id1: event1_2_id}) @@ -187,20 +180,17 @@ class ReceiptTestCase(HomeserverTestCase): # Test new room is reflected in what the method returns self.get_success( self.store.insert_receipt( - self.room_id2, receipt_type, OUR_USER_ID, [event2_1_id], {} + self.room_id2, ReceiptTypes.READ_PRIVATE, OUR_USER_ID, [event2_1_id], {} ) ) res = self.get_success( self.store.get_receipts_for_user( - OUR_USER_ID, [ReceiptTypes.READ, receipt_type] + OUR_USER_ID, [ReceiptTypes.READ, ReceiptTypes.READ_PRIVATE] ) ) self.assertEqual(res, {self.room_id1: event1_2_id, self.room_id2: event2_1_id}) - @parameterized.expand( - [ReceiptTypes.READ_PRIVATE, ReceiptTypes.UNSTABLE_READ_PRIVATE] - ) - def test_get_last_receipt_event_id_for_user(self, receipt_type: str) -> None: + def test_get_last_receipt_event_id_for_user(self) -> None: # Send some events into the first room event1_1_id = self.create_and_send_event( self.room_id1, UserID.from_string(OTHER_USER_ID) @@ -218,7 +208,7 @@ class ReceiptTestCase(HomeserverTestCase): # Send private read receipt for the second event self.get_success( self.store.insert_receipt( - self.room_id1, receipt_type, OUR_USER_ID, [event1_2_id], {} + self.room_id1, ReceiptTypes.READ_PRIVATE, OUR_USER_ID, [event1_2_id], {} ) ) @@ -227,7 +217,7 @@ class ReceiptTestCase(HomeserverTestCase): self.store.get_last_receipt_event_id_for_user( OUR_USER_ID, self.room_id1, - [ReceiptTypes.READ, receipt_type], + [ReceiptTypes.READ, ReceiptTypes.READ_PRIVATE], ) ) self.assertEqual(res, event1_2_id) @@ -243,7 +233,7 @@ class ReceiptTestCase(HomeserverTestCase): # Test we get the latest event when we want only the private receipt res = self.get_success( self.store.get_last_receipt_event_id_for_user( - OUR_USER_ID, self.room_id1, [receipt_type] + OUR_USER_ID, self.room_id1, [ReceiptTypes.READ_PRIVATE] ) ) self.assertEqual(res, event1_2_id) @@ -269,14 +259,14 @@ class ReceiptTestCase(HomeserverTestCase): # Test new room is reflected in what the method returns self.get_success( self.store.insert_receipt( - self.room_id2, receipt_type, OUR_USER_ID, [event2_1_id], {} + self.room_id2, ReceiptTypes.READ_PRIVATE, OUR_USER_ID, [event2_1_id], {} ) ) res = self.get_success( self.store.get_last_receipt_event_id_for_user( OUR_USER_ID, self.room_id2, - [ReceiptTypes.READ, receipt_type], + [ReceiptTypes.READ, ReceiptTypes.READ_PRIVATE], ) ) self.assertEqual(res, event2_1_id) From 18e40928010f990a8a9ce6db50a20b7add65bdbc Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 1 Sep 2022 14:46:47 +0200 Subject: [PATCH 038/107] Bump docker image to use stable poetry version (#13688) --- .github/workflows/tests.yml | 1 - changelog.d/13688.docker | 1 + docker/Dockerfile | 11 +---------- 3 files changed, 2 insertions(+), 11 deletions(-) create mode 100644 changelog.d/13688.docker diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 3e6d718a7c..49a010d9d8 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -283,7 +283,6 @@ jobs: - run: sudo apt-get -qq install xmlsec1 - uses: matrix-org/setup-python-poetry@v1 with: - python-version: ${{ matrix.python-version }} extras: "postgres" - run: .ci/scripts/test_export_data_command.sh diff --git a/changelog.d/13688.docker b/changelog.d/13688.docker new file mode 100644 index 0000000000..8935e2536f --- /dev/null +++ b/changelog.d/13688.docker @@ -0,0 +1 @@ +Update docker image to use a stable version of poetry. diff --git a/docker/Dockerfile b/docker/Dockerfile index fa58ae3acb..b87d263cff 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -46,17 +46,8 @@ RUN \ # We install poetry in its own build stage to avoid its dependencies conflicting with # synapse's dependencies. -# We use a specific commit from poetry's master branch instead of our usual 1.1.14, -# to incorporate fixes to some bugs in `poetry export`. This commit corresponds to -# https://github.com/python-poetry/poetry/pull/5156 and -# https://github.com/python-poetry/poetry/issues/5141 ; -# without it, we generate a requirements.txt with incorrect environment markers, -# which causes necessary packages to be omitted when we `pip install`. -# -# NB: In poetry 1.2 `poetry export` will be moved into a plugin; we'll need to also -# pip install poetry-plugin-export (https://github.com/python-poetry/poetry-plugin-export). RUN --mount=type=cache,target=/root/.cache/pip \ - pip install --user "poetry-core==1.1.0a7" "git+https://github.com/python-poetry/poetry.git@fb13b3a676f476177f7937ffa480ee5cff9a90a5" + pip install --user "poetry==1.2.0" WORKDIR /synapse From e8130f219b8aebb00441e4979d34eb3b5d3d6f03 Mon Sep 17 00:00:00 2001 From: Richard van der Hoff <1389908+richvdh@users.noreply.github.com> Date: Thu, 1 Sep 2022 13:54:02 +0100 Subject: [PATCH 039/107] Return keys for unwhitelisted servers from `/_matrix/key/v2/query` (#13683) --- changelog.d/13683.bugfix | 1 + synapse/rest/key/v2/remote_key_resource.py | 41 +++++++++++----------- 2 files changed, 22 insertions(+), 20 deletions(-) create mode 100644 changelog.d/13683.bugfix diff --git a/changelog.d/13683.bugfix b/changelog.d/13683.bugfix new file mode 100644 index 0000000000..538534fec1 --- /dev/null +++ b/changelog.d/13683.bugfix @@ -0,0 +1 @@ +Fix a long-standing bug which meant that keys for unwhitelisted servers were not returned by `/_matrix/key/v2/query`. diff --git a/synapse/rest/key/v2/remote_key_resource.py b/synapse/rest/key/v2/remote_key_resource.py index f597157581..7f8ad29566 100644 --- a/synapse/rest/key/v2/remote_key_resource.py +++ b/synapse/rest/key/v2/remote_key_resource.py @@ -135,13 +135,6 @@ class RemoteKey(DirectServeJsonResource): store_queries = [] for server_name, key_ids in query.items(): - if ( - self.federation_domain_whitelist is not None - and server_name not in self.federation_domain_whitelist - ): - logger.debug("Federation denied with %s", server_name) - continue - if not key_ids: key_ids = (None,) for key_id in key_ids: @@ -153,21 +146,28 @@ class RemoteKey(DirectServeJsonResource): time_now_ms = self.clock.time_msec() - # Note that the value is unused. + # Map server_name->key_id->int. Note that the value of the init is unused. + # XXX: why don't we just use a set? cache_misses: Dict[str, Dict[str, int]] = {} for (server_name, key_id, _), key_results in cached.items(): results = [(result["ts_added_ms"], result) for result in key_results] - if not results and key_id is not None: - cache_misses.setdefault(server_name, {})[key_id] = 0 + if key_id is None: + # all keys were requested. Just return what we have without worrying + # about validity + for _, result in results: + # Cast to bytes since postgresql returns a memoryview. + json_results.add(bytes(result["key_json"])) continue - if key_id is not None: + miss = False + if not results: + miss = True + else: ts_added_ms, most_recent_result = max(results) ts_valid_until_ms = most_recent_result["ts_valid_until_ms"] req_key = query.get(server_name, {}).get(key_id, {}) req_valid_until = req_key.get("minimum_valid_until_ts") - miss = False if req_valid_until is not None: if ts_valid_until_ms < req_valid_until: logger.debug( @@ -211,19 +211,20 @@ class RemoteKey(DirectServeJsonResource): ts_valid_until_ms, time_now_ms, ) - - if miss: - cache_misses.setdefault(server_name, {})[key_id] = 0 # Cast to bytes since postgresql returns a memoryview. json_results.add(bytes(most_recent_result["key_json"])) - else: - for _, result in results: - # Cast to bytes since postgresql returns a memoryview. - json_results.add(bytes(result["key_json"])) + + if miss and query_remote_on_cache_miss: + # only bother attempting to fetch keys from servers on our whitelist + if ( + self.federation_domain_whitelist is None + or server_name in self.federation_domain_whitelist + ): + cache_misses.setdefault(server_name, {})[key_id] = 0 # If there is a cache miss, request the missing keys, then recurse (and # ensure the result is sent). - if cache_misses and query_remote_on_cache_miss: + if cache_misses: await yieldable_gather_results( lambda t: self.fetcher.get_keys(*t), ( From 2318603772f2e1595f9b39369246d5955a069381 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 1 Sep 2022 14:54:52 +0200 Subject: [PATCH 040/107] Add some logging to help track down #13444 (#13679) --- changelog.d/13679.misc | 1 + synapse/federation/sender/__init__.py | 13 +++++++++++++ 2 files changed, 14 insertions(+) create mode 100644 changelog.d/13679.misc diff --git a/changelog.d/13679.misc b/changelog.d/13679.misc new file mode 100644 index 0000000000..a4fa94da9d --- /dev/null +++ b/changelog.d/13679.misc @@ -0,0 +1 @@ +Add some logging to help track down #13444. diff --git a/synapse/federation/sender/__init__.py b/synapse/federation/sender/__init__.py index 94a65ac65f..8bc60e3e3e 100644 --- a/synapse/federation/sender/__init__.py +++ b/synapse/federation/sender/__init__.py @@ -441,6 +441,19 @@ class FederationSender(AbstractFederationSender): destinations = await self._external_cache.get( "get_joined_hosts", str(sg) ) + if destinations is None: + # Add logging to help track down #13444 + logger.info( + "Unexpectedly did not have cached destinations for %s / %s", + sg, + event.event_id, + ) + else: + # Add logging to help track down #13444 + logger.info( + "Unexpectedly did not have cached prev group for %s", + event.event_id, + ) if destinations is None: try: From dcfb006f8a95366f9e388b313615039462fef0f9 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 1 Sep 2022 16:11:44 +0200 Subject: [PATCH 041/107] Use poetry 1.2.0 rather than prerelease for deb packages (#13695) --- debian/build_virtualenv | 2 +- debian/changelog | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/debian/build_virtualenv b/debian/build_virtualenv index f1ec609163..ed916ac97a 100755 --- a/debian/build_virtualenv +++ b/debian/build_virtualenv @@ -36,7 +36,7 @@ TEMP_VENV="$(mktemp -d)" python3 -m venv "$TEMP_VENV" source "$TEMP_VENV/bin/activate" pip install -U pip -pip install poetry==1.2.0b1 +pip install poetry==1.2.0 poetry export \ --extras all \ --extras test \ diff --git a/debian/changelog b/debian/changelog index 9398504a27..2b7b329b6b 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,9 @@ +matrix-synapse-py3 (1.66.0ubuntu1) UNRELEASED; urgency=medium + + * Use stable poetry 1.2.0 version, rather than a prerelease. + + -- Erik Johnston Thu, 01 Sep 2022 13:48:31 +0100 + matrix-synapse-py3 (1.66.0) stable; urgency=medium * New Synapse release 1.66.0. From c913e440c065adde1e055eb0baef5df40615ae9b Mon Sep 17 00:00:00 2001 From: Will Hunt Date: Thu, 1 Sep 2022 15:48:43 +0100 Subject: [PATCH 042/107] Add monthly active users documentation (#13617) * Add monthly active users documentation * changelog * Tidy up notes * more tidyup * Rewrite #1 * link back to mau docs * fix links * s/appservice|AS/application service * further review * a newline * Remove bit about shadow banned users. I think talking about them is confusing, and the current text doesn't imply they get any special treatment. * Update docs/usage/administration/monthly_active_users.md Co-authored-by: Patrick Cloke * Update docs/usage/administration/monthly_active_users.md Co-authored-by: Patrick Cloke Co-authored-by: Brendan Abolivier Co-authored-by: Patrick Cloke --- changelog.d/13617.doc | 1 + docs/SUMMARY.md | 1 + .../administration/monthly_active_users.md | 84 +++++++++++++++++++ .../configuration/config_documentation.md | 2 + 4 files changed, 88 insertions(+) create mode 100644 changelog.d/13617.doc create mode 100644 docs/usage/administration/monthly_active_users.md diff --git a/changelog.d/13617.doc b/changelog.d/13617.doc new file mode 100644 index 0000000000..5c7db7c3d7 --- /dev/null +++ b/changelog.d/13617.doc @@ -0,0 +1 @@ +Document how ["monthly active users"](https://matrix-org.github.io/synapse/latest/usage/administration/monthly_active_users.html) is calculated and used. diff --git a/docs/SUMMARY.md b/docs/SUMMARY.md index 2d56b084e2..16720bceb5 100644 --- a/docs/SUMMARY.md +++ b/docs/SUMMARY.md @@ -69,6 +69,7 @@ - [Manhole](manhole.md) - [Monitoring](metrics-howto.md) - [Reporting Homeserver Usage Statistics](usage/administration/monitoring/reporting_homeserver_usage_statistics.md) + - [Monthly Active Users](usage/administration/monthly_active_users.md) - [Understanding Synapse Through Grafana Graphs](usage/administration/understanding_synapse_through_grafana_graphs.md) - [Useful SQL for Admins](usage/administration/useful_sql_for_admins.md) - [Database Maintenance Tools](usage/administration/database_maintenance_tools.md) diff --git a/docs/usage/administration/monthly_active_users.md b/docs/usage/administration/monthly_active_users.md new file mode 100644 index 0000000000..d4e9037284 --- /dev/null +++ b/docs/usage/administration/monthly_active_users.md @@ -0,0 +1,84 @@ +# Monthly Active Users + +Synapse can be configured to record the number of monthly active users (also referred to as MAU) on a given homeserver. +For clarity's sake, MAU only tracks local users. + +Please note that the metrics recorded by the [Homeserver Usage Stats](../../usage/administration/monitoring/reporting_homeserver_usage_statistics.md) +are calculated differently. The `monthly_active_users` from the usage stats does not take into account any +of the rules below, and counts any users who have made a request to the homeserver in the last 30 days. + +See the [configuration manual](../../usage/configuration/config_documentation.md#limit_usage_by_mau) for details on how to configure MAU. + +## Calculating active users + +Individual user activity is measured in active days. If a user performs an action, the exact time of that action is then recorded. When +calculating the MAU figure, any users with a recorded action in the last 30 days are considered part of the cohort. Days are measured +as a rolling window from the current system time to 30 days ago. + +So for example, if Synapse were to calculate the active users on the 15th July at 13:25, it would include any activity from 15th June 13:25 onwards. + +A user is **never** considered active if they are either: + - Part of the trial day cohort (described below) + - Owned by an application service. + - Note: This **only** covers users that are part of an application service `namespaces.users` registration. The namespace + must also be marked as `exclusive`. + +Otherwise, any request to Synapse will mark the user as active. Please note that registration will not mark a user as active *unless* +they register with a 3pid that is included in the config field `mau_limits_reserved_threepids`. + +The Prometheus metric for MAU is refreshed every 5 minutes. + +Once an hour, Synapse checks to see if any users are inactive (with only activity timestamps later than 30 days). These users +are removed from the active users cohort. If they then become active, they are immediately restored to the cohort. + +It is important to note that **deactivated** users are not immediately removed from the pool of active users, but as these users won't +perform actions they will eventually be removed from the cohort. + +### Trial days + +If the config option `mau_trial_days` is set, a user must have been active this many days **after** registration to be active. A user is in the +trial period if their registration timestamp (also known as the `creation_ts`) is less than `mau_trial_days` old. + +As an example, if `mau_trial_days` is set to `3` and a user is active **after** 3 days (72 hours from registration time) then they will be counted as active. + +The `mau_appservice_trial_days` config further extends this rule by applying different durations depending on the `appservice_id` of the user. +Users registered by an application service will be recorded with an `appservice_id` matching the `id` key in the registration file for that service. + + +## Limiting usage of the homeserver when the maximum MAU is reached + +If both config options `limit_usage_by_mau` and `max_mau_value` is set, and the current MAU value exceeds the maximum value, the +homeserver will begin to block some actions. + +Individual users matching **any** of the below criteria never have their actions blocked: + - Considered part of the cohort of MAU users. + - Considered part of the trial period. + - Registered as a `support` user. + - Application service users if `track_appservice_user_ips` is NOT set. + +Please not that server admins are **not** exempt from blocking. + +The following actions are blocked when the MAU limit is exceeded: + - Logging in + - Sending events + - Creating rooms + - Syncing + +Registration is also blocked for all new signups *unless* the user is registering with a threepid included in the `mau_limits_reserved_threepids` +config value. + +When a request is blocked, the response will have the `errcode` `M_RESOURCE_LIMIT_EXCEEDED`. + +## Metrics + +Synapse records several different prometheus metrics for MAU. + +`synapse_admin_mau:current` records the current MAU figure for native (non-application-service) users. + +`synapse_admin_mau:max` records the maximum MAU as dictated by the `max_mau_value` config value. + +`synapse_admin_mau_current_mau_by_service` records the current MAU including application service users. The label `app_service` can be used +to filter by a specific service ID. This *also* includes non-application-service users under `app_service=native` . + +`synapse_admin_mau:registered_reserved_users` records the number of users specified in `mau_limits_reserved_threepids` which have +registered accounts on the homeserver. diff --git a/docs/usage/configuration/config_documentation.md b/docs/usage/configuration/config_documentation.md index bc9fc86bc8..88e8e1c66f 100644 --- a/docs/usage/configuration/config_documentation.md +++ b/docs/usage/configuration/config_documentation.md @@ -595,6 +595,8 @@ server owner wants to limit to the number of monthly active users. When enabled reached the server returns a `ResourceLimitError` with error type `Codes.RESOURCE_LIMIT_EXCEEDED`. Defaults to false. If this is enabled, a value for `max_mau_value` must also be set. +See [Monthly Active Users](../administration/monthly_active_users.md) for details on how to configure MAU. + Example configuration: ```yaml limit_usage_by_mau: true From 9d2823ab70e69a90f4a54256bd32cd85eab130d9 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 1 Sep 2022 17:07:01 +0200 Subject: [PATCH 043/107] Cache `is_partial_state_room` (#13693) Fixes #13613. --- changelog.d/13693.misc | 1 + synapse/storage/databases/main/room.py | 11 +++++++---- 2 files changed, 8 insertions(+), 4 deletions(-) create mode 100644 changelog.d/13693.misc diff --git a/changelog.d/13693.misc b/changelog.d/13693.misc new file mode 100644 index 0000000000..31490191c9 --- /dev/null +++ b/changelog.d/13693.misc @@ -0,0 +1 @@ +Add cache to `is_partial_state_room`. diff --git a/synapse/storage/databases/main/room.py b/synapse/storage/databases/main/room.py index 367424b4a8..bef66f1992 100644 --- a/synapse/storage/databases/main/room.py +++ b/synapse/storage/databases/main/room.py @@ -1185,8 +1185,9 @@ class RoomWorkerStore(CacheInvalidationWorkerStore): ) return False - @staticmethod - def _clear_partial_state_room_txn(txn: LoggingTransaction, room_id: str) -> None: + def _clear_partial_state_room_txn( + self, txn: LoggingTransaction, room_id: str + ) -> None: DatabasePool.simple_delete_txn( txn, table="partial_state_rooms_servers", @@ -1197,7 +1198,9 @@ class RoomWorkerStore(CacheInvalidationWorkerStore): table="partial_state_rooms", keyvalues={"room_id": room_id}, ) + self._invalidate_cache_and_stream(txn, self.is_partial_state_room, (room_id,)) + @cached() async def is_partial_state_room(self, room_id: str) -> bool: """Checks if this room has partial state. @@ -1771,9 +1774,8 @@ class RoomStore(RoomBackgroundUpdateStore, RoomWorkerStore): servers, ) - @staticmethod def _store_partial_state_room_txn( - txn: LoggingTransaction, room_id: str, servers: Collection[str] + self, txn: LoggingTransaction, room_id: str, servers: Collection[str] ) -> None: DatabasePool.simple_insert_txn( txn, @@ -1788,6 +1790,7 @@ class RoomStore(RoomBackgroundUpdateStore, RoomWorkerStore): keys=("room_id", "server_name"), values=((room_id, s) for s in servers), ) + self._invalidate_cache_and_stream(txn, self.is_partial_state_room, (room_id,)) async def maybe_store_room_on_outlier_membership( self, room_id: str, room_version: RoomVersion From f48f4dd59eb2d98940ac2000de07f3324cd5b188 Mon Sep 17 00:00:00 2001 From: reivilibre Date: Thu, 1 Sep 2022 15:27:06 +0000 Subject: [PATCH 044/107] Update the Grafana dashboard that is included with Synapse in the `contrib` directory. (#13697) * Add missing graph to contrib * Update with minor but plausible changes, including positioning changes * Newsfile Signed-off-by: Olivier Wilkinson (reivilibre) Signed-off-by: Olivier Wilkinson (reivilibre) --- changelog.d/13697.misc | 1 + contrib/grafana/synapse.json | 147 ++++++++++++++++++++++++++++++----- 2 files changed, 128 insertions(+), 20 deletions(-) create mode 100644 changelog.d/13697.misc diff --git a/changelog.d/13697.misc b/changelog.d/13697.misc new file mode 100644 index 0000000000..b9d2a60961 --- /dev/null +++ b/changelog.d/13697.misc @@ -0,0 +1 @@ +Update the Grafana dashboard that is included with Synapse in the `contrib` directory. \ No newline at end of file diff --git a/contrib/grafana/synapse.json b/contrib/grafana/synapse.json index 0f23fc17ea..248cd6d9ad 100644 --- a/contrib/grafana/synapse.json +++ b/contrib/grafana/synapse.json @@ -3244,6 +3244,104 @@ "yaxis": { "align": false } + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Average number of hosts being rate limited across each worker type.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 53 + }, + "id": 225, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "builder", + "expr": "avg by(job, rate_limiter_name) (synapse_rate_limit_sleep_affected_hosts{instance=\"$instance\", job=~\"$job\", index=~\"$index\"})", + "hide": false, + "legendFormat": "Slept by {{job}}:{{rate_limiter_name}}", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "builder", + "expr": "avg by(job, rate_limiter_name) (synapse_rate_limit_reject_affected_hosts{instance=\"$instance\", job=~\"$job\", index=~\"$index\"})", + "legendFormat": "Rejected by {{job}}:{{rate_limiter_name}}", + "range": true, + "refId": "A" + } + ], + "title": "Hosts being rate limited", + "type": "timeseries" } ], "targets": [ @@ -6404,7 +6502,7 @@ "h": 13, "w": 12, "x": 0, - "y": 10 + "y": 35 }, "hiddenSeries": false, "id": 12, @@ -6502,7 +6600,7 @@ "h": 13, "w": 12, "x": 12, - "y": 10 + "y": 35 }, "hiddenSeries": false, "id": 26, @@ -6601,7 +6699,7 @@ "h": 13, "w": 12, "x": 0, - "y": 23 + "y": 48 }, "hiddenSeries": false, "id": 13, @@ -6705,7 +6803,7 @@ "h": 13, "w": 12, "x": 12, - "y": 23 + "y": 48 }, "hiddenSeries": false, "id": 27, @@ -6803,7 +6901,7 @@ "h": 13, "w": 12, "x": 0, - "y": 36 + "y": 61 }, "hiddenSeries": false, "id": 28, @@ -6900,7 +6998,7 @@ "h": 13, "w": 12, "x": 12, - "y": 36 + "y": 61 }, "hiddenSeries": false, "id": 25, @@ -6935,7 +7033,7 @@ "datasource": { "uid": "$datasource" }, - "expr": "rate(synapse_util_metrics_block_time_seconds{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size]) / rate(synapse_util_metrics_block_count[$bucket_size])", + "expr": "rate(synapse_util_metrics_block_time_seconds{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size]) / rate(synapse_util_metrics_block_count{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])", "format": "time_series", "interval": "", "intervalFactor": 2, @@ -6960,11 +7058,13 @@ }, "yaxes": [ { - "format": "ms", + "$$hashKey": "object:180", + "format": "s", "logBase": 1, "show": true }, { + "$$hashKey": "object:181", "format": "short", "logBase": 1, "show": true @@ -6988,7 +7088,7 @@ "h": 15, "w": 12, "x": 0, - "y": 49 + "y": 74 }, "hiddenSeries": false, "id": 154, @@ -7009,7 +7109,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.4.3", + "pluginVersion": "9.0.4", "pointradius": 2, "points": false, "renderer": "flot", @@ -7109,7 +7209,7 @@ "h": 10, "w": 12, "x": 0, - "y": 36 + "y": 69 }, "hiddenSeries": false, "id": 1, @@ -7211,7 +7311,7 @@ "h": 10, "w": 12, "x": 12, - "y": 36 + "y": 69 }, "hiddenSeries": false, "id": 8, @@ -7311,7 +7411,7 @@ "h": 10, "w": 12, "x": 0, - "y": 46 + "y": 79 }, "hiddenSeries": false, "id": 38, @@ -7407,7 +7507,7 @@ "h": 10, "w": 12, "x": 12, - "y": 46 + "y": 79 }, "hiddenSeries": false, "id": 39, @@ -7415,11 +7515,16 @@ "alignAsTable": true, "avg": false, "current": false, - "max": false, + "hideEmpty": false, + "hideZero": false, + "max": true, "min": false, + "rightSide": false, "show": true, + "sort": "max", + "sortDesc": true, "total": false, - "values": false + "values": true }, "lines": true, "linewidth": 1, @@ -7467,11 +7572,13 @@ }, "yaxes": [ { + "$$hashKey": "object:101", "format": "rps", "logBase": 1, "show": true }, { + "$$hashKey": "object:102", "format": "short", "logBase": 1, "show": true @@ -7501,7 +7608,7 @@ "h": 9, "w": 12, "x": 0, - "y": 56 + "y": 89 }, "hiddenSeries": false, "id": 65, @@ -11757,8 +11864,8 @@ ] }, "time": { - "from": "2022-07-22T04:08:13.716Z", - "to": "2022-07-22T18:44:27.863Z" + "from": "now-3h", + "to": "now" }, "timepicker": { "now": true, @@ -11789,6 +11896,6 @@ "timezone": "", "title": "Synapse", "uid": "000000012", - "version": 124, + "version": 132, "weekStart": "" } \ No newline at end of file From 390b7ce946173b41a61f427ef25a9a1d0371ad0b Mon Sep 17 00:00:00 2001 From: Patrick Cloke Date: Thu, 1 Sep 2022 12:52:03 -0400 Subject: [PATCH 045/107] Disable calculating unread counts unless the config flag is enabled. (#13694) This avoids doing work that will never be used (since the resulting unread counts will never be sent in a /sync response). The negative of doing this is that unread counts will be incorrect when the feature is initially enabled. --- changelog.d/13694.bugfix | 1 + synapse/config/experimental.py | 3 ++ synapse/push/bulk_push_rule_evaluator.py | 7 +++- tests/storage/test_event_push_actions.py | 42 +++++++++++------------- 4 files changed, 30 insertions(+), 23 deletions(-) create mode 100644 changelog.d/13694.bugfix diff --git a/changelog.d/13694.bugfix b/changelog.d/13694.bugfix new file mode 100644 index 0000000000..48b9bb5f0a --- /dev/null +++ b/changelog.d/13694.bugfix @@ -0,0 +1 @@ +Fix a bug introduced in Synapse v1.20.0 that would cause the unstable unread counts from [MSC2654](https://github.com/matrix-org/matrix-spec-proposals/pull/2654) to be calculated even if the feature is disabled. diff --git a/synapse/config/experimental.py b/synapse/config/experimental.py index 260db49cad..702b81e636 100644 --- a/synapse/config/experimental.py +++ b/synapse/config/experimental.py @@ -71,6 +71,9 @@ class ExperimentalConfig(Config): self.msc3720_enabled: bool = experimental.get("msc3720_enabled", False) # MSC2654: Unread counts + # + # Note that enabling this will result in an incorrect unread count for + # previously calculated push actions. self.msc2654_enabled: bool = experimental.get("msc2654_enabled", False) # MSC2815 (allow room moderators to view redacted event content) diff --git a/synapse/push/bulk_push_rule_evaluator.py b/synapse/push/bulk_push_rule_evaluator.py index ccd512be54..d1caf8a0f7 100644 --- a/synapse/push/bulk_push_rule_evaluator.py +++ b/synapse/push/bulk_push_rule_evaluator.py @@ -262,7 +262,12 @@ class BulkPushRuleEvaluator: # This can happen due to out of band memberships return - count_as_unread = _should_count_as_unread(event, context) + # Disable counting as unread unless the experimental configuration is + # enabled, as it can cause additional (unwanted) rows to be added to the + # event_push_actions table. + count_as_unread = False + if self.hs.config.experimental.msc2654_enabled: + count_as_unread = _should_count_as_unread(event, context) rules_by_user = await self._get_rules_for_event(event) actions_by_user: Dict[str, Collection[Union[Mapping, str]]] = {} diff --git a/tests/storage/test_event_push_actions.py b/tests/storage/test_event_push_actions.py index 62fd4aeb2f..fc43d7edd1 100644 --- a/tests/storage/test_event_push_actions.py +++ b/tests/storage/test_event_push_actions.py @@ -67,9 +67,7 @@ class EventPushActionsStoreTestCase(HomeserverTestCase): last_event_id: str - def _assert_counts( - noitf_count: int, unread_count: int, highlight_count: int - ) -> None: + def _assert_counts(noitf_count: int, highlight_count: int) -> None: counts = self.get_success( self.store.db_pool.runInteraction( "get-unread-counts", @@ -82,7 +80,7 @@ class EventPushActionsStoreTestCase(HomeserverTestCase): counts, NotifCounts( notify_count=noitf_count, - unread_count=unread_count, + unread_count=0, highlight_count=highlight_count, ), ) @@ -112,27 +110,27 @@ class EventPushActionsStoreTestCase(HomeserverTestCase): ) ) - _assert_counts(0, 0, 0) + _assert_counts(0, 0) _create_event() - _assert_counts(1, 1, 0) + _assert_counts(1, 0) _rotate() - _assert_counts(1, 1, 0) + _assert_counts(1, 0) event_id = _create_event() - _assert_counts(2, 2, 0) + _assert_counts(2, 0) _rotate() - _assert_counts(2, 2, 0) + _assert_counts(2, 0) _create_event() _mark_read(event_id) - _assert_counts(1, 1, 0) + _assert_counts(1, 0) _mark_read(last_event_id) - _assert_counts(0, 0, 0) + _assert_counts(0, 0) _create_event() _rotate() - _assert_counts(1, 1, 0) + _assert_counts(1, 0) # Delete old event push actions, this should not affect the (summarised) count. # @@ -151,35 +149,35 @@ class EventPushActionsStoreTestCase(HomeserverTestCase): ) ) self.assertEqual(result, []) - _assert_counts(1, 1, 0) + _assert_counts(1, 0) _mark_read(last_event_id) - _assert_counts(0, 0, 0) + _assert_counts(0, 0) event_id = _create_event(True) - _assert_counts(1, 1, 1) + _assert_counts(1, 1) _rotate() - _assert_counts(1, 1, 1) + _assert_counts(1, 1) # Check that adding another notification and rotating after highlight # works. _create_event() _rotate() - _assert_counts(2, 2, 1) + _assert_counts(2, 1) # Check that sending read receipts at different points results in the # right counts. _mark_read(event_id) - _assert_counts(1, 1, 0) + _assert_counts(1, 0) _mark_read(last_event_id) - _assert_counts(0, 0, 0) + _assert_counts(0, 0) _create_event(True) - _assert_counts(1, 1, 1) + _assert_counts(1, 1) _mark_read(last_event_id) - _assert_counts(0, 0, 0) + _assert_counts(0, 0) _rotate() - _assert_counts(0, 0, 0) + _assert_counts(0, 0) def test_find_first_stream_ordering_after_ts(self) -> None: def add_event(so: int, ts: int) -> None: From 48a5c47a9f7379f3b7e85c5a8dcbae346eb22310 Mon Sep 17 00:00:00 2001 From: Patrick Cloke Date: Thu, 1 Sep 2022 14:57:47 -0400 Subject: [PATCH 046/107] Add a schema delta to drop unstable private read receipts. (#13692) Otherwise they'll be leaked due to the filtering code only respecting the stable identifiers for private read receipts. --- changelog.d/13692.removal | 1 + ...5remove_unstable_private_read_receipts.sql | 19 +++++++++++++++++++ 2 files changed, 20 insertions(+) create mode 100644 changelog.d/13692.removal create mode 100644 synapse/storage/schema/main/delta/72/05remove_unstable_private_read_receipts.sql diff --git a/changelog.d/13692.removal b/changelog.d/13692.removal new file mode 100644 index 0000000000..eb075d4517 --- /dev/null +++ b/changelog.d/13692.removal @@ -0,0 +1 @@ +Remove support for unstable [private read receipts](https://github.com/matrix-org/matrix-spec-proposals/pull/2285). diff --git a/synapse/storage/schema/main/delta/72/05remove_unstable_private_read_receipts.sql b/synapse/storage/schema/main/delta/72/05remove_unstable_private_read_receipts.sql new file mode 100644 index 0000000000..36b41049cd --- /dev/null +++ b/synapse/storage/schema/main/delta/72/05remove_unstable_private_read_receipts.sql @@ -0,0 +1,19 @@ +/* Copyright 2022 The Matrix.org Foundation C.I.C + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +-- Drop previously received private read receipts so they do not accidentally +-- get leaked to other users. +DELETE FROM receipts_linearized WHERE receipt_type = 'org.matrix.msc2285.read.private'; +DELETE FROM receipts_graph WHERE receipt_type = 'org.matrix.msc2285.read.private'; From 044900af6cc441d700d171098812786b4b312f59 Mon Sep 17 00:00:00 2001 From: Dirk Klimpel <5740567+dklimpel@users.noreply.github.com> Date: Thu, 1 Sep 2022 22:38:37 +0200 Subject: [PATCH 047/107] Fix two typos with colon in headlines (#13665) --- changelog.d/13665.doc | 1 + docs/usage/configuration/config_documentation.md | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) create mode 100644 changelog.d/13665.doc diff --git a/changelog.d/13665.doc b/changelog.d/13665.doc new file mode 100644 index 0000000000..6ee6434662 --- /dev/null +++ b/changelog.d/13665.doc @@ -0,0 +1 @@ +Remove unintentional colons from [config manual](https://matrix-org.github.io/synapse/latest/usage/configuration/config_documentation.html) headers. diff --git a/docs/usage/configuration/config_documentation.md b/docs/usage/configuration/config_documentation.md index 88e8e1c66f..396c560822 100644 --- a/docs/usage/configuration/config_documentation.md +++ b/docs/usage/configuration/config_documentation.md @@ -2486,7 +2486,7 @@ report_stats_endpoint: https://example.com/report-usage-stats/push Config settings related to the client/server API --- -### `room_prejoin_state:` +### `room_prejoin_state` Controls for the state that is shared with users who receive an invite to a room. By default, the following state event types are shared with users who @@ -2626,7 +2626,7 @@ Example configuration: key_refresh_interval: 2d ``` --- -### `trusted_key_servers:` +### `trusted_key_servers` The trusted servers to download signing keys from. From 0fdb685c2b95aa4b727f291c02821ceee6d40dad Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 2 Sep 2022 14:12:47 +0200 Subject: [PATCH 048/107] Only run trial CI on all python versions on non-PRs (#13698) --- .github/workflows/tests.yml | 10 ++++++++++ changelog.d/13698.misc | 1 + 2 files changed, 11 insertions(+) create mode 100644 changelog.d/13698.misc diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 49a010d9d8..720cb13907 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -82,6 +82,16 @@ jobs: python-version: ["3.7", "3.8", "3.9", "3.10"] database: ["sqlite"] extras: ["all"] + is_pr: + - ${{ startsWith(github.ref, 'refs/pull/') }} + + # If we're a PR then we only test min and max python. + exclude: + - is_pr: true + python-version: 3.8 + - is_pr: true + python-version: 3.9 + include: # Newest Python without optional deps - python-version: "3.10" diff --git a/changelog.d/13698.misc b/changelog.d/13698.misc new file mode 100644 index 0000000000..20b6b6d084 --- /dev/null +++ b/changelog.d/13698.misc @@ -0,0 +1 @@ +Only run trial CI on all python versions on non-PRs. From 4fee4a339d3bb1a90487a86ccbcca50a36b0ea74 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 2 Sep 2022 15:20:03 +0200 Subject: [PATCH 049/107] Update lock file for Poetry v1.2.0 (#13689) --- changelog.d/13689.misc | 1 + docs/development/contributing_guide.md | 2 + docs/development/dependencies.md | 9 +- docs/upgrade.md | 7 ++ poetry.lock | 146 +++++++++++++++---------- 5 files changed, 99 insertions(+), 66 deletions(-) create mode 100644 changelog.d/13689.misc diff --git a/changelog.d/13689.misc b/changelog.d/13689.misc new file mode 100644 index 0000000000..db6e48b150 --- /dev/null +++ b/changelog.d/13689.misc @@ -0,0 +1 @@ +Update poetry lock file for v1.2.0. diff --git a/docs/development/contributing_guide.md b/docs/development/contributing_guide.md index ab320cbd78..4e1df51164 100644 --- a/docs/development/contributing_guide.md +++ b/docs/development/contributing_guide.md @@ -62,6 +62,8 @@ pipx install poetry but see poetry's [installation instructions](https://python-poetry.org/docs/#installation) for other installation methods. +Synapse requires Poetry version 1.2.0 or later. + Next, open a terminal and install dependencies as follows: ```sh diff --git a/docs/development/dependencies.md b/docs/development/dependencies.md index 236856a6b0..b356870f27 100644 --- a/docs/development/dependencies.md +++ b/docs/development/dependencies.md @@ -243,14 +243,11 @@ doesn't require poetry. (It's what we use in CI too). However, you could try ## Check the version of poetry with `poetry --version`. -At the time of writing, the 1.2 series is beta only. We have seen some examples -where the lockfiles generated by 1.2 prereleasese aren't interpreted correctly -by poetry 1.1.x. For now, use poetry 1.1.14, which includes a critical -[change](https://github.com/python-poetry/poetry/pull/5973) needed to remain -[compatible with PyPI](https://github.com/pypi/warehouse/pull/11775). +The minimum version of poetry supported by Synapse is 1.2. It can also be useful to check the version of `poetry-core` in use. If you've -installed `poetry` with `pipx`, try `pipx runpip poetry list | grep poetry-core`. +installed `poetry` with `pipx`, try `pipx runpip poetry list | grep +poetry-core`. ## Clear caches: `poetry cache clear --all pypi`. diff --git a/docs/upgrade.md b/docs/upgrade.md index 51719f8c73..422a3da664 100644 --- a/docs/upgrade.md +++ b/docs/upgrade.md @@ -89,6 +89,13 @@ process, for example: dpkg -i matrix-synapse-py3_1.3.0+stretch1_amd64.deb ``` +# Upgrading to v1.67.0 + +## Minimum version of Poetry is now v1.2.0 + +The minimum supported version of poetry is now 1.2. This should only affect +those installing from a source checkout. + # Upgrading to v1.66.0 ## Delegation of email validation no longer supported diff --git a/poetry.lock b/poetry.lock index 651659ec98..35021390bf 100644 --- a/poetry.lock +++ b/poetry.lock @@ -7,10 +7,10 @@ optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" [package.extras] -dev = ["coverage[toml] (>=5.0.2)", "hypothesis", "pympler", "pytest (>=4.3.0)", "six", "mypy", "pytest-mypy-plugins", "zope.interface", "furo", "sphinx", "sphinx-notfound-page", "pre-commit", "cloudpickle"] -docs = ["furo", "sphinx", "zope.interface", "sphinx-notfound-page"] -tests = ["coverage[toml] (>=5.0.2)", "hypothesis", "pympler", "pytest (>=4.3.0)", "six", "mypy", "pytest-mypy-plugins", "zope.interface", "cloudpickle"] -tests_no_zope = ["coverage[toml] (>=5.0.2)", "hypothesis", "pympler", "pytest (>=4.3.0)", "six", "mypy", "pytest-mypy-plugins", "cloudpickle"] +dev = ["cloudpickle", "coverage[toml] (>=5.0.2)", "furo", "hypothesis", "mypy", "pre-commit", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins", "six", "sphinx", "sphinx-notfound-page", "zope.interface"] +docs = ["furo", "sphinx", "sphinx-notfound-page", "zope.interface"] +tests = ["cloudpickle", "coverage[toml] (>=5.0.2)", "hypothesis", "mypy", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins", "six", "zope.interface"] +tests_no_zope = ["cloudpickle", "coverage[toml] (>=5.0.2)", "hypothesis", "mypy", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins", "six"] [[package]] name = "authlib" @@ -39,7 +39,7 @@ attrs = ">=19.2.0" six = "*" [package.extras] -visualize = ["graphviz (>0.5.1)", "Twisted (>=16.1.1)"] +visualize = ["Twisted (>=16.1.1)", "graphviz (>0.5.1)"] [[package]] name = "bcrypt" @@ -177,7 +177,7 @@ optional = false python-versions = "*" [package.extras] -test = ["hypothesis (==3.55.3)", "flake8 (==3.7.8)"] +test = ["flake8 (==3.7.8)", "hypothesis (==3.55.3)"] [[package]] name = "constantly" @@ -199,12 +199,12 @@ python-versions = ">=3.6" cffi = ">=1.12" [package.extras] -docs = ["sphinx (>=1.6.5,!=1.8.0,!=3.1.0,!=3.1.1)", "sphinx-rtd-theme"] -docstest = ["pyenchant (>=1.6.11)", "twine (>=1.12.0)", "sphinxcontrib-spelling (>=4.0.1)"] +docs = ["sphinx (>=1.6.5,!=1.8.0,!=3.1.0,!=3.1.1)", "sphinx_rtd_theme"] +docstest = ["pyenchant (>=1.6.11)", "sphinxcontrib-spelling (>=4.0.1)", "twine (>=1.12.0)"] pep8test = ["black", "flake8", "flake8-import-order", "pep8-naming"] sdist = ["setuptools_rust (>=0.11.4)"] ssh = ["bcrypt (>=3.1.5)"] -test = ["pytest (>=6.2.0)", "pytest-cov", "pytest-subtests", "pytest-xdist", "pretend", "iso8601", "pytz", "hypothesis (>=1.11.4,!=3.79.2)"] +test = ["hypothesis (>=1.11.4,!=3.79.2)", "iso8601", "pretend", "pytest (>=6.2.0)", "pytest-cov", "pytest-subtests", "pytest-xdist", "pytz"] [[package]] name = "defusedxml" @@ -226,7 +226,7 @@ python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" wrapt = ">=1.10,<2" [package.extras] -dev = ["tox", "bump2version (<1)", "sphinx (<2)", "importlib-metadata (<3)", "importlib-resources (<4)", "configparser (<5)", "sphinxcontrib-websupport (<2)", "zipp (<2)", "PyTest (<5)", "PyTest-Cov (<2.6)", "pytest", "pytest-cov"] +dev = ["PyTest", "PyTest (<5)", "PyTest-Cov", "PyTest-Cov (<2.6)", "bump2version (<1)", "configparser (<5)", "importlib-metadata (<3)", "importlib-resources (<4)", "sphinx (<2)", "sphinxcontrib-websupport (<2)", "tox", "zipp (<2)"] [[package]] name = "docutils" @@ -245,7 +245,7 @@ optional = true python-versions = ">=3.7" [package.extras] -dev = ["tox", "coverage", "lxml", "xmlschema (>=1.8.0)", "sphinx", "memory-profiler", "flake8", "mypy (==0.910)"] +dev = ["Sphinx", "coverage", "flake8", "lxml", "memory-profiler", "mypy (==0.910)", "tox", "xmlschema (>=1.8.0)"] [[package]] name = "flake8" @@ -274,7 +274,7 @@ attrs = ">=19.2.0" flake8 = ">=3.0.0" [package.extras] -dev = ["coverage", "black", "hypothesis", "hypothesmith"] +dev = ["black", "coverage", "hypothesis", "hypothesmith"] [[package]] name = "flake8-comprehensions" @@ -367,8 +367,8 @@ typing-extensions = {version = ">=3.6.4", markers = "python_version < \"3.8\""} zipp = ">=0.5" [package.extras] -docs = ["sphinx", "jaraco.packaging (>=8.2)", "rst.linker (>=1.9)"] -testing = ["pytest (>=4.6)", "pytest-checkdocs (>=2.4)", "pytest-flake8", "pytest-cov", "pytest-enabler (>=1.0.1)", "packaging", "pep517", "pyfakefs", "flufl.flake8", "pytest-black (>=0.3.7)", "pytest-mypy", "importlib-resources (>=1.3)"] +docs = ["jaraco.packaging (>=8.2)", "rst.linker (>=1.9)", "sphinx"] +testing = ["flufl.flake8", "importlib-resources (>=1.3)", "packaging", "pep517", "pyfakefs", "pytest (>=4.6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=1.0.1)", "pytest-flake8", "pytest-mypy"] [[package]] name = "importlib-resources" @@ -382,8 +382,8 @@ python-versions = ">=3.6" zipp = {version = ">=3.1.0", markers = "python_version < \"3.10\""} [package.extras] -docs = ["sphinx", "jaraco.packaging (>=8.2)", "rst.linker (>=1.9)"] -testing = ["pytest (>=6)", "pytest-checkdocs (>=2.4)", "pytest-flake8", "pytest-cov", "pytest-enabler (>=1.0.1)", "pytest-black (>=0.3.7)", "pytest-mypy"] +docs = ["jaraco.packaging (>=8.2)", "rst.linker (>=1.9)", "sphinx"] +testing = ["pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=1.0.1)", "pytest-flake8", "pytest-mypy"] [[package]] name = "incremental" @@ -405,9 +405,9 @@ optional = false python-versions = ">=3.6,<4.0" [package.extras] -pipfile_deprecated_finder = ["pipreqs", "requirementslib"] -requirements_deprecated_finder = ["pipreqs", "pip-api"] colors = ["colorama (>=0.4.3,<0.5.0)"] +pipfile_deprecated_finder = ["pipreqs", "requirementslib"] +requirements_deprecated_finder = ["pip-api", "pipreqs"] [[package]] name = "jaeger-client" @@ -424,7 +424,7 @@ thrift = "*" tornado = ">=4.3" [package.extras] -tests = ["mock", "pycurl", "pytest", "pytest-cov", "coverage", "pytest-timeout", "pytest-tornado", "pytest-benchmark", "pytest-localserver", "flake8", "flake8-quotes", "flake8-typing-imports", "codecov", "tchannel (==2.1.0)", "opentracing_instrumentation (>=3,<4)", "prometheus_client (==0.11.0)", "mypy"] +tests = ["codecov", "coverage", "flake8", "flake8-quotes", "flake8-typing-imports", "mock", "mypy", "opentracing_instrumentation (>=3,<4)", "prometheus_client (==0.11.0)", "pycurl", "pytest", "pytest-benchmark[histogram]", "pytest-cov", "pytest-localserver", "pytest-timeout", "pytest-tornado", "tchannel (==2.1.0)"] [[package]] name = "jeepney" @@ -435,8 +435,8 @@ optional = false python-versions = ">=3.6" [package.extras] -trio = ["async-generator", "trio"] -test = ["async-timeout", "trio", "testpath", "pytest-asyncio", "pytest-trio", "pytest"] +test = ["async-timeout", "pytest", "pytest-asyncio", "pytest-trio", "testpath", "trio"] +trio = ["async_generator", "trio"] [[package]] name = "jinja2" @@ -486,8 +486,8 @@ pywin32-ctypes = {version = "<0.1.0 || >0.1.0,<0.1.1 || >0.1.1", markers = "sys_ SecretStorage = {version = ">=3.2", markers = "sys_platform == \"linux\""} [package.extras] -docs = ["sphinx", "jaraco.packaging (>=8.2)", "rst.linker (>=1.9)", "jaraco.tidelift (>=1.4)"] -testing = ["pytest (>=6)", "pytest-checkdocs (>=2.4)", "pytest-flake8", "pytest-cov", "pytest-enabler (>=1.0.1)", "pytest-black (>=0.3.7)", "pytest-mypy"] +docs = ["jaraco.packaging (>=8.2)", "jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "sphinx"] +testing = ["pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=1.0.1)", "pytest-flake8", "pytest-mypy"] [[package]] name = "ldap3" @@ -511,7 +511,7 @@ python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, != 3.4.*" [package.extras] cssselect = ["cssselect (>=0.7)"] html5 = ["html5lib"] -htmlsoup = ["beautifulsoup4"] +htmlsoup = ["BeautifulSoup4"] source = ["Cython (>=0.29.7)"] [[package]] @@ -535,8 +535,8 @@ attrs = "*" importlib-metadata = {version = ">=1.4", markers = "python_version < \"3.8\""} [package.extras] -test = ["aiounittest", "twisted", "tox"] -dev = ["twine (==4.0.1)", "build (==0.8.0)", "isort (==5.9.3)", "flake8 (==4.0.1)", "black (==22.3.0)", "mypy (==0.910)", "aiounittest", "twisted", "tox"] +dev = ["aiounittest", "black (==22.3.0)", "build (==0.8.0)", "flake8 (==4.0.1)", "isort (==5.9.3)", "mypy (==0.910)", "tox", "twine (==4.0.1)", "twisted"] +test = ["aiounittest", "tox", "twisted"] [[package]] name = "matrix-synapse-ldap3" @@ -552,7 +552,7 @@ service-identity = "*" Twisted = ">=15.1.0" [package.extras] -dev = ["isort (==5.9.3)", "flake8 (==4.0.1)", "black (==22.3.0)", "types-setuptools", "mypy (==0.910)", "ldaptor", "tox", "matrix-synapse"] +dev = ["black (==22.3.0)", "flake8 (==4.0.1)", "isort (==5.9.3)", "ldaptor", "matrix-synapse", "mypy (==0.910)", "tox", "types-setuptools"] [[package]] name = "mccabe" @@ -611,7 +611,7 @@ mypy = "0.950" "zope.schema" = "*" [package.extras] -test = ["pytest (>=4.6)", "pytest-cov", "lxml"] +test = ["lxml", "pytest (>=4.6)", "pytest-cov"] [[package]] name = "netaddr" @@ -630,7 +630,7 @@ optional = true python-versions = "*" [package.extras] -tests = ["doubles", "flake8", "flake8-quotes", "mock", "pytest", "pytest-cov", "pytest-mock", "sphinx", "sphinx-rtd-theme", "six (>=1.10.0,<2.0)", "gevent", "tornado"] +tests = ["Sphinx", "doubles", "flake8", "flake8-quotes", "gevent", "mock", "pytest", "pytest-cov", "pytest-mock", "six (>=1.10.0,<2.0)", "sphinx_rtd_theme", "tornado"] [[package]] name = "packaging" @@ -835,10 +835,10 @@ optional = false python-versions = ">=3.6" [package.extras] -tests = ["coverage[toml] (==5.0.4)", "pytest (>=6.0.0,<7.0.0)"] -docs = ["zope.interface", "sphinx-rtd-theme", "sphinx"] -dev = ["pre-commit", "mypy", "coverage[toml] (==5.0.4)", "pytest (>=6.0.0,<7.0.0)", "cryptography (>=3.3.1)", "zope.interface", "sphinx-rtd-theme", "sphinx"] crypto = ["cryptography (>=3.3.1)"] +dev = ["coverage[toml] (==5.0.4)", "cryptography (>=3.3.1)", "mypy", "pre-commit", "pytest (>=6.0.0,<7.0.0)", "sphinx", "sphinx-rtd-theme", "zope.interface"] +docs = ["sphinx", "sphinx-rtd-theme", "zope.interface"] +tests = ["coverage[toml] (==5.0.4)", "pytest (>=6.0.0,<7.0.0)"] [[package]] name = "pymacaroons" @@ -872,8 +872,8 @@ python-versions = ">=3.6" cffi = ">=1.4.1" [package.extras] -docs = ["sphinx (>=1.6.5)", "sphinx-rtd-theme"] -tests = ["pytest (>=3.2.1,!=3.3.0)", "hypothesis (>=3.27.0)"] +docs = ["sphinx (>=1.6.5)", "sphinx_rtd_theme"] +tests = ["hypothesis (>=3.27.0)", "pytest (>=3.2.1,!=3.3.0)"] [[package]] name = "pyopenssl" @@ -925,11 +925,12 @@ pyOpenSSL = "*" python-dateutil = "*" pytz = "*" requests = ">=1.0.0" +setuptools = "*" six = "*" xmlschema = ">=1.2.1" [package.extras] -s2repoze = ["paste", "zope.interface", "repoze.who"] +s2repoze = ["paste", "repoze.who", "zope.interface"] [[package]] name = "python-dateutil" @@ -1054,11 +1055,11 @@ celery = ["celery (>=3)"] chalice = ["chalice (>=1.16.0)"] django = ["django (>=1.8)"] falcon = ["falcon (>=1.4)"] -flask = ["flask (>=0.11)", "blinker (>=1.1)"] +flask = ["blinker (>=1.1)", "flask (>=0.11)"] httpx = ["httpx (>=0.16.0)"] -pure_eval = ["pure-eval", "executing", "asttokens"] +pure_eval = ["asttokens", "executing", "pure-eval"] pyspark = ["pyspark (>=2.4.4)"] -quart = ["quart (>=0.16.1)", "blinker (>=1.1)"] +quart = ["blinker (>=1.1)", "quart (>=0.16.1)"] rq = ["rq (>=0.6)"] sanic = ["sanic (>=0.8)"] sqlalchemy = ["sqlalchemy (>=1.2)"] @@ -1080,11 +1081,24 @@ pyasn1-modules = "*" six = "*" [package.extras] -dev = ["coverage[toml] (>=5.0.2)", "pytest", "sphinx", "furo", "idna", "pyopenssl"] -docs = ["sphinx", "furo"] +dev = ["coverage[toml] (>=5.0.2)", "furo", "idna", "pyOpenSSL", "pytest", "sphinx"] +docs = ["furo", "sphinx"] idna = ["idna"] tests = ["coverage[toml] (>=5.0.2)", "pytest"] +[[package]] +name = "setuptools" +version = "65.3.0" +description = "Easily download, build, install, upgrade, and uninstall Python packages" +category = "main" +optional = false +python-versions = ">=3.7" + +[package.extras] +docs = ["furo", "jaraco.packaging (>=9)", "jaraco.tidelift (>=1.4)", "pygments-github-lexers (==0.0.5)", "rst.linker (>=1.9)", "sphinx", "sphinx-favicon", "sphinx-hoverxref (<2)", "sphinx-inline-tabs", "sphinx-notfound-page (==0.8.3)", "sphinx-reredirects", "sphinxcontrib-towncrier"] +testing = ["build[virtualenv]", "filelock (>=3.4.0)", "flake8 (<5)", "flake8-2020", "ini2toml[lite] (>=0.9)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "mock", "pip (>=19.1)", "pip-run (>=8.8)", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=1.3)", "pytest-flake8", "pytest-mypy (>=0.9.1)", "pytest-perf", "pytest-xdist", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel"] +testing-integration = ["build[virtualenv]", "filelock (>=3.4.0)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "pytest", "pytest-enabler", "pytest-xdist", "tomli", "virtualenv (>=13.0.0)", "wheel"] + [[package]] name = "signedjson" version = "1.1.4" @@ -1199,6 +1213,7 @@ click = "*" click-default-group = "*" incremental = "*" jinja2 = "*" +setuptools = "*" tomli = {version = "*", markers = "python_version >= \"3.6\""} [package.extras] @@ -1236,7 +1251,7 @@ requests = ">=2.1.0" Twisted = {version = ">=18.7.0", extras = ["tls"]} [package.extras] -dev = ["pep8", "pyflakes", "httpbin (==0.5.0)"] +dev = ["httpbin (==0.5.0)", "pep8", "pyflakes"] docs = ["sphinx (>=1.4.8)"] [[package]] @@ -1281,20 +1296,20 @@ typing-extensions = ">=3.6.5" "zope.interface" = ">=4.4.2" [package.extras] -all_non_platform = ["cython-test-exception-raiser (>=1.0.2,<2)", "PyHamcrest (>=1.9.0)", "pyopenssl (>=16.0.0)", "service-identity (>=18.1.0)", "idna (>=2.4)", "pyasn1", "cryptography (>=2.6)", "appdirs (>=1.4.0)", "bcrypt (>=3.0.0)", "pyserial (>=3.0)", "h2 (>=3.0,<5.0)", "priority (>=1.1.0,<2.0)", "pywin32 (!=226)", "contextvars (>=2.4,<3)"] -conch = ["pyasn1", "cryptography (>=2.6)", "appdirs (>=1.4.0)", "bcrypt (>=3.0.0)"] -conch_nacl = ["pyasn1", "cryptography (>=2.6)", "appdirs (>=1.4.0)", "bcrypt (>=3.0.0)", "pynacl"] +all_non_platform = ["PyHamcrest (>=1.9.0)", "appdirs (>=1.4.0)", "bcrypt (>=3.0.0)", "contextvars (>=2.4,<3)", "cryptography (>=2.6)", "cython-test-exception-raiser (>=1.0.2,<2)", "h2 (>=3.0,<5.0)", "idna (>=2.4)", "priority (>=1.1.0,<2.0)", "pyasn1", "pyopenssl (>=16.0.0)", "pyserial (>=3.0)", "pywin32 (!=226)", "service-identity (>=18.1.0)"] +conch = ["appdirs (>=1.4.0)", "bcrypt (>=3.0.0)", "cryptography (>=2.6)", "pyasn1"] +conch_nacl = ["PyNaCl", "appdirs (>=1.4.0)", "bcrypt (>=3.0.0)", "cryptography (>=2.6)", "pyasn1"] contextvars = ["contextvars (>=2.4,<3)"] -dev = ["towncrier (>=19.2,<20.0)", "sphinx-rtd-theme (>=0.5,<1.0)", "readthedocs-sphinx-ext (>=2.1,<3.0)", "sphinx (>=4.1.2,<6)", "pyflakes (>=2.2,<3.0)", "twistedchecker (>=0.7,<1.0)", "coverage (>=6b1,<7)", "python-subunit (>=1.4,<2.0)", "pydoctor (>=21.9.0,<21.10.0)"] -dev_release = ["towncrier (>=19.2,<20.0)", "sphinx-rtd-theme (>=0.5,<1.0)", "readthedocs-sphinx-ext (>=2.1,<3.0)", "sphinx (>=4.1.2,<6)", "pydoctor (>=21.9.0,<21.10.0)"] +dev = ["coverage (>=6b1,<7)", "pydoctor (>=21.9.0,<21.10.0)", "pyflakes (>=2.2,<3.0)", "python-subunit (>=1.4,<2.0)", "readthedocs-sphinx-ext (>=2.1,<3.0)", "sphinx (>=4.1.2,<6)", "sphinx-rtd-theme (>=0.5,<1.0)", "towncrier (>=19.2,<20.0)", "twistedchecker (>=0.7,<1.0)"] +dev_release = ["pydoctor (>=21.9.0,<21.10.0)", "readthedocs-sphinx-ext (>=2.1,<3.0)", "sphinx (>=4.1.2,<6)", "sphinx-rtd-theme (>=0.5,<1.0)", "towncrier (>=19.2,<20.0)"] http2 = ["h2 (>=3.0,<5.0)", "priority (>=1.1.0,<2.0)"] -macos_platform = ["pyobjc-core", "pyobjc-framework-cfnetwork", "pyobjc-framework-cocoa", "cython-test-exception-raiser (>=1.0.2,<2)", "PyHamcrest (>=1.9.0)", "pyopenssl (>=16.0.0)", "service-identity (>=18.1.0)", "idna (>=2.4)", "pyasn1", "cryptography (>=2.6)", "appdirs (>=1.4.0)", "bcrypt (>=3.0.0)", "pyserial (>=3.0)", "h2 (>=3.0,<5.0)", "priority (>=1.1.0,<2.0)", "pywin32 (!=226)", "contextvars (>=2.4,<3)"] -mypy = ["mypy (==0.930)", "mypy-zope (==0.3.4)", "types-setuptools", "types-pyopenssl", "towncrier (>=19.2,<20.0)", "sphinx-rtd-theme (>=0.5,<1.0)", "readthedocs-sphinx-ext (>=2.1,<3.0)", "sphinx (>=4.1.2,<6)", "pyflakes (>=2.2,<3.0)", "twistedchecker (>=0.7,<1.0)", "coverage (>=6b1,<7)", "cython-test-exception-raiser (>=1.0.2,<2)", "PyHamcrest (>=1.9.0)", "pyopenssl (>=16.0.0)", "service-identity (>=18.1.0)", "idna (>=2.4)", "pyasn1", "cryptography (>=2.6)", "appdirs (>=1.4.0)", "bcrypt (>=3.0.0)", "pyserial (>=3.0)", "h2 (>=3.0,<5.0)", "priority (>=1.1.0,<2.0)", "pynacl", "pywin32 (!=226)", "python-subunit (>=1.4,<2.0)", "contextvars (>=2.4,<3)", "pydoctor (>=21.9.0,<21.10.0)"] -osx_platform = ["pyobjc-core", "pyobjc-framework-cfnetwork", "pyobjc-framework-cocoa", "cython-test-exception-raiser (>=1.0.2,<2)", "PyHamcrest (>=1.9.0)", "pyopenssl (>=16.0.0)", "service-identity (>=18.1.0)", "idna (>=2.4)", "pyasn1", "cryptography (>=2.6)", "appdirs (>=1.4.0)", "bcrypt (>=3.0.0)", "pyserial (>=3.0)", "h2 (>=3.0,<5.0)", "priority (>=1.1.0,<2.0)", "pywin32 (!=226)", "contextvars (>=2.4,<3)"] +macos_platform = ["PyHamcrest (>=1.9.0)", "appdirs (>=1.4.0)", "bcrypt (>=3.0.0)", "contextvars (>=2.4,<3)", "cryptography (>=2.6)", "cython-test-exception-raiser (>=1.0.2,<2)", "h2 (>=3.0,<5.0)", "idna (>=2.4)", "priority (>=1.1.0,<2.0)", "pyasn1", "pyobjc-core", "pyobjc-framework-CFNetwork", "pyobjc-framework-Cocoa", "pyopenssl (>=16.0.0)", "pyserial (>=3.0)", "pywin32 (!=226)", "service-identity (>=18.1.0)"] +mypy = ["PyHamcrest (>=1.9.0)", "PyNaCl", "appdirs (>=1.4.0)", "bcrypt (>=3.0.0)", "contextvars (>=2.4,<3)", "coverage (>=6b1,<7)", "cryptography (>=2.6)", "cython-test-exception-raiser (>=1.0.2,<2)", "h2 (>=3.0,<5.0)", "idna (>=2.4)", "mypy (==0.930)", "mypy-zope (==0.3.4)", "priority (>=1.1.0,<2.0)", "pyasn1", "pydoctor (>=21.9.0,<21.10.0)", "pyflakes (>=2.2,<3.0)", "pyopenssl (>=16.0.0)", "pyserial (>=3.0)", "python-subunit (>=1.4,<2.0)", "pywin32 (!=226)", "readthedocs-sphinx-ext (>=2.1,<3.0)", "service-identity (>=18.1.0)", "sphinx (>=4.1.2,<6)", "sphinx-rtd-theme (>=0.5,<1.0)", "towncrier (>=19.2,<20.0)", "twistedchecker (>=0.7,<1.0)", "types-pyOpenSSL", "types-setuptools"] +osx_platform = ["PyHamcrest (>=1.9.0)", "appdirs (>=1.4.0)", "bcrypt (>=3.0.0)", "contextvars (>=2.4,<3)", "cryptography (>=2.6)", "cython-test-exception-raiser (>=1.0.2,<2)", "h2 (>=3.0,<5.0)", "idna (>=2.4)", "priority (>=1.1.0,<2.0)", "pyasn1", "pyobjc-core", "pyobjc-framework-CFNetwork", "pyobjc-framework-Cocoa", "pyopenssl (>=16.0.0)", "pyserial (>=3.0)", "pywin32 (!=226)", "service-identity (>=18.1.0)"] serial = ["pyserial (>=3.0)", "pywin32 (!=226)"] -test = ["cython-test-exception-raiser (>=1.0.2,<2)", "PyHamcrest (>=1.9.0)"] -tls = ["pyopenssl (>=16.0.0)", "service-identity (>=18.1.0)", "idna (>=2.4)"] -windows_platform = ["pywin32 (!=226)", "cython-test-exception-raiser (>=1.0.2,<2)", "PyHamcrest (>=1.9.0)", "pyopenssl (>=16.0.0)", "service-identity (>=18.1.0)", "idna (>=2.4)", "pyasn1", "cryptography (>=2.6)", "appdirs (>=1.4.0)", "bcrypt (>=3.0.0)", "pyserial (>=3.0)", "h2 (>=3.0,<5.0)", "priority (>=1.1.0,<2.0)", "pywin32 (!=226)", "contextvars (>=2.4,<3)"] +test = ["PyHamcrest (>=1.9.0)", "cython-test-exception-raiser (>=1.0.2,<2)"] +tls = ["idna (>=2.4)", "pyopenssl (>=16.0.0)", "service-identity (>=18.1.0)"] +windows_platform = ["PyHamcrest (>=1.9.0)", "appdirs (>=1.4.0)", "bcrypt (>=3.0.0)", "contextvars (>=2.4,<3)", "cryptography (>=2.6)", "cython-test-exception-raiser (>=1.0.2,<2)", "h2 (>=3.0,<5.0)", "idna (>=2.4)", "priority (>=1.1.0,<2.0)", "pyasn1", "pyopenssl (>=16.0.0)", "pyserial (>=3.0)", "pywin32 (!=226)", "pywin32 (!=226)", "service-identity (>=18.1.0)"] [[package]] name = "twisted-iocpsupport" @@ -1472,7 +1487,7 @@ python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, <4" [package.extras] brotli = ["brotlipy (>=0.6.0)"] -secure = ["pyOpenSSL (>=0.14)", "cryptography (>=1.3.4)", "idna (>=2.0.0)", "certifi", "ipaddress"] +secure = ["certifi", "cryptography (>=1.3.4)", "idna (>=2.0.0)", "ipaddress", "pyOpenSSL (>=0.14)"] socks = ["PySocks (>=1.5.6,!=1.5.7,<2.0)"] [[package]] @@ -1504,8 +1519,8 @@ elementpath = ">=2.5.0,<3.0.0" [package.extras] codegen = ["elementpath (>=2.5.0,<3.0.0)", "jinja2"] -dev = ["tox", "coverage", "lxml", "elementpath (>=2.5.0,<3.0.0)", "memory-profiler", "sphinx", "sphinx-rtd-theme", "jinja2", "flake8", "mypy", "lxml-stubs"] -docs = ["elementpath (>=2.5.0,<3.0.0)", "sphinx", "sphinx-rtd-theme", "jinja2"] +dev = ["Sphinx", "coverage", "elementpath (>=2.5.0,<3.0.0)", "flake8", "jinja2", "lxml", "lxml-stubs", "memory-profiler", "mypy", "sphinx-rtd-theme", "tox"] +docs = ["Sphinx", "elementpath (>=2.5.0,<3.0.0)", "jinja2", "sphinx-rtd-theme"] [[package]] name = "zipp" @@ -1516,8 +1531,8 @@ optional = false python-versions = ">=3.7" [package.extras] -docs = ["sphinx", "jaraco.packaging (>=8.2)", "rst.linker (>=1.9)"] -testing = ["pytest (>=6)", "pytest-checkdocs (>=2.4)", "pytest-flake8", "pytest-cov", "pytest-enabler (>=1.0.1)", "jaraco.itertools", "func-timeout", "pytest-black (>=0.3.7)", "pytest-mypy"] +docs = ["jaraco.packaging (>=8.2)", "rst.linker (>=1.9)", "sphinx"] +testing = ["func-timeout", "jaraco.itertools", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=1.0.1)", "pytest-flake8", "pytest-mypy"] [[package]] name = "zope.event" @@ -1527,8 +1542,11 @@ category = "dev" optional = false python-versions = "*" +[package.dependencies] +setuptools = "*" + [package.extras] -docs = ["sphinx"] +docs = ["Sphinx"] test = ["zope.testrunner"] [[package]] @@ -1539,8 +1557,11 @@ category = "main" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" +[package.dependencies] +setuptools = "*" + [package.extras] -docs = ["sphinx", "repoze.sphinx.autointerface"] +docs = ["Sphinx", "repoze.sphinx.autointerface"] test = ["coverage (>=5.0.3)", "zope.event", "zope.testing"] testing = ["coverage (>=5.0.3)", "zope.event", "zope.testing"] @@ -1553,11 +1574,12 @@ optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" [package.dependencies] +setuptools = "*" "zope.event" = "*" "zope.interface" = ">=5.0.0" [package.extras] -docs = ["sphinx", "repoze.sphinx.autointerface"] +docs = ["Sphinx", "repoze.sphinx.autointerface"] test = ["zope.i18nmessageid", "zope.testing", "zope.testrunner"] [extras] @@ -2458,6 +2480,10 @@ service-identity = [ {file = "service-identity-21.1.0.tar.gz", hash = "sha256:6e6c6086ca271dc11b033d17c3a8bea9f24ebff920c587da090afc9519419d34"}, {file = "service_identity-21.1.0-py2.py3-none-any.whl", hash = "sha256:f0b0caac3d40627c3c04d7a51b6e06721857a0e10a8775f2d1d7e72901b3a7db"}, ] +setuptools = [ + {file = "setuptools-65.3.0-py3-none-any.whl", hash = "sha256:2e24e0bec025f035a2e72cdd1961119f557d78ad331bb00ff82efb2ab8da8e82"}, + {file = "setuptools-65.3.0.tar.gz", hash = "sha256:7732871f4f7fa58fb6bdcaeadb0161b2bd046c85905dbaa066bdcbcc81953b57"}, +] signedjson = [ {file = "signedjson-1.1.4-py3-none-any.whl", hash = "sha256:45569ec54241c65d2403fe3faf7169be5322547706a231e884ca2b427f23d228"}, {file = "signedjson-1.1.4.tar.gz", hash = "sha256:cd91c56af53f169ef032c62e9c4a3292dc158866933318d0592e3462db3d6492"}, From 36b184b78257fa4455293b7d7faa7a53a4261383 Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Fri, 2 Sep 2022 11:29:51 -0500 Subject: [PATCH 050/107] Update docs to make enabling metrics more clear (#13678) It was really easy to miss the `enable_metrics: True` step with the previous language. --- changelog.d/13678.doc | 1 + docs/metrics-howto.md | 11 +++++++---- 2 files changed, 8 insertions(+), 4 deletions(-) create mode 100644 changelog.d/13678.doc diff --git a/changelog.d/13678.doc b/changelog.d/13678.doc new file mode 100644 index 0000000000..8b85dfe643 --- /dev/null +++ b/changelog.d/13678.doc @@ -0,0 +1 @@ +Update docs to make enabling metrics more clear. diff --git a/docs/metrics-howto.md b/docs/metrics-howto.md index 4a77d5604c..279303a798 100644 --- a/docs/metrics-howto.md +++ b/docs/metrics-howto.md @@ -7,7 +7,13 @@ 1. Enable Synapse metrics: - There are two methods of enabling metrics in Synapse. + In `homeserver.yaml`, make sure `enable_metrics` is + set to `True`. + +1. Enable the `/_synapse/metrics` Synapse endpoint that Prometheus uses to + collect data: + + There are two methods of enabling the metrics endpoint in Synapse. The first serves the metrics as a part of the usual web server and can be enabled by adding the \"metrics\" resource to the existing @@ -41,9 +47,6 @@ - '0.0.0.0' ``` - For both options, you will need to ensure that `enable_metrics` is - set to `True`. - 1. Restart Synapse. 1. Add a Prometheus target for Synapse. From 877bdfa889fa07d09f385df297a9282d51d50dae Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Fri, 2 Sep 2022 14:05:39 -0500 Subject: [PATCH 051/107] Clarify `(room_id, event_id)` global uniqueness (#13701) Summarized from @richvdh's reply at https://github.com/matrix-org/synapse/pull/13589#discussion_r961116999 --- changelog.d/13701.doc | 1 + docs/development/database_schema.md | 24 ++++++++++++++++++++++++ 2 files changed, 25 insertions(+) create mode 100644 changelog.d/13701.doc diff --git a/changelog.d/13701.doc b/changelog.d/13701.doc new file mode 100644 index 0000000000..b438e066d8 --- /dev/null +++ b/changelog.d/13701.doc @@ -0,0 +1 @@ +Clarify `(room_id, event_id)` global uniqueness and how we should scope our database schemas. diff --git a/docs/development/database_schema.md b/docs/development/database_schema.md index d996a7caa2..e9b925ddd8 100644 --- a/docs/development/database_schema.md +++ b/docs/development/database_schema.md @@ -191,3 +191,27 @@ There are three separate aspects to this: flavour will be accepted by SQLite 3.22, but will give a column whose default value is the **string** `"FALSE"` - which, when cast back to a boolean in Python, evaluates to `True`. + + +## `event_id` global uniqueness + +In room versions `1` and `2` it's possible to end up with two events with the +same `event_id` (in the same or different rooms). After room version `3`, that +can only happen with a hash collision, which we basically hope will never +happen. + +There are several places in Synapse and even Matrix APIs like [`GET +/_matrix/federation/v1/event/{eventId}`](https://spec.matrix.org/v1.1/server-server-api/#get_matrixfederationv1eventeventid) +where we assume that event IDs are globally unique. + +But hash collisions are still possible, and by treating event IDs as room +scoped, we can reduce the possibility of a hash collision. When scoping +`event_id` in the database schema, it should be also accompanied by `room_id` +(`PRIMARY KEY (room_id, event_id)`) and lookups should be done through the pair +`(room_id, event_id)`. + +There has been a lot of debate on this in places like +https://github.com/matrix-org/matrix-spec-proposals/issues/2779 and +[MSC2848](https://github.com/matrix-org/matrix-spec-proposals/pull/2848) which +has no resolution yet (as of 2022-09-01). + From ad7fc8e92f53217469342f4593907cd4ce13a930 Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Sat, 3 Sep 2022 11:08:28 +0100 Subject: [PATCH 052/107] fix grammar --- README.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.rst b/README.rst index 84e5310309..d116cd51fb 100644 --- a/README.rst +++ b/README.rst @@ -3,7 +3,7 @@ Synapse |support| |development| |documentation| |license| |pypi| |python| ========================================================================= Synapse is an open-source `Matrix `_ homeserver written and -maintained by the Matrix.org Foundation. We began rapid development began in 2014, +maintained by the Matrix.org Foundation. We began rapid development in 2014, reaching v1.0.0 in 2019. Development on Synapse and the Matrix protocol itself continues in earnest today. From 898fef2789c9b1a20ef53c7d588f536f51f0fe2f Mon Sep 17 00:00:00 2001 From: Brendan Abolivier Date: Mon, 5 Sep 2022 12:26:43 +0200 Subject: [PATCH 053/107] Share some metrics between the Prometheus exporter and the phone home stats (#13671) --- changelog.d/13671.misc | 1 + synapse/app/_base.py | 5 +- synapse/app/phone_stats_home.py | 13 +++- synapse/metrics/common_usage_metrics.py | 79 +++++++++++++++++++++++++ synapse/server.py | 6 ++ tests/test_phone_home.py | 46 +++++++++++++- 6 files changed, 146 insertions(+), 4 deletions(-) create mode 100644 changelog.d/13671.misc create mode 100644 synapse/metrics/common_usage_metrics.py diff --git a/changelog.d/13671.misc b/changelog.d/13671.misc new file mode 100644 index 0000000000..f1c62b5b1e --- /dev/null +++ b/changelog.d/13671.misc @@ -0,0 +1 @@ +Introduce a `CommonUsageMetrics` class to share some usage metrics between the Prometheus exporter and the phone home stats. diff --git a/synapse/app/_base.py b/synapse/app/_base.py index 4742435d3b..9a24bed0a0 100644 --- a/synapse/app/_base.py +++ b/synapse/app/_base.py @@ -511,9 +511,10 @@ async def start(hs: "HomeServer") -> None: setup_sentry(hs) setup_sdnotify(hs) - # If background tasks are running on the main process, start collecting the - # phone home stats. + # If background tasks are running on the main process or this is the worker in + # charge of them, start collecting the phone home stats and shared usage metrics. if hs.config.worker.run_background_tasks: + await hs.get_common_usage_metrics_manager().setup() start_phone_stats_home(hs) # We now freeze all allocated objects in the hopes that (almost) diff --git a/synapse/app/phone_stats_home.py b/synapse/app/phone_stats_home.py index 40dbdace8e..51c8d15711 100644 --- a/synapse/app/phone_stats_home.py +++ b/synapse/app/phone_stats_home.py @@ -51,6 +51,16 @@ async def phone_stats_home( stats: JsonDict, stats_process: List[Tuple[int, "resource.struct_rusage"]] = _stats_process, ) -> None: + """Collect usage statistics and send them to the configured endpoint. + + Args: + hs: the HomeServer object to use for gathering usage data. + stats: the dict in which to store the statistics sent to the configured + endpoint. Mostly used in tests to figure out the data that is supposed to + be sent. + stats_process: statistics about resource usage of the process. + """ + logger.info("Gathering stats for reporting") now = int(hs.get_clock().time()) # Ensure the homeserver has started. @@ -83,6 +93,7 @@ async def phone_stats_home( # store = hs.get_datastores().main + common_metrics = await hs.get_common_usage_metrics_manager().get_metrics() stats["homeserver"] = hs.config.server.server_name stats["server_context"] = hs.config.server.server_context @@ -104,7 +115,7 @@ async def phone_stats_home( room_count = await store.get_room_count() stats["total_room_count"] = room_count - stats["daily_active_users"] = await store.count_daily_users() + stats["daily_active_users"] = common_metrics.daily_active_users stats["monthly_active_users"] = await store.count_monthly_users() daily_active_e2ee_rooms = await store.count_daily_active_e2ee_rooms() stats["daily_active_e2ee_rooms"] = daily_active_e2ee_rooms diff --git a/synapse/metrics/common_usage_metrics.py b/synapse/metrics/common_usage_metrics.py new file mode 100644 index 0000000000..0a22ea3d92 --- /dev/null +++ b/synapse/metrics/common_usage_metrics.py @@ -0,0 +1,79 @@ +# Copyright 2022 The Matrix.org Foundation C.I.C +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from typing import TYPE_CHECKING + +import attr + +from synapse.metrics.background_process_metrics import run_as_background_process + +if TYPE_CHECKING: + from synapse.server import HomeServer + +from prometheus_client import Gauge + +# Gauge to expose daily active users metrics +current_dau_gauge = Gauge( + "synapse_admin_daily_active_users", + "Current daily active users count", +) + + +@attr.s(auto_attribs=True) +class CommonUsageMetrics: + """Usage metrics shared between the phone home stats and the prometheus exporter.""" + + daily_active_users: int + + +class CommonUsageMetricsManager: + """Collects common usage metrics.""" + + def __init__(self, hs: "HomeServer") -> None: + self._store = hs.get_datastores().main + self._clock = hs.get_clock() + + async def get_metrics(self) -> CommonUsageMetrics: + """Get the CommonUsageMetrics object. If no collection has happened yet, do it + before returning the metrics. + + Returns: + The CommonUsageMetrics object to read common metrics from. + """ + return await self._collect() + + async def setup(self) -> None: + """Keep the gauges for common usage metrics up to date.""" + await self._update_gauges() + self._clock.looping_call( + run_as_background_process, + 5 * 60 * 1000, + desc="common_usage_metrics_update_gauges", + func=self._update_gauges, + ) + + async def _collect(self) -> CommonUsageMetrics: + """Collect the common metrics and either create the CommonUsageMetrics object to + use if it doesn't exist yet, or update it. + """ + dau_count = await self._store.count_daily_users() + + return CommonUsageMetrics( + daily_active_users=dau_count, + ) + + async def _update_gauges(self) -> None: + """Update the Prometheus gauges.""" + metrics = await self._collect() + + current_dau_gauge.set(float(metrics.daily_active_users)) diff --git a/synapse/server.py b/synapse/server.py index c2e55bf0b1..5a99c0b344 100644 --- a/synapse/server.py +++ b/synapse/server.py @@ -105,6 +105,7 @@ from synapse.handlers.typing import FollowerTypingHandler, TypingWriterHandler from synapse.handlers.user_directory import UserDirectoryHandler from synapse.http.client import InsecureInterceptableContextFactory, SimpleHttpClient from synapse.http.matrixfederationclient import MatrixFederationHttpClient +from synapse.metrics.common_usage_metrics import CommonUsageMetricsManager from synapse.module_api import ModuleApi from synapse.notifier import Notifier from synapse.push.bulk_push_rule_evaluator import BulkPushRuleEvaluator @@ -829,3 +830,8 @@ class HomeServer(metaclass=abc.ABCMeta): self.config.ratelimiting.rc_message, self.config.ratelimiting.rc_admin_redaction, ) + + @cache_in_self + def get_common_usage_metrics_manager(self) -> CommonUsageMetricsManager: + """Usage metrics shared between phone home stats and the prometheus exporter.""" + return CommonUsageMetricsManager(self) diff --git a/tests/test_phone_home.py b/tests/test_phone_home.py index b01cae6e5d..cc1a98f1c4 100644 --- a/tests/test_phone_home.py +++ b/tests/test_phone_home.py @@ -15,8 +15,14 @@ import resource from unittest import mock +from twisted.test.proto_helpers import MemoryReactor + from synapse.app.phone_stats_home import phone_stats_home +from synapse.rest import admin +from synapse.rest.client import login, sync +from synapse.server import HomeServer from synapse.types import JsonDict +from synapse.util import Clock from tests.unittest import HomeserverTestCase @@ -47,5 +53,43 @@ class PhoneHomeStatsTestCase(HomeserverTestCase): stats: JsonDict = {} self.reactor.advance(1) # `old_resource` has type `Mock` instead of `struct_rusage` - self.get_success(phone_stats_home(self.hs, stats, past_stats)) # type: ignore[arg-type] + self.get_success( + phone_stats_home(self.hs, stats, past_stats) # type: ignore[arg-type] + ) self.assertApproximates(stats["cpu_average"], 100, tolerance=2.5) + + +class CommonMetricsTestCase(HomeserverTestCase): + servlets = [ + admin.register_servlets, + login.register_servlets, + sync.register_servlets, + ] + + def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None: + self.metrics_manager = hs.get_common_usage_metrics_manager() + self.get_success(self.metrics_manager.setup()) + + def test_dau(self) -> None: + """Tests that the daily active users count is correctly updated.""" + self._assert_metric_value("daily_active_users", 0) + + self.register_user("user", "password") + tok = self.login("user", "password") + self.make_request("GET", "/sync", access_token=tok) + + self.pump(1) + + self._assert_metric_value("daily_active_users", 1) + + def _assert_metric_value(self, metric_name: str, expected: int) -> None: + """Compare the given value to the current value of the common usage metric with + the given name. + + Args: + metric_name: The metric to look up. + expected: Expected value for this metric. + """ + metrics = self.get_success(self.metrics_manager.get_metrics()) + value = getattr(metrics, metric_name) + self.assertEqual(value, expected) From 8cb9261598becb3ed790b4268ff713f5e1122a21 Mon Sep 17 00:00:00 2001 From: David Robertson Date: Mon, 5 Sep 2022 12:13:13 +0100 Subject: [PATCH 054/107] Fix typechecking with latest `types-jsonschema` (#13712) --- changelog.d/13712.misc | 1 + synapse/api/filtering.py | 8 ++++---- 2 files changed, 5 insertions(+), 4 deletions(-) create mode 100644 changelog.d/13712.misc diff --git a/changelog.d/13712.misc b/changelog.d/13712.misc new file mode 100644 index 0000000000..2c4f6b19f6 --- /dev/null +++ b/changelog.d/13712.misc @@ -0,0 +1 @@ +Fix typechecking with latest types-jsonschema. diff --git a/synapse/api/filtering.py b/synapse/api/filtering.py index b007147519..102889ac49 100644 --- a/synapse/api/filtering.py +++ b/synapse/api/filtering.py @@ -140,13 +140,13 @@ USER_FILTER_SCHEMA = { @FormatChecker.cls_checks("matrix_room_id") -def matrix_room_id_validator(room_id_str: str) -> RoomID: - return RoomID.from_string(room_id_str) +def matrix_room_id_validator(room_id_str: str) -> bool: + return RoomID.is_valid(room_id_str) @FormatChecker.cls_checks("matrix_user_id") -def matrix_user_id_validator(user_id_str: str) -> UserID: - return UserID.from_string(user_id_str) +def matrix_user_id_validator(user_id_str: str) -> bool: + return UserID.is_valid(user_id_str) class Filtering: From c7b18d9d44c90acfd4ceaec1fa2f8275e03f14af Mon Sep 17 00:00:00 2001 From: reivilibre Date: Mon, 5 Sep 2022 11:16:59 +0000 Subject: [PATCH 055/107] Extend the release script to wait for GitHub Actions to finish and to be usable as a guide for the whole process. (#13483) --- changelog.d/13483.misc | 1 + scripts-dev/release.py | 146 +++++++++++++++++++++++++++++++++++++++-- 2 files changed, 143 insertions(+), 4 deletions(-) create mode 100644 changelog.d/13483.misc diff --git a/changelog.d/13483.misc b/changelog.d/13483.misc new file mode 100644 index 0000000000..4fe6dbbea4 --- /dev/null +++ b/changelog.d/13483.misc @@ -0,0 +1 @@ +Extend the release script to wait for GitHub Actions to finish and to be usable as a guide for the whole process. \ No newline at end of file diff --git a/scripts-dev/release.py b/scripts-dev/release.py index 46220c4dd3..6603bc593b 100755 --- a/scripts-dev/release.py +++ b/scripts-dev/release.py @@ -18,10 +18,12 @@ """ import glob +import json import os import re import subprocess import sys +import time import urllib.request from os import path from tempfile import TemporaryDirectory @@ -71,18 +73,21 @@ def cli() -> None: ./scripts-dev/release.py tag - # ... wait for assets to build ... + # wait for assets to build, either manually or with: + ./scripts-dev/release.py wait-for-actions ./scripts-dev/release.py publish ./scripts-dev/release.py upload - # Optional: generate some nice links for the announcement - ./scripts-dev/release.py merge-back + # Optional: generate some nice links for the announcement ./scripts-dev/release.py announce + Alternatively, `./scripts-dev/release.py full` will do all the above + as well as guiding you through the manual steps. + If the env var GH_TOKEN (or GITHUB_TOKEN) is set, or passed into the `tag`/`publish` command, then a new draft release will be created/published. """ @@ -90,6 +95,10 @@ def cli() -> None: @cli.command() def prepare() -> None: + _prepare() + + +def _prepare() -> None: """Do the initial stages of creating a release, including creating release branch, updating changelog and pushing to GitHub. """ @@ -284,6 +293,10 @@ def prepare() -> None: @cli.command() @click.option("--gh-token", envvar=["GH_TOKEN", "GITHUB_TOKEN"]) def tag(gh_token: Optional[str]) -> None: + _tag(gh_token) + + +def _tag(gh_token: Optional[str]) -> None: """Tags the release and generates a draft GitHub release""" # Make sure we're in a git repo. @@ -374,6 +387,10 @@ def tag(gh_token: Optional[str]) -> None: @cli.command() @click.option("--gh-token", envvar=["GH_TOKEN", "GITHUB_TOKEN"], required=True) def publish(gh_token: str) -> None: + _publish(gh_token) + + +def _publish(gh_token: str) -> None: """Publish release on GitHub.""" # Make sure we're in a git repo. @@ -411,6 +428,10 @@ def publish(gh_token: str) -> None: @cli.command() def upload() -> None: + _upload() + + +def _upload() -> None: """Upload release to pypi.""" current_version = get_package_version() @@ -479,8 +500,75 @@ def _merge_into(repo: Repo, source: str, target: str) -> None: repo.remote().push() +@cli.command() +@click.option("--gh-token", envvar=["GH_TOKEN", "GITHUB_TOKEN"], required=False) +def wait_for_actions(gh_token: Optional[str]) -> None: + _wait_for_actions(gh_token) + + +def _wait_for_actions(gh_token: Optional[str]) -> None: + # Find out the version and tag name. + current_version = get_package_version() + tag_name = f"v{current_version}" + + # Authentication is optional on this endpoint, + # but use a token if we have one to reduce the chance of being rate-limited. + url = f"https://api.github.com/repos/matrix-org/synapse/actions/runs?branch={tag_name}" + headers = {"Accept": "application/vnd.github+json"} + if gh_token is not None: + headers["authorization"] = f"token {gh_token}" + req = urllib.request.Request(url, headers=headers) + + time.sleep(10 * 60) + while True: + time.sleep(5 * 60) + response = urllib.request.urlopen(req) + resp = json.loads(response.read()) + + if len(resp["workflow_runs"]) == 0: + continue + + if all( + workflow["status"] != "in_progress" for workflow in resp["workflow_runs"] + ): + success = ( + workflow["status"] == "completed" for workflow in resp["workflow_runs"] + ) + if success: + _notify("Workflows successful. You can now continue the release.") + else: + _notify("Workflows failed.") + click.confirm("Continue anyway?", abort=True) + + break + + +def _notify(message: str) -> None: + # Send a bell character. Most terminals will play a sound or show a notification + # for this. + click.echo(f"\a{message}") + + # Try and run notify-send, but don't raise an Exception if this fails + # (This is best-effort) + # TODO Support other platforms? + subprocess.run( + [ + "notify-send", + "--app-name", + "Synapse Release Script", + "--expire-time", + "3600000", + message, + ] + ) + + @cli.command() def merge_back() -> None: + _merge_back() + + +def _merge_back() -> None: """Merge the release branch back into the appropriate branches. All branches will be automatically pulled from the remote and the results will be pushed to the remote.""" @@ -519,6 +607,10 @@ def merge_back() -> None: @cli.command() def announce() -> None: + _announce() + + +def _announce() -> None: """Generate markdown to announce the release.""" current_version = get_package_version() @@ -548,10 +640,56 @@ Announce the release in - #homeowners:matrix.org (Synapse Announcements), bumping the version in the topic - #synapse:matrix.org (Synapse Admins), bumping the version in the topic - #synapse-dev:matrix.org -- #synapse-package-maintainers:matrix.org""" +- #synapse-package-maintainers:matrix.org + +Ask the designated people to do the blog and tweets.""" ) +@cli.command() +@click.option("--gh-token", envvar=["GH_TOKEN", "GITHUB_TOKEN"], required=True) +def full(gh_token: str) -> None: + click.echo("1. If this is a security release, read the security wiki page.") + click.echo("2. Check for any release blockers before proceeding.") + click.echo(" https://github.com/matrix-org/synapse/labels/X-Release-Blocker") + + click.confirm("Ready?", abort=True) + + click.echo("\n*** prepare ***") + _prepare() + + click.echo("Deploy to matrix.org and ensure that it hasn't fallen over.") + click.echo("Remember to silence the alerts to prevent alert spam.") + click.confirm("Deployed?", abort=True) + + click.echo("\n*** tag ***") + _tag(gh_token) + + click.echo("\n*** wait for actions ***") + _wait_for_actions(gh_token) + + click.echo("\n*** publish ***") + _publish(gh_token) + + click.echo("\n*** upload ***") + _upload() + + click.echo("\n*** merge back ***") + _merge_back() + + click.echo("\nUpdate the Debian repository") + click.confirm("Started updating Debian repository?", abort=True) + + click.echo("\nWait for all release methods to be ready.") + # Docker should be ready because it was done by the workflows earlier + # PyPI should be ready because we just ran upload(). + # TODO Automatically poll until the Debs have made it to packages.matrix.org + click.confirm("Debs ready?", abort=True) + + click.echo("\n*** announce ***") + _announce() + + def get_package_version() -> version.Version: version_string = subprocess.check_output(["poetry", "version", "--short"]).decode( "utf-8" From 8edf3f66d50adc79d432e0868e6d21ed3b136bd0 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 5 Sep 2022 14:31:42 +0200 Subject: [PATCH 056/107] Reduce number of CI jobs run on PRs (#13713) * Reduce number of CI jobs run on PRs * Newsfile * Also limit sytest jobs * Fix typo * Fix up * Fixup --- .ci/scripts/calculate_jobs.py | 128 ++++++++++++++++++++++++++++++++++ .github/workflows/tests.yml | 97 ++++++++------------------ changelog.d/13713.misc | 1 + 3 files changed, 160 insertions(+), 66 deletions(-) create mode 100755 .ci/scripts/calculate_jobs.py create mode 100644 changelog.d/13713.misc diff --git a/.ci/scripts/calculate_jobs.py b/.ci/scripts/calculate_jobs.py new file mode 100755 index 0000000000..b1f604eeb0 --- /dev/null +++ b/.ci/scripts/calculate_jobs.py @@ -0,0 +1,128 @@ +#!/usr/bin/env python +# Copyright 2022 The Matrix.org Foundation C.I.C. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Calculate the trial jobs to run based on if we're in a PR or not. + +import json +import os + +IS_PR = os.environ["GITHUB_REF"].startswith("refs/pull/") + +# First calculate the various trial jobs. +# +# For each type of test we only run on Py3.7 on PRs + +trial_sqlite_tests = [ + { + "python-version": "3.7", + "database": "sqlite", + "extras": "all", + } +] + +if not IS_PR: + trial_sqlite_tests.extend( + { + "python-version": version, + "database": "sqlite", + "extras": "all", + } + for version in ("3.8", "3.9", "3.10") + ) + + +trial_postgres_tests = [ + { + "python-version": "3.7", + "database": "postgres", + "postgres-version": "10", + "extras": "all", + } +] + +if not IS_PR: + trial_postgres_tests.append( + { + "python-version": "3.10", + "database": "postgres", + "postgres-version": "14", + "extras": "all", + } + ) + +trial_no_extra_tests = [ + { + "python-version": "3.7", + "database": "sqlite", + "extras": "", + } +] + +print("::group::Calculated trial jobs") +print( + json.dumps( + trial_sqlite_tests + trial_postgres_tests + trial_no_extra_tests, indent=4 + ) +) +print("::endgroup::") + +test_matrix = json.dumps( + trial_sqlite_tests + trial_postgres_tests + trial_no_extra_tests +) +print(f"::set-output name=trial_test_matrix::{test_matrix}") + + +# First calculate the various sytest jobs. +# +# For each type of test we only run on focal on PRs + + +sytest_tests = [ + { + "sytest-tag": "focal", + }, + { + "sytest-tag": "focal", + "postgres": "postgres", + }, + { + "sytest-tag": "focal", + "postgres": "multi-postgres", + "workers": "workers", + }, +] + +if not IS_PR: + sytest_tests.extend( + [ + { + "sytest-tag": "testing", + "postgres": "postgres", + }, + { + "sytest-tag": "buster", + "postgres": "multi-postgres", + "workers": "workers", + }, + ] + ) + + +print("::group::Calculated sytest jobs") +print(json.dumps(sytest_tests, indent=4)) +print("::endgroup::") + +test_matrix = json.dumps(sytest_tests) +print(f"::set-output name=sytest_test_matrix::{test_matrix}") diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 720cb13907..3ce4ffb036 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -73,63 +73,48 @@ jobs: steps: - run: "true" - trial: + calculate-test-jobs: if: ${{ !cancelled() && !failure() }} # Allow previous steps to be skipped, but not fail needs: linting-done runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - uses: actions/setup-python@v2 + - id: get-matrix + run: .ci/scripts/calculate_jobs.py + outputs: + trial_test_matrix: ${{ steps.get-matrix.outputs.trial_test_matrix }} + sytest_test_matrix: ${{ steps.get-matrix.outputs.sytest_test_matrix }} + + trial: + if: ${{ !cancelled() && !failure() }} # Allow previous steps to be skipped, but not fail + needs: calculate-test-jobs + runs-on: ubuntu-latest strategy: matrix: - python-version: ["3.7", "3.8", "3.9", "3.10"] - database: ["sqlite"] - extras: ["all"] - is_pr: - - ${{ startsWith(github.ref, 'refs/pull/') }} - - # If we're a PR then we only test min and max python. - exclude: - - is_pr: true - python-version: 3.8 - - is_pr: true - python-version: 3.9 - - include: - # Newest Python without optional deps - - python-version: "3.10" - extras: "" - - # Oldest Python with PostgreSQL - - python-version: "3.7" - database: "postgres" - postgres-version: "10" - extras: "all" - - # Newest Python with newest PostgreSQL - - python-version: "3.10" - database: "postgres" - postgres-version: "14" - extras: "all" + job: ${{ fromJson(needs.calculate-test-jobs.outputs.trial_test_matrix) }} steps: - uses: actions/checkout@v2 - run: sudo apt-get -qq install xmlsec1 - - name: Set up PostgreSQL ${{ matrix.postgres-version }} - if: ${{ matrix.postgres-version }} + - name: Set up PostgreSQL ${{ matrix.job.postgres-version }} + if: ${{ matrix.job.postgres-version }} run: | docker run -d -p 5432:5432 \ -e POSTGRES_PASSWORD=postgres \ -e POSTGRES_INITDB_ARGS="--lc-collate C --lc-ctype C --encoding UTF8" \ - postgres:${{ matrix.postgres-version }} + postgres:${{ matrix.job.postgres-version }} - uses: matrix-org/setup-python-poetry@v1 with: - python-version: ${{ matrix.python-version }} - extras: ${{ matrix.extras }} + python-version: ${{ matrix.job.python-version }} + extras: ${{ matrix.job.extras }} - name: Await PostgreSQL - if: ${{ matrix.postgres-version }} + if: ${{ matrix.job.postgres-version }} timeout-minutes: 2 run: until pg_isready -h localhost; do sleep 1; done - run: poetry run trial --jobs=2 tests env: - SYNAPSE_POSTGRES: ${{ matrix.database == 'postgres' || '' }} + SYNAPSE_POSTGRES: ${{ matrix.job.database == 'postgres' || '' }} SYNAPSE_POSTGRES_HOST: localhost SYNAPSE_POSTGRES_USER: postgres SYNAPSE_POSTGRES_PASSWORD: postgres @@ -208,45 +193,25 @@ jobs: sytest: if: ${{ !failure() && !cancelled() }} - needs: linting-done + needs: calculate-test-jobs runs-on: ubuntu-latest container: - image: matrixdotorg/sytest-synapse:${{ matrix.sytest-tag }} + image: matrixdotorg/sytest-synapse:${{ matrix.job.sytest-tag }} volumes: - ${{ github.workspace }}:/src env: SYTEST_BRANCH: ${{ github.head_ref }} - POSTGRES: ${{ matrix.postgres && 1}} - MULTI_POSTGRES: ${{ (matrix.postgres == 'multi-postgres') && 1}} - WORKERS: ${{ matrix.workers && 1 }} - REDIS: ${{ matrix.redis && 1 }} - BLACKLIST: ${{ matrix.workers && 'synapse-blacklist-with-workers' }} + POSTGRES: ${{ matrix.job.postgres && 1}} + MULTI_POSTGRES: ${{ (matrix.job.postgres == 'multi-postgres') && 1}} + WORKERS: ${{ matrix.job.workers && 1 }} + REDIS: 1 + BLACKLIST: ${{ matrix.job.workers && 'synapse-blacklist-with-workers' }} TOP: ${{ github.workspace }} strategy: fail-fast: false matrix: - include: - - sytest-tag: focal - - - sytest-tag: focal - postgres: postgres - - - sytest-tag: testing - postgres: postgres - - - sytest-tag: focal - postgres: multi-postgres - workers: workers - - - sytest-tag: buster - postgres: multi-postgres - workers: workers - - - sytest-tag: buster - postgres: postgres - workers: workers - redis: redis + job: ${{ fromJson(needs.calculate-test-jobs.outputs.sytest_test_matrix) }} steps: - uses: actions/checkout@v2 @@ -262,7 +227,7 @@ jobs: uses: actions/upload-artifact@v2 if: ${{ always() }} with: - name: Sytest Logs - ${{ job.status }} - (${{ join(matrix.*, ', ') }}) + name: Sytest Logs - ${{ job.status }} - (${{ join(matrix.job.*, ', ') }}) path: | /logs/results.tap /logs/**/*.log* diff --git a/changelog.d/13713.misc b/changelog.d/13713.misc new file mode 100644 index 0000000000..1044099542 --- /dev/null +++ b/changelog.d/13713.misc @@ -0,0 +1 @@ +Reduce number of CI checks we run for PRs. From 32fc3b7ba4702a0068a82bdd0595e2f426967d4d Mon Sep 17 00:00:00 2001 From: Patrick Cloke Date: Tue, 6 Sep 2022 03:50:02 -0400 Subject: [PATCH 057/107] Remove configuration options for direct TCP replication. (#13647) Removes the ability to configure legacy direct TCP replication. Workers now require Redis to run. --- .github/workflows/tests.yml | 1 - changelog.d/13647.removal | 1 + docs/upgrade.md | 15 +++++ .../configuration/config_documentation.md | 2 - docs/workers.md | 22 ++----- synapse/app/homeserver.py | 11 ---- synapse/config/server.py | 16 ++++- synapse/config/workers.py | 8 ++- synapse/replication/tcp/handler.py | 58 +++++++------------ tests/app/test_openid_listener.py | 4 +- tests/test_server.py | 2 +- tests/utils.py | 1 - 12 files changed, 63 insertions(+), 78 deletions(-) create mode 100644 changelog.d/13647.removal diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 3ce4ffb036..bc1de2893c 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -204,7 +204,6 @@ jobs: POSTGRES: ${{ matrix.job.postgres && 1}} MULTI_POSTGRES: ${{ (matrix.job.postgres == 'multi-postgres') && 1}} WORKERS: ${{ matrix.job.workers && 1 }} - REDIS: 1 BLACKLIST: ${{ matrix.job.workers && 'synapse-blacklist-with-workers' }} TOP: ${{ github.workspace }} diff --git a/changelog.d/13647.removal b/changelog.d/13647.removal new file mode 100644 index 0000000000..0190a65dba --- /dev/null +++ b/changelog.d/13647.removal @@ -0,0 +1 @@ +Remove the ability to use direct TCP replication with workers. Direct TCP replication was deprecated in Synapse v1.18.0. Workers now require using Redis. diff --git a/docs/upgrade.md b/docs/upgrade.md index 422a3da664..c6219d06e8 100644 --- a/docs/upgrade.md +++ b/docs/upgrade.md @@ -91,6 +91,21 @@ process, for example: # Upgrading to v1.67.0 +## Direct TCP replication is no longer supported: migrate to Redis + +Redis support was added in v1.13.0 with it becoming the recommended method in +v1.18.0. It replaced the old direct TCP connections (which was deprecated as of +v1.18.0) to the main process. With Redis, rather than all the workers connecting +to the main process, all the workers and the main process connect to Redis, +which relays replication commands between processes. This can give a significant +CPU saving on the main process and is a prerequisite for upcoming +performance improvements. + +To migrate to Redis add the [`redis` config](./workers.md#shared-configuration), +and remove the TCP `replication` listener from config of the master and +`worker_replication_port` from worker config. Note that a HTTP listener with a +`replication` resource is still required. + ## Minimum version of Poetry is now v1.2.0 The minimum supported version of poetry is now 1.2. This should only affect diff --git a/docs/usage/configuration/config_documentation.md b/docs/usage/configuration/config_documentation.md index 396c560822..757957a1d5 100644 --- a/docs/usage/configuration/config_documentation.md +++ b/docs/usage/configuration/config_documentation.md @@ -431,8 +431,6 @@ Sub-options for each listener include: * `metrics`: (see the docs [here](../../metrics-howto.md)), - * `replication`: (deprecated as of Synapse 1.18, see the docs [here](../../workers.md)). - * `tls`: set to true to enable TLS for this listener. Will use the TLS key/cert specified in tls_private_key_path / tls_certificate_path. * `x_forwarded`: Only valid for an 'http' listener. Set to true to use the X-Forwarded-For header as the client IP. Useful when Synapse is diff --git a/docs/workers.md b/docs/workers.md index 176bb1475e..40b1852313 100644 --- a/docs/workers.md +++ b/docs/workers.md @@ -32,13 +32,8 @@ stream between all configured Synapse processes. Additionally, processes may make HTTP requests to each other, primarily for operations which need to wait for a reply ─ such as sending an event. -Redis support was added in v1.13.0 with it becoming the recommended method in -v1.18.0. It replaced the old direct TCP connections (which is deprecated as of -v1.18.0) to the main process. With Redis, rather than all the workers connecting -to the main process, all the workers and the main process connect to Redis, -which relays replication commands between processes. This can give a significant -cpu saving on the main process and will be a prerequisite for upcoming -performance improvements. +All the workers and the main process connect to Redis, which relays replication +commands between processes. If Redis support is enabled Synapse will use it as a shared cache, as well as a pub/sub mechanism. @@ -330,7 +325,6 @@ effects of bursts of events from that bridge on events sent by normal users. Additionally, the writing of specific streams (such as events) can be moved off of the main process to a particular worker. -(This is only supported with Redis-based replication.) To enable this, the worker must have a HTTP replication listener configured, have a `worker_name` and be listed in the `instance_map` config. The same worker @@ -600,15 +594,9 @@ equivalent to `synapse.app.generic_worker`: ## Migration from old config -There are two main independent changes that have been made: introducing Redis -support and merging apps into `synapse.app.generic_worker`. Both these changes -are backwards compatible and so no changes to the config are required, however -server admins are encouraged to plan to migrate to Redis as the old style direct -TCP replication config is deprecated. - -To migrate to Redis add the `redis` config as above, and optionally remove the -TCP `replication` listener from master and `worker_replication_port` from worker -config. +A main change that has occurred is the merging of worker apps into +`synapse.app.generic_worker`. This change is backwards compatible and so no +changes to the config are required. To migrate apps to use `synapse.app.generic_worker` simply update the `worker_app` option in the worker configs, and where worker are started (e.g. diff --git a/synapse/app/homeserver.py b/synapse/app/homeserver.py index e57a926032..883f2fd2ec 100644 --- a/synapse/app/homeserver.py +++ b/synapse/app/homeserver.py @@ -57,7 +57,6 @@ from synapse.http.site import SynapseSite from synapse.logging.context import LoggingContext from synapse.metrics import METRICS_PREFIX, MetricsResource, RegistryProxy from synapse.replication.http import REPLICATION_PREFIX, ReplicationRestResource -from synapse.replication.tcp.resource import ReplicationStreamProtocolFactory from synapse.rest import ClientRestResource from synapse.rest.admin import AdminRestResource from synapse.rest.health import HealthResource @@ -290,16 +289,6 @@ class SynapseHomeServer(HomeServer): manhole_settings=self.config.server.manhole_settings, manhole_globals={"hs": self}, ) - elif listener.type == "replication": - services = listen_tcp( - listener.bind_addresses, - listener.port, - ReplicationStreamProtocolFactory(self), - ) - for s in services: - self.get_reactor().addSystemEventTrigger( - "before", "shutdown", s.stopListening - ) elif listener.type == "metrics": if not self.config.metrics.enable_metrics: logger.warning( diff --git a/synapse/config/server.py b/synapse/config/server.py index 085fe22c51..c91df636d9 100644 --- a/synapse/config/server.py +++ b/synapse/config/server.py @@ -36,6 +36,12 @@ from ._util import validate_config logger = logging.Logger(__name__) +DIRECT_TCP_ERROR = """ +Using direct TCP replication for workers is no longer supported. + +Please see https://matrix-org.github.io/synapse/latest/upgrade.html#direct-tcp-replication-is-no-longer-supported-migrate-to-redis +""" + # by default, we attempt to listen on both '::' *and* '0.0.0.0' because some OSes # (Windows, macOS, other BSD/Linux where net.ipv6.bindv6only is set) will only listen # on IPv6 when '::' is set. @@ -165,7 +171,6 @@ KNOWN_LISTENER_TYPES = { "http", "metrics", "manhole", - "replication", } KNOWN_RESOURCES = { @@ -515,7 +520,9 @@ class ServerConfig(Config): ): raise ConfigError("allowed_avatar_mimetypes must be a list") - self.listeners = [parse_listener_def(x) for x in config.get("listeners", [])] + self.listeners = [ + parse_listener_def(i, x) for i, x in enumerate(config.get("listeners", [])) + ] # no_tls is not really supported any more, but let's grandfather it in # here. @@ -880,9 +887,12 @@ def read_gc_thresholds( ) -def parse_listener_def(listener: Any) -> ListenerConfig: +def parse_listener_def(num: int, listener: Any) -> ListenerConfig: """parse a listener config from the config file""" listener_type = listener["type"] + # Raise a helpful error if direct TCP replication is still configured. + if listener_type == "replication": + raise ConfigError(DIRECT_TCP_ERROR, ("listeners", str(num), "type")) port = listener.get("port") if not isinstance(port, int): diff --git a/synapse/config/workers.py b/synapse/config/workers.py index f2716422b5..0fb725dd8f 100644 --- a/synapse/config/workers.py +++ b/synapse/config/workers.py @@ -27,7 +27,7 @@ from ._base import ( RoutableShardedWorkerHandlingConfig, ShardedWorkerHandlingConfig, ) -from .server import ListenerConfig, parse_listener_def +from .server import DIRECT_TCP_ERROR, ListenerConfig, parse_listener_def _FEDERATION_SENDER_WITH_SEND_FEDERATION_ENABLED_ERROR = """ The send_federation config option must be disabled in the main @@ -128,7 +128,8 @@ class WorkerConfig(Config): self.worker_app = None self.worker_listeners = [ - parse_listener_def(x) for x in config.get("worker_listeners", []) + parse_listener_def(i, x) + for i, x in enumerate(config.get("worker_listeners", [])) ] self.worker_daemonize = bool(config.get("worker_daemonize")) self.worker_pid_file = config.get("worker_pid_file") @@ -142,7 +143,8 @@ class WorkerConfig(Config): self.worker_replication_host = config.get("worker_replication_host", None) # The port on the main synapse for TCP replication - self.worker_replication_port = config.get("worker_replication_port", None) + if "worker_replication_port" in config: + raise ConfigError(DIRECT_TCP_ERROR, ("worker_replication_port",)) # The port on the main synapse for HTTP replication endpoint self.worker_replication_http_port = config.get("worker_replication_http_port") diff --git a/synapse/replication/tcp/handler.py b/synapse/replication/tcp/handler.py index e1cbfa50eb..0f166d16aa 100644 --- a/synapse/replication/tcp/handler.py +++ b/synapse/replication/tcp/handler.py @@ -35,7 +35,6 @@ from twisted.internet.protocol import ReconnectingClientFactory from synapse.metrics import LaterGauge from synapse.metrics.background_process_metrics import run_as_background_process -from synapse.replication.tcp.client import DirectTcpReplicationClientFactory from synapse.replication.tcp.commands import ( ClearUserSyncsCommand, Command, @@ -332,46 +331,31 @@ class ReplicationCommandHandler: def start_replication(self, hs: "HomeServer") -> None: """Helper method to start replication.""" - if hs.config.redis.redis_enabled: - from synapse.replication.tcp.redis import ( - RedisDirectTcpReplicationClientFactory, - ) + from synapse.replication.tcp.redis import RedisDirectTcpReplicationClientFactory - # First let's ensure that we have a ReplicationStreamer started. - hs.get_replication_streamer() + # First let's ensure that we have a ReplicationStreamer started. + hs.get_replication_streamer() - # We need two connections to redis, one for the subscription stream and - # one to send commands to (as you can't send further redis commands to a - # connection after SUBSCRIBE is called). + # We need two connections to redis, one for the subscription stream and + # one to send commands to (as you can't send further redis commands to a + # connection after SUBSCRIBE is called). - # First create the connection for sending commands. - outbound_redis_connection = hs.get_outbound_redis_connection() + # First create the connection for sending commands. + outbound_redis_connection = hs.get_outbound_redis_connection() - # Now create the factory/connection for the subscription stream. - self._factory = RedisDirectTcpReplicationClientFactory( - hs, - outbound_redis_connection, - channel_names=self._channels_to_subscribe_to, - ) - hs.get_reactor().connectTCP( - hs.config.redis.redis_host, - hs.config.redis.redis_port, - self._factory, - timeout=30, - bindAddress=None, - ) - else: - client_name = hs.get_instance_name() - self._factory = DirectTcpReplicationClientFactory(hs, client_name, self) - host = hs.config.worker.worker_replication_host - port = hs.config.worker.worker_replication_port - hs.get_reactor().connectTCP( - host, - port, - self._factory, - timeout=30, - bindAddress=None, - ) + # Now create the factory/connection for the subscription stream. + self._factory = RedisDirectTcpReplicationClientFactory( + hs, + outbound_redis_connection, + channel_names=self._channels_to_subscribe_to, + ) + hs.get_reactor().connectTCP( + hs.config.redis.redis_host, + hs.config.redis.redis_port, + self._factory, + timeout=30, + bindAddress=None, + ) def get_streams(self) -> Dict[str, Stream]: """Get a map from stream name to all streams.""" diff --git a/tests/app/test_openid_listener.py b/tests/app/test_openid_listener.py index 264e101082..c7dae58eb5 100644 --- a/tests/app/test_openid_listener.py +++ b/tests/app/test_openid_listener.py @@ -61,7 +61,7 @@ class FederationReaderOpenIDListenerTests(HomeserverTestCase): } # Listen with the config - self.hs._listen_http(parse_listener_def(config)) + self.hs._listen_http(parse_listener_def(0, config)) # Grab the resource from the site that was told to listen site = self.reactor.tcpServers[0][1] @@ -109,7 +109,7 @@ class SynapseHomeserverOpenIDListenerTests(HomeserverTestCase): } # Listen with the config - self.hs._listener_http(self.hs.config, parse_listener_def(config)) + self.hs._listener_http(self.hs.config, parse_listener_def(0, config)) # Grab the resource from the site that was told to listen site = self.reactor.tcpServers[0][1] diff --git a/tests/test_server.py b/tests/test_server.py index 23975d59c3..7c66448245 100644 --- a/tests/test_server.py +++ b/tests/test_server.py @@ -228,7 +228,7 @@ class OptionsResourceTests(unittest.TestCase): site = SynapseSite( "test", "site_tag", - parse_listener_def({"type": "http", "port": 0}), + parse_listener_def(0, {"type": "http", "port": 0}), self.resource, "1.0", max_request_body_size=4096, diff --git a/tests/utils.py b/tests/utils.py index d2c6d1e852..65db437697 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -135,7 +135,6 @@ def default_config( "enable_registration_captcha": False, "macaroon_secret_key": "not even a little secret", "password_providers": [], - "worker_replication_url": "", "worker_app": None, "block_non_admin_invites": False, "federation_domain_whitelist": None, From cdf7fb737b12cc06cc5693806ee8fa21ade9723d Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 6 Sep 2022 09:01:26 +0100 Subject: [PATCH 058/107] 1.67.0rc1 --- CHANGES.md | 84 +++++++++++++++++++++++++++++++++++++++ changelog.d/13241.removal | 1 - changelog.d/13483.misc | 1 - changelog.d/13509.bugfix | 1 - changelog.d/13540.misc | 1 - changelog.d/13546.bugfix | 1 - changelog.d/13569.removal | 1 - changelog.d/13573.misc | 1 - changelog.d/13575.misc | 1 - changelog.d/13583.bugfix | 1 - changelog.d/13585.bugfix | 1 - changelog.d/13586.misc | 1 - changelog.d/13588.misc | 1 - changelog.d/13591.misc | 1 - changelog.d/13592.misc | 1 - changelog.d/13597.misc | 1 - changelog.d/13600.misc | 1 - changelog.d/13602.doc | 1 - changelog.d/13603.misc | 1 - changelog.d/13605.misc | 1 - changelog.d/13606.misc | 1 - changelog.d/13608.misc | 1 - changelog.d/13614.feature | 1 - changelog.d/13615.feature | 1 - changelog.d/13616.bugfix | 1 - changelog.d/13617.doc | 1 - changelog.d/13627.misc | 1 - changelog.d/13632.bugfix | 1 - changelog.d/13634.feature | 1 - changelog.d/13639.misc | 1 - changelog.d/13640.doc | 1 - changelog.d/13645.doc | 1 - changelog.d/13647.removal | 1 - changelog.d/13653.removal | 1 - changelog.d/13656.doc | 1 - changelog.d/13657.bugfix | 1 - changelog.d/13658.bugfix | 1 - changelog.d/13660.bugfix | 1 - changelog.d/13662.misc | 1 - changelog.d/13665.doc | 1 - changelog.d/13671.misc | 1 - changelog.d/13678.doc | 1 - changelog.d/13679.misc | 1 - changelog.d/13683.bugfix | 1 - changelog.d/13688.docker | 1 - changelog.d/13689.misc | 1 - changelog.d/13692.removal | 1 - changelog.d/13693.misc | 1 - changelog.d/13694.bugfix | 1 - changelog.d/13697.misc | 1 - changelog.d/13698.misc | 1 - changelog.d/13701.doc | 1 - changelog.d/13712.misc | 1 - changelog.d/13713.misc | 1 - debian/changelog | 8 +++- pyproject.toml | 2 +- 56 files changed, 91 insertions(+), 56 deletions(-) delete mode 100644 changelog.d/13241.removal delete mode 100644 changelog.d/13483.misc delete mode 100644 changelog.d/13509.bugfix delete mode 100644 changelog.d/13540.misc delete mode 100644 changelog.d/13546.bugfix delete mode 100644 changelog.d/13569.removal delete mode 100644 changelog.d/13573.misc delete mode 100644 changelog.d/13575.misc delete mode 100644 changelog.d/13583.bugfix delete mode 100644 changelog.d/13585.bugfix delete mode 100644 changelog.d/13586.misc delete mode 100644 changelog.d/13588.misc delete mode 100644 changelog.d/13591.misc delete mode 100644 changelog.d/13592.misc delete mode 100644 changelog.d/13597.misc delete mode 100644 changelog.d/13600.misc delete mode 100644 changelog.d/13602.doc delete mode 100644 changelog.d/13603.misc delete mode 100644 changelog.d/13605.misc delete mode 100644 changelog.d/13606.misc delete mode 100644 changelog.d/13608.misc delete mode 100644 changelog.d/13614.feature delete mode 100644 changelog.d/13615.feature delete mode 100644 changelog.d/13616.bugfix delete mode 100644 changelog.d/13617.doc delete mode 100644 changelog.d/13627.misc delete mode 100644 changelog.d/13632.bugfix delete mode 100644 changelog.d/13634.feature delete mode 100644 changelog.d/13639.misc delete mode 100644 changelog.d/13640.doc delete mode 100644 changelog.d/13645.doc delete mode 100644 changelog.d/13647.removal delete mode 100644 changelog.d/13653.removal delete mode 100644 changelog.d/13656.doc delete mode 100644 changelog.d/13657.bugfix delete mode 100644 changelog.d/13658.bugfix delete mode 100644 changelog.d/13660.bugfix delete mode 100644 changelog.d/13662.misc delete mode 100644 changelog.d/13665.doc delete mode 100644 changelog.d/13671.misc delete mode 100644 changelog.d/13678.doc delete mode 100644 changelog.d/13679.misc delete mode 100644 changelog.d/13683.bugfix delete mode 100644 changelog.d/13688.docker delete mode 100644 changelog.d/13689.misc delete mode 100644 changelog.d/13692.removal delete mode 100644 changelog.d/13693.misc delete mode 100644 changelog.d/13694.bugfix delete mode 100644 changelog.d/13697.misc delete mode 100644 changelog.d/13698.misc delete mode 100644 changelog.d/13701.doc delete mode 100644 changelog.d/13712.misc delete mode 100644 changelog.d/13713.misc diff --git a/CHANGES.md b/CHANGES.md index 0b10e90186..3bb00b6cea 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,3 +1,87 @@ +Synapse 1.67.0rc1 (2022-09-06) +============================== + +Features +-------- + +- Support setting the registration shared secret in a file, via a new `registration_shared_secret_path` configuration option. ([\#13614](https://github.com/matrix-org/synapse/issues/13614)) +- Change the default startup behaviour so that any missing "additional" configuration files (signing key, etc) are generated automatically. ([\#13615](https://github.com/matrix-org/synapse/issues/13615)) +- Improve performance of sending messages in rooms with thousands of local users. ([\#13634](https://github.com/matrix-org/synapse/issues/13634)) + + +Bugfixes +-------- + +- Fix a bug introduced in Synapse 1.13 where the [List Rooms admin API](https://matrix-org.github.io/synapse/develop/admin_api/rooms.html#list-room-api) would return integers instead of booleans for the `federatable` and `public` fields when using a Sqlite database. ([\#13509](https://github.com/matrix-org/synapse/issues/13509)) +- Fix bug that user cannot `/forget` rooms after the last member has left the room. ([\#13546](https://github.com/matrix-org/synapse/issues/13546)) +- Faster Room Joins: fix `/make_knock` blocking indefinitely when the room in question is a partial-stated room. ([\#13583](https://github.com/matrix-org/synapse/issues/13583)) +- Fix loading the current stream position behind the actual position. ([\#13585](https://github.com/matrix-org/synapse/issues/13585)) +- Fix a longstanding bug in `register_new_matrix_user` which meant it was always necessary to explicitly give a server URL. ([\#13616](https://github.com/matrix-org/synapse/issues/13616)) +- Fix the running of MSC1763 retention purge_jobs in deployments with background jobs running on a worker by forcing them back onto the main worker. Contributed by Brad @ Beeper. ([\#13632](https://github.com/matrix-org/synapse/issues/13632)) +- Fix a long-standing bug that downloaded media for URL previews was not deleted while database background updates were running. ([\#13657](https://github.com/matrix-org/synapse/issues/13657)) +- Fix MSC3030 `/timestamp_to_event` endpoint to return the correct next event when the events have the same timestamp. ([\#13658](https://github.com/matrix-org/synapse/issues/13658)) +- Fix bug where we wedge media plugins if clients disconnect early. Introduced in v1.22.0. ([\#13660](https://github.com/matrix-org/synapse/issues/13660)) +- Fix a long-standing bug which meant that keys for unwhitelisted servers were not returned by `/_matrix/key/v2/query`. ([\#13683](https://github.com/matrix-org/synapse/issues/13683)) +- Fix a bug introduced in Synapse v1.20.0 that would cause the unstable unread counts from [MSC2654](https://github.com/matrix-org/matrix-spec-proposals/pull/2654) to be calculated even if the feature is disabled. ([\#13694](https://github.com/matrix-org/synapse/issues/13694)) + + +Updates to the Docker image +--------------------------- + +- Update docker image to use a stable version of poetry. ([\#13688](https://github.com/matrix-org/synapse/issues/13688)) + + +Improved Documentation +---------------------- + +- Improve the description of the ["chain cover index"](https://matrix-org.github.io/synapse/latest/auth_chain_difference_algorithm.html) used internally by Synapse. ([\#13602](https://github.com/matrix-org/synapse/issues/13602)) +- Document how ["monthly active users"](https://matrix-org.github.io/synapse/latest/usage/administration/monthly_active_users.html) is calculated and used. ([\#13617](https://github.com/matrix-org/synapse/issues/13617)) +- Improve documentation around user registration. ([\#13640](https://github.com/matrix-org/synapse/issues/13640)) +- Remove documentation of legacy `frontend_proxy` worker app. ([\#13645](https://github.com/matrix-org/synapse/issues/13645)) +- Clarify documentation that HTTP replication traffic can be protected with a shared secret. ([\#13656](https://github.com/matrix-org/synapse/issues/13656)) +- Remove unintentional colons from [config manual](https://matrix-org.github.io/synapse/latest/usage/configuration/config_documentation.html) headers. ([\#13665](https://github.com/matrix-org/synapse/issues/13665)) +- Update docs to make enabling metrics more clear. ([\#13678](https://github.com/matrix-org/synapse/issues/13678)) +- Clarify `(room_id, event_id)` global uniqueness and how we should scope our database schemas. ([\#13701](https://github.com/matrix-org/synapse/issues/13701)) + + +Deprecations and Removals +------------------------- + +- Drop support for calling `/_matrix/client/v3/rooms/{roomId}/invite` without an `id_access_token`, which was not permitted by the spec. Contributed by @Vetchu. ([\#13241](https://github.com/matrix-org/synapse/issues/13241)) +- Remove redundant `_get_joined_users_from_context` cache. Contributed by Nick @ Beeper (@fizzadar). ([\#13569](https://github.com/matrix-org/synapse/issues/13569)) +- Remove the ability to use direct TCP replication with workers. Direct TCP replication was deprecated in Synapse v1.18.0. Workers now require using Redis. ([\#13647](https://github.com/matrix-org/synapse/issues/13647)) +- Remove support for unstable [private read receipts](https://github.com/matrix-org/matrix-spec-proposals/pull/2285). ([\#13653](https://github.com/matrix-org/synapse/issues/13653), [\#13692](https://github.com/matrix-org/synapse/issues/13692)) + + +Internal Changes +---------------- + +- Extend the release script to wait for GitHub Actions to finish and to be usable as a guide for the whole process. ([\#13483](https://github.com/matrix-org/synapse/issues/13483)) +- Add experimental configuration option to allow disabling legacy Prometheus metric names. ([\#13540](https://github.com/matrix-org/synapse/issues/13540)) +- Cache user IDs instead of profiles to reduce cache memory usage. Contributed by Nick @ Beeper (@fizzadar). ([\#13573](https://github.com/matrix-org/synapse/issues/13573), [\#13600](https://github.com/matrix-org/synapse/issues/13600)) +- Optimize how Synapse calculates domains to fetch from during backfill. ([\#13575](https://github.com/matrix-org/synapse/issues/13575)) +- Comment about a better future where we can get the state diff between two events. ([\#13586](https://github.com/matrix-org/synapse/issues/13586)) +- Instrument `_check_sigs_and_hash_and_fetch` to trace time spent in child concurrent calls for understandable traces in Jaeger. ([\#13588](https://github.com/matrix-org/synapse/issues/13588)) +- Improve performance of `@cachedList`. ([\#13591](https://github.com/matrix-org/synapse/issues/13591)) +- Minor speed up of fetching large numbers of push rules. ([\#13592](https://github.com/matrix-org/synapse/issues/13592)) +- Optimise push action fetching queries. Contributed by Nick @ Beeper (@fizzadar). ([\#13597](https://github.com/matrix-org/synapse/issues/13597)) +- Rename `event_map` to `unpersisted_events` when computing the auth differences. ([\#13603](https://github.com/matrix-org/synapse/issues/13603)) +- Refactor `get_users_in_room(room_id)` mis-use with dedicated `get_current_hosts_in_room(room_id)` function. ([\#13605](https://github.com/matrix-org/synapse/issues/13605)) +- Use dedicated `get_local_users_in_room(room_id)` function to find local users when calculating `join_authorised_via_users_server` of a `/make_join` request. ([\#13606](https://github.com/matrix-org/synapse/issues/13606)) +- Refactor `get_users_in_room(room_id)` mis-use to lookup single local user with dedicated `check_local_user_in_room(...)` function. ([\#13608](https://github.com/matrix-org/synapse/issues/13608)) +- Drop unused column `application_services_state.last_txn`. ([\#13627](https://github.com/matrix-org/synapse/issues/13627)) +- Improve readability of Complement CI logs by printing failure results last. ([\#13639](https://github.com/matrix-org/synapse/issues/13639)) +- Generalise the `@cancellable` annotation so it can be used on functions other than just servlet methods. ([\#13662](https://github.com/matrix-org/synapse/issues/13662)) +- Introduce a `CommonUsageMetrics` class to share some usage metrics between the Prometheus exporter and the phone home stats. ([\#13671](https://github.com/matrix-org/synapse/issues/13671)) +- Add some logging to help track down #13444. ([\#13679](https://github.com/matrix-org/synapse/issues/13679)) +- Update poetry lock file for v1.2.0. ([\#13689](https://github.com/matrix-org/synapse/issues/13689)) +- Add cache to `is_partial_state_room`. ([\#13693](https://github.com/matrix-org/synapse/issues/13693)) +- Update the Grafana dashboard that is included with Synapse in the `contrib` directory. ([\#13697](https://github.com/matrix-org/synapse/issues/13697)) +- Only run trial CI on all python versions on non-PRs. ([\#13698](https://github.com/matrix-org/synapse/issues/13698)) +- Fix typechecking with latest types-jsonschema. ([\#13712](https://github.com/matrix-org/synapse/issues/13712)) +- Reduce number of CI checks we run for PRs. ([\#13713](https://github.com/matrix-org/synapse/issues/13713)) + + Synapse 1.66.0 (2022-08-31) =========================== diff --git a/changelog.d/13241.removal b/changelog.d/13241.removal deleted file mode 100644 index 60b0e7969c..0000000000 --- a/changelog.d/13241.removal +++ /dev/null @@ -1 +0,0 @@ -Drop support for calling `/_matrix/client/v3/rooms/{roomId}/invite` without an `id_access_token`, which was not permitted by the spec. Contributed by @Vetchu. \ No newline at end of file diff --git a/changelog.d/13483.misc b/changelog.d/13483.misc deleted file mode 100644 index 4fe6dbbea4..0000000000 --- a/changelog.d/13483.misc +++ /dev/null @@ -1 +0,0 @@ -Extend the release script to wait for GitHub Actions to finish and to be usable as a guide for the whole process. \ No newline at end of file diff --git a/changelog.d/13509.bugfix b/changelog.d/13509.bugfix deleted file mode 100644 index 6dcb9741d9..0000000000 --- a/changelog.d/13509.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix a bug introduced in Synapse 1.13 where the [List Rooms admin API](https://matrix-org.github.io/synapse/develop/admin_api/rooms.html#list-room-api) would return integers instead of booleans for the `federatable` and `public` fields when using a Sqlite database. diff --git a/changelog.d/13540.misc b/changelog.d/13540.misc deleted file mode 100644 index 07ace50b12..0000000000 --- a/changelog.d/13540.misc +++ /dev/null @@ -1 +0,0 @@ -Add experimental configuration option to allow disabling legacy Prometheus metric names. \ No newline at end of file diff --git a/changelog.d/13546.bugfix b/changelog.d/13546.bugfix deleted file mode 100644 index 83bc3a61d2..0000000000 --- a/changelog.d/13546.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix bug that user cannot `/forget` rooms after the last member has left the room. \ No newline at end of file diff --git a/changelog.d/13569.removal b/changelog.d/13569.removal deleted file mode 100644 index af9d407671..0000000000 --- a/changelog.d/13569.removal +++ /dev/null @@ -1 +0,0 @@ -Remove redundant `_get_joined_users_from_context` cache. Contributed by Nick @ Beeper (@fizzadar). diff --git a/changelog.d/13573.misc b/changelog.d/13573.misc deleted file mode 100644 index 1ce9c0c081..0000000000 --- a/changelog.d/13573.misc +++ /dev/null @@ -1 +0,0 @@ -Cache user IDs instead of profiles to reduce cache memory usage. Contributed by Nick @ Beeper (@fizzadar). diff --git a/changelog.d/13575.misc b/changelog.d/13575.misc deleted file mode 100644 index 3841472617..0000000000 --- a/changelog.d/13575.misc +++ /dev/null @@ -1 +0,0 @@ -Optimize how Synapse calculates domains to fetch from during backfill. diff --git a/changelog.d/13583.bugfix b/changelog.d/13583.bugfix deleted file mode 100644 index 1e4ce5904b..0000000000 --- a/changelog.d/13583.bugfix +++ /dev/null @@ -1 +0,0 @@ -Faster Room Joins: fix `/make_knock` blocking indefinitely when the room in question is a partial-stated room. \ No newline at end of file diff --git a/changelog.d/13585.bugfix b/changelog.d/13585.bugfix deleted file mode 100644 index 664b986c59..0000000000 --- a/changelog.d/13585.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix loading the current stream position behind the actual position. diff --git a/changelog.d/13586.misc b/changelog.d/13586.misc deleted file mode 100644 index daa71470f6..0000000000 --- a/changelog.d/13586.misc +++ /dev/null @@ -1 +0,0 @@ -Comment about a better future where we can get the state diff between two events. diff --git a/changelog.d/13588.misc b/changelog.d/13588.misc deleted file mode 100644 index eca1416ceb..0000000000 --- a/changelog.d/13588.misc +++ /dev/null @@ -1 +0,0 @@ -Instrument `_check_sigs_and_hash_and_fetch` to trace time spent in child concurrent calls for understandable traces in Jaeger. diff --git a/changelog.d/13591.misc b/changelog.d/13591.misc deleted file mode 100644 index 080e865e55..0000000000 --- a/changelog.d/13591.misc +++ /dev/null @@ -1 +0,0 @@ -Improve performance of `@cachedList`. diff --git a/changelog.d/13592.misc b/changelog.d/13592.misc deleted file mode 100644 index 8f48d557e5..0000000000 --- a/changelog.d/13592.misc +++ /dev/null @@ -1 +0,0 @@ -Minor speed up of fetching large numbers of push rules. diff --git a/changelog.d/13597.misc b/changelog.d/13597.misc deleted file mode 100644 index eb5e971008..0000000000 --- a/changelog.d/13597.misc +++ /dev/null @@ -1 +0,0 @@ - Optimise push action fetching queries. Contributed by Nick @ Beeper (@fizzadar). diff --git a/changelog.d/13600.misc b/changelog.d/13600.misc deleted file mode 100644 index 1ce9c0c081..0000000000 --- a/changelog.d/13600.misc +++ /dev/null @@ -1 +0,0 @@ -Cache user IDs instead of profiles to reduce cache memory usage. Contributed by Nick @ Beeper (@fizzadar). diff --git a/changelog.d/13602.doc b/changelog.d/13602.doc deleted file mode 100644 index dbba082163..0000000000 --- a/changelog.d/13602.doc +++ /dev/null @@ -1 +0,0 @@ -Improve the description of the ["chain cover index"](https://matrix-org.github.io/synapse/latest/auth_chain_difference_algorithm.html) used internally by Synapse. diff --git a/changelog.d/13603.misc b/changelog.d/13603.misc deleted file mode 100644 index d08eb6cc0a..0000000000 --- a/changelog.d/13603.misc +++ /dev/null @@ -1 +0,0 @@ -Rename `event_map` to `unpersisted_events` when computing the auth differences. diff --git a/changelog.d/13605.misc b/changelog.d/13605.misc deleted file mode 100644 index 88d518383b..0000000000 --- a/changelog.d/13605.misc +++ /dev/null @@ -1 +0,0 @@ -Refactor `get_users_in_room(room_id)` mis-use with dedicated `get_current_hosts_in_room(room_id)` function. diff --git a/changelog.d/13606.misc b/changelog.d/13606.misc deleted file mode 100644 index 58a4467798..0000000000 --- a/changelog.d/13606.misc +++ /dev/null @@ -1 +0,0 @@ -Use dedicated `get_local_users_in_room(room_id)` function to find local users when calculating `join_authorised_via_users_server` of a `/make_join` request. diff --git a/changelog.d/13608.misc b/changelog.d/13608.misc deleted file mode 100644 index 19bcc45e33..0000000000 --- a/changelog.d/13608.misc +++ /dev/null @@ -1 +0,0 @@ -Refactor `get_users_in_room(room_id)` mis-use to lookup single local user with dedicated `check_local_user_in_room(...)` function. diff --git a/changelog.d/13614.feature b/changelog.d/13614.feature deleted file mode 100644 index fa177ead09..0000000000 --- a/changelog.d/13614.feature +++ /dev/null @@ -1 +0,0 @@ -Support setting the registration shared secret in a file, via a new `registration_shared_secret_path` configuration option. diff --git a/changelog.d/13615.feature b/changelog.d/13615.feature deleted file mode 100644 index c2c568f1eb..0000000000 --- a/changelog.d/13615.feature +++ /dev/null @@ -1 +0,0 @@ -Change the default startup behaviour so that any missing "additional" configuration files (signing key, etc) are generated automatically. diff --git a/changelog.d/13616.bugfix b/changelog.d/13616.bugfix deleted file mode 100644 index f2c48d1d8d..0000000000 --- a/changelog.d/13616.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix a longstanding bug in `register_new_matrix_user` which meant it was always necessary to explicitly give a server URL. diff --git a/changelog.d/13617.doc b/changelog.d/13617.doc deleted file mode 100644 index 5c7db7c3d7..0000000000 --- a/changelog.d/13617.doc +++ /dev/null @@ -1 +0,0 @@ -Document how ["monthly active users"](https://matrix-org.github.io/synapse/latest/usage/administration/monthly_active_users.html) is calculated and used. diff --git a/changelog.d/13627.misc b/changelog.d/13627.misc deleted file mode 100644 index 1a15709aff..0000000000 --- a/changelog.d/13627.misc +++ /dev/null @@ -1 +0,0 @@ -Drop unused column `application_services_state.last_txn`. diff --git a/changelog.d/13632.bugfix b/changelog.d/13632.bugfix deleted file mode 100644 index e4b7b403cd..0000000000 --- a/changelog.d/13632.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix the running of MSC1763 retention purge_jobs in deployments with background jobs running on a worker by forcing them back onto the main worker. Contributed by Brad @ Beeper. diff --git a/changelog.d/13634.feature b/changelog.d/13634.feature deleted file mode 100644 index 0a8827205d..0000000000 --- a/changelog.d/13634.feature +++ /dev/null @@ -1 +0,0 @@ -Improve performance of sending messages in rooms with thousands of local users. diff --git a/changelog.d/13639.misc b/changelog.d/13639.misc deleted file mode 100644 index de4e4d1206..0000000000 --- a/changelog.d/13639.misc +++ /dev/null @@ -1 +0,0 @@ -Improve readability of Complement CI logs by printing failure results last. diff --git a/changelog.d/13640.doc b/changelog.d/13640.doc deleted file mode 100644 index fa049371cf..0000000000 --- a/changelog.d/13640.doc +++ /dev/null @@ -1 +0,0 @@ -Improve documentation around user registration. diff --git a/changelog.d/13645.doc b/changelog.d/13645.doc deleted file mode 100644 index 04c302ec2c..0000000000 --- a/changelog.d/13645.doc +++ /dev/null @@ -1 +0,0 @@ -Remove documentation of legacy `frontend_proxy` worker app. diff --git a/changelog.d/13647.removal b/changelog.d/13647.removal deleted file mode 100644 index 0190a65dba..0000000000 --- a/changelog.d/13647.removal +++ /dev/null @@ -1 +0,0 @@ -Remove the ability to use direct TCP replication with workers. Direct TCP replication was deprecated in Synapse v1.18.0. Workers now require using Redis. diff --git a/changelog.d/13653.removal b/changelog.d/13653.removal deleted file mode 100644 index eb075d4517..0000000000 --- a/changelog.d/13653.removal +++ /dev/null @@ -1 +0,0 @@ -Remove support for unstable [private read receipts](https://github.com/matrix-org/matrix-spec-proposals/pull/2285). diff --git a/changelog.d/13656.doc b/changelog.d/13656.doc deleted file mode 100644 index 61013a0daf..0000000000 --- a/changelog.d/13656.doc +++ /dev/null @@ -1 +0,0 @@ -Clarify documentation that HTTP replication traffic can be protected with a shared secret. diff --git a/changelog.d/13657.bugfix b/changelog.d/13657.bugfix deleted file mode 100644 index d314d9c52f..0000000000 --- a/changelog.d/13657.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix a long-standing bug that downloaded media for URL previews was not deleted while database background updates were running. diff --git a/changelog.d/13658.bugfix b/changelog.d/13658.bugfix deleted file mode 100644 index 8740f066bb..0000000000 --- a/changelog.d/13658.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix MSC3030 `/timestamp_to_event` endpoint to return the correct next event when the events have the same timestamp. diff --git a/changelog.d/13660.bugfix b/changelog.d/13660.bugfix deleted file mode 100644 index 43859a4d65..0000000000 --- a/changelog.d/13660.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix bug where we wedge media plugins if clients disconnect early. Introduced in v1.22.0. diff --git a/changelog.d/13662.misc b/changelog.d/13662.misc deleted file mode 100644 index 3dea4a1c2c..0000000000 --- a/changelog.d/13662.misc +++ /dev/null @@ -1 +0,0 @@ -Generalise the `@cancellable` annotation so it can be used on functions other than just servlet methods. \ No newline at end of file diff --git a/changelog.d/13665.doc b/changelog.d/13665.doc deleted file mode 100644 index 6ee6434662..0000000000 --- a/changelog.d/13665.doc +++ /dev/null @@ -1 +0,0 @@ -Remove unintentional colons from [config manual](https://matrix-org.github.io/synapse/latest/usage/configuration/config_documentation.html) headers. diff --git a/changelog.d/13671.misc b/changelog.d/13671.misc deleted file mode 100644 index f1c62b5b1e..0000000000 --- a/changelog.d/13671.misc +++ /dev/null @@ -1 +0,0 @@ -Introduce a `CommonUsageMetrics` class to share some usage metrics between the Prometheus exporter and the phone home stats. diff --git a/changelog.d/13678.doc b/changelog.d/13678.doc deleted file mode 100644 index 8b85dfe643..0000000000 --- a/changelog.d/13678.doc +++ /dev/null @@ -1 +0,0 @@ -Update docs to make enabling metrics more clear. diff --git a/changelog.d/13679.misc b/changelog.d/13679.misc deleted file mode 100644 index a4fa94da9d..0000000000 --- a/changelog.d/13679.misc +++ /dev/null @@ -1 +0,0 @@ -Add some logging to help track down #13444. diff --git a/changelog.d/13683.bugfix b/changelog.d/13683.bugfix deleted file mode 100644 index 538534fec1..0000000000 --- a/changelog.d/13683.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix a long-standing bug which meant that keys for unwhitelisted servers were not returned by `/_matrix/key/v2/query`. diff --git a/changelog.d/13688.docker b/changelog.d/13688.docker deleted file mode 100644 index 8935e2536f..0000000000 --- a/changelog.d/13688.docker +++ /dev/null @@ -1 +0,0 @@ -Update docker image to use a stable version of poetry. diff --git a/changelog.d/13689.misc b/changelog.d/13689.misc deleted file mode 100644 index db6e48b150..0000000000 --- a/changelog.d/13689.misc +++ /dev/null @@ -1 +0,0 @@ -Update poetry lock file for v1.2.0. diff --git a/changelog.d/13692.removal b/changelog.d/13692.removal deleted file mode 100644 index eb075d4517..0000000000 --- a/changelog.d/13692.removal +++ /dev/null @@ -1 +0,0 @@ -Remove support for unstable [private read receipts](https://github.com/matrix-org/matrix-spec-proposals/pull/2285). diff --git a/changelog.d/13693.misc b/changelog.d/13693.misc deleted file mode 100644 index 31490191c9..0000000000 --- a/changelog.d/13693.misc +++ /dev/null @@ -1 +0,0 @@ -Add cache to `is_partial_state_room`. diff --git a/changelog.d/13694.bugfix b/changelog.d/13694.bugfix deleted file mode 100644 index 48b9bb5f0a..0000000000 --- a/changelog.d/13694.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix a bug introduced in Synapse v1.20.0 that would cause the unstable unread counts from [MSC2654](https://github.com/matrix-org/matrix-spec-proposals/pull/2654) to be calculated even if the feature is disabled. diff --git a/changelog.d/13697.misc b/changelog.d/13697.misc deleted file mode 100644 index b9d2a60961..0000000000 --- a/changelog.d/13697.misc +++ /dev/null @@ -1 +0,0 @@ -Update the Grafana dashboard that is included with Synapse in the `contrib` directory. \ No newline at end of file diff --git a/changelog.d/13698.misc b/changelog.d/13698.misc deleted file mode 100644 index 20b6b6d084..0000000000 --- a/changelog.d/13698.misc +++ /dev/null @@ -1 +0,0 @@ -Only run trial CI on all python versions on non-PRs. diff --git a/changelog.d/13701.doc b/changelog.d/13701.doc deleted file mode 100644 index b438e066d8..0000000000 --- a/changelog.d/13701.doc +++ /dev/null @@ -1 +0,0 @@ -Clarify `(room_id, event_id)` global uniqueness and how we should scope our database schemas. diff --git a/changelog.d/13712.misc b/changelog.d/13712.misc deleted file mode 100644 index 2c4f6b19f6..0000000000 --- a/changelog.d/13712.misc +++ /dev/null @@ -1 +0,0 @@ -Fix typechecking with latest types-jsonschema. diff --git a/changelog.d/13713.misc b/changelog.d/13713.misc deleted file mode 100644 index 1044099542..0000000000 --- a/changelog.d/13713.misc +++ /dev/null @@ -1 +0,0 @@ -Reduce number of CI checks we run for PRs. diff --git a/debian/changelog b/debian/changelog index 2b7b329b6b..4fa54dc5ee 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,8 +1,12 @@ -matrix-synapse-py3 (1.66.0ubuntu1) UNRELEASED; urgency=medium +matrix-synapse-py3 (1.67.0~rc1) stable; urgency=medium + [ Erik Johnston ] * Use stable poetry 1.2.0 version, rather than a prerelease. - -- Erik Johnston Thu, 01 Sep 2022 13:48:31 +0100 + [ Synapse Packaging team ] + * New Synapse release 1.67.0rc1. + + -- Synapse Packaging team Tue, 06 Sep 2022 09:01:06 +0100 matrix-synapse-py3 (1.66.0) stable; urgency=medium diff --git a/pyproject.toml b/pyproject.toml index a41d88ea74..dfb5c788be 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -54,7 +54,7 @@ skip_gitignore = true [tool.poetry] name = "matrix-synapse" -version = "1.66.0" +version = "1.67.0rc1" description = "Homeserver for the Matrix decentralised comms protocol" authors = ["Matrix.org Team and Contributors "] license = "Apache-2.0" From 571f565c1fa1e00462f9c96c3907d3304a37e39f Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 6 Sep 2022 09:25:32 +0100 Subject: [PATCH 059/107] Update changelog --- CHANGES.md | 13 +++++++++++++ docs/upgrade.md | 9 +++++++++ 2 files changed, 22 insertions(+) diff --git a/CHANGES.md b/CHANGES.md index 3bb00b6cea..f0d3db7849 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,6 +1,19 @@ Synapse 1.67.0rc1 (2022-09-06) ============================== +This release removes using the deprecated direct TCP replication configuration +for workers. Server admins should use Redis instead. See the [upgrade +notes](https://matrix-org.github.io/synapse/v1.67/upgrade.html#upgrading-to-v1670). + +The minimum version of `poetry` supported for managing source checkouts is now +1.2.0. + +Notice: from the next major release (v1.68.0) installing Synapse from a source +checkout will require a recent Rust compiler. Those using packages or +`pip install matrix-synapse` will not be affected. See the [upgrade +notes](https://matrix-org.github.io/synapse/v1.67/upgrade.html#upgrading-to-v1670). + + Features -------- diff --git a/docs/upgrade.md b/docs/upgrade.md index c6219d06e8..023ca0a30b 100644 --- a/docs/upgrade.md +++ b/docs/upgrade.md @@ -111,6 +111,15 @@ and remove the TCP `replication` listener from config of the master and The minimum supported version of poetry is now 1.2. This should only affect those installing from a source checkout. +## Rust requirement in the next release + +From the next major release (v1.68.0) installing Synapse from a source checkout +will require a recent Rust compiler. Those using packages or +`pip install matrix-synapse` will not be affected. + +The simplest way of installing Rust is via [rustup.rs](https://rustup.rs/) + + # Upgrading to v1.66.0 ## Delegation of email validation no longer supported From b455c2a5ec4874a6897c0448631d8ab8f36f9339 Mon Sep 17 00:00:00 2001 From: reivilibre Date: Tue, 6 Sep 2022 11:21:21 +0000 Subject: [PATCH 060/107] Update Grafana dashboard to not use legacy metric names. (#13714) --- changelog.d/13714.misc | 1 + contrib/grafana/synapse.json | 138 +++++++++++++------------- synapse/metrics/_legacy_exposition.py | 4 +- synapse/util/caches/__init__.py | 4 +- 4 files changed, 74 insertions(+), 73 deletions(-) create mode 100644 changelog.d/13714.misc diff --git a/changelog.d/13714.misc b/changelog.d/13714.misc new file mode 100644 index 0000000000..07ace50b12 --- /dev/null +++ b/changelog.d/13714.misc @@ -0,0 +1 @@ +Add experimental configuration option to allow disabling legacy Prometheus metric names. \ No newline at end of file diff --git a/contrib/grafana/synapse.json b/contrib/grafana/synapse.json index 248cd6d9ad..58061e2fce 100644 --- a/contrib/grafana/synapse.json +++ b/contrib/grafana/synapse.json @@ -335,7 +335,7 @@ "datasource": { "uid": "$datasource" }, - "expr": "sum(rate(synapse_storage_events_persisted_events{instance=\"$instance\"}[$bucket_size]))", + "expr": "sum(rate(synapse_storage_events_persisted_events_total{instance=\"$instance\"}[$bucket_size]))", "hide": false, "instant": false, "legendFormat": "Events", @@ -1423,7 +1423,7 @@ "datasource": { "uid": "$datasource" }, - "expr": "rate(synapse_background_process_ru_utime_seconds{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])+rate(synapse_background_process_ru_stime_seconds{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])", + "expr": "rate(synapse_background_process_ru_utime_seconds_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])+rate(synapse_background_process_ru_stime_seconds_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])", "format": "time_series", "hide": false, "instant": false, @@ -1804,7 +1804,7 @@ "datasource": { "uid": "$datasource" }, - "expr": "sum(rate(synapse_storage_events_persisted_events{instance=\"$instance\"}[$bucket_size])) without (job,index)", + "expr": "sum(rate(synapse_storage_events_persisted_events_total{instance=\"$instance\"}[$bucket_size])) without (job,index)", "format": "time_series", "interval": "", "intervalFactor": 2, @@ -2437,7 +2437,7 @@ "uid": "$datasource" }, "exemplar": false, - "expr": "sum(rate(synapse_state_res_db_for_biggest_room_seconds{instance=\"$instance\"}[1m]))", + "expr": "sum(rate(synapse_state_res_db_for_biggest_room_seconds_total{instance=\"$instance\"}[1m]))", "format": "time_series", "hide": false, "instant": false, @@ -2451,7 +2451,7 @@ "uid": "$datasource" }, "exemplar": false, - "expr": "sum(rate(synapse_state_res_cpu_for_biggest_room_seconds{instance=\"$instance\"}[1m]))", + "expr": "sum(rate(synapse_state_res_cpu_for_biggest_room_seconds_total{instance=\"$instance\"}[1m]))", "format": "time_series", "hide": false, "instant": false, @@ -3425,7 +3425,7 @@ "datasource": { "uid": "$datasource" }, - "expr": "rate(synapse_background_process_ru_utime_seconds{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])+rate(synapse_background_process_ru_stime_seconds{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])", + "expr": "rate(synapse_background_process_ru_utime_seconds_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])+rate(synapse_background_process_ru_stime_seconds_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])", "format": "time_series", "interval": "", "intervalFactor": 1, @@ -3518,7 +3518,7 @@ "datasource": { "uid": "$datasource" }, - "expr": "rate(synapse_background_process_db_txn_duration_seconds{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size]) + rate(synapse_background_process_db_sched_duration_seconds{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])", + "expr": "rate(synapse_background_process_db_txn_duration_seconds_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size]) + rate(synapse_background_process_db_sched_duration_seconds_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])", "format": "time_series", "hide": false, "intervalFactor": 1, @@ -3726,7 +3726,7 @@ "datasource": { "uid": "$datasource" }, - "expr": "sum(rate(synapse_federation_client_sent_transactions{instance=\"$instance\"}[$bucket_size]))", + "expr": "sum(rate(synapse_federation_client_sent_transactions_total{instance=\"$instance\"}[$bucket_size]))", "format": "time_series", "intervalFactor": 1, "legendFormat": "successful txn rate", @@ -3736,7 +3736,7 @@ "datasource": { "uid": "$datasource" }, - "expr": "sum(rate(synapse_util_metrics_block_count{block_name=\"_send_new_transaction\",instance=\"$instance\"}[$bucket_size]) - ignoring (block_name) rate(synapse_federation_client_sent_transactions{instance=\"$instance\"}[$bucket_size]))", + "expr": "sum(rate(synapse_util_metrics_block_count_total{block_name=\"_send_new_transaction\",instance=\"$instance\"}[$bucket_size]) - ignoring (block_name) rate(synapse_federation_client_sent_transactions_total{instance=\"$instance\"}[$bucket_size]))", "legendFormat": "failed txn rate", "refId": "B" } @@ -3826,7 +3826,7 @@ "datasource": { "uid": "$datasource" }, - "expr": "sum(rate(synapse_federation_server_received_pdus{instance=~\"$instance\"}[$bucket_size]))", + "expr": "sum(rate(synapse_federation_server_received_pdus_total{instance=~\"$instance\"}[$bucket_size]))", "format": "time_series", "intervalFactor": 1, "legendFormat": "pdus", @@ -3836,7 +3836,7 @@ "datasource": { "uid": "$datasource" }, - "expr": "sum(rate(synapse_federation_server_received_edus{instance=~\"$instance\"}[$bucket_size]))", + "expr": "sum(rate(synapse_federation_server_received_edus_total{instance=~\"$instance\"}[$bucket_size]))", "format": "time_series", "intervalFactor": 1, "legendFormat": "edus", @@ -3928,7 +3928,7 @@ "datasource": { "uid": "$datasource" }, - "expr": "sum(rate(synapse_federation_client_sent_pdu_destinations:total{instance=\"$instance\"}[$bucket_size]))", + "expr": "sum(rate(synapse_federation_client_sent_pdu_destinations:total_total{instance=\"$instance\"}[$bucket_size]))", "format": "time_series", "interval": "", "intervalFactor": 1, @@ -3939,7 +3939,7 @@ "datasource": { "uid": "$datasource" }, - "expr": "sum(rate(synapse_federation_client_sent_edus{instance=\"$instance\"}[$bucket_size]))", + "expr": "sum(rate(synapse_federation_client_sent_edus_total{instance=\"$instance\"}[$bucket_size]))", "format": "time_series", "intervalFactor": 1, "legendFormat": "edus", @@ -5042,7 +5042,7 @@ "datasource": { "uid": "$datasource" }, - "expr": "rate(synapse_http_httppusher_http_pushes_processed{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size]) and on (instance, job, index) (synapse_http_httppusher_http_pushes_failed + synapse_http_httppusher_http_pushes_processed) > 0", + "expr": "rate(synapse_http_httppusher_http_pushes_processed_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size]) and on (instance, job, index) (synapse_http_httppusher_http_pushes_failed_total + synapse_http_httppusher_http_pushes_processed_total) > 0", "format": "time_series", "interval": "", "intervalFactor": 2, @@ -5054,7 +5054,7 @@ "datasource": { "uid": "$datasource" }, - "expr": "rate(synapse_http_httppusher_http_pushes_failed{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size]) and on (instance, job, index) (synapse_http_httppusher_http_pushes_failed + synapse_http_httppusher_http_pushes_processed) > 0", + "expr": "rate(synapse_http_httppusher_http_pushes_failed_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size]) and on (instance, job, index) (synapse_http_httppusher_http_pushes_failed_total + synapse_http_httppusher_http_pushes_processed_total) > 0", "format": "time_series", "intervalFactor": 2, "legendFormat": "failed {{job}}", @@ -5268,12 +5268,12 @@ "uid": "${DS_PROMETHEUS}" }, "exemplar": true, - "expr": "sum(rate(synapse_push_bulk_push_rule_evaluator_push_rules_state_size_counter{job=\"$job\",index=~\"$index\",instance=\"$instance\"}[$bucket_size]))", + "expr": "sum(rate(synapse_push_bulk_push_rule_evaluator_push_rules_state_size_counter_total{job=\"$job\",index=~\"$index\",instance=\"$instance\"}[$bucket_size]))", "format": "time_series", "interval": "", "intervalFactor": 2, "legendFormat": "{{index}}", - "metric": "synapse_push_bulk_push_rule_evaluator_push_rules_state_size_counter", + "metric": "synapse_push_bulk_push_rule_evaluator_push_rules_state_size_counter_total", "refId": "A", "step": 2 } @@ -5369,12 +5369,12 @@ "uid": "$datasource" }, "exemplar": true, - "expr": "sum(rate(synapse_push_bulk_push_rule_evaluator_push_rules_invalidation_counter{job=\"$job\",index=~\"$index\",instance=\"$instance\"}[$bucket_size]))", + "expr": "sum(rate(synapse_push_bulk_push_rule_evaluator_push_rules_invalidation_counter_total{job=\"$job\",index=~\"$index\",instance=\"$instance\"}[$bucket_size]))", "format": "time_series", "interval": "", "intervalFactor": 2, "legendFormat": "{{index}}", - "metric": "synapse_push_bulk_push_rule_evaluator_push_rules_invalidation_counter", + "metric": "synapse_push_bulk_push_rule_evaluator_push_rules_invalidation_counter_total", "refId": "A", "step": 2 } @@ -5475,12 +5475,12 @@ "uid": "${DS_PROMETHEUS}" }, "exemplar": true, - "expr": "sum(rate(synapse_util_caches_cache:hits{job=\"$job\",index=~\"$index\",name=\"push_rules_delta_state_cache_metric\",instance=\"$instance\"}[$bucket_size]))/sum(rate(synapse_util_caches_cache:total{job=\"$job\",index=~\"$index\", name=\"push_rules_delta_state_cache_metric\",instance=\"$instance\"}[$bucket_size]))", + "expr": "sum(rate(synapse_util_caches_cache_hits{job=\"$job\",index=~\"$index\",name=\"push_rules_delta_state_cache_metric\",instance=\"$instance\"}[$bucket_size]))/sum(rate(synapse_util_caches_cache{job=\"$job\",index=~\"$index\", name=\"push_rules_delta_state_cache_metric\",instance=\"$instance\"}[$bucket_size]))", "format": "time_series", "interval": "", "intervalFactor": 2, "legendFormat": "Hit Rate", - "metric": "synapse_push_bulk_push_rule_evaluator_push_rules_invalidation_counter", + "metric": "synapse_push_bulk_push_rule_evaluator_push_rules_invalidation_counter_total", "refId": "A", "step": 2 }, @@ -5490,7 +5490,7 @@ "uid": "${DS_PROMETHEUS}" }, "exemplar": true, - "expr": "sum(rate(synapse_util_caches_cache:total{job=\"$job\",index=~\"$index\", name=\"push_rules_delta_state_cache_metric\",instance=\"$instance\"}[$bucket_size]))", + "expr": "sum(rate(synapse_util_caches_cache{job=\"$job\",index=~\"$index\", name=\"push_rules_delta_state_cache_metric\",instance=\"$instance\"}[$bucket_size]))", "format": "time_series", "interval": "", "intervalFactor": 2, @@ -5598,12 +5598,12 @@ "uid": "${DS_PROMETHEUS}" }, "exemplar": true, - "expr": "sum(rate(synapse_util_caches_cache:hits{job=\"$job\",index=~\"$index\",name=\"room_push_rule_cache\",instance=\"$instance\"}[$bucket_size]))/sum(rate(synapse_util_caches_cache:total{job=\"$job\",index=~\"$index\", name=\"room_push_rule_cache\",instance=\"$instance\"}[$bucket_size]))", + "expr": "sum(rate(synapse_util_caches_cache_hits{job=\"$job\",index=~\"$index\",name=\"room_push_rule_cache\",instance=\"$instance\"}[$bucket_size]))/sum(rate(synapse_util_caches_cache{job=\"$job\",index=~\"$index\", name=\"room_push_rule_cache\",instance=\"$instance\"}[$bucket_size]))", "format": "time_series", "interval": "", "intervalFactor": 2, "legendFormat": "Hit Rate", - "metric": "synapse_push_bulk_push_rule_evaluator_push_rules_invalidation_counter", + "metric": "synapse_push_bulk_push_rule_evaluator_push_rules_invalidation_counter_total", "refId": "A", "step": 2 }, @@ -5613,7 +5613,7 @@ "uid": "${DS_PROMETHEUS}" }, "exemplar": true, - "expr": "sum(rate(synapse_util_caches_cache:total{job=\"$job\",index=~\"$index\", name=\"room_push_rule_cache\",instance=\"$instance\"}[$bucket_size]))", + "expr": "sum(rate(synapse_util_caches_cache{job=\"$job\",index=~\"$index\", name=\"room_push_rule_cache\",instance=\"$instance\"}[$bucket_size]))", "format": "time_series", "interval": "", "intervalFactor": 2, @@ -5719,12 +5719,12 @@ "uid": "${DS_PROMETHEUS}" }, "exemplar": true, - "expr": "sum(rate(synapse_util_caches_cache:hits{job=\"$job\",index=~\"$index\",name=\"_get_rules_for_room\",instance=\"$instance\"}[$bucket_size]))/sum(rate(synapse_util_caches_cache:total{job=\"$job\",index=~\"$index\", name=\"_get_rules_for_room\",instance=\"$instance\"}[$bucket_size]))", + "expr": "sum(rate(synapse_util_caches_cache_hits{job=\"$job\",index=~\"$index\",name=\"_get_rules_for_room\",instance=\"$instance\"}[$bucket_size]))/sum(rate(synapse_util_caches_cache{job=\"$job\",index=~\"$index\", name=\"_get_rules_for_room\",instance=\"$instance\"}[$bucket_size]))", "format": "time_series", "interval": "", "intervalFactor": 2, "legendFormat": "Hit Rate", - "metric": "synapse_push_bulk_push_rule_evaluator_push_rules_invalidation_counter", + "metric": "synapse_push_bulk_push_rule_evaluator_push_rules_invalidation_counter_total", "refId": "A", "step": 2 }, @@ -5734,7 +5734,7 @@ "uid": "${DS_PROMETHEUS}" }, "exemplar": true, - "expr": "sum(rate(synapse_util_caches_cache:total{job=\"$job\",index=~\"$index\", name=\"_get_rules_for_room\",instance=\"$instance\"}[$bucket_size]))", + "expr": "sum(rate(synapse_util_caches_cache{job=\"$job\",index=~\"$index\", name=\"_get_rules_for_room\",instance=\"$instance\"}[$bucket_size]))", "format": "time_series", "interval": "", "intervalFactor": 2, @@ -6087,7 +6087,7 @@ "datasource": { "uid": "$datasource" }, - "expr": "topk(10, rate(synapse_storage_transaction_time_count{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size]))", + "expr": "topk(10, rate(synapse_storage_transaction_time_count_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size]))", "format": "time_series", "interval": "", "intervalFactor": 2, @@ -6187,7 +6187,7 @@ "datasource": { "uid": "$datasource" }, - "expr": "rate(synapse_storage_transaction_time_sum{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])", + "expr": "rate(synapse_storage_transaction_time_sum_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])", "format": "time_series", "instant": false, "interval": "", @@ -6287,7 +6287,7 @@ "datasource": { "uid": "$datasource" }, - "expr": "rate(synapse_storage_transaction_time_sum{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])/rate(synapse_storage_transaction_time_count{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])", + "expr": "rate(synapse_storage_transaction_time_sum_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])/rate(synapse_storage_transaction_time_count_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])", "format": "time_series", "instant": false, "interval": "", @@ -6538,7 +6538,7 @@ "datasource": { "uid": "$datasource" }, - "expr": "rate(synapse_util_metrics_block_ru_utime_seconds{instance=\"$instance\",job=~\"$job\",index=~\"$index\",block_name!=\"wrapped_request_handler\"}[$bucket_size]) + rate(synapse_util_metrics_block_ru_stime_seconds[$bucket_size])", + "expr": "rate(synapse_util_metrics_block_ru_utime_seconds_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\",block_name!=\"wrapped_request_handler\"}[$bucket_size]) + rate(synapse_util_metrics_block_ru_stime_seconds_total[$bucket_size])", "format": "time_series", "interval": "", "intervalFactor": 2, @@ -6636,7 +6636,7 @@ "datasource": { "uid": "$datasource" }, - "expr": "(rate(synapse_util_metrics_block_ru_utime_seconds{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size]) + rate(synapse_util_metrics_block_ru_stime_seconds[$bucket_size])) / rate(synapse_util_metrics_block_count[$bucket_size])", + "expr": "(rate(synapse_util_metrics_block_ru_utime_seconds_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size]) + rate(synapse_util_metrics_block_ru_stime_seconds_total[$bucket_size])) / rate(synapse_util_metrics_block_count_total[$bucket_size])", "format": "time_series", "interval": "", "intervalFactor": 2, @@ -6737,7 +6737,7 @@ "uid": "${DS_PROMETHEUS}" }, "exemplar": true, - "expr": "rate(synapse_util_metrics_block_db_txn_duration_seconds{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])", + "expr": "rate(synapse_util_metrics_block_db_txn_duration_seconds_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])", "format": "time_series", "interval": "", "intervalFactor": 2, @@ -6839,7 +6839,7 @@ "datasource": { "uid": "$datasource" }, - "expr": "rate(synapse_util_metrics_block_db_txn_duration_seconds{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size]) / rate(synapse_util_metrics_block_db_txn_count{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])", + "expr": "rate(synapse_util_metrics_block_db_txn_duration_seconds_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size]) / rate(synapse_util_metrics_block_db_txn_count_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])", "format": "time_series", "interval": "", "intervalFactor": 2, @@ -6936,7 +6936,7 @@ "datasource": { "uid": "$datasource" }, - "expr": "rate(synapse_util_metrics_block_db_txn_duration_seconds{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size]) / rate(synapse_util_metrics_block_db_txn_count{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])", + "expr": "rate(synapse_util_metrics_block_db_txn_duration_seconds_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size]) / rate(synapse_util_metrics_block_db_txn_count_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])", "format": "time_series", "interval": "", "intervalFactor": 2, @@ -7033,7 +7033,7 @@ "datasource": { "uid": "$datasource" }, - "expr": "rate(synapse_util_metrics_block_time_seconds{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size]) / rate(synapse_util_metrics_block_count{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])", + "expr": "rate(synapse_util_metrics_block_time_seconds_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size]) / rate(synapse_util_metrics_block_count_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])", "format": "time_series", "interval": "", "intervalFactor": 2, @@ -7122,7 +7122,7 @@ "datasource": { "uid": "$datasource" }, - "expr": "rate(synapse_util_metrics_block_count{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])", + "expr": "rate(synapse_util_metrics_block_count_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])", "interval": "", "legendFormat": "{{job}}-{{index}} {{block_name}}", "refId": "A" @@ -7246,7 +7246,7 @@ "datasource": { "uid": "$datasource" }, - "expr": "rate(synapse_util_caches_cache:hits{job=~\"$job\",index=~\"$index\",instance=\"$instance\"}[$bucket_size])/rate(synapse_util_caches_cache:total{job=~\"$job\",index=~\"$index\",instance=\"$instance\"}[$bucket_size])", + "expr": "rate(synapse_util_caches_cache_hits{job=~\"$job\",index=~\"$index\",instance=\"$instance\"}[$bucket_size])/rate(synapse_util_caches_cache{job=~\"$job\",index=~\"$index\",instance=\"$instance\"}[$bucket_size])", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{name}} {{job}}-{{index}}", @@ -7347,7 +7347,7 @@ "datasource": { "uid": "$datasource" }, - "expr": "synapse_util_caches_cache:size{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}", + "expr": "synapse_util_caches_cache_size{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}", "format": "time_series", "hide": false, "interval": "", @@ -7447,7 +7447,7 @@ "datasource": { "uid": "$datasource" }, - "expr": "rate(synapse_util_caches_cache:total{job=~\"$job\",index=~\"$index\",instance=\"$instance\"}[$bucket_size])", + "expr": "rate(synapse_util_caches_cache{job=~\"$job\",index=~\"$index\",instance=\"$instance\"}[$bucket_size])", "format": "time_series", "interval": "", "intervalFactor": 2, @@ -7547,7 +7547,7 @@ "datasource": { "uid": "$datasource" }, - "expr": "topk(10, rate(synapse_util_caches_cache:total{job=~\"$job\",index=~\"$index\",instance=\"$instance\"}[$bucket_size]) - rate(synapse_util_caches_cache:hits{job=~\"$job\",index=~\"$index\",instance=\"$instance\"}[$bucket_size]))", + "expr": "topk(10, rate(synapse_util_caches_cache{job=~\"$job\",index=~\"$index\",instance=\"$instance\"}[$bucket_size]) - rate(synapse_util_caches_cache_hits{job=~\"$job\",index=~\"$index\",instance=\"$instance\"}[$bucket_size]))", "format": "time_series", "interval": "", "intervalFactor": 2, @@ -7643,7 +7643,7 @@ "datasource": { "uid": "$datasource" }, - "expr": "rate(synapse_util_caches_cache:evicted_size{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])", + "expr": "rate(synapse_util_caches_cache_evicted_size{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size])", "format": "time_series", "interval": "", "intervalFactor": 1, @@ -7763,7 +7763,7 @@ "datasource": { "uid": "$datasource" }, - "expr": "synapse_util_caches_response_cache:size{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}", + "expr": "synapse_util_caches_response_cache_size{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}", "interval": "", "legendFormat": "{{name}} {{job}}-{{index}}", "refId": "A" @@ -7853,7 +7853,7 @@ "datasource": { "uid": "$datasource" }, - "expr": "rate(synapse_util_caches_response_cache:hits{instance=\"$instance\", job=~\"$job\", index=~\"$index\"}[$bucket_size])/rate(synapse_util_caches_response_cache:total{instance=\"$instance\", job=~\"$job\", index=~\"$index\"}[$bucket_size])", + "expr": "rate(synapse_util_caches_response_cache_hits{instance=\"$instance\", job=~\"$job\", index=~\"$index\"}[$bucket_size])/rate(synapse_util_caches_response_cache{instance=\"$instance\", job=~\"$job\", index=~\"$index\"}[$bucket_size])", "interval": "", "legendFormat": "{{name}} {{job}}-{{index}}", "refId": "A" @@ -9556,7 +9556,7 @@ "datasource": { "uid": "$datasource" }, - "expr": "synapse_forward_extremities_bucket{instance=\"$instance\"} and on (index, instance, job) (synapse_storage_events_persisted_events > 0)", + "expr": "synapse_forward_extremities_bucket{instance=\"$instance\"} and on (index, instance, job) (synapse_storage_events_persisted_events_total > 0)", "format": "heatmap", "intervalFactor": 1, "legendFormat": "{{le}}", @@ -9716,7 +9716,7 @@ "datasource": { "uid": "$datasource" }, - "expr": "rate(synapse_storage_events_forward_extremities_persisted_bucket{instance=\"$instance\"}[$bucket_size]) and on (index, instance, job) (synapse_storage_events_persisted_events > 0)", + "expr": "rate(synapse_storage_events_forward_extremities_persisted_bucket{instance=\"$instance\"}[$bucket_size]) and on (index, instance, job) (synapse_storage_events_persisted_events_total > 0)", "format": "heatmap", "intervalFactor": 1, "legendFormat": "{{le}}", @@ -9793,7 +9793,7 @@ "datasource": { "uid": "$datasource" }, - "expr": "histogram_quantile(0.5, rate(synapse_storage_events_forward_extremities_persisted_bucket{instance=\"$instance\"}[$bucket_size]) and on (index, instance, job) (synapse_storage_events_persisted_events > 0))", + "expr": "histogram_quantile(0.5, rate(synapse_storage_events_forward_extremities_persisted_bucket{instance=\"$instance\"}[$bucket_size]) and on (index, instance, job) (synapse_storage_events_persisted_events_total > 0))", "format": "time_series", "intervalFactor": 1, "legendFormat": "50%", @@ -9803,7 +9803,7 @@ "datasource": { "uid": "$datasource" }, - "expr": "histogram_quantile(0.75, rate(synapse_storage_events_forward_extremities_persisted_bucket{instance=\"$instance\"}[$bucket_size]) and on (index, instance, job) (synapse_storage_events_persisted_events > 0))", + "expr": "histogram_quantile(0.75, rate(synapse_storage_events_forward_extremities_persisted_bucket{instance=\"$instance\"}[$bucket_size]) and on (index, instance, job) (synapse_storage_events_persisted_events_total > 0))", "format": "time_series", "intervalFactor": 1, "legendFormat": "75%", @@ -9813,7 +9813,7 @@ "datasource": { "uid": "$datasource" }, - "expr": "histogram_quantile(0.90, rate(synapse_storage_events_forward_extremities_persisted_bucket{instance=\"$instance\"}[$bucket_size]) and on (index, instance, job) (synapse_storage_events_persisted_events > 0))", + "expr": "histogram_quantile(0.90, rate(synapse_storage_events_forward_extremities_persisted_bucket{instance=\"$instance\"}[$bucket_size]) and on (index, instance, job) (synapse_storage_events_persisted_events_total > 0))", "format": "time_series", "intervalFactor": 1, "legendFormat": "90%", @@ -9823,7 +9823,7 @@ "datasource": { "uid": "$datasource" }, - "expr": "histogram_quantile(0.99, rate(synapse_storage_events_forward_extremities_persisted_bucket{instance=\"$instance\"}[$bucket_size]) and on (index, instance, job) (synapse_storage_events_persisted_events > 0))", + "expr": "histogram_quantile(0.99, rate(synapse_storage_events_forward_extremities_persisted_bucket{instance=\"$instance\"}[$bucket_size]) and on (index, instance, job) (synapse_storage_events_persisted_events_total > 0))", "format": "time_series", "intervalFactor": 1, "legendFormat": "99%", @@ -9905,7 +9905,7 @@ "datasource": { "uid": "$datasource" }, - "expr": "rate(synapse_storage_events_stale_forward_extremities_persisted_bucket{instance=\"$instance\"}[$bucket_size]) and on (index, instance, job) (synapse_storage_events_persisted_events > 0)", + "expr": "rate(synapse_storage_events_stale_forward_extremities_persisted_bucket{instance=\"$instance\"}[$bucket_size]) and on (index, instance, job) (synapse_storage_events_persisted_events_total > 0)", "format": "heatmap", "intervalFactor": 1, "legendFormat": "{{le}}", @@ -9982,7 +9982,7 @@ "datasource": { "uid": "$datasource" }, - "expr": "histogram_quantile(0.5, rate(synapse_storage_events_stale_forward_extremities_persisted_bucket{instance=\"$instance\"}[$bucket_size]) and on (index, instance, job) (synapse_storage_events_persisted_events > 0))", + "expr": "histogram_quantile(0.5, rate(synapse_storage_events_stale_forward_extremities_persisted_bucket{instance=\"$instance\"}[$bucket_size]) and on (index, instance, job) (synapse_storage_events_persisted_events_total > 0))", "format": "time_series", "intervalFactor": 1, "legendFormat": "50%", @@ -9992,7 +9992,7 @@ "datasource": { "uid": "$datasource" }, - "expr": "histogram_quantile(0.75, rate(synapse_storage_events_stale_forward_extremities_persisted_bucket{instance=\"$instance\"}[$bucket_size]) and on (index, instance, job) (synapse_storage_events_persisted_events > 0))", + "expr": "histogram_quantile(0.75, rate(synapse_storage_events_stale_forward_extremities_persisted_bucket{instance=\"$instance\"}[$bucket_size]) and on (index, instance, job) (synapse_storage_events_persisted_events_total > 0))", "format": "time_series", "intervalFactor": 1, "legendFormat": "75%", @@ -10002,7 +10002,7 @@ "datasource": { "uid": "$datasource" }, - "expr": "histogram_quantile(0.90, rate(synapse_storage_events_stale_forward_extremities_persisted_bucket{instance=\"$instance\"}[$bucket_size]) and on (index, instance, job) (synapse_storage_events_persisted_events > 0))", + "expr": "histogram_quantile(0.90, rate(synapse_storage_events_stale_forward_extremities_persisted_bucket{instance=\"$instance\"}[$bucket_size]) and on (index, instance, job) (synapse_storage_events_persisted_events_total > 0))", "format": "time_series", "intervalFactor": 1, "legendFormat": "90%", @@ -10012,7 +10012,7 @@ "datasource": { "uid": "$datasource" }, - "expr": "histogram_quantile(0.99, rate(synapse_storage_events_stale_forward_extremities_persisted_bucket{instance=\"$instance\"}[$bucket_size]) and on (index, instance, job) (synapse_storage_events_persisted_events > 0))", + "expr": "histogram_quantile(0.99, rate(synapse_storage_events_stale_forward_extremities_persisted_bucket{instance=\"$instance\"}[$bucket_size]) and on (index, instance, job) (synapse_storage_events_persisted_events_total > 0))", "format": "time_series", "intervalFactor": 1, "legendFormat": "99%", @@ -10297,7 +10297,7 @@ "datasource": { "uid": "$datasource" }, - "expr": "sum(rate(synapse_storage_events_state_resolutions_during_persistence{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size]))", + "expr": "sum(rate(synapse_storage_events_state_resolutions_during_persistence_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size]))", "interval": "", "legendFormat": "State res ", "refId": "A" @@ -10306,7 +10306,7 @@ "datasource": { "uid": "$datasource" }, - "expr": "sum(rate(synapse_storage_events_potential_times_prune_extremities{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size]))", + "expr": "sum(rate(synapse_storage_events_potential_times_prune_extremities_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size]))", "interval": "", "legendFormat": "Potential to prune", "refId": "B" @@ -10315,7 +10315,7 @@ "datasource": { "uid": "$datasource" }, - "expr": "sum(rate(synapse_storage_events_times_pruned_extremities{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size]))", + "expr": "sum(rate(synapse_storage_events_times_pruned_extremities_total{instance=\"$instance\",job=~\"$job\",index=~\"$index\"}[$bucket_size]))", "interval": "", "legendFormat": "Pruned", "refId": "C" @@ -11069,7 +11069,7 @@ "datasource": { "uid": "$datasource" }, - "expr": "rate(synapse_handler_presence_notified_presence{job=\"$job\",index=~\"$index\",instance=\"$instance\"}[$bucket_size])", + "expr": "rate(synapse_handler_presence_notified_presence_total{job=\"$job\",index=~\"$index\",instance=\"$instance\"}[$bucket_size])", "interval": "", "legendFormat": "Notified", "refId": "A" @@ -11078,7 +11078,7 @@ "datasource": { "uid": "$datasource" }, - "expr": "rate(synapse_handler_presence_federation_presence_out{job=\"$job\",index=~\"$index\",instance=\"$instance\"}[$bucket_size])", + "expr": "rate(synapse_handler_presence_federation_presence_out_total{job=\"$job\",index=~\"$index\",instance=\"$instance\"}[$bucket_size])", "interval": "", "legendFormat": "Remote ping", "refId": "B" @@ -11087,7 +11087,7 @@ "datasource": { "uid": "$datasource" }, - "expr": "rate(synapse_handler_presence_presence_updates{job=\"$job\",index=~\"$index\",instance=\"$instance\"}[$bucket_size])", + "expr": "rate(synapse_handler_presence_presence_updates_total{job=\"$job\",index=~\"$index\",instance=\"$instance\"}[$bucket_size])", "interval": "", "legendFormat": "Total updates", "refId": "C" @@ -11096,7 +11096,7 @@ "datasource": { "uid": "$datasource" }, - "expr": "rate(synapse_handler_presence_federation_presence{job=\"$job\",index=~\"$index\",instance=\"$instance\"}[$bucket_size])", + "expr": "rate(synapse_handler_presence_federation_presence_total{job=\"$job\",index=~\"$index\",instance=\"$instance\"}[$bucket_size])", "interval": "", "legendFormat": "Remote updates", "refId": "D" @@ -11105,7 +11105,7 @@ "datasource": { "uid": "$datasource" }, - "expr": "rate(synapse_handler_presence_bump_active_time{job=\"$job\",index=~\"$index\",instance=\"$instance\"}[$bucket_size])", + "expr": "rate(synapse_handler_presence_bump_active_time_total{job=\"$job\",index=~\"$index\",instance=\"$instance\"}[$bucket_size])", "interval": "", "legendFormat": "Bump active time", "refId": "E" @@ -11789,7 +11789,7 @@ "name": "instance", "options": [], "query": { - "query": "label_values(synapse_util_metrics_block_ru_utime_seconds, instance)", + "query": "label_values(synapse_util_metrics_block_ru_utime_seconds_total, instance)", "refId": "Prometheus-instance-Variable-Query" }, "refresh": 2, @@ -11818,7 +11818,7 @@ "name": "job", "options": [], "query": { - "query": "label_values(synapse_util_metrics_block_ru_utime_seconds, job)", + "query": "label_values(synapse_util_metrics_block_ru_utime_seconds_total, job)", "refId": "Prometheus-job-Variable-Query" }, "refresh": 2, @@ -11848,7 +11848,7 @@ "name": "index", "options": [], "query": { - "query": "label_values(synapse_util_metrics_block_ru_utime_seconds, index)", + "query": "label_values(synapse_util_metrics_block_ru_utime_seconds_total, index)", "refId": "Prometheus-index-Variable-Query" }, "refresh": 2, @@ -11896,6 +11896,6 @@ "timezone": "", "title": "Synapse", "uid": "000000012", - "version": 132, + "version": 133, "weekStart": "" -} \ No newline at end of file +} diff --git a/synapse/metrics/_legacy_exposition.py b/synapse/metrics/_legacy_exposition.py index ff640a49af..6f00ff2a47 100644 --- a/synapse/metrics/_legacy_exposition.py +++ b/synapse/metrics/_legacy_exposition.py @@ -88,11 +88,11 @@ LEGACY_METRIC_NAMES = { "synapse_util_caches_cache_hits": "synapse_util_caches_cache:hits", "synapse_util_caches_cache_size": "synapse_util_caches_cache:size", "synapse_util_caches_cache_evicted_size": "synapse_util_caches_cache:evicted_size", - "synapse_util_caches_cache_total": "synapse_util_caches_cache:total", + "synapse_util_caches_cache": "synapse_util_caches_cache:total", "synapse_util_caches_response_cache_size": "synapse_util_caches_response_cache:size", "synapse_util_caches_response_cache_hits": "synapse_util_caches_response_cache:hits", "synapse_util_caches_response_cache_evicted_size": "synapse_util_caches_response_cache:evicted_size", - "synapse_util_caches_response_cache_total": "synapse_util_caches_response_cache:total", + "synapse_util_caches_response_cache": "synapse_util_caches_response_cache:total", } diff --git a/synapse/util/caches/__init__.py b/synapse/util/caches/__init__.py index bdf9b0dc8c..d4a2b77c29 100644 --- a/synapse/util/caches/__init__.py +++ b/synapse/util/caches/__init__.py @@ -37,7 +37,7 @@ collectors_by_name: Dict[str, "CacheMetric"] = {} cache_size = Gauge("synapse_util_caches_cache_size", "", ["name"]) cache_hits = Gauge("synapse_util_caches_cache_hits", "", ["name"]) cache_evicted = Gauge("synapse_util_caches_cache_evicted_size", "", ["name", "reason"]) -cache_total = Gauge("synapse_util_caches_cache_total", "", ["name"]) +cache_total = Gauge("synapse_util_caches_cache", "", ["name"]) cache_max_size = Gauge("synapse_util_caches_cache_max_size", "", ["name"]) cache_memory_usage = Gauge( "synapse_util_caches_cache_size_bytes", @@ -50,7 +50,7 @@ response_cache_hits = Gauge("synapse_util_caches_response_cache_hits", "", ["nam response_cache_evicted = Gauge( "synapse_util_caches_response_cache_evicted_size", "", ["name", "reason"] ) -response_cache_total = Gauge("synapse_util_caches_response_cache_total", "", ["name"]) +response_cache_total = Gauge("synapse_util_caches_response_cache", "", ["name"]) class EvictionReason(Enum): From b5effc72016021cc38f8d7949420d9246787fe11 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 6 Sep 2022 12:43:04 +0100 Subject: [PATCH 061/107] Update trial old deps CI to use poetry 1.2.0 (#13707) --- .../{test_old_deps.sh => prepare_old_deps.sh} | 25 ++--------- .github/workflows/tests.yml | 45 ++++++++++++++++--- changelog.d/13707.misc | 1 + 3 files changed, 42 insertions(+), 29 deletions(-) rename .ci/scripts/{test_old_deps.sh => prepare_old_deps.sh} (71%) create mode 100644 changelog.d/13707.misc diff --git a/.ci/scripts/test_old_deps.sh b/.ci/scripts/prepare_old_deps.sh similarity index 71% rename from .ci/scripts/test_old_deps.sh rename to .ci/scripts/prepare_old_deps.sh index 478c8d639a..7e4f060b17 100755 --- a/.ci/scripts/test_old_deps.sh +++ b/.ci/scripts/prepare_old_deps.sh @@ -5,18 +5,8 @@ # - creates a venv with these old versions using poetry; and finally # - invokes `trial` to run the tests with old deps. -# Prevent tzdata from asking for user input -export DEBIAN_FRONTEND=noninteractive - set -ex -apt-get update -apt-get install -y \ - python3 python3-dev python3-pip python3-venv pipx \ - libxml2-dev libxslt-dev xmlsec1 zlib1g-dev libjpeg-dev libwebp-dev - -export LANG="C.UTF-8" - # Prevent virtualenv from auto-updating pip to an incompatible version export VIRTUALENV_NO_DOWNLOAD=1 @@ -33,12 +23,6 @@ export VIRTUALENV_NO_DOWNLOAD=1 # a `cryptography` compiled against OpenSSL 1.1. # - Omit systemd: we're not logging to journal here. -# TODO: also replace caret bounds, see https://python-poetry.org/docs/dependency-specification/#version-constraints -# We don't use these yet, but IIRC they are the default bound used when you `poetry add`. -# The sed expression 's/\^/==/g' ought to do the trick. But it would also change -# `python = "^3.7"` to `python = "==3.7", which would mean we fail because olddeps -# runs on 3.8 (#12343). - sed -i \ -e "s/[~>]=/==/g" \ -e '/^python = "^/!s/\^/==/g' \ @@ -55,7 +39,7 @@ sed -i \ # toml file. This means we don't have to ensure compatibility between old deps and # dev tools. -pip install --user toml +pip install toml wheel REMOVE_DEV_DEPENDENCIES=" import toml @@ -69,8 +53,8 @@ with open('pyproject.toml', 'w') as f: " python3 -c "$REMOVE_DEV_DEPENDENCIES" -pipx install poetry==1.1.14 -~/.local/bin/poetry lock +pip install poetry==1.2.0 +poetry lock echo "::group::Patched pyproject.toml" cat pyproject.toml @@ -78,6 +62,3 @@ echo "::endgroup::" echo "::group::Lockfile after patch" cat poetry.lock echo "::endgroup::" - -~/.local/bin/poetry install -E "all test" -~/.local/bin/poetry run trial --jobs=2 tests diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index bc1de2893c..16fb4b43e2 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -135,16 +135,47 @@ jobs: # Note: sqlite only; no postgres if: ${{ !cancelled() && !failure() }} # Allow previous steps to be skipped, but not fail needs: linting-done - runs-on: ubuntu-latest + runs-on: ubuntu-20.04 steps: - uses: actions/checkout@v2 - - name: Test with old deps - uses: docker://ubuntu:focal # For old python and sqlite - # Note: focal seems to be using 3.8, but the oldest is 3.7? - # See https://github.com/matrix-org/synapse/issues/12343 + + # There aren't wheels for some of the older deps, so we need to install + # their build dependencies + - run: | + sudo apt-get -qq install build-essential libffi-dev python-dev \ + libxml2-dev libxslt-dev xmlsec1 zlib1g-dev libjpeg-dev libwebp-dev + + - uses: actions/setup-python@v4 with: - workdir: /github/workspace - entrypoint: .ci/scripts/test_old_deps.sh + python-version: '3.7' + + # Calculating the old-deps actually takes a bunch of time, so we cache the + # pyproject.toml / poetry.lock. We need to cache pyproject.toml as + # otherwise the `poetry install` step will error due to the poetry.lock + # file being outdated. + # + # This caches the output of `Prepare old deps`, which should generate the + # same `pyproject.toml` and `poetry.lock` for a given `pyproject.toml` input. + - uses: actions/cache@v3 + id: cache-poetry-old-deps + name: Cache poetry.lock + with: + path: | + poetry.lock + pyproject.toml + key: poetry-old-deps2-${{ hashFiles('pyproject.toml') }} + - name: Prepare old deps + if: steps.cache-poetry-old-deps.outputs.cache-hit != 'true' + run: .ci/scripts/prepare_old_deps.sh + + # We only now install poetry so that `setup-python-poetry` caches the + # right poetry.lock's dependencies. + - uses: matrix-org/setup-python-poetry@v1 + with: + python-version: '3.7' + extras: "all test" + + - run: poetry run trial -j 2 tests - name: Dump logs # Logs are most useful when the command fails, always include them. if: ${{ always() }} diff --git a/changelog.d/13707.misc b/changelog.d/13707.misc new file mode 100644 index 0000000000..e72c322d2e --- /dev/null +++ b/changelog.d/13707.misc @@ -0,0 +1 @@ +Update trial old deps CI to use poetry 1.2.0. From ec2fe7bb53f3e37694da421f37be709b19aea499 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 6 Sep 2022 13:04:22 +0100 Subject: [PATCH 062/107] Fixup changelog --- CHANGES.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index f0d3db7849..ad277f32e5 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -30,9 +30,9 @@ Bugfixes - Faster Room Joins: fix `/make_knock` blocking indefinitely when the room in question is a partial-stated room. ([\#13583](https://github.com/matrix-org/synapse/issues/13583)) - Fix loading the current stream position behind the actual position. ([\#13585](https://github.com/matrix-org/synapse/issues/13585)) - Fix a longstanding bug in `register_new_matrix_user` which meant it was always necessary to explicitly give a server URL. ([\#13616](https://github.com/matrix-org/synapse/issues/13616)) -- Fix the running of MSC1763 retention purge_jobs in deployments with background jobs running on a worker by forcing them back onto the main worker. Contributed by Brad @ Beeper. ([\#13632](https://github.com/matrix-org/synapse/issues/13632)) +- Fix the running of [MSC1763](https://github.com/matrix-org/matrix-spec-proposals/pull/1763) retention purge_jobs in deployments with background jobs running on a worker by forcing them back onto the main worker. Contributed by Brad @ Beeper. ([\#13632](https://github.com/matrix-org/synapse/issues/13632)) - Fix a long-standing bug that downloaded media for URL previews was not deleted while database background updates were running. ([\#13657](https://github.com/matrix-org/synapse/issues/13657)) -- Fix MSC3030 `/timestamp_to_event` endpoint to return the correct next event when the events have the same timestamp. ([\#13658](https://github.com/matrix-org/synapse/issues/13658)) +- Fix [MSC3030](https://github.com/matrix-org/matrix-spec-proposals/pull/3030) `/timestamp_to_event` endpoint to return the correct next event when the events have the same timestamp. ([\#13658](https://github.com/matrix-org/synapse/issues/13658)) - Fix bug where we wedge media plugins if clients disconnect early. Introduced in v1.22.0. ([\#13660](https://github.com/matrix-org/synapse/issues/13660)) - Fix a long-standing bug which meant that keys for unwhitelisted servers were not returned by `/_matrix/key/v2/query`. ([\#13683](https://github.com/matrix-org/synapse/issues/13683)) - Fix a bug introduced in Synapse v1.20.0 that would cause the unstable unread counts from [MSC2654](https://github.com/matrix-org/matrix-spec-proposals/pull/2654) to be calculated even if the feature is disabled. ([\#13694](https://github.com/matrix-org/synapse/issues/13694)) From a4ecb8e35309d780f5d4e93fb4998b90c9068e8a Mon Sep 17 00:00:00 2001 From: David Robertson Date: Tue, 6 Sep 2022 14:29:16 +0100 Subject: [PATCH 063/107] Actually fix typechecking with latest types-jsonschema (#13724) --- changelog.d/13724.misc | 1 + synapse/api/filtering.py | 8 ++++---- 2 files changed, 5 insertions(+), 4 deletions(-) create mode 100644 changelog.d/13724.misc diff --git a/changelog.d/13724.misc b/changelog.d/13724.misc new file mode 100644 index 0000000000..2c4f6b19f6 --- /dev/null +++ b/changelog.d/13724.misc @@ -0,0 +1 @@ +Fix typechecking with latest types-jsonschema. diff --git a/synapse/api/filtering.py b/synapse/api/filtering.py index 102889ac49..f7f46f8d80 100644 --- a/synapse/api/filtering.py +++ b/synapse/api/filtering.py @@ -140,13 +140,13 @@ USER_FILTER_SCHEMA = { @FormatChecker.cls_checks("matrix_room_id") -def matrix_room_id_validator(room_id_str: str) -> bool: - return RoomID.is_valid(room_id_str) +def matrix_room_id_validator(room_id: object) -> bool: + return isinstance(room_id, str) and RoomID.is_valid(room_id) @FormatChecker.cls_checks("matrix_user_id") -def matrix_user_id_validator(user_id_str: str) -> bool: - return UserID.is_valid(user_id_str) +def matrix_user_id_validator(user_id: object) -> bool: + return isinstance(user_id, str) and UserID.is_valid(user_id) class Filtering: From 3d201151152ca8ba9b9aae8da5b76a26044cc85f Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 6 Sep 2022 15:21:55 +0100 Subject: [PATCH 064/107] Fix trial-olddeps (#13725) --- changelog.d/13725.misc | 1 + poetry.lock | 2 +- pyproject.toml | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) create mode 100644 changelog.d/13725.misc diff --git a/changelog.d/13725.misc b/changelog.d/13725.misc new file mode 100644 index 0000000000..e72c322d2e --- /dev/null +++ b/changelog.d/13725.misc @@ -0,0 +1 @@ +Update trial old deps CI to use poetry 1.2.0. diff --git a/poetry.lock b/poetry.lock index 35021390bf..44df7d395c 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1600,7 +1600,7 @@ url_preview = ["lxml"] [metadata] lock-version = "1.1" python-versions = "^3.7.1" -content-hash = "7de518bf27967b3547eab8574342cfb67f87d6b47b4145c13de11112141dbf2d" +content-hash = "0df36bf75561fef340a7af704ed379b235f07a7d4a231aaccec5e7afb87159ca" [metadata.files] attrs = [ diff --git a/pyproject.toml b/pyproject.toml index a41d88ea74..8b2b5060b1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -126,7 +126,7 @@ pyOpenSSL = ">=16.0.0" PyYAML = ">=3.11" pyasn1 = ">=0.1.9" pyasn1-modules = ">=0.0.7" -bcrypt = ">=3.1.0" +bcrypt = ">=3.1.7" Pillow = ">=5.4.0" sortedcontainers = ">=1.4.4" pymacaroons = ">=0.13.0" From c9b7e9735508bb148c6ad59c433d71e5b8b360ad Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 6 Sep 2022 19:01:37 +0100 Subject: [PATCH 065/107] Add a stub Rust crate (#12595) --- .dockerignore | 4 ++ .github/workflows/release-artifacts.yml | 65 +++++++++++++++++++++++-- .github/workflows/tests.yml | 19 +++++++- .gitignore | 7 +++ Cargo.toml | 5 ++ build_rust.py | 20 ++++++++ changelog.d/12595.misc | 1 + debian/build_virtualenv | 7 ++- debian/changelog | 4 ++ debian/rules | 2 + docker/Dockerfile | 14 +++++- docker/Dockerfile-dhvirtualenv | 10 ++++ docs/deprecation_policy.md | 13 +++++ docs/development/contributing_guide.md | 10 +++- docs/setup/installation.md | 4 ++ mypy.ini | 6 ++- poetry.lock | 35 ++++++++++++- pyproject.toml | 39 ++++++++++++++- rust/Cargo.toml | 21 ++++++++ rust/src/lib.rs | 15 ++++++ stubs/synapse/__init__.pyi | 0 stubs/synapse/synapse_rust.pyi | 1 + tests/test_rust.py | 11 +++++ 23 files changed, 302 insertions(+), 11 deletions(-) create mode 100644 Cargo.toml create mode 100644 build_rust.py create mode 100644 changelog.d/12595.misc create mode 100644 rust/Cargo.toml create mode 100644 rust/src/lib.rs create mode 100644 stubs/synapse/__init__.pyi create mode 100644 stubs/synapse/synapse_rust.pyi create mode 100644 tests/test_rust.py diff --git a/.dockerignore b/.dockerignore index 7809863ef3..8eb1e4df8a 100644 --- a/.dockerignore +++ b/.dockerignore @@ -4,8 +4,12 @@ # things to include !docker !synapse +!rust !README.rst !pyproject.toml !poetry.lock +!build_rust.py + +rust/target **/__pycache__ diff --git a/.github/workflows/release-artifacts.yml b/.github/workflows/release-artifacts.yml index ed4fc6179d..0708d631cd 100644 --- a/.github/workflows/release-artifacts.yml +++ b/.github/workflows/release-artifacts.yml @@ -15,7 +15,7 @@ on: concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true - + permissions: contents: write @@ -89,9 +89,67 @@ jobs: name: debs path: debs/* + build-wheels: + name: Build wheels on ${{ matrix.os }} + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: [ubuntu-20.04, macos-10.15] + is_pr: + - ${{ startsWith(github.ref, 'refs/pull/') }} + + exclude: + # Don't build macos wheels on PR CI. + - is_pr: true + os: "macos-10.15" + + steps: + - uses: actions/checkout@v3 + + - uses: actions/setup-python@v3 + + - name: Install cibuildwheel + run: python -m pip install cibuildwheel==2.9.0 poetry==1.2.0 + + # Only build a single wheel in CI. + - name: Set env vars. + run: | + echo "CIBW_BUILD="cp37-manylinux_x86_64"" >> $GITHUB_ENV + if: startsWith(github.ref, 'refs/pull/') + + - name: Build wheels + run: python -m cibuildwheel --output-dir wheelhouse + env: + # Skip testing for platforms which various libraries don't have wheels + # for, and so need extra build deps. + CIBW_TEST_SKIP: pp39-* *i686* *musl* pp37-macosx* + + - uses: actions/upload-artifact@v3 + with: + name: Wheel + path: ./wheelhouse/*.whl + build-sdist: - name: "Build pypi distribution files" - uses: "matrix-org/backend-meta/.github/workflows/packaging.yml@v1" + name: Build sdist + runs-on: ubuntu-latest + if: ${{ !startsWith(github.ref, 'refs/pull/') }} + + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-python@v4 + with: + python-version: '3.10' + + - run: pip install build + + - name: Build sdist + run: python -m build --sdist + + - uses: actions/upload-artifact@v2 + with: + name: Sdist + path: dist/*.tar.gz + # if it's a tag, create a release and attach the artifacts to it attach-assets: @@ -99,6 +157,7 @@ jobs: if: ${{ !failure() && !cancelled() && startsWith(github.ref, 'refs/tags/') }} needs: - build-debs + - build-wheels - build-sdist runs-on: ubuntu-latest steps: diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 16fb4b43e2..5f96bdfa7f 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -139,6 +139,12 @@ jobs: steps: - uses: actions/checkout@v2 + - name: Install Rust + uses: actions-rs/toolchain@v1 + with: + toolchain: 1.61.0 + override: true + # There aren't wheels for some of the older deps, so we need to install # their build dependencies - run: | @@ -175,7 +181,7 @@ jobs: python-version: '3.7' extras: "all test" - - run: poetry run trial -j 2 tests + - run: poetry run trial -j2 tests - name: Dump logs # Logs are most useful when the command fails, always include them. if: ${{ always() }} @@ -247,6 +253,11 @@ jobs: - uses: actions/checkout@v2 - name: Prepare test blacklist run: cat sytest-blacklist .ci/worker-blacklist > synapse-blacklist-with-workers + - name: Install Rust + uses: actions-rs/toolchain@v1 + with: + toolchain: 1.61.0 + override: true - name: Run SyTest run: /bootstrap.sh synapse working-directory: /src @@ -353,6 +364,12 @@ jobs: with: path: synapse + - name: Install Rust + uses: actions-rs/toolchain@v1 + with: + toolchain: 1.61.0 + override: true + - name: Prepare Complement's Prerequisites run: synapse/.ci/scripts/setup_complement_prerequisites.sh diff --git a/.gitignore b/.gitignore index e58affb241..31a60bb7bd 100644 --- a/.gitignore +++ b/.gitignore @@ -60,3 +60,10 @@ book/ # complement /complement-* /master.tar.gz + +# rust +/target/ +/synapse/*.so + +# Poetry will create a setup.py, which we don't want to include. +/setup.py diff --git a/Cargo.toml b/Cargo.toml new file mode 100644 index 0000000000..de141bdee9 --- /dev/null +++ b/Cargo.toml @@ -0,0 +1,5 @@ +# We make the whole Synapse folder a workspace so that we can run `cargo` +# commands from the root (rather than having to cd into rust/). + +[workspace] +members = ["rust"] diff --git a/build_rust.py b/build_rust.py new file mode 100644 index 0000000000..5c5e557ee8 --- /dev/null +++ b/build_rust.py @@ -0,0 +1,20 @@ +# A build script for poetry that adds the rust extension. + +import os +from typing import Any, Dict + +from setuptools_rust import Binding, RustExtension + + +def build(setup_kwargs: Dict[str, Any]) -> None: + original_project_dir = os.path.dirname(os.path.realpath(__file__)) + cargo_toml_path = os.path.join(original_project_dir, "rust", "Cargo.toml") + + extension = RustExtension( + target="synapse.synapse_rust", + path=cargo_toml_path, + binding=Binding.PyO3, + py_limited_api=True, + ) + setup_kwargs.setdefault("rust_extensions", []).append(extension) + setup_kwargs["zip_safe"] = False diff --git a/changelog.d/12595.misc b/changelog.d/12595.misc new file mode 100644 index 0000000000..2e0dd68a0f --- /dev/null +++ b/changelog.d/12595.misc @@ -0,0 +1 @@ +Add a stub Rust crate. diff --git a/debian/build_virtualenv b/debian/build_virtualenv index ed916ac97a..dd97e888ba 100755 --- a/debian/build_virtualenv +++ b/debian/build_virtualenv @@ -61,7 +61,7 @@ dh_virtualenv \ --extras="all,systemd,test" \ --requirements="exported_requirements.txt" -PACKAGE_BUILD_DIR="debian/matrix-synapse-py3" +PACKAGE_BUILD_DIR="$(pwd)/debian/matrix-synapse-py3" VIRTUALENV_DIR="${PACKAGE_BUILD_DIR}${DH_VIRTUALENV_INSTALL_ROOT}/matrix-synapse" TARGET_PYTHON="${VIRTUALENV_DIR}/bin/python" @@ -78,9 +78,14 @@ case "$DEB_BUILD_OPTIONS" in cp -r tests "$tmpdir" + # To avoid pulling in the unbuilt Synapse in the local directory + pushd / + PYTHONPATH="$tmpdir" \ "${TARGET_PYTHON}" -m twisted.trial --reporter=text -j2 tests + popd + ;; esac diff --git a/debian/changelog b/debian/changelog index 2b7b329b6b..bd2d56e738 100644 --- a/debian/changelog +++ b/debian/changelog @@ -12,11 +12,15 @@ matrix-synapse-py3 (1.66.0) stable; urgency=medium matrix-synapse-py3 (1.66.0~rc2+nmu1) UNRELEASED; urgency=medium + [ Jörg Behrmann ] * Update debhelper to compatibility level 12. * Drop the preinst script stopping synapse. * Allocate a group for the system user. * Change dpkg-statoverride to --force-statoverride-add. + [ Erik Johnston ] + * Disable `dh_auto_configure` as it broke during Rust build. + -- Jörg Behrmann Tue, 23 Aug 2022 17:17:00 +0100 matrix-synapse-py3 (1.66.0~rc2) stable; urgency=medium diff --git a/debian/rules b/debian/rules index 3b79d56074..914d068f2a 100755 --- a/debian/rules +++ b/debian/rules @@ -12,6 +12,8 @@ override_dh_installsystemd: # we don't really want to strip the symbols from our object files. override_dh_strip: +override_dh_auto_configure: + # many libraries pulled from PyPI have allocatable sections after # non-allocatable ones on which dwz errors out. For those without the issue the # gains are only marginal diff --git a/docker/Dockerfile b/docker/Dockerfile index b87d263cff..a057bf397b 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -92,11 +92,20 @@ RUN \ libxml++2.6-dev \ libxslt1-dev \ openssl \ - rustc \ zlib1g-dev \ git \ + curl \ && rm -rf /var/lib/apt/lists/* + +# Install rust and ensure its in the PATH +ENV RUSTUP_HOME=/rust +ENV CARGO_HOME=/cargo +ENV PATH=/cargo/bin:/rust/bin:$PATH +RUN mkdir /rust /cargo + +RUN curl -sSf https://sh.rustup.rs | sh -s -- -y --no-modify-path --default-toolchain stable + # To speed up rebuilds, install all of the dependencies before we copy over # the whole synapse project, so that this layer in the Docker cache can be # used while you develop on the source @@ -108,8 +117,9 @@ RUN --mount=type=cache,target=/root/.cache/pip \ # Copy over the rest of the synapse source code. COPY synapse /synapse/synapse/ +COPY rust /synapse/rust/ # ... and what we need to `pip install`. -COPY pyproject.toml README.rst /synapse/ +COPY pyproject.toml README.rst build_rust.py /synapse/ # Repeat of earlier build argument declaration, as this is a new build stage. ARG TEST_ONLY_IGNORE_POETRY_LOCKFILE diff --git a/docker/Dockerfile-dhvirtualenv b/docker/Dockerfile-dhvirtualenv index fbc1d2346f..ca3a259081 100644 --- a/docker/Dockerfile-dhvirtualenv +++ b/docker/Dockerfile-dhvirtualenv @@ -72,6 +72,7 @@ RUN apt-get update -qq -o Acquire::Languages=none \ && env DEBIAN_FRONTEND=noninteractive apt-get install \ -yqq --no-install-recommends -o Dpkg::Options::=--force-unsafe-io \ build-essential \ + curl \ debhelper \ devscripts \ libsystemd-dev \ @@ -85,6 +86,15 @@ RUN apt-get update -qq -o Acquire::Languages=none \ libpq-dev \ xmlsec1 +# Install rust and ensure it's in the PATH +ENV RUSTUP_HOME=/rust +ENV CARGO_HOME=/cargo +ENV PATH=/cargo/bin:/rust/bin:$PATH +RUN mkdir /rust /cargo + +RUN curl -sSf https://sh.rustup.rs | sh -s -- -y --no-modify-path --default-toolchain stable + + COPY --from=builder /dh-virtualenv_1.2.2-1_all.deb / # install dhvirtualenv. Update the apt cache again first, in case we got a diff --git a/docs/deprecation_policy.md b/docs/deprecation_policy.md index 359dac07c3..b8a46e3d60 100644 --- a/docs/deprecation_policy.md +++ b/docs/deprecation_policy.md @@ -18,6 +18,12 @@ documented at [https://endoflife.date/python](https://endoflife.date/python) and [https://endoflife.date/postgresql](https://endoflife.date/postgresql). +A Rust compiler is required to build Synapse from source. For any given release +the minimum required version may be bumped up to a recent Rust version, and so +people building from source should ensure they can fetch recent versions of Rust +(e.g. by using [rustup](https://rustup.rs/)). + + Context ------- @@ -31,3 +37,10 @@ long process. By following the upstream support life cycles Synapse can ensure that its dependencies continue to get security patches, while not requiring system admins to constantly update their platform dependencies to the latest versions. + +For Rust, the situation is a bit different given that a) the Rust foundation +does not generally support older Rust versions, and b) the library ecosystem +generally bump their minimum support Rust versions frequently. In general, the +Synapse team will try to avoid updating the dependency on Rust to the absolute +latest version, but introducing a formal policy is hard given the constraints of +the ecosystem. diff --git a/docs/development/contributing_guide.md b/docs/development/contributing_guide.md index 4e1df51164..cb0d727efa 100644 --- a/docs/development/contributing_guide.md +++ b/docs/development/contributing_guide.md @@ -28,6 +28,9 @@ The source code of Synapse is hosted on GitHub. You will also need [a recent ver For some tests, you will need [a recent version of Docker](https://docs.docker.com/get-docker/). +A recent version of the Rust compiler is needed to build the native modules. The +easiest way of installing the latest version is to use [rustup](https://rustup.rs/). + # 3. Get the source. @@ -114,6 +117,11 @@ Some documentation also exists in [Synapse's GitHub Wiki](https://github.com/matrix-org/synapse/wiki), although this is primarily contributed to by community authors. +When changes are made to any Rust code then you must call either `poetry install` +or `maturin develop` (if installed) to rebuild the Rust code. Using [`maturin`](https://github.com/PyO3/maturin) +is quicker than `poetry install`, so is recommended when making frequent +changes to the Rust code. + # 8. Test, test, test! @@ -195,7 +203,7 @@ The database file can then be inspected with: sqlite3 _trial_temp/test.db ``` -Note that the database file is cleared at the beginning of each test run. Thus it +Note that the database file is cleared at the beginning of each test run. Thus it will always only contain the data generated by the *last run test*. Though generally when debugging, one is only running a single test anyway. diff --git a/docs/setup/installation.md b/docs/setup/installation.md index bb78b3267a..90737520ba 100644 --- a/docs/setup/installation.md +++ b/docs/setup/installation.md @@ -196,6 +196,10 @@ System requirements: - Python 3.7 or later, up to Python 3.10. - At least 1GB of free RAM if you want to join large public rooms like #matrix:matrix.org +If building on an uncommon architecture for which pre-built wheels are +unavailable, you will need to have a recent Rust compiler installed. The easiest +way of installing the latest version is to use [rustup](https://rustup.rs/). + To install the Synapse homeserver run: ```sh diff --git a/mypy.ini b/mypy.ini index e2034e411f..64f9097206 100644 --- a/mypy.ini +++ b/mypy.ini @@ -16,7 +16,8 @@ files = docker/, scripts-dev/, synapse/, - tests/ + tests/, + build_rust.py # Note: Better exclusion syntax coming in mypy > 0.910 # https://github.com/python/mypy/pull/11329 @@ -181,3 +182,6 @@ ignore_missing_imports = True [mypy-incremental.*] ignore_missing_imports = True + +[mypy-setuptools_rust.*] +ignore_missing_imports = True diff --git a/poetry.lock b/poetry.lock index 44df7d395c..cdc69f8ea9 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1035,6 +1035,18 @@ python-versions = ">=3.6" cryptography = ">=2.0" jeepney = ">=0.6" +[[package]] +name = "semantic-version" +version = "2.10.0" +description = "A library implementing the 'SemVer' scheme." +category = "main" +optional = false +python-versions = ">=2.7" + +[package.extras] +dev = ["Django (>=1.11)", "check-manifest", "colorama (<=0.4.1)", "coverage", "flake8", "nose2", "readme-renderer (<25.0)", "tox", "wheel", "zest.releaser[recommended]"] +doc = ["Sphinx", "sphinx-rtd-theme"] + [[package]] name = "sentry-sdk" version = "1.5.11" @@ -1099,6 +1111,19 @@ docs = ["furo", "jaraco.packaging (>=9)", "jaraco.tidelift (>=1.4)", "pygments-g testing = ["build[virtualenv]", "filelock (>=3.4.0)", "flake8 (<5)", "flake8-2020", "ini2toml[lite] (>=0.9)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "mock", "pip (>=19.1)", "pip-run (>=8.8)", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=1.3)", "pytest-flake8", "pytest-mypy (>=0.9.1)", "pytest-perf", "pytest-xdist", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel"] testing-integration = ["build[virtualenv]", "filelock (>=3.4.0)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "pytest", "pytest-enabler", "pytest-xdist", "tomli", "virtualenv (>=13.0.0)", "wheel"] +[[package]] +name = "setuptools-rust" +version = "1.5.1" +description = "Setuptools Rust extension plugin" +category = "main" +optional = false +python-versions = ">=3.7" + +[package.dependencies] +semantic-version = ">=2.8.2,<3" +setuptools = ">=62.4" +typing-extensions = ">=3.7.4.3" + [[package]] name = "signedjson" version = "1.1.4" @@ -1600,7 +1625,7 @@ url_preview = ["lxml"] [metadata] lock-version = "1.1" python-versions = "^3.7.1" -content-hash = "0df36bf75561fef340a7af704ed379b235f07a7d4a231aaccec5e7afb87159ca" +content-hash = "79cfa09d59f9f8b5ef24318fb860df1915f54328692aa56d04331ecbdd92a8cb" [metadata.files] attrs = [ @@ -2472,6 +2497,10 @@ secretstorage = [ {file = "SecretStorage-3.3.1-py3-none-any.whl", hash = "sha256:422d82c36172d88d6a0ed5afdec956514b189ddbfb72fefab0c8a1cee4eaf71f"}, {file = "SecretStorage-3.3.1.tar.gz", hash = "sha256:fd666c51a6bf200643495a04abb261f83229dcb6fd8472ec393df7ffc8b6f195"}, ] +semantic-version = [ + {file = "semantic_version-2.10.0-py2.py3-none-any.whl", hash = "sha256:de78a3b8e0feda74cabc54aab2da702113e33ac9d9eb9d2389bcf1f58b7d9177"}, + {file = "semantic_version-2.10.0.tar.gz", hash = "sha256:bdabb6d336998cbb378d4b9db3a4b56a1e3235701dc05ea2690d9a997ed5041c"}, +] sentry-sdk = [ {file = "sentry-sdk-1.5.11.tar.gz", hash = "sha256:6c01d9d0b65935fd275adc120194737d1df317dce811e642cbf0394d0d37a007"}, {file = "sentry_sdk-1.5.11-py2.py3-none-any.whl", hash = "sha256:c17179183cac614e900cbd048dab03f49a48e2820182ec686c25e7ce46f8548f"}, @@ -2484,6 +2513,10 @@ setuptools = [ {file = "setuptools-65.3.0-py3-none-any.whl", hash = "sha256:2e24e0bec025f035a2e72cdd1961119f557d78ad331bb00ff82efb2ab8da8e82"}, {file = "setuptools-65.3.0.tar.gz", hash = "sha256:7732871f4f7fa58fb6bdcaeadb0161b2bd046c85905dbaa066bdcbcc81953b57"}, ] +setuptools-rust = [ + {file = "setuptools-rust-1.5.1.tar.gz", hash = "sha256:0e05e456645d59429cb1021370aede73c0760e9360bbfdaaefb5bced530eb9d7"}, + {file = "setuptools_rust-1.5.1-py3-none-any.whl", hash = "sha256:306b236ff3aa5229180e58292610d0c2c51bb488191122d2fc559ae4caeb7d5e"}, +] signedjson = [ {file = "signedjson-1.1.4-py3-none-any.whl", hash = "sha256:45569ec54241c65d2403fe3faf7169be5322547706a231e884ca2b427f23d228"}, {file = "signedjson-1.1.4.tar.gz", hash = "sha256:cd91c56af53f169ef032c62e9c4a3292dc158866933318d0592e3462db3d6492"}, diff --git a/pyproject.toml b/pyproject.toml index 8b2b5060b1..7cc9de5bc7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -52,6 +52,9 @@ include_trailing_comma = true combine_as_imports = true skip_gitignore = true +[tool.maturin] +manifest-path = "rust/Cargo.toml" + [tool.poetry] name = "matrix-synapse" version = "1.66.0" @@ -82,7 +85,16 @@ include = [ { path = "sytest-blacklist", format = "sdist" }, { path = "tests", format = "sdist" }, { path = "UPGRADE.rst", format = "sdist" }, + { path = "Cargo.toml", format = "sdist" }, + { path = "rust/Cargo.toml", format = "sdist" }, + { path = "rust/Cargo.lock", format = "sdist" }, + { path = "rust/src/**", format = "sdist" }, ] +exclude = [ + { path = "synapse/*.so", format = "sdist"} +] + +build = "build_rust.py" [tool.poetry.scripts] synapse_homeserver = "synapse.app.homeserver:main" @@ -161,6 +173,15 @@ importlib_metadata = { version = ">=1.4", python = "<3.8" } # This is the most recent version of Pydantic with available on common distros. pydantic = ">=1.7.4" +# This is for building the rust components during "poetry install", which +# currently ignores the `build-system.requires` directive (c.f. +# https://github.com/python-poetry/poetry/issues/6154). Both `pip install` and +# `poetry build` do the right thing without this explicit dependency. +# +# This isn't really a dev-dependency, as `poetry install --no-dev` will fail, +# but the alternative is to add it to the main list of deps where it isn't +# needed. +setuptools_rust = ">=1.3" # Optional Dependencies @@ -285,5 +306,21 @@ twine = "*" towncrier = ">=18.6.0rc1" [build-system] -requires = ["poetry-core>=1.0.0"] +requires = ["poetry-core>=1.0.0", "setuptools_rust>=1.3"] build-backend = "poetry.core.masonry.api" + + +[tool.cibuildwheel] +# Skip unsupported platforms (by us or by Rust). +skip = "cp36* *-musllinux_i686" + +# We need a rust compiler +before-all = "curl https://sh.rustup.rs -sSf | sh -s -- --default-toolchain stable -y" +environment= { PATH = "$PATH:$HOME/.cargo/bin" } + +# For some reason if we don't manually clean the build directory we +# can end up polluting the next build with a .so that is for the wrong +# Python version. +before-build = "rm -rf {project}/build" +build-frontend = "build" +test-command = "python -c 'from synapse.synapse_rust import sum_as_string; print(sum_as_string(1, 2))'" diff --git a/rust/Cargo.toml b/rust/Cargo.toml new file mode 100644 index 0000000000..0a9760cafc --- /dev/null +++ b/rust/Cargo.toml @@ -0,0 +1,21 @@ +[package] +# We name the package `synapse` so that things like logging have the right +# logging target. +name = "synapse" + +# dummy version. See pyproject.toml for the Synapse's version number. +version = "0.1.0" + +edition = "2021" +rust-version = "1.61.0" + +[lib] +name = "synapse" +crate-type = ["cdylib"] + +[package.metadata.maturin] +# This is where we tell maturin where to place the built library. +name = "synapse.synapse_rust" + +[dependencies] +pyo3 = { version = "0.16.5", features = ["extension-module", "macros", "abi3", "abi3-py37"] } diff --git a/rust/src/lib.rs b/rust/src/lib.rs new file mode 100644 index 0000000000..fc4eb39154 --- /dev/null +++ b/rust/src/lib.rs @@ -0,0 +1,15 @@ +use pyo3::prelude::*; + +/// Formats the sum of two numbers as string. +#[pyfunction] +#[pyo3(text_signature = "(a, b, /)")] +fn sum_as_string(a: usize, b: usize) -> PyResult { + Ok((a + b).to_string()) +} + +/// The entry point for defining the Python module. +#[pymodule] +fn synapse_rust(_py: Python<'_>, m: &PyModule) -> PyResult<()> { + m.add_function(wrap_pyfunction!(sum_as_string, m)?)?; + Ok(()) +} diff --git a/stubs/synapse/__init__.pyi b/stubs/synapse/__init__.pyi new file mode 100644 index 0000000000..e69de29bb2 diff --git a/stubs/synapse/synapse_rust.pyi b/stubs/synapse/synapse_rust.pyi new file mode 100644 index 0000000000..5b51ba05d7 --- /dev/null +++ b/stubs/synapse/synapse_rust.pyi @@ -0,0 +1 @@ +def sum_as_string(a: int, b: int) -> str: ... diff --git a/tests/test_rust.py b/tests/test_rust.py new file mode 100644 index 0000000000..55d8b6b28c --- /dev/null +++ b/tests/test_rust.py @@ -0,0 +1,11 @@ +from synapse.synapse_rust import sum_as_string + +from tests import unittest + + +class RustTestCase(unittest.TestCase): + """Basic tests to ensure that we can call into Rust code.""" + + def test_basic(self): + result = sum_as_string(1, 2) + self.assertEqual("3", result) From 26bc26586b4b95d63ce7e453e9312469843f796e Mon Sep 17 00:00:00 2001 From: reivilibre Date: Tue, 6 Sep 2022 19:28:44 +0000 Subject: [PATCH 066/107] Remove the unspecced room_id field in the /hierarchy response. (#13506) This is a re-do of 57d334a13d983406ea452dfa203bbe4837509c4e (#13365), which was backed out in 12abd724974a2311d5311272d26d2f8aa11734a9 (#13501). The `room_id` field represented the parent space for each room and was made redundant by changes in the API shape where the `children_state` is now nested underneath each `room`. The room ID of each child is in the `state_key` field and is still available. --- changelog.d/13506.bugfix | 1 + synapse/handlers/room_summary.py | 1 - 2 files changed, 1 insertion(+), 1 deletion(-) create mode 100644 changelog.d/13506.bugfix diff --git a/changelog.d/13506.bugfix b/changelog.d/13506.bugfix new file mode 100644 index 0000000000..2e43668865 --- /dev/null +++ b/changelog.d/13506.bugfix @@ -0,0 +1 @@ +Fix a bug introduced in Synapse v1.41.0 where the `/hierarchy` API returned non-standard information (a `room_id` field under each entry in `children_state`). \ No newline at end of file diff --git a/synapse/handlers/room_summary.py b/synapse/handlers/room_summary.py index 732b0310bc..ebd445adca 100644 --- a/synapse/handlers/room_summary.py +++ b/synapse/handlers/room_summary.py @@ -453,7 +453,6 @@ class RoomSummaryHandler: "type": e.type, "state_key": e.state_key, "content": e.content, - "room_id": e.room_id, "sender": e.sender, "origin_server_ts": e.origin_server_ts, } From bb5b47b62a11b14a3458e5a8aafd9ddaf1294199 Mon Sep 17 00:00:00 2001 From: Connor Davis Date: Wed, 7 Sep 2022 05:54:44 -0400 Subject: [PATCH 067/107] Add Admin API to Fetch Messages Within a Particular Window (#13672) This adds two new admin APIs that allow us to fetch messages from a room within a particular time. --- changelog.d/13672.feature | 1 + docs/admin_api/rooms.md | 145 ++++++++++++++++++++++++++++++ synapse/handlers/pagination.py | 37 +++++--- synapse/rest/admin/__init__.py | 4 + synapse/rest/admin/rooms.py | 104 ++++++++++++++++++++++ tests/rest/admin/test_room.py | 158 ++++++++++++++++++++++++++++++++- 6 files changed, 435 insertions(+), 14 deletions(-) create mode 100644 changelog.d/13672.feature diff --git a/changelog.d/13672.feature b/changelog.d/13672.feature new file mode 100644 index 0000000000..2334e6fe15 --- /dev/null +++ b/changelog.d/13672.feature @@ -0,0 +1 @@ +Add admin APIs to fetch messages within a particular window of time. diff --git a/docs/admin_api/rooms.md b/docs/admin_api/rooms.md index 7526956bec..8f727b363e 100644 --- a/docs/admin_api/rooms.md +++ b/docs/admin_api/rooms.md @@ -393,6 +393,151 @@ A response body like the following is returned: } ``` +# Room Messages API + +The Room Messages admin API allows server admins to get all messages +sent to a room in a given timeframe. There are various parameters available +that allow for filtering and ordering the returned list. This API supports pagination. + +To use it, you will need to authenticate by providing an `access_token` +for a server admin: see [Admin API](../usage/administration/admin_api). + +This endpoint mirrors the [Matrix Spec defined Messages API](https://spec.matrix.org/v1.1/client-server-api/#get_matrixclientv3roomsroomidmessages). + +The API is: +``` +GET /_synapse/admin/v1/rooms//messages +``` + +**Parameters** + +The following path parameters are required: + +* `room_id` - The ID of the room you wish you fetch messages from. + +The following query parameters are available: + +* `from` (required) - The token to start returning events from. This token can be obtained from a prev_batch + or next_batch token returned by the /sync endpoint, or from an end token returned by a previous request to this endpoint. +* `to` - The token to spot returning events at. +* `limit` - The maximum number of events to return. Defaults to `10`. +* `filter` - A JSON RoomEventFilter to filter returned events with. +* `dir` - The direction to return events from. Either `f` for forwards or `b` for backwards. Setting + this value to `b` will reverse the above sort order. Defaults to `f`. + +**Response** + +The following fields are possible in the JSON response body: + +* `chunk` - A list of room events. The order depends on the dir parameter. + Note that an empty chunk does not necessarily imply that no more events are available. Clients should continue to paginate until no end property is returned. +* `end` - A token corresponding to the end of chunk. This token can be passed back to this endpoint to request further events. + If no further events are available, this property is omitted from the response. +* `start` - A token corresponding to the start of chunk. +* `state` - A list of state events relevant to showing the chunk. + +**Example** + +For more details on each chunk, read [the Matrix specification](https://spec.matrix.org/v1.1/client-server-api/#get_matrixclientv3roomsroomidmessages). + +```json +{ + "chunk": [ + { + "content": { + "body": "This is an example text message", + "format": "org.matrix.custom.html", + "formatted_body": "This is an example text message", + "msgtype": "m.text" + }, + "event_id": "$143273582443PhrSn:example.org", + "origin_server_ts": 1432735824653, + "room_id": "!636q39766251:example.com", + "sender": "@example:example.org", + "type": "m.room.message", + "unsigned": { + "age": 1234 + } + }, + { + "content": { + "name": "The room name" + }, + "event_id": "$143273582443PhrSn:example.org", + "origin_server_ts": 1432735824653, + "room_id": "!636q39766251:example.com", + "sender": "@example:example.org", + "state_key": "", + "type": "m.room.name", + "unsigned": { + "age": 1234 + } + }, + { + "content": { + "body": "Gangnam Style", + "info": { + "duration": 2140786, + "h": 320, + "mimetype": "video/mp4", + "size": 1563685, + "thumbnail_info": { + "h": 300, + "mimetype": "image/jpeg", + "size": 46144, + "w": 300 + }, + "thumbnail_url": "mxc://example.org/FHyPlCeYUSFFxlgbQYZmoEoe", + "w": 480 + }, + "msgtype": "m.video", + "url": "mxc://example.org/a526eYUSFFxlgbQYZmo442" + }, + "event_id": "$143273582443PhrSn:example.org", + "origin_server_ts": 1432735824653, + "room_id": "!636q39766251:example.com", + "sender": "@example:example.org", + "type": "m.room.message", + "unsigned": { + "age": 1234 + } + } + ], + "end": "t47409-4357353_219380_26003_2265", + "start": "t47429-4392820_219380_26003_2265" +} +``` + +# Room Timestamp to Event API + +The Room Timestamp to Event API endpoint fetches the `event_id` of the closest event to the given +timestamp (`ts` query parameter) in the given direction (`dir` query parameter). + +Useful for cases like jump to date so you can start paginating messages from +a given date in the archive. + +The API is: +``` + GET /_synapse/admin/v1/rooms//timestamp_to_event +``` + +**Parameters** + +The following path parameters are required: + +* `room_id` - The ID of the room you wish to check. + +The following query parameters are available: + +* `ts` - a timestamp in milliseconds where we will find the closest event in + the given direction. +* `dir` - can be `f` or `b` to indicate forwards and backwards in time from the + given timestamp. Defaults to `f`. + +**Response** + +* `event_id` - converted from timestamp + # Block Room API The Block Room admin API allows server admins to block and unblock rooms, and query to see if a given room is blocked. diff --git a/synapse/handlers/pagination.py b/synapse/handlers/pagination.py index a0c39778ab..1f83bab836 100644 --- a/synapse/handlers/pagination.py +++ b/synapse/handlers/pagination.py @@ -26,6 +26,7 @@ from synapse.events.utils import SerializeEventConfig from synapse.handlers.room import ShutdownRoomResponse from synapse.logging.opentracing import trace from synapse.metrics.background_process_metrics import run_as_background_process +from synapse.rest.admin._base import assert_user_is_admin from synapse.storage.state import StateFilter from synapse.streams.config import PaginationConfig from synapse.types import JsonDict, Requester, StreamKeyType @@ -423,6 +424,7 @@ class PaginationHandler: pagin_config: PaginationConfig, as_client_event: bool = True, event_filter: Optional[Filter] = None, + use_admin_priviledge: bool = False, ) -> JsonDict: """Get messages in a room. @@ -432,10 +434,16 @@ class PaginationHandler: pagin_config: The pagination config rules to apply, if any. as_client_event: True to get events in client-server format. event_filter: Filter to apply to results or None + use_admin_priviledge: if `True`, return all events, regardless + of whether `user` has access to them. To be used **ONLY** + from the admin API. Returns: Pagination API results """ + if use_admin_priviledge: + await assert_user_is_admin(self.auth, requester) + user_id = requester.user.to_string() if pagin_config.from_token: @@ -458,12 +466,14 @@ class PaginationHandler: room_token = from_token.room_key async with self.pagination_lock.read(room_id): - ( - membership, - member_event_id, - ) = await self.auth.check_user_in_room_or_world_readable( - room_id, requester, allow_departed_users=True - ) + (membership, member_event_id) = (None, None) + if not use_admin_priviledge: + ( + membership, + member_event_id, + ) = await self.auth.check_user_in_room_or_world_readable( + room_id, requester, allow_departed_users=True + ) if pagin_config.direction == "b": # if we're going backwards, we might need to backfill. This @@ -475,7 +485,7 @@ class PaginationHandler: room_id, room_token.stream ) - if membership == Membership.LEAVE: + if not use_admin_priviledge and membership == Membership.LEAVE: # If they have left the room then clamp the token to be before # they left the room, to save the effort of loading from the # database. @@ -528,12 +538,13 @@ class PaginationHandler: if event_filter: events = await event_filter.filter(events) - events = await filter_events_for_client( - self._storage_controllers, - user_id, - events, - is_peeking=(member_event_id is None), - ) + if not use_admin_priviledge: + events = await filter_events_for_client( + self._storage_controllers, + user_id, + events, + is_peeking=(member_event_id is None), + ) # if after the filter applied there are no more events # return immediately - but there might be more in next_token batch diff --git a/synapse/rest/admin/__init__.py b/synapse/rest/admin/__init__.py index fa3266720b..bac754e1b1 100644 --- a/synapse/rest/admin/__init__.py +++ b/synapse/rest/admin/__init__.py @@ -61,9 +61,11 @@ from synapse.rest.admin.rooms import ( MakeRoomAdminRestServlet, RoomEventContextServlet, RoomMembersRestServlet, + RoomMessagesRestServlet, RoomRestServlet, RoomRestV2Servlet, RoomStateRestServlet, + RoomTimestampToEventRestServlet, ) from synapse.rest.admin.server_notice_servlet import SendServerNoticeServlet from synapse.rest.admin.statistics import UserMediaStatisticsRestServlet @@ -271,6 +273,8 @@ def register_servlets(hs: "HomeServer", http_server: HttpServer) -> None: DestinationResetConnectionRestServlet(hs).register(http_server) DestinationRestServlet(hs).register(http_server) ListDestinationsRestServlet(hs).register(http_server) + RoomMessagesRestServlet(hs).register(http_server) + RoomTimestampToEventRestServlet(hs).register(http_server) # Some servlets only get registered for the main process. if hs.config.worker.worker_app is None: diff --git a/synapse/rest/admin/rooms.py b/synapse/rest/admin/rooms.py index 3d870629c4..747e6fda83 100644 --- a/synapse/rest/admin/rooms.py +++ b/synapse/rest/admin/rooms.py @@ -35,6 +35,7 @@ from synapse.rest.admin._base import ( ) from synapse.storage.databases.main.room import RoomSortOrder from synapse.storage.state import StateFilter +from synapse.streams.config import PaginationConfig from synapse.types import JsonDict, RoomID, UserID, create_requester from synapse.util import json_decoder @@ -858,3 +859,106 @@ class BlockRoomRestServlet(RestServlet): await self._store.unblock_room(room_id) return HTTPStatus.OK, {"block": block} + + +class RoomMessagesRestServlet(RestServlet): + """ + Get messages list of a room. + """ + + PATTERNS = admin_patterns("/rooms/(?P[^/]*)/messages$") + + def __init__(self, hs: "HomeServer"): + self._hs = hs + self._clock = hs.get_clock() + self._pagination_handler = hs.get_pagination_handler() + self._auth = hs.get_auth() + self._store = hs.get_datastores().main + + async def on_GET( + self, request: SynapseRequest, room_id: str + ) -> Tuple[int, JsonDict]: + requester = await self._auth.get_user_by_req(request) + await assert_user_is_admin(self._auth, requester) + + pagination_config = await PaginationConfig.from_request( + self._store, request, default_limit=10 + ) + # Twisted will have processed the args by now. + assert request.args is not None + as_client_event = b"raw" not in request.args + filter_str = parse_string(request, "filter", encoding="utf-8") + if filter_str: + filter_json = urlparse.unquote(filter_str) + event_filter: Optional[Filter] = Filter( + self._hs, json_decoder.decode(filter_json) + ) + if ( + event_filter + and event_filter.filter_json.get("event_format", "client") + == "federation" + ): + as_client_event = False + else: + event_filter = None + + msgs = await self._pagination_handler.get_messages( + room_id=room_id, + requester=requester, + pagin_config=pagination_config, + as_client_event=as_client_event, + event_filter=event_filter, + use_admin_priviledge=True, + ) + + return HTTPStatus.OK, msgs + + +class RoomTimestampToEventRestServlet(RestServlet): + """ + API endpoint to fetch the `event_id` of the closest event to the given + timestamp (`ts` query parameter) in the given direction (`dir` query + parameter). + + Useful for cases like jump to date so you can start paginating messages from + a given date in the archive. + + `ts` is a timestamp in milliseconds where we will find the closest event in + the given direction. + + `dir` can be `f` or `b` to indicate forwards and backwards in time from the + given timestamp. + + GET /_synapse/admin/v1/rooms//timestamp_to_event?ts=&dir= + { + "event_id": ... + } + """ + + PATTERNS = admin_patterns("/rooms/(?P[^/]*)/timestamp_to_event$") + + def __init__(self, hs: "HomeServer"): + self._auth = hs.get_auth() + self._store = hs.get_datastores().main + self._timestamp_lookup_handler = hs.get_timestamp_lookup_handler() + + async def on_GET( + self, request: SynapseRequest, room_id: str + ) -> Tuple[int, JsonDict]: + requester = await self._auth.get_user_by_req(request) + await assert_user_is_admin(self._auth, requester) + + timestamp = parse_integer(request, "ts", required=True) + direction = parse_string(request, "dir", default="f", allowed_values=["f", "b"]) + + ( + event_id, + origin_server_ts, + ) = await self._timestamp_lookup_handler.get_event_for_timestamp( + requester, room_id, timestamp, direction + ) + + return HTTPStatus.OK, { + "event_id": event_id, + "origin_server_ts": origin_server_ts, + } diff --git a/tests/rest/admin/test_room.py b/tests/rest/admin/test_room.py index 9d71a97524..d156be82b0 100644 --- a/tests/rest/admin/test_room.py +++ b/tests/rest/admin/test_room.py @@ -11,6 +11,8 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +import json +import time import urllib.parse from typing import List, Optional from unittest.mock import Mock @@ -22,10 +24,11 @@ from twisted.test.proto_helpers import MemoryReactor import synapse.rest.admin from synapse.api.constants import EventTypes, Membership, RoomTypes from synapse.api.errors import Codes -from synapse.handlers.pagination import PaginationHandler +from synapse.handlers.pagination import PaginationHandler, PurgeStatus from synapse.rest.client import directory, events, login, room from synapse.server import HomeServer from synapse.util import Clock +from synapse.util.stringutils import random_string from tests import unittest @@ -1793,6 +1796,159 @@ class RoomTestCase(unittest.HomeserverTestCase): self.assertEqual(Codes.FORBIDDEN, channel.json_body["errcode"]) +class RoomMessagesTestCase(unittest.HomeserverTestCase): + servlets = [ + synapse.rest.admin.register_servlets, + login.register_servlets, + room.register_servlets, + ] + + def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None: + self.admin_user = self.register_user("admin", "pass", admin=True) + self.admin_user_tok = self.login("admin", "pass") + + self.user = self.register_user("foo", "pass") + self.user_tok = self.login("foo", "pass") + self.room_id = self.helper.create_room_as(self.user, tok=self.user_tok) + + def test_timestamp_to_event(self) -> None: + """Test that providing the current timestamp can get the last event.""" + self.helper.send(self.room_id, body="message 1", tok=self.user_tok) + second_event_id = self.helper.send( + self.room_id, body="message 2", tok=self.user_tok + )["event_id"] + ts = str(round(time.time() * 1000)) + + channel = self.make_request( + "GET", + "/_synapse/admin/v1/rooms/%s/timestamp_to_event?dir=b&ts=%s" + % (self.room_id, ts), + access_token=self.admin_user_tok, + ) + self.assertEqual(200, channel.code) + self.assertIn("event_id", channel.json_body) + self.assertEqual(second_event_id, channel.json_body["event_id"]) + + def test_topo_token_is_accepted(self) -> None: + """Test Topo Token is accepted.""" + token = "t1-0_0_0_0_0_0_0_0_0" + channel = self.make_request( + "GET", + "/_synapse/admin/v1/rooms/%s/messages?from=%s" % (self.room_id, token), + access_token=self.admin_user_tok, + ) + self.assertEqual(200, channel.code) + self.assertIn("start", channel.json_body) + self.assertEqual(token, channel.json_body["start"]) + self.assertIn("chunk", channel.json_body) + self.assertIn("end", channel.json_body) + + def test_stream_token_is_accepted_for_fwd_pagianation(self) -> None: + """Test that stream token is accepted for forward pagination.""" + token = "s0_0_0_0_0_0_0_0_0" + channel = self.make_request( + "GET", + "/_synapse/admin/v1/rooms/%s/messages?from=%s" % (self.room_id, token), + access_token=self.admin_user_tok, + ) + self.assertEqual(200, channel.code) + self.assertIn("start", channel.json_body) + self.assertEqual(token, channel.json_body["start"]) + self.assertIn("chunk", channel.json_body) + self.assertIn("end", channel.json_body) + + def test_room_messages_purge(self) -> None: + """Test room messages can be retrieved by an admin that isn't in the room.""" + store = self.hs.get_datastores().main + pagination_handler = self.hs.get_pagination_handler() + + # Send a first message in the room, which will be removed by the purge. + first_event_id = self.helper.send( + self.room_id, body="message 1", tok=self.user_tok + )["event_id"] + first_token = self.get_success( + store.get_topological_token_for_event(first_event_id) + ) + first_token_str = self.get_success(first_token.to_string(store)) + + # Send a second message in the room, which won't be removed, and which we'll + # use as the marker to purge events before. + second_event_id = self.helper.send( + self.room_id, body="message 2", tok=self.user_tok + )["event_id"] + second_token = self.get_success( + store.get_topological_token_for_event(second_event_id) + ) + second_token_str = self.get_success(second_token.to_string(store)) + + # Send a third event in the room to ensure we don't fall under any edge case + # due to our marker being the latest forward extremity in the room. + self.helper.send(self.room_id, body="message 3", tok=self.user_tok) + + # Check that we get the first and second message when querying /messages. + channel = self.make_request( + "GET", + "/_synapse/admin/v1/rooms/%s/messages?from=%s&dir=b&filter=%s" + % ( + self.room_id, + second_token_str, + json.dumps({"types": [EventTypes.Message]}), + ), + access_token=self.admin_user_tok, + ) + self.assertEqual(channel.code, 200, channel.json_body) + + chunk = channel.json_body["chunk"] + self.assertEqual(len(chunk), 2, [event["content"] for event in chunk]) + + # Purge every event before the second event. + purge_id = random_string(16) + pagination_handler._purges_by_id[purge_id] = PurgeStatus() + self.get_success( + pagination_handler._purge_history( + purge_id=purge_id, + room_id=self.room_id, + token=second_token_str, + delete_local_events=True, + ) + ) + + # Check that we only get the second message through /message now that the first + # has been purged. + channel = self.make_request( + "GET", + "/_synapse/admin/v1/rooms/%s/messages?from=%s&dir=b&filter=%s" + % ( + self.room_id, + second_token_str, + json.dumps({"types": [EventTypes.Message]}), + ), + access_token=self.admin_user_tok, + ) + self.assertEqual(channel.code, 200, channel.json_body) + + chunk = channel.json_body["chunk"] + self.assertEqual(len(chunk), 1, [event["content"] for event in chunk]) + + # Check that we get no event, but also no error, when querying /messages with + # the token that was pointing at the first event, because we don't have it + # anymore. + channel = self.make_request( + "GET", + "/_synapse/admin/v1/rooms/%s/messages?from=%s&dir=b&filter=%s" + % ( + self.room_id, + first_token_str, + json.dumps({"types": [EventTypes.Message]}), + ), + access_token=self.admin_user_tok, + ) + self.assertEqual(channel.code, 200, channel.json_body) + + chunk = channel.json_body["chunk"] + self.assertEqual(len(chunk), 0, [event["content"] for event in chunk]) + + class JoinAliasRoomTestCase(unittest.HomeserverTestCase): servlets = [ From c2fe48a6ffb99f553f3eaecb8f15bcbedb58add0 Mon Sep 17 00:00:00 2001 From: reivilibre Date: Wed, 7 Sep 2022 10:08:20 +0000 Subject: [PATCH 068/107] Rename the `EventFormatVersions` enum values so that they line up with room version numbers. (#13706) --- changelog.d/13706.misc | 1 + synapse/api/room_versions.py | 45 ++++++++++--------- synapse/event_auth.py | 4 +- synapse/events/__init__.py | 12 ++--- synapse/events/builder.py | 4 +- synapse/events/validator.py | 2 +- synapse/federation/federation_base.py | 2 +- synapse/federation/federation_client.py | 2 +- .../databases/main/event_federation.py | 2 +- .../storage/databases/main/events_worker.py | 6 +-- .../databases/main/test_events_worker.py | 2 +- tests/storage/test_event_federation.py | 2 +- tests/test_event_auth.py | 4 +- 13 files changed, 47 insertions(+), 41 deletions(-) create mode 100644 changelog.d/13706.misc diff --git a/changelog.d/13706.misc b/changelog.d/13706.misc new file mode 100644 index 0000000000..65c854c7a9 --- /dev/null +++ b/changelog.d/13706.misc @@ -0,0 +1 @@ +Rename the `EventFormatVersions` enum values so that they line up with room version numbers. \ No newline at end of file diff --git a/synapse/api/room_versions.py b/synapse/api/room_versions.py index a0e4ab6db6..e37acb0f1e 100644 --- a/synapse/api/room_versions.py +++ b/synapse/api/room_versions.py @@ -19,18 +19,23 @@ import attr class EventFormatVersions: """This is an internal enum for tracking the version of the event format, - independently from the room version. + independently of the room version. + + To reduce confusion, the event format versions are named after the room + versions that they were used or introduced in. + The concept of an 'event format version' is specific to Synapse (the + specification does not mention this term.) """ - V1 = 1 # $id:server event id format - V2 = 2 # MSC1659-style $hash event id format: introduced for room v3 - V3 = 3 # MSC1884-style $hash format: introduced for room v4 + ROOM_V1_V2 = 1 # $id:server event id format: used for room v1 and v2 + ROOM_V3 = 2 # MSC1659-style $hash event id format: used for room v3 + ROOM_V4_PLUS = 3 # MSC1884-style $hash format: introduced for room v4 KNOWN_EVENT_FORMAT_VERSIONS = { - EventFormatVersions.V1, - EventFormatVersions.V2, - EventFormatVersions.V3, + EventFormatVersions.ROOM_V1_V2, + EventFormatVersions.ROOM_V3, + EventFormatVersions.ROOM_V4_PLUS, } @@ -92,7 +97,7 @@ class RoomVersions: V1 = RoomVersion( "1", RoomDisposition.STABLE, - EventFormatVersions.V1, + EventFormatVersions.ROOM_V1_V2, StateResolutionVersions.V1, enforce_key_validity=False, special_case_aliases_auth=True, @@ -110,7 +115,7 @@ class RoomVersions: V2 = RoomVersion( "2", RoomDisposition.STABLE, - EventFormatVersions.V1, + EventFormatVersions.ROOM_V1_V2, StateResolutionVersions.V2, enforce_key_validity=False, special_case_aliases_auth=True, @@ -128,7 +133,7 @@ class RoomVersions: V3 = RoomVersion( "3", RoomDisposition.STABLE, - EventFormatVersions.V2, + EventFormatVersions.ROOM_V3, StateResolutionVersions.V2, enforce_key_validity=False, special_case_aliases_auth=True, @@ -146,7 +151,7 @@ class RoomVersions: V4 = RoomVersion( "4", RoomDisposition.STABLE, - EventFormatVersions.V3, + EventFormatVersions.ROOM_V4_PLUS, StateResolutionVersions.V2, enforce_key_validity=False, special_case_aliases_auth=True, @@ -164,7 +169,7 @@ class RoomVersions: V5 = RoomVersion( "5", RoomDisposition.STABLE, - EventFormatVersions.V3, + EventFormatVersions.ROOM_V4_PLUS, StateResolutionVersions.V2, enforce_key_validity=True, special_case_aliases_auth=True, @@ -182,7 +187,7 @@ class RoomVersions: V6 = RoomVersion( "6", RoomDisposition.STABLE, - EventFormatVersions.V3, + EventFormatVersions.ROOM_V4_PLUS, StateResolutionVersions.V2, enforce_key_validity=True, special_case_aliases_auth=False, @@ -200,7 +205,7 @@ class RoomVersions: MSC2176 = RoomVersion( "org.matrix.msc2176", RoomDisposition.UNSTABLE, - EventFormatVersions.V3, + EventFormatVersions.ROOM_V4_PLUS, StateResolutionVersions.V2, enforce_key_validity=True, special_case_aliases_auth=False, @@ -218,7 +223,7 @@ class RoomVersions: V7 = RoomVersion( "7", RoomDisposition.STABLE, - EventFormatVersions.V3, + EventFormatVersions.ROOM_V4_PLUS, StateResolutionVersions.V2, enforce_key_validity=True, special_case_aliases_auth=False, @@ -236,7 +241,7 @@ class RoomVersions: V8 = RoomVersion( "8", RoomDisposition.STABLE, - EventFormatVersions.V3, + EventFormatVersions.ROOM_V4_PLUS, StateResolutionVersions.V2, enforce_key_validity=True, special_case_aliases_auth=False, @@ -254,7 +259,7 @@ class RoomVersions: V9 = RoomVersion( "9", RoomDisposition.STABLE, - EventFormatVersions.V3, + EventFormatVersions.ROOM_V4_PLUS, StateResolutionVersions.V2, enforce_key_validity=True, special_case_aliases_auth=False, @@ -272,7 +277,7 @@ class RoomVersions: MSC3787 = RoomVersion( "org.matrix.msc3787", RoomDisposition.UNSTABLE, - EventFormatVersions.V3, + EventFormatVersions.ROOM_V4_PLUS, StateResolutionVersions.V2, enforce_key_validity=True, special_case_aliases_auth=False, @@ -290,7 +295,7 @@ class RoomVersions: V10 = RoomVersion( "10", RoomDisposition.STABLE, - EventFormatVersions.V3, + EventFormatVersions.ROOM_V4_PLUS, StateResolutionVersions.V2, enforce_key_validity=True, special_case_aliases_auth=False, @@ -308,7 +313,7 @@ class RoomVersions: MSC2716v4 = RoomVersion( "org.matrix.msc2716v4", RoomDisposition.UNSTABLE, - EventFormatVersions.V3, + EventFormatVersions.ROOM_V4_PLUS, StateResolutionVersions.V2, enforce_key_validity=True, special_case_aliases_auth=False, diff --git a/synapse/event_auth.py b/synapse/event_auth.py index 389b0c5d53..c7d5ef92fc 100644 --- a/synapse/event_auth.py +++ b/synapse/event_auth.py @@ -109,7 +109,7 @@ def validate_event_for_room_version(event: "EventBase") -> None: if not is_invite_via_3pid: raise AuthError(403, "Event not signed by sender's server") - if event.format_version in (EventFormatVersions.V1,): + if event.format_version in (EventFormatVersions.ROOM_V1_V2,): # Only older room versions have event IDs to check. event_id_domain = get_domain_from_id(event.event_id) @@ -716,7 +716,7 @@ def check_redaction( if user_level >= redact_level: return False - if room_version_obj.event_format == EventFormatVersions.V1: + if room_version_obj.event_format == EventFormatVersions.ROOM_V1_V2: redacter_domain = get_domain_from_id(event.event_id) if not isinstance(event.redacts, str): return False diff --git a/synapse/events/__init__.py b/synapse/events/__init__.py index 39ad2793d9..b2c9119fd0 100644 --- a/synapse/events/__init__.py +++ b/synapse/events/__init__.py @@ -442,7 +442,7 @@ class EventBase(metaclass=abc.ABCMeta): class FrozenEvent(EventBase): - format_version = EventFormatVersions.V1 # All events of this type are V1 + format_version = EventFormatVersions.ROOM_V1_V2 # All events of this type are V1 def __init__( self, @@ -490,7 +490,7 @@ class FrozenEvent(EventBase): class FrozenEventV2(EventBase): - format_version = EventFormatVersions.V2 # All events of this type are V2 + format_version = EventFormatVersions.ROOM_V3 # All events of this type are V2 def __init__( self, @@ -567,7 +567,7 @@ class FrozenEventV2(EventBase): class FrozenEventV3(FrozenEventV2): """FrozenEventV3, which differs from FrozenEventV2 only in the event_id format""" - format_version = EventFormatVersions.V3 # All events of this type are V3 + format_version = EventFormatVersions.ROOM_V4_PLUS # All events of this type are V3 @property def event_id(self) -> str: @@ -597,11 +597,11 @@ def _event_type_from_format_version( `FrozenEvent` """ - if format_version == EventFormatVersions.V1: + if format_version == EventFormatVersions.ROOM_V1_V2: return FrozenEvent - elif format_version == EventFormatVersions.V2: + elif format_version == EventFormatVersions.ROOM_V3: return FrozenEventV2 - elif format_version == EventFormatVersions.V3: + elif format_version == EventFormatVersions.ROOM_V4_PLUS: return FrozenEventV3 else: raise Exception("No event format %r" % (format_version,)) diff --git a/synapse/events/builder.py b/synapse/events/builder.py index 17f624b68f..746bd3978d 100644 --- a/synapse/events/builder.py +++ b/synapse/events/builder.py @@ -137,7 +137,7 @@ class EventBuilder: # The types of auth/prev events changes between event versions. prev_events: Union[List[str], List[Tuple[str, Dict[str, str]]]] auth_events: Union[List[str], List[Tuple[str, Dict[str, str]]]] - if format_version == EventFormatVersions.V1: + if format_version == EventFormatVersions.ROOM_V1_V2: auth_events = await self._store.add_event_hashes(auth_event_ids) prev_events = await self._store.add_event_hashes(prev_event_ids) else: @@ -253,7 +253,7 @@ def create_local_event_from_event_dict( time_now = int(clock.time_msec()) - if format_version == EventFormatVersions.V1: + if format_version == EventFormatVersions.ROOM_V1_V2: event_dict["event_id"] = _create_event_id(clock, hostname) event_dict["origin"] = hostname diff --git a/synapse/events/validator.py b/synapse/events/validator.py index 27c8beba25..a6f0104396 100644 --- a/synapse/events/validator.py +++ b/synapse/events/validator.py @@ -45,7 +45,7 @@ class EventValidator: """ self.validate_builder(event) - if event.format_version == EventFormatVersions.V1: + if event.format_version == EventFormatVersions.ROOM_V1_V2: EventID.from_string(event.event_id) required = [ diff --git a/synapse/federation/federation_base.py b/synapse/federation/federation_base.py index 4269a98db2..abe2c1971a 100644 --- a/synapse/federation/federation_base.py +++ b/synapse/federation/federation_base.py @@ -194,7 +194,7 @@ async def _check_sigs_on_pdu( # event id's domain (normally only the case for joins/leaves), and add additional # checks. Only do this if the room version has a concept of event ID domain # (ie, the room version uses old-style non-hash event IDs). - if room_version.event_format == EventFormatVersions.V1: + if room_version.event_format == EventFormatVersions.ROOM_V1_V2: event_domain = get_domain_from_id(pdu.event_id) if event_domain != sender_domain: try: diff --git a/synapse/federation/federation_client.py b/synapse/federation/federation_client.py index 7ee2974bb1..4a4289ee7c 100644 --- a/synapse/federation/federation_client.py +++ b/synapse/federation/federation_client.py @@ -1190,7 +1190,7 @@ class FederationClient(FederationBase): # Otherwise, consider it a legitimate error and raise. err = e.to_synapse_error() if self._is_unknown_endpoint(e, err): - if room_version.event_format != EventFormatVersions.V1: + if room_version.event_format != EventFormatVersions.ROOM_V1_V2: raise SynapseError( 400, "User's homeserver does not support this room version", diff --git a/synapse/storage/databases/main/event_federation.py b/synapse/storage/databases/main/event_federation.py index c836078da6..e687f87eca 100644 --- a/synapse/storage/databases/main/event_federation.py +++ b/synapse/storage/databases/main/event_federation.py @@ -1606,7 +1606,7 @@ class EventFederationWorkerStore(SignatureWorkerStore, EventsWorkerStore, SQLBas logger.info("Invalid prev_events for %s", event_id) continue - if room_version.event_format == EventFormatVersions.V1: + if room_version.event_format == EventFormatVersions.ROOM_V1_V2: for prev_event_tuple in prev_events: if ( not isinstance(prev_event_tuple, list) diff --git a/synapse/storage/databases/main/events_worker.py b/synapse/storage/databases/main/events_worker.py index 9b997c304d..84f17a9945 100644 --- a/synapse/storage/databases/main/events_worker.py +++ b/synapse/storage/databases/main/events_worker.py @@ -1156,7 +1156,7 @@ class EventsWorkerStore(SQLBaseStore): if format_version is None: # This means that we stored the event before we had the concept # of a event format version, so it must be a V1 event. - format_version = EventFormatVersions.V1 + format_version = EventFormatVersions.ROOM_V1_V2 room_version_id = row.room_version_id @@ -1186,10 +1186,10 @@ class EventsWorkerStore(SQLBaseStore): # # So, the following approximations should be adequate. - if format_version == EventFormatVersions.V1: + if format_version == EventFormatVersions.ROOM_V1_V2: # if it's event format v1 then it must be room v1 or v2 room_version = RoomVersions.V1 - elif format_version == EventFormatVersions.V2: + elif format_version == EventFormatVersions.ROOM_V3: # if it's event format v2 then it must be room v3 room_version = RoomVersions.V3 else: diff --git a/tests/storage/databases/main/test_events_worker.py b/tests/storage/databases/main/test_events_worker.py index 46d829b062..67401272ac 100644 --- a/tests/storage/databases/main/test_events_worker.py +++ b/tests/storage/databases/main/test_events_worker.py @@ -254,7 +254,7 @@ class DatabaseOutageTestCase(unittest.HomeserverTestCase): "room_id": self.room_id, "json": json.dumps(event_json), "internal_metadata": "{}", - "format_version": EventFormatVersions.V3, + "format_version": EventFormatVersions.ROOM_V4_PLUS, }, ) ) diff --git a/tests/storage/test_event_federation.py b/tests/storage/test_event_federation.py index d92a9ac5b7..a6679e1312 100644 --- a/tests/storage/test_event_federation.py +++ b/tests/storage/test_event_federation.py @@ -513,7 +513,7 @@ class EventFederationWorkerStoreTestCase(tests.unittest.HomeserverTestCase): def prev_event_format(prev_event_id: str) -> Union[Tuple[str, dict], str]: """Account for differences in prev_events format across room versions""" - if room_version.event_format == EventFormatVersions.V1: + if room_version.event_format == EventFormatVersions.ROOM_V1_V2: return prev_event_id, {} return prev_event_id diff --git a/tests/test_event_auth.py b/tests/test_event_auth.py index e42d7b9ba0..f4d9fba0a1 100644 --- a/tests/test_event_auth.py +++ b/tests/test_event_auth.py @@ -821,7 +821,7 @@ def _alias_event(room_version: RoomVersion, sender: str, **kwargs) -> EventBase: def _build_auth_dict_for_room_version( room_version: RoomVersion, auth_events: Iterable[EventBase] ) -> List: - if room_version.event_format == EventFormatVersions.V1: + if room_version.event_format == EventFormatVersions.ROOM_V1_V2: return [(e.event_id, "not_used") for e in auth_events] else: return [e.event_id for e in auth_events] @@ -871,7 +871,7 @@ event_count = 0 def _maybe_get_event_id_dict_for_room_version(room_version: RoomVersion) -> dict: """If this room version needs it, generate an event id""" - if room_version.event_format != EventFormatVersions.V1: + if room_version.event_format != EventFormatVersions.ROOM_V1_V2: return {} global event_count From d3d9ca156e323fe194b1bcb1af1628f65a2f3c1c Mon Sep 17 00:00:00 2001 From: reivilibre Date: Wed, 7 Sep 2022 11:03:32 +0000 Subject: [PATCH 069/107] Cancel the processing of key query requests when they time out. (#13680) --- changelog.d/13680.feature | 1 + synapse/api/auth.py | 5 +++ synapse/handlers/device.py | 3 ++ synapse/handlers/e2e_keys.py | 40 +++++++++++-------- synapse/rest/client/keys.py | 6 ++- synapse/storage/controllers/state.py | 4 ++ synapse/storage/databases/main/devices.py | 4 ++ .../storage/databases/main/end_to_end_keys.py | 5 ++- .../databases/main/event_federation.py | 2 + .../storage/databases/main/events_worker.py | 4 ++ synapse/storage/databases/main/roommember.py | 2 + synapse/storage/databases/main/state.py | 2 + synapse/storage/databases/main/stream.py | 2 + synapse/storage/databases/state/store.py | 3 ++ .../util/partial_state_events_tracker.py | 3 ++ synapse/types.py | 5 +++ tests/http/server/_base.py | 10 ++++- tests/rest/client/test_keys.py | 29 ++++++++++++++ 18 files changed, 110 insertions(+), 20 deletions(-) create mode 100644 changelog.d/13680.feature diff --git a/changelog.d/13680.feature b/changelog.d/13680.feature new file mode 100644 index 0000000000..4234c7e082 --- /dev/null +++ b/changelog.d/13680.feature @@ -0,0 +1 @@ +Cancel the processing of key query requests when they time out. \ No newline at end of file diff --git a/synapse/api/auth.py b/synapse/api/auth.py index 9a1aea083f..8e54ef84b2 100644 --- a/synapse/api/auth.py +++ b/synapse/api/auth.py @@ -38,6 +38,7 @@ from synapse.logging.opentracing import ( trace, ) from synapse.types import Requester, create_requester +from synapse.util.cancellation import cancellable if TYPE_CHECKING: from synapse.server import HomeServer @@ -118,6 +119,7 @@ class Auth: errcode=Codes.NOT_JOINED, ) + @cancellable async def get_user_by_req( self, request: SynapseRequest, @@ -166,6 +168,7 @@ class Auth: parent_span.set_tag("appservice_id", requester.app_service.id) return requester + @cancellable async def _wrapped_get_user_by_req( self, request: SynapseRequest, @@ -281,6 +284,7 @@ class Auth: 403, "Application service has not registered this user (%s)" % user_id ) + @cancellable async def _get_appservice_user(self, request: Request) -> Optional[Requester]: """ Given a request, reads the request parameters to determine: @@ -523,6 +527,7 @@ class Auth: return bool(query_params) or bool(auth_headers) @staticmethod + @cancellable def get_access_token_from_request(request: Request) -> str: """Extracts the access_token from the request. diff --git a/synapse/handlers/device.py b/synapse/handlers/device.py index 9c2c3a0e68..c5ac169644 100644 --- a/synapse/handlers/device.py +++ b/synapse/handlers/device.py @@ -52,6 +52,7 @@ from synapse.types import ( from synapse.util import stringutils from synapse.util.async_helpers import Linearizer from synapse.util.caches.expiringcache import ExpiringCache +from synapse.util.cancellation import cancellable from synapse.util.metrics import measure_func from synapse.util.retryutils import NotRetryingDestination @@ -124,6 +125,7 @@ class DeviceWorkerHandler: return device + @cancellable async def get_device_changes_in_shared_rooms( self, user_id: str, room_ids: Collection[str], from_token: StreamToken ) -> Collection[str]: @@ -163,6 +165,7 @@ class DeviceWorkerHandler: @trace @measure_func("device.get_user_ids_changed") + @cancellable async def get_user_ids_changed( self, user_id: str, from_token: StreamToken ) -> JsonDict: diff --git a/synapse/handlers/e2e_keys.py b/synapse/handlers/e2e_keys.py index c938339ddd..ec81639c78 100644 --- a/synapse/handlers/e2e_keys.py +++ b/synapse/handlers/e2e_keys.py @@ -37,7 +37,8 @@ from synapse.types import ( get_verify_key_from_cross_signing_key, ) from synapse.util import json_decoder, unwrapFirstError -from synapse.util.async_helpers import Linearizer +from synapse.util.async_helpers import Linearizer, delay_cancellation +from synapse.util.cancellation import cancellable from synapse.util.retryutils import NotRetryingDestination if TYPE_CHECKING: @@ -91,6 +92,7 @@ class E2eKeysHandler: ) @trace + @cancellable async def query_devices( self, query_body: JsonDict, @@ -208,22 +210,26 @@ class E2eKeysHandler: r[user_id] = remote_queries[user_id] # Now fetch any devices that we don't have in our cache + # TODO It might make sense to propagate cancellations into the + # deferreds which are querying remote homeservers. await make_deferred_yieldable( - defer.gatherResults( - [ - run_in_background( - self._query_devices_for_destination, - results, - cross_signing_keys, - failures, - destination, - queries, - timeout, - ) - for destination, queries in remote_queries_not_in_cache.items() - ], - consumeErrors=True, - ).addErrback(unwrapFirstError) + delay_cancellation( + defer.gatherResults( + [ + run_in_background( + self._query_devices_for_destination, + results, + cross_signing_keys, + failures, + destination, + queries, + timeout, + ) + for destination, queries in remote_queries_not_in_cache.items() + ], + consumeErrors=True, + ).addErrback(unwrapFirstError) + ) ) ret = {"device_keys": results, "failures": failures} @@ -347,6 +353,7 @@ class E2eKeysHandler: return + @cancellable async def get_cross_signing_keys_from_cache( self, query: Iterable[str], from_user_id: Optional[str] ) -> Dict[str, Dict[str, dict]]: @@ -393,6 +400,7 @@ class E2eKeysHandler: } @trace + @cancellable async def query_local_devices( self, query: Mapping[str, Optional[List[str]]] ) -> Dict[str, Dict[str, dict]]: diff --git a/synapse/rest/client/keys.py b/synapse/rest/client/keys.py index a395694fa5..f653d2a3e1 100644 --- a/synapse/rest/client/keys.py +++ b/synapse/rest/client/keys.py @@ -27,9 +27,9 @@ from synapse.http.servlet import ( ) from synapse.http.site import SynapseRequest from synapse.logging.opentracing import log_kv, set_tag +from synapse.rest.client._base import client_patterns, interactive_auth_handler from synapse.types import JsonDict, StreamToken - -from ._base import client_patterns, interactive_auth_handler +from synapse.util.cancellation import cancellable if TYPE_CHECKING: from synapse.server import HomeServer @@ -156,6 +156,7 @@ class KeyQueryServlet(RestServlet): self.auth = hs.get_auth() self.e2e_keys_handler = hs.get_e2e_keys_handler() + @cancellable async def on_POST(self, request: SynapseRequest) -> Tuple[int, JsonDict]: requester = await self.auth.get_user_by_req(request, allow_guest=True) user_id = requester.user.to_string() @@ -199,6 +200,7 @@ class KeyChangesServlet(RestServlet): self.device_handler = hs.get_device_handler() self.store = hs.get_datastores().main + @cancellable async def on_GET(self, request: SynapseRequest) -> Tuple[int, JsonDict]: requester = await self.auth.get_user_by_req(request, allow_guest=True) diff --git a/synapse/storage/controllers/state.py b/synapse/storage/controllers/state.py index ba5380ce3e..bbe568bf05 100644 --- a/synapse/storage/controllers/state.py +++ b/synapse/storage/controllers/state.py @@ -36,6 +36,7 @@ from synapse.storage.util.partial_state_events_tracker import ( PartialStateEventsTracker, ) from synapse.types import MutableStateMap, StateMap +from synapse.util.cancellation import cancellable if TYPE_CHECKING: from synapse.server import HomeServer @@ -229,6 +230,7 @@ class StateStorageController: @trace @tag_args + @cancellable async def get_state_ids_for_events( self, event_ids: Collection[str], @@ -350,6 +352,7 @@ class StateStorageController: @trace @tag_args + @cancellable async def get_state_group_for_events( self, event_ids: Collection[str], @@ -398,6 +401,7 @@ class StateStorageController: event_id, room_id, prev_group, delta_ids, current_state_ids ) + @cancellable async def get_current_state_ids( self, room_id: str, diff --git a/synapse/storage/databases/main/devices.py b/synapse/storage/databases/main/devices.py index ca0fe8c4be..5d700ca6c3 100644 --- a/synapse/storage/databases/main/devices.py +++ b/synapse/storage/databases/main/devices.py @@ -53,6 +53,7 @@ from synapse.util import json_decoder, json_encoder from synapse.util.caches.descriptors import cached, cachedList from synapse.util.caches.lrucache import LruCache from synapse.util.caches.stream_change_cache import StreamChangeCache +from synapse.util.cancellation import cancellable from synapse.util.iterutils import batch_iter from synapse.util.stringutils import shortstr @@ -668,6 +669,7 @@ class DeviceWorkerStore(EndToEndKeyWorkerStore): ... @trace + @cancellable async def get_user_devices_from_cache( self, query_list: List[Tuple[str, Optional[str]]] ) -> Tuple[Set[str], Dict[str, Dict[str, JsonDict]]]: @@ -743,6 +745,7 @@ class DeviceWorkerStore(EndToEndKeyWorkerStore): return self._device_list_stream_cache.get_all_entities_changed(from_key) + @cancellable async def get_users_whose_devices_changed( self, from_key: int, @@ -1221,6 +1224,7 @@ class DeviceWorkerStore(EndToEndKeyWorkerStore): desc="get_min_device_lists_changes_in_room", ) + @cancellable async def get_device_list_changes_in_rooms( self, room_ids: Collection[str], from_id: int ) -> Optional[Set[str]]: diff --git a/synapse/storage/databases/main/end_to_end_keys.py b/synapse/storage/databases/main/end_to_end_keys.py index 46c0d06157..8e9e1b0b4b 100644 --- a/synapse/storage/databases/main/end_to_end_keys.py +++ b/synapse/storage/databases/main/end_to_end_keys.py @@ -50,6 +50,7 @@ from synapse.storage.util.id_generators import StreamIdGenerator from synapse.types import JsonDict from synapse.util import json_encoder from synapse.util.caches.descriptors import cached, cachedList +from synapse.util.cancellation import cancellable from synapse.util.iterutils import batch_iter if TYPE_CHECKING: @@ -135,6 +136,7 @@ class EndToEndKeyWorkerStore(EndToEndKeyBackgroundStore, CacheInvalidationWorker return now_stream_id, [] @trace + @cancellable async def get_e2e_device_keys_for_cs_api( self, query_list: List[Tuple[str, Optional[str]]] ) -> Dict[str, Dict[str, JsonDict]]: @@ -197,6 +199,7 @@ class EndToEndKeyWorkerStore(EndToEndKeyBackgroundStore, CacheInvalidationWorker ... @trace + @cancellable async def get_e2e_device_keys_and_signatures( self, query_list: Collection[Tuple[str, Optional[str]]], @@ -887,6 +890,7 @@ class EndToEndKeyWorkerStore(EndToEndKeyBackgroundStore, CacheInvalidationWorker return keys + @cancellable async def get_e2e_cross_signing_keys_bulk( self, user_ids: List[str], from_user_id: Optional[str] = None ) -> Dict[str, Optional[Dict[str, JsonDict]]]: @@ -902,7 +906,6 @@ class EndToEndKeyWorkerStore(EndToEndKeyBackgroundStore, CacheInvalidationWorker keys were not found, either their user ID will not be in the dict, or their user ID will map to None. """ - result = await self._get_bare_e2e_cross_signing_keys_bulk(user_ids) if from_user_id: diff --git a/synapse/storage/databases/main/event_federation.py b/synapse/storage/databases/main/event_federation.py index e687f87eca..ca47a22bf1 100644 --- a/synapse/storage/databases/main/event_federation.py +++ b/synapse/storage/databases/main/event_federation.py @@ -48,6 +48,7 @@ from synapse.types import JsonDict from synapse.util import json_encoder from synapse.util.caches.descriptors import cached from synapse.util.caches.lrucache import LruCache +from synapse.util.cancellation import cancellable from synapse.util.iterutils import batch_iter if TYPE_CHECKING: @@ -976,6 +977,7 @@ class EventFederationWorkerStore(SignatureWorkerStore, EventsWorkerStore, SQLBas return int(min_depth) if min_depth is not None else None + @cancellable async def get_forward_extremities_for_room_at_stream_ordering( self, room_id: str, stream_ordering: int ) -> List[str]: diff --git a/synapse/storage/databases/main/events_worker.py b/synapse/storage/databases/main/events_worker.py index 84f17a9945..52914febf9 100644 --- a/synapse/storage/databases/main/events_worker.py +++ b/synapse/storage/databases/main/events_worker.py @@ -81,6 +81,7 @@ from synapse.util import unwrapFirstError from synapse.util.async_helpers import ObservableDeferred, delay_cancellation from synapse.util.caches.descriptors import cached, cachedList from synapse.util.caches.lrucache import AsyncLruCache +from synapse.util.cancellation import cancellable from synapse.util.iterutils import batch_iter from synapse.util.metrics import Measure @@ -339,6 +340,7 @@ class EventsWorkerStore(SQLBaseStore): ) -> Optional[EventBase]: ... + @cancellable async def get_event( self, event_id: str, @@ -433,6 +435,7 @@ class EventsWorkerStore(SQLBaseStore): @trace @tag_args + @cancellable async def get_events_as_list( self, event_ids: Collection[str], @@ -584,6 +587,7 @@ class EventsWorkerStore(SQLBaseStore): return events + @cancellable async def _get_events_from_cache_or_db( self, event_ids: Iterable[str], allow_rejected: bool = False ) -> Dict[str, EventCacheEntry]: diff --git a/synapse/storage/databases/main/roommember.py b/synapse/storage/databases/main/roommember.py index 4f0adb136a..a77e49dc66 100644 --- a/synapse/storage/databases/main/roommember.py +++ b/synapse/storage/databases/main/roommember.py @@ -55,6 +55,7 @@ from synapse.types import JsonDict, PersistedEventPosition, StateMap, get_domain from synapse.util.async_helpers import Linearizer from synapse.util.caches import intern_string from synapse.util.caches.descriptors import _CacheContext, cached, cachedList +from synapse.util.cancellation import cancellable from synapse.util.iterutils import batch_iter from synapse.util.metrics import Measure @@ -770,6 +771,7 @@ class RoomMemberWorkerStore(EventsWorkerStore): _get_users_server_still_shares_room_with_txn, ) + @cancellable async def get_rooms_for_user( self, user_id: str, on_invalidate: Optional[Callable[[], None]] = None ) -> FrozenSet[str]: diff --git a/synapse/storage/databases/main/state.py b/synapse/storage/databases/main/state.py index 0b10af0e58..e607ccfdc9 100644 --- a/synapse/storage/databases/main/state.py +++ b/synapse/storage/databases/main/state.py @@ -36,6 +36,7 @@ from synapse.storage.state import StateFilter from synapse.types import JsonDict, JsonMapping, StateMap from synapse.util.caches import intern_string from synapse.util.caches.descriptors import cached, cachedList +from synapse.util.cancellation import cancellable from synapse.util.iterutils import batch_iter if TYPE_CHECKING: @@ -281,6 +282,7 @@ class StateGroupWorkerStore(EventsWorkerStore, SQLBaseStore): ) # FIXME: how should this be cached? + @cancellable async def get_partial_filtered_current_state_ids( self, room_id: str, state_filter: Optional[StateFilter] = None ) -> StateMap[str]: diff --git a/synapse/storage/databases/main/stream.py b/synapse/storage/databases/main/stream.py index a347430aa7..3f9bfaeac5 100644 --- a/synapse/storage/databases/main/stream.py +++ b/synapse/storage/databases/main/stream.py @@ -72,6 +72,7 @@ from synapse.storage.util.id_generators import MultiWriterIdGenerator from synapse.types import PersistedEventPosition, RoomStreamToken from synapse.util.caches.descriptors import cached from synapse.util.caches.stream_change_cache import StreamChangeCache +from synapse.util.cancellation import cancellable if TYPE_CHECKING: from synapse.server import HomeServer @@ -597,6 +598,7 @@ class StreamWorkerStore(EventsWorkerStore, SQLBaseStore): return ret, key + @cancellable async def get_membership_changes_for_user( self, user_id: str, diff --git a/synapse/storage/databases/state/store.py b/synapse/storage/databases/state/store.py index bb64543c1f..f8cfcaca83 100644 --- a/synapse/storage/databases/state/store.py +++ b/synapse/storage/databases/state/store.py @@ -31,6 +31,7 @@ from synapse.storage.util.sequence import build_sequence_generator from synapse.types import MutableStateMap, StateKey, StateMap from synapse.util.caches.descriptors import cached from synapse.util.caches.dictionary_cache import DictionaryCache +from synapse.util.cancellation import cancellable if TYPE_CHECKING: from synapse.server import HomeServer @@ -156,6 +157,7 @@ class StateGroupDataStore(StateBackgroundUpdateStore, SQLBaseStore): "get_state_group_delta", _get_state_group_delta_txn ) + @cancellable async def _get_state_groups_from_groups( self, groups: List[int], state_filter: StateFilter ) -> Dict[int, StateMap[str]]: @@ -235,6 +237,7 @@ class StateGroupDataStore(StateBackgroundUpdateStore, SQLBaseStore): return state_filter.filter_state(state_dict_ids), not missing_types + @cancellable async def _get_state_for_groups( self, groups: Iterable[int], state_filter: Optional[StateFilter] = None ) -> Dict[int, MutableStateMap[str]]: diff --git a/synapse/storage/util/partial_state_events_tracker.py b/synapse/storage/util/partial_state_events_tracker.py index b4bf49dace..8d8894d1d5 100644 --- a/synapse/storage/util/partial_state_events_tracker.py +++ b/synapse/storage/util/partial_state_events_tracker.py @@ -24,6 +24,7 @@ from synapse.logging.opentracing import trace_with_opname from synapse.storage.databases.main.events_worker import EventsWorkerStore from synapse.storage.databases.main.room import RoomWorkerStore from synapse.util import unwrapFirstError +from synapse.util.cancellation import cancellable logger = logging.getLogger(__name__) @@ -60,6 +61,7 @@ class PartialStateEventsTracker: o.callback(None) @trace_with_opname("PartialStateEventsTracker.await_full_state") + @cancellable async def await_full_state(self, event_ids: Collection[str]) -> None: """Wait for all the given events to have full state. @@ -154,6 +156,7 @@ class PartialCurrentStateTracker: o.callback(None) @trace_with_opname("PartialCurrentStateTracker.await_full_state") + @cancellable async def await_full_state(self, room_id: str) -> None: # We add the deferred immediately so that the DB call to check for # partial state doesn't race when we unpartial the room. diff --git a/synapse/types.py b/synapse/types.py index 668d48d646..ec44601f54 100644 --- a/synapse/types.py +++ b/synapse/types.py @@ -52,6 +52,7 @@ from twisted.internet.interfaces import ( ) from synapse.api.errors import Codes, SynapseError +from synapse.util.cancellation import cancellable from synapse.util.stringutils import parse_and_validate_server_name if TYPE_CHECKING: @@ -699,7 +700,11 @@ class StreamToken: START: ClassVar["StreamToken"] @classmethod + @cancellable async def from_string(cls, store: "DataStore", string: str) -> "StreamToken": + """ + Creates a RoomStreamToken from its textual representation. + """ try: keys = string.split(cls._SEPARATOR) while len(keys) < len(attr.fields(cls)): diff --git a/tests/http/server/_base.py b/tests/http/server/_base.py index 5726e60cee..5071f83574 100644 --- a/tests/http/server/_base.py +++ b/tests/http/server/_base.py @@ -140,6 +140,8 @@ def make_request_with_cancellation_test( method: str, path: str, content: Union[bytes, str, JsonDict] = b"", + *, + token: Optional[str] = None, ) -> FakeChannel: """Performs a request repeatedly, disconnecting at successive `await`s, until one completes. @@ -211,7 +213,13 @@ def make_request_with_cancellation_test( with deferred_patch.patch(): # Start the request. channel = make_request( - reactor, site, method, path, content, await_result=False + reactor, + site, + method, + path, + content, + await_result=False, + access_token=token, ) request = channel.request diff --git a/tests/rest/client/test_keys.py b/tests/rest/client/test_keys.py index bbc8e74243..741fecea77 100644 --- a/tests/rest/client/test_keys.py +++ b/tests/rest/client/test_keys.py @@ -19,6 +19,7 @@ from synapse.rest import admin from synapse.rest.client import keys, login from tests import unittest +from tests.http.server._base import make_request_with_cancellation_test class KeyQueryTestCase(unittest.HomeserverTestCase): @@ -89,3 +90,31 @@ class KeyQueryTestCase(unittest.HomeserverTestCase): Codes.BAD_JSON, channel.result, ) + + def test_key_query_cancellation(self) -> None: + """ + Tests that /keys/query is cancellable and does not swallow the + CancelledError. + """ + self.register_user("alice", "wonderland") + alice_token = self.login("alice", "wonderland") + + bob = self.register_user("bob", "uncle") + + channel = make_request_with_cancellation_test( + "test_key_query_cancellation", + self.reactor, + self.site, + "POST", + "/_matrix/client/r0/keys/query", + { + "device_keys": { + # Empty list means we request keys for all bob's devices + bob: [], + }, + }, + token=alice_token, + ) + + self.assertEqual(200, channel.code, msg=channel.result["body"]) + self.assertIn(bob, channel.json_body["device_keys"]) From b58386e37e30e920332e4b04011b528a66a39fad Mon Sep 17 00:00:00 2001 From: David Robertson Date: Wed, 7 Sep 2022 12:16:10 +0100 Subject: [PATCH 070/107] A second batch of Pydantic models for rest/client/account.py (#13687) --- changelog.d/13687.feature | 1 + synapse/http/servlet.py | 19 ++++++++++-- synapse/rest/client/account.py | 54 ++++++++++++++++------------------ synapse/rest/client/models.py | 24 ++++++++++++--- 4 files changed, 64 insertions(+), 34 deletions(-) create mode 100644 changelog.d/13687.feature diff --git a/changelog.d/13687.feature b/changelog.d/13687.feature new file mode 100644 index 0000000000..dac53ec122 --- /dev/null +++ b/changelog.d/13687.feature @@ -0,0 +1 @@ +Improve validation of request bodies for the following client-server API endpoints: [`/account/3pid/msisdn/requestToken`](https://spec.matrix.org/v1.3/client-server-api/#post_matrixclientv3account3pidmsisdnrequesttoken) and [`/org.matrix.msc3720/account_status`](https://github.com/matrix-org/matrix-spec-proposals/blob/babolivier/user_status/proposals/3720-account-status.md#post-_matrixclientv1account_status). \ No newline at end of file diff --git a/synapse/http/servlet.py b/synapse/http/servlet.py index 26aaabfb34..80acbdcf3c 100644 --- a/synapse/http/servlet.py +++ b/synapse/http/servlet.py @@ -28,7 +28,8 @@ from typing import ( overload, ) -from pydantic import BaseModel, ValidationError +from pydantic import BaseModel, MissingError, PydanticValueError, ValidationError +from pydantic.error_wrappers import ErrorWrapper from typing_extensions import Literal from twisted.web.server import Request @@ -714,7 +715,21 @@ def parse_and_validate_json_object_from_request( try: instance = model_type.parse_obj(content) except ValidationError as e: - raise SynapseError(HTTPStatus.BAD_REQUEST, str(e), errcode=Codes.BAD_JSON) + # Choose a matrix error code. The catch-all is BAD_JSON, but we try to find a + # more specific error if possible (which occasionally helps us to be spec- + # compliant) This is a bit awkward because the spec's error codes aren't very + # clear-cut: BAD_JSON arguably overlaps with MISSING_PARAM and INVALID_PARAM. + errcode = Codes.BAD_JSON + + raw_errors = e.raw_errors + if len(raw_errors) == 1 and isinstance(raw_errors[0], ErrorWrapper): + raw_error = raw_errors[0].exc + if isinstance(raw_error, MissingError): + errcode = Codes.MISSING_PARAM + elif isinstance(raw_error, PydanticValueError): + errcode = Codes.INVALID_PARAM + + raise SynapseError(HTTPStatus.BAD_REQUEST, str(e), errcode=errcode) return instance diff --git a/synapse/rest/client/account.py b/synapse/rest/client/account.py index 1f9a8ccc23..a09aaf3448 100644 --- a/synapse/rest/client/account.py +++ b/synapse/rest/client/account.py @@ -15,7 +15,7 @@ # limitations under the License. import logging import random -from typing import TYPE_CHECKING, Optional, Tuple +from typing import TYPE_CHECKING, List, Optional, Tuple from urllib.parse import urlparse from pydantic import StrictBool, StrictStr, constr @@ -41,7 +41,11 @@ from synapse.http.servlet import ( from synapse.http.site import SynapseRequest from synapse.metrics import threepid_send_requests from synapse.push.mailer import Mailer -from synapse.rest.client.models import AuthenticationData, EmailRequestTokenBody +from synapse.rest.client.models import ( + AuthenticationData, + EmailRequestTokenBody, + MsisdnRequestTokenBody, +) from synapse.rest.models import RequestBodyModel from synapse.types import JsonDict from synapse.util.msisdn import phone_number_to_msisdn @@ -400,23 +404,16 @@ class MsisdnThreepidRequestTokenRestServlet(RestServlet): self.identity_handler = hs.get_identity_handler() async def on_POST(self, request: SynapseRequest) -> Tuple[int, JsonDict]: - body = parse_json_object_from_request(request) - assert_params_in_dict( - body, ["client_secret", "country", "phone_number", "send_attempt"] + body = parse_and_validate_json_object_from_request( + request, MsisdnRequestTokenBody ) - client_secret = body["client_secret"] - assert_valid_client_secret(client_secret) - - country = body["country"] - phone_number = body["phone_number"] - send_attempt = body["send_attempt"] - next_link = body.get("next_link") # Optional param - - msisdn = phone_number_to_msisdn(country, phone_number) + msisdn = phone_number_to_msisdn(body.country, body.phone_number) if not await check_3pid_allowed(self.hs, "msisdn", msisdn): raise SynapseError( 403, + # TODO: is this error message accurate? Looks like we've only rejected + # this phone number, not necessarily all phone numbers "Account phone numbers are not authorized on this server", Codes.THREEPID_DENIED, ) @@ -425,9 +422,9 @@ class MsisdnThreepidRequestTokenRestServlet(RestServlet): request, "msisdn", msisdn ) - if next_link: + if body.next_link: # Raise if the provided next_link value isn't valid - assert_valid_next_link(self.hs, next_link) + assert_valid_next_link(self.hs, body.next_link) existing_user_id = await self.store.get_user_id_by_threepid("msisdn", msisdn) @@ -454,15 +451,15 @@ class MsisdnThreepidRequestTokenRestServlet(RestServlet): ret = await self.identity_handler.requestMsisdnToken( self.hs.config.registration.account_threepid_delegate_msisdn, - country, - phone_number, - client_secret, - send_attempt, - next_link, + body.country, + body.phone_number, + body.client_secret, + body.send_attempt, + body.next_link, ) threepid_send_requests.labels(type="msisdn", reason="add_threepid").observe( - send_attempt + body.send_attempt ) return 200, ret @@ -845,17 +842,18 @@ class AccountStatusRestServlet(RestServlet): self._auth = hs.get_auth() self._account_handler = hs.get_account_handler() + class PostBody(RequestBodyModel): + # TODO: we could validate that each user id is an mxid here, and/or parse it + # as a UserID + user_ids: List[StrictStr] + async def on_POST(self, request: SynapseRequest) -> Tuple[int, JsonDict]: await self._auth.get_user_by_req(request) - body = parse_json_object_from_request(request) - if "user_ids" not in body: - raise SynapseError( - 400, "Required parameter 'user_ids' is missing", Codes.MISSING_PARAM - ) + body = parse_and_validate_json_object_from_request(request, self.PostBody) statuses, failures = await self._account_handler.get_account_statuses( - body["user_ids"], + body.user_ids, allow_remote=True, ) diff --git a/synapse/rest/client/models.py b/synapse/rest/client/models.py index 3150602997..6278450c70 100644 --- a/synapse/rest/client/models.py +++ b/synapse/rest/client/models.py @@ -25,8 +25,8 @@ class AuthenticationData(RequestBodyModel): (The name "Authentication Data" is taken directly from the spec.) - Additional keys will be present, depending on the `type` field. Use `.dict()` to - access them. + Additional keys will be present, depending on the `type` field. Use + `.dict(exclude_unset=True)` to access them. """ class Config: @@ -36,7 +36,7 @@ class AuthenticationData(RequestBodyModel): type: Optional[StrictStr] = None -class EmailRequestTokenBody(RequestBodyModel): +class ThreePidRequestTokenBody(RequestBodyModel): if TYPE_CHECKING: client_secret: StrictStr else: @@ -47,7 +47,7 @@ class EmailRequestTokenBody(RequestBodyModel): max_length=255, strict=True, ) - email: StrictStr + id_server: Optional[StrictStr] id_access_token: Optional[StrictStr] next_link: Optional[StrictStr] @@ -61,9 +61,25 @@ class EmailRequestTokenBody(RequestBodyModel): raise ValueError("id_access_token is required if an id_server is supplied.") return token + +class EmailRequestTokenBody(ThreePidRequestTokenBody): + email: StrictStr + # Canonicalise the email address. The addresses are all stored canonicalised # in the database. This allows the user to reset his password without having to # know the exact spelling (eg. upper and lower case) of address in the database. # Without this, an email stored in the database as "foo@bar.com" would cause # user requests for "FOO@bar.com" to raise a Not Found error. _email_validator = validator("email", allow_reuse=True)(validate_email) + + +if TYPE_CHECKING: + ISO3116_1_Alpha_2 = StrictStr +else: + # Per spec: two-letter uppercase ISO-3166-1-alpha-2 + ISO3116_1_Alpha_2 = constr(regex="[A-Z]{2}", strict=True) + + +class MsisdnRequestTokenBody(ThreePidRequestTokenBody): + country: ISO3116_1_Alpha_2 + phone_number: StrictStr From 77f39864511d0c0217d41691cc5395fde9e0bced Mon Sep 17 00:00:00 2001 From: David Robertson Date: Wed, 7 Sep 2022 13:07:42 +0100 Subject: [PATCH 071/107] Define SQLite compat policy (#13728) --- changelog.d/13728.doc | 1 + docs/deprecation_policy.md | 15 +++++++++++---- 2 files changed, 12 insertions(+), 4 deletions(-) create mode 100644 changelog.d/13728.doc diff --git a/changelog.d/13728.doc b/changelog.d/13728.doc new file mode 100644 index 0000000000..75ca7b7ec3 --- /dev/null +++ b/changelog.d/13728.doc @@ -0,0 +1 @@ +Define Synapse's compatability policy for SQLite versions. diff --git a/docs/deprecation_policy.md b/docs/deprecation_policy.md index b8a46e3d60..46c18d7d32 100644 --- a/docs/deprecation_policy.md +++ b/docs/deprecation_policy.md @@ -1,9 +1,9 @@ Deprecation Policy for Platform Dependencies ============================================ -Synapse has a number of platform dependencies, including Python and PostgreSQL. -This document outlines the policy towards which versions we support, and when we -drop support for versions in the future. +Synapse has a number of platform dependencies, including Python, Rust, +PostgreSQL and SQLite. This document outlines the policy towards which versions +we support, and when we drop support for versions in the future. Policy @@ -17,12 +17,14 @@ Details on the upstream support life cycles for Python and PostgreSQL are documented at [https://endoflife.date/python](https://endoflife.date/python) and [https://endoflife.date/postgresql](https://endoflife.date/postgresql). - A Rust compiler is required to build Synapse from source. For any given release the minimum required version may be bumped up to a recent Rust version, and so people building from source should ensure they can fetch recent versions of Rust (e.g. by using [rustup](https://rustup.rs/)). +The oldest supported version of SQLite is the version +[provided](https://packages.debian.org/buster/libsqlite3-0) by +[Debian oldstable](https://wiki.debian.org/DebianOldStable). Context ------- @@ -44,3 +46,8 @@ generally bump their minimum support Rust versions frequently. In general, the Synapse team will try to avoid updating the dependency on Rust to the absolute latest version, but introducing a formal policy is hard given the constraints of the ecosystem. + +On a similar note, SQLite does not generally have a concept of "supported +release"; bugfixes are published for the latest minor release only. We chose to +track Debian's oldstable as this is relatively conservative, predictably updated +and is consistent with the `.deb` packages released by Matrix.org. \ No newline at end of file From c46fecd1f29b1d2b8756ab2dc5979c24822bfb93 Mon Sep 17 00:00:00 2001 From: David Robertson Date: Wed, 7 Sep 2022 14:46:11 +0100 Subject: [PATCH 072/107] Correct out-of-date doc for `event_cache_size` (#13726) --- changelog.d/13726.doc | 1 + docs/usage/configuration/config_documentation.md | 6 ++++-- 2 files changed, 5 insertions(+), 2 deletions(-) create mode 100644 changelog.d/13726.doc diff --git a/changelog.d/13726.doc b/changelog.d/13726.doc new file mode 100644 index 0000000000..ab840e1a92 --- /dev/null +++ b/changelog.d/13726.doc @@ -0,0 +1 @@ +Fix a mistake in the config manual: the `event_cache_size` _is_ scaled by `caches.global_factor`. The documentation was incorrect since Synapse 1.22. diff --git a/docs/usage/configuration/config_documentation.md b/docs/usage/configuration/config_documentation.md index 757957a1d5..ae490d13a4 100644 --- a/docs/usage/configuration/config_documentation.md +++ b/docs/usage/configuration/config_documentation.md @@ -1069,8 +1069,10 @@ Options related to caching. --- ### `event_cache_size` -The number of events to cache in memory. Not affected by -`caches.global_factor` and is not part of the `caches` section. Defaults to 10K. +The number of events to cache in memory. Defaults to 10K. Like other caches, +this is affected by `caches.global_factor` (see below). + +Note that this option is not part of the `caches` section. Example configuration: ```yaml From dc0e896b68beb7518fc81f897ac1006bf9f1858c Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 7 Sep 2022 14:56:59 +0100 Subject: [PATCH 073/107] Add some rust caching to CI (#13735) --- .github/workflows/tests.yml | 5 +++++ changelog.d/13735.misc | 1 + 2 files changed, 6 insertions(+) create mode 100644 changelog.d/13735.misc diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 5f96bdfa7f..50dc8e30d4 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -144,6 +144,7 @@ jobs: with: toolchain: 1.61.0 override: true + - uses: Swatinem/rust-cache@v2 # There aren't wheels for some of the older deps, so we need to install # their build dependencies @@ -253,11 +254,14 @@ jobs: - uses: actions/checkout@v2 - name: Prepare test blacklist run: cat sytest-blacklist .ci/worker-blacklist > synapse-blacklist-with-workers + - name: Install Rust uses: actions-rs/toolchain@v1 with: toolchain: 1.61.0 override: true + - uses: Swatinem/rust-cache@v2 + - name: Run SyTest run: /bootstrap.sh synapse working-directory: /src @@ -369,6 +373,7 @@ jobs: with: toolchain: 1.61.0 override: true + - uses: Swatinem/rust-cache@v2 - name: Prepare Complement's Prerequisites run: synapse/.ci/scripts/setup_complement_prerequisites.sh diff --git a/changelog.d/13735.misc b/changelog.d/13735.misc new file mode 100644 index 0000000000..2e0dd68a0f --- /dev/null +++ b/changelog.d/13735.misc @@ -0,0 +1 @@ +Add a stub Rust crate. From 8d7fcf9b761941d73af4f150e1173cafe86714a2 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Wed, 7 Sep 2022 15:07:06 +0100 Subject: [PATCH 074/107] Fix latest deps CI (#13734) --- .github/workflows/latest_deps.yml | 20 +++++++++++++++++--- .github/workflows/twisted_trunk.yml | 24 ++++++++++++++++++++++++ changelog.d/13734.misc | 1 + 3 files changed, 42 insertions(+), 3 deletions(-) create mode 100644 changelog.d/13734.misc diff --git a/.github/workflows/latest_deps.yml b/.github/workflows/latest_deps.yml index 7dac617c4b..07229e56bd 100644 --- a/.github/workflows/latest_deps.yml +++ b/.github/workflows/latest_deps.yml @@ -5,7 +5,7 @@ # # As an overview this workflow: # - checks out develop, -# - installs from source, pulling in the dependencies like a fresh `pip install` would, and +# - installs from source, pulling in the dependencies like a fresh `pip install` would, and # - runs mypy and test suites in that checkout. # # Based on the twisted trunk CI job. @@ -26,12 +26,19 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 + - name: Install Rust + uses: actions-rs/toolchain@v1 + with: + toolchain: stable + override: true + - uses: Swatinem/rust-cache@v2 + # The dev dependencies aren't exposed in the wheel metadata (at least with current # poetry-core versions), so we install with poetry. - uses: matrix-org/setup-python-poetry@v1 with: python-version: "3.x" - poetry-version: "1.2.0b1" + poetry-version: "1.2.0" extras: "all" # Dump installed versions for debugging. - run: poetry run pip list > before.txt @@ -53,6 +60,14 @@ jobs: steps: - uses: actions/checkout@v2 + + - name: Install Rust + uses: actions-rs/toolchain@v1 + with: + toolchain: stable + override: true + - uses: Swatinem/rust-cache@v2 + - run: sudo apt-get -qq install xmlsec1 - name: Set up PostgreSQL ${{ matrix.postgres-version }} if: ${{ matrix.postgres-version }} @@ -187,4 +202,3 @@ jobs: with: update_existing: true filename: .ci/latest_deps_build_failed_issue_template.md - diff --git a/.github/workflows/twisted_trunk.yml b/.github/workflows/twisted_trunk.yml index 0906101cc1..8fa2fbdea0 100644 --- a/.github/workflows/twisted_trunk.yml +++ b/.github/workflows/twisted_trunk.yml @@ -16,6 +16,14 @@ jobs: steps: - uses: actions/checkout@v2 + + - name: Install Rust + uses: actions-rs/toolchain@v1 + with: + toolchain: stable + override: true + - uses: Swatinem/rust-cache@v2 + - uses: matrix-org/setup-python-poetry@v1 with: python-version: "3.x" @@ -34,6 +42,14 @@ jobs: steps: - uses: actions/checkout@v2 - run: sudo apt-get -qq install xmlsec1 + + - name: Install Rust + uses: actions-rs/toolchain@v1 + with: + toolchain: stable + override: true + - uses: Swatinem/rust-cache@v2 + - uses: matrix-org/setup-python-poetry@v1 with: python-version: "3.x" @@ -66,6 +82,14 @@ jobs: steps: - uses: actions/checkout@v2 + + - name: Install Rust + uses: actions-rs/toolchain@v1 + with: + toolchain: stable + override: true + - uses: Swatinem/rust-cache@v2 + - name: Patch dependencies # Note: The poetry commands want to create a virtualenv in /src/.venv/, # but the sytest-synapse container expects it to be in /venv/. diff --git a/changelog.d/13734.misc b/changelog.d/13734.misc new file mode 100644 index 0000000000..2e0dd68a0f --- /dev/null +++ b/changelog.d/13734.misc @@ -0,0 +1 @@ +Add a stub Rust crate. From d4d3249ded000219a4f875943632c3d0f928d58d Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Wed, 7 Sep 2022 11:41:52 -0500 Subject: [PATCH 075/107] Instrument `get_metadata_for_events` for tracing (#13730) When backfilling, `_get_state_ids_after_missing_prev_event` calls [`get_metadata_for_events`](https://github.com/matrix-org/synapse/blob/26bc26586b4b95d63ce7e453e9312469843f796e/synapse/handlers/federation_event.py#L1133). For `#matrix:matrix.org`, it's called with 77k `state_events` which means 77 calls to the database and takes 28 seconds. --- changelog.d/13730.misc | 1 + synapse/storage/databases/main/state.py | 2 ++ 2 files changed, 3 insertions(+) create mode 100644 changelog.d/13730.misc diff --git a/changelog.d/13730.misc b/changelog.d/13730.misc new file mode 100644 index 0000000000..06da6581a4 --- /dev/null +++ b/changelog.d/13730.misc @@ -0,0 +1 @@ +Instrument `get_metadata_for_events` for understandable traces in Jaeger. diff --git a/synapse/storage/databases/main/state.py b/synapse/storage/databases/main/state.py index e607ccfdc9..af7bebee80 100644 --- a/synapse/storage/databases/main/state.py +++ b/synapse/storage/databases/main/state.py @@ -23,6 +23,7 @@ from synapse.api.errors import NotFoundError, UnsupportedRoomVersionError from synapse.api.room_versions import KNOWN_ROOM_VERSIONS, RoomVersion from synapse.events import EventBase from synapse.events.snapshot import EventContext +from synapse.logging.opentracing import trace from synapse.storage._base import SQLBaseStore from synapse.storage.database import ( DatabasePool, @@ -143,6 +144,7 @@ class StateGroupWorkerStore(EventsWorkerStore, SQLBaseStore): return room_version + @trace async def get_metadata_for_events( self, event_ids: Collection[str] ) -> Dict[str, EventMetadata]: From b7e4bfd005a804ed0cd3d204c367b11565b76df6 Mon Sep 17 00:00:00 2001 From: reivilibre Date: Thu, 8 Sep 2022 10:18:03 +0000 Subject: [PATCH 076/107] Fix a bug where Synapse fails to start if a signing key file contains an empty line. (#13738) --- changelog.d/13738.bugfix | 1 + synapse/config/key.py | 13 ++++++++++++- 2 files changed, 13 insertions(+), 1 deletion(-) create mode 100644 changelog.d/13738.bugfix diff --git a/changelog.d/13738.bugfix b/changelog.d/13738.bugfix new file mode 100644 index 0000000000..d64fa0b4de --- /dev/null +++ b/changelog.d/13738.bugfix @@ -0,0 +1 @@ +Fix a bug where Synapse fails to start if a signing key file contains an empty line. \ No newline at end of file diff --git a/synapse/config/key.py b/synapse/config/key.py index cc75efdf8f..f3dc4df695 100644 --- a/synapse/config/key.py +++ b/synapse/config/key.py @@ -217,7 +217,18 @@ class KeyConfig(Config): signing_keys = self.read_file(signing_key_path, name) try: - return read_signing_keys(signing_keys.splitlines(True)) + loaded_signing_keys = read_signing_keys( + [ + signing_key_line + for signing_key_line in signing_keys.splitlines(keepends=False) + if signing_key_line.strip() + ] + ) + + if not loaded_signing_keys: + raise ConfigError(f"No signing keys in file {signing_key_path}") + + return loaded_signing_keys except Exception as e: raise ConfigError("Error reading %s: %s" % (name, str(e))) From a7c71686ca6c6b8e2dd3ba005669a4707e56869f Mon Sep 17 00:00:00 2001 From: David Robertson Date: Thu, 8 Sep 2022 12:00:03 +0100 Subject: [PATCH 077/107] Add minimum version bump for sqlite to the release notes (#13742) * Notify that SQLite min version will be bumped * Mention in upgrade notes Co-authored-by: reivilibre --- CHANGES.md | 7 ++++++- docs/upgrade.md | 15 +++++++++++++++ 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index ad277f32e5..c2c628d1a5 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -8,11 +8,16 @@ notes](https://matrix-org.github.io/synapse/v1.67/upgrade.html#upgrading-to-v167 The minimum version of `poetry` supported for managing source checkouts is now 1.2.0. -Notice: from the next major release (v1.68.0) installing Synapse from a source +**Notice:** from the next major release (v1.68.0) installing Synapse from a source checkout will require a recent Rust compiler. Those using packages or `pip install matrix-synapse` will not be affected. See the [upgrade notes](https://matrix-org.github.io/synapse/v1.67/upgrade.html#upgrading-to-v1670). +**Notice:** from the next major release (v1.68.0), running Synapse with a SQLite +database will require SQLite version 3.27.0 or higher. (The [current minimum + version is SQLite 3.22.0](https://github.com/matrix-org/synapse/blob/release-v1.67/synapse/storage/engines/sqlite.py#L69-L78).) +See [#12983](https://github.com/matrix-org/synapse/issues/12983) for more details. + Features -------- diff --git a/docs/upgrade.md b/docs/upgrade.md index 023ca0a30b..9f165551fa 100644 --- a/docs/upgrade.md +++ b/docs/upgrade.md @@ -119,6 +119,21 @@ will require a recent Rust compiler. Those using packages or The simplest way of installing Rust is via [rustup.rs](https://rustup.rs/) +## SQLite version requirement in the next release + +From the next major release (v1.68.0) Synapse will require SQLite 3.27.0 or +higher. Synapse v1.67.0 will be the last major release supporting SQLite +versions 3.22 to 3.26. + +Those using docker images or Debian packages from Matrix.org will not be +affected. If you have installed from source, you should check the version of +SQLite used by Python with: + +```shell +python -c "import sqlite3; print(sqlite3.sqlite_version)" +``` + +If this is too old, refer to your distribution for advice on upgrading. # Upgrading to v1.66.0 From 9d118425627c332d30eae8ebe5c3fc519416100f Mon Sep 17 00:00:00 2001 From: David Robertson Date: Thu, 8 Sep 2022 12:04:29 +0100 Subject: [PATCH 078/107] Also cite upgrade notes --- CHANGES.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index c2c628d1a5..0300fcbd30 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -16,7 +16,7 @@ notes](https://matrix-org.github.io/synapse/v1.67/upgrade.html#upgrading-to-v167 **Notice:** from the next major release (v1.68.0), running Synapse with a SQLite database will require SQLite version 3.27.0 or higher. (The [current minimum version is SQLite 3.22.0](https://github.com/matrix-org/synapse/blob/release-v1.67/synapse/storage/engines/sqlite.py#L69-L78).) -See [#12983](https://github.com/matrix-org/synapse/issues/12983) for more details. +See [#12983](https://github.com/matrix-org/synapse/issues/12983) and the [upgrade notes](https://matrix-org.github.io/synapse/v1.67/upgrade.html#upgrading-to-v1670). for more details. Features From 1cc729c177ec734b3e7c56c80bc2b32e100c0fe9 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Thu, 8 Sep 2022 13:58:31 +0100 Subject: [PATCH 079/107] Fix latest deps (#13743) --- .github/workflows/latest_deps.yml | 14 ++++++++++++++ changelog.d/13743.misc | 1 + 2 files changed, 15 insertions(+) create mode 100644 changelog.d/13743.misc diff --git a/.github/workflows/latest_deps.yml b/.github/workflows/latest_deps.yml index 07229e56bd..8366ac9393 100644 --- a/.github/workflows/latest_deps.yml +++ b/.github/workflows/latest_deps.yml @@ -84,6 +84,12 @@ jobs: if: ${{ matrix.postgres-version }} timeout-minutes: 2 run: until pg_isready -h localhost; do sleep 1; done + + # We nuke the local copy, as we've installed synapse into the virtualenv + # (rather than use an editable install, which we no longer support). If we + # don't do this then python can't find the native lib. + - run: rm -rf synapse/ + - run: python -m twisted.trial --jobs=2 tests env: SYNAPSE_POSTGRES: ${{ matrix.database == 'postgres' || '' }} @@ -128,6 +134,14 @@ jobs: steps: - uses: actions/checkout@v2 + + - name: Install Rust + uses: actions-rs/toolchain@v1 + with: + toolchain: stable + override: true + - uses: Swatinem/rust-cache@v2 + - name: Ensure sytest runs `pip install` # Delete the lockfile so sytest will `pip install` rather than `poetry install` run: rm /src/poetry.lock diff --git a/changelog.d/13743.misc b/changelog.d/13743.misc new file mode 100644 index 0000000000..2e0dd68a0f --- /dev/null +++ b/changelog.d/13743.misc @@ -0,0 +1 @@ +Add a stub Rust crate. From 526f84bc2ea78a44ef612bf79caf5cfb7966c1ed Mon Sep 17 00:00:00 2001 From: reivilibre Date: Thu, 8 Sep 2022 14:01:42 +0000 Subject: [PATCH 080/107] Fix Prometheus recording rules to not use legacy metric names. (#13718) --- changelog.d/13718.misc | 1 + contrib/prometheus/synapse-v1.rules | 21 --------------------- contrib/prometheus/synapse-v2.rules | 22 +++++++++++----------- synapse/app/phone_stats_home.py | 6 +++--- synapse/federation/sender/__init__.py | 4 ++-- synapse/metrics/_legacy_exposition.py | 5 +++++ 6 files changed, 22 insertions(+), 37 deletions(-) create mode 100644 changelog.d/13718.misc delete mode 100644 contrib/prometheus/synapse-v1.rules diff --git a/changelog.d/13718.misc b/changelog.d/13718.misc new file mode 100644 index 0000000000..07ace50b12 --- /dev/null +++ b/changelog.d/13718.misc @@ -0,0 +1 @@ +Add experimental configuration option to allow disabling legacy Prometheus metric names. \ No newline at end of file diff --git a/contrib/prometheus/synapse-v1.rules b/contrib/prometheus/synapse-v1.rules deleted file mode 100644 index 4c900ba537..0000000000 --- a/contrib/prometheus/synapse-v1.rules +++ /dev/null @@ -1,21 +0,0 @@ -synapse_federation_transaction_queue_pendingEdus:total = sum(synapse_federation_transaction_queue_pendingEdus or absent(synapse_federation_transaction_queue_pendingEdus)*0) -synapse_federation_transaction_queue_pendingPdus:total = sum(synapse_federation_transaction_queue_pendingPdus or absent(synapse_federation_transaction_queue_pendingPdus)*0) - -synapse_http_server_request_count:method{servlet=""} = sum(synapse_http_server_request_count) by (method) -synapse_http_server_request_count:servlet{method=""} = sum(synapse_http_server_request_count) by (servlet) - -synapse_http_server_request_count:total{servlet=""} = sum(synapse_http_server_request_count:by_method) by (servlet) - -synapse_cache:hit_ratio_5m = rate(synapse_util_caches_cache:hits[5m]) / rate(synapse_util_caches_cache:total[5m]) -synapse_cache:hit_ratio_30s = rate(synapse_util_caches_cache:hits[30s]) / rate(synapse_util_caches_cache:total[30s]) - -synapse_federation_client_sent{type="EDU"} = synapse_federation_client_sent_edus + 0 -synapse_federation_client_sent{type="PDU"} = synapse_federation_client_sent_pdu_destinations:count + 0 -synapse_federation_client_sent{type="Query"} = sum(synapse_federation_client_sent_queries) by (job) - -synapse_federation_server_received{type="EDU"} = synapse_federation_server_received_edus + 0 -synapse_federation_server_received{type="PDU"} = synapse_federation_server_received_pdus + 0 -synapse_federation_server_received{type="Query"} = sum(synapse_federation_server_received_queries) by (job) - -synapse_federation_transaction_queue_pending{type="EDU"} = synapse_federation_transaction_queue_pending_edus + 0 -synapse_federation_transaction_queue_pending{type="PDU"} = synapse_federation_transaction_queue_pending_pdus + 0 diff --git a/contrib/prometheus/synapse-v2.rules b/contrib/prometheus/synapse-v2.rules index 7e405bf7f0..a5e6a735cd 100644 --- a/contrib/prometheus/synapse-v2.rules +++ b/contrib/prometheus/synapse-v2.rules @@ -20,18 +20,18 @@ groups: expr: 'sum(synapse_http_server_request_count:by_method) by (servlet)' - record: 'synapse_cache:hit_ratio_5m' - expr: 'rate(synapse_util_caches_cache:hits[5m]) / rate(synapse_util_caches_cache:total[5m])' + expr: 'rate(synapse_util_caches_cache_hits[5m]) / rate(synapse_util_caches_cache[5m])' - record: 'synapse_cache:hit_ratio_30s' - expr: 'rate(synapse_util_caches_cache:hits[30s]) / rate(synapse_util_caches_cache:total[30s])' + expr: 'rate(synapse_util_caches_cache_hits[30s]) / rate(synapse_util_caches_cache[30s])' - record: 'synapse_federation_client_sent' labels: type: "EDU" - expr: 'synapse_federation_client_sent_edus + 0' + expr: 'synapse_federation_client_sent_edus_total + 0' - record: 'synapse_federation_client_sent' labels: type: "PDU" - expr: 'synapse_federation_client_sent_pdu_destinations:count + 0' + expr: 'synapse_federation_client_sent_pdu_destinations_count_total + 0' - record: 'synapse_federation_client_sent' labels: type: "Query" @@ -40,11 +40,11 @@ groups: - record: 'synapse_federation_server_received' labels: type: "EDU" - expr: 'synapse_federation_server_received_edus + 0' + expr: 'synapse_federation_server_received_edus_total + 0' - record: 'synapse_federation_server_received' labels: type: "PDU" - expr: 'synapse_federation_server_received_pdus + 0' + expr: 'synapse_federation_server_received_pdus_total + 0' - record: 'synapse_federation_server_received' labels: type: "Query" @@ -60,19 +60,19 @@ groups: expr: 'synapse_federation_transaction_queue_pending_pdus + 0' - record: synapse_storage_events_persisted_by_source_type - expr: sum without(type, origin_type, origin_entity) (synapse_storage_events_persisted_events_sep{origin_type="remote"}) + expr: sum without(type, origin_type, origin_entity) (synapse_storage_events_persisted_events_sep_total{origin_type="remote"}) labels: type: remote - record: synapse_storage_events_persisted_by_source_type - expr: sum without(type, origin_type, origin_entity) (synapse_storage_events_persisted_events_sep{origin_entity="*client*",origin_type="local"}) + expr: sum without(type, origin_type, origin_entity) (synapse_storage_events_persisted_events_sep_total{origin_entity="*client*",origin_type="local"}) labels: type: local - record: synapse_storage_events_persisted_by_source_type - expr: sum without(type, origin_type, origin_entity) (synapse_storage_events_persisted_events_sep{origin_entity!="*client*",origin_type="local"}) + expr: sum without(type, origin_type, origin_entity) (synapse_storage_events_persisted_events_sep_total{origin_entity!="*client*",origin_type="local"}) labels: type: bridges - record: synapse_storage_events_persisted_by_event_type - expr: sum without(origin_entity, origin_type) (synapse_storage_events_persisted_events_sep) + expr: sum without(origin_entity, origin_type) (synapse_storage_events_persisted_events_sep_total) - record: synapse_storage_events_persisted_by_origin - expr: sum without(type) (synapse_storage_events_persisted_events_sep) + expr: sum without(type) (synapse_storage_events_persisted_events_sep_total) diff --git a/synapse/app/phone_stats_home.py b/synapse/app/phone_stats_home.py index 51c8d15711..53db1e85b3 100644 --- a/synapse/app/phone_stats_home.py +++ b/synapse/app/phone_stats_home.py @@ -32,15 +32,15 @@ logger = logging.getLogger("synapse.app.homeserver") _stats_process: List[Tuple[int, "resource.struct_rusage"]] = [] # Gauges to expose monthly active user control metrics -current_mau_gauge = Gauge("synapse_admin_mau:current", "Current MAU") +current_mau_gauge = Gauge("synapse_admin_mau_current", "Current MAU") current_mau_by_service_gauge = Gauge( "synapse_admin_mau_current_mau_by_service", "Current MAU by service", ["app_service"], ) -max_mau_gauge = Gauge("synapse_admin_mau:max", "MAU Limit") +max_mau_gauge = Gauge("synapse_admin_mau_max", "MAU Limit") registered_reserved_users_mau_gauge = Gauge( - "synapse_admin_mau:registered_reserved_users", + "synapse_admin_mau_registered_reserved_users", "Registered users with reserved threepids", ) diff --git a/synapse/federation/sender/__init__.py b/synapse/federation/sender/__init__.py index 8bc60e3e3e..a6cb3ba58f 100644 --- a/synapse/federation/sender/__init__.py +++ b/synapse/federation/sender/__init__.py @@ -62,12 +62,12 @@ if TYPE_CHECKING: logger = logging.getLogger(__name__) sent_pdus_destination_dist_count = Counter( - "synapse_federation_client_sent_pdu_destinations:count", + "synapse_federation_client_sent_pdu_destinations_count", "Number of PDUs queued for sending to one or more destinations", ) sent_pdus_destination_dist_total = Counter( - "synapse_federation_client_sent_pdu_destinations:total", + "synapse_federation_client_sent_pdu_destinations", "Total number of PDUs queued for sending across all destinations", ) diff --git a/synapse/metrics/_legacy_exposition.py b/synapse/metrics/_legacy_exposition.py index 6f00ff2a47..133f1603dd 100644 --- a/synapse/metrics/_legacy_exposition.py +++ b/synapse/metrics/_legacy_exposition.py @@ -93,6 +93,11 @@ LEGACY_METRIC_NAMES = { "synapse_util_caches_response_cache_hits": "synapse_util_caches_response_cache:hits", "synapse_util_caches_response_cache_evicted_size": "synapse_util_caches_response_cache:evicted_size", "synapse_util_caches_response_cache": "synapse_util_caches_response_cache:total", + "synapse_federation_client_sent_pdu_destinations": "synapse_federation_client_sent_pdu_destinations:total", + "synapse_federation_client_sent_pdu_destinations_count": "synapse_federation_client_sent_pdu_destinations:count", + "synapse_admin_mau_current": "synapse_admin_mau:current", + "synapse_admin_mau_max": "synapse_admin_mau:max", + "synapse_admin_mau_registered_reserved_users": "synapse_admin_mau:registered_reserved_users", } From cf11919ddd4f48b2f59062542ba62969042f80aa Mon Sep 17 00:00:00 2001 From: reivilibre Date: Thu, 8 Sep 2022 14:30:48 +0000 Subject: [PATCH 081/107] Fix cache metrics not being updated when not using the legacy exposition module. (#13717) --- changelog.d/13717.misc | 1 + synapse/metrics/_legacy_exposition.py | 7 --- synapse/util/caches/__init__.py | 62 +++++++++++++++++++++------ synapse/util/metrics.py | 34 ++++++++++++++- 4 files changed, 82 insertions(+), 22 deletions(-) create mode 100644 changelog.d/13717.misc diff --git a/changelog.d/13717.misc b/changelog.d/13717.misc new file mode 100644 index 0000000000..07ace50b12 --- /dev/null +++ b/changelog.d/13717.misc @@ -0,0 +1 @@ +Add experimental configuration option to allow disabling legacy Prometheus metric names. \ No newline at end of file diff --git a/synapse/metrics/_legacy_exposition.py b/synapse/metrics/_legacy_exposition.py index 133f1603dd..563d8cc2c6 100644 --- a/synapse/metrics/_legacy_exposition.py +++ b/synapse/metrics/_legacy_exposition.py @@ -34,8 +34,6 @@ from prometheus_client.core import Sample from twisted.web.resource import Resource from twisted.web.server import Request -from synapse.util import caches - CONTENT_TYPE_LATEST = "text/plain; version=0.0.4; charset=utf-8" @@ -107,11 +105,6 @@ def generate_latest(registry: CollectorRegistry, emit_help: bool = False) -> byt by prometheus-client. """ - # Trigger the cache metrics to be rescraped, which updates the common - # metrics but do not produce metrics themselves - for collector in caches.collectors_by_name.values(): - collector.collect() - output = [] for metric in registry.collect(): diff --git a/synapse/util/caches/__init__.py b/synapse/util/caches/__init__.py index d4a2b77c29..35c0be08b0 100644 --- a/synapse/util/caches/__init__.py +++ b/synapse/util/caches/__init__.py @@ -20,9 +20,11 @@ from sys import intern from typing import Any, Callable, Dict, List, Optional, Sized, TypeVar import attr +from prometheus_client import REGISTRY from prometheus_client.core import Gauge from synapse.config.cache import add_resizable_cache +from synapse.util.metrics import DynamicCollectorRegistry logger = logging.getLogger(__name__) @@ -30,27 +32,62 @@ logger = logging.getLogger(__name__) # Whether to track estimated memory usage of the LruCaches. TRACK_MEMORY_USAGE = False +# We track cache metrics in a special registry that lets us update the metrics +# just before they are returned from the scrape endpoint. +CACHE_METRIC_REGISTRY = DynamicCollectorRegistry() caches_by_name: Dict[str, Sized] = {} -collectors_by_name: Dict[str, "CacheMetric"] = {} -cache_size = Gauge("synapse_util_caches_cache_size", "", ["name"]) -cache_hits = Gauge("synapse_util_caches_cache_hits", "", ["name"]) -cache_evicted = Gauge("synapse_util_caches_cache_evicted_size", "", ["name", "reason"]) -cache_total = Gauge("synapse_util_caches_cache", "", ["name"]) -cache_max_size = Gauge("synapse_util_caches_cache_max_size", "", ["name"]) +cache_size = Gauge( + "synapse_util_caches_cache_size", "", ["name"], registry=CACHE_METRIC_REGISTRY +) +cache_hits = Gauge( + "synapse_util_caches_cache_hits", "", ["name"], registry=CACHE_METRIC_REGISTRY +) +cache_evicted = Gauge( + "synapse_util_caches_cache_evicted_size", + "", + ["name", "reason"], + registry=CACHE_METRIC_REGISTRY, +) +cache_total = Gauge( + "synapse_util_caches_cache", "", ["name"], registry=CACHE_METRIC_REGISTRY +) +cache_max_size = Gauge( + "synapse_util_caches_cache_max_size", "", ["name"], registry=CACHE_METRIC_REGISTRY +) cache_memory_usage = Gauge( "synapse_util_caches_cache_size_bytes", "Estimated memory usage of the caches", ["name"], + registry=CACHE_METRIC_REGISTRY, ) -response_cache_size = Gauge("synapse_util_caches_response_cache_size", "", ["name"]) -response_cache_hits = Gauge("synapse_util_caches_response_cache_hits", "", ["name"]) -response_cache_evicted = Gauge( - "synapse_util_caches_response_cache_evicted_size", "", ["name", "reason"] +response_cache_size = Gauge( + "synapse_util_caches_response_cache_size", + "", + ["name"], + registry=CACHE_METRIC_REGISTRY, ) -response_cache_total = Gauge("synapse_util_caches_response_cache", "", ["name"]) +response_cache_hits = Gauge( + "synapse_util_caches_response_cache_hits", + "", + ["name"], + registry=CACHE_METRIC_REGISTRY, +) +response_cache_evicted = Gauge( + "synapse_util_caches_response_cache_evicted_size", + "", + ["name", "reason"], + registry=CACHE_METRIC_REGISTRY, +) +response_cache_total = Gauge( + "synapse_util_caches_response_cache", "", ["name"], registry=CACHE_METRIC_REGISTRY +) + + +# Register our custom cache metrics registry with the global registry +REGISTRY.register(CACHE_METRIC_REGISTRY) class EvictionReason(Enum): @@ -168,9 +205,8 @@ def register_cache( add_resizable_cache(cache_name, resize_callback) metric = CacheMetric(cache, cache_type, cache_name, collect_callback) - metric_name = "cache_%s_%s" % (cache_type, cache_name) caches_by_name[cache_name] = cache - collectors_by_name[metric_name] = metric + CACHE_METRIC_REGISTRY.register_hook(metric.collect) return metric diff --git a/synapse/util/metrics.py b/synapse/util/metrics.py index bc3b4938ea..9687120ebf 100644 --- a/synapse/util/metrics.py +++ b/synapse/util/metrics.py @@ -15,9 +15,9 @@ import logging from functools import wraps from types import TracebackType -from typing import Awaitable, Callable, Optional, Type, TypeVar +from typing import Awaitable, Callable, Generator, List, Optional, Type, TypeVar -from prometheus_client import Counter +from prometheus_client import CollectorRegistry, Counter, Metric from typing_extensions import Concatenate, ParamSpec, Protocol from synapse.logging.context import ( @@ -208,3 +208,33 @@ class Measure: metrics.real_time_sum += duration # TODO: Add other in flight metrics. + + +class DynamicCollectorRegistry(CollectorRegistry): + """ + Custom Prometheus Collector registry that calls a hook first, allowing you + to update metrics on-demand. + + Don't forget to register this registry with the main registry! + """ + + def __init__(self) -> None: + super().__init__() + self._pre_update_hooks: List[Callable[[], None]] = [] + + def collect(self) -> Generator[Metric, None, None]: + """ + Collects metrics, calling pre-update hooks first. + """ + + for pre_update_hook in self._pre_update_hooks: + pre_update_hook() + + yield from super().collect() + + def register_hook(self, hook: Callable[[], None]) -> None: + """ + Registers a hook that is called before metric collection. + """ + + self._pre_update_hooks.append(hook) From 8ef0c8ff14fcf613c5df1cfc30b38236de1695a7 Mon Sep 17 00:00:00 2001 From: Sean Quah <8349537+squahtx@users.noreply.github.com> Date: Thu, 8 Sep 2022 15:54:36 +0100 Subject: [PATCH 082/107] Fix error in `is_mine_id` when encountering a malformed ID (#13746) Previously, `is_mine_id` would raise an exception when passed an ID with no colons. Return `False` instead. Fixes #13040. Signed-off-by: Sean Quah --- changelog.d/13746.bugfix | 1 + synapse/server.py | 12 +++++++++++- tests/test_types.py | 26 +++++++++++++++++++++++++- 3 files changed, 37 insertions(+), 2 deletions(-) create mode 100644 changelog.d/13746.bugfix diff --git a/changelog.d/13746.bugfix b/changelog.d/13746.bugfix new file mode 100644 index 0000000000..b692af8fd5 --- /dev/null +++ b/changelog.d/13746.bugfix @@ -0,0 +1 @@ +Fix a long standing bug where Synapse would fail to handle malformed user IDs or room aliases gracefully in certain cases. diff --git a/synapse/server.py b/synapse/server.py index 5a99c0b344..df3a1cb405 100644 --- a/synapse/server.py +++ b/synapse/server.py @@ -341,7 +341,17 @@ class HomeServer(metaclass=abc.ABCMeta): return domain_specific_string.domain == self.hostname def is_mine_id(self, string: str) -> bool: - return string.split(":", 1)[1] == self.hostname + """Determines whether a user ID or room alias originates from this homeserver. + + Returns: + `True` if the hostname part of the user ID or room alias matches this + homeserver. + `False` otherwise, or if the user ID or room alias is malformed. + """ + localpart_hostname = string.split(":", 1) + if len(localpart_hostname) < 2: + return False + return localpart_hostname[1] == self.hostname @cache_in_self def get_clock(self) -> Clock: diff --git a/tests/test_types.py b/tests/test_types.py index d8d82a517e..1111169384 100644 --- a/tests/test_types.py +++ b/tests/test_types.py @@ -13,11 +13,35 @@ # limitations under the License. from synapse.api.errors import SynapseError -from synapse.types import RoomAlias, UserID, map_username_to_mxid_localpart +from synapse.types import ( + RoomAlias, + UserID, + get_domain_from_id, + get_localpart_from_id, + map_username_to_mxid_localpart, +) from tests import unittest +class IsMineIDTests(unittest.HomeserverTestCase): + def test_is_mine_id(self) -> None: + self.assertTrue(self.hs.is_mine_id("@user:test")) + self.assertTrue(self.hs.is_mine_id("#room:test")) + self.assertTrue(self.hs.is_mine_id("invalid:test")) + + self.assertFalse(self.hs.is_mine_id("@user:test\0")) + self.assertFalse(self.hs.is_mine_id("@user")) + + def test_two_colons(self) -> None: + """Test handling of IDs containing more than one colon.""" + # The domain starts after the first colon. + # These functions must interpret things consistently. + self.assertFalse(self.hs.is_mine_id("@user:test:test")) + self.assertEqual("user", get_localpart_from_id("@user:test:test")) + self.assertEqual("test:test", get_domain_from_id("@user:test:test")) + + class UserIDTestCase(unittest.HomeserverTestCase): def test_parse(self): user = UserID.from_string("@1234abcd:test") From 89e8b98b6522dc9fafd5c32efe7a84dc9a31246a Mon Sep 17 00:00:00 2001 From: Sean Quah <8349537+squahtx@users.noreply.github.com> Date: Thu, 8 Sep 2022 15:55:03 +0100 Subject: [PATCH 083/107] Avoid raising errors due to malformed IDs in `get_current_hosts_in_room` (#13748) Handle malformed user IDs with no colons in `get_current_hosts_in_room`. It's not currently possible for a malformed user ID to join a room, so this error would never be hit. Signed-off-by: Sean Quah --- changelog.d/13748.misc | 1 + synapse/storage/databases/main/roommember.py | 5 ++++- 2 files changed, 5 insertions(+), 1 deletion(-) create mode 100644 changelog.d/13748.misc diff --git a/changelog.d/13748.misc b/changelog.d/13748.misc new file mode 100644 index 0000000000..2f419bb659 --- /dev/null +++ b/changelog.d/13748.misc @@ -0,0 +1 @@ +Avoid raising an error due to malformed user IDs in `get_current_hosts_in_room`. Malformed user IDs cannot currently join a room, so this error would not be hit. diff --git a/synapse/storage/databases/main/roommember.py b/synapse/storage/databases/main/roommember.py index a77e49dc66..e6b87010a9 100644 --- a/synapse/storage/databases/main/roommember.py +++ b/synapse/storage/databases/main/roommember.py @@ -1044,6 +1044,8 @@ class RoomMemberWorkerStore(EventsWorkerStore): # We use a `Set` just for fast lookups domain_set: Set[str] = set() for u in users: + if ":" not in u: + continue domain = get_domain_from_id(u) if domain not in domain_set: domain_set.add(domain) @@ -1077,7 +1079,8 @@ class RoomMemberWorkerStore(EventsWorkerStore): ORDER BY min(e.depth) ASC; """ txn.execute(sql, (room_id,)) - return [d for d, in txn] + # `server_domain` will be `NULL` for malformed MXIDs with no colons. + return [d for d, in txn if d is not None] return await self.db_pool.runInteraction( "get_current_hosts_in_room", get_current_hosts_in_room_txn From 906cead9ca5fc95fd64680ec18f775f4de6ea97f Mon Sep 17 00:00:00 2001 From: Sean Quah <8349537+squahtx@users.noreply.github.com> Date: Thu, 8 Sep 2022 15:55:29 +0100 Subject: [PATCH 084/107] Update docstrings to explain the impact of partial state (#13750) Update the docstrings for `get_users_in_room` and `get_current_hosts_in_room` to explain the impact of partial state. Signed-off-by: Sean Quah --- changelog.d/13750.misc | 1 + synapse/storage/databases/main/roommember.py | 17 ++++++++++++++++- 2 files changed, 17 insertions(+), 1 deletion(-) create mode 100644 changelog.d/13750.misc diff --git a/changelog.d/13750.misc b/changelog.d/13750.misc new file mode 100644 index 0000000000..3bccc21fc5 --- /dev/null +++ b/changelog.d/13750.misc @@ -0,0 +1 @@ +Update the docstrings for `get_users_in_room` and `get_current_hosts_in_room` to explain the impact of partial state. diff --git a/synapse/storage/databases/main/roommember.py b/synapse/storage/databases/main/roommember.py index e6b87010a9..6e1ff5626b 100644 --- a/synapse/storage/databases/main/roommember.py +++ b/synapse/storage/databases/main/roommember.py @@ -192,8 +192,15 @@ class RoomMemberWorkerStore(EventsWorkerStore): (aka. with the lowest depth). This is done to match the sort in `get_current_hosts_in_room()` and so we can re-use the cache but it's not horrible to have here either. - """ + Uses `m.room.member`s in the room state at the current forward extremities to + determine which users are in the room. + + Will return inaccurate results for rooms with partial state, since the state for + the forward extremities of those rooms will exclude most members. We may also + calculate room state incorrectly for such rooms and believe that a member is or + is not in the room when the opposite is true. + """ return await self.db_pool.runInteraction( "get_users_in_room", self.get_users_in_room_txn, room_id ) @@ -1022,6 +1029,14 @@ class RoomMemberWorkerStore(EventsWorkerStore): longest is good because they're most likely to have anything we ask about. + Uses `m.room.member`s in the room state at the current forward extremities to + determine which hosts are in the room. + + Will return inaccurate results for rooms with partial state, since the state for + the forward extremities of those rooms will exclude most members. We may also + calculate room state incorrectly for such rooms and believe that a host is or + is not in the room when the opposite is true. + Returns: Returns a list of servers sorted by longest in the room first. (aka. sorted by join with the lowest depth first). From f799eac7ea96f943ad1272a5a81f845dfa08a254 Mon Sep 17 00:00:00 2001 From: Dirk Klimpel <5740567+dklimpel@users.noreply.github.com> Date: Thu, 8 Sep 2022 17:41:48 +0200 Subject: [PATCH 085/107] Add timestamp to user's consent (#13741) Co-authored-by: reivilibre --- changelog.d/13741.feature | 1 + docs/admin_api/user_admin_api.md | 2 ++ synapse/handlers/admin.py | 1 + .../storage/databases/main/registration.py | 6 +++- .../delta/72/06add_consent_ts_to_users.sql | 16 +++++++++ tests/rest/admin/test_user.py | 1 + tests/storage/test_registration.py | 33 +++++++++++++++---- 7 files changed, 52 insertions(+), 8 deletions(-) create mode 100644 changelog.d/13741.feature create mode 100644 synapse/storage/schema/main/delta/72/06add_consent_ts_to_users.sql diff --git a/changelog.d/13741.feature b/changelog.d/13741.feature new file mode 100644 index 0000000000..dff46f373f --- /dev/null +++ b/changelog.d/13741.feature @@ -0,0 +1 @@ +Document the timestamp when a user accepts the consent, if [consent tracking](https://matrix-org.github.io/synapse/latest/consent_tracking.html) is used. \ No newline at end of file diff --git a/docs/admin_api/user_admin_api.md b/docs/admin_api/user_admin_api.md index c1ca0c8a64..975f05c929 100644 --- a/docs/admin_api/user_admin_api.md +++ b/docs/admin_api/user_admin_api.md @@ -42,6 +42,7 @@ It returns a JSON body like the following: "appservice_id": null, "consent_server_notice_sent": null, "consent_version": null, + "consent_ts": null, "external_ids": [ { "auth_provider": "", @@ -364,6 +365,7 @@ The following actions are **NOT** performed. The list may be incomplete. - Remove the user's creation (registration) timestamp - [Remove rate limit overrides](#override-ratelimiting-for-users) - Remove from monthly active users +- Remove user's consent information (consent version and timestamp) ## Reset password diff --git a/synapse/handlers/admin.py b/synapse/handlers/admin.py index d4fe7df533..cf9f19608a 100644 --- a/synapse/handlers/admin.py +++ b/synapse/handlers/admin.py @@ -70,6 +70,7 @@ class AdminHandler: "appservice_id", "consent_server_notice_sent", "consent_version", + "consent_ts", "user_type", "is_guest", } diff --git a/synapse/storage/databases/main/registration.py b/synapse/storage/databases/main/registration.py index 7fb9c801da..ac821878b0 100644 --- a/synapse/storage/databases/main/registration.py +++ b/synapse/storage/databases/main/registration.py @@ -175,6 +175,7 @@ class RegistrationWorkerStore(CacheInvalidationWorkerStore): "is_guest", "admin", "consent_version", + "consent_ts", "consent_server_notice_sent", "appservice_id", "creation_ts", @@ -2227,7 +2228,10 @@ class RegistrationStore(StatsStore, RegistrationBackgroundUpdateStore): txn, table="users", keyvalues={"name": user_id}, - updatevalues={"consent_version": consent_version}, + updatevalues={ + "consent_version": consent_version, + "consent_ts": self._clock.time_msec(), + }, ) self._invalidate_cache_and_stream(txn, self.get_user_by_id, (user_id,)) diff --git a/synapse/storage/schema/main/delta/72/06add_consent_ts_to_users.sql b/synapse/storage/schema/main/delta/72/06add_consent_ts_to_users.sql new file mode 100644 index 0000000000..609eb1750f --- /dev/null +++ b/synapse/storage/schema/main/delta/72/06add_consent_ts_to_users.sql @@ -0,0 +1,16 @@ +/* Copyright 2022 The Matrix.org Foundation C.I.C + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +ALTER TABLE users ADD consent_ts bigint; diff --git a/tests/rest/admin/test_user.py b/tests/rest/admin/test_user.py index 1afd082707..ec5ccf6fca 100644 --- a/tests/rest/admin/test_user.py +++ b/tests/rest/admin/test_user.py @@ -2580,6 +2580,7 @@ class UserRestTestCase(unittest.HomeserverTestCase): self.assertIn("appservice_id", content) self.assertIn("consent_server_notice_sent", content) self.assertIn("consent_version", content) + self.assertIn("consent_ts", content) self.assertIn("external_ids", content) # This key was removed intentionally. Ensure it is not accidentally re-included. diff --git a/tests/storage/test_registration.py b/tests/storage/test_registration.py index a49ac1525e..853a93afab 100644 --- a/tests/storage/test_registration.py +++ b/tests/storage/test_registration.py @@ -11,15 +11,18 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +from twisted.test.proto_helpers import MemoryReactor from synapse.api.constants import UserTypes from synapse.api.errors import ThreepidValidationError +from synapse.server import HomeServer +from synapse.util import Clock from tests.unittest import HomeserverTestCase class RegistrationStoreTestCase(HomeserverTestCase): - def prepare(self, reactor, clock, hs): + def prepare(self, reactor: MemoryReactor, clock: Clock, hs: HomeServer) -> None: self.store = hs.get_datastores().main self.user_id = "@my-user:test" @@ -27,7 +30,7 @@ class RegistrationStoreTestCase(HomeserverTestCase): self.pwhash = "{xx1}123456789" self.device_id = "akgjhdjklgshg" - def test_register(self): + def test_register(self) -> None: self.get_success(self.store.register_user(self.user_id, self.pwhash)) self.assertEqual( @@ -38,6 +41,7 @@ class RegistrationStoreTestCase(HomeserverTestCase): "admin": 0, "is_guest": 0, "consent_version": None, + "consent_ts": None, "consent_server_notice_sent": None, "appservice_id": None, "creation_ts": 0, @@ -48,7 +52,20 @@ class RegistrationStoreTestCase(HomeserverTestCase): (self.get_success(self.store.get_user_by_id(self.user_id))), ) - def test_add_tokens(self): + def test_consent(self) -> None: + self.get_success(self.store.register_user(self.user_id, self.pwhash)) + before_consent = self.clock.time_msec() + self.reactor.advance(5) + self.get_success(self.store.user_set_consent_version(self.user_id, "1")) + self.reactor.advance(5) + + user = self.get_success(self.store.get_user_by_id(self.user_id)) + assert user + self.assertEqual(user["consent_version"], "1") + self.assertGreater(user["consent_ts"], before_consent) + self.assertLess(user["consent_ts"], self.clock.time_msec()) + + def test_add_tokens(self) -> None: self.get_success(self.store.register_user(self.user_id, self.pwhash)) self.get_success( self.store.add_access_token_to_user( @@ -58,11 +75,12 @@ class RegistrationStoreTestCase(HomeserverTestCase): result = self.get_success(self.store.get_user_by_access_token(self.tokens[1])) + assert result self.assertEqual(result.user_id, self.user_id) self.assertEqual(result.device_id, self.device_id) self.assertIsNotNone(result.token_id) - def test_user_delete_access_tokens(self): + def test_user_delete_access_tokens(self) -> None: # add some tokens self.get_success(self.store.register_user(self.user_id, self.pwhash)) self.get_success( @@ -87,6 +105,7 @@ class RegistrationStoreTestCase(HomeserverTestCase): # check the one not associated with the device was not deleted user = self.get_success(self.store.get_user_by_access_token(self.tokens[0])) + assert user self.assertEqual(self.user_id, user.user_id) # now delete the rest @@ -95,11 +114,11 @@ class RegistrationStoreTestCase(HomeserverTestCase): user = self.get_success(self.store.get_user_by_access_token(self.tokens[0])) self.assertIsNone(user, "access token was not deleted without device_id") - def test_is_support_user(self): + def test_is_support_user(self) -> None: TEST_USER = "@test:test" SUPPORT_USER = "@support:test" - res = self.get_success(self.store.is_support_user(None)) + res = self.get_success(self.store.is_support_user(None)) # type: ignore[arg-type] self.assertFalse(res) self.get_success( self.store.register_user(user_id=TEST_USER, password_hash=None) @@ -115,7 +134,7 @@ class RegistrationStoreTestCase(HomeserverTestCase): res = self.get_success(self.store.is_support_user(SUPPORT_USER)) self.assertTrue(res) - def test_3pid_inhibit_invalid_validation_session_error(self): + def test_3pid_inhibit_invalid_validation_session_error(self) -> None: """Tests that enabling the configuration option to inhibit 3PID errors on /requestToken also inhibits validation errors caused by an unknown session ID. """ From 5261d2e2e869d10863bcd13e45f16e8cac99e3d0 Mon Sep 17 00:00:00 2001 From: reivilibre Date: Thu, 8 Sep 2022 17:50:15 +0000 Subject: [PATCH 086/107] Remove unused Prometheus recording rules from `synapse-v2.rules` and add comments describing where the rest are used. (#13756) --- changelog.d/13756.misc | 1 + contrib/prometheus/synapse-v2.rules | 31 ++++++++--------------------- 2 files changed, 9 insertions(+), 23 deletions(-) create mode 100644 changelog.d/13756.misc diff --git a/changelog.d/13756.misc b/changelog.d/13756.misc new file mode 100644 index 0000000000..06e9cd09bf --- /dev/null +++ b/changelog.d/13756.misc @@ -0,0 +1 @@ +Remove unused Prometheus recording rules from `synapse-v2.rules` and add comments describing where the rest are used. \ No newline at end of file diff --git a/contrib/prometheus/synapse-v2.rules b/contrib/prometheus/synapse-v2.rules index a5e6a735cd..cbe6f7beba 100644 --- a/contrib/prometheus/synapse-v2.rules +++ b/contrib/prometheus/synapse-v2.rules @@ -1,29 +1,7 @@ groups: - name: synapse rules: - - record: "synapse_federation_transaction_queue_pendingEdus:total" - expr: "sum(synapse_federation_transaction_queue_pendingEdus or absent(synapse_federation_transaction_queue_pendingEdus)*0)" - - record: "synapse_federation_transaction_queue_pendingPdus:total" - expr: "sum(synapse_federation_transaction_queue_pendingPdus or absent(synapse_federation_transaction_queue_pendingPdus)*0)" - - record: 'synapse_http_server_request_count:method' - labels: - servlet: "" - expr: "sum(synapse_http_server_request_count) by (method)" - - record: 'synapse_http_server_request_count:servlet' - labels: - method: "" - expr: 'sum(synapse_http_server_request_count) by (servlet)' - - - record: 'synapse_http_server_request_count:total' - labels: - servlet: "" - expr: 'sum(synapse_http_server_request_count:by_method) by (servlet)' - - - record: 'synapse_cache:hit_ratio_5m' - expr: 'rate(synapse_util_caches_cache_hits[5m]) / rate(synapse_util_caches_cache[5m])' - - record: 'synapse_cache:hit_ratio_30s' - expr: 'rate(synapse_util_caches_cache_hits[30s]) / rate(synapse_util_caches_cache[30s])' - + # These 3 rules are used in the included Prometheus console - record: 'synapse_federation_client_sent' labels: type: "EDU" @@ -37,6 +15,7 @@ groups: type: "Query" expr: 'sum(synapse_federation_client_sent_queries) by (job)' + # These 3 rules are used in the included Prometheus console - record: 'synapse_federation_server_received' labels: type: "EDU" @@ -50,6 +29,7 @@ groups: type: "Query" expr: 'sum(synapse_federation_server_received_queries) by (job)' + # These 2 rules are used in the included Prometheus console - record: 'synapse_federation_transaction_queue_pending' labels: type: "EDU" @@ -59,6 +39,7 @@ groups: type: "PDU" expr: 'synapse_federation_transaction_queue_pending_pdus + 0' + # These 3 rules are used in the included Grafana dashboard - record: synapse_storage_events_persisted_by_source_type expr: sum without(type, origin_type, origin_entity) (synapse_storage_events_persisted_events_sep_total{origin_type="remote"}) labels: @@ -71,8 +52,12 @@ groups: expr: sum without(type, origin_type, origin_entity) (synapse_storage_events_persisted_events_sep_total{origin_entity!="*client*",origin_type="local"}) labels: type: bridges + + # This rule is used in the included Grafana dashboard - record: synapse_storage_events_persisted_by_event_type expr: sum without(origin_entity, origin_type) (synapse_storage_events_persisted_events_sep_total) + + # This rule is used in the included Grafana dashboard - record: synapse_storage_events_persisted_by_origin expr: sum without(type) (synapse_storage_events_persisted_events_sep_total) From 69fa29700e260f55238bfa480d3dd2b91a4353c0 Mon Sep 17 00:00:00 2001 From: Sean Quah <8349537+squahtx@users.noreply.github.com> Date: Thu, 8 Sep 2022 20:13:39 +0100 Subject: [PATCH 087/107] Re-type hint some collections in `/sync` code as read-only (#13754) Signed-off-by: Sean Quah --- changelog.d/13754.misc | 1 + synapse/handlers/sync.py | 20 ++++++++++---------- 2 files changed, 11 insertions(+), 10 deletions(-) create mode 100644 changelog.d/13754.misc diff --git a/changelog.d/13754.misc b/changelog.d/13754.misc new file mode 100644 index 0000000000..662ee00e99 --- /dev/null +++ b/changelog.d/13754.misc @@ -0,0 +1 @@ +Re-type hint some collections as read-only. diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py index 2d95b1fa24..5293fa4d0e 100644 --- a/synapse/handlers/sync.py +++ b/synapse/handlers/sync.py @@ -15,6 +15,7 @@ import itertools import logging from typing import ( TYPE_CHECKING, + AbstractSet, Any, Collection, Dict, @@ -1413,10 +1414,10 @@ class SyncHandler: async def _generate_sync_entry_for_device_list( self, sync_result_builder: "SyncResultBuilder", - newly_joined_rooms: Set[str], - newly_joined_or_invited_or_knocked_users: Set[str], - newly_left_rooms: Set[str], - newly_left_users: Set[str], + newly_joined_rooms: AbstractSet[str], + newly_joined_or_invited_or_knocked_users: AbstractSet[str], + newly_left_rooms: AbstractSet[str], + newly_left_users: AbstractSet[str], ) -> DeviceListUpdates: """Generate the DeviceListUpdates section of sync @@ -1434,8 +1435,7 @@ class SyncHandler: user_id = sync_result_builder.sync_config.user.to_string() since_token = sync_result_builder.since_token - # We're going to mutate these fields, so lets copy them rather than - # assume they won't get used later. + # Take a copy since these fields will be mutated later. newly_joined_or_invited_or_knocked_users = set( newly_joined_or_invited_or_knocked_users ) @@ -1635,8 +1635,8 @@ class SyncHandler: async def _generate_sync_entry_for_presence( self, sync_result_builder: "SyncResultBuilder", - newly_joined_rooms: Set[str], - newly_joined_or_invited_users: Set[str], + newly_joined_rooms: AbstractSet[str], + newly_joined_or_invited_users: AbstractSet[str], ) -> None: """Generates the presence portion of the sync response. Populates the `sync_result_builder` with the result. @@ -1694,7 +1694,7 @@ class SyncHandler: self, sync_result_builder: "SyncResultBuilder", account_data_by_room: Dict[str, Dict[str, JsonDict]], - ) -> Tuple[Set[str], Set[str], Set[str], Set[str]]: + ) -> Tuple[AbstractSet[str], AbstractSet[str], AbstractSet[str], AbstractSet[str]]: """Generates the rooms portion of the sync response. Populates the `sync_result_builder` with the result. @@ -2534,7 +2534,7 @@ class SyncResultBuilder: archived: List[ArchivedSyncResult] = attr.Factory(list) to_device: List[JsonDict] = attr.Factory(list) - def calculate_user_changes(self) -> Tuple[Set[str], Set[str]]: + def calculate_user_changes(self) -> Tuple[AbstractSet[str], AbstractSet[str]]: """Work out which other users have joined or left rooms we are joined to. This data only is only useful for an incremental sync. From f2d2481e56f06005de5ae8429eca3bb31834079e Mon Sep 17 00:00:00 2001 From: David Robertson Date: Fri, 9 Sep 2022 11:14:10 +0100 Subject: [PATCH 088/107] Require SQLite >= 3.27.0 (#13760) --- changelog.d/13760.removal | 1 + synapse/storage/database.py | 47 +++---- synapse/storage/databases/main/lock.py | 123 ++++++------------ synapse/storage/databases/main/stats.py | 78 +++++------ .../storage/databases/main/transactions.py | 30 ++--- synapse/storage/engines/_base.py | 8 -- synapse/storage/engines/postgres.py | 7 - synapse/storage/engines/sqlite.py | 13 +- tests/storage/test_base.py | 1 - 9 files changed, 103 insertions(+), 205 deletions(-) create mode 100644 changelog.d/13760.removal diff --git a/changelog.d/13760.removal b/changelog.d/13760.removal new file mode 100644 index 0000000000..624e7c3678 --- /dev/null +++ b/changelog.d/13760.removal @@ -0,0 +1 @@ +Synapse will now refuse to start if configured to use SQLite < 3.27. diff --git a/synapse/storage/database.py b/synapse/storage/database.py index b394a6658b..e881bff7fb 100644 --- a/synapse/storage/database.py +++ b/synapse/storage/database.py @@ -533,15 +533,14 @@ class DatabasePool: if isinstance(self.engine, Sqlite3Engine): self._unsafe_to_upsert_tables.add("user_directory_search") - if self.engine.can_native_upsert: - # Check ASAP (and then later, every 1s) to see if we have finished - # background updates of tables that aren't safe to update. - self._clock.call_later( - 0.0, - run_as_background_process, - "upsert_safety_check", - self._check_safe_to_upsert, - ) + # Check ASAP (and then later, every 1s) to see if we have finished + # background updates of tables that aren't safe to update. + self._clock.call_later( + 0.0, + run_as_background_process, + "upsert_safety_check", + self._check_safe_to_upsert, + ) def name(self) -> str: "Return the name of this database" @@ -1160,11 +1159,8 @@ class DatabasePool: attempts = 0 while True: try: - # We can autocommit if we are going to use native upserts - autocommit = ( - self.engine.can_native_upsert - and table not in self._unsafe_to_upsert_tables - ) + # We can autocommit if it is safe to upsert + autocommit = table not in self._unsafe_to_upsert_tables return await self.runInteraction( desc, @@ -1199,7 +1195,7 @@ class DatabasePool: ) -> bool: """ Pick the UPSERT method which works best on the platform. Either the - native one (Pg9.5+, recent SQLites), or fall back to an emulated method. + native one (Pg9.5+, SQLite >= 3.24), or fall back to an emulated method. Args: txn: The transaction to use. @@ -1207,14 +1203,15 @@ class DatabasePool: keyvalues: The unique key tables and their new values values: The nonunique columns and their new values insertion_values: additional key/values to use only when inserting - lock: True to lock the table when doing the upsert. + lock: True to lock the table when doing the upsert. Unused when performing + a native upsert. Returns: Returns True if a row was inserted or updated (i.e. if `values` is not empty then this always returns True) """ insertion_values = insertion_values or {} - if self.engine.can_native_upsert and table not in self._unsafe_to_upsert_tables: + if table not in self._unsafe_to_upsert_tables: return self.simple_upsert_txn_native_upsert( txn, table, keyvalues, values, insertion_values=insertion_values ) @@ -1365,14 +1362,12 @@ class DatabasePool: value_names: The value column names value_values: A list of each row's value column values. Ignored if value_names is empty. - lock: True to lock the table when doing the upsert. Unused if the database engine - supports native upserts. + lock: True to lock the table when doing the upsert. Unused when performing + a native upsert. """ - # We can autocommit if we are going to use native upserts - autocommit = ( - self.engine.can_native_upsert and table not in self._unsafe_to_upsert_tables - ) + # We can autocommit if it safe to upsert + autocommit = table not in self._unsafe_to_upsert_tables await self.runInteraction( desc, @@ -1406,10 +1401,10 @@ class DatabasePool: value_names: The value column names value_values: A list of each row's value column values. Ignored if value_names is empty. - lock: True to lock the table when doing the upsert. Unused if the database engine - supports native upserts. + lock: True to lock the table when doing the upsert. Unused when performing + a native upsert. """ - if self.engine.can_native_upsert and table not in self._unsafe_to_upsert_tables: + if table not in self._unsafe_to_upsert_tables: return self.simple_upsert_many_txn_native_upsert( txn, table, key_names, key_values, value_names, value_values ) diff --git a/synapse/storage/databases/main/lock.py b/synapse/storage/databases/main/lock.py index 2d7633fbd5..7270ef09da 100644 --- a/synapse/storage/databases/main/lock.py +++ b/synapse/storage/databases/main/lock.py @@ -129,91 +129,48 @@ class LockStore(SQLBaseStore): now = self._clock.time_msec() token = random_string(6) - if self.db_pool.engine.can_native_upsert: - - def _try_acquire_lock_txn(txn: LoggingTransaction) -> bool: - # We take out the lock if either a) there is no row for the lock - # already, b) the existing row has timed out, or c) the row is - # for this instance (which means the process got killed and - # restarted) - sql = """ - INSERT INTO worker_locks (lock_name, lock_key, instance_name, token, last_renewed_ts) - VALUES (?, ?, ?, ?, ?) - ON CONFLICT (lock_name, lock_key) - DO UPDATE - SET - token = EXCLUDED.token, - instance_name = EXCLUDED.instance_name, - last_renewed_ts = EXCLUDED.last_renewed_ts - WHERE - worker_locks.last_renewed_ts < ? - OR worker_locks.instance_name = EXCLUDED.instance_name - """ - txn.execute( - sql, - ( - lock_name, - lock_key, - self._instance_name, - token, - now, - now - _LOCK_TIMEOUT_MS, - ), - ) - - # We only acquired the lock if we inserted or updated the table. - return bool(txn.rowcount) - - did_lock = await self.db_pool.runInteraction( - "try_acquire_lock", - _try_acquire_lock_txn, - # We can autocommit here as we're executing a single query, this - # will avoid serialization errors. - db_autocommit=True, - ) - if not did_lock: - return None - - else: - # If we're on an old SQLite we emulate the above logic by first - # clearing out any existing stale locks and then upserting. - - def _try_acquire_lock_emulated_txn(txn: LoggingTransaction) -> bool: - sql = """ - DELETE FROM worker_locks - WHERE - lock_name = ? - AND lock_key = ? - AND (last_renewed_ts < ? OR instance_name = ?) - """ - txn.execute( - sql, - (lock_name, lock_key, now - _LOCK_TIMEOUT_MS, self._instance_name), - ) - - inserted = self.db_pool.simple_upsert_txn_emulated( - txn, - table="worker_locks", - keyvalues={ - "lock_name": lock_name, - "lock_key": lock_key, - }, - values={}, - insertion_values={ - "token": token, - "last_renewed_ts": self._clock.time_msec(), - "instance_name": self._instance_name, - }, - ) - - return inserted - - did_lock = await self.db_pool.runInteraction( - "try_acquire_lock_emulated", _try_acquire_lock_emulated_txn + def _try_acquire_lock_txn(txn: LoggingTransaction) -> bool: + # We take out the lock if either a) there is no row for the lock + # already, b) the existing row has timed out, or c) the row is + # for this instance (which means the process got killed and + # restarted) + sql = """ + INSERT INTO worker_locks (lock_name, lock_key, instance_name, token, last_renewed_ts) + VALUES (?, ?, ?, ?, ?) + ON CONFLICT (lock_name, lock_key) + DO UPDATE + SET + token = EXCLUDED.token, + instance_name = EXCLUDED.instance_name, + last_renewed_ts = EXCLUDED.last_renewed_ts + WHERE + worker_locks.last_renewed_ts < ? + OR worker_locks.instance_name = EXCLUDED.instance_name + """ + txn.execute( + sql, + ( + lock_name, + lock_key, + self._instance_name, + token, + now, + now - _LOCK_TIMEOUT_MS, + ), ) - if not did_lock: - return None + # We only acquired the lock if we inserted or updated the table. + return bool(txn.rowcount) + + did_lock = await self.db_pool.runInteraction( + "try_acquire_lock", + _try_acquire_lock_txn, + # We can autocommit here as we're executing a single query, this + # will avoid serialization errors. + db_autocommit=True, + ) + if not did_lock: + return None lock = Lock( self._reactor, diff --git a/synapse/storage/databases/main/stats.py b/synapse/storage/databases/main/stats.py index b4c652acf3..356d4ca788 100644 --- a/synapse/storage/databases/main/stats.py +++ b/synapse/storage/databases/main/stats.py @@ -446,59 +446,41 @@ class StatsStore(StateDeltasStore): absolutes: Absolute (set) fields additive_relatives: Fields that will be added onto if existing row present. """ - if self.database_engine.can_native_upsert: - absolute_updates = [ - "%(field)s = EXCLUDED.%(field)s" % {"field": field} - for field in absolutes.keys() - ] + absolute_updates = [ + "%(field)s = EXCLUDED.%(field)s" % {"field": field} + for field in absolutes.keys() + ] - relative_updates = [ - "%(field)s = EXCLUDED.%(field)s + COALESCE(%(table)s.%(field)s, 0)" - % {"table": table, "field": field} - for field in additive_relatives.keys() - ] + relative_updates = [ + "%(field)s = EXCLUDED.%(field)s + COALESCE(%(table)s.%(field)s, 0)" + % {"table": table, "field": field} + for field in additive_relatives.keys() + ] - insert_cols = [] - qargs = [] + insert_cols = [] + qargs = [] - for (key, val) in chain( - keyvalues.items(), absolutes.items(), additive_relatives.items() - ): - insert_cols.append(key) - qargs.append(val) + for (key, val) in chain( + keyvalues.items(), absolutes.items(), additive_relatives.items() + ): + insert_cols.append(key) + qargs.append(val) - sql = """ - INSERT INTO %(table)s (%(insert_cols_cs)s) - VALUES (%(insert_vals_qs)s) - ON CONFLICT (%(key_columns)s) DO UPDATE SET %(updates)s - """ % { - "table": table, - "insert_cols_cs": ", ".join(insert_cols), - "insert_vals_qs": ", ".join( - ["?"] * (len(keyvalues) + len(absolutes) + len(additive_relatives)) - ), - "key_columns": ", ".join(keyvalues), - "updates": ", ".join(chain(absolute_updates, relative_updates)), - } + sql = """ + INSERT INTO %(table)s (%(insert_cols_cs)s) + VALUES (%(insert_vals_qs)s) + ON CONFLICT (%(key_columns)s) DO UPDATE SET %(updates)s + """ % { + "table": table, + "insert_cols_cs": ", ".join(insert_cols), + "insert_vals_qs": ", ".join( + ["?"] * (len(keyvalues) + len(absolutes) + len(additive_relatives)) + ), + "key_columns": ", ".join(keyvalues), + "updates": ", ".join(chain(absolute_updates, relative_updates)), + } - txn.execute(sql, qargs) - else: - self.database_engine.lock_table(txn, table) - retcols = list(chain(absolutes.keys(), additive_relatives.keys())) - current_row = self.db_pool.simple_select_one_txn( - txn, table, keyvalues, retcols, allow_none=True - ) - if current_row is None: - merged_dict = {**keyvalues, **absolutes, **additive_relatives} - self.db_pool.simple_insert_txn(txn, table, merged_dict) - else: - for (key, val) in additive_relatives.items(): - if current_row[key] is None: - current_row[key] = val - else: - current_row[key] += val - current_row.update(absolutes) - self.db_pool.simple_update_one_txn(txn, table, keyvalues, current_row) + txn.execute(sql, qargs) async def _calculate_and_set_initial_state_for_room(self, room_id: str) -> None: """Calculate and insert an entry into room_stats_current. diff --git a/synapse/storage/databases/main/transactions.py b/synapse/storage/databases/main/transactions.py index ba79e19f7f..f8c6877ee8 100644 --- a/synapse/storage/databases/main/transactions.py +++ b/synapse/storage/databases/main/transactions.py @@ -221,25 +221,15 @@ class TransactionWorkerStore(CacheInvalidationWorkerStore): retry_interval: how long until next retry in ms """ - if self.database_engine.can_native_upsert: - await self.db_pool.runInteraction( - "set_destination_retry_timings", - self._set_destination_retry_timings_native, - destination, - failure_ts, - retry_last_ts, - retry_interval, - db_autocommit=True, # Safe as its a single upsert - ) - else: - await self.db_pool.runInteraction( - "set_destination_retry_timings", - self._set_destination_retry_timings_emulated, - destination, - failure_ts, - retry_last_ts, - retry_interval, - ) + await self.db_pool.runInteraction( + "set_destination_retry_timings", + self._set_destination_retry_timings_native, + destination, + failure_ts, + retry_last_ts, + retry_interval, + db_autocommit=True, # Safe as it's a single upsert + ) def _set_destination_retry_timings_native( self, @@ -249,8 +239,6 @@ class TransactionWorkerStore(CacheInvalidationWorkerStore): retry_last_ts: int, retry_interval: int, ) -> None: - assert self.database_engine.can_native_upsert - # Upsert retry time interval if retry_interval is zero (i.e. we're # resetting it) or greater than the existing retry interval. # diff --git a/synapse/storage/engines/_base.py b/synapse/storage/engines/_base.py index 971ff82693..0d16a419a4 100644 --- a/synapse/storage/engines/_base.py +++ b/synapse/storage/engines/_base.py @@ -43,14 +43,6 @@ class BaseDatabaseEngine(Generic[ConnectionType], metaclass=abc.ABCMeta): def single_threaded(self) -> bool: ... - @property - @abc.abstractmethod - def can_native_upsert(self) -> bool: - """ - Do we support native UPSERTs? - """ - ... - @property @abc.abstractmethod def supports_using_any_list(self) -> bool: diff --git a/synapse/storage/engines/postgres.py b/synapse/storage/engines/postgres.py index 517f9d5f98..7f7d006ac2 100644 --- a/synapse/storage/engines/postgres.py +++ b/synapse/storage/engines/postgres.py @@ -158,13 +158,6 @@ class PostgresEngine(BaseDatabaseEngine[psycopg2.extensions.connection]): cursor.close() db_conn.commit() - @property - def can_native_upsert(self) -> bool: - """ - Can we use native UPSERTs? - """ - return True - @property def supports_using_any_list(self) -> bool: """Do we support using `a = ANY(?)` and passing a list""" diff --git a/synapse/storage/engines/sqlite.py b/synapse/storage/engines/sqlite.py index 621f2c5efe..095ae0a096 100644 --- a/synapse/storage/engines/sqlite.py +++ b/synapse/storage/engines/sqlite.py @@ -48,14 +48,6 @@ class Sqlite3Engine(BaseDatabaseEngine[sqlite3.Connection]): def single_threaded(self) -> bool: return True - @property - def can_native_upsert(self) -> bool: - """ - Do we support native UPSERTs? This requires SQLite3 3.24+, plus some - more work we haven't done yet to tell what was inserted vs updated. - """ - return sqlite3.sqlite_version_info >= (3, 24, 0) - @property def supports_using_any_list(self) -> bool: """Do we support using `a = ANY(?)` and passing a list""" @@ -70,12 +62,11 @@ class Sqlite3Engine(BaseDatabaseEngine[sqlite3.Connection]): self, db_conn: sqlite3.Connection, allow_outdated_version: bool = False ) -> None: if not allow_outdated_version: - version = sqlite3.sqlite_version_info # Synapse is untested against older SQLite versions, and we don't want # to let users upgrade to a version of Synapse with broken support for their # sqlite version, because it risks leaving them with a half-upgraded db. - if version < (3, 22, 0): - raise RuntimeError("Synapse requires sqlite 3.22 or above.") + if sqlite3.sqlite_version_info < (3, 27, 0): + raise RuntimeError("Synapse requires sqlite 3.27 or above.") def check_new_database(self, txn: Cursor) -> None: """Gets called when setting up a brand new database. This allows us to diff --git a/tests/storage/test_base.py b/tests/storage/test_base.py index cce8e75c74..40e58f8199 100644 --- a/tests/storage/test_base.py +++ b/tests/storage/test_base.py @@ -54,7 +54,6 @@ class SQLBaseStoreTestCase(unittest.TestCase): sqlite_config = {"name": "sqlite3"} engine = create_engine(sqlite_config) fake_engine = Mock(wraps=engine) - fake_engine.can_native_upsert = False fake_engine.in_transaction.return_value = False db = DatabasePool(Mock(), Mock(config=sqlite_config), fake_engine) From c85c5ace525c3cadac8501c2eba4abbc91d9f09d Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 9 Sep 2022 11:29:04 +0100 Subject: [PATCH 089/107] Add rust to CI (#13763) --- .github/workflows/tests.yml | 83 ++++++++++++++++++++++++++++++++++++- changelog.d/13763.misc | 1 + rust/src/lib.rs | 1 + 3 files changed, 84 insertions(+), 1 deletion(-) create mode 100644 changelog.d/13763.misc diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 50dc8e30d4..7c4ae3d7ff 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -10,6 +10,23 @@ concurrency: cancel-in-progress: true jobs: + # Job to detect what has changed so we don't run e.g. Rust checks on PRs that + # don't modify Rust code. + changes: + runs-on: ubuntu-latest + outputs: + rust: ${{ !startsWith(github.ref, 'refs/pull/') || steps.filter.outputs.rust }} + steps: + - uses: dorny/paths-filter@v2 + id: filter + # We only check on PRs + if: startsWith(github.ref, 'refs/pull/') + with: + filters: | + rust: + - 'rust/**' + - 'Cargo.toml' + check-sampleconfig: runs-on: ubuntu-latest steps: @@ -65,10 +82,54 @@ jobs: extras: "all" - run: poetry run scripts-dev/check_pydantic_models.py + lint-clippy: + runs-on: ubuntu-latest + needs: changes + if: ${{ needs.changes.outputs.rust == 'true' }} + + steps: + - uses: actions/checkout@v2 + + - name: Install Rust + uses: actions-rs/toolchain@v1 + with: + toolchain: 1.61.0 + override: true + components: clippy + - uses: Swatinem/rust-cache@v2 + + - run: cargo clippy + + lint-rustfmt: + runs-on: ubuntu-latest + needs: changes + if: ${{ needs.changes.outputs.rust == 'true' }} + + steps: + - uses: actions/checkout@v2 + + - name: Install Rust + uses: actions-rs/toolchain@v1 + with: + toolchain: 1.61.0 + override: true + components: rustfmt + - uses: Swatinem/rust-cache@v2 + + - run: cargo fmt --check + # Dummy step to gate other tests on without repeating the whole list linting-done: if: ${{ !cancelled() }} # Run this even if prior jobs were skipped - needs: [lint, lint-crlf, lint-newsfile, lint-pydantic, check-sampleconfig, check-schema-delta] + needs: + - lint + - lint-crlf + - lint-newsfile + - lint-pydantic + - check-sampleconfig + - check-schema-delta + - lint-clippy + - lint-rustfmt runs-on: ubuntu-latest steps: - run: "true" @@ -384,6 +445,25 @@ jobs: shell: bash name: Run Complement Tests + cargo-test: + if: ${{ needs.changes.outputs.rust == 'true' }} + runs-on: ubuntu-latest + needs: + - linting-done + - changes + + steps: + - uses: actions/checkout@v2 + + - name: Install Rust + uses: actions-rs/toolchain@v1 + with: + toolchain: 1.61.0 + override: true + - uses: Swatinem/rust-cache@v2 + + - run: cargo test + # a job which marks all the other jobs as complete, thus allowing PRs to be merged. tests-done: if: ${{ always() }} @@ -398,6 +478,7 @@ jobs: - export-data - portdb - complement + - cargo-test runs-on: ubuntu-latest steps: - uses: matrix-org/done-action@v2 diff --git a/changelog.d/13763.misc b/changelog.d/13763.misc new file mode 100644 index 0000000000..2e0dd68a0f --- /dev/null +++ b/changelog.d/13763.misc @@ -0,0 +1 @@ +Add a stub Rust crate. diff --git a/rust/src/lib.rs b/rust/src/lib.rs index fc4eb39154..142fc2ed93 100644 --- a/rust/src/lib.rs +++ b/rust/src/lib.rs @@ -11,5 +11,6 @@ fn sum_as_string(a: usize, b: usize) -> PyResult { #[pymodule] fn synapse_rust(_py: Python<'_>, m: &PyModule) -> PyResult<()> { m.add_function(wrap_pyfunction!(sum_as_string, m)?)?; + Ok(()) } From 3d9f82efcb9c337197c9f50a88ec3fb541ee08ff Mon Sep 17 00:00:00 2001 From: Patrick Cloke Date: Fri, 9 Sep 2022 07:08:41 -0400 Subject: [PATCH 090/107] Use an upsert for `receipts_graph`. (#13752) Instead of a delete, then insert. This was previously done for `receipts_linearized` in 2dc430d36ef793b38d6d79ec8db4ea60588df2ee (#7607). --- changelog.d/13752.misc | 1 + synapse/storage/databases/main/receipts.py | 12 ++++-------- 2 files changed, 5 insertions(+), 8 deletions(-) create mode 100644 changelog.d/13752.misc diff --git a/changelog.d/13752.misc b/changelog.d/13752.misc new file mode 100644 index 0000000000..7624861b9f --- /dev/null +++ b/changelog.d/13752.misc @@ -0,0 +1 @@ +User an additional database query when persisting receipts. diff --git a/synapse/storage/databases/main/receipts.py b/synapse/storage/databases/main/receipts.py index 124c70ad37..3838409519 100644 --- a/synapse/storage/databases/main/receipts.py +++ b/synapse/storage/databases/main/receipts.py @@ -812,7 +812,7 @@ class ReceiptsWorkerStore(SQLBaseStore): # FIXME: This shouldn't invalidate the whole cache txn.call_after(self._get_linearized_receipts_for_room.invalidate, (room_id,)) - self.db_pool.simple_delete_txn( + self.db_pool.simple_upsert_txn( txn, table="receipts_graph", keyvalues={ @@ -820,17 +820,13 @@ class ReceiptsWorkerStore(SQLBaseStore): "receipt_type": receipt_type, "user_id": user_id, }, - ) - self.db_pool.simple_insert_txn( - txn, - table="receipts_graph", values={ - "room_id": room_id, - "receipt_type": receipt_type, - "user_id": user_id, "event_ids": json_encoder.encode(event_ids), "data": json_encoder.encode(data), }, + # receipts_graph has a unique constraint on + # (user_id, room_id, receipt_type), so no need to lock + lock=False, ) From f694bb71b7ea7841a5b5db3d884dfda5a3f78023 Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Fri, 9 Sep 2022 11:30:06 -0500 Subject: [PATCH 091/107] Strip number suffix from instance name to consolidate services that traces are spread over (#13729) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The problem with many services is that it makes it hard to find which service has the trace you want, see https://github.com/jaegertracing/jaeger-ui/issues/985 Previously, we split traces out into services based on their instance name like `matrix.org client_reader-1`, etc but there are many worker instances of the same `client_reader` so there is a lot to click through. With this PR, all of the traces are just collected under the worker type like `client_reader`, `event_persister` 😇 Note: A Synapse worker instance name is an opaque string with the number convention only being our own thing for the `matrix.org` deployment. But seems pretty sensible to group things this way. --- changelog.d/13729.misc | 1 + synapse/logging/opentracing.py | 13 ++++++++++++- 2 files changed, 13 insertions(+), 1 deletion(-) create mode 100644 changelog.d/13729.misc diff --git a/changelog.d/13729.misc b/changelog.d/13729.misc new file mode 100644 index 0000000000..c6a6f617e3 --- /dev/null +++ b/changelog.d/13729.misc @@ -0,0 +1 @@ +Strip number suffix from instance name to consolidate services that traces are spread over. diff --git a/synapse/logging/opentracing.py b/synapse/logging/opentracing.py index 482316a1ff..adf3f54770 100644 --- a/synapse/logging/opentracing.py +++ b/synapse/logging/opentracing.py @@ -203,6 +203,9 @@ if TYPE_CHECKING: # Helper class +# Matches the number suffix in an instance name like "matrix.org client_reader-8" +STRIP_INSTANCE_NUMBER_SUFFIX_REGEX = re.compile(r"[_-]?\d+$") + class _DummyTagNames: """wrapper of opentracings tags. We need to have them if we @@ -441,9 +444,17 @@ def init_tracer(hs: "HomeServer") -> None: from jaeger_client.metrics.prometheus import PrometheusMetricsFactory + # Instance names are opaque strings but by stripping off the number suffix, + # we can get something that looks like a "worker type", e.g. + # "client_reader-1" -> "client_reader" so we don't spread the traces across + # so many services. + instance_name_by_type = re.sub( + STRIP_INSTANCE_NUMBER_SUFFIX_REGEX, "", hs.get_instance_name() + ) + config = JaegerConfig( config=hs.config.tracing.jaeger_config, - service_name=f"{hs.config.server.server_name} {hs.get_instance_name()}", + service_name=f"{hs.config.server.server_name} {instance_name_by_type}", scope_manager=LogContextScopeManager(), metrics_factory=PrometheusMetricsFactory(), ) From a911ffb42cc88adc8084a04acf6fd651efba278f Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Fri, 9 Sep 2022 11:31:37 -0500 Subject: [PATCH 092/107] Tag trace with instance name (#13761) We tag the Synapse instance name so that it's an easy jumping off point into the logs. Can also be used to filter for an instance that is under load. As suggested by @clokep and @reivilibre in, - https://github.com/matrix-org/synapse/pull/13729#discussion_r964719258 - https://github.com/matrix-org/synapse/pull/13729#discussion_r964733578 --- changelog.d/13761.misc | 1 + synapse/api/auth.py | 7 +++++++ synapse/logging/opentracing.py | 6 ++++-- 3 files changed, 12 insertions(+), 2 deletions(-) create mode 100644 changelog.d/13761.misc diff --git a/changelog.d/13761.misc b/changelog.d/13761.misc new file mode 100644 index 0000000000..f7aa8c459a --- /dev/null +++ b/changelog.d/13761.misc @@ -0,0 +1 @@ +Tag traces with the instance name to be able to easily jump into the right logs and filter traces by instance. diff --git a/synapse/api/auth.py b/synapse/api/auth.py index 8e54ef84b2..4a75eb6b21 100644 --- a/synapse/api/auth.py +++ b/synapse/api/auth.py @@ -32,6 +32,7 @@ from synapse.appservice import ApplicationService from synapse.http import get_request_user_agent from synapse.http.site import SynapseRequest from synapse.logging.opentracing import ( + SynapseTags, active_span, force_tracing, start_active_span, @@ -161,6 +162,12 @@ class Auth: parent_span.set_tag( "authenticated_entity", requester.authenticated_entity ) + # We tag the Synapse instance name so that it's an easy jumping + # off point into the logs. Can also be used to filter for an + # instance that is under load. + parent_span.set_tag( + SynapseTags.INSTANCE_NAME, self.hs.get_instance_name() + ) parent_span.set_tag("user_id", requester.user.to_string()) if requester.device_id is not None: parent_span.set_tag("device_id", requester.device_id) diff --git a/synapse/logging/opentracing.py b/synapse/logging/opentracing.py index adf3f54770..ca2735dd6d 100644 --- a/synapse/logging/opentracing.py +++ b/synapse/logging/opentracing.py @@ -298,6 +298,8 @@ class SynapseTags: # Whether the sync response has new data to be returned to the client. SYNC_RESULT = "sync.new_data" + INSTANCE_NAME = "instance_name" + # incoming HTTP request ID (as written in the logs) REQUEST_ID = "request_id" @@ -1043,11 +1045,11 @@ def trace_servlet( # with JsonResource). scope.span.set_operation_name(request.request_metrics.name) - # set the tags *after* the servlet completes, in case it decided to - # prioritise the span (tags will get dropped on unprioritised spans) request_tags[ SynapseTags.REQUEST_TAG ] = request.request_metrics.start_context.tag + # set the tags *after* the servlet completes, in case it decided to + # prioritise the span (tags will get dropped on unprioritised spans) for k, v in request_tags.items(): scope.span.set_tag(k, v) From 4c4889cac0e6f7df4689287b9fddea1bf8b15b7f Mon Sep 17 00:00:00 2001 From: Nick Mills-Barrett Date: Fri, 9 Sep 2022 19:00:21 +0100 Subject: [PATCH 093/107] Concurrently collect room unread counts for push badges (#13765) Most of the time this function is heavily cached, but when that isn't the case fetching the counts room by room slows down push delivery on users with many (thousands) of rooms. Signed off by Nick @ Beeper. --- changelog.d/13765.misc | 1 + synapse/push/push_tools.py | 13 ++++++++++--- 2 files changed, 11 insertions(+), 3 deletions(-) create mode 100644 changelog.d/13765.misc diff --git a/changelog.d/13765.misc b/changelog.d/13765.misc new file mode 100644 index 0000000000..fdda5cf3b6 --- /dev/null +++ b/changelog.d/13765.misc @@ -0,0 +1 @@ +Concurrently fetch room push actions when calculating badge counts. Contributed by Nick @ Beeper (@fizzadar). diff --git a/synapse/push/push_tools.py b/synapse/push/push_tools.py index 6661887d9f..658bf373b7 100644 --- a/synapse/push/push_tools.py +++ b/synapse/push/push_tools.py @@ -17,6 +17,7 @@ from synapse.events import EventBase from synapse.push.presentable_names import calculate_room_name, name_from_member_event from synapse.storage.controllers import StorageControllers from synapse.storage.databases.main import DataStore +from synapse.util.async_helpers import concurrently_execute async def get_badge_count(store: DataStore, user_id: str, group_by_room: bool) -> int: @@ -25,13 +26,19 @@ async def get_badge_count(store: DataStore, user_id: str, group_by_room: bool) - badge = len(invites) - for room_id in joins: - notifs = await ( - store.get_unread_event_push_actions_by_room_for_user( + room_notifs = [] + + async def get_room_unread_count(room_id: str) -> None: + room_notifs.append( + await store.get_unread_event_push_actions_by_room_for_user( room_id, user_id, ) ) + + await concurrently_execute(get_room_unread_count, joins, 10) + + for notifs in room_notifs: if notifs.notify_count == 0: continue From ebfeac7c5ded851a2639911ec6adf9d0fcdb029a Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 12 Sep 2022 11:03:42 +0100 Subject: [PATCH 094/107] Check if Rust lib needs rebuilding. (#13759) This protects against the common mistake of failing to remember to rebuild Rust code after making changes. --- changelog.d/13759.misc | 1 + rust/Cargo.toml | 4 ++ rust/build.rs | 45 ++++++++++++++++++ rust/src/lib.rs | 10 +++- stubs/synapse/synapse_rust.pyi | 1 + synapse/__init__.py | 5 ++ synapse/util/rust.py | 84 ++++++++++++++++++++++++++++++++++ 7 files changed, 149 insertions(+), 1 deletion(-) create mode 100644 changelog.d/13759.misc create mode 100644 rust/build.rs create mode 100644 synapse/util/rust.py diff --git a/changelog.d/13759.misc b/changelog.d/13759.misc new file mode 100644 index 0000000000..f91c512483 --- /dev/null +++ b/changelog.d/13759.misc @@ -0,0 +1 @@ +Add a check for editable installs if the Rust library needs rebuilding. diff --git a/rust/Cargo.toml b/rust/Cargo.toml index 0a9760cafc..deddf3cec2 100644 --- a/rust/Cargo.toml +++ b/rust/Cargo.toml @@ -19,3 +19,7 @@ name = "synapse.synapse_rust" [dependencies] pyo3 = { version = "0.16.5", features = ["extension-module", "macros", "abi3", "abi3-py37"] } + +[build-dependencies] +blake2 = "0.10.4" +hex = "0.4.3" diff --git a/rust/build.rs b/rust/build.rs new file mode 100644 index 0000000000..2117975e56 --- /dev/null +++ b/rust/build.rs @@ -0,0 +1,45 @@ +//! This build script calculates the hash of all files in the `src/` +//! directory and adds it as an environment variable during build time. +//! +//! This is used so that the python code can detect when the built native module +//! does not match the source in-tree, helping to detect the case where the +//! source has been updated but the library hasn't been rebuilt. + +use std::path::PathBuf; + +use blake2::{Blake2b512, Digest}; + +fn main() -> Result<(), std::io::Error> { + let mut dirs = vec![PathBuf::from("src")]; + + let mut paths = Vec::new(); + while let Some(path) = dirs.pop() { + let mut entries = std::fs::read_dir(path)? + .map(|res| res.map(|e| e.path())) + .collect::, std::io::Error>>()?; + + entries.sort(); + + for entry in entries { + if entry.is_dir() { + dirs.push(entry) + } else { + paths.push(entry.to_str().expect("valid rust paths").to_string()); + } + } + } + + paths.sort(); + + let mut hasher = Blake2b512::new(); + + for path in paths { + let bytes = std::fs::read(path)?; + hasher.update(bytes); + } + + let hex_digest = hex::encode(hasher.finalize()); + println!("cargo:rustc-env=SYNAPSE_RUST_DIGEST={hex_digest}"); + + Ok(()) +} diff --git a/rust/src/lib.rs b/rust/src/lib.rs index 142fc2ed93..ba42465fb8 100644 --- a/rust/src/lib.rs +++ b/rust/src/lib.rs @@ -1,5 +1,13 @@ use pyo3::prelude::*; +/// Returns the hash of all the rust source files at the time it was compiled. +/// +/// Used by python to detect if the rust library is outdated. +#[pyfunction] +fn get_rust_file_digest() -> &'static str { + env!("SYNAPSE_RUST_DIGEST") +} + /// Formats the sum of two numbers as string. #[pyfunction] #[pyo3(text_signature = "(a, b, /)")] @@ -11,6 +19,6 @@ fn sum_as_string(a: usize, b: usize) -> PyResult { #[pymodule] fn synapse_rust(_py: Python<'_>, m: &PyModule) -> PyResult<()> { m.add_function(wrap_pyfunction!(sum_as_string, m)?)?; - + m.add_function(wrap_pyfunction!(get_rust_file_digest, m)?)?; Ok(()) } diff --git a/stubs/synapse/synapse_rust.pyi b/stubs/synapse/synapse_rust.pyi index 5b51ba05d7..8658d3138f 100644 --- a/stubs/synapse/synapse_rust.pyi +++ b/stubs/synapse/synapse_rust.pyi @@ -1 +1,2 @@ def sum_as_string(a: int, b: int) -> str: ... +def get_rust_file_digest() -> str: ... diff --git a/synapse/__init__.py b/synapse/__init__.py index b1369aca8f..1bed6393bd 100644 --- a/synapse/__init__.py +++ b/synapse/__init__.py @@ -20,6 +20,8 @@ import json import os import sys +from synapse.util.rust import check_rust_lib_up_to_date + # Check that we're not running on an unsupported Python version. if sys.version_info < (3, 7): print("Synapse requires Python 3.7 or above.") @@ -78,3 +80,6 @@ if bool(os.environ.get("SYNAPSE_TEST_PATCH_LOG_CONTEXTS", False)): from synapse.util.patch_inline_callbacks import do_patch do_patch() + + +check_rust_lib_up_to_date() diff --git a/synapse/util/rust.py b/synapse/util/rust.py new file mode 100644 index 0000000000..30ecb9ffd9 --- /dev/null +++ b/synapse/util/rust.py @@ -0,0 +1,84 @@ +# Copyright 2022 The Matrix.org Foundation C.I.C. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import sys +from hashlib import blake2b + +import synapse +from synapse.synapse_rust import get_rust_file_digest + + +def check_rust_lib_up_to_date() -> None: + """For editable installs check if the rust library is outdated and needs to + be rebuilt. + """ + + if not _dist_is_editable(): + return + + synapse_dir = os.path.dirname(synapse.__file__) + synapse_root = os.path.abspath(os.path.join(synapse_dir, "..")) + + # Double check we've not gone into site-packages... + if os.path.basename(synapse_root) == "site-packages": + return + + # ... and it looks like the root of a python project. + if not os.path.exists("pyproject.toml"): + return + + # Get the hash of all Rust source files + hash = _hash_rust_files_in_directory(os.path.join(synapse_root, "rust", "src")) + + if hash != get_rust_file_digest(): + raise Exception("Rust module outdated. Please rebuild using `poetry install`") + + +def _hash_rust_files_in_directory(directory: str) -> str: + """Get the hash of all files in a directory (recursively)""" + + directory = os.path.abspath(directory) + + paths = [] + + dirs = [directory] + while dirs: + dir = dirs.pop() + with os.scandir(dir) as d: + for entry in d: + if entry.is_dir(): + dirs.append(entry.path) + else: + paths.append(entry.path) + + # We sort to make sure that we get a consistent and well-defined ordering. + paths.sort() + + hasher = blake2b() + + for path in paths: + with open(os.path.join(directory, path), "rb") as f: + hasher.update(f.read()) + + return hasher.hexdigest() + + +def _dist_is_editable() -> bool: + """Is distribution an editable install?""" + for path_item in sys.path: + egg_link = os.path.join(path_item, "matrix-synapse.egg-link") + if os.path.isfile(egg_link): + return True + return False From da41a7cd618d11b05c2c04c39068fd4b1e1b7894 Mon Sep 17 00:00:00 2001 From: Nick Mills-Barrett Date: Mon, 12 Sep 2022 12:58:33 +0100 Subject: [PATCH 095/107] Remove check current state membership up to date (#13745) * Remove checks for membership column in current_state_events * Add schema script to force through the `current_state_events_membership` background job Contributed by Nick @ Beeper (@fizzadar). --- changelog.d/13745.misc | 1 + synapse/storage/databases/main/roommember.py | 202 ++++-------------- ..._update_current_state_events_membership.py | 52 +++++ 3 files changed, 100 insertions(+), 155 deletions(-) create mode 100644 changelog.d/13745.misc create mode 100644 synapse/storage/schema/main/delta/72/07force_update_current_state_events_membership.py diff --git a/changelog.d/13745.misc b/changelog.d/13745.misc new file mode 100644 index 0000000000..e97a789c0e --- /dev/null +++ b/changelog.d/13745.misc @@ -0,0 +1 @@ +Remove old queries to join room memberships to current state events. Contributed by Nick @ Beeper (@fizzadar). diff --git a/synapse/storage/databases/main/roommember.py b/synapse/storage/databases/main/roommember.py index 6e1ff5626b..fdb4684e12 100644 --- a/synapse/storage/databases/main/roommember.py +++ b/synapse/storage/databases/main/roommember.py @@ -32,10 +32,7 @@ import attr from synapse.api.constants import EventTypes, Membership from synapse.metrics import LaterGauge -from synapse.metrics.background_process_metrics import ( - run_as_background_process, - wrap_as_background_process, -) +from synapse.metrics.background_process_metrics import wrap_as_background_process from synapse.storage._base import SQLBaseStore, db_to_json, make_in_list_sql_clause from synapse.storage.database import ( DatabasePool, @@ -91,16 +88,6 @@ class RoomMemberWorkerStore(EventsWorkerStore): # at a time. Keyed by room_id. self._joined_host_linearizer = Linearizer("_JoinedHostsCache") - # Is the current_state_events.membership up to date? Or is the - # background update still running? - self._current_state_events_membership_up_to_date = False - - txn = db_conn.cursor( - txn_name="_check_safe_current_state_events_membership_updated" - ) - self._check_safe_current_state_events_membership_updated_txn(txn) - txn.close() - if ( self.hs.config.worker.run_background_tasks and self.hs.config.metrics.metrics_flags.known_servers @@ -157,34 +144,6 @@ class RoomMemberWorkerStore(EventsWorkerStore): self._known_servers_count = max([count, 1]) return self._known_servers_count - def _check_safe_current_state_events_membership_updated_txn( - self, txn: LoggingTransaction - ) -> None: - """Checks if it is safe to assume the new current_state_events - membership column is up to date - """ - - pending_update = self.db_pool.simple_select_one_txn( - txn, - table="background_updates", - keyvalues={"update_name": _CURRENT_STATE_MEMBERSHIP_UPDATE_NAME}, - retcols=["update_name"], - allow_none=True, - ) - - self._current_state_events_membership_up_to_date = not pending_update - - # If the update is still running, reschedule to run. - if pending_update: - self._clock.call_later( - 15.0, - run_as_background_process, - "_check_safe_current_state_events_membership_updated", - self.db_pool.runInteraction, - "_check_safe_current_state_events_membership_updated", - self._check_safe_current_state_events_membership_updated_txn, - ) - @cached(max_entries=100000, iterable=True) async def get_users_in_room(self, room_id: str) -> List[str]: """ @@ -212,31 +171,14 @@ class RoomMemberWorkerStore(EventsWorkerStore): `get_current_hosts_in_room()` and so we can re-use the cache but it's not horrible to have here either. """ - # If we can assume current_state_events.membership is up to date - # then we can avoid a join, which is a Very Good Thing given how - # frequently this function gets called. - if self._current_state_events_membership_up_to_date: - sql = """ - SELECT c.state_key FROM current_state_events as c - /* Get the depth of the event from the events table */ - INNER JOIN events AS e USING (event_id) - WHERE c.type = 'm.room.member' AND c.room_id = ? AND membership = ? - /* Sorted by lowest depth first */ - ORDER BY e.depth ASC; - """ - else: - sql = """ - SELECT c.state_key FROM room_memberships as m - /* Get the depth of the event from the events table */ - INNER JOIN events AS e USING (event_id) - INNER JOIN current_state_events as c - ON m.event_id = c.event_id - AND m.room_id = c.room_id - AND m.user_id = c.state_key - WHERE c.type = 'm.room.member' AND c.room_id = ? AND m.membership = ? - /* Sorted by lowest depth first */ - ORDER BY e.depth ASC; - """ + sql = """ + SELECT c.state_key FROM current_state_events as c + /* Get the depth of the event from the events table */ + INNER JOIN events AS e USING (event_id) + WHERE c.type = 'm.room.member' AND c.room_id = ? AND membership = ? + /* Sorted by lowest depth first */ + ORDER BY e.depth ASC; + """ txn.execute(sql, (room_id, Membership.JOIN)) return [r[0] for r in txn] @@ -353,28 +295,14 @@ class RoomMemberWorkerStore(EventsWorkerStore): # We do this all in one transaction to keep the cache small. # FIXME: get rid of this when we have room_stats - # If we can assume current_state_events.membership is up to date - # then we can avoid a join, which is a Very Good Thing given how - # frequently this function gets called. - if self._current_state_events_membership_up_to_date: - # Note, rejected events will have a null membership field, so - # we we manually filter them out. - sql = """ - SELECT count(*), membership FROM current_state_events - WHERE type = 'm.room.member' AND room_id = ? - AND membership IS NOT NULL - GROUP BY membership - """ - else: - sql = """ - SELECT count(*), m.membership FROM room_memberships as m - INNER JOIN current_state_events as c - ON m.event_id = c.event_id - AND m.room_id = c.room_id - AND m.user_id = c.state_key - WHERE c.type = 'm.room.member' AND c.room_id = ? - GROUP BY m.membership - """ + # Note, rejected events will have a null membership field, so + # we we manually filter them out. + sql = """ + SELECT count(*), membership FROM current_state_events + WHERE type = 'm.room.member' AND room_id = ? + AND membership IS NOT NULL + GROUP BY membership + """ txn.execute(sql, (room_id,)) res: Dict[str, MemberSummary] = {} @@ -383,30 +311,18 @@ class RoomMemberWorkerStore(EventsWorkerStore): # we order by membership and then fairly arbitrarily by event_id so # heroes are consistent - if self._current_state_events_membership_up_to_date: - # Note, rejected events will have a null membership field, so - # we we manually filter them out. - sql = """ - SELECT state_key, membership, event_id - FROM current_state_events - WHERE type = 'm.room.member' AND room_id = ? - AND membership IS NOT NULL - ORDER BY - CASE membership WHEN ? THEN 1 WHEN ? THEN 2 ELSE 3 END ASC, - event_id ASC - LIMIT ? - """ - else: - sql = """ - SELECT c.state_key, m.membership, c.event_id - FROM room_memberships as m - INNER JOIN current_state_events as c USING (room_id, event_id) - WHERE c.type = 'm.room.member' AND c.room_id = ? - ORDER BY - CASE m.membership WHEN ? THEN 1 WHEN ? THEN 2 ELSE 3 END ASC, - c.event_id ASC - LIMIT ? - """ + # Note, rejected events will have a null membership field, so + # we we manually filter them out. + sql = """ + SELECT state_key, membership, event_id + FROM current_state_events + WHERE type = 'm.room.member' AND room_id = ? + AND membership IS NOT NULL + ORDER BY + CASE membership WHEN ? THEN 1 WHEN ? THEN 2 ELSE 3 END ASC, + event_id ASC + LIMIT ? + """ # 6 is 5 (number of heroes) plus 1, in case one of them is the calling user. txn.execute(sql, (room_id, Membership.JOIN, Membership.INVITE, 6)) @@ -649,27 +565,15 @@ class RoomMemberWorkerStore(EventsWorkerStore): # We use `current_state_events` here and not `local_current_membership` # as a) this gets called with remote users and b) this only gets called # for rooms the server is participating in. - if self._current_state_events_membership_up_to_date: - sql = """ - SELECT room_id, e.instance_name, e.stream_ordering - FROM current_state_events AS c - INNER JOIN events AS e USING (room_id, event_id) - WHERE - c.type = 'm.room.member' - AND c.state_key = ? - AND c.membership = ? - """ - else: - sql = """ - SELECT room_id, e.instance_name, e.stream_ordering - FROM current_state_events AS c - INNER JOIN room_memberships AS m USING (room_id, event_id) - INNER JOIN events AS e USING (room_id, event_id) - WHERE - c.type = 'm.room.member' - AND c.state_key = ? - AND m.membership = ? - """ + sql = """ + SELECT room_id, e.instance_name, e.stream_ordering + FROM current_state_events AS c + INNER JOIN events AS e USING (room_id, event_id) + WHERE + c.type = 'm.room.member' + AND c.state_key = ? + AND c.membership = ? + """ txn.execute(sql, (user_id, Membership.JOIN)) return frozenset( @@ -707,27 +611,15 @@ class RoomMemberWorkerStore(EventsWorkerStore): user_ids, ) - if self._current_state_events_membership_up_to_date: - sql = f""" - SELECT c.state_key, room_id, e.instance_name, e.stream_ordering - FROM current_state_events AS c - INNER JOIN events AS e USING (room_id, event_id) - WHERE - c.type = 'm.room.member' - AND c.membership = ? - AND {clause} - """ - else: - sql = f""" - SELECT c.state_key, room_id, e.instance_name, e.stream_ordering - FROM current_state_events AS c - INNER JOIN room_memberships AS m USING (room_id, event_id) - INNER JOIN events AS e USING (room_id, event_id) - WHERE - c.type = 'm.room.member' - AND m.membership = ? - AND {clause} - """ + sql = f""" + SELECT c.state_key, room_id, e.instance_name, e.stream_ordering + FROM current_state_events AS c + INNER JOIN events AS e USING (room_id, event_id) + WHERE + c.type = 'm.room.member' + AND c.membership = ? + AND {clause} + """ txn.execute(sql, [Membership.JOIN] + args) diff --git a/synapse/storage/schema/main/delta/72/07force_update_current_state_events_membership.py b/synapse/storage/schema/main/delta/72/07force_update_current_state_events_membership.py new file mode 100644 index 0000000000..b5853d125c --- /dev/null +++ b/synapse/storage/schema/main/delta/72/07force_update_current_state_events_membership.py @@ -0,0 +1,52 @@ +# Copyright 2022 Beeper +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +""" +Forces through the `current_state_events_membership` background job so checks +for its completion can be removed. + +Note the background job must still remain defined in the database class. +""" + + +def run_upgrade(cur, database_engine, *args, **kwargs): + cur.execute("SELECT update_name FROM background_updates") + rows = cur.fetchall() + for row in rows: + if row[0] == "current_state_events_membership": + break + # No pending background job so nothing to do here + else: + return + + # Populate membership field for all current_state_events, this may take + # a while but was originally handled via a background update in 2019. + cur.execute( + """ + UPDATE current_state_events + SET membership = ( + SELECT membership FROM room_memberships + WHERE event_id = current_state_events.event_id + ) + """ + ) + + # Finally, delete the background job because we've handled it above + cur.execute( + """ + DELETE FROM background_updates + WHERE update_name = 'current_state_events_membership' + """ + ) From dd7484b5628dc297d0324198c9f5765247f982b5 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Mon, 12 Sep 2022 13:26:33 +0100 Subject: [PATCH 096/107] Fix CI on non-PR builds (#13769) Mark cargo-test as skippable since it only runs on Rust code change. --- .github/workflows/tests.yml | 1 + changelog.d/13769.misc | 1 + 2 files changed, 2 insertions(+) create mode 100644 changelog.d/13769.misc diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 7c4ae3d7ff..b4dd74ea4f 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -488,3 +488,4 @@ jobs: # The newsfile lint may be skipped on non PR builds skippable: lint-newsfile + cargo-test diff --git a/changelog.d/13769.misc b/changelog.d/13769.misc new file mode 100644 index 0000000000..2e0dd68a0f --- /dev/null +++ b/changelog.d/13769.misc @@ -0,0 +1 @@ +Add a stub Rust crate. From 75713374454ef36ec549018344996e06a67dd2bf Mon Sep 17 00:00:00 2001 From: Brendan Abolivier Date: Mon, 12 Sep 2022 15:11:18 +0200 Subject: [PATCH 097/107] Fix typo in ratelimiting documentation (#13727) --- changelog.d/13727.doc | 1 + docs/usage/configuration/config_documentation.md | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) create mode 100644 changelog.d/13727.doc diff --git a/changelog.d/13727.doc b/changelog.d/13727.doc new file mode 100644 index 0000000000..ba530b409d --- /dev/null +++ b/changelog.d/13727.doc @@ -0,0 +1 @@ +Fix a typo in the documentation for the login ratelimiting configuration. diff --git a/docs/usage/configuration/config_documentation.md b/docs/usage/configuration/config_documentation.md index ae490d13a4..cd546041b2 100644 --- a/docs/usage/configuration/config_documentation.md +++ b/docs/usage/configuration/config_documentation.md @@ -1393,7 +1393,7 @@ This option specifies several limits for login: client is attempting to log into. Defaults to `per_second: 0.17`, `burst_count: 3`. -* `failted_attempts` ratelimits login requests based on the account the +* `failed_attempts` ratelimits login requests based on the account the client is attempting to log into, based on the amount of failed login attempts for this account. Defaults to `per_second: 0.17`, `burst_count: 3`. From fa2f3d8d0c915c6ebd84ef53526ac04c91fd4a19 Mon Sep 17 00:00:00 2001 From: Mathieu Velten Date: Mon, 12 Sep 2022 19:31:23 +0200 Subject: [PATCH 098/107] Fix GHA skippable syntax (#13778) Signed-off-by: Mathieu Velten --- .github/workflows/tests.yml | 3 ++- changelog.d/13778.misc | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) create mode 100644 changelog.d/13778.misc diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index b4dd74ea4f..bf70f8373e 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -486,6 +486,7 @@ jobs: needs: ${{ toJSON(needs) }} # The newsfile lint may be skipped on non PR builds - skippable: + # Cargo test is skipped if there is no changes on Rust code + skippable: | lint-newsfile cargo-test diff --git a/changelog.d/13778.misc b/changelog.d/13778.misc new file mode 100644 index 0000000000..2e0dd68a0f --- /dev/null +++ b/changelog.d/13778.misc @@ -0,0 +1 @@ +Add a stub Rust crate. From cdbb6412327b542e0dead792717fe58253291131 Mon Sep 17 00:00:00 2001 From: Nick Mills-Barrett Date: Tue, 13 Sep 2022 08:16:37 +0100 Subject: [PATCH 099/107] Add receipts event stream ordering (#13703) --- changelog.d/13703.misc | 1 + synapse/_scripts/synapse_port_db.py | 2 + synapse/storage/databases/main/receipts.py | 74 ++++++++++++++++++- .../72/05receipts_event_stream_ordering.sql | 19 +++++ 4 files changed, 95 insertions(+), 1 deletion(-) create mode 100644 changelog.d/13703.misc create mode 100644 synapse/storage/schema/main/delta/72/05receipts_event_stream_ordering.sql diff --git a/changelog.d/13703.misc b/changelog.d/13703.misc new file mode 100644 index 0000000000..685a29b17d --- /dev/null +++ b/changelog.d/13703.misc @@ -0,0 +1 @@ +Add & populate `event_stream_ordering` column on receipts table for future optimisation of push action processing. Contributed by Nick @ Beeper (@fizzadar). diff --git a/synapse/_scripts/synapse_port_db.py b/synapse/_scripts/synapse_port_db.py index 543bba27c2..30983c47fb 100755 --- a/synapse/_scripts/synapse_port_db.py +++ b/synapse/_scripts/synapse_port_db.py @@ -67,6 +67,7 @@ from synapse.storage.databases.main.media_repository import ( ) from synapse.storage.databases.main.presence import PresenceBackgroundUpdateStore from synapse.storage.databases.main.pusher import PusherWorkerStore +from synapse.storage.databases.main.receipts import ReceiptsBackgroundUpdateStore from synapse.storage.databases.main.registration import ( RegistrationBackgroundUpdateStore, find_max_generated_user_id_localpart, @@ -203,6 +204,7 @@ class Store( PushRuleStore, PusherWorkerStore, PresenceBackgroundUpdateStore, + ReceiptsBackgroundUpdateStore, ): def execute(self, f: Callable[..., R], *args: Any, **kwargs: Any) -> Awaitable[R]: return self.db_pool.runInteraction(f.__name__, f, *args, **kwargs) diff --git a/synapse/storage/databases/main/receipts.py b/synapse/storage/databases/main/receipts.py index 3838409519..719a12b0ae 100644 --- a/synapse/storage/databases/main/receipts.py +++ b/synapse/storage/databases/main/receipts.py @@ -675,6 +675,7 @@ class ReceiptsWorkerStore(SQLBaseStore): values={ "stream_id": stream_id, "event_id": event_id, + "event_stream_ordering": stream_ordering, "data": json_encoder.encode(data), }, # receipts_linearized has a unique constraint on @@ -830,5 +831,76 @@ class ReceiptsWorkerStore(SQLBaseStore): ) -class ReceiptsStore(ReceiptsWorkerStore): +class ReceiptsBackgroundUpdateStore(SQLBaseStore): + POPULATE_RECEIPT_EVENT_STREAM_ORDERING = "populate_event_stream_ordering" + + def __init__( + self, + database: DatabasePool, + db_conn: LoggingDatabaseConnection, + hs: "HomeServer", + ): + super().__init__(database, db_conn, hs) + + self.db_pool.updates.register_background_update_handler( + self.POPULATE_RECEIPT_EVENT_STREAM_ORDERING, + self._populate_receipt_event_stream_ordering, + ) + + async def _populate_receipt_event_stream_ordering( + self, progress: JsonDict, batch_size: int + ) -> int: + def _populate_receipt_event_stream_ordering_txn( + txn: LoggingTransaction, + ) -> bool: + + if "max_stream_id" in progress: + max_stream_id = progress["max_stream_id"] + else: + txn.execute("SELECT max(stream_id) FROM receipts_linearized") + res = txn.fetchone() + if res is None or res[0] is None: + return True + else: + max_stream_id = res[0] + + start = progress.get("stream_id", 0) + stop = start + batch_size + + sql = """ + UPDATE receipts_linearized + SET event_stream_ordering = ( + SELECT stream_ordering + FROM events + WHERE event_id = receipts_linearized.event_id + ) + WHERE stream_id >= ? AND stream_id < ? + """ + txn.execute(sql, (start, stop)) + + self.db_pool.updates._background_update_progress_txn( + txn, + self.POPULATE_RECEIPT_EVENT_STREAM_ORDERING, + { + "stream_id": stop, + "max_stream_id": max_stream_id, + }, + ) + + return stop > max_stream_id + + finished = await self.db_pool.runInteraction( + "_remove_devices_from_device_inbox_txn", + _populate_receipt_event_stream_ordering_txn, + ) + + if finished: + await self.db_pool.updates._end_background_update( + self.POPULATE_RECEIPT_EVENT_STREAM_ORDERING + ) + + return batch_size + + +class ReceiptsStore(ReceiptsWorkerStore, ReceiptsBackgroundUpdateStore): pass diff --git a/synapse/storage/schema/main/delta/72/05receipts_event_stream_ordering.sql b/synapse/storage/schema/main/delta/72/05receipts_event_stream_ordering.sql new file mode 100644 index 0000000000..2a822f4509 --- /dev/null +++ b/synapse/storage/schema/main/delta/72/05receipts_event_stream_ordering.sql @@ -0,0 +1,19 @@ +/* Copyright 2022 Beeper + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +ALTER TABLE receipts_linearized ADD COLUMN event_stream_ordering BIGINT; + +INSERT INTO background_updates (update_name, progress_json) VALUES + ('populate_event_stream_ordering', '{}'); From 4b678b20a2f55d6c11ed1acfd8f517d15ef49c6e Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 13 Sep 2022 09:20:28 +0100 Subject: [PATCH 100/107] 1.67.0 --- CHANGES.md | 10 ++++++++-- debian/changelog | 6 ++++++ pyproject.toml | 2 +- 3 files changed, 15 insertions(+), 3 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 0300fcbd30..244dc6b0d6 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,5 +1,5 @@ -Synapse 1.67.0rc1 (2022-09-06) -============================== +Synapse 1.67.0 (2022-09-13) +=========================== This release removes using the deprecated direct TCP replication configuration for workers. Server admins should use Redis instead. See the [upgrade @@ -19,6 +19,12 @@ database will require SQLite version 3.27.0 or higher. (The [current minimum See [#12983](https://github.com/matrix-org/synapse/issues/12983) and the [upgrade notes](https://matrix-org.github.io/synapse/v1.67/upgrade.html#upgrading-to-v1670). for more details. +No significant changes since 1.67.0rc1. + + +Synapse 1.67.0rc1 (2022-09-06) +============================== + Features -------- diff --git a/debian/changelog b/debian/changelog index 4fa54dc5ee..77c8f7ad0b 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,9 @@ +matrix-synapse-py3 (1.67.0) stable; urgency=medium + + * New Synapse release 1.67.0. + + -- Synapse Packaging team Tue, 13 Sep 2022 09:19:56 +0100 + matrix-synapse-py3 (1.67.0~rc1) stable; urgency=medium [ Erik Johnston ] diff --git a/pyproject.toml b/pyproject.toml index dfb5c788be..97be0f15eb 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -54,7 +54,7 @@ skip_gitignore = true [tool.poetry] name = "matrix-synapse" -version = "1.67.0rc1" +version = "1.67.0" description = "Homeserver for the Matrix decentralised comms protocol" authors = ["Matrix.org Team and Contributors "] license = "Apache-2.0" From 80bb098d8775cc2ad1bf5abd150913577e643481 Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Tue, 13 Sep 2022 09:55:10 +0100 Subject: [PATCH 101/107] Fixup changelog --- CHANGES.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 244dc6b0d6..be44903bfe 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -8,15 +8,15 @@ notes](https://matrix-org.github.io/synapse/v1.67/upgrade.html#upgrading-to-v167 The minimum version of `poetry` supported for managing source checkouts is now 1.2.0. -**Notice:** from the next major release (v1.68.0) installing Synapse from a source +**Notice:** from the next major release (1.68.0) installing Synapse from a source checkout will require a recent Rust compiler. Those using packages or `pip install matrix-synapse` will not be affected. See the [upgrade notes](https://matrix-org.github.io/synapse/v1.67/upgrade.html#upgrading-to-v1670). -**Notice:** from the next major release (v1.68.0), running Synapse with a SQLite +**Notice:** from the next major release (1.68.0), running Synapse with a SQLite database will require SQLite version 3.27.0 or higher. (The [current minimum version is SQLite 3.22.0](https://github.com/matrix-org/synapse/blob/release-v1.67/synapse/storage/engines/sqlite.py#L69-L78).) -See [#12983](https://github.com/matrix-org/synapse/issues/12983) and the [upgrade notes](https://matrix-org.github.io/synapse/v1.67/upgrade.html#upgrading-to-v1670). for more details. +See [#12983](https://github.com/matrix-org/synapse/issues/12983) and the [upgrade notes](https://matrix-org.github.io/synapse/v1.67/upgrade.html#upgrading-to-v1670) for more details. No significant changes since 1.67.0rc1. From 41df25bbbd29caeb539269a436fbe6bc57fad93c Mon Sep 17 00:00:00 2001 From: Richard van der Hoff <1389908+richvdh@users.noreply.github.com> Date: Tue, 13 Sep 2022 10:01:21 +0100 Subject: [PATCH 102/107] installation.md: require libpq on M1 macs (#13480) --- changelog.d/13480.doc | 1 + docs/setup/installation.md | 5 +++-- 2 files changed, 4 insertions(+), 2 deletions(-) create mode 100644 changelog.d/13480.doc diff --git a/changelog.d/13480.doc b/changelog.d/13480.doc new file mode 100644 index 0000000000..ae5df16367 --- /dev/null +++ b/changelog.d/13480.doc @@ -0,0 +1 @@ +Note that `libpq` is required on ARM-based Macs. diff --git a/docs/setup/installation.md b/docs/setup/installation.md index 90737520ba..96833effc6 100644 --- a/docs/setup/installation.md +++ b/docs/setup/installation.md @@ -303,9 +303,10 @@ You may need to install the latest Xcode developer tools: xcode-select --install ``` -On ARM-based Macs you may need to explicitly install libjpeg which is a pillow dependency. You can use Homebrew (https://brew.sh): +On ARM-based Macs you may need to install libjpeg and libpq. +You can use Homebrew (https://brew.sh): ```sh - brew install jpeg + brew install jpeg libpq ``` On macOS Catalina (10.15) you may need to explicitly install OpenSSL From 540afb0bfcc407328a75b0b9910f26b407dbcaed Mon Sep 17 00:00:00 2001 From: David Robertson Date: Tue, 13 Sep 2022 10:17:23 +0100 Subject: [PATCH 103/107] Simplify CI tests DAG (#13784) * Simplify CI tests DAG * Changelog --- .github/workflows/tests.yml | 4 ---- changelog.d/13784.misc | 1 + 2 files changed, 1 insertion(+), 4 deletions(-) create mode 100644 changelog.d/13784.misc diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index bf70f8373e..a5a217d015 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -468,10 +468,6 @@ jobs: tests-done: if: ${{ always() }} needs: - - check-sampleconfig - - lint - - lint-crlf - - lint-newsfile - trial - trial-olddeps - sytest diff --git a/changelog.d/13784.misc b/changelog.d/13784.misc new file mode 100644 index 0000000000..e7a542cd80 --- /dev/null +++ b/changelog.d/13784.misc @@ -0,0 +1 @@ +Simplify the dependency DAG in the tests workflow. From b60d47ab2c55580fc1941497964cd33c27838231 Mon Sep 17 00:00:00 2001 From: David Robertson Date: Tue, 13 Sep 2022 10:53:11 +0100 Subject: [PATCH 104/107] Updates to the schema dump script (#13770) --- changelog.d/13770.misc | 1 + scripts-dev/make_full_schema.sh | 48 +++++++------------ .../schema/state/delta/30/state_stream.sql | 4 ++ 3 files changed, 21 insertions(+), 32 deletions(-) create mode 100644 changelog.d/13770.misc diff --git a/changelog.d/13770.misc b/changelog.d/13770.misc new file mode 100644 index 0000000000..36ac91400a --- /dev/null +++ b/changelog.d/13770.misc @@ -0,0 +1 @@ +Update the script which makes full schema dumps. diff --git a/scripts-dev/make_full_schema.sh b/scripts-dev/make_full_schema.sh index f0e22d4ca2..61394360ce 100755 --- a/scripts-dev/make_full_schema.sh +++ b/scripts-dev/make_full_schema.sh @@ -9,8 +9,10 @@ export PGHOST="localhost" POSTGRES_DB_NAME="synapse_full_schema.$$" -SQLITE_FULL_SCHEMA_OUTPUT_FILE="full.sql.sqlite" -POSTGRES_FULL_SCHEMA_OUTPUT_FILE="full.sql.postgres" +SQLITE_SCHEMA_FILE="schema.sql.sqlite" +SQLITE_ROWS_FILE="rows.sql.sqlite" +POSTGRES_SCHEMA_FILE="full.sql.postgres" +POSTGRES_ROWS_FILE="rows.sql.postgres" REQUIRED_DEPS=("matrix-synapse" "psycopg2") @@ -22,7 +24,7 @@ usage() { echo " Username to connect to local postgres instance. The password will be requested" echo " during script execution." echo "-c" - echo " CI mode. Enables coverage tracking and prints every command that the script runs." + echo " CI mode. Prints every command that the script runs." echo "-o " echo " Directory to output full schema files to." echo "-h" @@ -37,11 +39,6 @@ while getopts "p:co:h" opt; do c) # Print all commands that are being executed set -x - - # Modify required dependencies for coverage - REQUIRED_DEPS+=("coverage" "coverage-enable-subprocess") - - COVERAGE=1 ;; o) command -v realpath > /dev/null || (echo "The -o flag requires the 'realpath' binary to be installed" && exit 1) @@ -102,6 +99,7 @@ SQLITE_DB=$TMPDIR/homeserver.db POSTGRES_CONFIG=$TMPDIR/postgres.conf # Ensure these files are delete on script exit +# TODO: the trap should also drop the temp postgres DB trap 'rm -rf $TMPDIR' EXIT cat > "$SQLITE_CONFIG" < "$OUTPUT_DIR/$SQLITE_FULL_SCHEMA_OUTPUT_FILE" +echo "Dumping SQLite3 schema to '$OUTPUT_DIR/$SQLITE_SCHEMA_FILE' and '$OUTPUT_DIR/$SQLITE_ROWS_FILE'..." +sqlite3 "$SQLITE_DB" ".schema --indent" > "$OUTPUT_DIR/$SQLITE_SCHEMA_FILE" +sqlite3 "$SQLITE_DB" ".dump --data-only --nosys" > "$OUTPUT_DIR/$SQLITE_ROWS_FILE" -echo "Dumping Postgres schema to '$OUTPUT_DIR/$POSTGRES_FULL_SCHEMA_OUTPUT_FILE'..." -pg_dump --format=plain --no-tablespaces --no-acl --no-owner $POSTGRES_DB_NAME | sed -e '/^--/d' -e 's/public\.//g' -e '/^SET /d' -e '/^SELECT /d' > "$OUTPUT_DIR/$POSTGRES_FULL_SCHEMA_OUTPUT_FILE" +echo "Dumping Postgres schema to '$OUTPUT_DIR/$POSTGRES_SCHEMA_FILE' and '$OUTPUT_DIR/$POSTGRES_ROWS_FILE'..." +pg_dump --format=plain --schema-only --no-tablespaces --no-acl --no-owner "$POSTGRES_DB_NAME" | sed -e '/^$/d' -e '/^--/d' -e 's/public\.//g' -e '/^SET /d' -e '/^SELECT /d' > "$OUTPUT_DIR/$POSTGRES_SCHEMA_FILE" +pg_dump --format=plain --data-only --inserts --no-tablespaces --no-acl --no-owner "$POSTGRES_DB_NAME" | sed -e '/^$/d' -e '/^--/d' -e 's/public\.//g' -e '/^SET /d' -e '/^SELECT /d' > "$OUTPUT_DIR/$POSTGRES_ROWS_FILE" echo "Cleaning up temporary Postgres database..." dropdb $POSTGRES_DB_NAME diff --git a/synapse/storage/schema/state/delta/30/state_stream.sql b/synapse/storage/schema/state/delta/30/state_stream.sql index e85699e82e..bdaf8b02d5 100644 --- a/synapse/storage/schema/state/delta/30/state_stream.sql +++ b/synapse/storage/schema/state/delta/30/state_stream.sql @@ -26,6 +26,10 @@ * (event, state) pair, we can use that stream_ordering to identify when * the new state was assigned for the event. */ + +/* NB: This table belongs to the `main` logical database; it should not be present + * in `state`. + */ CREATE TABLE IF NOT EXISTS ex_outlier_stream( event_stream_ordering BIGINT PRIMARY KEY NOT NULL, event_id TEXT NOT NULL, From 12dacecabd27680dc77c17724953ecda0801b5ea Mon Sep 17 00:00:00 2001 From: Mathieu Velten Date: Tue, 13 Sep 2022 16:14:28 +0200 Subject: [PATCH 105/107] Make sequence `cache_invalidation_stream_seq` begin at `2` (#13766) Signed-off-by: Mathieu Velten Co-authored-by: Sean Quah <8349537+squahtx@users.noreply.github.com> --- changelog.d/13766.bugfix | 1 + synapse/storage/schema/__init__.py | 1 + ...n_cache_invalidation_seq_at_2.sql.postgres | 23 +++++++++++++++++++ 3 files changed, 25 insertions(+) create mode 100644 changelog.d/13766.bugfix create mode 100644 synapse/storage/schema/main/delta/72/08begin_cache_invalidation_seq_at_2.sql.postgres diff --git a/changelog.d/13766.bugfix b/changelog.d/13766.bugfix new file mode 100644 index 0000000000..c708e54f9c --- /dev/null +++ b/changelog.d/13766.bugfix @@ -0,0 +1 @@ +Fix a long-standing bug where the `cache_invalidation_stream_seq` sequence would begin at 1 instead of 2. diff --git a/synapse/storage/schema/__init__.py b/synapse/storage/schema/__init__.py index 256f745dc0..32cda5e3ba 100644 --- a/synapse/storage/schema/__init__.py +++ b/synapse/storage/schema/__init__.py @@ -76,6 +76,7 @@ Changes in SCHEMA_VERSION = 72: - event_edges.(room_id, is_state) are no longer written to. - Tables related to groups are dropped. - Unused column application_services_state.last_txn is dropped + - Cache invalidation stream id sequence now begins at 2 to match code expectation. """ diff --git a/synapse/storage/schema/main/delta/72/08begin_cache_invalidation_seq_at_2.sql.postgres b/synapse/storage/schema/main/delta/72/08begin_cache_invalidation_seq_at_2.sql.postgres new file mode 100644 index 0000000000..69931fe971 --- /dev/null +++ b/synapse/storage/schema/main/delta/72/08begin_cache_invalidation_seq_at_2.sql.postgres @@ -0,0 +1,23 @@ +/* Copyright 2022 The Matrix.org Foundation C.I.C + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +-- The sequence needs to begin at 2 because a bunch of code assumes that +-- get_next_id_txn will return values >= 2, cf this comment: +-- https://github.com/matrix-org/synapse/blob/b93bd95e8ab64d27ae26841020f62ee61272a5f2/synapse/storage/util/id_generators.py#L344 + +SELECT setval('cache_invalidation_stream_seq', ( + SELECT COALESCE(MAX(last_value), 1) FROM cache_invalidation_stream_seq +)); From 21687ec189f404bcee98ae61b008afc8c5094400 Mon Sep 17 00:00:00 2001 From: reivilibre Date: Wed, 14 Sep 2022 08:28:12 +0000 Subject: [PATCH 106/107] Fix a long-standing spec compliance bug where Synapse would accept a trailing slash on the end of `/get_missing_events` federation requests. (#13789) * Don't accept a trailing slash on the end of /get_missing_events * Newsfile Signed-off-by: Olivier Wilkinson (reivilibre) Signed-off-by: Olivier Wilkinson (reivilibre) --- changelog.d/13789.bugfix | 1 + synapse/federation/transport/server/federation.py | 3 +-- 2 files changed, 2 insertions(+), 2 deletions(-) create mode 100644 changelog.d/13789.bugfix diff --git a/changelog.d/13789.bugfix b/changelog.d/13789.bugfix new file mode 100644 index 0000000000..9e1e3e0fa7 --- /dev/null +++ b/changelog.d/13789.bugfix @@ -0,0 +1 @@ +Fix a long-standing spec compliance bug where Synapse would accept a trailing slash on the end of `/get_missing_events` federation requests. \ No newline at end of file diff --git a/synapse/federation/transport/server/federation.py b/synapse/federation/transport/server/federation.py index f7884bfbe0..6bb4659c4c 100644 --- a/synapse/federation/transport/server/federation.py +++ b/synapse/federation/transport/server/federation.py @@ -549,8 +549,7 @@ class FederationClientKeysClaimServlet(BaseFederationServerServlet): class FederationGetMissingEventsServlet(BaseFederationServerServlet): - # TODO(paul): Why does this path alone end with "/?" optional? - PATH = "/get_missing_events/(?P[^/]*)/?" + PATH = "/get_missing_events/(?P[^/]*)" async def on_POST( self, From c73774467edb04c372caecb9e843542654f7610b Mon Sep 17 00:00:00 2001 From: Sean Quah <8349537+squahtx@users.noreply.github.com> Date: Wed, 14 Sep 2022 10:42:57 +0100 Subject: [PATCH 107/107] Fix bug in device list caching when remote users leave rooms (#13749) When a remote user leaves the last room shared with the homeserver, we have to mark their device list as unsubscribed, otherwise we would hold on to a stale device list in our cache. Crucially, the device list would remain cached even after the remote user rejoined the room, which could lead to E2EE failures until the next change to the remote user's device list. Fixes #13651. Signed-off-by: Sean Quah --- changelog.d/13749.bugfix | 1 + synapse/handlers/device.py | 11 -------- synapse/handlers/e2e_keys.py | 26 +++++++++++++++++++ synapse/storage/controllers/persist_events.py | 20 +++++++++++--- tests/handlers/test_e2e_keys.py | 8 +++++- 5 files changed, 51 insertions(+), 15 deletions(-) create mode 100644 changelog.d/13749.bugfix diff --git a/changelog.d/13749.bugfix b/changelog.d/13749.bugfix new file mode 100644 index 0000000000..8ffafec07b --- /dev/null +++ b/changelog.d/13749.bugfix @@ -0,0 +1 @@ +Fix a long standing bug where device lists would remain cached when remote users left and rejoined the last room shared with the local homeserver. diff --git a/synapse/handlers/device.py b/synapse/handlers/device.py index c5ac169644..901e2310b7 100644 --- a/synapse/handlers/device.py +++ b/synapse/handlers/device.py @@ -45,7 +45,6 @@ from synapse.types import ( JsonDict, StreamKeyType, StreamToken, - UserID, get_domain_from_id, get_verify_key_from_cross_signing_key, ) @@ -324,8 +323,6 @@ class DeviceHandler(DeviceWorkerHandler): self.device_list_updater.incoming_device_list_update, ) - hs.get_distributor().observe("user_left_room", self.user_left_room) - # Whether `_handle_new_device_update_async` is currently processing. self._handle_new_device_update_is_processing = False @@ -569,14 +566,6 @@ class DeviceHandler(DeviceWorkerHandler): StreamKeyType.DEVICE_LIST, position, users=[from_user_id] ) - async def user_left_room(self, user: UserID, room_id: str) -> None: - user_id = user.to_string() - room_ids = await self.store.get_rooms_for_user(user_id) - if not room_ids: - # We no longer share rooms with this user, so we'll no longer - # receive device updates. Mark this in DB. - await self.store.mark_remote_user_device_list_as_unsubscribed(user_id) - async def store_dehydrated_device( self, user_id: str, diff --git a/synapse/handlers/e2e_keys.py b/synapse/handlers/e2e_keys.py index ec81639c78..8eed63ccf3 100644 --- a/synapse/handlers/e2e_keys.py +++ b/synapse/handlers/e2e_keys.py @@ -175,6 +175,32 @@ class E2eKeysHandler: user_ids_not_in_cache, remote_results, ) = await self.store.get_user_devices_from_cache(query_list) + + # Check that the homeserver still shares a room with all cached users. + # Note that this check may be slightly racy when a remote user leaves a + # room after we have fetched their cached device list. In the worst case + # we will do extra federation queries for devices that we had cached. + cached_users = set(remote_results.keys()) + valid_cached_users = ( + await self.store.get_users_server_still_shares_room_with( + remote_results.keys() + ) + ) + invalid_cached_users = cached_users - valid_cached_users + if invalid_cached_users: + # Fix up results. If we get here, there is either a bug in device + # list tracking, or we hit the race mentioned above. + user_ids_not_in_cache.update(invalid_cached_users) + for invalid_user_id in invalid_cached_users: + remote_results.pop(invalid_user_id) + # This log message may be removed if it turns out it's almost + # entirely triggered by races. + logger.error( + "Devices for %s were cached, but the server no longer shares " + "any rooms with them. The cached device lists are stale.", + invalid_cached_users, + ) + for user_id, devices in remote_results.items(): user_devices = results.setdefault(user_id, {}) for device_id, device in devices.items(): diff --git a/synapse/storage/controllers/persist_events.py b/synapse/storage/controllers/persist_events.py index dad3731b9b..501dbbc990 100644 --- a/synapse/storage/controllers/persist_events.py +++ b/synapse/storage/controllers/persist_events.py @@ -598,9 +598,9 @@ class EventsPersistenceStorageController: # room state_delta_for_room: Dict[str, DeltaState] = {} - # Set of remote users which were in rooms the server has left. We - # should check if we still share any rooms and if not we mark their - # device lists as stale. + # Set of remote users which were in rooms the server has left or who may + # have left rooms the server is in. We should check if we still share any + # rooms and if not we mark their device lists as stale. potentially_left_users: Set[str] = set() if not backfilled: @@ -725,6 +725,20 @@ class EventsPersistenceStorageController: current_state = {} delta.no_longer_in_room = True + # Add all remote users that might have left rooms. + potentially_left_users.update( + user_id + for event_type, user_id in delta.to_delete + if event_type == EventTypes.Member + and not self.is_mine_id(user_id) + ) + potentially_left_users.update( + user_id + for event_type, user_id in delta.to_insert.keys() + if event_type == EventTypes.Member + and not self.is_mine_id(user_id) + ) + state_delta_for_room[room_id] = delta await self.persist_events_store._persist_events_and_state_updates( diff --git a/tests/handlers/test_e2e_keys.py b/tests/handlers/test_e2e_keys.py index 1e6ad4b663..95698bc275 100644 --- a/tests/handlers/test_e2e_keys.py +++ b/tests/handlers/test_e2e_keys.py @@ -891,6 +891,12 @@ class E2eKeysHandlerTestCase(unittest.HomeserverTestCase): new_callable=mock.MagicMock, return_value=make_awaitable(["some_room_id"]), ) + mock_get_users = mock.patch.object( + self.store, + "get_users_server_still_shares_room_with", + new_callable=mock.MagicMock, + return_value=make_awaitable({remote_user_id}), + ) mock_request = mock.patch.object( self.hs.get_federation_client(), "query_user_devices", @@ -898,7 +904,7 @@ class E2eKeysHandlerTestCase(unittest.HomeserverTestCase): return_value=make_awaitable(response_body), ) - with mock_get_rooms, mock_request as mocked_federation_request: + with mock_get_rooms, mock_get_users, mock_request as mocked_federation_request: # Make the first query and sanity check it succeeds. response_1 = self.get_success( e2e_handler.query_devices(