+
+
+
+
+
+
+
+
+
+
diff --git a/debian/hash_password.ronn b/debian/hash_password.ronn
index 5d0df53802..b68d4a210e 100644
--- a/debian/hash_password.ronn
+++ b/debian/hash_password.ronn
@@ -29,8 +29,8 @@ A sample YAML file accepted by `hash_password` is described below:
## OPTIONS
* `-p`, `--password`:
- Read the password form the command line if [password] is supplied.
- If not, prompt the user and read the password form the `STDIN`.
+ Read the password form the command line if [password] is supplied, or from `STDIN`.
+ If not, prompt the user and read the password from the tty prompt.
It is not recommended to type the password on the command line
directly. Use the STDIN instead.
@@ -45,7 +45,14 @@ Hash from the command line:
$ hash_password -p "p@ssw0rd"
$2b$12$VJNqWQYfsWTEwcELfoSi4Oa8eA17movHqqi8.X8fWFpum7SxZ9MFe
-Hash from the STDIN:
+Hash from the stdin:
+
+ $ cat password_file | hash_password
+ Password:
+ Confirm password:
+ $2b$12$AszlvfmJl2esnyhmn8m/kuR2tdXgROWtWxnX.rcuAbM8ErLoUhybG
+
+Hash from the prompt:
$ hash_password
Password:
diff --git a/debian/register_new_matrix_user.ronn b/debian/register_new_matrix_user.ronn
index 0410b1f4cd..aa305ec671 100644
--- a/debian/register_new_matrix_user.ronn
+++ b/debian/register_new_matrix_user.ronn
@@ -31,8 +31,12 @@ A sample YAML file accepted by `register_new_matrix_user` is described below:
Local part of the new user. Will prompt if omitted.
* `-p`, `--password`:
- New password for user. Will prompt if omitted. Supplying the password
- on the command line is not recommended. Use the STDIN instead.
+ New password for user. Will prompt if this option and `--password-file` are omitted.
+ Supplying the password on the command line is not recommended.
+
+ * `--password-file`:
+ File containing the new password for user. If set, overrides `--password`.
+ This is a more secure alternative to specifying the password on the command line.
* `-a`, `--admin`:
Register new user as an admin. Will prompt if omitted.
@@ -44,6 +48,9 @@ A sample YAML file accepted by `register_new_matrix_user` is described below:
Shared secret as defined in server config file. This is an optional
parameter as it can be also supplied via the YAML file.
+ * `--exists-ok`:
+ Do not fail if the user already exists. The user account will be not updated in this case.
+
* `server_url`:
URL of the home server. Defaults to 'https://localhost:8448'.
diff --git a/debian/templates b/debian/templates
index cab05715d0..7bfd3c2e9f 100644
--- a/debian/templates
+++ b/debian/templates
@@ -5,7 +5,7 @@ _Description: Name of the server:
servers via federation. This is normally the public hostname of the
server running synapse, but can be different if you set up delegation.
Please refer to the delegation documentation in this case:
- https://github.com/element-hq/synapse/blob/master/docs/delegate.md.
+ https://element-hq.github.io/synapse/latest/delegate.html.
Template: matrix-synapse/report-stats
Type: boolean
diff --git a/dev-docs/Makefile b/dev-docs/Makefile
deleted file mode 100644
index d4bb2cbb9e..0000000000
--- a/dev-docs/Makefile
+++ /dev/null
@@ -1,20 +0,0 @@
-# Minimal makefile for Sphinx documentation
-#
-
-# You can set these variables from the command line, and also
-# from the environment for the first two.
-SPHINXOPTS ?=
-SPHINXBUILD ?= sphinx-build
-SOURCEDIR = .
-BUILDDIR = _build
-
-# Put it first so that "make" without argument is like "make help".
-help:
- @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
-
-.PHONY: help Makefile
-
-# Catch-all target: route all unknown targets to Sphinx using the new
-# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS).
-%: Makefile
- @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
diff --git a/dev-docs/conf.py b/dev-docs/conf.py
deleted file mode 100644
index 826d578c0b..0000000000
--- a/dev-docs/conf.py
+++ /dev/null
@@ -1,50 +0,0 @@
-# Configuration file for the Sphinx documentation builder.
-#
-# For the full list of built-in configuration values, see the documentation:
-# https://www.sphinx-doc.org/en/master/usage/configuration.html
-
-# -- Project information -----------------------------------------------------
-# https://www.sphinx-doc.org/en/master/usage/configuration.html#project-information
-
-project = "Synapse development"
-copyright = "2023, The Matrix.org Foundation C.I.C."
-author = "The Synapse Maintainers and Community"
-
-# -- General configuration ---------------------------------------------------
-# https://www.sphinx-doc.org/en/master/usage/configuration.html#general-configuration
-
-extensions = [
- "autodoc2",
- "myst_parser",
-]
-
-templates_path = ["_templates"]
-exclude_patterns = ["_build", "Thumbs.db", ".DS_Store"]
-
-
-# -- Options for Autodoc2 ----------------------------------------------------
-
-autodoc2_docstring_parser_regexes = [
- # this will render all docstrings as 'MyST' Markdown
- (r".*", "myst"),
-]
-
-autodoc2_packages = [
- {
- "path": "../synapse",
- # Don't render documentation for everything as a matter of course
- "auto_mode": False,
- },
-]
-
-
-# -- Options for MyST (Markdown) ---------------------------------------------
-
-# myst_heading_anchors = 2
-
-
-# -- Options for HTML output -------------------------------------------------
-# https://www.sphinx-doc.org/en/master/usage/configuration.html#options-for-html-output
-
-html_theme = "furo"
-html_static_path = ["_static"]
diff --git a/dev-docs/index.rst b/dev-docs/index.rst
deleted file mode 100644
index 1ef210460a..0000000000
--- a/dev-docs/index.rst
+++ /dev/null
@@ -1,22 +0,0 @@
-.. Synapse Developer Documentation documentation master file, created by
- sphinx-quickstart on Mon Mar 13 08:59:51 2023.
- You can adapt this file completely to your liking, but it should at least
- contain the root `toctree` directive.
-
-Welcome to the Synapse Developer Documentation!
-===========================================================
-
-.. toctree::
- :maxdepth: 2
- :caption: Contents:
-
- modules/federation_sender
-
-
-
-Indices and tables
-==================
-
-* :ref:`genindex`
-* :ref:`modindex`
-* :ref:`search`
diff --git a/dev-docs/modules/federation_sender.md b/dev-docs/modules/federation_sender.md
deleted file mode 100644
index dac6852c16..0000000000
--- a/dev-docs/modules/federation_sender.md
+++ /dev/null
@@ -1,5 +0,0 @@
-Federation Sender
-=================
-
-```{autodoc2-docstring} synapse.federation.sender
-```
diff --git a/docker/Dockerfile b/docker/Dockerfile
index d4cb9414ff..1da196b12e 100644
--- a/docker/Dockerfile
+++ b/docker/Dockerfile
@@ -27,7 +27,7 @@ ARG PYTHON_VERSION=3.11
###
# We hardcode the use of Debian bookworm here because this could change upstream
# and other Dockerfiles used for testing are expecting bookworm.
-FROM docker.io/library/python:${PYTHON_VERSION}-slim-bookworm as requirements
+FROM docker.io/library/python:${PYTHON_VERSION}-slim-bookworm AS requirements
# RUN --mount is specific to buildkit and is documented at
# https://github.com/moby/buildkit/blob/master/frontend/dockerfile/docs/syntax.md#build-mounts-run---mount.
@@ -87,7 +87,7 @@ RUN if [ -z "$TEST_ONLY_IGNORE_POETRY_LOCKFILE" ]; then \
###
### Stage 1: builder
###
-FROM docker.io/library/python:${PYTHON_VERSION}-slim-bookworm as builder
+FROM docker.io/library/python:${PYTHON_VERSION}-slim-bookworm AS builder
# install the OS build deps
RUN \
@@ -163,7 +163,7 @@ FROM docker.io/library/python:${PYTHON_VERSION}-slim-bookworm
LABEL org.opencontainers.image.url='https://matrix.org/docs/projects/server/synapse'
LABEL org.opencontainers.image.documentation='https://github.com/element-hq/synapse/blob/master/docker/README.md'
LABEL org.opencontainers.image.source='https://github.com/element-hq/synapse.git'
-LABEL org.opencontainers.image.licenses='Apache-2.0'
+LABEL org.opencontainers.image.licenses='AGPL-3.0-or-later'
RUN \
--mount=type=cache,target=/var/cache/apt,sharing=locked \
diff --git a/docker/Dockerfile-dhvirtualenv b/docker/Dockerfile-dhvirtualenv
index b7679924c2..9266f134be 100644
--- a/docker/Dockerfile-dhvirtualenv
+++ b/docker/Dockerfile-dhvirtualenv
@@ -24,7 +24,7 @@ ARG distro=""
# https://launchpad.net/~jyrki-pulliainen/+archive/ubuntu/dh-virtualenv, but
# it's not obviously easier to use that than to build our own.)
-FROM docker.io/library/${distro} as builder
+FROM docker.io/library/${distro} AS builder
RUN apt-get update -qq -o Acquire::Languages=none
RUN env DEBIAN_FRONTEND=noninteractive apt-get install \
@@ -73,6 +73,8 @@ RUN apt-get update -qq -o Acquire::Languages=none \
curl \
debhelper \
devscripts \
+ # Required for building cffi from source.
+ libffi-dev \
libsystemd-dev \
lsb-release \
pkg-config \
diff --git a/docker/build_debian.sh b/docker/build_debian.sh
index 9eae38af91..00e0856c7d 100644
--- a/docker/build_debian.sh
+++ b/docker/build_debian.sh
@@ -11,6 +11,9 @@ DIST=$(cut -d ':' -f2 <<< "${distro:?}")
cp -aT /synapse/source /synapse/build
cd /synapse/build
+# Delete any existing `.so` files to ensure a clean build.
+rm -f /synapse/build/synapse/*.so
+
# if this is a prerelease, set the Section accordingly.
#
# When the package is later added to the package repo, reprepro will use the
diff --git a/docker/complement/conf/workers-shared-extra.yaml.j2 b/docker/complement/conf/workers-shared-extra.yaml.j2
index 32eada4419..6588b3ce14 100644
--- a/docker/complement/conf/workers-shared-extra.yaml.j2
+++ b/docker/complement/conf/workers-shared-extra.yaml.j2
@@ -92,8 +92,6 @@ allow_device_name_lookup_over_federation: true
## Experimental Features ##
experimental_features:
- # client-side support for partial state in /send_join responses
- faster_joins: true
# Enable support for polls
msc3381_polls_enabled: true
# Enable deleting device-specific notification settings stored in account data
@@ -104,6 +102,8 @@ experimental_features:
msc3874_enabled: true
# no UIA for x-signing upload for the first time
msc3967_enabled: true
+ # Expose a room summary for public rooms
+ msc3266_enabled: true
server_notices:
system_mxid_localpart: _server
diff --git a/docker/conf/homeserver.yaml b/docker/conf/homeserver.yaml
index c412ba2e87..2890990705 100644
--- a/docker/conf/homeserver.yaml
+++ b/docker/conf/homeserver.yaml
@@ -176,7 +176,6 @@ app_service_config_files:
{% endif %}
macaroon_secret_key: "{{ SYNAPSE_MACAROON_SECRET_KEY }}"
-expire_access_token: False
## Signing Keys ##
diff --git a/docker/configure_workers_and_start.py b/docker/configure_workers_and_start.py
index 77534a4f4f..15d8d7b558 100755
--- a/docker/configure_workers_and_start.py
+++ b/docker/configure_workers_and_start.py
@@ -117,7 +117,7 @@ WORKERS_CONFIG: Dict[str, Dict[str, Any]] = {
},
"media_repository": {
"app": "synapse.app.generic_worker",
- "listener_resources": ["media"],
+ "listener_resources": ["media", "client"],
"endpoint_patterns": [
"^/_matrix/media/",
"^/_synapse/admin/v1/purge_media_cache$",
@@ -125,6 +125,8 @@ WORKERS_CONFIG: Dict[str, Dict[str, Any]] = {
"^/_synapse/admin/v1/user/.*/media.*$",
"^/_synapse/admin/v1/media/.*$",
"^/_synapse/admin/v1/quarantine_media/.*$",
+ "^/_matrix/client/v1/media/.*$",
+ "^/_matrix/federation/v1/media/.*$",
],
# The first configured media worker will run the media background jobs
"shared_extra_conf": {
@@ -211,6 +213,8 @@ WORKERS_CONFIG: Dict[str, Dict[str, Any]] = {
"^/_matrix/federation/(v1|v2)/make_leave/",
"^/_matrix/federation/(v1|v2)/send_join/",
"^/_matrix/federation/(v1|v2)/send_leave/",
+ "^/_matrix/federation/v1/make_knock/",
+ "^/_matrix/federation/v1/send_knock/",
"^/_matrix/federation/(v1|v2)/invite/",
"^/_matrix/federation/(v1|v2)/query_auth/",
"^/_matrix/federation/(v1|v2)/event_auth/",
diff --git a/docs/admin_api/experimental_features.md b/docs/admin_api/experimental_features.md
index 07b630915d..ef1b58c9ba 100644
--- a/docs/admin_api/experimental_features.md
+++ b/docs/admin_api/experimental_features.md
@@ -1,21 +1,17 @@
# Experimental Features API
This API allows a server administrator to enable or disable some experimental features on a per-user
-basis. The currently supported features are:
-- [MSC3026](https://github.com/matrix-org/matrix-spec-proposals/pull/3026): busy
-presence state enabled
-- [MSC3881](https://github.com/matrix-org/matrix-spec-proposals/pull/3881): enable remotely toggling push notifications
-for another client
-- [MSC3967](https://github.com/matrix-org/matrix-spec-proposals/pull/3967): do not require
-UIA when first uploading cross-signing keys.
-
+basis. The currently supported features are:
+- [MSC3881](https://github.com/matrix-org/matrix-spec-proposals/pull/3881): enable remotely toggling push notifications
+for another client
+- [MSC3575](https://github.com/matrix-org/matrix-spec-proposals/pull/3575): enable experimental sliding sync support
To use it, you will need to authenticate by providing an `access_token`
for a server admin: see [Admin API](../usage/administration/admin_api/).
## Enabling/Disabling Features
-This API allows a server administrator to enable experimental features for a given user. The request must
+This API allows a server administrator to enable experimental features for a given user. The request must
provide a body containing the user id and listing the features to enable/disable in the following format:
```json
{
@@ -35,7 +31,7 @@ PUT /_synapse/admin/v1/experimental_features/-
+
- hash_password(1) + +
- hash_password(1) +
NAME
+
+ hash_password - Calculate the hash of a new password, so that passwords can be reset
+
SYNOPSIS
+ +
+hash_password [-p
+ |
+
+--password [password]] [-c
+ |
+
+--config file] |
+
DESCRIPTION
+ +hash_password calculates the hash of a supplied password using bcrypt.
+ +hash_password takes a password as an parameter either on the command line
+or the STDIN if not supplied.
It accepts an YAML file which can be used to specify parameters like the
+number of rounds for bcrypt and password_config section having the pepper
+value used for the hashing. By default bcrypt_rounds is set to 12.
The hashed password is written on the STDOUT.
FILES
+ +A sample YAML file accepted by hash_password is described below:
bcrypt_rounds: 17 + password_config: + pepper: "random hashing pepper"
+ +OPTIONS
+ +-
+
-
+
-p,--password+
+ - Read the password form the command line if [password] is supplied, or from
STDIN. +If not, prompt the user and read the password from the tty prompt. +It is not recommended to type the password on the command line +directly. Use the STDIN instead.
+ -
+
-c,--config+
+ - Read the supplied YAML file containing the options
bcrypt_rounds+and thepassword_configsection containing thepeppervalue.
+
EXAMPLES
+ +Hash from the command line:
+ +$ hash_password -p "p@ssw0rd"
+$2b$12$VJNqWQYfsWTEwcELfoSi4Oa8eA17movHqqi8.X8fWFpum7SxZ9MFe
+Hash from the stdin:
+ +$ cat password_file | hash_password
+Password:
+Confirm password:
+$2b$12$AszlvfmJl2esnyhmn8m/kuR2tdXgROWtWxnX.rcuAbM8ErLoUhybG
+Hash from the prompt:
+ +$ hash_password
+Password:
+Confirm password:
+$2b$12$AszlvfmJl2esnyhmn8m/kuR2tdXgROWtWxnX.rcuAbM8ErLoUhybG
+Using a config file:
+ +$ hash_password -c config.yml
+Password:
+Confirm password:
+$2b$12$CwI.wBNr.w3kmiUlV3T5s.GT2wH7uebDCovDrCOh18dFedlANK99O
+COPYRIGHT
+ +This man page was written by Rahul De «rahulde@swecha.net» +for Debian GNU/Linux distribution.
+ +SEE ALSO
+ +synctl(1), synapse_port_db(1), register_new_matrix_user(1), synapse_review_recent_signups(1)
+ +-
+
+
- August 2024 +
- hash_password(1) +
You have asked us to register this email with a new Matrix account, but this email is already registered with an existing account.
+ +Please reset your password if needed.
+ +If this was not you, you can safely disregard this email.
+ +Thank you.
+{% endblock %} diff --git a/synapse/res/templates/already_in_use.txt b/synapse/res/templates/already_in_use.txt new file mode 100644 index 0000000000..c60401a940 --- /dev/null +++ b/synapse/res/templates/already_in_use.txt @@ -0,0 +1,10 @@ +Hello there, + +You have asked us to register this email with a new Matrix account, +but this email is already registered with an existing account. + +Please reset your password if needed. + +If this was not you, you can safely disregard this email. + +Thank you. diff --git a/synapse/rest/__init__.py b/synapse/rest/__init__.py index 534dc0e276..c5cdc36955 100644 --- a/synapse/rest/__init__.py +++ b/synapse/rest/__init__.py @@ -18,7 +18,8 @@ # [This file includes modifications made by New Vector Limited] # # -from typing import TYPE_CHECKING, Callable +import logging +from typing import TYPE_CHECKING, Callable, Dict, Iterable, List, Optional, Tuple from synapse.http.server import HttpServer, JsonResource from synapse.rest import admin @@ -53,7 +54,7 @@ from synapse.rest.client import ( register, relations, rendezvous, - report_event, + reporting, room, room_keys, room_upgrade_rest_servlet, @@ -67,11 +68,64 @@ from synapse.rest.client import ( voip, ) +logger = logging.getLogger(__name__) + if TYPE_CHECKING: from synapse.server import HomeServer RegisterServletsFunc = Callable[["HomeServer", HttpServer], None] +CLIENT_SERVLET_FUNCTIONS: Tuple[RegisterServletsFunc, ...] = ( + versions.register_servlets, + initial_sync.register_servlets, + room.register_deprecated_servlets, + events.register_servlets, + room.register_servlets, + login.register_servlets, + profile.register_servlets, + presence.register_servlets, + directory.register_servlets, + voip.register_servlets, + pusher.register_servlets, + push_rule.register_servlets, + logout.register_servlets, + sync.register_servlets, + filter.register_servlets, + account.register_servlets, + register.register_servlets, + auth.register_servlets, + receipts.register_servlets, + read_marker.register_servlets, + room_keys.register_servlets, + keys.register_servlets, + tokenrefresh.register_servlets, + tags.register_servlets, + account_data.register_servlets, + reporting.register_servlets, + openid.register_servlets, + notifications.register_servlets, + devices.register_servlets, + thirdparty.register_servlets, + sendtodevice.register_servlets, + user_directory.register_servlets, + room_upgrade_rest_servlet.register_servlets, + capabilities.register_servlets, + account_validity.register_servlets, + relations.register_servlets, + password_policy.register_servlets, + knock.register_servlets, + appservice_ping.register_servlets, + admin.register_servlets_for_client_rest_resource, + mutual_rooms.register_servlets, + login_token_request.register_servlets, + rendezvous.register_servlets, + auth_issuer.register_servlets, +) + +SERVLET_GROUPS: Dict[str, Iterable[RegisterServletsFunc]] = { + "client": CLIENT_SERVLET_FUNCTIONS, +} + class ClientRestResource(JsonResource): """Matrix Client API REST resource. @@ -83,76 +137,56 @@ class ClientRestResource(JsonResource): * etc """ - def __init__(self, hs: "HomeServer"): + def __init__(self, hs: "HomeServer", servlet_groups: Optional[List[str]] = None): JsonResource.__init__(self, hs, canonical_json=False) - self.register_servlets(self, hs) + if hs.config.media.can_load_media_repo: + # This import is here to prevent a circular import failure + from synapse.rest.client import media + + SERVLET_GROUPS["media"] = (media.register_servlets,) + self.register_servlets(self, hs, servlet_groups) @staticmethod - def register_servlets(client_resource: HttpServer, hs: "HomeServer") -> None: + def register_servlets( + client_resource: HttpServer, + hs: "HomeServer", + servlet_groups: Optional[Iterable[str]] = None, + ) -> None: # Some servlets are only registered on the main process (and not worker # processes). is_main_process = hs.config.worker.worker_app is None - versions.register_servlets(hs, client_resource) + if not servlet_groups: + servlet_groups = SERVLET_GROUPS.keys() - # Deprecated in r0 - initial_sync.register_servlets(hs, client_resource) - room.register_deprecated_servlets(hs, client_resource) + for servlet_group in servlet_groups: + # Fail on unknown servlet groups. + if servlet_group not in SERVLET_GROUPS: + if servlet_group == "media": + logger.warn( + "media.can_load_media_repo needs to be configured for the media servlet to be available" + ) + raise RuntimeError( + f"Attempting to register unknown client servlet: '{servlet_group}'" + ) - # Partially deprecated in r0 - events.register_servlets(hs, client_resource) + for servletfunc in SERVLET_GROUPS[servlet_group]: + if not is_main_process and servletfunc in [ + pusher.register_servlets, + logout.register_servlets, + auth.register_servlets, + tokenrefresh.register_servlets, + reporting.register_servlets, + openid.register_servlets, + thirdparty.register_servlets, + room_upgrade_rest_servlet.register_servlets, + account_validity.register_servlets, + admin.register_servlets_for_client_rest_resource, + mutual_rooms.register_servlets, + login_token_request.register_servlets, + rendezvous.register_servlets, + auth_issuer.register_servlets, + ]: + continue - room.register_servlets(hs, client_resource) - login.register_servlets(hs, client_resource) - profile.register_servlets(hs, client_resource) - presence.register_servlets(hs, client_resource) - directory.register_servlets(hs, client_resource) - voip.register_servlets(hs, client_resource) - if is_main_process: - pusher.register_servlets(hs, client_resource) - push_rule.register_servlets(hs, client_resource) - if is_main_process: - logout.register_servlets(hs, client_resource) - sync.register_servlets(hs, client_resource) - filter.register_servlets(hs, client_resource) - account.register_servlets(hs, client_resource) - register.register_servlets(hs, client_resource) - if is_main_process: - auth.register_servlets(hs, client_resource) - receipts.register_servlets(hs, client_resource) - read_marker.register_servlets(hs, client_resource) - room_keys.register_servlets(hs, client_resource) - keys.register_servlets(hs, client_resource) - if is_main_process: - tokenrefresh.register_servlets(hs, client_resource) - tags.register_servlets(hs, client_resource) - account_data.register_servlets(hs, client_resource) - if is_main_process: - report_event.register_servlets(hs, client_resource) - openid.register_servlets(hs, client_resource) - notifications.register_servlets(hs, client_resource) - devices.register_servlets(hs, client_resource) - if is_main_process: - thirdparty.register_servlets(hs, client_resource) - sendtodevice.register_servlets(hs, client_resource) - user_directory.register_servlets(hs, client_resource) - if is_main_process: - room_upgrade_rest_servlet.register_servlets(hs, client_resource) - capabilities.register_servlets(hs, client_resource) - if is_main_process: - account_validity.register_servlets(hs, client_resource) - relations.register_servlets(hs, client_resource) - password_policy.register_servlets(hs, client_resource) - knock.register_servlets(hs, client_resource) - appservice_ping.register_servlets(hs, client_resource) - - # moving to /_synapse/admin - if is_main_process: - admin.register_servlets_for_client_rest_resource(hs, client_resource) - - # unstable - if is_main_process: - mutual_rooms.register_servlets(hs, client_resource) - login_token_request.register_servlets(hs, client_resource) - rendezvous.register_servlets(hs, client_resource) - auth_issuer.register_servlets(hs, client_resource) + servletfunc(hs, client_resource) diff --git a/synapse/rest/admin/__init__.py b/synapse/rest/admin/__init__.py index 6da1d79168..cdaee17451 100644 --- a/synapse/rest/admin/__init__.py +++ b/synapse/rest/admin/__init__.py @@ -101,6 +101,7 @@ from synapse.rest.admin.users import ( ResetPasswordRestServlet, SearchUsersRestServlet, ShadowBanRestServlet, + SuspendAccountRestServlet, UserAdminServlet, UserByExternalId, UserByThreePid, @@ -327,6 +328,8 @@ def register_servlets(hs: "HomeServer", http_server: HttpServer) -> None: BackgroundUpdateRestServlet(hs).register(http_server) BackgroundUpdateStartJobRestServlet(hs).register(http_server) ExperimentalFeaturesRestServlet(hs).register(http_server) + if hs.config.experimental.msc3823_account_suspension: + SuspendAccountRestServlet(hs).register(http_server) def register_servlets_for_client_rest_resource( diff --git a/synapse/rest/admin/experimental_features.py b/synapse/rest/admin/experimental_features.py index 52eb9e62db..d7913896d9 100644 --- a/synapse/rest/admin/experimental_features.py +++ b/synapse/rest/admin/experimental_features.py @@ -31,7 +31,9 @@ from synapse.rest.admin import admin_patterns, assert_requester_is_admin from synapse.types import JsonDict, UserID if TYPE_CHECKING: - from synapse.server import HomeServer + from typing_extensions import assert_never + + from synapse.server import HomeServer, HomeServerConfig class ExperimentalFeature(str, Enum): @@ -39,9 +41,16 @@ class ExperimentalFeature(str, Enum): Currently supported per-user features """ - MSC3026 = "msc3026" MSC3881 = "msc3881" - MSC3967 = "msc3967" + MSC3575 = "msc3575" + + def is_globally_enabled(self, config: "HomeServerConfig") -> bool: + if self is ExperimentalFeature.MSC3881: + return config.experimental.msc3881_enabled + if self is ExperimentalFeature.MSC3575: + return config.experimental.msc3575_enabled + + assert_never(self) class ExperimentalFeaturesRestServlet(RestServlet): diff --git a/synapse/rest/admin/federation.py b/synapse/rest/admin/federation.py index 045153e0cb..d85a04b825 100644 --- a/synapse/rest/admin/federation.py +++ b/synapse/rest/admin/federation.py @@ -23,7 +23,7 @@ from http import HTTPStatus from typing import TYPE_CHECKING, Tuple from synapse.api.constants import Direction -from synapse.api.errors import Codes, NotFoundError, SynapseError +from synapse.api.errors import NotFoundError, SynapseError from synapse.federation.transport.server import Authenticator from synapse.http.servlet import RestServlet, parse_enum, parse_integer, parse_string from synapse.http.site import SynapseRequest @@ -64,20 +64,6 @@ class ListDestinationsRestServlet(RestServlet): start = parse_integer(request, "from", default=0) limit = parse_integer(request, "limit", default=100) - if start < 0: - raise SynapseError( - HTTPStatus.BAD_REQUEST, - "Query parameter from must be a string representing a positive integer.", - errcode=Codes.INVALID_PARAM, - ) - - if limit < 0: - raise SynapseError( - HTTPStatus.BAD_REQUEST, - "Query parameter limit must be a string representing a positive integer.", - errcode=Codes.INVALID_PARAM, - ) - destination = parse_string(request, "destination") order_by = parse_string( @@ -198,20 +184,6 @@ class DestinationMembershipRestServlet(RestServlet): start = parse_integer(request, "from", default=0) limit = parse_integer(request, "limit", default=100) - if start < 0: - raise SynapseError( - HTTPStatus.BAD_REQUEST, - "Query parameter from must be a string representing a positive integer.", - errcode=Codes.INVALID_PARAM, - ) - - if limit < 0: - raise SynapseError( - HTTPStatus.BAD_REQUEST, - "Query parameter limit must be a string representing a positive integer.", - errcode=Codes.INVALID_PARAM, - ) - direction = parse_enum(request, "dir", Direction, default=Direction.FORWARDS) rooms, total = await self._store.get_destination_rooms_paginate( diff --git a/synapse/rest/admin/media.py b/synapse/rest/admin/media.py index 27f0808658..ee6a681285 100644 --- a/synapse/rest/admin/media.py +++ b/synapse/rest/admin/media.py @@ -315,25 +315,13 @@ class DeleteMediaByDateSize(RestServlet): size_gt = parse_integer(request, "size_gt", default=0) keep_profiles = parse_boolean(request, "keep_profiles", default=True) - if before_ts < 0: - raise SynapseError( - HTTPStatus.BAD_REQUEST, - "Query parameter before_ts must be a positive integer.", - errcode=Codes.INVALID_PARAM, - ) - elif before_ts < 30000000000: # Dec 1970 in milliseconds, Aug 2920 in seconds + if before_ts < 30000000000: # Dec 1970 in milliseconds, Aug 2920 in seconds raise SynapseError( HTTPStatus.BAD_REQUEST, "Query parameter before_ts you provided is from the year 1970. " + "Double check that you are providing a timestamp in milliseconds.", errcode=Codes.INVALID_PARAM, ) - if size_gt < 0: - raise SynapseError( - HTTPStatus.BAD_REQUEST, - "Query parameter size_gt must be a string representing a positive integer.", - errcode=Codes.INVALID_PARAM, - ) # This check is useless, we keep it for the legacy endpoint only. if server_name is not None and self.server_name != server_name: @@ -392,20 +380,6 @@ class UserMediaRestServlet(RestServlet): start = parse_integer(request, "from", default=0) limit = parse_integer(request, "limit", default=100) - if start < 0: - raise SynapseError( - HTTPStatus.BAD_REQUEST, - "Query parameter from must be a string representing a positive integer.", - errcode=Codes.INVALID_PARAM, - ) - - if limit < 0: - raise SynapseError( - HTTPStatus.BAD_REQUEST, - "Query parameter limit must be a string representing a positive integer.", - errcode=Codes.INVALID_PARAM, - ) - # If neither `order_by` nor `dir` is set, set the default order # to newest media is on top for backward compatibility. if b"order_by" not in request.args and b"dir" not in request.args: @@ -450,20 +424,6 @@ class UserMediaRestServlet(RestServlet): start = parse_integer(request, "from", default=0) limit = parse_integer(request, "limit", default=100) - if start < 0: - raise SynapseError( - HTTPStatus.BAD_REQUEST, - "Query parameter from must be a string representing a positive integer.", - errcode=Codes.INVALID_PARAM, - ) - - if limit < 0: - raise SynapseError( - HTTPStatus.BAD_REQUEST, - "Query parameter limit must be a string representing a positive integer.", - errcode=Codes.INVALID_PARAM, - ) - # If neither `order_by` nor `dir` is set, set the default order # to newest media is on top for backward compatibility. if b"order_by" not in request.args and b"dir" not in request.args: diff --git a/synapse/rest/admin/registration_tokens.py b/synapse/rest/admin/registration_tokens.py index 0867f7a51c..bec2331590 100644 --- a/synapse/rest/admin/registration_tokens.py +++ b/synapse/rest/admin/registration_tokens.py @@ -181,8 +181,7 @@ class NewRegistrationTokenRestServlet(RestServlet): uses_allowed = body.get("uses_allowed", None) if not ( - uses_allowed is None - or (type(uses_allowed) is int and uses_allowed >= 0) # noqa: E721 + uses_allowed is None or (type(uses_allowed) is int and uses_allowed >= 0) # noqa: E721 ): raise SynapseError( HTTPStatus.BAD_REQUEST, diff --git a/synapse/rest/admin/rooms.py b/synapse/rest/admin/rooms.py index 4252f98a6c..01f9de9ffa 100644 --- a/synapse/rest/admin/rooms.py +++ b/synapse/rest/admin/rooms.py @@ -21,7 +21,6 @@ import logging from http import HTTPStatus from typing import TYPE_CHECKING, List, Optional, Tuple, cast -from urllib import parse as urlparse import attr @@ -36,8 +35,10 @@ from synapse.http.servlet import ( ResolveRoomIdMixin, RestServlet, assert_params_in_dict, + parse_boolean, parse_enum, parse_integer, + parse_json, parse_json_object_from_request, parse_string, ) @@ -51,7 +52,6 @@ from synapse.storage.databases.main.room import RoomSortOrder from synapse.streams.config import PaginationConfig from synapse.types import JsonDict, RoomID, ScheduledTask, UserID, create_requester from synapse.types.state import StateFilter -from synapse.util import json_decoder if TYPE_CHECKING: from synapse.api.auth import Auth @@ -243,13 +243,23 @@ class ListRoomRestServlet(RestServlet): errcode=Codes.INVALID_PARAM, ) + public_rooms = parse_boolean(request, "public_rooms") + empty_rooms = parse_boolean(request, "empty_rooms") + direction = parse_enum(request, "dir", Direction, default=Direction.FORWARDS) reverse_order = True if direction == Direction.BACKWARDS else False # Return list of rooms according to parameters rooms, total_rooms = await self.store.get_rooms_paginate( - start, limit, order_by, reverse_order, search_term + start, + limit, + order_by, + reverse_order, + search_term, + public_rooms, + empty_rooms, ) + response = { # next_token should be opaque, so return a value the client can parse "offset": start, @@ -776,14 +786,8 @@ class RoomEventContextServlet(RestServlet): limit = parse_integer(request, "limit", default=10) # picking the API shape for symmetry with /messages - filter_str = parse_string(request, "filter", encoding="utf-8") - if filter_str: - filter_json = urlparse.unquote(filter_str) - event_filter: Optional[Filter] = Filter( - self._hs, json_decoder.decode(filter_json) - ) - else: - event_filter = None + filter_json = parse_json(request, "filter", encoding="utf-8") + event_filter = Filter(self._hs, filter_json) if filter_json else None event_context = await self.room_context_handler.get_event_context( requester, @@ -914,21 +918,16 @@ class RoomMessagesRestServlet(RestServlet): ) # Twisted will have processed the args by now. assert request.args is not None + + filter_json = parse_json(request, "filter", encoding="utf-8") + event_filter = Filter(self._hs, filter_json) if filter_json else None + as_client_event = b"raw" not in request.args - filter_str = parse_string(request, "filter", encoding="utf-8") - if filter_str: - filter_json = urlparse.unquote(filter_str) - event_filter: Optional[Filter] = Filter( - self._hs, json_decoder.decode(filter_json) - ) - if ( - event_filter - and event_filter.filter_json.get("event_format", "client") - == "federation" - ): - as_client_event = False - else: - event_filter = None + if ( + event_filter + and event_filter.filter_json.get("event_format", "client") == "federation" + ): + as_client_event = False msgs = await self._pagination_handler.get_messages( room_id=room_id, diff --git a/synapse/rest/admin/statistics.py b/synapse/rest/admin/statistics.py index 832f20402e..0adc5b7005 100644 --- a/synapse/rest/admin/statistics.py +++ b/synapse/rest/admin/statistics.py @@ -64,37 +64,11 @@ class UserMediaStatisticsRestServlet(RestServlet): ) start = parse_integer(request, "from", default=0) - if start < 0: - raise SynapseError( - HTTPStatus.BAD_REQUEST, - "Query parameter from must be a string representing a positive integer.", - errcode=Codes.INVALID_PARAM, - ) - limit = parse_integer(request, "limit", default=100) - if limit < 0: - raise SynapseError( - HTTPStatus.BAD_REQUEST, - "Query parameter limit must be a string representing a positive integer.", - errcode=Codes.INVALID_PARAM, - ) - from_ts = parse_integer(request, "from_ts", default=0) - if from_ts < 0: - raise SynapseError( - HTTPStatus.BAD_REQUEST, - "Query parameter from_ts must be a string representing a positive integer.", - errcode=Codes.INVALID_PARAM, - ) - until_ts = parse_integer(request, "until_ts") + if until_ts is not None: - if until_ts < 0: - raise SynapseError( - HTTPStatus.BAD_REQUEST, - "Query parameter until_ts must be a string representing a positive integer.", - errcode=Codes.INVALID_PARAM, - ) if until_ts <= from_ts: raise SynapseError( HTTPStatus.BAD_REQUEST, diff --git a/synapse/rest/admin/users.py b/synapse/rest/admin/users.py index 4e34e46512..076994b87e 100644 --- a/synapse/rest/admin/users.py +++ b/synapse/rest/admin/users.py @@ -27,11 +27,13 @@ from typing import TYPE_CHECKING, Dict, List, Optional, Tuple, Union import attr +from synapse._pydantic_compat import StrictBool from synapse.api.constants import Direction, UserTypes from synapse.api.errors import Codes, NotFoundError, SynapseError from synapse.http.servlet import ( RestServlet, assert_params_in_dict, + parse_and_validate_json_object_from_request, parse_boolean, parse_enum, parse_integer, @@ -49,10 +51,12 @@ from synapse.rest.client._base import client_patterns from synapse.storage.databases.main.registration import ExternalIDReuseException from synapse.storage.databases.main.stats import UserSortOrder from synapse.types import JsonDict, JsonMapping, UserID +from synapse.types.rest import RequestBodyModel if TYPE_CHECKING: from synapse.server import HomeServer + logger = logging.getLogger(__name__) @@ -93,20 +97,6 @@ class UsersRestServletV2(RestServlet): start = parse_integer(request, "from", default=0) limit = parse_integer(request, "limit", default=100) - if start < 0: - raise SynapseError( - HTTPStatus.BAD_REQUEST, - "Query parameter from must be a string representing a positive integer.", - errcode=Codes.INVALID_PARAM, - ) - - if limit < 0: - raise SynapseError( - HTTPStatus.BAD_REQUEST, - "Query parameter limit must be a string representing a positive integer.", - errcode=Codes.INVALID_PARAM, - ) - user_id = parse_string(request, "user_id") name = parse_string(request, "name", encoding="utf-8") @@ -746,6 +736,36 @@ class DeactivateAccountRestServlet(RestServlet): return HTTPStatus.OK, {"id_server_unbind_result": id_server_unbind_result} +class SuspendAccountRestServlet(RestServlet): + PATTERNS = admin_patterns("/suspend/(?P
"""
+ )
+
+ channel = self.make_request(
+ "GET",
+ "/_matrix/client/v1/media/preview_url?url=http://matrix.org",
+ shorthand=False,
+ await_result=False,
+ )
+ self.pump()
+
+ # Respond with the HTML.
+ client = self.reactor.tcpClients[0][2].buildProtocol(None)
+ server = AccumulatingProtocol()
+ server.makeConnection(FakeTransport(client, self.reactor))
+ client.makeConnection(FakeTransport(server, self.reactor))
+ client.dataReceived(
+ (
+ b"HTTP/1.0 200 OK\r\nContent-Length: %d\r\n"
+ b'Content-Type: text/html; charset="utf8"\r\n\r\n'
+ )
+ % (len(result),)
+ + result
+ )
+ self.pump()
+
+ # Respond with the photo.
+ client = self.reactor.tcpClients[1][2].buildProtocol(None)
+ server = AccumulatingProtocol()
+ server.makeConnection(FakeTransport(client, self.reactor))
+ client.makeConnection(FakeTransport(server, self.reactor))
+ client.dataReceived(
+ (
+ b"HTTP/1.0 200 OK\r\nContent-Length: %d\r\n"
+ b"Content-Type: image/png\r\n\r\n"
+ )
+ % (len(SMALL_PNG),)
+ + SMALL_PNG
+ )
+ self.pump()
+
+ # The image should be in the result.
+ self.assertEqual(channel.code, 200)
+ self._assert_small_png(channel.json_body)
+
+ def test_nonexistent_image(self) -> None:
+ """If the preview image doesn't exist, ensure some data is returned."""
+ self.lookups["matrix.org"] = [(IPv4Address, "10.1.2.3")]
+
+ result = (
+ b"""
"""
+ )
+
+ channel = self.make_request(
+ "GET",
+ "/_matrix/client/v1/media/preview_url?url=http://matrix.org",
+ shorthand=False,
+ await_result=False,
+ )
+ self.pump()
+
+ client = self.reactor.tcpClients[0][2].buildProtocol(None)
+ server = AccumulatingProtocol()
+ server.makeConnection(FakeTransport(client, self.reactor))
+ client.makeConnection(FakeTransport(server, self.reactor))
+ client.dataReceived(
+ (
+ b"HTTP/1.0 200 OK\r\nContent-Length: %d\r\n"
+ b'Content-Type: text/html; charset="utf8"\r\n\r\n'
+ )
+ % (len(result),)
+ + result
+ )
+
+ self.pump()
+
+ # There should not be a second connection.
+ self.assertEqual(len(self.reactor.tcpClients), 1)
+
+ # The image should not be in the result.
+ self.assertEqual(channel.code, 200)
+ self.assertNotIn("og:image", channel.json_body)
+
+ @unittest.override_config(
+ {"url_preview_url_blacklist": [{"netloc": "cdn.matrix.org"}]}
+ )
+ def test_image_blocked(self) -> None:
+ """If the preview image doesn't exist, ensure some data is returned."""
+ self.lookups["matrix.org"] = [(IPv4Address, "10.1.2.3")]
+ self.lookups["cdn.matrix.org"] = [(IPv4Address, "10.1.2.4")]
+
+ result = (
+ b""""""
+ )
+
+ channel = self.make_request(
+ "GET",
+ "/_matrix/client/v1/media/preview_url?url=http://matrix.org",
+ shorthand=False,
+ await_result=False,
+ )
+ self.pump()
+
+ client = self.reactor.tcpClients[0][2].buildProtocol(None)
+ server = AccumulatingProtocol()
+ server.makeConnection(FakeTransport(client, self.reactor))
+ client.makeConnection(FakeTransport(server, self.reactor))
+ client.dataReceived(
+ (
+ b"HTTP/1.0 200 OK\r\nContent-Length: %d\r\n"
+ b'Content-Type: text/html; charset="utf8"\r\n\r\n'
+ )
+ % (len(result),)
+ + result
+ )
+ self.pump()
+
+ # There should not be a second connection.
+ self.assertEqual(len(self.reactor.tcpClients), 1)
+
+ # The image should not be in the result.
+ self.assertEqual(channel.code, 200)
+ self.assertNotIn("og:image", channel.json_body)
+
+ def test_oembed_failure(self) -> None:
+ """If the autodiscovered oEmbed URL fails, ensure some data is returned."""
+ self.lookups["matrix.org"] = [(IPv4Address, "10.1.2.3")]
+
+ result = b"""
+ 
'
+ }
+ )
+
+ channel = self.make_request(
+ "GET",
+ f"/_matrix/client/v1/media/preview_url?{query_params}",
+ shorthand=False,
+ )
+ self.pump()
+
+ self.assertEqual(channel.code, 500)
+
+ def test_inline_data_url(self) -> None:
+ """
+ An inline image (as a data URL) should be parsed properly.
+ """
+ self.lookups["matrix.org"] = [(IPv4Address, "10.1.2.3")]
+
+ data = base64.b64encode(SMALL_PNG)
+
+ end_content = (
+ b"" b'
' b""
+ ) % (data,)
+
+ channel = self.make_request(
+ "GET",
+ "/_matrix/client/v1/media/preview_url?url=http://matrix.org",
+ shorthand=False,
+ await_result=False,
+ )
+ self.pump()
+
+ client = self.reactor.tcpClients[0][2].buildProtocol(None)
+ server = AccumulatingProtocol()
+ server.makeConnection(FakeTransport(client, self.reactor))
+ client.makeConnection(FakeTransport(server, self.reactor))
+ client.dataReceived(
+ (
+ b"HTTP/1.0 200 OK\r\nContent-Length: %d\r\n"
+ b'Content-Type: text/html; charset="utf8"\r\n\r\n'
+ )
+ % (len(end_content),)
+ + end_content
+ )
+
+ self.pump()
+ self.assertEqual(channel.code, 200)
+ self._assert_small_png(channel.json_body)
+
+ def test_oembed_photo(self) -> None:
+ """Test an oEmbed endpoint which returns a 'photo' type which redirects the preview to a new URL."""
+ self.lookups["publish.twitter.com"] = [(IPv4Address, "10.1.2.3")]
+ self.lookups["cdn.twitter.com"] = [(IPv4Address, "10.1.2.3")]
+
+ result = {
+ "version": "1.0",
+ "type": "photo",
+ "url": "http://cdn.twitter.com/matrixdotorg",
+ }
+ oembed_content = json.dumps(result).encode("utf-8")
+
+ channel = self.make_request(
+ "GET",
+ "/_matrix/client/v1/media/preview_url?url=http://twitter.com/matrixdotorg/status/12345",
+ shorthand=False,
+ await_result=False,
+ )
+ self.pump()
+
+ client = self.reactor.tcpClients[0][2].buildProtocol(None)
+ server = AccumulatingProtocol()
+ server.makeConnection(FakeTransport(client, self.reactor))
+ client.makeConnection(FakeTransport(server, self.reactor))
+ client.dataReceived(
+ (
+ b"HTTP/1.0 200 OK\r\nContent-Length: %d\r\n"
+ b'Content-Type: application/json; charset="utf8"\r\n\r\n'
+ )
+ % (len(oembed_content),)
+ + oembed_content
+ )
+
+ self.pump()
+
+ # Ensure a second request is made to the photo URL.
+ client = self.reactor.tcpClients[1][2].buildProtocol(None)
+ server = AccumulatingProtocol()
+ server.makeConnection(FakeTransport(client, self.reactor))
+ client.makeConnection(FakeTransport(server, self.reactor))
+ client.dataReceived(
+ (
+ b"HTTP/1.0 200 OK\r\nContent-Length: %d\r\n"
+ b"Content-Type: image/png\r\n\r\n"
+ )
+ % (len(SMALL_PNG),)
+ + SMALL_PNG
+ )
+
+ self.pump()
+
+ # Ensure the URL is what was requested.
+ self.assertIn(b"/matrixdotorg", server.data)
+
+ self.assertEqual(channel.code, 200)
+ body = channel.json_body
+ self.assertEqual(body["og:url"], "http://twitter.com/matrixdotorg/status/12345")
+ self._assert_small_png(body)
+
+ def test_oembed_rich(self) -> None:
+ """Test an oEmbed endpoint which returns HTML content via the 'rich' type."""
+ self.lookups["publish.twitter.com"] = [(IPv4Address, "10.1.2.3")]
+
+ result = {
+ "version": "1.0",
+ "type": "rich",
+ # Note that this provides the author, not the title.
+ "author_name": "Alice",
+ "html": "Content Preview
",
+ }
+ end_content = json.dumps(result).encode("utf-8")
+
+ channel = self.make_request(
+ "GET",
+ "/_matrix/client/v1/media/preview_url?url=http://twitter.com/matrixdotorg/status/12345",
+ shorthand=False,
+ await_result=False,
+ )
+ self.pump()
+
+ client = self.reactor.tcpClients[0][2].buildProtocol(None)
+ server = AccumulatingProtocol()
+ server.makeConnection(FakeTransport(client, self.reactor))
+ client.makeConnection(FakeTransport(server, self.reactor))
+ client.dataReceived(
+ (
+ b"HTTP/1.0 200 OK\r\nContent-Length: %d\r\n"
+ b'Content-Type: application/json; charset="utf8"\r\n\r\n'
+ )
+ % (len(end_content),)
+ + end_content
+ )
+
+ self.pump()
+
+ # Double check that the proper host is being connected to. (Note that
+ # twitter.com can't be resolved so this is already implicitly checked.)
+ self.assertIn(b"\r\nHost: publish.twitter.com\r\n", server.data)
+
+ self.assertEqual(channel.code, 200)
+ body = channel.json_body
+ self.assertEqual(
+ body,
+ {
+ "og:url": "http://twitter.com/matrixdotorg/status/12345",
+ "og:title": "Alice",
+ "og:description": "Content Preview",
+ },
+ )
+
+ def test_oembed_format(self) -> None:
+ """Test an oEmbed endpoint which requires the format in the URL."""
+ self.lookups["www.hulu.com"] = [(IPv4Address, "10.1.2.3")]
+
+ result = {
+ "version": "1.0",
+ "type": "rich",
+ "html": "Content Preview
",
+ }
+ end_content = json.dumps(result).encode("utf-8")
+
+ channel = self.make_request(
+ "GET",
+ "/_matrix/client/v1/media/preview_url?url=http://www.hulu.com/watch/12345",
+ shorthand=False,
+ await_result=False,
+ )
+ self.pump()
+
+ client = self.reactor.tcpClients[0][2].buildProtocol(None)
+ server = AccumulatingProtocol()
+ server.makeConnection(FakeTransport(client, self.reactor))
+ client.makeConnection(FakeTransport(server, self.reactor))
+ client.dataReceived(
+ (
+ b"HTTP/1.0 200 OK\r\nContent-Length: %d\r\n"
+ b'Content-Type: application/json; charset="utf8"\r\n\r\n'
+ )
+ % (len(end_content),)
+ + end_content
+ )
+
+ self.pump()
+
+ # The {format} should have been turned into json.
+ self.assertIn(b"/api/oembed.json", server.data)
+ # A URL parameter of format=json should be provided.
+ self.assertIn(b"format=json", server.data)
+
+ self.assertEqual(channel.code, 200)
+ body = channel.json_body
+ self.assertEqual(
+ body,
+ {
+ "og:url": "http://www.hulu.com/watch/12345",
+ "og:description": "Content Preview",
+ },
+ )
+
+ @unittest.override_config(
+ {"url_preview_url_blacklist": [{"netloc": "publish.twitter.com"}]}
+ )
+ def test_oembed_blocked(self) -> None:
+ """The oEmbed URL should not be downloaded if the oEmbed URL is blocked."""
+ self.lookups["twitter.com"] = [(IPv4Address, "10.1.2.3")]
+
+ channel = self.make_request(
+ "GET",
+ "/_matrix/client/v1/media/preview_url?url=http://twitter.com/matrixdotorg/status/12345",
+ shorthand=False,
+ await_result=False,
+ )
+ self.pump()
+ self.assertEqual(channel.code, 403, channel.result)
+
+ def test_oembed_autodiscovery(self) -> None:
+ """
+ Autodiscovery works by finding the link in the HTML response and then requesting an oEmbed URL.
+ 1. Request a preview of a URL which is not known to the oEmbed code.
+ 2. It returns HTML including a link to an oEmbed preview.
+ 3. The oEmbed preview is requested and returns a URL for an image.
+ 4. The image is requested for thumbnailing.
+ """
+ # This is a little cheesy in that we use the www subdomain (which isn't the
+ # list of oEmbed patterns) to get "raw" HTML response.
+ self.lookups["www.twitter.com"] = [(IPv4Address, "10.1.2.3")]
+ self.lookups["publish.twitter.com"] = [(IPv4Address, "10.1.2.3")]
+ self.lookups["cdn.twitter.com"] = [(IPv4Address, "10.1.2.3")]
+
+ result = b"""
+
+ """
+
+ channel = self.make_request(
+ "GET",
+ "/_matrix/client/v1/media/preview_url?url=http://www.twitter.com/matrixdotorg/status/12345",
+ shorthand=False,
+ await_result=False,
+ )
+ self.pump()
+
+ client = self.reactor.tcpClients[0][2].buildProtocol(None)
+ server = AccumulatingProtocol()
+ server.makeConnection(FakeTransport(client, self.reactor))
+ client.makeConnection(FakeTransport(server, self.reactor))
+ client.dataReceived(
+ (
+ b"HTTP/1.0 200 OK\r\nContent-Length: %d\r\n"
+ b'Content-Type: text/html; charset="utf8"\r\n\r\n'
+ )
+ % (len(result),)
+ + result
+ )
+ self.pump()
+
+ # The oEmbed response.
+ result2 = {
+ "version": "1.0",
+ "type": "photo",
+ "url": "http://cdn.twitter.com/matrixdotorg",
+ }
+ oembed_content = json.dumps(result2).encode("utf-8")
+
+ # Ensure a second request is made to the oEmbed URL.
+ client = self.reactor.tcpClients[1][2].buildProtocol(None)
+ server = AccumulatingProtocol()
+ server.makeConnection(FakeTransport(client, self.reactor))
+ client.makeConnection(FakeTransport(server, self.reactor))
+ client.dataReceived(
+ (
+ b"HTTP/1.0 200 OK\r\nContent-Length: %d\r\n"
+ b'Content-Type: application/json; charset="utf8"\r\n\r\n'
+ )
+ % (len(oembed_content),)
+ + oembed_content
+ )
+ self.pump()
+
+ # Ensure the URL is what was requested.
+ self.assertIn(b"/oembed?", server.data)
+
+ # Ensure a third request is made to the photo URL.
+ client = self.reactor.tcpClients[2][2].buildProtocol(None)
+ server = AccumulatingProtocol()
+ server.makeConnection(FakeTransport(client, self.reactor))
+ client.makeConnection(FakeTransport(server, self.reactor))
+ client.dataReceived(
+ (
+ b"HTTP/1.0 200 OK\r\nContent-Length: %d\r\n"
+ b"Content-Type: image/png\r\n\r\n"
+ )
+ % (len(SMALL_PNG),)
+ + SMALL_PNG
+ )
+ self.pump()
+
+ # Ensure the URL is what was requested.
+ self.assertIn(b"/matrixdotorg", server.data)
+
+ self.assertEqual(channel.code, 200)
+ body = channel.json_body
+ self.assertEqual(
+ body["og:url"], "http://www.twitter.com/matrixdotorg/status/12345"
+ )
+ self._assert_small_png(body)
+
+ @unittest.override_config(
+ {"url_preview_url_blacklist": [{"netloc": "publish.twitter.com"}]}
+ )
+ def test_oembed_autodiscovery_blocked(self) -> None:
+ """
+ If the discovered oEmbed URL is blocked, it should be discarded.
+ """
+ # This is a little cheesy in that we use the www subdomain (which isn't the
+ # list of oEmbed patterns) to get "raw" HTML response.
+ self.lookups["www.twitter.com"] = [(IPv4Address, "10.1.2.3")]
+ self.lookups["publish.twitter.com"] = [(IPv4Address, "10.1.2.4")]
+
+ result = b"""
+