kaslo/synapse
1
0
Fork 0
Code Wiki Activity

Merge commit '24229fac0' into anoa/dinsic_release_1_23_1

Browse Source
This commit is contained in:
Andrew Morgan
2020-12-31 13:40:07 +00:00
parent 8c2f42374b 24229fac05
commit 47c25048c5
33 changed files with 1415 additions and 115 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
changelog.d/8455.bugfix Normal file
View File

@@ -0,0 +1 @@
Fix fetching of E2E cross signing keys over federation when only one of the master key and device signing key is cached already.

1
changelog.d/8519.feature Normal file
View File

@@ -0,0 +1 @@
Add an admin api to delete a single file or files were not used for a defined time from server. Contributed by @dklimpel.

1
changelog.d/8539.feature Normal file
View File

@@ -0,0 +1 @@
Split admin API for reported events (`GET /_synapse/admin/v1/event_reports`) into detail and list endpoints. This is a breaking change to #8217 which was introduced in Synapse v1.21.0. Those who already use this API should check their scripts. Contributed by @dklimpel.

1
changelog.d/8580.bugfix Normal file
View File

@@ -0,0 +1 @@
Fix a bug where Synapse would blindly forward bad responses from federation to clients when retrieving profile information.

1
changelog.d/8582.doc Normal file
View File

@@ -0,0 +1 @@
Instructions for Azure AD in the OpenID Connect documentation. Contributed by peterk.

1
changelog.d/8620.bugfix Normal file
View File

@@ -0,0 +1 @@
Fix a bug where the account validity endpoint would silently fail if the user ID did not have an expiration time. It now returns a 400 error.

1
changelog.d/8634.misc Normal file
View File

@@ -0,0 +1 @@
Correct Synapse's PyPI package name in the OpenID Connect installation instructions.

1
changelog.d/8643.bugfix Normal file
View File

@@ -0,0 +1 @@
Fix a bug in the `joined_rooms` admin API if the user has never joined any rooms. The bug was introduced, along with the API, in v1.21.0.

1
changelog.d/8644.misc Normal file
View File

@@ -0,0 +1 @@
Add field `total` to device list in admin API.

1
changelog.d/8657.doc Normal file
View File

@@ -0,0 +1 @@
Fix the filepath of Dex's example config and the link to Dex's Getting Started guide in the OpenID Connect docs.

144
docs/admin_api/event_reports.rst
View File

@@ -17,67 +17,26 @@ It returns a JSON body like the following:
{
"event_reports": [
{
"content": {
"reason": "foo",
"score": -100
},
"event_id": "$bNUFCwGzWca1meCGkjp-zwslF-GfVcXukvRLI1_FaVY",
"event_json": {
"auth_events": [
"$YK4arsKKcc0LRoe700pS8DSjOvUT4NDv0HfInlMFw2M",
"$oggsNXxzPFRE3y53SUNd7nsj69-QzKv03a1RucHu-ws"
],
"content": {
"body": "matrix.org: This Week in Matrix",
"format": "org.matrix.custom.html",
"formatted_body": "<strong>matrix.org</strong>:<br><a href=\"https://matrix.org/blog/\"><strong>This Week in Matrix</strong></a>",
"msgtype": "m.notice"
},
"depth": 546,
"hashes": {
"sha256": "xK1//xnmvHJIOvbgXlkI8eEqdvoMmihVDJ9J4SNlsAw"
},
"origin": "matrix.org",
"origin_server_ts": 1592291711430,
"prev_events": [
"$YK4arsKKcc0LRoe700pS8DSjOvUT4NDv0HfInlMFw2M"
],
"prev_state": [],
"room_id": "!ERAgBpSOcCCuTJqQPk:matrix.org",
"sender": "@foobar:matrix.org",
"signatures": {
"matrix.org": {
"ed25519:a_JaEG": "cs+OUKW/iHx5pEidbWxh0UiNNHwe46Ai9LwNz+Ah16aWDNszVIe2gaAcVZfvNsBhakQTew51tlKmL2kspXk/Dg"
}
},
"type": "m.room.message",
"unsigned": {
"age_ts": 1592291711430,
}
},
"id": 2,
"reason": "foo",
"score": -100,
"received_ts": 1570897107409,
"room_alias": "#alias1:matrix.org",
"canonical_alias": "#alias1:matrix.org",
"room_id": "!ERAgBpSOcCCuTJqQPk:matrix.org",
"name": "Matrix HQ",
"sender": "@foobar:matrix.org",
"user_id": "@foo:matrix.org"
},
{
"content": {
"reason": "bar",
"score": -100
},
"event_id": "$3IcdZsDaN_En-S1DF4EMCy3v4gNRKeOJs8W5qTOKj4I",
"event_json": {
// hidden items
// see above
},
"id": 3,
"reason": "bar",
"score": -100,
"received_ts": 1598889612059,
"room_alias": "#alias2:matrix.org",
"canonical_alias": "#alias2:matrix.org",
"room_id": "!eGvUQuTCkHGVwNMOjv:matrix.org",
"name": "Your room name here",
"sender": "@foobar:matrix.org",
"user_id": "@bar:matrix.org"
}
@@ -113,17 +72,94 @@ The following fields are returned in the JSON response body:
- ``id``: integer - ID of event report.
- ``received_ts``: integer - The timestamp (in milliseconds since the unix epoch) when this report was sent.
- ``room_id``: string - The ID of the room in which the event being reported is located.
- ``name``: string - The name of the room.
- ``event_id``: string - The ID of the reported event.
- ``user_id``: string - This is the user who reported the event and wrote the reason.
- ``reason``: string - Comment made by the ``user_id`` in this report. May be blank.
- ``content``: object - Content of reported event.
- ``reason``: string - Comment made by the ``user_id`` in this report. May be blank.
- ``score``: integer - Content is reported based upon a negative score, where -100 is "most offensive" and 0 is "inoffensive".
- ``score``: integer - Content is reported based upon a negative score, where -100 is "most offensive" and 0 is "inoffensive".
- ``sender``: string - This is the ID of the user who sent the original message/event that was reported.
- ``room_alias``: string - The alias of the room. ``null`` if the room does not have a canonical alias set.
- ``event_json``: object - Details of the original event that was reported.
- ``canonical_alias``: string - The canonical alias of the room. ``null`` if the room does not have a canonical alias set.
- ``next_token``: integer - Indication for pagination. See above.
- ``total``: integer - Total number of event reports related to the query (``user_id`` and ``room_id``).
Show details of a specific event report
=======================================
This API returns information about a specific event report.
The api is::
GET /_synapse/admin/v1/event_reports/<report_id>
To use it, you will need to authenticate by providing an ``access_token`` for a
server admin: see `README.rst <README.rst>`_.
It returns a JSON body like the following:
.. code:: jsonc
{
"event_id": "$bNUFCwGzWca1meCGkjp-zwslF-GfVcXukvRLI1_FaVY",
"event_json": {
"auth_events": [
"$YK4arsKKcc0LRoe700pS8DSjOvUT4NDv0HfInlMFw2M",
"$oggsNXxzPFRE3y53SUNd7nsj69-QzKv03a1RucHu-ws"
],
"content": {
"body": "matrix.org: This Week in Matrix",
"format": "org.matrix.custom.html",
"formatted_body": "<strong>matrix.org</strong>:<br><a href=\"https://matrix.org/blog/\"><strong>This Week in Matrix</strong></a>",
"msgtype": "m.notice"
},
"depth": 546,
"hashes": {
"sha256": "xK1//xnmvHJIOvbgXlkI8eEqdvoMmihVDJ9J4SNlsAw"
},
"origin": "matrix.org",
"origin_server_ts": 1592291711430,
"prev_events": [
"$YK4arsKKcc0LRoe700pS8DSjOvUT4NDv0HfInlMFw2M"
],
"prev_state": [],
"room_id": "!ERAgBpSOcCCuTJqQPk:matrix.org",
"sender": "@foobar:matrix.org",
"signatures": {
"matrix.org": {
"ed25519:a_JaEG": "cs+OUKW/iHx5pEidbWxh0UiNNHwe46Ai9LwNz+Ah16aWDNszVIe2gaAcVZfvNsBhakQTew51tlKmL2kspXk/Dg"
}
},
"type": "m.room.message",
"unsigned": {
"age_ts": 1592291711430,
}
},
"id": <report_id>,
"reason": "foo",
"score": -100,
"received_ts": 1570897107409,
"canonical_alias": "#alias1:matrix.org",
"room_id": "!ERAgBpSOcCCuTJqQPk:matrix.org",
"name": "Matrix HQ",
"sender": "@foobar:matrix.org",
"user_id": "@foo:matrix.org"
}
**URL parameters:**
- ``report_id``: string - The ID of the event report.
**Response**
The following fields are returned in the JSON response body:
- ``id``: integer - ID of event report.
- ``received_ts``: integer - The timestamp (in milliseconds since the unix epoch) when this report was sent.
- ``room_id``: string - The ID of the room in which the event being reported is located.
- ``name``: string - The name of the room.
- ``event_id``: string - The ID of the reported event.
- ``user_id``: string - This is the user who reported the event and wrote the reason.
- ``reason``: string - Comment made by the ``user_id`` in this report. May be blank.
- ``score``: integer - Content is reported based upon a negative score, where -100 is "most offensive" and 0 is "inoffensive".
- ``sender``: string - This is the ID of the user who sent the original message/event that was reported.
- ``canonical_alias``: string - The canonical alias of the room. ``null`` if the room does not have a canonical alias set.
- ``event_json``: object - Details of the original event that was reported.

79
docs/admin_api/media_admin_api.md
View File

@@ -100,3 +100,82 @@ Response:
"num_quarantined": 10 # The number of media items successfully quarantined
}
```
# Delete local media
This API deletes the *local* media from the disk of your own server.
This includes any local thumbnails and copies of media downloaded from
remote homeservers.
This API will not affect media that has been uploaded to external
media repositories (e.g https://github.com/turt2live/matrix-media-repo/).
See also [purge_remote_media.rst](purge_remote_media.rst).
## Delete a specific local media
Delete a specific `media_id`.
Request:
```
DELETE /_synapse/admin/v1/media/<server_name>/<media_id>
{}
```
URL Parameters
* `server_name`: string - The name of your local server (e.g `matrix.org`)
* `media_id`: string - The ID of the media (e.g `abcdefghijklmnopqrstuvwx`)
Response:
```json
{
"deleted_media": [
"abcdefghijklmnopqrstuvwx"
],
"total": 1
}
```
The following fields are returned in the JSON response body:
* `deleted_media`: an array of strings - List of deleted `media_id`
* `total`: integer - Total number of deleted `media_id`
## Delete local media by date or size
Request:
```
POST /_synapse/admin/v1/media/<server_name>/delete?before_ts=<before_ts>
{}
```
URL Parameters
* `server_name`: string - The name of your local server (e.g `matrix.org`).
* `before_ts`: string representing a positive integer - Unix timestamp in ms.
Files that were last used before this timestamp will be deleted. It is the timestamp of
last access and not the timestamp creation.
* `size_gt`: Optional - string representing a positive integer - Size of the media in bytes.
Files that are larger will be deleted. Defaults to `0`.
* `keep_profiles`: Optional - string representing a boolean - Switch to also delete files
that are still used in image data (e.g user profile, room avatar).
If `false` these files will be deleted. Defaults to `true`.
Response:
```json
{
"deleted_media": [
"abcdefghijklmnopqrstuvwx",
"abcdefghijklmnopqrstuvwz"
],
"total": 2
}
```
The following fields are returned in the JSON response body:
* `deleted_media`: an array of strings - List of deleted `media_id`
* `total`: integer - Total number of deleted `media_id`

5
docs/admin_api/user_admin_api.rst
View File

@@ -375,7 +375,8 @@ A response body like the following is returned:
"last_seen_ts": 1474491775025,
"user_id": "<user_id>"
}
]
],
"total": 2
}
**Parameters**
@@ -400,6 +401,8 @@ The following fields are returned in the JSON response body:
devices was last seen. (May be a few minutes out of date, for efficiency reasons).
- ``user_id`` - Owner of device.
- ``total`` - Total number of user's devices.
Delete multiple devices
------------------
Deletes the given devices for a specific ``user_id``, and invalidates

33
docs/openid.md
View File

@@ -37,7 +37,7 @@ as follows:
provided by `matrix.org` so no further action is needed.
* If you installed Synapse into a virtualenv, run `/path/to/env/bin/pip
install synapse[oidc]` to install the necessary dependencies.
install matrix-synapse[oidc]` to install the necessary dependencies.
* For other installation mechanisms, see the documentation provided by the
maintainer.
@@ -52,14 +52,39 @@ specific providers.
Here are a few configs for providers that should work with Synapse.
### Microsoft Azure Active Directory
Azure AD can act as an OpenID Connect Provider. Register a new application under
*App registrations* in the Azure AD management console. The RedirectURI for your
application should point to your matrix server: `[synapse public baseurl]/_synapse/oidc/callback`
Go to *Certificates & secrets* and register a new client secret. Make note of your
Directory (tenant) ID as it will be used in the Azure links.
Edit your Synapse config file and change the `oidc_config` section:
```yaml
oidc_config:
enabled: true
issuer: "https://login.microsoftonline.com/<tenant id>/v2.0"
client_id: "<client id>"
client_secret: "<client secret>"
scopes: ["openid", "profile"]
authorization_endpoint: "https://login.microsoftonline.com/<tenant id>/oauth2/v2.0/authorize"
token_endpoint: "https://login.microsoftonline.com/<tenant id>/oauth2/v2.0/token"
userinfo_endpoint: "https://graph.microsoft.com/oidc/userinfo"
user_mapping_provider:
config:
localpart_template: "{{ user.preferred_username.split('@')[0] }}"
display_name_template: "{{ user.name }}"
```
### [Dex][dex-idp]
[Dex][dex-idp] is a simple, open-source, certified OpenID Connect Provider.
Although it is designed to help building a full-blown provider with an
external database, it can be configured with static passwords in a config file.
Follow the [Getting Started
guide](https://github.com/dexidp/dex/blob/master/Documentation/getting-started.md)
Follow the [Getting Started guide](https://dexidp.io/docs/getting-started/)
to install Dex.
Edit `examples/config-dev.yaml` config file from the Dex repo to add a client:
@@ -73,7 +98,7 @@ staticClients:
name: 'Synapse'
```
Run with `dex serve examples/config-dex.yaml`.
Run with `dex serve examples/config-dev.yaml`.
Synapse config:

1
mypy.ini
View File

@@ -17,6 +17,7 @@ files =
synapse/federation,
synapse/handlers/_base.py,
synapse/handlers/account_data.py,
synapse/handlers/account_validity.py,
synapse/handlers/appservice.py,
synapse/handlers/auth.py,
synapse/handlers/cas_handler.py,

29
synapse/handlers/account_validity.py
View File

@@ -18,9 +18,9 @@ import email.utils
import logging
from email.mime.multipart import MIMEMultipart
from email.mime.text import MIMEText
from typing import List, Optional, Tuple
from typing import TYPE_CHECKING, List, Optional, Tuple
from synapse.api.errors import StoreError
from synapse.api.errors import StoreError, SynapseError
from synapse.logging.context import make_deferred_yieldable
from synapse.metrics.background_process_metrics import (
run_as_background_process,
@@ -29,11 +29,14 @@ from synapse.metrics.background_process_metrics import (
from synapse.types import UserID
from synapse.util import stringutils
if TYPE_CHECKING:
from synapse.app.homeserver import HomeServer
logger = logging.getLogger(__name__)
class AccountValidityHandler:
def __init__(self, hs):
def __init__(self, hs: "HomeServer"):
self.hs = hs
self.config = hs.config
self.store = self.hs.get_datastore()
@@ -92,7 +95,7 @@ class AccountValidityHandler:
self.clock.looping_call(mark_expired_users_as_inactive, 60 * 60 * 1000)
@wrap_as_background_process("send_renewals")
async def _send_renewal_emails(self):
async def _send_renewal_emails(self) -> None:
"""Gets the list of users whose account is expiring in the amount of time
configured in the ``renew_at`` parameter from the ``account_validity``
configuration, and sends renewal emails to all of these users as long as they
@@ -106,11 +109,25 @@ class AccountValidityHandler:
user_id=user["user_id"], expiration_ts=user["expiration_ts_ms"]
)
async def send_renewal_email_to_user(self, user_id: str):
async def send_renewal_email_to_user(self, user_id: str) -> None:
"""
Send a renewal email for a specific user.
Args:
user_id: The user ID to send a renewal email for.
Raises:
SynapseError if the user is not set to renew.
"""
expiration_ts = await self.store.get_expiration_ts_for_user(user_id)
# If this user isn't set to be expired, raise an error.
if expiration_ts is None:
raise SynapseError(400, "User has no expiration time: %s" % (user_id,))
await self._send_renewal_email(user_id, expiration_ts)
async def _send_renewal_email(self, user_id: str, expiration_ts: int):
async def _send_renewal_email(self, user_id: str, expiration_ts: int) -> None:
"""Sends out a renewal email to every email address attached to the given user
with a unique link allowing them to renew their account.

27
synapse/handlers/e2e_keys.py
View File

@@ -129,6 +129,11 @@ class E2eKeysHandler:
if user_id in local_query:
results[user_id] = keys
# Get cached cross-signing keys
cross_signing_keys = await self.get_cross_signing_keys_from_cache(
device_keys_query, from_user_id
)
# Now attempt to get any remote devices from our local cache.
remote_queries_not_in_cache = {}
if remote_queries:
@@ -155,16 +160,28 @@ class E2eKeysHandler:
unsigned["device_display_name"] = device_display_name
user_devices[device_id] = result
# check for missing cross-signing keys.
for user_id in remote_queries.keys():
cached_cross_master = user_id in cross_signing_keys["master_keys"]
cached_cross_selfsigning = (
user_id in cross_signing_keys["self_signing_keys"]
)
# check if we are missing only one of cross-signing master or
# self-signing key, but the other one is cached.
# as we need both, this will issue a federation request.
# if we don't have any of the keys, either the user doesn't have
# cross-signing set up, or the cached device list
# is not (yet) updated.
if cached_cross_master ^ cached_cross_selfsigning:
user_ids_not_in_cache.add(user_id)
# add those users to the list to fetch over federation.
for user_id in user_ids_not_in_cache:
domain = get_domain_from_id(user_id)
r = remote_queries_not_in_cache.setdefault(domain, {})
r[user_id] = remote_queries[user_id]
# Get cached cross-signing keys
cross_signing_keys = await self.get_cross_signing_keys_from_cache(
device_keys_query, from_user_id
)
# Now fetch any devices that we don't have in our cache
@trace
async def do_remote_query(destination):

11
synapse/handlers/profile.py
View File

@@ -196,6 +196,13 @@ class ProfileHandler(BaseHandler):
except RequestSendFailed as e:
raise SynapseError(502, "Failed to fetch profile") from e
except HttpResponseException as e:
if e.code < 500 and e.code != 404:
# Other codes are not allowed in c2s API
logger.info(
"Server replied with wrong response: %s %s", e.code, e.msg
)
raise SynapseError(502, "Failed to fetch profile")
raise e.to_synapse_error()
async def get_profile_from_cache(self, user_id: str) -> JsonDict:
@@ -222,7 +229,7 @@ class ProfileHandler(BaseHandler):
profile = await self.store.get_from_remote_profile_cache(user_id)
return profile or {}
async def get_displayname(self, target_user: UserID) -> str:
async def get_displayname(self, target_user: UserID) -> Optional[str]:
if self.hs.is_mine(target_user):
try:
displayname = await self.store.get_profile_displayname(
@@ -357,7 +364,7 @@ class ProfileHandler(BaseHandler):
# start a profile replication push
run_in_background(self._replicate_profiles)
async def get_avatar_url(self, target_user: UserID) -> str:
async def get_avatar_url(self, target_user: UserID) -> Optional[str]:
if self.hs.is_mine(target_user):
try:
avatar_url = await self.store.get_profile_avatar_url(

6
synapse/rest/admin/__init__.py
View File

@@ -31,7 +31,10 @@ from synapse.rest.admin.devices import (
DeviceRestServlet,
DevicesRestServlet,
)
from synapse.rest.admin.event_reports import EventReportsRestServlet
from synapse.rest.admin.event_reports import (
EventReportDetailRestServlet,
EventReportsRestServlet,
)
from synapse.rest.admin.groups import DeleteGroupAdminRestServlet
from synapse.rest.admin.media import ListMediaInRoom, register_servlets_for_media_repo
from synapse.rest.admin.purge_room_servlet import PurgeRoomServlet
@@ -222,6 +225,7 @@ def register_servlets(hs, http_server):
DevicesRestServlet(hs).register(http_server)
DeleteDevicesRestServlet(hs).register(http_server)
EventReportsRestServlet(hs).register(http_server)
EventReportDetailRestServlet(hs).register(http_server)
def register_servlets_for_client_rest_resource(hs, http_server):

2
synapse/rest/admin/devices.py
View File

@@ -119,7 +119,7 @@ class DevicesRestServlet(RestServlet):
raise NotFoundError("Unknown user")
devices = await self.device_handler.get_devices_by_user(target_user.to_string())
return 200, {"devices": devices}
return 200, {"devices": devices, "total": len(devices)}
class DeleteDevicesRestServlet(RestServlet):

46
synapse/rest/admin/event_reports.py
View File

@@ -15,7 +15,7 @@
import logging
from synapse.api.errors import Codes, SynapseError
from synapse.api.errors import Codes, NotFoundError, SynapseError
from synapse.http.servlet import RestServlet, parse_integer, parse_string
from synapse.rest.admin._base import admin_patterns, assert_requester_is_admin
@@ -86,3 +86,47 @@ class EventReportsRestServlet(RestServlet):
ret["next_token"] = start + len(event_reports)
return 200, ret
class EventReportDetailRestServlet(RestServlet):
"""
Get a specific reported event that is known to the homeserver. Results are returned
in a dictionary containing report information.
The requester must have administrator access in Synapse.
GET /_synapse/admin/v1/event_reports/<report_id>
returns:
200 OK with details report if success otherwise an error.
Args:
The parameter `report_id` is the ID of the event report in the database.
Returns:
JSON blob of information about the event report
"""
PATTERNS = admin_patterns("/event_reports/(?P<report_id>[^/]*)$")
def __init__(self, hs):
self.hs = hs
self.auth = hs.get_auth()
self.store = hs.get_datastore()
async def on_GET(self, request, report_id):
await assert_requester_is_admin(self.auth, request)
message = (
"The report_id parameter must be a string representing a positive integer."
)
try:
report_id = int(report_id)
except ValueError:
raise SynapseError(400, message, errcode=Codes.INVALID_PARAM)
if report_id < 0:
raise SynapseError(400, message, errcode=Codes.INVALID_PARAM)
ret = await self.store.get_event_report(report_id)
if not ret:
raise NotFoundError("Event report not found")
return 200, ret

81
synapse/rest/admin/media.py
View File

@@ -16,9 +16,10 @@
import logging
from synapse.api.errors import AuthError
from synapse.http.servlet import RestServlet, parse_integer
from synapse.api.errors import AuthError, Codes, NotFoundError, SynapseError
from synapse.http.servlet import RestServlet, parse_boolean, parse_integer
from synapse.rest.admin._base import (
admin_patterns,
assert_requester_is_admin,
assert_user_is_admin,
historical_admin_path_patterns,
@@ -150,6 +151,80 @@ class PurgeMediaCacheRestServlet(RestServlet):
return 200, ret
class DeleteMediaByID(RestServlet):
"""Delete local media by a given ID. Removes it from this server.
"""
PATTERNS = admin_patterns("/media/(?P<server_name>[^/]+)/(?P<media_id>[^/]+)")
def __init__(self, hs):
self.store = hs.get_datastore()
self.auth = hs.get_auth()
self.server_name = hs.hostname
self.media_repository = hs.get_media_repository()
async def on_DELETE(self, request, server_name: str, media_id: str):
await assert_requester_is_admin(self.auth, request)
if self.server_name != server_name:
raise SynapseError(400, "Can only delete local media")
if await self.store.get_local_media(media_id) is None:
raise NotFoundError("Unknown media")
logging.info("Deleting local media by ID: %s", media_id)
deleted_media, total = await self.media_repository.delete_local_media(media_id)
return 200, {"deleted_media": deleted_media, "total": total}
class DeleteMediaByDateSize(RestServlet):
"""Delete local media and local copies of remote media by
timestamp and size.
"""
PATTERNS = admin_patterns("/media/(?P<server_name>[^/]+)/delete")
def __init__(self, hs):
self.store = hs.get_datastore()
self.auth = hs.get_auth()
self.server_name = hs.hostname
self.media_repository = hs.get_media_repository()
async def on_POST(self, request, server_name: str):
await assert_requester_is_admin(self.auth, request)
before_ts = parse_integer(request, "before_ts", required=True)
size_gt = parse_integer(request, "size_gt", default=0)
keep_profiles = parse_boolean(request, "keep_profiles", default=True)
if before_ts < 0:
raise SynapseError(
400,
"Query parameter before_ts must be a string representing a positive integer.",
errcode=Codes.INVALID_PARAM,
)
if size_gt < 0:
raise SynapseError(
400,
"Query parameter size_gt must be a string representing a positive integer.",
errcode=Codes.INVALID_PARAM,
)
if self.server_name != server_name:
raise SynapseError(400, "Can only delete local media")
logging.info(
"Deleting local media by timestamp: %s, size larger than: %s, keep profile media: %s"
% (before_ts, size_gt, keep_profiles)
)
deleted_media, total = await self.media_repository.delete_old_local_media(
before_ts, size_gt, keep_profiles
)
return 200, {"deleted_media": deleted_media, "total": total}
def register_servlets_for_media_repo(hs, http_server):
"""
Media repo specific APIs.
@@ -159,3 +234,5 @@ def register_servlets_for_media_repo(hs, http_server):
QuarantineMediaByID(hs).register(http_server)
QuarantineMediaByUser(hs).register(http_server)
ListMediaInRoom(hs).register(http_server)
DeleteMediaByID(hs).register(http_server)
DeleteMediaByDateSize(hs).register(http_server)

7
synapse/rest/admin/users.py
View File

@@ -702,9 +702,10 @@ class UserMembershipRestServlet(RestServlet):
if not self.is_mine(UserID.from_string(user_id)):
raise SynapseError(400, "Can only lookup local users")
room_ids = await self.store.get_rooms_for_user(user_id)
if not room_ids:
raise NotFoundError("User not found")
user = await self.store.get_user_by_id(user_id)
if user is None:
raise NotFoundError("Unknown user")
room_ids = await self.store.get_rooms_for_user(user_id)
ret = {"joined_rooms": list(room_ids), "total": len(room_ids)}
return 200, ret

17
synapse/rest/media/v1/filepath.py
View File

@@ -69,6 +69,23 @@ class MediaFilePaths:
local_media_thumbnail = _wrap_in_base_path(local_media_thumbnail_rel)
def local_media_thumbnail_dir(self, media_id: str) -> str:
"""
Retrieve the local store path of thumbnails of a given media_id
Args:
media_id: The media ID to query.
Returns:
Path of local_thumbnails from media_id
"""
return os.path.join(
self.base_path,
"local_thumbnails",
media_id[0:2],
media_id[2:4],
media_id[4:],
)
def remote_media_filepath_rel(self, server_name, file_id):
return os.path.join(
"remote_content", server_name, file_id[0:2], file_id[2:4], file_id[4:]

72
synapse/rest/media/v1/media_repository.py
View File

@@ -18,7 +18,7 @@ import errno
import logging
import os
import shutil
from typing import IO, Dict, Optional, Tuple
from typing import IO, Dict, List, Optional, Tuple
import twisted.internet.error
import twisted.web.http
@@ -767,6 +767,76 @@ class MediaRepository:
return {"deleted": deleted}
async def delete_local_media(self, media_id: str) -> Tuple[List[str], int]:
"""
Delete the given local or remote media ID from this server
Args:
media_id: The media ID to delete.
Returns:
A tuple of (list of deleted media IDs, total deleted media IDs).
"""
return await self._remove_local_media_from_disk([media_id])
async def delete_old_local_media(
self, before_ts: int, size_gt: int = 0, keep_profiles: bool = True,
) -> Tuple[List[str], int]:
"""
Delete local or remote media from this server by size and timestamp. Removes
media files, any thumbnails and cached URLs.
Args:
before_ts: Unix timestamp in ms.
Files that were last used before this timestamp will be deleted
size_gt: Size of the media in bytes. Files that are larger will be deleted
keep_profiles: Switch to delete also files that are still used in image data
(e.g user profile, room avatar)
If false these files will be deleted
Returns:
A tuple of (list of deleted media IDs, total deleted media IDs).
"""
old_media = await self.store.get_local_media_before(
before_ts, size_gt, keep_profiles,
)
return await self._remove_local_media_from_disk(old_media)
async def _remove_local_media_from_disk(
self, media_ids: List[str]
) -> Tuple[List[str], int]:
"""
Delete local or remote media from this server. Removes media files,
any thumbnails and cached URLs.
Args:
media_ids: List of media_id to delete
Returns:
A tuple of (list of deleted media IDs, total deleted media IDs).
"""
removed_media = []
for media_id in media_ids:
logger.info("Deleting media with ID '%s'", media_id)
full_path = self.filepaths.local_media_filepath(media_id)
try:
os.remove(full_path)
except OSError as e:
logger.warning("Failed to remove file: %r: %s", full_path, e)
if e.errno == errno.ENOENT:
pass
else:
continue
thumbnail_dir = self.filepaths.local_media_thumbnail_dir(media_id)
shutil.rmtree(thumbnail_dir, ignore_errors=True)
await self.store.delete_remote_media(self.server_name, media_id)
await self.store.delete_url_cache((media_id,))
await self.store.delete_url_cache_media((media_id,))
removed_media.append(media_id)
return removed_media, len(removed_media)
class MediaRepositoryResource(Resource):
"""File uploading and downloading.

53
synapse/storage/databases/main/media_repository.py
View File

@@ -93,6 +93,7 @@ class MediaRepositoryStore(MediaRepositoryBackgroundUpdateStore):
def __init__(self, database: DatabasePool, db_conn, hs):
super().__init__(database, db_conn, hs)
self.server_name = hs.hostname
async def get_local_media(self, media_id: str) -> Optional[Dict[str, Any]]:
"""Get the metadata for a local piece of media
@@ -115,6 +116,58 @@ class MediaRepositoryStore(MediaRepositoryBackgroundUpdateStore):
desc="get_local_media",
)
async def get_local_media_before(
self, before_ts: int, size_gt: int, keep_profiles: bool,
) -> Optional[List[str]]:
# to find files that have never been accessed (last_access_ts IS NULL)
# compare with `created_ts`
sql = """
SELECT media_id
FROM local_media_repository AS lmr
WHERE
( last_access_ts < ?
OR ( created_ts < ? AND last_access_ts IS NULL ) )
AND media_length > ?
"""
if keep_profiles:
sql_keep = """
AND (
NOT EXISTS
(SELECT 1
FROM profiles
WHERE profiles.avatar_url = '{media_prefix}' || lmr.media_id)
AND NOT EXISTS
(SELECT 1
FROM groups
WHERE groups.avatar_url = '{media_prefix}' || lmr.media_id)
AND NOT EXISTS
(SELECT 1
FROM room_memberships
WHERE room_memberships.avatar_url = '{media_prefix}' || lmr.media_id)
AND NOT EXISTS
(SELECT 1
FROM user_directory
WHERE user_directory.avatar_url = '{media_prefix}' || lmr.media_id)
AND NOT EXISTS
(SELECT 1
FROM room_stats_state
WHERE room_stats_state.avatar = '{media_prefix}' || lmr.media_id)
)
""".format(
media_prefix="mxc://%s/" % (self.server_name,),
)
sql += sql_keep
def _get_local_media_before_txn(txn):
txn.execute(sql, (before_ts, before_ts, size_gt))
return [row[0] for row in txn]
return await self.db_pool.runInteraction(
"get_local_media_before", _get_local_media_before_txn
)
async def store_local_media(
self,
media_id,

4
synapse/storage/databases/main/profile.py
View File

@@ -45,7 +45,7 @@ class ProfileWorkerStore(SQLBaseStore):
)
@cached(max_entries=5000)
async def get_profile_displayname(self, user_localpart: str) -> str:
async def get_profile_displayname(self, user_localpart: str) -> Optional[str]:
return await self.db_pool.simple_select_one_onecol(
table="profiles",
keyvalues={"user_id": user_localpart},
@@ -54,7 +54,7 @@ class ProfileWorkerStore(SQLBaseStore):
)
@cached(max_entries=5000)
async def get_profile_avatar_url(self, user_localpart: str) -> str:
async def get_profile_avatar_url(self, user_localpart: str) -> Optional[str]:
return await self.db_pool.simple_select_one_onecol(
table="profiles",
keyvalues={"user_id": user_localpart},

4
synapse/storage/databases/main/registration.py
View File

@@ -295,13 +295,13 @@ class RegistrationWorkerStore(CacheInvalidationWorkerStore):
desc="get_renewal_token_for_user",
)
async def get_users_expiring_soon(self) -> List[Dict[str, int]]:
async def get_users_expiring_soon(self) -> List[Dict[str, Any]]:
"""Selects users whose account will expire in the [now, now + renew_at] time
window (see configuration for account_validity for information on what renew_at
refers to).
Returns:
A list of dictionaries mapping user ID to expiration time (in milliseconds).
A list of dictionaries, each with a user ID and expiration time (in milliseconds).
"""
def select_users_txn(txn, now_ms, renew_at):

102
synapse/storage/databases/main/room.py
View File

@@ -1433,6 +1433,65 @@ class RoomStore(RoomBackgroundUpdateStore, RoomWorkerStore, SearchStore):
desc="add_event_report",
)
async def get_event_report(self, report_id: int) -> Optional[Dict[str, Any]]:
"""Retrieve an event report
Args:
report_id: ID of reported event in database
Returns:
event_report: json list of information from event report
"""
def _get_event_report_txn(txn, report_id):
sql = """
SELECT
er.id,
er.received_ts,
er.room_id,
er.event_id,
er.user_id,
er.content,
events.sender,
room_stats_state.canonical_alias,
room_stats_state.name,
event_json.json AS event_json
FROM event_reports AS er
LEFT JOIN events
ON events.event_id = er.event_id
JOIN event_json
ON event_json.event_id = er.event_id
JOIN room_stats_state
ON room_stats_state.room_id = er.room_id
WHERE er.id = ?
"""
txn.execute(sql, [report_id])
row = txn.fetchone()
if not row:
return None
event_report = {
"id": row[0],
"received_ts": row[1],
"room_id": row[2],
"event_id": row[3],
"user_id": row[4],
"score": db_to_json(row[5]).get("score"),
"reason": db_to_json(row[5]).get("reason"),
"sender": row[6],
"canonical_alias": row[7],
"name": row[8],
"event_json": db_to_json(row[9]),
}
return event_report
return await self.db_pool.runInteraction(
"get_event_report", _get_event_report_txn, report_id
)
async def get_event_reports_paginate(
self,
start: int,
@@ -1490,18 +1549,15 @@ class RoomStore(RoomBackgroundUpdateStore, RoomWorkerStore, SearchStore):
er.room_id,
er.event_id,
er.user_id,
er.reason,
er.content,
events.sender,
room_aliases.room_alias,
event_json.json AS event_json
room_stats_state.canonical_alias,
room_stats_state.name
FROM event_reports AS er
LEFT JOIN room_aliases
ON room_aliases.room_id = er.room_id
JOIN events
LEFT JOIN events
ON events.event_id = er.event_id
JOIN event_json
ON event_json.event_id = er.event_id
JOIN room_stats_state
ON room_stats_state.room_id = er.room_id
{where_clause}
ORDER BY er.received_ts {order}
LIMIT ?
@@ -1512,15 +1568,29 @@ class RoomStore(RoomBackgroundUpdateStore, RoomWorkerStore, SearchStore):
args += [limit, start]
txn.execute(sql, args)
event_reports = self.db_pool.cursor_to_dict(txn)
if count > 0:
for row in event_reports:
try:
row["content"] = db_to_json(row["content"])
row["event_json"] = db_to_json(row["event_json"])
except Exception:
continue
event_reports = []
for row in txn:
try:
s = db_to_json(row[5]).get("score")
r = db_to_json(row[5]).get("reason")
except Exception:
logger.error("Unable to parse json from event_reports: %s", row[0])
continue
event_reports.append(
{
"id": row[0],
"received_ts": row[1],
"room_id": row[2],
"event_id": row[3],
"user_id": row[4],
"score": s,
"reason": r,
"sender": row[6],
"canonical_alias": row[7],
"name": row[8],
}
)
return event_reports, count

17
tests/rest/admin/test_device.py
View File

@@ -393,6 +393,22 @@ class DevicesRestTestCase(unittest.HomeserverTestCase):
self.assertEqual(400, channel.code, msg=channel.json_body)
self.assertEqual("Can only lookup local users", channel.json_body["error"])
def test_user_has_no_devices(self):
"""
Tests that a normal lookup for devices is successfully
if user has no devices
"""
# Get devices
request, channel = self.make_request(
"GET", self.url, access_token=self.admin_user_tok,
)
self.render(request)
self.assertEqual(200, channel.code, msg=channel.json_body)
self.assertEqual(0, channel.json_body["total"])
self.assertEqual(0, len(channel.json_body["devices"]))
def test_get_devices(self):
"""
Tests that a normal lookup for devices is successfully
@@ -409,6 +425,7 @@ class DevicesRestTestCase(unittest.HomeserverTestCase):
self.render(request)
self.assertEqual(200, channel.code, msg=channel.json_body)
self.assertEqual(number_devices, channel.json_body["total"])
self.assertEqual(number_devices, len(channel.json_body["devices"]))
self.assertEqual(self.other_user, channel.json_body["devices"][0]["user_id"])
# Check that all fields are available

196
tests/rest/admin/test_event_reports.py
View File

@@ -70,6 +70,16 @@ class EventReportsTestCase(unittest.HomeserverTestCase):
self.url = "/_synapse/admin/v1/event_reports"
def test_no_auth(self):
"""
Try to get an event report without authentication.
"""
request, channel = self.make_request("GET", self.url, b"{}")
self.render(request)
self.assertEqual(401, int(channel.result["code"]), msg=channel.result["body"])
self.assertEqual(Codes.MISSING_TOKEN, channel.json_body["errcode"])
def test_requester_is_no_admin(self):
"""
If the user is not a server admin, an error 403 is returned.
@@ -266,7 +276,7 @@ class EventReportsTestCase(unittest.HomeserverTestCase):
def test_limit_is_negative(self):
"""
Testing that a negative list parameter returns a 400
Testing that a negative limit parameter returns a 400
"""
request, channel = self.make_request(
@@ -360,7 +370,7 @@ class EventReportsTestCase(unittest.HomeserverTestCase):
self.assertEqual(200, int(channel.result["code"]), msg=channel.result["body"])
def _check_fields(self, content):
"""Checks that all attributes are present in a event report
"""Checks that all attributes are present in an event report
"""
for c in content:
self.assertIn("id", c)
@@ -368,15 +378,175 @@ class EventReportsTestCase(unittest.HomeserverTestCase):
self.assertIn("room_id", c)
self.assertIn("event_id", c)
self.assertIn("user_id", c)
self.assertIn("reason", c)
self.assertIn("content", c)
self.assertIn("sender", c)
self.assertIn("room_alias", c)
self.assertIn("event_json", c)
self.assertIn("score", c["content"])
self.assertIn("reason", c["content"])
self.assertIn("auth_events", c["event_json"])
self.assertIn("type", c["event_json"])
self.assertIn("room_id", c["event_json"])
self.assertIn("sender", c["event_json"])
self.assertIn("content", c["event_json"])
self.assertIn("canonical_alias", c)
self.assertIn("name", c)
self.assertIn("score", c)
self.assertIn("reason", c)
class EventReportDetailTestCase(unittest.HomeserverTestCase):
servlets = [
synapse.rest.admin.register_servlets,
login.register_servlets,
room.register_servlets,
report_event.register_servlets,
]
def prepare(self, reactor, clock, hs):
self.store = hs.get_datastore()
self.admin_user = self.register_user("admin", "pass", admin=True)
self.admin_user_tok = self.login("admin", "pass")
self.other_user = self.register_user("user", "pass")
self.other_user_tok = self.login("user", "pass")
self.room_id1 = self.helper.create_room_as(
self.other_user, tok=self.other_user_tok, is_public=True
)
self.helper.join(self.room_id1, user=self.admin_user, tok=self.admin_user_tok)
self._create_event_and_report(
room_id=self.room_id1, user_tok=self.other_user_tok,
)
# first created event report gets `id`=2
self.url = "/_synapse/admin/v1/event_reports/2"
def test_no_auth(self):
"""
Try to get event report without authentication.
"""
request, channel = self.make_request("GET", self.url, b"{}")
self.render(request)
self.assertEqual(401, int(channel.result["code"]), msg=channel.result["body"])
self.assertEqual(Codes.MISSING_TOKEN, channel.json_body["errcode"])
def test_requester_is_no_admin(self):
"""
If the user is not a server admin, an error 403 is returned.
"""
request, channel = self.make_request(
"GET", self.url, access_token=self.other_user_tok,
)
self.render(request)
self.assertEqual(403, int(channel.result["code"]), msg=channel.result["body"])
self.assertEqual(Codes.FORBIDDEN, channel.json_body["errcode"])
def test_default_success(self):
"""
Testing get a reported event
"""
request, channel = self.make_request(
"GET", self.url, access_token=self.admin_user_tok,
)
self.render(request)
self.assertEqual(200, int(channel.result["code"]), msg=channel.result["body"])
self._check_fields(channel.json_body)
def test_invalid_report_id(self):
"""
Testing that an invalid `report_id` returns a 400.
"""
# `report_id` is negative
request, channel = self.make_request(
"GET",
"/_synapse/admin/v1/event_reports/-123",
access_token=self.admin_user_tok,
)
self.render(request)
self.assertEqual(400, int(channel.result["code"]), msg=channel.result["body"])
self.assertEqual(Codes.INVALID_PARAM, channel.json_body["errcode"])
self.assertEqual(
"The report_id parameter must be a string representing a positive integer.",
channel.json_body["error"],
)
# `report_id` is a non-numerical string
request, channel = self.make_request(
"GET",
"/_synapse/admin/v1/event_reports/abcdef",
access_token=self.admin_user_tok,
)
self.render(request)
self.assertEqual(400, int(channel.result["code"]), msg=channel.result["body"])
self.assertEqual(Codes.INVALID_PARAM, channel.json_body["errcode"])
self.assertEqual(
"The report_id parameter must be a string representing a positive integer.",
channel.json_body["error"],
)
# `report_id` is undefined
request, channel = self.make_request(
"GET",
"/_synapse/admin/v1/event_reports/",
access_token=self.admin_user_tok,
)
self.render(request)
self.assertEqual(400, int(channel.result["code"]), msg=channel.result["body"])
self.assertEqual(Codes.INVALID_PARAM, channel.json_body["errcode"])
self.assertEqual(
"The report_id parameter must be a string representing a positive integer.",
channel.json_body["error"],
)
def test_report_id_not_found(self):
"""
Testing that a not existing `report_id` returns a 404.
"""
request, channel = self.make_request(
"GET",
"/_synapse/admin/v1/event_reports/123",
access_token=self.admin_user_tok,
)
self.render(request)
self.assertEqual(404, int(channel.result["code"]), msg=channel.result["body"])
self.assertEqual(Codes.NOT_FOUND, channel.json_body["errcode"])
self.assertEqual("Event report not found", channel.json_body["error"])
def _create_event_and_report(self, room_id, user_tok):
"""Create and report events
"""
resp = self.helper.send(room_id, tok=user_tok)
event_id = resp["event_id"]
request, channel = self.make_request(
"POST",
"rooms/%s/report/%s" % (room_id, event_id),
json.dumps({"score": -100, "reason": "this makes me sad"}),
access_token=user_tok,
)
self.render(request)
self.assertEqual(200, int(channel.result["code"]), msg=channel.result["body"])
def _check_fields(self, content):
"""Checks that all attributes are present in a event report
"""
self.assertIn("id", content)
self.assertIn("received_ts", content)
self.assertIn("room_id", content)
self.assertIn("event_id", content)
self.assertIn("user_id", content)
self.assertIn("sender", content)
self.assertIn("canonical_alias", content)
self.assertIn("name", content)
self.assertIn("event_json", content)
self.assertIn("score", content)
self.assertIn("reason", content)
self.assertIn("auth_events", content["event_json"])
self.assertIn("type", content["event_json"])
self.assertIn("room_id", content["event_json"])
self.assertIn("sender", content["event_json"])
self.assertIn("content", content["event_json"])

568
tests/rest/admin/test_media.py Normal file
View File

@@ -0,0 +1,568 @@
# -*- coding: utf-8 -*-
# Copyright 2020 Dirk Klimpel
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import json
import os
from binascii import unhexlify
import synapse.rest.admin
from synapse.api.errors import Codes
from synapse.rest.client.v1 import login, profile, room
from synapse.rest.media.v1.filepath import MediaFilePaths
from tests import unittest
class DeleteMediaByIDTestCase(unittest.HomeserverTestCase):
servlets = [
synapse.rest.admin.register_servlets,
synapse.rest.admin.register_servlets_for_media_repo,
login.register_servlets,
]
def prepare(self, reactor, clock, hs):
self.handler = hs.get_device_handler()
self.media_repo = hs.get_media_repository_resource()
self.server_name = hs.hostname
self.admin_user = self.register_user("admin", "pass", admin=True)
self.admin_user_tok = self.login("admin", "pass")
self.filepaths = MediaFilePaths(hs.config.media_store_path)
def test_no_auth(self):
"""
Try to delete media without authentication.
"""
url = "/_synapse/admin/v1/media/%s/%s" % (self.server_name, "12345")
request, channel = self.make_request("DELETE", url, b"{}")
self.render(request)
self.assertEqual(401, int(channel.result["code"]), msg=channel.result["body"])
self.assertEqual(Codes.MISSING_TOKEN, channel.json_body["errcode"])
def test_requester_is_no_admin(self):
"""
If the user is not a server admin, an error is returned.
"""
self.other_user = self.register_user("user", "pass")
self.other_user_token = self.login("user", "pass")
url = "/_synapse/admin/v1/media/%s/%s" % (self.server_name, "12345")
request, channel = self.make_request(
"DELETE", url, access_token=self.other_user_token,
)
self.render(request)
self.assertEqual(403, int(channel.result["code"]), msg=channel.result["body"])
self.assertEqual(Codes.FORBIDDEN, channel.json_body["errcode"])
def test_media_does_not_exist(self):
"""
Tests that a lookup for a media that does not exist returns a 404
"""
url = "/_synapse/admin/v1/media/%s/%s" % (self.server_name, "12345")
request, channel = self.make_request(
"DELETE", url, access_token=self.admin_user_tok,
)
self.render(request)
self.assertEqual(404, channel.code, msg=channel.json_body)
self.assertEqual(Codes.NOT_FOUND, channel.json_body["errcode"])
def test_media_is_not_local(self):
"""
Tests that a lookup for a media that is not a local returns a 400
"""
url = "/_synapse/admin/v1/media/%s/%s" % ("unknown_domain", "12345")
request, channel = self.make_request(
"DELETE", url, access_token=self.admin_user_tok,
)
self.render(request)
self.assertEqual(400, channel.code, msg=channel.json_body)
self.assertEqual("Can only delete local media", channel.json_body["error"])
def test_delete_media(self):
"""
Tests that delete a media is successfully
"""
download_resource = self.media_repo.children[b"download"]
upload_resource = self.media_repo.children[b"upload"]
image_data = unhexlify(
b"89504e470d0a1a0a0000000d4948445200000001000000010806"
b"0000001f15c4890000000a49444154789c63000100000500010d"
b"0a2db40000000049454e44ae426082"
)
# Upload some media into the room
response = self.helper.upload_media(
upload_resource, image_data, tok=self.admin_user_tok, expect_code=200
)
# Extract media ID from the response
server_and_media_id = response["content_uri"][6:] # Cut off 'mxc://'
server_name, media_id = server_and_media_id.split("/")
self.assertEqual(server_name, self.server_name)
# Attempt to access media
request, channel = self.make_request(
"GET",
server_and_media_id,
shorthand=False,
access_token=self.admin_user_tok,
)
request.render(download_resource)
self.pump(1.0)
# Should be successful
self.assertEqual(
200,
channel.code,
msg=(
"Expected to receive a 200 on accessing media: %s" % server_and_media_id
),
)
# Test if the file exists
local_path = self.filepaths.local_media_filepath(media_id)
self.assertTrue(os.path.exists(local_path))
url = "/_synapse/admin/v1/media/%s/%s" % (self.server_name, media_id)
# Delete media
request, channel = self.make_request(
"DELETE", url, access_token=self.admin_user_tok,
)
self.render(request)
self.assertEqual(200, channel.code, msg=channel.json_body)
self.assertEqual(1, channel.json_body["total"])
self.assertEqual(
media_id, channel.json_body["deleted_media"][0],
)
# Attempt to access media
request, channel = self.make_request(
"GET",
server_and_media_id,
shorthand=False,
access_token=self.admin_user_tok,
)
request.render(download_resource)
self.pump(1.0)
self.assertEqual(
404,
channel.code,
msg=(
"Expected to receive a 404 on accessing deleted media: %s"
% server_and_media_id
),
)
# Test if the file is deleted
self.assertFalse(os.path.exists(local_path))
class DeleteMediaByDateSizeTestCase(unittest.HomeserverTestCase):
servlets = [
synapse.rest.admin.register_servlets,
synapse.rest.admin.register_servlets_for_media_repo,
login.register_servlets,
profile.register_servlets,
room.register_servlets,
]
def prepare(self, reactor, clock, hs):
self.handler = hs.get_device_handler()
self.media_repo = hs.get_media_repository_resource()
self.server_name = hs.hostname
self.clock = hs.clock
self.admin_user = self.register_user("admin", "pass", admin=True)
self.admin_user_tok = self.login("admin", "pass")
self.filepaths = MediaFilePaths(hs.config.media_store_path)
self.url = "/_synapse/admin/v1/media/%s/delete" % self.server_name
def test_no_auth(self):
"""
Try to delete media without authentication.
"""
request, channel = self.make_request("POST", self.url, b"{}")
self.render(request)
self.assertEqual(401, int(channel.result["code"]), msg=channel.result["body"])
self.assertEqual(Codes.MISSING_TOKEN, channel.json_body["errcode"])
def test_requester_is_no_admin(self):
"""
If the user is not a server admin, an error is returned.
"""
self.other_user = self.register_user("user", "pass")
self.other_user_token = self.login("user", "pass")
request, channel = self.make_request(
"POST", self.url, access_token=self.other_user_token,
)
self.render(request)
self.assertEqual(403, int(channel.result["code"]), msg=channel.result["body"])
self.assertEqual(Codes.FORBIDDEN, channel.json_body["errcode"])
def test_media_is_not_local(self):
"""
Tests that a lookup for media that is not local returns a 400
"""
url = "/_synapse/admin/v1/media/%s/delete" % "unknown_domain"
request, channel = self.make_request(
"POST", url + "?before_ts=1234", access_token=self.admin_user_tok,
)
self.render(request)
self.assertEqual(400, channel.code, msg=channel.json_body)
self.assertEqual("Can only delete local media", channel.json_body["error"])
def test_missing_parameter(self):
"""
If the parameter `before_ts` is missing, an error is returned.
"""
request, channel = self.make_request(
"POST", self.url, access_token=self.admin_user_tok,
)
self.render(request)
self.assertEqual(400, int(channel.result["code"]), msg=channel.result["body"])
self.assertEqual(Codes.MISSING_PARAM, channel.json_body["errcode"])
self.assertEqual(
"Missing integer query parameter b'before_ts'", channel.json_body["error"]
)
def test_invalid_parameter(self):
"""
If parameters are invalid, an error is returned.
"""
request, channel = self.make_request(
"POST", self.url + "?before_ts=-1234", access_token=self.admin_user_tok,
)
self.render(request)
self.assertEqual(400, int(channel.result["code"]), msg=channel.result["body"])
self.assertEqual(Codes.INVALID_PARAM, channel.json_body["errcode"])
self.assertEqual(
"Query parameter before_ts must be a string representing a positive integer.",
channel.json_body["error"],
)
request, channel = self.make_request(
"POST",
self.url + "?before_ts=1234&size_gt=-1234",
access_token=self.admin_user_tok,
)
self.render(request)
self.assertEqual(400, int(channel.result["code"]), msg=channel.result["body"])
self.assertEqual(Codes.INVALID_PARAM, channel.json_body["errcode"])
self.assertEqual(
"Query parameter size_gt must be a string representing a positive integer.",
channel.json_body["error"],
)
request, channel = self.make_request(
"POST",
self.url + "?before_ts=1234&keep_profiles=not_bool",
access_token=self.admin_user_tok,
)
self.render(request)
self.assertEqual(400, int(channel.result["code"]), msg=channel.result["body"])
self.assertEqual(Codes.UNKNOWN, channel.json_body["errcode"])
self.assertEqual(
"Boolean query parameter b'keep_profiles' must be one of ['true', 'false']",
channel.json_body["error"],
)
def test_delete_media_never_accessed(self):
"""
Tests that media deleted if it is older than `before_ts` and never accessed
`last_access_ts` is `NULL` and `created_ts` < `before_ts`
"""
# upload and do not access
server_and_media_id = self._create_media()
self.pump(1.0)
# test that the file exists
media_id = server_and_media_id.split("/")[1]
local_path = self.filepaths.local_media_filepath(media_id)
self.assertTrue(os.path.exists(local_path))
# timestamp after upload/create
now_ms = self.clock.time_msec()
request, channel = self.make_request(
"POST",
self.url + "?before_ts=" + str(now_ms),
access_token=self.admin_user_tok,
)
self.render(request)
self.assertEqual(200, channel.code, msg=channel.json_body)
self.assertEqual(1, channel.json_body["total"])
self.assertEqual(
media_id, channel.json_body["deleted_media"][0],
)
self._access_media(server_and_media_id, False)
def test_keep_media_by_date(self):
"""
Tests that media is not deleted if it is newer than `before_ts`
"""
# timestamp before upload
now_ms = self.clock.time_msec()
server_and_media_id = self._create_media()
self._access_media(server_and_media_id)
request, channel = self.make_request(
"POST",
self.url + "?before_ts=" + str(now_ms),
access_token=self.admin_user_tok,
)
self.render(request)
self.assertEqual(200, channel.code, msg=channel.json_body)
self.assertEqual(0, channel.json_body["total"])
self._access_media(server_and_media_id)
# timestamp after upload
now_ms = self.clock.time_msec()
request, channel = self.make_request(
"POST",
self.url + "?before_ts=" + str(now_ms),
access_token=self.admin_user_tok,
)
self.render(request)
self.assertEqual(200, channel.code, msg=channel.json_body)
self.assertEqual(1, channel.json_body["total"])
self.assertEqual(
server_and_media_id.split("/")[1], channel.json_body["deleted_media"][0],
)
self._access_media(server_and_media_id, False)
def test_keep_media_by_size(self):
"""
Tests that media is not deleted if its size is smaller than or equal
to `size_gt`
"""
server_and_media_id = self._create_media()
self._access_media(server_and_media_id)
now_ms = self.clock.time_msec()
request, channel = self.make_request(
"POST",
self.url + "?before_ts=" + str(now_ms) + "&size_gt=67",
access_token=self.admin_user_tok,
)
self.render(request)
self.assertEqual(200, channel.code, msg=channel.json_body)
self.assertEqual(0, channel.json_body["total"])
self._access_media(server_and_media_id)
now_ms = self.clock.time_msec()
request, channel = self.make_request(
"POST",
self.url + "?before_ts=" + str(now_ms) + "&size_gt=66",
access_token=self.admin_user_tok,
)
self.render(request)
self.assertEqual(200, channel.code, msg=channel.json_body)
self.assertEqual(1, channel.json_body["total"])
self.assertEqual(
server_and_media_id.split("/")[1], channel.json_body["deleted_media"][0],
)
self._access_media(server_and_media_id, False)
def test_keep_media_by_user_avatar(self):
"""
Tests that we do not delete media if is used as a user avatar
Tests parameter `keep_profiles`
"""
server_and_media_id = self._create_media()
self._access_media(server_and_media_id)
# set media as avatar
request, channel = self.make_request(
"PUT",
"/profile/%s/avatar_url" % (self.admin_user,),
content=json.dumps({"avatar_url": "mxc://%s" % (server_and_media_id,)}),
access_token=self.admin_user_tok,
)
self.render(request)
self.assertEqual(200, channel.code, msg=channel.json_body)
now_ms = self.clock.time_msec()
request, channel = self.make_request(
"POST",
self.url + "?before_ts=" + str(now_ms) + "&keep_profiles=true",
access_token=self.admin_user_tok,
)
self.render(request)
self.assertEqual(200, channel.code, msg=channel.json_body)
self.assertEqual(0, channel.json_body["total"])
self._access_media(server_and_media_id)
now_ms = self.clock.time_msec()
request, channel = self.make_request(
"POST",
self.url + "?before_ts=" + str(now_ms) + "&keep_profiles=false",
access_token=self.admin_user_tok,
)
self.render(request)
self.assertEqual(200, channel.code, msg=channel.json_body)
self.assertEqual(1, channel.json_body["total"])
self.assertEqual(
server_and_media_id.split("/")[1], channel.json_body["deleted_media"][0],
)
self._access_media(server_and_media_id, False)
def test_keep_media_by_room_avatar(self):
"""
Tests that we do not delete media if it is used as a room avatar
Tests parameter `keep_profiles`
"""
server_and_media_id = self._create_media()
self._access_media(server_and_media_id)
# set media as room avatar
room_id = self.helper.create_room_as(self.admin_user, tok=self.admin_user_tok)
request, channel = self.make_request(
"PUT",
"/rooms/%s/state/m.room.avatar" % (room_id,),
content=json.dumps({"url": "mxc://%s" % (server_and_media_id,)}),
access_token=self.admin_user_tok,
)
self.render(request)
self.assertEqual(200, channel.code, msg=channel.json_body)
now_ms = self.clock.time_msec()
request, channel = self.make_request(
"POST",
self.url + "?before_ts=" + str(now_ms) + "&keep_profiles=true",
access_token=self.admin_user_tok,
)
self.render(request)
self.assertEqual(200, channel.code, msg=channel.json_body)
self.assertEqual(0, channel.json_body["total"])
self._access_media(server_and_media_id)
now_ms = self.clock.time_msec()
request, channel = self.make_request(
"POST",
self.url + "?before_ts=" + str(now_ms) + "&keep_profiles=false",
access_token=self.admin_user_tok,
)
self.render(request)
self.assertEqual(200, channel.code, msg=channel.json_body)
self.assertEqual(1, channel.json_body["total"])
self.assertEqual(
server_and_media_id.split("/")[1], channel.json_body["deleted_media"][0],
)
self._access_media(server_and_media_id, False)
def _create_media(self):
"""
Create a media and return media_id and server_and_media_id
"""
upload_resource = self.media_repo.children[b"upload"]
# file size is 67 Byte
image_data = unhexlify(
b"89504e470d0a1a0a0000000d4948445200000001000000010806"
b"0000001f15c4890000000a49444154789c63000100000500010d"
b"0a2db40000000049454e44ae426082"
)
# Upload some media into the room
response = self.helper.upload_media(
upload_resource, image_data, tok=self.admin_user_tok, expect_code=200
)
# Extract media ID from the response
server_and_media_id = response["content_uri"][6:] # Cut off 'mxc://'
server_name = server_and_media_id.split("/")[0]
# Check that new media is a local and not remote
self.assertEqual(server_name, self.server_name)
return server_and_media_id
def _access_media(self, server_and_media_id, expect_success=True):
"""
Try to access a media and check the result
"""
download_resource = self.media_repo.children[b"download"]
media_id = server_and_media_id.split("/")[1]
local_path = self.filepaths.local_media_filepath(media_id)
request, channel = self.make_request(
"GET",
server_and_media_id,
shorthand=False,
access_token=self.admin_user_tok,
)
request.render(download_resource)
self.pump(1.0)
if expect_success:
self.assertEqual(
200,
channel.code,
msg=(
"Expected to receive a 200 on accessing media: %s"
% server_and_media_id
),
)
# Test that the file exists
self.assertTrue(os.path.exists(local_path))
else:
self.assertEqual(
404,
channel.code,
msg=(
"Expected to receive a 404 on accessing deleted media: %s"
% (server_and_media_id)
),
)
# Test that the file is deleted
self.assertFalse(os.path.exists(local_path))

16
tests/rest/admin/test_user.py
View File

@@ -1016,7 +1016,6 @@ class UserMembershipRestTestCase(unittest.HomeserverTestCase):
servlets = [
synapse.rest.admin.register_servlets,
login.register_servlets,
sync.register_servlets,
room.register_servlets,
]
@@ -1082,6 +1081,21 @@ class UserMembershipRestTestCase(unittest.HomeserverTestCase):
self.assertEqual(400, channel.code, msg=channel.json_body)
self.assertEqual("Can only lookup local users", channel.json_body["error"])
def test_no_memberships(self):
"""
Tests that a normal lookup for rooms is successfully
if user has no memberships
"""
# Get rooms
request, channel = self.make_request(
"GET", self.url, access_token=self.admin_user_tok,
)
self.render(request)
self.assertEqual(200, channel.code, msg=channel.json_body)
self.assertEqual(0, channel.json_body["total"])
self.assertEqual(0, len(channel.json_body["joined_rooms"]))
def test_get_rooms(self):
"""
Tests that a normal lookup for rooms is successfully