kaslo/synapse
1
0
Fork 0
Code Wiki Activity

Merge commit 'de5cafe98' into dinsic

Browse Source
This commit is contained in:
Andrew Morgan
2020-12-31 11:43:55 +00:00
parent 4c7d889be7 de5cafe980
commit 8f05de1d97
48 changed files with 317 additions and 233 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
changelog.d/8504.bugfix Normal file
View File

@@ -0,0 +1 @@
Expose the `uk.half-shot.msc2778.login.application_service` to clients from the login API. This feature was added in v1.21.0, but was not exposed as a potential login flow.

1
changelog.d/8545.bugfix Normal file
View File

@@ -0,0 +1 @@
Fix a long standing bug where email notifications for encrypted messages were blank.

1
changelog.d/8567.bugfix Normal file
View File

@@ -0,0 +1 @@
Fix increase in the number of `There was no active span...` errors logged when using OpenTracing.

1
changelog.d/8568.misc Normal file
View File

@@ -0,0 +1 @@
Add `get_immediate` method to `DeferredCache`.

1
changelog.d/8569.misc Normal file
View File

@@ -0,0 +1 @@
Fix mypy not properly checking across the codebase, additionally, fix a typing assertion error in `handlers/auth.py`.

1
changelog.d/8578.misc Normal file
View File

@@ -0,0 +1 @@
Support macOS on the `synmark` benchmark runner.

1
changelog.d/8583.misc Normal file
View File

@@ -0,0 +1 @@
Update `mypy` static type checker to 0.790.

1
changelog.d/8585.bugfix Normal file
View File

@@ -0,0 +1 @@
Fix a bug that prevented errors encountered during execution of the `synapse_port_db` from being correctly printed.

1
changelog.d/8589.removal Normal file
View File

@@ -0,0 +1 @@
Drop unused `device_max_stream_id` table.

1
changelog.d/8590.misc Normal file
View File

@@ -0,0 +1 @@
Implement [MSC2409](https://github.com/matrix-org/matrix-doc/pull/2409) to send typing, read receipts, and presence events to appservices.

1
changelog.d/8591.misc Normal file
View File

@@ -0,0 +1 @@
Move metric registration code down into `LruCache`.

1
changelog.d/8592.misc Normal file
View File

@@ -0,0 +1 @@
Remove extraneous unittest logging decorators from unit tests.

1
changelog.d/8599.feature Normal file
View File

@@ -0,0 +1 @@
Allow running background tasks in a separate worker process.

1
changelog.d/8609.misc Normal file
View File

@@ -0,0 +1 @@
Add type hints to profile and base handler.

4
mypy.ini
View File

@@ -15,8 +15,9 @@ files =
synapse/events/builder.py,
synapse/events/spamcheck.py,
synapse/federation,
synapse/handlers/appservice.py,
synapse/handlers/_base.py,
synapse/handlers/account_data.py,
synapse/handlers/appservice.py,
synapse/handlers/auth.py,
synapse/handlers/cas_handler.py,
synapse/handlers/deactivate_account.py,
@@ -32,6 +33,7 @@ files =
synapse/handlers/pagination.py,
synapse/handlers/password_policy.py,
synapse/handlers/presence.py,
synapse/handlers/profile.py,
synapse/handlers/read_marker.py,
synapse/handlers/room.py,
synapse/handlers/room_member.py,

5
scripts/synapse_port_db
View File

@@ -22,6 +22,7 @@ import logging
import sys
import time
import traceback
from typing import Optional
import yaml
@@ -153,7 +154,7 @@ IGNORED_TABLES = {
# Error returned by the run function. Used at the top-level part of the script to
# handle errors and return codes.
end_error = None
end_error = None # type: Optional[str]
# The exec_info for the error, if any. If error is defined but not exec_info the script
# will show only the error message without the stacktrace, if exec_info is defined but
# not the error then the script will show nothing outside of what's printed in the run
@@ -637,7 +638,7 @@ class Porter(object):
self.progress.done()
except Exception as e:
global end_error_exec_info
end_error = e
end_error = str(e)
end_error_exec_info = sys.exc_info()
logger.exception("")
finally:

2
setup.py
View File

@@ -102,6 +102,8 @@ CONDITIONAL_REQUIREMENTS["lint"] = [
"flake8",
]
CONDITIONAL_REQUIREMENTS["mypy"] = ["mypy==0.790", "mypy-zope"]
# Dependencies which are exclusively required by unit test code. This is
# NOT a list of all modules that are necessary to run the unit tests.
# Tests assume that all optional dependencies are installed.

20
synapse/handlers/_base.py
View File

@@ -14,6 +14,7 @@
# limitations under the License.
import logging
from typing import TYPE_CHECKING, Optional
import synapse.state
import synapse.storage
@@ -22,6 +23,9 @@ from synapse.api.constants import EventTypes, Membership
from synapse.api.ratelimiting import Ratelimiter
from synapse.types import UserID
if TYPE_CHECKING:
from synapse.app.homeserver import HomeServer
logger = logging.getLogger(__name__)
@@ -30,11 +34,7 @@ class BaseHandler:
Common base class for the event handlers.
"""
def __init__(self, hs):
"""
Args:
hs (synapse.server.HomeServer):
"""
def __init__(self, hs: "HomeServer"):
self.store = hs.get_datastore() # type: synapse.storage.DataStore
self.auth = hs.get_auth()
self.notifier = hs.get_notifier()
@@ -56,7 +56,7 @@ class BaseHandler:
clock=self.clock,
rate_hz=self.hs.config.rc_admin_redaction.per_second,
burst_count=self.hs.config.rc_admin_redaction.burst_count,
)
) # type: Optional[Ratelimiter]
else:
self.admin_redaction_ratelimiter = None
@@ -127,15 +127,15 @@ class BaseHandler:
if guest_access != "can_join":
if context:
current_state_ids = await context.get_current_state_ids()
current_state = await self.store.get_events(
current_state_dict = await self.store.get_events(
list(current_state_ids.values())
)
current_state = list(current_state_dict.values())
else:
current_state = await self.state_handler.get_current_state(
current_state_map = await self.state_handler.get_current_state(
event.room_id
)
current_state = list(current_state.values())
current_state = list(current_state_map.values())
logger.info("maybe_kick_guest_users %r", current_state)
await self.kick_guest_users(current_state)

15
synapse/handlers/account_validity.py
View File

@@ -22,7 +22,10 @@ from typing import List, Optional, Tuple
from synapse.api.errors import StoreError
from synapse.logging.context import make_deferred_yieldable
from synapse.metrics.background_process_metrics import run_as_background_process
from synapse.metrics.background_process_metrics import (
run_as_background_process,
wrap_as_background_process,
)
from synapse.types import UserID
from synapse.util import stringutils
@@ -73,15 +76,8 @@ class AccountValidityHandler:
self._raw_from = email.utils.parseaddr(self._from_string)[1]
# Check the renewal emails to send and send them every 30min.
def send_emails():
# run as a background process to make sure that the database transactions
# have a logcontext to report to
return run_as_background_process(
"send_renewals", self._send_renewal_emails
)
if hs.config.run_background_tasks:
self.clock.looping_call(send_emails, 30 * 60 * 1000)
self.clock.looping_call(self._send_renewal_emails, 30 * 60 * 1000)
# Mark users as inactive when they expired. Check once every hour
if self._account_validity_enabled:
@@ -95,6 +91,7 @@ class AccountValidityHandler:
self.clock.looping_call(mark_expired_users_as_inactive, 60 * 60 * 1000)
@wrap_as_background_process("send_renewals")
async def _send_renewal_emails(self):
"""Gets the list of users whose account is expiring in the amount of time
configured in the ``renew_at`` parameter from the ``account_validity``

8
synapse/handlers/initial_sync.py
View File

@@ -293,6 +293,10 @@ class InitialSyncHandler(BaseHandler):
user_id, room_id, pagin_config, membership, is_peeking
)
elif membership == Membership.LEAVE:
# The member_event_id will always be available if membership is set
# to leave.
assert member_event_id
result = await self._room_initial_sync_parted(
user_id, room_id, pagin_config, membership, member_event_id, is_peeking
)
@@ -315,7 +319,7 @@ class InitialSyncHandler(BaseHandler):
user_id: str,
room_id: str,
pagin_config: PaginationConfig,
membership: Membership,
membership: str,
member_event_id: str,
is_peeking: bool,
) -> JsonDict:
@@ -367,7 +371,7 @@ class InitialSyncHandler(BaseHandler):
user_id: str,
room_id: str,
pagin_config: PaginationConfig,
membership: Membership,
membership: str,
is_peeking: bool,
) -> JsonDict:
current_state = await self.state.get_current_state(room_id=room_id)

86
synapse/handlers/profile.py
View File

@@ -13,10 +13,9 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import logging
import random
from typing import List
from typing import TYPE_CHECKING, List, Optional
from signedjson.sign import sign_json
@@ -31,11 +30,23 @@ from synapse.api.errors import (
SynapseError,
)
from synapse.logging.context import run_in_background
from synapse.metrics.background_process_metrics import run_as_background_process
from synapse.types import UserID, create_requester, get_domain_from_id
from synapse.metrics.background_process_metrics import (
run_as_background_process,
wrap_as_background_process,
)
from synapse.types import (
JsonDict,
Requester,
UserID,
create_requester,
get_domain_from_id,
)
from ._base import BaseHandler
if TYPE_CHECKING:
from synapse.app.homeserver import HomeServer
logger = logging.getLogger(__name__)
MAX_DISPLAYNAME_LEN = 256
@@ -54,7 +65,7 @@ class ProfileHandler(BaseHandler):
PROFILE_REPLICATE_INTERVAL = 2 * 60 * 1000
def __init__(self, hs):
def __init__(self, hs: "HomeServer"):
super().__init__(hs)
self.federation = hs.get_federation_client()
@@ -72,7 +83,7 @@ class ProfileHandler(BaseHandler):
if hs.config.run_background_tasks:
self.clock.looping_call(
self._start_update_remote_profile_cache, self.PROFILE_UPDATE_MS
self._update_remote_profile_cache, self.PROFILE_UPDATE_MS
)
if len(self.hs.config.replicate_user_profiles_to) > 0:
@@ -156,7 +167,7 @@ class ProfileHandler(BaseHandler):
)
raise
async def get_profile(self, user_id):
async def get_profile(self, user_id: str) -> JsonDict:
target_user = UserID.from_string(user_id)
if self.hs.is_mine(target_user):
@@ -187,7 +198,7 @@ class ProfileHandler(BaseHandler):
except HttpResponseException as e:
raise e.to_synapse_error()
async def get_profile_from_cache(self, user_id):
async def get_profile_from_cache(self, user_id: str) -> JsonDict:
"""Get the profile information from our local cache. If the user is
ours then the profile information will always be corect. Otherwise,
it may be out of date/missing.
@@ -211,7 +222,7 @@ class ProfileHandler(BaseHandler):
profile = await self.store.get_from_remote_profile_cache(user_id)
return profile or {}
async def get_displayname(self, target_user):
async def get_displayname(self, target_user: UserID) -> str:
if self.hs.is_mine(target_user):
try:
displayname = await self.store.get_profile_displayname(
@@ -239,15 +250,19 @@ class ProfileHandler(BaseHandler):
return result["displayname"]
async def set_displayname(
self, target_user, requester, new_displayname, by_admin=False
):
self,
target_user: UserID,
requester: Requester,
new_displayname: str,
by_admin: bool = False,
) -> None:
"""Set the displayname of a user
Args:
target_user (UserID): the user whose displayname is to be changed.
requester (Requester): The user attempting to make this change.
new_displayname (str): The displayname to give this user.
by_admin (bool): Whether this change was made by an administrator.
target_user: the user whose displayname is to be changed.
requester: The user attempting to make this change.
new_displayname: The displayname to give this user.
by_admin: Whether this change was made by an administrator.
"""
if not self.hs.is_mine(target_user):
raise SynapseError(400, "User is not hosted on this homeserver")
@@ -272,8 +287,9 @@ class ProfileHandler(BaseHandler):
400, "Displayname is too long (max %i)" % (MAX_DISPLAYNAME_LEN,)
)
displayname_to_set = new_displayname # type: Optional[str]
if new_displayname == "":
new_displayname = None
displayname_to_set = None
if len(self.hs.config.replicate_user_profiles_to) > 0:
cur_batchnum = (
@@ -290,7 +306,7 @@ class ProfileHandler(BaseHandler):
requester = create_requester(target_user)
await self.store.set_profile_displayname(
target_user.localpart, new_displayname, new_batchnum
target_user.localpart, displayname_to_set, new_batchnum
)
if self.hs.config.user_directory_search_all_users:
@@ -341,7 +357,7 @@ class ProfileHandler(BaseHandler):
# start a profile replication push
run_in_background(self._replicate_profiles)
async def get_avatar_url(self, target_user):
async def get_avatar_url(self, target_user: UserID) -> str:
if self.hs.is_mine(target_user):
try:
avatar_url = await self.store.get_profile_avatar_url(
@@ -368,15 +384,19 @@ class ProfileHandler(BaseHandler):
return result["avatar_url"]
async def set_avatar_url(
self, target_user, requester, new_avatar_url, by_admin=False
self,
target_user: UserID,
requester: Requester,
new_avatar_url: str,
by_admin: bool = False,
):
"""Set a new avatar URL for a user.
Args:
target_user (UserID): the user whose avatar URL is to be changed.
requester (Requester): The user attempting to make this change.
new_avatar_url (str): The avatar URL to give this user.
by_admin (bool): Whether this change was made by an administrator.
target_user: the user whose avatar URL is to be changed.
requester: The user attempting to make this change.
new_avatar_url: The avatar URL to give this user.
by_admin: Whether this change was made by an administrator.
"""
if not self.hs.is_mine(target_user):
raise SynapseError(400, "User is not hosted on this homeserver")
@@ -470,7 +490,7 @@ class ProfileHandler(BaseHandler):
raise SynapseError(400, "Invalid avatar URL '%s' supplied" % mxc)
return avatar_pieces[-1]
async def on_profile_query(self, args):
async def on_profile_query(self, args: JsonDict) -> JsonDict:
user = UserID.from_string(args["user_id"])
if not self.hs.is_mine(user):
raise SynapseError(400, "User is not hosted on this homeserver")
@@ -495,7 +515,9 @@ class ProfileHandler(BaseHandler):
return response
async def _update_join_states(self, requester, target_user):
async def _update_join_states(
self, requester: Requester, target_user: UserID
) -> None:
if not self.hs.is_mine(target_user):
return
@@ -526,15 +548,17 @@ class ProfileHandler(BaseHandler):
"Failed to update join event for room %s - %s", room_id, str(e)
)
async def check_profile_query_allowed(self, target_user, requester=None):
async def check_profile_query_allowed(
self, target_user: UserID, requester: Optional[UserID] = None
) -> None:
"""Checks whether a profile query is allowed. If the
'require_auth_for_profile_requests' config flag is set to True and a
'requester' is provided, the query is only allowed if the two users
share a room.
Args:
target_user (UserID): The owner of the queried profile.
requester (None|UserID): The user querying for the profile.
target_user: The owner of the queried profile.
requester: The user querying for the profile.
Raises:
SynapseError(403): The two users share no room, or ne user couldn't
@@ -573,11 +597,7 @@ class ProfileHandler(BaseHandler):
raise SynapseError(403, "Profile isn't available", Codes.FORBIDDEN)
raise
def _start_update_remote_profile_cache(self):
return run_as_background_process(
"Update remote profile", self._update_remote_profile_cache
)
@wrap_as_background_process("Update remote profile")
async def _update_remote_profile_cache(self):
"""Called periodically to check profiles of remote users we haven't
checked in a while.

11
synapse/metrics/background_process_metrics.py
View File

@@ -24,6 +24,7 @@ from prometheus_client.core import REGISTRY, Counter, Gauge
from twisted.internet import defer
from synapse.logging.context import LoggingContext, PreserveLoggingContext
from synapse.logging.opentracing import start_active_span
if TYPE_CHECKING:
import resource
@@ -197,14 +198,14 @@ def run_as_background_process(desc: str, func, *args, **kwargs):
with BackgroundProcessLoggingContext(desc) as context:
context.request = "%s-%i" % (desc, count)
try:
result = func(*args, **kwargs)
with start_active_span(desc, tags={"request_id": context.request}):
result = func(*args, **kwargs)
if inspect.isawaitable(result):
result = await result
if inspect.isawaitable(result):
result = await result
return result
return result
except Exception:
logger.exception(
"Background process '%s' threw an exception", desc,

2
synapse/push/bulk_push_rule_evaluator.py
View File

@@ -496,6 +496,6 @@ class _Invalidation(namedtuple("_Invalidation", ("cache", "room_id"))):
# dedupe when we add callbacks to lru cache nodes, otherwise the number
# of callbacks would grow.
def __call__(self):
rules = self.cache.get(self.room_id, None, update_metrics=False)
rules = self.cache.get_immediate(self.room_id, None, update_metrics=False)
if rules:
rules.invalidate_all()

16
synapse/push/mailer.py
View File

@@ -387,8 +387,8 @@ class Mailer:
return ret
async def get_message_vars(self, notif, event, room_state_ids):
if event.type != EventTypes.Message:
return
if event.type != EventTypes.Message and event.type != EventTypes.Encrypted:
return None
sender_state_event_id = room_state_ids[("m.room.member", event.sender)]
sender_state_event = await self.store.get_event(sender_state_event_id)
@@ -399,10 +399,8 @@ class Mailer:
# sender_hash % the number of default images to choose from
sender_hash = string_ordinal_total(event.sender)
msgtype = event.content.get("msgtype")
ret = {
"msgtype": msgtype,
"event_type": event.type,
"is_historical": event.event_id != notif["event_id"],
"id": event.event_id,
"ts": event.origin_server_ts,
@@ -411,6 +409,14 @@ class Mailer:
"sender_hash": sender_hash,
}
# Encrypted messages don't have any additional useful information.
if event.type == EventTypes.Encrypted:
return ret
msgtype = event.content.get("msgtype")
ret["msgtype"] = msgtype
if msgtype == "m.text":
self.add_text_message_vars(ret, event)
elif msgtype == "m.image":

56
synapse/res/templates/notif.html
View File

@@ -1,41 +1,47 @@
{% for message in notif.messages %}
{%- for message in notif.messages %}
<tr class="{{ "historical_message" if message.is_historical else "message" }}">
<td class="sender_avatar">
{% if loop.index0 == 0 or notif.messages[loop.index0 - 1].sender_name != notif.messages[loop.index0].sender_name %}
{% if message.sender_avatar_url %}
{%- if loop.index0 == 0 or notif.messages[loop.index0 - 1].sender_name != notif.messages[loop.index0].sender_name %}
{%- if message.sender_avatar_url %}
<img alt="" class="sender_avatar" src="{{ message.sender_avatar_url|mxc_to_http(32,32) }}" />
{% else %}
{% if message.sender_hash % 3 == 0 %}
{%- else %}
{%- if message.sender_hash % 3 == 0 %}
<img class="sender_avatar" src="https://riot.im/img/external/avatar-1.png" />
{% elif message.sender_hash % 3 == 1 %}
{%- elif message.sender_hash % 3 == 1 %}
<img class="sender_avatar" src="https://riot.im/img/external/avatar-2.png" />
{% else %}
{%- else %}
<img class="sender_avatar" src="https://riot.im/img/external/avatar-3.png" />
{% endif %}
{% endif %}
{% endif %}
{%- endif %}
{%- endif %}
{%- endif %}
</td>
<td class="message_contents">
{% if loop.index0 == 0 or notif.messages[loop.index0 - 1].sender_name != notif.messages[loop.index0].sender_name %}
<div class="sender_name">{% if message.msgtype == "m.emote" %}*{% endif %} {{ message.sender_name }}</div>
{% endif %}
{%- if loop.index0 == 0 or notif.messages[loop.index0 - 1].sender_name != notif.messages[loop.index0].sender_name %}
<div class="sender_name">{%- if message.msgtype == "m.emote" %}*{%- endif %} {{ message.sender_name }}</div>
{%- endif %}
<div class="message_body">
{% if message.msgtype == "m.text" %}
{{ message.body_text_html }}
{% elif message.msgtype == "m.emote" %}
{{ message.body_text_html }}
{% elif message.msgtype == "m.notice" %}
{{ message.body_text_html }}
{% elif message.msgtype == "m.image" %}
<img src="{{ message.image_url|mxc_to_http(640, 480, scale) }}" />
{% elif message.msgtype == "m.file" %}
<span class="filename">{{ message.body_text_plain }}</span>
{% endif %}
{%- if message.event_type == "m.room.encrypted" %}
An encrypted message.
{%- elif message.event_type == "m.room.message" %}
{%- if message.msgtype == "m.text" %}
{{ message.body_text_html }}
{%- elif message.msgtype == "m.emote" %}
{{ message.body_text_html }}
{%- elif message.msgtype == "m.notice" %}
{{ message.body_text_html }}
{%- elif message.msgtype == "m.image" %}
<img src="{{ message.image_url|mxc_to_http(640, 480, scale) }}" />
{%- elif message.msgtype == "m.file" %}
<span class="filename">{{ message.body_text_plain }}</span>
{%- else %}
A message with unrecognised content.
{%- endif %}
{%- endif %}
</div>
</td>
<td class="message_time">{{ message.ts|format_ts("%H:%M") }}</td>
</tr>
{% endfor %}
{%- endfor %}
<tr class="notif_link">
<td></td>
<td>

24
synapse/res/templates/notif.txt
View File

@@ -1,16 +1,22 @@
{% for message in notif.messages %}
{% if message.msgtype == "m.emote" %}* {% endif %}{{ message.sender_name }} ({{ message.ts|format_ts("%H:%M") }})
{% if message.msgtype == "m.text" %}
{%- for message in notif.messages %}
{%- if message.event_type == "m.room.encrypted" %}
An encrypted message.
{%- elif message.event_type == "m.room.message" %}
{%- if message.msgtype == "m.emote" %}* {%- endif %}{{ message.sender_name }} ({{ message.ts|format_ts("%H:%M") }})
{%- if message.msgtype == "m.text" %}
{{ message.body_text_plain }}
{% elif message.msgtype == "m.emote" %}
{%- elif message.msgtype == "m.emote" %}
{{ message.body_text_plain }}
{% elif message.msgtype == "m.notice" %}
{%- elif message.msgtype == "m.notice" %}
{{ message.body_text_plain }}
{% elif message.msgtype == "m.image" %}
{%- elif message.msgtype == "m.image" %}
{{ message.body_text_plain }}
{% elif message.msgtype == "m.file" %}
{%- elif message.msgtype == "m.file" %}
{{ message.body_text_plain }}
{% endif %}
{% endfor %}
{%- else %}
A message with unrecognised content.
{%- endif %}
{%- endif %}
{%- endfor %}
View {{ room.title }} at {{ notif.link }}

26
synapse/res/templates/notif_mail.html
View File

@@ -2,8 +2,8 @@
<html lang="en">
<head>
<style type="text/css">
{% include 'mail.css' without context %}
{% include "mail-%s.css" % app_name ignore missing without context %}
{%- include 'mail.css' without context %}
{%- include "mail-%s.css" % app_name ignore missing without context %}
</style>
</head>
<body>
@@ -18,21 +18,21 @@
<div class="summarytext">{{ summary_text }}</div>
</td>
<td class="logo">
{% if app_name == "Riot" %}
{%- if app_name == "Riot" %}
<img src="http://riot.im/img/external/riot-logo-email.png" width="83" height="83" alt="[Riot]"/>
{% elif app_name == "Vector" %}
{%- elif app_name == "Vector" %}
<img src="http://matrix.org/img/vector-logo-email.png" width="64" height="83" alt="[Vector]"/>
{% elif app_name == "Element" %}
{%- elif app_name == "Element" %}
<img src="https://static.element.io/images/email-logo.png" width="83" height="83" alt="[Element]"/>
{% else %}
{%- else %}
<img src="http://matrix.org/img/matrix-120x51.png" width="120" height="51" alt="[matrix]"/>
{% endif %}
{%- endif %}
</td>
</tr>
</table>
{% for room in rooms %}
{% include 'room.html' with context %}
{% endfor %}
{%- for room in rooms %}
{%- include 'room.html' with context %}
{%- endfor %}
<div class="footer">
<a href="{{ unsubscribe_link }}">Unsubscribe</a>
<br/>
@@ -41,12 +41,12 @@
Sending email at {{ reason.now|format_ts("%c") }} due to activity in room {{ reason.room_name }} because
an event was received at {{ reason.received_at|format_ts("%c") }}
which is more than {{ "%.1f"|format(reason.delay_before_mail_ms / (60*1000)) }} ({{ reason.delay_before_mail_ms }}) mins ago,
{% if reason.last_sent_ts %}
{%- if reason.last_sent_ts %}
and the last time we sent a mail for this room was {{ reason.last_sent_ts|format_ts("%c") }},
which is more than {{ "%.1f"|format(reason.throttle_ms / (60*1000)) }} (current throttle_ms) mins ago.
{% else %}
{%- else %}
and we don't have a last time we sent a mail for this room.
{% endif %}
{%- endif %}
</div>
</div>
</td>

6
synapse/res/templates/notif_mail.txt
View File

@@ -2,9 +2,9 @@ Hi {{ user_display_name }},
{{ summary_text }}
{% for room in rooms %}
{% include 'room.txt' with context %}
{% endfor %}
{%- for room in rooms %}
{%- include 'room.txt' with context %}
{%- endfor %}
You can disable these notifications at {{ unsubscribe_link }}

26
synapse/res/templates/room.html
View File

@@ -1,23 +1,23 @@
<table class="room">
<tr class="room_header">
<td class="room_avatar">
{% if room.avatar_url %}
{%- if room.avatar_url %}
<img alt="" src="{{ room.avatar_url|mxc_to_http(48,48) }}" />
{% else %}
{% if room.hash % 3 == 0 %}
{%- else %}
{%- if room.hash % 3 == 0 %}
<img alt="" src="https://riot.im/img/external/avatar-1.png" />
{% elif room.hash % 3 == 1 %}
{%- elif room.hash % 3 == 1 %}
<img alt="" src="https://riot.im/img/external/avatar-2.png" />
{% else %}
{%- else %}
<img alt="" src="https://riot.im/img/external/avatar-3.png" />
{% endif %}
{% endif %}
{%- endif %}
{%- endif %}
</td>
<td class="room_name" colspan="2">
{{ room.title }}
</td>
</tr>
{% if room.invite %}
{%- if room.invite %}
<tr>
<td></td>
<td>
@@ -25,9 +25,9 @@
</td>
<td></td>
</tr>
{% else %}
{% for notif in room.notifs %}
{% include 'notif.html' with context %}
{% endfor %}
{% endif %}
{%- else %}
{%- for notif in room.notifs %}
{%- include 'notif.html' with context %}
{%- endfor %}
{%- endif %}
</table>

12
synapse/res/templates/room.txt
View File

@@ -1,9 +1,9 @@
{{ room.title }}
{% if room.invite %}
{%- if room.invite %}
You've been invited, join at {{ room.link }}
{% else %}
{% for notif in room.notifs %}
{% include 'notif.txt' with context %}
{% endfor %}
{% endif %}
{%- else %}
{%- for notif in room.notifs %}
{%- include 'notif.txt' with context %}
{%- endfor %}
{%- endif %}

2
synapse/rest/client/v1/login.py
View File

@@ -110,6 +110,8 @@ class LoginRestServlet(RestServlet):
({"type": t} for t in self.auth_handler.get_supported_login_types())
)
flows.append({"type": LoginRestServlet.APPSERVICE_TYPE})
return 200, {"flows": flows}
def on_OPTIONS(self, request: SynapseRequest):

11
synapse/storage/databases/main/events_worker.py
View File

@@ -33,7 +33,10 @@ from synapse.api.room_versions import (
from synapse.events import EventBase, make_event_from_dict
from synapse.events.utils import prune_event
from synapse.logging.context import PreserveLoggingContext, current_context
from synapse.metrics.background_process_metrics import run_as_background_process
from synapse.metrics.background_process_metrics import (
run_as_background_process,
wrap_as_background_process,
)
from synapse.replication.slave.storage._slaved_id_tracker import SlavedIdTracker
from synapse.replication.tcp.streams import BackfillStream
from synapse.replication.tcp.streams.events import EventsStream
@@ -140,10 +143,7 @@ class EventsWorkerStore(SQLBaseStore):
if hs.config.run_background_tasks:
# We periodically clean out old transaction ID mappings
self._clock.looping_call(
run_as_background_process,
5 * 60 * 1000,
"_cleanup_old_transaction_ids",
self._cleanup_old_transaction_ids,
self._cleanup_old_transaction_ids, 5 * 60 * 1000,
)
self._get_event_cache = LruCache(
@@ -1374,6 +1374,7 @@ class EventsWorkerStore(SQLBaseStore):
return mapping
@wrap_as_background_process("_cleanup_old_transaction_ids")
async def _cleanup_old_transaction_ids(self):
"""Cleans out transaction id mappings older than 24hrs.
"""

4
synapse/storage/databases/main/profile.py
View File

@@ -131,7 +131,7 @@ class ProfileWorkerStore(SQLBaseStore):
)
async def set_profile_displayname(
self, user_localpart: str, new_displayname: str, batchnum: int
self, user_localpart: str, new_displayname: Optional[str], batchnum: int
) -> None:
# Invalidate the read cache for this user
self.get_profile_displayname.invalidate((user_localpart,))
@@ -266,7 +266,7 @@ class ProfileWorkerStore(SQLBaseStore):
async def get_remote_profile_cache_entries_that_expire(
self, last_checked: int
) -> Dict[str, str]:
) -> List[Dict[str, str]]:
"""Get all users who haven't been checked since `last_checked`
"""

2
synapse/storage/databases/main/pusher.py
View File

@@ -303,7 +303,7 @@ class PusherStore(PusherWorkerStore):
lock=False,
)
user_has_pusher = self.get_if_user_has_pusher.cache.get(
user_has_pusher = self.get_if_user_has_pusher.cache.get_immediate(
(user_id,), None, update_metrics=False
)

11
synapse/storage/databases/main/receipts.py
View File

@@ -25,7 +25,6 @@ from synapse.storage.database import DatabasePool
from synapse.storage.util.id_generators import StreamIdGenerator
from synapse.types import JsonDict
from synapse.util import json_encoder
from synapse.util.async_helpers import ObservableDeferred
from synapse.util.caches.descriptors import cached, cachedList
from synapse.util.caches.stream_change_cache import StreamChangeCache
@@ -413,18 +412,10 @@ class ReceiptsWorkerStore(SQLBaseStore, metaclass=abc.ABCMeta):
if receipt_type != "m.read":
return
# Returns either an ObservableDeferred or the raw result
res = self.get_users_with_read_receipts_in_room.cache.get(
res = self.get_users_with_read_receipts_in_room.cache.get_immediate(
room_id, None, update_metrics=False
)
# first handle the ObservableDeferred case
if isinstance(res, ObservableDeferred):
if res.has_called():
res = res.get_result()
else:
res = None
if res and user_id in res:
# We'd only be adding to the set, so no point invalidating if the
# user is already there

18
synapse/storage/databases/main/roommember.py
View File

@@ -20,7 +20,10 @@ from synapse.api.constants import EventTypes, Membership
from synapse.events import EventBase
from synapse.events.snapshot import EventContext
from synapse.metrics import LaterGauge
from synapse.metrics.background_process_metrics import run_as_background_process
from synapse.metrics.background_process_metrics import (
run_as_background_process,
wrap_as_background_process,
)
from synapse.storage._base import SQLBaseStore, db_to_json, make_in_list_sql_clause
from synapse.storage.database import DatabasePool
from synapse.storage.databases.main.events_worker import EventsWorkerStore
@@ -67,16 +70,10 @@ class RoomMemberWorkerStore(EventsWorkerStore):
):
self._known_servers_count = 1
self.hs.get_clock().looping_call(
run_as_background_process,
60 * 1000,
"_count_known_servers",
self._count_known_servers,
self._count_known_servers, 60 * 1000,
)
self.hs.get_clock().call_later(
1000,
run_as_background_process,
"_count_known_servers",
self._count_known_servers,
1000, self._count_known_servers,
)
LaterGauge(
"synapse_federation_known_servers",
@@ -85,6 +82,7 @@ class RoomMemberWorkerStore(EventsWorkerStore):
lambda: self._known_servers_count,
)
@wrap_as_background_process("_count_known_servers")
async def _count_known_servers(self):
"""
Count the servers that this server knows about.
@@ -531,7 +529,7 @@ class RoomMemberWorkerStore(EventsWorkerStore):
# If we do then we can reuse that result and simply update it with
# any membership changes in `delta_ids`
if context.prev_group and context.delta_ids:
prev_res = self._get_joined_users_from_context.cache.get(
prev_res = self._get_joined_users_from_context.cache.get_immediate(
(room_id, context.prev_group), None
)
if prev_res and isinstance(prev_res, dict):

5
synapse/storage/databases/main/schema/delta/59/19as_device_stream.sql → synapse/storage/databases/main/schema/delta/58/21as_device_stream.sql
View File

@@ -13,6 +13,5 @@
* limitations under the License.
*/
ALTER TABLE application_services_state
ADD COLUMN read_receipt_stream_id INT,
ADD COLUMN presence_stream_id INT;
ALTER TABLE application_services_state ADD COLUMN read_receipt_stream_id INT;
ALTER TABLE application_services_state ADD COLUMN presence_stream_id INT;

1
synapse/storage/databases/main/schema/delta/58/21drop_device_max_stream_id.sql Normal file
View File

@@ -0,0 +1 @@
DROP TABLE device_max_stream_id;

35
synapse/util/caches/deferred_cache.py
View File

@@ -17,7 +17,16 @@
import enum
import threading
from typing import Callable, Generic, Iterable, MutableMapping, Optional, TypeVar, cast
from typing import (
Callable,
Generic,
Iterable,
MutableMapping,
Optional,
TypeVar,
Union,
cast,
)
from prometheus_client import Gauge
@@ -33,7 +42,7 @@ cache_pending_metric = Gauge(
["name"],
)
T = TypeVar("T")
KT = TypeVar("KT")
VT = TypeVar("VT")
@@ -119,21 +128,21 @@ class DeferredCache(Generic[KT, VT]):
def get(
self,
key: KT,
default=_Sentinel.sentinel,
callback: Optional[Callable[[], None]] = None,
update_metrics: bool = True,
):
) -> Union[ObservableDeferred, VT]:
"""Looks the key up in the caches.
Args:
key(tuple)
default: What is returned if key is not in the caches. If not
specified then function throws KeyError instead
callback(fn): Gets called when the entry in the cache is invalidated
update_metrics (bool): whether to update the cache hit rate metrics
Returns:
Either an ObservableDeferred or the result itself
Raises:
KeyError if the key is not found in the cache
"""
callbacks = [callback] if callback else []
val = self._pending_deferred_cache.get(key, _Sentinel.sentinel)
@@ -145,13 +154,19 @@ class DeferredCache(Generic[KT, VT]):
m.inc_hits()
return val.deferred
val = self.cache.get(
key, default, callbacks=callbacks, update_metrics=update_metrics
val2 = self.cache.get(
key, _Sentinel.sentinel, callbacks=callbacks, update_metrics=update_metrics
)
if val is _Sentinel.sentinel:
if val2 is _Sentinel.sentinel:
raise KeyError()
else:
return val
return val2
def get_immediate(
self, key: KT, default: T, update_metrics: bool = True
) -> Union[VT, T]:
"""If we have a *completed* cached value, return it."""
return self.cache.get(key, default, update_metrics=update_metrics)
def set(
self,

10
synapse/util/caches/lrucache.py
View File

@@ -124,6 +124,10 @@ class LruCache(Generic[KT, VT]):
else:
self.max_size = int(max_size)
# register_cache might call our "set_cache_factor" callback; there's nothing to
# do yet when we get resized.
self._on_resize = None # type: Optional[Callable[[],None]]
if cache_name is not None:
metrics = register_cache(
"lru_cache",
@@ -332,7 +336,10 @@ class LruCache(Generic[KT, VT]):
return key in cache
self.sentinel = object()
# make sure that we clear out any excess entries after we get resized.
self._on_resize = evict
self.get = cache_get
self.set = cache_set
self.setdefault = cache_set_default
@@ -383,6 +390,7 @@ class LruCache(Generic[KT, VT]):
new_size = int(self._original_max_size * factor)
if new_size != self.max_size:
self.max_size = new_size
self._on_resize()
if self._on_resize:
self._on_resize()
return True
return False

7
synmark/__init__.py
View File

@@ -15,7 +15,10 @@
import sys
from twisted.internet import epollreactor
try:
from twisted.internet.epollreactor import EPollReactor as Reactor
except ImportError:
from twisted.internet.pollreactor import PollReactor as Reactor
from twisted.internet.main import installReactor
from synapse.config.homeserver import HomeServerConfig
@@ -63,7 +66,7 @@ def make_reactor():
Instantiate and install a Twisted reactor suitable for testing (i.e. not the
default global one).
"""
reactor = epollreactor.EPollReactor()
reactor = Reactor()
if "twisted.internet.reactor" in sys.modules:
del sys.modules["twisted.internet.reactor"]

15
tests/push/test_email.py
View File

@@ -158,8 +158,21 @@ class EmailPusherTests(HomeserverTestCase):
# We should get emailed about those messages
self._check_for_mail()
def test_encrypted_message(self):
room = self.helper.create_room_as(self.user_id, tok=self.access_token)
self.helper.invite(
room=room, src=self.user_id, tok=self.access_token, targ=self.others[0].id
)
self.helper.join(room=room, user=self.others[0].id, tok=self.others[0].token)
# The other user sends some messages
self.helper.send_event(room, "m.room.encrypted", {}, tok=self.others[0].token)
# We should get emailed about that message
self._check_for_mail()
def _check_for_mail(self):
"Check that the user receives an email notification"
"""Check that the user receives an email notification"""
# Get the stream ordering before it gets sent
pushers = self.get_success(

54
tests/rest/client/test_room_access_rules.py
View File

@@ -22,6 +22,7 @@ from mock import Mock
from twisted.internet import defer
from synapse.api.constants import EventTypes, JoinRules, Membership, RoomCreationPreset
from synapse.api.errors import SynapseError
from synapse.rest import admin
from synapse.rest.client.v1 import directory, login, room
from synapse.third_party_rules.access_rules import (
@@ -783,8 +784,8 @@ class RoomAccessTestCase(unittest.HomeserverTestCase):
allowed_requester = create_requester("@user:allowed_domain")
forbidden_requester = create_requester("@user:forbidden_domain")
# Create a join event for a forbidden user
forbidden_join_event, forbidden_join_event_context = self.get_success(
# Assert a join event from a forbidden user to a restricted room is rejected
self.get_failure(
event_creator.create_event(
forbidden_requester,
{
@@ -794,11 +795,12 @@ class RoomAccessTestCase(unittest.HomeserverTestCase):
"content": {"membership": Membership.JOIN},
"state_key": forbidden_requester.user.to_string(),
},
)
),
SynapseError,
)
# Create a join event for an allowed user
allowed_join_event, allowed_join_event_context = self.get_success(
# A join event from an non-forbidden user to a restricted room is allowed
self.get_success(
event_creator.create_event(
allowed_requester,
{
@@ -811,26 +813,10 @@ class RoomAccessTestCase(unittest.HomeserverTestCase):
)
)
# Assert a join event from a forbidden user to a restricted room is rejected
can_join = self.get_success(
self.third_party_event_rules.check_event_allowed(
forbidden_join_event, forbidden_join_event_context
)
)
self.assertFalse(can_join)
# But a join event from an non-forbidden user to a restricted room is allowed
can_join = self.get_success(
self.third_party_event_rules.check_event_allowed(
allowed_join_event, allowed_join_event_context
)
)
self.assertTrue(can_join)
# Test that forbidden users can only join unrestricted rooms if they have an invite
# Recreate the forbidden join event for the unrestricted room instead
forbidden_join_event, forbidden_join_event_context = self.get_success(
# A forbidden user without an invite should not be able to join an unrestricted room
self.get_failure(
event_creator.create_event(
forbidden_requester,
{
@@ -840,17 +826,10 @@ class RoomAccessTestCase(unittest.HomeserverTestCase):
"content": {"membership": Membership.JOIN},
"state_key": forbidden_requester.user.to_string(),
},
)
),
SynapseError,
)
# A forbidden user without an invite should not be able to join an unrestricted room
can_join = self.get_success(
self.third_party_event_rules.check_event_allowed(
forbidden_join_event, forbidden_join_event_context
)
)
self.assertFalse(can_join)
# However, if we then invite this user...
self.helper.invite(
room=self.unrestricted_room,
@@ -861,7 +840,8 @@ class RoomAccessTestCase(unittest.HomeserverTestCase):
# And create another join event, making sure that its context states it's coming
# in after the above invite was made...
forbidden_join_event, forbidden_join_event_context = self.get_success(
# Then the forbidden user should be able to join!
self.get_success(
event_creator.create_event(
forbidden_requester,
{
@@ -874,14 +854,6 @@ class RoomAccessTestCase(unittest.HomeserverTestCase):
)
)
# Then the forbidden user should be able to join!
can_join = self.get_success(
self.third_party_event_rules.check_event_allowed(
forbidden_join_event, forbidden_join_event_context
)
)
self.assertTrue(can_join)
def test_freezing_a_room(self):
"""Tests that the power levels in a room change to prevent new events from
non-admin users when the last admin of a room leaves.

1
tests/rest/client/v2_alpha/test_account.py
View File

@@ -352,7 +352,6 @@ class DeactivateTestCase(unittest.HomeserverTestCase):
self.render(request)
self.assertEqual(request.code, 401)
@unittest.INFO
def test_pending_invites(self):
"""Tests that deactivating a user rejects every pending invite for them."""
store = self.hs.get_datastore()

1
tests/rest/client/v2_alpha/test_auth.py
View File

@@ -104,7 +104,6 @@ class FallbackAuthTests(unittest.HomeserverTestCase):
self.assertEqual(len(attempts), 1)
self.assertEqual(attempts[0][0]["response"], "a")
@unittest.INFO
def test_fallback_captcha(self):
"""Ensure that fallback auth via a captcha works."""
# Returns a 401 as per the spec

27
tests/util/caches/test_deferred_cache.py
View File

@@ -38,6 +38,22 @@ class DeferredCacheTestCase(unittest.TestCase):
self.assertEquals(cache.get("foo"), 123)
def test_get_immediate(self):
cache = DeferredCache("test")
d1 = defer.Deferred()
cache.set("key1", d1)
# get_immediate should return default
v = cache.get_immediate("key1", 1)
self.assertEqual(v, 1)
# now complete the set
d1.callback(2)
# get_immediate should return result
v = cache.get_immediate("key1", 1)
self.assertEqual(v, 2)
def test_invalidate(self):
cache = DeferredCache("test")
cache.prefill(("foo",), 123)
@@ -80,9 +96,11 @@ class DeferredCacheTestCase(unittest.TestCase):
# now do the invalidation
cache.invalidate_all()
# lookup should return none
self.assertIsNone(cache.get("key1", None))
self.assertIsNone(cache.get("key2", None))
# lookup should fail
with self.assertRaises(KeyError):
cache.get("key1")
with self.assertRaises(KeyError):
cache.get("key2")
# both callbacks should have been callbacked
self.assertTrue(callback_record[0], "Invalidation callback for key1 not called")
@@ -90,7 +108,8 @@ class DeferredCacheTestCase(unittest.TestCase):
# letting the other lookup complete should do nothing
d1.callback("result1")
self.assertIsNone(cache.get("key1", None))
with self.assertRaises(KeyError):
cache.get("key1", None)
def test_eviction(self):
cache = DeferredCache(

8
tests/util/test_lrucache.py
View File

@@ -19,7 +19,8 @@ from mock import Mock
from synapse.util.caches.lrucache import LruCache
from synapse.util.caches.treecache import TreeCache
from .. import unittest
from tests import unittest
from tests.unittest import override_config
class LruCacheTestCase(unittest.HomeserverTestCase):
@@ -83,6 +84,11 @@ class LruCacheTestCase(unittest.HomeserverTestCase):
cache.clear()
self.assertEquals(len(cache), 0)
@override_config({"caches": {"per_cache_factors": {"mycache": 10}}})
def test_special_size(self):
cache = LruCache(10, "mycache")
self.assertEqual(cache.max_size, 100)
class LruCacheCallbacksTestCase(unittest.HomeserverTestCase):
def test_get(self):

5
tox.ini
View File

@@ -111,7 +111,7 @@ commands =
[testenv:packaging]
skip_install=True
deps =
check-manifest==0.41
check-manifest
commands =
check-manifest
@@ -131,7 +131,6 @@ skip_install = True
deps = towncrier>=18.6.0rc1
commands =
python -m towncrier.check --compare-with=origin/dinsic
basepython = python3.6
[testenv:check-sampleconfig]
commands = {toxinidir}/scripts-dev/generate_sample_config --check
@@ -161,7 +160,7 @@ commands=
[testenv:mypy]
deps =
{[base]deps}
extras = all, mypy
extras = all,mypy
commands = mypy
# To find all folders that pass mypy you run: