Merge commit '4ecba9bd5' into anoa/dinsic_release_1_31_0

changelog.d/9612.docker

@@ -0,0 +1 @@
+Include [opencontainers labels](https://github.com/opencontainers/image-spec/blob/master/annotations.md#pre-defined-annotation-keys) in the Docker image.

changelog.d/9644.feature

@@ -0,0 +1 @@
+Implement the busy presence state as described in [MSC3026](https://github.com/matrix-org/matrix-doc/pull/3026).

changelog.d/9649.misc

@@ -0,0 +1 @@
+Fixed some antipattern issues to improve code quality.

changelog.d/9652.feature

@@ -0,0 +1 @@
+Add initial experimental support for a "space summary" API.

changelog.d/9665.misc

@@ -0,0 +1 @@
+Import `HomeServer` from the proper module.

docker/Dockerfile

@@ -18,6 +18,11 @@ ARG PYTHON_VERSION=3.8
 ###
 FROM docker.io/python:${PYTHON_VERSION}-slim as builder
 
+LABEL org.opencontainers.image.url='https://matrix.org/docs/projects/server/synapse'
+LABEL org.opencontainers.image.documentation='https://github.com/matrix-org/synapse/blob/master/docker/README.md'
+LABEL org.opencontainers.image.source='https://github.com/matrix-org/synapse.git'
+LABEL org.opencontainers.image.licenses='Apache-2.0'
+
 # install the OS build deps
 RUN apt-get update && apt-get install -y \
     build-essential \

scripts/move_remote_media_to_new_store.py

@@ -51,7 +51,7 @@ def main(src_repo, dest_repo):
         parts = line.split("|")
         if len(parts) != 2:
             print("Unable to parse input line %s" % line, file=sys.stderr)
-            exit(1)
+            sys.exit(1)
 
         move_media(parts[0], parts[1], src_paths, dest_paths)
 

synapse/api/constants.py

@@ -51,6 +51,7 @@ class PresenceState:
     OFFLINE = "offline"
     UNAVAILABLE = "unavailable"
     ONLINE = "online"
+    BUSY = "org.matrix.msc3026.busy"
 
 
 class JoinRules:

synapse/app/generic_worker.py

@@ -302,6 +302,8 @@ class GenericWorkerPresence(BasePresenceHandler):
             self.send_stop_syncing, UPDATE_SYNCING_USERS_MS
         )
 
+        self._busy_presence_enabled = hs.config.experimental.msc3026_enabled
+
         hs.get_reactor().addSystemEventTrigger(
             "before",
             "shutdown",
@@ -439,8 +441,12 @@ class GenericWorkerPresence(BasePresenceHandler):
             PresenceState.ONLINE,
             PresenceState.UNAVAILABLE,
             PresenceState.OFFLINE,
+            PresenceState.BUSY,
         )
-        if presence not in valid_presence:
+
+        if presence not in valid_presence or (
+            presence == PresenceState.BUSY and not self._busy_presence_enabled
+        ):
             raise SynapseError(400, "Invalid presence state")
 
         user_id = target_user.to_string()

synapse/config/experimental.py

@@ -34,6 +34,7 @@ class ExperimentalConfig(Config):
 
         # MSC2858 (multiple SSO identity providers)
         self.msc2858_enabled = experimental.get("msc2858_enabled", False)  # type: bool
-
         # Spaces (MSC1772, MSC2946, etc)
         self.spaces_enabled = experimental.get("spaces_enabled", False)  # type: bool
+        # MSC3026 (busy presence state)
+        self.msc3026_enabled = experimental.get("msc3026_enabled", False)  # type: bool

synapse/crypto/keyring.py

@@ -57,7 +57,7 @@ from synapse.util.metrics import Measure
 from synapse.util.retryutils import NotRetryingDestination
 
 if TYPE_CHECKING:
-    from synapse.app.homeserver import HomeServer
+    from synapse.server import HomeServer
 
 logger = logging.getLogger(__name__)
 

synapse/federation/federation_client.py

@@ -64,7 +64,7 @@ from synapse.util.caches.expiringcache import ExpiringCache
 from synapse.util.retryutils import NotRetryingDestination
 
 if TYPE_CHECKING:
-    from synapse.app.homeserver import HomeServer
+    from synapse.server import HomeServer
 
 logger = logging.getLogger(__name__)
 

synapse/federation/transport/server.py

@@ -18,7 +18,7 @@
 import functools
 import logging
 import re
-from typing import Optional, Tuple, Type
+from typing import Container, Mapping, Optional, Sequence, Tuple, Type
 
 import synapse
 from synapse.api.constants import MAX_GROUP_CATEGORYID_LENGTH, MAX_GROUP_ROLEID_LENGTH
@@ -29,7 +29,7 @@ from synapse.api.urls import (
     FEDERATION_V1_PREFIX,
     FEDERATION_V2_PREFIX,
 )
-from synapse.http.server import JsonResource
+from synapse.http.server import HttpServer, JsonResource
 from synapse.http.servlet import (
     assert_params_in_dict,
     parse_boolean_from_args,
@@ -46,7 +46,8 @@ from synapse.logging.opentracing import (
     whitelisted_homeserver,
 )
 from synapse.server import HomeServer
-from synapse.types import ThirdPartyInstanceID, get_domain_from_id
+from synapse.types import JsonDict, ThirdPartyInstanceID, get_domain_from_id
+from synapse.util.ratelimitutils import FederationRateLimiter
 from synapse.util.stringutils import parse_and_validate_server_name
 from synapse.util.versionstring import get_version_string
 
@@ -1457,6 +1458,40 @@ class FederationGroupsSettingJoinPolicyServlet(BaseFederationServlet):
         return 200, new_content
 
 
+class FederationSpaceSummaryServlet(BaseFederationServlet):
+    PREFIX = FEDERATION_UNSTABLE_PREFIX + "/org.matrix.msc2946"
+    PATH = "/spaces/(?P<room_id>[^/]*)"
+
+    async def on_POST(
+        self,
+        origin: str,
+        content: JsonDict,
+        query: Mapping[bytes, Sequence[bytes]],
+        room_id: str,
+    ) -> Tuple[int, JsonDict]:
+        suggested_only = content.get("suggested_only", False)
+        if not isinstance(suggested_only, bool):
+            raise SynapseError(
+                400, "'suggested_only' must be a boolean", Codes.BAD_JSON
+            )
+
+        exclude_rooms = content.get("exclude_rooms", [])
+        if not isinstance(exclude_rooms, list) or any(
+            not isinstance(x, str) for x in exclude_rooms
+        ):
+            raise SynapseError(400, "bad value for 'exclude_rooms'", Codes.BAD_JSON)
+
+        max_rooms_per_space = content.get("max_rooms_per_space")
+        if max_rooms_per_space is not None and not isinstance(max_rooms_per_space, int):
+            raise SynapseError(
+                400, "bad value for 'max_rooms_per_space'", Codes.BAD_JSON
+            )
+
+        return 200, await self.handler.federation_space_summary(
+            room_id, suggested_only, max_rooms_per_space, exclude_rooms
+        )
+
+
 class RoomComplexityServlet(BaseFederationServlet):
     """
     Indicates to other servers how complex (and therefore likely
@@ -1558,18 +1593,24 @@ DEFAULT_SERVLET_GROUPS = (
 )
 
 
-def register_servlets(hs, resource, authenticator, ratelimiter, servlet_groups=None):
+def register_servlets(
+    hs: HomeServer,
+    resource: HttpServer,
+    authenticator: Authenticator,
+    ratelimiter: FederationRateLimiter,
+    servlet_groups: Optional[Container[str]] = None,
+):
     """Initialize and register servlet classes.
 
     Will by default register all servlets. For custom behaviour, pass in
     a list of servlet_groups to register.
 
     Args:
-        hs (synapse.server.HomeServer): homeserver
-        resource (JsonResource): resource class to register to
-        authenticator (Authenticator): authenticator to use
-        ratelimiter (util.ratelimitutils.FederationRateLimiter): ratelimiter to use
-        servlet_groups (list[str], optional): List of servlet groups to register.
+        hs: homeserver
+        resource: resource class to register to
+        authenticator: authenticator to use
+        ratelimiter: ratelimiter to use
+        servlet_groups: List of servlet groups to register.
             Defaults to ``DEFAULT_SERVLET_GROUPS``.
     """
     if not servlet_groups:
@@ -1584,6 +1625,14 @@ def register_servlets(hs, resource, authenticator, ratelimiter, servlet_groups=N
                 server_name=hs.hostname,
             ).register(resource)
 
+        if hs.config.experimental.spaces_enabled:
+            FederationSpaceSummaryServlet(
+                handler=hs.get_space_summary_handler(),
+                authenticator=authenticator,
+                ratelimiter=ratelimiter,
+                server_name=hs.hostname,
+            ).register(resource)
+
     if "openid" in servlet_groups:
         for servletclass in OPENID_SERVLET_CLASSES:
             servletclass(

synapse/groups/attestations.py

@@ -46,7 +46,7 @@ from synapse.metrics.background_process_metrics import run_as_background_process
 from synapse.types import JsonDict, get_domain_from_id
 
 if TYPE_CHECKING:
-    from synapse.app.homeserver import HomeServer
+    from synapse.server import HomeServer
 
 logger = logging.getLogger(__name__)
 

synapse/groups/groups_server.py

@@ -25,7 +25,7 @@ from synapse.types import GroupID, JsonDict, RoomID, UserID, get_domain_from_id
 from synapse.util.async_helpers import concurrently_execute
 
 if TYPE_CHECKING:
-    from synapse.app.homeserver import HomeServer
+    from synapse.server import HomeServer
 
 logger = logging.getLogger(__name__)
 

synapse/handlers/_base.py

@@ -24,7 +24,7 @@ from synapse.api.ratelimiting import Ratelimiter
 from synapse.types import UserID
 
 if TYPE_CHECKING:
-    from synapse.app.homeserver import HomeServer
+    from synapse.server import HomeServer
 
 logger = logging.getLogger(__name__)
 

synapse/handlers/account_data.py

@@ -25,7 +25,7 @@ from synapse.replication.http.account_data import (
 from synapse.types import JsonDict, UserID
 
 if TYPE_CHECKING:
-    from synapse.app.homeserver import HomeServer
+    from synapse.server import HomeServer
 
 
 class AccountDataHandler:

synapse/handlers/account_validity.py

@@ -30,7 +30,7 @@ from synapse.types import UserID
 from synapse.util import stringutils
 
 if TYPE_CHECKING:
-    from synapse.app.homeserver import HomeServer
+    from synapse.server import HomeServer
 
 logger = logging.getLogger(__name__)
 

synapse/handlers/acme.py

@@ -24,7 +24,7 @@ from twisted.web.resource import Resource
 from synapse.app import check_bind_error
 
 if TYPE_CHECKING:
-    from synapse.app.homeserver import HomeServer
+    from synapse.server import HomeServer
 
 logger = logging.getLogger(__name__)
 

synapse/handlers/admin.py

@@ -25,7 +25,7 @@ from synapse.visibility import filter_events_for_client
 from ._base import BaseHandler
 
 if TYPE_CHECKING:
-    from synapse.app.homeserver import HomeServer
+    from synapse.server import HomeServer
 
 logger = logging.getLogger(__name__)
 

synapse/handlers/appservice.py

@@ -38,7 +38,7 @@ from synapse.types import Collection, JsonDict, RoomAlias, RoomStreamToken, User
 from synapse.util.metrics import Measure
 
 if TYPE_CHECKING:
-    from synapse.app.homeserver import HomeServer
+    from synapse.server import HomeServer
 
 logger = logging.getLogger(__name__)
 

synapse/handlers/auth.py

@@ -70,7 +70,7 @@ from synapse.util.msisdn import phone_number_to_msisdn
 from synapse.util.threepids import canonicalise_email
 
 if TYPE_CHECKING:
-    from synapse.app.homeserver import HomeServer
+    from synapse.server import HomeServer
 
 logger = logging.getLogger(__name__)
 

synapse/handlers/cas_handler.py

@@ -27,7 +27,7 @@ from synapse.http.site import SynapseRequest
 from synapse.types import UserID, map_username_to_mxid_localpart
 
 if TYPE_CHECKING:
-    from synapse.app.homeserver import HomeServer
+    from synapse.server import HomeServer
 
 logger = logging.getLogger(__name__)
 

synapse/handlers/deactivate_account.py

@@ -23,7 +23,7 @@ from synapse.types import Requester, UserID, create_requester
 from ._base import BaseHandler
 
 if TYPE_CHECKING:
-    from synapse.app.homeserver import HomeServer
+    from synapse.server import HomeServer
 
 logger = logging.getLogger(__name__)
 

synapse/handlers/device.py

@@ -45,7 +45,7 @@ from synapse.util.retryutils import NotRetryingDestination
 from ._base import BaseHandler
 
 if TYPE_CHECKING:
-    from synapse.app.homeserver import HomeServer
+    from synapse.server import HomeServer
 
 logger = logging.getLogger(__name__)
 

synapse/handlers/devicemessage.py

@@ -32,7 +32,7 @@ from synapse.util import json_encoder
 from synapse.util.stringutils import random_string
 
 if TYPE_CHECKING:
-    from synapse.app.homeserver import HomeServer
+    from synapse.server import HomeServer
 
 
 logger = logging.getLogger(__name__)

synapse/handlers/e2e_keys.py

@@ -42,7 +42,7 @@ from synapse.util.caches.expiringcache import ExpiringCache
 from synapse.util.retryutils import NotRetryingDestination
 
 if TYPE_CHECKING:
-    from synapse.app.homeserver import HomeServer
+    from synapse.server import HomeServer
 
 logger = logging.getLogger(__name__)
 

synapse/handlers/e2e_room_keys.py

@@ -29,7 +29,7 @@ from synapse.types import JsonDict
 from synapse.util.async_helpers import Linearizer
 
 if TYPE_CHECKING:
-    from synapse.app.homeserver import HomeServer
+    from synapse.server import HomeServer
 
 logger = logging.getLogger(__name__)
 

synapse/handlers/groups_local.py

@@ -21,7 +21,7 @@ from synapse.api.errors import HttpResponseException, RequestSendFailed, Synapse
 from synapse.types import GroupID, JsonDict, get_domain_from_id
 
 if TYPE_CHECKING:
-    from synapse.app.homeserver import HomeServer
+    from synapse.server import HomeServer
 
 logger = logging.getLogger(__name__)
 

synapse/handlers/password_policy.py

@@ -21,7 +21,7 @@ from typing import TYPE_CHECKING
 from synapse.api.errors import Codes, PasswordRefusedError
 
 if TYPE_CHECKING:
-    from synapse.app.homeserver import HomeServer
+    from synapse.server import HomeServer
 
 logger = logging.getLogger(__name__)
 

synapse/handlers/presence.py

@@ -104,6 +104,8 @@ class BasePresenceHandler(abc.ABC):
         self.clock = hs.get_clock()
         self.store = hs.get_datastore()
 
+        self._busy_presence_enabled = hs.config.experimental.msc3026_enabled
+
         active_presence = self.store.take_presence_startup_info()
         self.user_to_current_state = {state.user_id: state for state in active_presence}
 
@@ -730,8 +732,12 @@ class PresenceHandler(BasePresenceHandler):
             PresenceState.ONLINE,
             PresenceState.UNAVAILABLE,
             PresenceState.OFFLINE,
+            PresenceState.BUSY,
         )
-        if presence not in valid_presence:
+
+        if presence not in valid_presence or (
+            presence == PresenceState.BUSY and not self._busy_presence_enabled
+        ):
             raise SynapseError(400, "Invalid presence state")
 
         user_id = target_user.to_string()
@@ -744,7 +750,9 @@ class PresenceHandler(BasePresenceHandler):
             msg = status_msg if presence != PresenceState.OFFLINE else None
             new_fields["status_msg"] = msg
 
-        if presence == PresenceState.ONLINE:
+        if presence == PresenceState.ONLINE or (
+            presence == PresenceState.BUSY and self._busy_presence_enabled
+        ):
             new_fields["last_active_ts"] = self.clock.time_msec()
 
         await self._update_states([prev_state.copy_and_replace(**new_fields)])

synapse/handlers/profile.py

@@ -45,7 +45,7 @@ from synapse.types import (
 from ._base import BaseHandler
 
 if TYPE_CHECKING:
-    from synapse.app.homeserver import HomeServer
+    from synapse.server import HomeServer
 
 logger = logging.getLogger(__name__)
 

synapse/handlers/read_marker.py

@@ -21,7 +21,7 @@ from synapse.util.async_helpers import Linearizer
 from ._base import BaseHandler
 
 if TYPE_CHECKING:
-    from synapse.app.homeserver import HomeServer
+    from synapse.server import HomeServer
 
 logger = logging.getLogger(__name__)
 

synapse/handlers/receipts.py

@@ -20,7 +20,7 @@ from synapse.handlers._base import BaseHandler
 from synapse.types import JsonDict, ReadReceipt, get_domain_from_id
 
 if TYPE_CHECKING:
-    from synapse.app.homeserver import HomeServer
+    from synapse.server import HomeServer
 
 logger = logging.getLogger(__name__)
 

synapse/handlers/register.py

@@ -38,7 +38,7 @@ from synapse.types import RoomAlias, UserID, create_requester
 from ._base import BaseHandler
 
 if TYPE_CHECKING:
-    from synapse.app.homeserver import HomeServer
+    from synapse.server import HomeServer
 
 logger = logging.getLogger(__name__)
 

synapse/handlers/room_list.py

@@ -29,7 +29,7 @@ from synapse.util.caches.response_cache import ResponseCache
 from ._base import BaseHandler
 
 if TYPE_CHECKING:
-    from synapse.app.homeserver import HomeServer
+    from synapse.server import HomeServer
 
 logger = logging.getLogger(__name__)
 

synapse/handlers/room_member_worker.py

@@ -29,7 +29,7 @@ from synapse.replication.http.membership import (
 from synapse.types import JsonDict, Requester, UserID
 
 if TYPE_CHECKING:
-    from synapse.app.homeserver import HomeServer
+    from synapse.server import HomeServer
 
 logger = logging.getLogger(__name__)
 

synapse/handlers/search.py

@@ -30,7 +30,7 @@ from synapse.visibility import filter_events_for_client
 from ._base import BaseHandler
 
 if TYPE_CHECKING:
-    from synapse.app.homeserver import HomeServer
+    from synapse.server import HomeServer
 
 logger = logging.getLogger(__name__)
 

synapse/handlers/set_password.py

@@ -22,7 +22,7 @@ from synapse.types import Requester
 from ._base import BaseHandler
 
 if TYPE_CHECKING:
-    from synapse.app.homeserver import HomeServer
+    from synapse.server import HomeServer
 
 logger = logging.getLogger(__name__)
 

synapse/handlers/space_summary.py

@@ -16,7 +16,9 @@
 import itertools
 import logging
 from collections import deque
-from typing import TYPE_CHECKING, Iterable, List, Optional, Set
+from typing import TYPE_CHECKING, Iterable, List, Optional, Sequence, Set, Tuple
+
+import attr
 
 from synapse.api.constants import EventContentFields, EventTypes, HistoryVisibility
 from synapse.api.errors import AuthError
@@ -54,7 +56,7 @@ class SpaceSummaryHandler:
         max_rooms_per_space: Optional[int] = None,
     ) -> JsonDict:
         """
-        Implementation of the space summary API
+        Implementation of the space summary C-S API
 
         Args:
             requester:  user id of the user making this request
@@ -66,7 +68,7 @@ class SpaceSummaryHandler:
 
             max_rooms_per_space: an optional limit on the number of child rooms we will
                 return. This does not apply to the root room (ie, room_id), and
-                is overridden by ROOMS_PER_SPACE_LIMIT.
+                is overridden by MAX_ROOMS_PER_SPACE.
 
         Returns:
             summary dict to return
@@ -76,68 +78,154 @@ class SpaceSummaryHandler:
         await self._auth.check_user_in_room_or_world_readable(room_id, requester)
 
         # the queue of rooms to process
-        room_queue = deque((room_id,))
+        room_queue = deque((_RoomQueueEntry(room_id),))
 
         processed_rooms = set()  # type: Set[str]
 
         rooms_result = []  # type: List[JsonDict]
         events_result = []  # type: List[JsonDict]
 
-        now = self._clock.time_msec()
+        while room_queue and len(rooms_result) < MAX_ROOMS:
+            queue_entry = room_queue.popleft()
+            room_id = queue_entry.room_id
+            logger.debug("Processing room %s", room_id)
+            processed_rooms.add(room_id)
+
+            # The client-specified max_rooms_per_space limit doesn't apply to the
+            # room_id specified in the request, so we ignore it if this is the
+            # first room we are processing.
+            max_children = max_rooms_per_space if processed_rooms else None
+
+            rooms, events = await self._summarize_local_room(
+                requester, room_id, suggested_only, max_children
+            )
+
+            rooms_result.extend(rooms)
+            events_result.extend(events)
+
+            # add any children that we haven't already processed to the queue
+            for edge_event in events:
+                if edge_event["state_key"] not in processed_rooms:
+                    room_queue.append(_RoomQueueEntry(edge_event["state_key"]))
+
+        return {"rooms": rooms_result, "events": events_result}
+
+    async def federation_space_summary(
+        self,
+        room_id: str,
+        suggested_only: bool,
+        max_rooms_per_space: Optional[int],
+        exclude_rooms: Iterable[str],
+    ) -> JsonDict:
+        """
+        Implementation of the space summary Federation API
+
+        Args:
+            room_id: room id to start the summary at
+
+            suggested_only: whether we should only return children with the "suggested"
+                flag set.
+
+            max_rooms_per_space: an optional limit on the number of child rooms we will
+                return. Unlike the C-S API, this applies to the root room (room_id).
+                It is clipped to MAX_ROOMS_PER_SPACE.
+
+            exclude_rooms: a list of rooms to skip over (presumably because the
+                calling server has already seen them).
+
+        Returns:
+            summary dict to return
+        """
+        # the queue of rooms to process
+        room_queue = deque((room_id,))
+
+        # the set of rooms that we should not walk further. Initialise it with the
+        # excluded-rooms list; we will add other rooms as we process them so that
+        # we do not loop.
+        processed_rooms = set(exclude_rooms)  # type: Set[str]
+
+        rooms_result = []  # type: List[JsonDict]
+        events_result = []  # type: List[JsonDict]
 
         while room_queue and len(rooms_result) < MAX_ROOMS:
             room_id = room_queue.popleft()
             logger.debug("Processing room %s", room_id)
             processed_rooms.add(room_id)
 
-            try:
-                await self._auth.check_user_in_room_or_world_readable(
-                    room_id, requester
-                )
-            except AuthError:
-                logger.info(
-                    "user %s cannot view room %s, omitting from summary",
-                    requester,
-                    room_id,
-                )
-                continue
+            rooms, events = await self._summarize_local_room(
+                None, room_id, suggested_only, max_rooms_per_space
+            )
 
-            room_entry = await self._build_room_entry(room_id)
-            rooms_result.append(room_entry)
+            rooms_result.extend(rooms)
+            events_result.extend(events)
 
-            # look for child rooms/spaces.
-            child_events = await self._get_child_events(room_id)
-
-            if suggested_only:
-                # we only care about suggested children
-                child_events = filter(_is_suggested_child_event, child_events)
-
-            # The client-specified max_rooms_per_space limit doesn't apply to the
-            # room_id specified in the request, so we ignore it if this is the
-            # first room we are processing. Otherwise, apply any client-specified
-            # limit, capping to our built-in limit.
-            if max_rooms_per_space is not None and len(processed_rooms) > 1:
-                max_rooms = min(MAX_ROOMS_PER_SPACE, max_rooms_per_space)
-            else:
-                max_rooms = MAX_ROOMS_PER_SPACE
-
-            for edge_event in itertools.islice(child_events, max_rooms):
-                edge_room_id = edge_event.state_key
-
-                events_result.append(
-                    await self._event_serializer.serialize_event(
-                        edge_event,
-                        time_now=now,
-                        event_format=format_event_for_client_v2,
-                    )
-                )
-
-                # if we haven't yet visited the target of this link, add it to the queue
-                if edge_room_id not in processed_rooms:
-                    room_queue.append(edge_room_id)
+            # add any children that we haven't already processed to the queue
+            for edge_event in events:
+                if edge_event["state_key"] not in processed_rooms:
+                    room_queue.append(edge_event["state_key"])
 
         return {"rooms": rooms_result, "events": events_result}
 
+    async def _summarize_local_room(
+        self,
+        requester: Optional[str],
+        room_id: str,
+        suggested_only: bool,
+        max_children: Optional[int],
+    ) -> Tuple[Sequence[JsonDict], Sequence[JsonDict]]:
+        if not await self._is_room_accessible(room_id, requester):
+            return (), ()
+
+        room_entry = await self._build_room_entry(room_id)
+
+        # look for child rooms/spaces.
+        child_events = await self._get_child_events(room_id)
+
+        if suggested_only:
+            # we only care about suggested children
+            child_events = filter(_is_suggested_child_event, child_events)
+
+        if max_children is None or max_children > MAX_ROOMS_PER_SPACE:
+            max_children = MAX_ROOMS_PER_SPACE
+
+        now = self._clock.time_msec()
+        events_result = []  # type: List[JsonDict]
+        for edge_event in itertools.islice(child_events, max_children):
+            events_result.append(
+                await self._event_serializer.serialize_event(
+                    edge_event,
+                    time_now=now,
+                    event_format=format_event_for_client_v2,
+                )
+            )
+        return (room_entry,), events_result
+
+    async def _is_room_accessible(self, room_id: str, requester: Optional[str]) -> bool:
+        # if we have an authenticated requesting user, first check if they are in the
+        # room
+        if requester:
+            try:
+                await self._auth.check_user_in_room(room_id, requester)
+                return True
+            except AuthError:
+                pass
+
+        # otherwise, check if the room is peekable
+        hist_vis_ev = await self._state_handler.get_current_state(
+            room_id, EventTypes.RoomHistoryVisibility, ""
+        )
+        if hist_vis_ev:
+            hist_vis = hist_vis_ev.content.get("history_visibility")
+            if hist_vis == HistoryVisibility.WORLD_READABLE:
+                return True
+
+        logger.info(
+            "room %s is unpeekable and user %s is not a member, omitting from summary",
+            room_id,
+            requester,
+        )
+        return False
+
     async def _build_room_entry(self, room_id: str) -> JsonDict:
         """Generate en entry suitable for the 'rooms' list in the summary response"""
         stats = await self._store.get_room_with_stats(room_id)
@@ -191,6 +279,11 @@ class SpaceSummaryHandler:
         return (e for e in events if e.content.get("via"))
 
 
+@attr.s(frozen=True, slots=True)
+class _RoomQueueEntry:
+    room_id = attr.ib(type=str)
+
+
 def _is_suggested_child_event(edge_event: EventBase) -> bool:
     suggested = edge_event.content.get("suggested")
     if isinstance(suggested, bool) and suggested:

synapse/handlers/state_deltas.py

@@ -17,7 +17,7 @@ import logging
 from typing import TYPE_CHECKING, Optional
 
 if TYPE_CHECKING:
-    from synapse.app.homeserver import HomeServer
+    from synapse.server import HomeServer
 
 logger = logging.getLogger(__name__)
 

synapse/handlers/stats.py

@@ -26,7 +26,7 @@ from synapse.metrics.background_process_metrics import run_as_background_process
 from synapse.types import JsonDict
 
 if TYPE_CHECKING:
-    from synapse.app.homeserver import HomeServer
+    from synapse.server import HomeServer
 
 logger = logging.getLogger(__name__)
 

synapse/handlers/user_directory.py

@@ -25,7 +25,7 @@ from synapse.types import JsonDict
 from synapse.util.metrics import Measure
 
 if TYPE_CHECKING:
-    from synapse.app.homeserver import HomeServer
+    from synapse.server import HomeServer
 
 logger = logging.getLogger(__name__)
 

synapse/http/client.py

@@ -77,7 +77,7 @@ from synapse.util import json_decoder
 from synapse.util.async_helpers import timeout_deferred
 
 if TYPE_CHECKING:
-    from synapse.app.homeserver import HomeServer
+    from synapse.server import HomeServer
 
 logger = logging.getLogger(__name__)
 

synapse/push/__init__.py

@@ -21,7 +21,7 @@ import attr
 from synapse.types import JsonDict, RoomStreamToken
 
 if TYPE_CHECKING:
-    from synapse.app.homeserver import HomeServer
+    from synapse.server import HomeServer
 
 
 @attr.s(slots=True)

synapse/push/action_generator.py

@@ -22,7 +22,7 @@ from synapse.push.bulk_push_rule_evaluator import BulkPushRuleEvaluator
 from synapse.util.metrics import Measure
 
 if TYPE_CHECKING:
-    from synapse.app.homeserver import HomeServer
+    from synapse.server import HomeServer
 
 logger = logging.getLogger(__name__)
 

synapse/push/bulk_push_rule_evaluator.py

@@ -33,7 +33,7 @@ from synapse.util.caches.lrucache import LruCache
 from .push_rule_evaluator import PushRuleEvaluatorForEvent
 
 if TYPE_CHECKING:
-    from synapse.app.homeserver import HomeServer
+    from synapse.server import HomeServer
 
 logger = logging.getLogger(__name__)
 

synapse/push/emailpusher.py

@@ -24,7 +24,7 @@ from synapse.push import Pusher, PusherConfig, ThrottleParams
 from synapse.push.mailer import Mailer
 
 if TYPE_CHECKING:
-    from synapse.app.homeserver import HomeServer
+    from synapse.server import HomeServer
 
 logger = logging.getLogger(__name__)
 

synapse/push/httppusher.py

@@ -31,7 +31,7 @@ from synapse.push import Pusher, PusherConfig, PusherConfigException
 from . import push_rule_evaluator, push_tools
 
 if TYPE_CHECKING:
-    from synapse.app.homeserver import HomeServer
+    from synapse.server import HomeServer
 
 logger = logging.getLogger(__name__)
 
@@ -290,7 +290,7 @@ class HttpPusher(Pusher):
         if rejected is False:
             return False
 
-        if isinstance(rejected, list) or isinstance(rejected, tuple):
+        if isinstance(rejected, (list, tuple)):
             for pk in rejected:
                 if pk != self.pushkey:
                     # for sanity, we only remove the pushkey if it

synapse/push/mailer.py

@@ -40,7 +40,7 @@ from synapse.util.async_helpers import concurrently_execute
 from synapse.visibility import filter_events_for_client
 
 if TYPE_CHECKING:
-    from synapse.app.homeserver import HomeServer
+    from synapse.server import HomeServer
 
 logger = logging.getLogger(__name__)
 

synapse/push/pusher.py

@@ -22,7 +22,7 @@ from synapse.push.httppusher import HttpPusher
 from synapse.push.mailer import Mailer
 
 if TYPE_CHECKING:
-    from synapse.app.homeserver import HomeServer
+    from synapse.server import HomeServer
 
 logger = logging.getLogger(__name__)
 

synapse/replication/slave/storage/pushers.py

@@ -24,7 +24,7 @@ from ._base import BaseSlavedStore
 from ._slaved_id_tracker import SlavedIdTracker
 
 if TYPE_CHECKING:
-    from synapse.app.homeserver import HomeServer
+    from synapse.server import HomeServer
 
 
 class SlavedPusherStore(PusherWorkerStore, BaseSlavedStore):

synapse/replication/tcp/streams/_base.py

@@ -33,7 +33,7 @@ import attr
 from synapse.replication.http.streams import ReplicationGetStreamUpdates
 
 if TYPE_CHECKING:
-    from synapse.app.homeserver import HomeServer
+    from synapse.server import HomeServer
 
 logger = logging.getLogger(__name__)
 

synapse/rest/admin/media.py

@@ -28,7 +28,7 @@ from synapse.rest.admin._base import (
 from synapse.types import JsonDict
 
 if TYPE_CHECKING:
-    from synapse.app.homeserver import HomeServer
+    from synapse.server import HomeServer
 
 logger = logging.getLogger(__name__)
 

synapse/rest/client/v1/room.py

@@ -20,8 +20,6 @@ import re
 from typing import TYPE_CHECKING, Optional, Tuple
 from urllib import parse as urlparse
 
-from twisted.web.server import Request
-
 from synapse.api.constants import EventTypes, Membership
 from synapse.api.errors import (
     AuthError,
@@ -42,6 +40,7 @@ from synapse.http.servlet import (
     parse_list_from_args,
     parse_string,
 )
+from synapse.http.site import SynapseRequest
 from synapse.logging.opentracing import set_tag
 from synapse.rest.client.transactions import HttpTransactionCache
 from synapse.rest.client.v2_alpha._base import client_patterns
@@ -59,7 +58,7 @@ from synapse.util import json_decoder
 from synapse.util.stringutils import parse_and_validate_server_name, random_string
 
 if TYPE_CHECKING:
-    from synapse.app.homeserver import HomeServer
+    from synapse.server import HomeServer
 
 logger = logging.getLogger(__name__)
 
@@ -1009,7 +1008,9 @@ class RoomSpaceSummaryRestServlet(RestServlet):
         self._auth = hs.get_auth()
         self._space_summary_handler = hs.get_space_summary_handler()
 
-    async def on_GET(self, request: Request, room_id: str) -> Tuple[int, JsonDict]:
+    async def on_GET(
+        self, request: SynapseRequest, room_id: str
+    ) -> Tuple[int, JsonDict]:
         requester = await self._auth.get_user_by_req(request, allow_guest=True)
 
         return 200, await self._space_summary_handler.get_space_summary(
@@ -1019,7 +1020,9 @@ class RoomSpaceSummaryRestServlet(RestServlet):
             max_rooms_per_space=parse_integer(request, "max_rooms_per_space"),
         )
 
-    async def on_POST(self, request: Request, room_id: str) -> Tuple[int, JsonDict]:
+    async def on_POST(
+        self, request: SynapseRequest, room_id: str
+    ) -> Tuple[int, JsonDict]:
         requester = await self._auth.get_user_by_req(request, allow_guest=True)
         content = parse_json_object_from_request(request)
 

synapse/rest/client/v2_alpha/account.py

@@ -47,7 +47,7 @@ from synapse.util.threepids import canonicalise_email, check_3pid_allowed
 from ._base import client_patterns, interactive_auth_handler
 
 if TYPE_CHECKING:
-    from synapse.app.homeserver import HomeServer
+    from synapse.server import HomeServer
 
 
 logger = logging.getLogger(__name__)

synapse/rest/client/v2_alpha/groups.py

@@ -38,7 +38,7 @@ from synapse.types import GroupID, JsonDict
 from ._base import client_patterns
 
 if TYPE_CHECKING:
-    from synapse.app.homeserver import HomeServer
+    from synapse.server import HomeServer
 
 logger = logging.getLogger(__name__)
 

synapse/rest/client/versions.py

@@ -84,6 +84,8 @@ class VersionsRestServlet(RestServlet):
                     "io.element.e2ee_forced.public": self.e2ee_forced_public,
                     "io.element.e2ee_forced.private": self.e2ee_forced_private,
                     "io.element.e2ee_forced.trusted_private": self.e2ee_forced_trusted_private,
+                    # Supports the busy presence state described in MSC3026.
+                    "org.matrix.msc3026.busy_presence": self.config.experimental.msc3026_enabled,
                 },
             },
         )

synapse/rest/media/v1/config_resource.py

@@ -23,7 +23,7 @@ from synapse.http.server import DirectServeJsonResource, respond_with_json
 from synapse.http.site import SynapseRequest
 
 if TYPE_CHECKING:
-    from synapse.app.homeserver import HomeServer
+    from synapse.server import HomeServer
 
 
 class MediaConfigResource(DirectServeJsonResource):

synapse/rest/media/v1/download_resource.py

@@ -24,8 +24,8 @@ from synapse.http.servlet import parse_boolean
 from ._base import parse_media_id, respond_404
 
 if TYPE_CHECKING:
-    from synapse.app.homeserver import HomeServer
     from synapse.rest.media.v1.media_repository import MediaRepository
+    from synapse.server import HomeServer
 
 logger = logging.getLogger(__name__)
 

synapse/rest/media/v1/media_repository.py

@@ -58,7 +58,7 @@ from .thumbnailer import Thumbnailer, ThumbnailError
 from .upload_resource import UploadResource
 
 if TYPE_CHECKING:
-    from synapse.app.homeserver import HomeServer
+    from synapse.server import HomeServer
 
 logger = logging.getLogger(__name__)
 

synapse/rest/media/v1/preview_url_resource.py

@@ -54,8 +54,8 @@ from ._base import FileInfo
 if TYPE_CHECKING:
     from lxml import etree
 
-    from synapse.app.homeserver import HomeServer
     from synapse.rest.media.v1.media_repository import MediaRepository
+    from synapse.server import HomeServer
 
 logger = logging.getLogger(__name__)
 

synapse/rest/media/v1/storage_provider.py

@@ -29,7 +29,7 @@ from .media_storage import FileResponder
 logger = logging.getLogger(__name__)
 
 if TYPE_CHECKING:
-    from synapse.app.homeserver import HomeServer
+    from synapse.server import HomeServer
 
 
 class StorageProvider(metaclass=abc.ABCMeta):

synapse/rest/media/v1/thumbnail_resource.py

@@ -34,8 +34,8 @@ from ._base import (
 )
 
 if TYPE_CHECKING:
-    from synapse.app.homeserver import HomeServer
     from synapse.rest.media.v1.media_repository import MediaRepository
+    from synapse.server import HomeServer
 
 logger = logging.getLogger(__name__)
 

synapse/rest/media/v1/upload_resource.py

@@ -26,8 +26,8 @@ from synapse.http.site import SynapseRequest
 from synapse.rest.media.v1.media_storage import SpamMediaException
 
 if TYPE_CHECKING:
-    from synapse.app.homeserver import HomeServer
     from synapse.rest.media.v1.media_repository import MediaRepository
+    from synapse.server import HomeServer
 
 logger = logging.getLogger(__name__)
 

synapse/storage/__init__.py

@@ -36,7 +36,7 @@ from synapse.storage.purge_events import PurgeEventsStorage
 from synapse.storage.state import StateGroupStorage
 
 if TYPE_CHECKING:
-    from synapse.app.homeserver import HomeServer
+    from synapse.server import HomeServer
 
 
 __all__ = ["Databases", "DataStore"]

synapse/storage/_base.py

@@ -27,7 +27,7 @@ from synapse.types import Collection, StreamToken, get_domain_from_id
 from synapse.util import json_decoder
 
 if TYPE_CHECKING:
-    from synapse.app.homeserver import HomeServer
+    from synapse.server import HomeServer
 
 logger = logging.getLogger(__name__)
 

synapse/storage/background_updates.py

@@ -23,7 +23,7 @@ from synapse.util import json_encoder
 from . import engines
 
 if TYPE_CHECKING:
-    from synapse.app.homeserver import HomeServer
+    from synapse.server import HomeServer
     from synapse.storage.database import DatabasePool, LoggingTransaction
 
 logger = logging.getLogger(__name__)

synapse/storage/databases/main/appservice.py

@@ -32,7 +32,7 @@ from synapse.types import JsonDict
 from synapse.util import json_encoder
 
 if TYPE_CHECKING:
-    from synapse.app.homeserver import HomeServer
+    from synapse.server import HomeServer
 
 logger = logging.getLogger(__name__)
 

synapse/storage/databases/main/pusher.py

@@ -27,7 +27,7 @@ from synapse.util import json_encoder
 from synapse.util.caches.descriptors import cached, cachedList
 
 if TYPE_CHECKING:
-    from synapse.app.homeserver import HomeServer
+    from synapse.server import HomeServer
 
 logger = logging.getLogger(__name__)
 

synapse/storage/purge_events.py

@@ -20,7 +20,7 @@ from typing import TYPE_CHECKING, Set
 from synapse.storage.databases import Databases
 
 if TYPE_CHECKING:
-    from synapse.app.homeserver import HomeServer
+    from synapse.server import HomeServer
 
 logger = logging.getLogger(__name__)
 

synapse/storage/state.py

@@ -32,7 +32,7 @@ from synapse.events import EventBase
 from synapse.types import MutableStateMap, StateMap
 
 if TYPE_CHECKING:
-    from synapse.app.homeserver import HomeServer
+    from synapse.server import HomeServer
     from synapse.storage.databases import Databases
 
 logger = logging.getLogger(__name__)

synapse/util/frozenutils.py

@@ -36,7 +36,7 @@ def freeze(o):
 
 def unfreeze(o):
     if isinstance(o, (dict, frozendict)):
-        return dict({k: unfreeze(v) for k, v in o.items()})
+        return {k: unfreeze(v) for k, v in o.items()}
 
     if isinstance(o, (bytes, str)):
         return o

tests/handlers/test_presence.py

@@ -310,6 +310,26 @@ class PresenceTimeoutTestCase(unittest.TestCase):
         self.assertIsNotNone(new_state)
         self.assertEquals(new_state.state, PresenceState.UNAVAILABLE)
 
+    def test_busy_no_idle(self):
+        """
+        Tests that a user setting their presence to busy but idling doesn't turn their
+        presence state into unavailable.
+        """
+        user_id = "@foo:bar"
+        now = 5000000
+
+        state = UserPresenceState.default(user_id)
+        state = state.copy_and_replace(
+            state=PresenceState.BUSY,
+            last_active_ts=now - IDLE_TIMER - 1,
+            last_user_sync_ts=now,
+        )
+
+        new_state = handle_timeout(state, is_mine=True, syncing_user_ids=set(), now=now)
+
+        self.assertIsNotNone(new_state)
+        self.assertEquals(new_state.state, PresenceState.BUSY)
+
     def test_sync_timeout(self):
         user_id = "@foo:bar"
         now = 5000000