Up no output timeout

Apple want to POST the OIDC auth response back to us rather than using query-params; add the necessary support to make that work.

.circleci/config.yml

@@ -22,11 +22,11 @@ jobs:
     steps:
       - checkout
       - docker_prepare
-      - run: docker login --username $DOCKER_HUB_USERNAME --password $DOCKER_HUB_PASSWORD
+      # - run: docker login --username $DOCKER_HUB_USERNAME --password $DOCKER_HUB_PASSWORD
       # for `latest`, we don't want the arm images to disappear, so don't update the tag
       # until all of the platforms are built.
       - docker_build:
-          tag: -t matrixdotorg/synapse:latest
+          tag: -t 127.0.0.1:5000/synapse:erikj-test
           platforms: linux/amd64,linux/arm/v7,linux/arm64
 
 workflows:
@@ -41,7 +41,7 @@ workflows:
       - dockerhubuploadlatest:
           filters:
             branches:
-              only: master
+              only: erikj/arm_docker_cache
 
 commands:
   docker_prepare:
@@ -52,9 +52,9 @@ commands:
         default: "v0.4.1"
     steps:
       - setup_remote_docker:
-          # 19.03.13 was the most recent available on circleci at the time of
+          # 20.10.2 was the most recent available on circleci at the time of
           # writing.
-          version: 19.03.13
+          version: 20.10.2
       - run: apk add --no-cache curl
       - run: mkdir -vp ~/.docker/cli-plugins/ ~/dockercache
       - run: curl --silent -L "https://github.com/docker/buildx/releases/download/<< parameters.buildx_version >>/buildx-<< parameters.buildx_version >>.linux-amd64" > ~/.docker/cli-plugins/docker-buildx
@@ -64,7 +64,10 @@ commands:
       # create a context named `builder` for the builds
       - run: docker context create builder
       # create a buildx builder using the new context, and set it as the default
-      - run: docker buildx create builder --use
+      - run: docker buildx create --driver docker-container --driver-opt network=host builder --use
+      # Start a registry so that have somewhere to store our temporary docker
+      # images (as multi arch builds don't work with stand local docker store)
+      - run: docker run -d -p 127.0.0.1:5000:5000 --name registry registry:2
 
   docker_build:
     description: Builds and pushed images to dockerhub using buildx
@@ -75,4 +78,7 @@ commands:
       tag:
         type: string
     steps:
-      - run: docker buildx build -f docker/Dockerfile --push --platform << parameters.platforms >> --label gitsha1=${CIRCLE_SHA1} << parameters.tag >> --progress=plain .
+      - run: docker buildx build -f docker/Dockerfile-cargo-cache --push -t 127.0.0.1:5000/cargo_cache --platform << parameters.platforms >> --progress=plain .
+      - run:
+          command: docker buildx build -f docker/Dockerfile --push --platform << parameters.platforms >> --label gitsha1=${CIRCLE_SHA1} << parameters.tag >> --build-arg BASE_IMAGE=127.0.0.1:5000/cargo_cache --build-arg CARGO_NET_OFFLINE=true --progress=plain  .
+          no_output_timeout: 30m

INSTALL.md

@@ -151,29 +151,15 @@ sudo pacman -S base-devel python python-pip \
 
 ##### CentOS/Fedora
 
-Installing prerequisites on CentOS 8 or Fedora>26:
+Installing prerequisites on CentOS or Fedora Linux:
 
 ```sh
 sudo dnf install libtiff-devel libjpeg-devel libzip-devel freetype-devel \
-                 libwebp-devel tk-devel redhat-rpm-config \
-                 python3-virtualenv libffi-devel openssl-devel
+                 libwebp-devel libxml2-devel libxslt-devel libpq-devel \
+                 python3-virtualenv libffi-devel openssl-devel python3-devel
 sudo dnf groupinstall "Development Tools"
 ```
 
-Installing prerequisites on CentOS 7 or Fedora<=25:
-
-```sh
-sudo yum install libtiff-devel libjpeg-devel libzip-devel freetype-devel \
-                 lcms2-devel libwebp-devel tcl-devel tk-devel redhat-rpm-config \
-                 python3-virtualenv libffi-devel openssl-devel
-sudo yum groupinstall "Development Tools"
-```
-
-Note that Synapse does not support versions of SQLite before 3.11, and CentOS 7
-uses SQLite 3.7. You may be able to work around this by installing a more
-recent SQLite version, but it is recommended that you instead use a Postgres
-database: see [docs/postgres.md](docs/postgres.md).
-
 ##### macOS
 
 Installing prerequisites on macOS:

changelog.d/9003.misc

@@ -0,0 +1 @@
+Fix 'object name reserved for internal use' errors with recent versions of SQLite.

changelog.d/9123.misc

@@ -0,0 +1 @@
+Add experimental support for running Synapse with PyPy.

changelog.d/9150.feature

@@ -0,0 +1 @@
+New API /_synapse/admin/rooms/{roomId}/context/{eventId}.

changelog.d/9240.misc

@@ -0,0 +1 @@
+Deny access to additional IP addresses by default.

changelog.d/9257.bugfix

@@ -0,0 +1 @@
+Fix long-standing bug where sending email push would fail for rooms that the server had since left.

changelog.d/9291.doc

@@ -0,0 +1 @@
+Add note to `auto_join_rooms` config option explaining existing rooms must be publicly joinable.

changelog.d/9296.bugfix

@@ -0,0 +1 @@
+Fix bug in Synapse 1.27.0rc1 which meant the "session expired" error page during SSO registration was badly formatted.

changelog.d/9299.misc

@@ -0,0 +1 @@
+Update the `Cursor` type hints to better match PEP 249.

changelog.d/9300.feature

@@ -0,0 +1 @@
+Further improvements to the user experience of registration via single sign-on.

changelog.d/9301.feature

@@ -0,0 +1 @@
+Further improvements to the user experience of registration via single sign-on.

changelog.d/9305.misc

@@ -0,0 +1 @@
+Add debug logging for SRV lookups. Contributed by @Bubu.

changelog.d/9307.misc

@@ -0,0 +1 @@
+Improve logging for OIDC login flow.

changelog.d/9308.doc

@@ -0,0 +1 @@
+Correct name of Synapse's service file in TURN howto.

changelog.d/9311.feature

@@ -0,0 +1 @@
+Add hook to spam checker modules that allow checking file uploads and remote downloads.

changelog.d/9317.doc

@@ -0,0 +1 @@
+Fix the braces in the `oidc_providers` section of the sample config.

changelog.d/9321.bugfix

@@ -0,0 +1 @@
+Assert a maximum length for the `client_secret` parameter for spec compliance.

changelog.d/9322.doc

@@ -0,0 +1 @@
+Update installation instructions on Fedora.

changelog.d/9326.misc

@@ -0,0 +1 @@
+Share the code for handling required attributes between the CAS and SAML handlers.

changelog.d/9333.bugfix

@@ -0,0 +1 @@
+Fix additional errors when previewing URLs: "AttributeError 'NoneType' object has no attribute 'xpath'" and "ValueError: Unicode strings with encoding declaration are not supported. Please use bytes input or XML fragments without declaration.".

changelog.d/9361.bugfix

@@ -0,0 +1 @@
+Fix a bug causing Synapse to impose the wrong type constraints on fields when processing responses from appservices to `/_matrix/app/v1/thirdparty/user/{protocol}`.

changelog.d/9362.misc

@@ -0,0 +1 @@
+Clean up the code to load the metadata for OpenID Connect identity providers.

changelog.d/9376.feature

@@ -0,0 +1 @@
+Add support for receiving OpenID Connect authentication responses via form `POST`s rather than `GET`s.

changelog.d/9377.misc

@@ -0,0 +1 @@
+Convert tests to use `HomeserverTestCase`.

changelog.d/9381.misc

@@ -0,0 +1 @@
+Update the version of black used to 20.8b1.

changelog.d/9384.misc

@@ -0,0 +1 @@
+Allow OIDC config to override discovered values.

changelog.d/9391.bugfix

@@ -0,0 +1 @@
+Fix bug where Synapse would occaisonally stop reconnecting after the connection was lost.

changelog.d/9394.misc

@@ -0,0 +1 @@
+Remove some dead code from the acceptance of room invites path.

changelog.d/9395.bugfix

@@ -0,0 +1 @@
+Fix a long-standing bug when upgrading a room: "TypeError: '>' not supported between instances of 'NoneType' and 'int'".

changelog.d/9396.misc

@@ -0,0 +1 @@
+Convert tests to use `HomeserverTestCase`.

changelog.d/9404.doc

@@ -0,0 +1 @@
+Update docs for using Gitea as OpenID provider.

changelog.d/9407.doc

@@ -0,0 +1 @@
+Document that pusher instances are shardable.

changelog.d/9423.bugfix

@@ -0,0 +1 @@
+Fix building docker images for 32-bit ARM.

contrib/cmdclient/console.py

@@ -92,7 +92,7 @@ class SynapseCmd(cmd.Cmd):
         return self.config["user"].split(":")[1]
 
     def do_config(self, line):
-        """ Show the config for this client: "config"
+        """Show the config for this client: "config"
         Edit a key value mapping: "config key value" e.g. "config token 1234"
         Config variables:
             user: The username to auth with.
@@ -360,7 +360,7 @@ class SynapseCmd(cmd.Cmd):
             print(e)
 
     def do_topic(self, line):
-        """"topic [set|get] <roomid> [<newtopic>]"
+        """ "topic [set|get] <roomid> [<newtopic>]"
         Set the topic for a room: topic set <roomid> <newtopic>
         Get the topic for a room: topic get <roomid>
         """
@@ -690,7 +690,7 @@ class SynapseCmd(cmd.Cmd):
         self._do_presence_state(2, line)
 
     def _parse(self, line, keys, force_keys=False):
-        """ Parses the given line.
+        """Parses the given line.
 
         Args:
             line : The line to parse
@@ -721,7 +721,7 @@ class SynapseCmd(cmd.Cmd):
         query_params={"access_token": None},
         alt_text=None,
     ):
-        """ Runs an HTTP request and pretty prints the output.
+        """Runs an HTTP request and pretty prints the output.
 
         Args:
             method: HTTP method

contrib/cmdclient/http.py

@@ -23,11 +23,10 @@ from twisted.web.http_headers import Headers
 
 
 class HttpClient:
-    """ Interface for talking json over http
-    """
+    """Interface for talking json over http"""
 
     def put_json(self, url, data):
-        """ Sends the specifed json data using PUT
+        """Sends the specifed json data using PUT
 
         Args:
             url (str): The URL to PUT data to.
@@ -41,7 +40,7 @@ class HttpClient:
         pass
 
     def get_json(self, url, args=None):
-        """ Gets some json from the given host homeserver and path
+        """Gets some json from the given host homeserver and path
 
         Args:
             url (str): The URL to GET data from.
@@ -58,7 +57,7 @@ class HttpClient:
 
 
 class TwistedHttpClient(HttpClient):
-    """ Wrapper around the twisted HTTP client api.
+    """Wrapper around the twisted HTTP client api.
 
     Attributes:
         agent (twisted.web.client.Agent): The twisted Agent used to send the
@@ -87,8 +86,7 @@ class TwistedHttpClient(HttpClient):
         defer.returnValue(json.loads(body))
 
     def _create_put_request(self, url, json_data, headers_dict={}):
-        """ Wrapper of _create_request to issue a PUT request
-        """
+        """Wrapper of _create_request to issue a PUT request"""
 
         if "Content-Type" not in headers_dict:
             raise defer.error(RuntimeError("Must include Content-Type header for PUTs"))
@@ -98,8 +96,7 @@ class TwistedHttpClient(HttpClient):
         )
 
     def _create_get_request(self, url, headers_dict={}):
-        """ Wrapper of _create_request to issue a GET request
-        """
+        """Wrapper of _create_request to issue a GET request"""
         return self._create_request("GET", url, headers_dict=headers_dict)
 
     @defer.inlineCallbacks
@@ -127,8 +124,7 @@ class TwistedHttpClient(HttpClient):
 
     @defer.inlineCallbacks
     def _create_request(self, method, url, producer=None, headers_dict={}):
-        """ Creates and sends a request to the given url
-        """
+        """Creates and sends a request to the given url"""
         headers_dict["User-Agent"] = ["Synapse Cmd Client"]
 
         retries_left = 5
@@ -185,8 +181,7 @@ class _RawProducer:
 
 
 class _JsonProducer:
-    """ Used by the twisted http client to create the HTTP body from json
-    """
+    """Used by the twisted http client to create the HTTP body from json"""
 
     def __init__(self, jsn):
         self.data = jsn

contrib/experiments/cursesio.py

@@ -63,8 +63,7 @@ class CursesStdIO:
         self.redraw()
 
     def redraw(self):
-        """ method for redisplaying lines
-            based on internal list of lines """
+        """method for redisplaying lines based on internal list of lines"""
 
         self.stdscr.clear()
         self.paintStatus(self.statusText)

contrib/experiments/test_messaging.py

@@ -56,7 +56,7 @@ def excpetion_errback(failure):
 
 
 class InputOutput:
-    """ This is responsible for basic I/O so that a user can interact with
+    """This is responsible for basic I/O so that a user can interact with
     the example app.
     """
 
@@ -68,8 +68,7 @@ class InputOutput:
         self.server = server
 
     def on_line(self, line):
-        """ This is where we process commands.
-        """
+        """This is where we process commands."""
 
         try:
             m = re.match(r"^join (\S+)$", line)
@@ -133,7 +132,7 @@ class IOLoggerHandler(logging.Handler):
 
 
 class Room:
-    """ Used to store (in memory) the current membership state of a room, and
+    """Used to store (in memory) the current membership state of a room, and
     which home servers we should send PDUs associated with the room to.
     """
 
@@ -148,8 +147,7 @@ class Room:
         self.have_got_metadata = False
 
     def add_participant(self, participant):
-        """ Someone has joined the room
-        """
+        """Someone has joined the room"""
         self.participants.add(participant)
         self.invited.discard(participant)
 
@@ -160,14 +158,13 @@ class Room:
             self.oldest_server = server
 
     def add_invited(self, invitee):
-        """ Someone has been invited to the room
-        """
+        """Someone has been invited to the room"""
         self.invited.add(invitee)
         self.servers.add(origin_from_ucid(invitee))
 
 
 class HomeServer(ReplicationHandler):
-    """ A very basic home server implentation that allows people to join a
+    """A very basic home server implentation that allows people to join a
     room and then invite other people.
     """
 
@@ -181,8 +178,7 @@ class HomeServer(ReplicationHandler):
         self.output = output
 
     def on_receive_pdu(self, pdu):
-        """ We just received a PDU
-        """
+        """We just received a PDU"""
         pdu_type = pdu.pdu_type
 
         if pdu_type == "sy.room.message":
@@ -199,23 +195,20 @@ class HomeServer(ReplicationHandler):
             )
 
     def _on_message(self, pdu):
-        """ We received a message
-        """
+        """We received a message"""
         self.output.print_line(
             "#%s %s %s" % (pdu.context, pdu.content["sender"], pdu.content["body"])
         )
 
     def _on_join(self, context, joinee):
-        """ Someone has joined a room, either a remote user or a local user
-        """
+        """Someone has joined a room, either a remote user or a local user"""
         room = self._get_or_create_room(context)
         room.add_participant(joinee)
 
         self.output.print_line("#%s %s %s" % (context, joinee, "*** JOINED"))
 
     def _on_invite(self, origin, context, invitee):
-        """ Someone has been invited
-        """
+        """Someone has been invited"""
         room = self._get_or_create_room(context)
         room.add_invited(invitee)
 
@@ -228,8 +221,7 @@ class HomeServer(ReplicationHandler):
 
     @defer.inlineCallbacks
     def send_message(self, room_name, sender, body):
-        """ Send a message to a room!
-        """
+        """Send a message to a room!"""
         destinations = yield self.get_servers_for_context(room_name)
 
         try:
@@ -247,8 +239,7 @@ class HomeServer(ReplicationHandler):
 
     @defer.inlineCallbacks
     def join_room(self, room_name, sender, joinee):
-        """ Join a room!
-        """
+        """Join a room!"""
         self._on_join(room_name, joinee)
 
         destinations = yield self.get_servers_for_context(room_name)
@@ -269,8 +260,7 @@ class HomeServer(ReplicationHandler):
 
     @defer.inlineCallbacks
     def invite_to_room(self, room_name, sender, invitee):
-        """ Invite someone to a room!
-        """
+        """Invite someone to a room!"""
         self._on_invite(self.server_name, room_name, invitee)
 
         destinations = yield self.get_servers_for_context(room_name)

contrib/jitsimeetbridge/jitsimeetbridge.py

@@ -193,15 +193,12 @@ class TrivialXmppClient:
         time.sleep(7)
         print("SSRC spammer started")
         while self.running:
-            ssrcMsg = (
-                "<presence to='%(tojid)s' xmlns='jabber:client'><x xmlns='http://jabber.org/protocol/muc'/><c xmlns='http://jabber.org/protocol/caps' hash='sha-1' node='http://jitsi.org/jitsimeet' ver='0WkSdhFnAUxrz4ImQQLdB80GFlE='/><nick xmlns='http://jabber.org/protocol/nick'>%(nick)s</nick><stats xmlns='http://jitsi.org/jitmeet/stats'><stat name='bitrate_download' value='175'/><stat name='bitrate_upload' value='176'/><stat name='packetLoss_total' value='0'/><stat name='packetLoss_download' value='0'/><stat name='packetLoss_upload' value='0'/></stats><media xmlns='http://estos.de/ns/mjs'><source type='audio' ssrc='%(assrc)s' direction='sendre'/><source type='video' ssrc='%(vssrc)s' direction='sendre'/></media></presence>"
-                % {
-                    "tojid": "%s@%s/%s" % (ROOMNAME, ROOMDOMAIN, self.shortJid),
-                    "nick": self.userId,
-                    "assrc": self.ssrcs["audio"],
-                    "vssrc": self.ssrcs["video"],
-                }
-            )
+            ssrcMsg = "<presence to='%(tojid)s' xmlns='jabber:client'><x xmlns='http://jabber.org/protocol/muc'/><c xmlns='http://jabber.org/protocol/caps' hash='sha-1' node='http://jitsi.org/jitsimeet' ver='0WkSdhFnAUxrz4ImQQLdB80GFlE='/><nick xmlns='http://jabber.org/protocol/nick'>%(nick)s</nick><stats xmlns='http://jitsi.org/jitmeet/stats'><stat name='bitrate_download' value='175'/><stat name='bitrate_upload' value='176'/><stat name='packetLoss_total' value='0'/><stat name='packetLoss_download' value='0'/><stat name='packetLoss_upload' value='0'/></stats><media xmlns='http://estos.de/ns/mjs'><source type='audio' ssrc='%(assrc)s' direction='sendre'/><source type='video' ssrc='%(vssrc)s' direction='sendre'/></media></presence>" % {
+                "tojid": "%s@%s/%s" % (ROOMNAME, ROOMDOMAIN, self.shortJid),
+                "nick": self.userId,
+                "assrc": self.ssrcs["audio"],
+                "vssrc": self.ssrcs["video"],
+            }
             res = self.sendIq(ssrcMsg)
             print("reply from ssrc announce: ", res)
             time.sleep(10)

docker/Dockerfile

@@ -12,11 +12,13 @@
 #
 
 ARG PYTHON_VERSION=3.8
+ARG BASE_IMAGE=docker.io/python:${PYTHON_VERSION}-slim
+ARG CARGO_NET_OFFLINE=false
 
 ###
 ### Stage 0: builder
 ###
-FROM docker.io/python:${PYTHON_VERSION}-slim as builder
+FROM ${BASE_IMAGE} as builder
 
 # install the OS build deps
 RUN apt-get update && apt-get install -y \
@@ -32,9 +34,16 @@ RUN apt-get update && apt-get install -y \
     zlib1g-dev \
  && rm -rf /var/lib/apt/lists/*
 
+ENV CARGO_NET_OFFLINE=${CARGO_NET_OFFLINE}
+
 # Build dependencies that are not available as wheels, to speed up rebuilds
 RUN pip install --prefix="/install" --no-warn-script-location \
-        cryptography \
+        lxml
+
+RUN pip install --prefix="/install" --no-warn-script-location \
+        cryptography
+
+RUN pip install --prefix="/install" --no-warn-script-location \
         frozendict \
         jaeger-client \
         opentracing \

docker/Dockerfile-cargo-cache

@@ -0,0 +1,21 @@
+# A docker file that caches the cargo index for the cryptography deps. This is
+# mainly useful for multi-arch builds where fetching the index from the internet
+# fails for 32bit archs built on 64 bit platforms.
+
+ARG PYTHON_VERSION=3.8
+
+FROM --platform=$BUILDPLATFORM docker.io/python:${PYTHON_VERSION}-slim as builder
+
+RUN apt-get update && apt-get install -y \
+    rustc \
+    && rm -rf /var/lib/apt/lists/*
+
+RUN pip download --no-binary cryptography --no-deps cryptography
+
+RUN tar -xf cryptography*.tar.gz --wildcards cryptography*/src/rust/
+
+RUN cd cryptography*/src/rust && cargo fetch
+
+FROM docker.io/python:${PYTHON_VERSION}-slim
+
+COPY --from=builder /root/.cargo /root/.cargo

docs/admin_api/rooms.md

@@ -10,6 +10,7 @@
   * [Undoing room shutdowns](#undoing-room-shutdowns)
 - [Make Room Admin API](#make-room-admin-api)
 - [Forward Extremities Admin API](#forward-extremities-admin-api)
+- [Event Context API](#event-context-api)
 
 # List Room API
 
@@ -594,3 +595,121 @@ that were deleted.
   "deleted": 1
 }
 ```
+
+# Event Context API
+
+This API lets a client find the context of an event. This is designed primarily to investigate abuse reports.
+
+```
+GET /_synapse/admin/v1/rooms/<room_id>/context/<event_id>
+```
+
+This API mimmicks [GET /_matrix/client/r0/rooms/{roomId}/context/{eventId}](https://matrix.org/docs/spec/client_server/r0.6.1#get-matrix-client-r0-rooms-roomid-context-eventid). Please refer to the link for all details on parameters and reseponse.
+
+Example response:
+
+```json
+{
+  "end": "t29-57_2_0_2",
+  "events_after": [
+    {
+      "content": {
+        "body": "This is an example text message",
+        "msgtype": "m.text",
+        "format": "org.matrix.custom.html",
+        "formatted_body": "<b>This is an example text message</b>"
+      },
+      "type": "m.room.message",
+      "event_id": "$143273582443PhrSn:example.org",
+      "room_id": "!636q39766251:example.com",
+      "sender": "@example:example.org",
+      "origin_server_ts": 1432735824653,
+      "unsigned": {
+        "age": 1234
+      }
+    }
+  ],
+  "event": {
+    "content": {
+      "body": "filename.jpg",
+      "info": {
+        "h": 398,
+        "w": 394,
+        "mimetype": "image/jpeg",
+        "size": 31037
+      },
+      "url": "mxc://example.org/JWEIFJgwEIhweiWJE",
+      "msgtype": "m.image"
+    },
+    "type": "m.room.message",
+    "event_id": "$f3h4d129462ha:example.com",
+    "room_id": "!636q39766251:example.com",
+    "sender": "@example:example.org",
+    "origin_server_ts": 1432735824653,
+    "unsigned": {
+      "age": 1234
+    }
+  },
+  "events_before": [
+    {
+      "content": {
+        "body": "something-important.doc",
+        "filename": "something-important.doc",
+        "info": {
+          "mimetype": "application/msword",
+          "size": 46144
+        },
+        "msgtype": "m.file",
+        "url": "mxc://example.org/FHyPlCeYUSFFxlgbQYZmoEoe"
+      },
+      "type": "m.room.message",
+      "event_id": "$143273582443PhrSn:example.org",
+      "room_id": "!636q39766251:example.com",
+      "sender": "@example:example.org",
+      "origin_server_ts": 1432735824653,
+      "unsigned": {
+        "age": 1234
+      }
+    }
+  ],
+  "start": "t27-54_2_0_2",
+  "state": [
+    {
+      "content": {
+        "creator": "@example:example.org",
+        "room_version": "1",
+        "m.federate": true,
+        "predecessor": {
+          "event_id": "$something:example.org",
+          "room_id": "!oldroom:example.org"
+        }
+      },
+      "type": "m.room.create",
+      "event_id": "$143273582443PhrSn:example.org",
+      "room_id": "!636q39766251:example.com",
+      "sender": "@example:example.org",
+      "origin_server_ts": 1432735824653,
+      "unsigned": {
+        "age": 1234
+      },
+      "state_key": ""
+    },
+    {
+      "content": {
+        "membership": "join",
+        "avatar_url": "mxc://example.org/SEsfnsuifSDFSSEF",
+        "displayname": "Alice Margatroid"
+      },
+      "type": "m.room.member",
+      "event_id": "$143273582443PhrSn:example.org",
+      "room_id": "!636q39766251:example.com",
+      "sender": "@example:example.org",
+      "origin_server_ts": 1432735824653,
+      "unsigned": {
+        "age": 1234
+      },
+      "state_key": "@alice:example.org"
+    }
+  ]
+}
+```

docs/code_style.md

@@ -8,16 +8,16 @@ errors in code.
 
 The necessary tools are detailed below.
 
+First install them with:
+
+    pip install -e ".[lint,mypy]"
+
 -   **black**
 
     The Synapse codebase uses [black](https://pypi.org/project/black/)
     as an opinionated code formatter, ensuring all comitted code is
     properly formatted.
 
-    First install `black` with:
-
-        pip install --upgrade black
-
     Have `black` auto-format your code (it shouldn't change any
     functionality) with:
 
@@ -28,10 +28,6 @@ The necessary tools are detailed below.
     `flake8` is a code checking tool. We require code to pass `flake8`
     before being merged into the codebase.
 
-    Install `flake8` with:
-
-        pip install --upgrade flake8 flake8-comprehensions
-
     Check all application and test code with:
 
         flake8 synapse tests
@@ -41,10 +37,6 @@ The necessary tools are detailed below.
     `isort` ensures imports are nicely formatted, and can suggest and
     auto-fix issues such as double-importing.
 
-    Install `isort` with:
-
-        pip install --upgrade isort
-
     Auto-fix imports with:
 
         isort -rc synapse tests

docs/openid.md

@@ -365,7 +365,7 @@ login mechanism needs an attribute to uniquely identify users, and that endpoint
 does not return a `sub` property, an alternative `subject_claim` has to be set.
 
 1. Create a new application.
-2. Add this Callback URL: `[synapse public baseurl]/_synapse/oidc/callback`
+2. Add this Callback URL: `[synapse public baseurl]/_synapse/client/oidc/callback`
 
 Synapse config:
 
@@ -388,3 +388,25 @@ oidc_providers:
         localpart_template: "{{ user.login }}"
         display_name_template: "{{ user.full_name }}" 
 ```
+
+### XWiki
+
+Install [OpenID Connect Provider](https://extensions.xwiki.org/xwiki/bin/view/Extension/OpenID%20Connect/OpenID%20Connect%20Provider/) extension in your [XWiki](https://www.xwiki.org) instance.
+
+Synapse config:
+
+```yaml
+oidc_providers:
+  - idp_id: xwiki
+    idp_name: "XWiki"
+    issuer: "https://myxwikihost/xwiki/oidc/"
+    client_id: "your-client-id" # TO BE FILLED
+    # Needed until https://github.com/matrix-org/synapse/issues/9212 is fixed
+    client_secret: "dontcare"
+    scopes: ["openid", "profile"]
+    user_profile_method: "userinfo_endpoint"
+    user_mapping_provider:
+      config:
+        localpart_template: "{{ user.preferred_username }}"
+        display_name_template: "{{ user.name }}"
+```

docs/sample_config.yaml

@@ -165,6 +165,7 @@ pid_file: DATADIR/homeserver.pid
 #  - '100.64.0.0/10'
 #  - '192.0.0.0/24'
 #  - '169.254.0.0/16'
+#  - '192.88.99.0/24'
 #  - '198.18.0.0/15'
 #  - '192.0.2.0/24'
 #  - '198.51.100.0/24'
@@ -173,6 +174,9 @@ pid_file: DATADIR/homeserver.pid
 #  - '::1/128'
 #  - 'fe80::/10'
 #  - 'fc00::/7'
+#  - '2001:db8::/32'
+#  - 'ff00::/8'
+#  - 'fec0::/10'
 
 # List of IP address CIDR ranges that should be allowed for federation,
 # identity servers, push servers, and for checking key validity for
@@ -990,6 +994,7 @@ media_store_path: "DATADIR/media_store"
 #  - '100.64.0.0/10'
 #  - '192.0.0.0/24'
 #  - '169.254.0.0/16'
+#  - '192.88.99.0/24'
 #  - '198.18.0.0/15'
 #  - '192.0.2.0/24'
 #  - '198.51.100.0/24'
@@ -998,6 +1003,9 @@ media_store_path: "DATADIR/media_store"
 #  - '::1/128'
 #  - 'fe80::/10'
 #  - 'fc00::/7'
+#  - '2001:db8::/32'
+#  - 'ff00::/8'
+#  - 'fec0::/10'
 
 # List of IP address CIDR ranges that the URL preview spider is allowed
 # to access even if they are specified in url_preview_ip_range_blacklist.
@@ -1318,6 +1326,8 @@ account_threepid_delegates:
 # By default, any room aliases included in this list will be created
 # as a publicly joinable room when the first user registers for the
 # homeserver. This behaviour can be customised with the settings below.
+# If the room already exists, make certain it is a publicly joinable
+# room. The join rule of the room must be set to 'public'.
 #
 #auto_join_rooms:
 #  - "#example:example.com"
@@ -1860,9 +1870,9 @@ oidc_providers:
   #  user_mapping_provider:
   #    config:
   #      subject_claim: "id"
-  #      localpart_template: "{ user.login }"
-  #      display_name_template: "{ user.name }"
-  #      email_template: "{ user.email }"
+  #      localpart_template: "{{ user.login }}"
+  #      display_name_template: "{{ user.name }}"
+  #      email_template: "{{ user.email }}"
 
   # For use with Keycloak
   #
@@ -1889,8 +1899,8 @@ oidc_providers:
   #  user_mapping_provider:
   #    config:
   #      subject_claim: "id"
-  #      localpart_template: "{ user.login }"
-  #      display_name_template: "{ user.name }"
+  #      localpart_template: "{{ user.login }}"
+  #      display_name_template: "{{ user.name }}"
 
 
 # Enable Central Authentication Service (CAS) for registration and login.
@@ -2222,7 +2232,7 @@ ui_auth:
     # session to be active.
     #
     # This defaults to 0, meaning the user is queried for their credentials
-    # before every action, but this can be overridden to alow a single
+    # before every action, but this can be overridden to allow a single
     # validation to be re-used.  This weakens the protections afforded by
     # the user-interactive authentication process, by allowing for multiple
     # (and potentially different) operations to use the same validation session.

docs/spam_checker.md

@@ -61,6 +61,9 @@ class ExampleSpamChecker:
 
     async def check_registration_for_spam(self, email_threepid, username, request_info):
         return RegistrationBehaviour.ALLOW  # allow all registrations
+
+    async def check_media_file_for_spam(self, file_wrapper, file_info):
+        return False  # allow all media
 ```
 
 ## Configuration

docs/turn-howto.md

@@ -187,7 +187,7 @@ After updating the homeserver configuration, you must restart synapse:
     ```
   * If you use systemd:
     ```
-    systemctl restart synapse.service
+    systemctl restart matrix-synapse.service
     ```
 ... and then reload any clients (or wait an hour for them to refresh their
 settings).

docs/workers.md

@@ -373,7 +373,15 @@ Handles sending push notifications to sygnal and email. Doesn't handle any
 REST endpoints itself, but you should set `start_pushers: False` in the
 shared configuration file to stop the main synapse sending push notifications.
 
-Note this worker cannot be load-balanced: only one instance should be active.
+To run multiple instances at once the `pusher_instances` option should list all
+pusher instances by their worker name, e.g.:
+
+```yaml
+pusher_instances:
+    - pusher_worker1
+    - pusher_worker2
+```
+
 
 ### `synapse.app.appservice`
 

scripts-dev/make_full_schema.sh

@@ -162,12 +162,23 @@ else
 fi
 
 # Delete schema_version, applied_schema_deltas and applied_module_schemas tables
+# Also delete any shadow tables from fts4
 # This needs to be done after synapse_port_db is run
 echo "Dropping unwanted db tables..."
 SQL="
 DROP TABLE schema_version;
 DROP TABLE applied_schema_deltas;
 DROP TABLE applied_module_schemas;
+DROP TABLE event_search_content;
+DROP TABLE event_search_segments;
+DROP TABLE event_search_segdir;
+DROP TABLE event_search_docsize;
+DROP TABLE event_search_stat;
+DROP TABLE user_directory_search_content;
+DROP TABLE user_directory_search_segments;
+DROP TABLE user_directory_search_segdir;
+DROP TABLE user_directory_search_docsize;
+DROP TABLE user_directory_search_stat;
 "
 sqlite3 "$SQLITE_DB" <<< "$SQL"
 psql $POSTGRES_DB_NAME -U "$POSTGRES_USERNAME" -w <<< "$SQL"

scripts-dev/mypy_synapse_plugin.py

@@ -87,7 +87,9 @@ def cached_function_method_signature(ctx: MethodSigContext) -> CallableType:
     arg_kinds.append(ARG_NAMED_OPT)  # Arg is an optional kwarg.
 
     signature = signature.copy_modified(
-        arg_types=arg_types, arg_names=arg_names, arg_kinds=arg_kinds,
+        arg_types=arg_types,
+        arg_names=arg_names,
+        arg_kinds=arg_kinds,
     )
 
     return signature

setup.py

@@ -97,7 +97,7 @@ CONDITIONAL_REQUIREMENTS["all"] = list(ALL_OPTIONAL_REQUIREMENTS)
 # We pin black so that our tests don't start failing on new releases.
 CONDITIONAL_REQUIREMENTS["lint"] = [
     "isort==5.7.0",
-    "black==19.10b0",
+    "black==20.8b1",
     "flake8-comprehensions",
     "flake8",
 ]

stubs/sortedcontainers/sorteddict.pyi

@@ -89,12 +89,16 @@ class SortedDict(Dict[_KT, _VT]):
     def __reduce__(
         self,
     ) -> Tuple[
-        Type[SortedDict[_KT, _VT]], Tuple[Callable[[_KT], Any], List[Tuple[_KT, _VT]]],
+        Type[SortedDict[_KT, _VT]],
+        Tuple[Callable[[_KT], Any], List[Tuple[_KT, _VT]]],
     ]: ...
     def __repr__(self) -> str: ...
     def _check(self) -> None: ...
     def islice(
-        self, start: Optional[int] = ..., stop: Optional[int] = ..., reverse=bool,
+        self,
+        start: Optional[int] = ...,
+        stop: Optional[int] = ...,
+        reverse=bool,
     ) -> Iterator[_KT]: ...
     def bisect_left(self, value: _KT) -> int: ...
     def bisect_right(self, value: _KT) -> int: ...

stubs/sortedcontainers/sortedlist.pyi

@@ -31,7 +31,9 @@ class SortedList(MutableSequence[_T]):
 
     DEFAULT_LOAD_FACTOR: int = ...
     def __init__(
-        self, iterable: Optional[Iterable[_T]] = ..., key: Optional[_Key[_T]] = ...,
+        self,
+        iterable: Optional[Iterable[_T]] = ...,
+        key: Optional[_Key[_T]] = ...,
     ): ...
     # NB: currently mypy does not honour return type, see mypy #3307
     @overload
@@ -76,10 +78,18 @@ class SortedList(MutableSequence[_T]):
     def __len__(self) -> int: ...
     def reverse(self) -> None: ...
     def islice(
-        self, start: Optional[int] = ..., stop: Optional[int] = ..., reverse=bool,
+        self,
+        start: Optional[int] = ...,
+        stop: Optional[int] = ...,
+        reverse=bool,
     ) -> Iterator[_T]: ...
     def _islice(
-        self, min_pos: int, min_idx: int, max_pos: int, max_idx: int, reverse: bool,
+        self,
+        min_pos: int,
+        min_idx: int,
+        max_pos: int,
+        max_idx: int,
+        reverse: bool,
     ) -> Iterator[_T]: ...
     def irange(
         self,

synapse/api/auth.py

@@ -168,7 +168,7 @@ class Auth:
         rights: str = "access",
         allow_expired: bool = False,
     ) -> synapse.types.Requester:
-        """ Get a registered user's ID.
+        """Get a registered user's ID.
 
         Args:
             request: An HTTP request with an access_token query parameter.
@@ -294,9 +294,12 @@ class Auth:
         return user_id, app_service
 
     async def get_user_by_access_token(
-        self, token: str, rights: str = "access", allow_expired: bool = False,
+        self,
+        token: str,
+        rights: str = "access",
+        allow_expired: bool = False,
     ) -> TokenLookupResult:
-        """ Validate access token and get user_id from it
+        """Validate access token and get user_id from it
 
         Args:
             token: The access token to get the user by
@@ -489,7 +492,7 @@ class Auth:
         return service
 
     async def is_server_admin(self, user: UserID) -> bool:
-        """ Check if the given user is a local server admin.
+        """Check if the given user is a local server admin.
 
         Args:
             user: user to check
@@ -500,7 +503,10 @@ class Auth:
         return await self.store.is_server_admin(user)
 
     def compute_auth_events(
-        self, event, current_state_ids: StateMap[str], for_verification: bool = False,
+        self,
+        event,
+        current_state_ids: StateMap[str],
+        for_verification: bool = False,
     ) -> List[str]:
         """Given an event and current state return the list of event IDs used
         to auth an event.

synapse/api/constants.py

@@ -128,8 +128,7 @@ class UserTypes:
 
 
 class RelationTypes:
-    """The types of relations known to this server.
-    """
+    """The types of relations known to this server."""
 
     ANNOTATION = "m.annotation"
     REPLACE = "m.replace"

synapse/api/errors.py

@@ -390,8 +390,7 @@ class InvalidCaptchaError(SynapseError):
 
 
 class LimitExceededError(SynapseError):
-    """A client has sent too many requests and is being throttled.
-    """
+    """A client has sent too many requests and is being throttled."""
 
     def __init__(
         self,
@@ -408,8 +407,7 @@ class LimitExceededError(SynapseError):
 
 
 class RoomKeysVersionError(SynapseError):
-    """A client has tried to upload to a non-current version of the room_keys store
-    """
+    """A client has tried to upload to a non-current version of the room_keys store"""
 
     def __init__(self, current_version: str):
         """
@@ -426,7 +424,9 @@ class UnsupportedRoomVersionError(SynapseError):
 
     def __init__(self, msg: str = "Homeserver does not support this room version"):
         super().__init__(
-            code=400, msg=msg, errcode=Codes.UNSUPPORTED_ROOM_VERSION,
+            code=400,
+            msg=msg,
+            errcode=Codes.UNSUPPORTED_ROOM_VERSION,
         )
 
 
@@ -461,8 +461,7 @@ class IncompatibleRoomVersionError(SynapseError):
 
 
 class PasswordRefusedError(SynapseError):
-    """A password has been refused, either during password reset/change or registration.
-    """
+    """A password has been refused, either during password reset/change or registration."""
 
     def __init__(
         self,
@@ -470,7 +469,9 @@ class PasswordRefusedError(SynapseError):
         errcode: str = Codes.WEAK_PASSWORD,
     ):
         super().__init__(
-            code=400, msg=msg, errcode=errcode,
+            code=400,
+            msg=msg,
+            errcode=errcode,
         )
 
 
@@ -493,7 +494,7 @@ class RequestSendFailed(RuntimeError):
 
 
 def cs_error(msg: str, code: str = Codes.UNKNOWN, **kwargs):
-    """ Utility method for constructing an error response for client-server
+    """Utility method for constructing an error response for client-server
     interactions.
 
     Args:
@@ -510,7 +511,7 @@ def cs_error(msg: str, code: str = Codes.UNKNOWN, **kwargs):
 
 
 class FederationError(RuntimeError):
-    """  This class is used to inform remote homeservers about erroneous
+    """This class is used to inform remote homeservers about erroneous
     PDUs they sent us.
 
     FATAL: The remote server could not interpret the source event.

synapse/api/presence.py

@@ -56,8 +56,7 @@ class UserPresenceState(
 
     @classmethod
     def default(cls, user_id):
-        """Returns a default presence state.
-        """
+        """Returns a default presence state."""
         return cls(
             user_id=user_id,
             state=PresenceState.OFFLINE,

synapse/app/_base.py

@@ -58,7 +58,7 @@ def register_sighup(func, *args, **kwargs):
 
 
 def start_worker_reactor(appname, config, run_command=reactor.run):
-    """ Run the reactor in the main process
+    """Run the reactor in the main process
 
     Daemonizes if necessary, and then configures some resources, before starting
     the reactor. Pulls configuration from the 'worker' settings in 'config'.
@@ -93,7 +93,7 @@ def start_reactor(
     logger,
     run_command=reactor.run,
 ):
-    """ Run the reactor in the main process
+    """Run the reactor in the main process
 
     Daemonizes if necessary, and then configures some resources, before starting
     the reactor
@@ -313,9 +313,7 @@ async def start(hs: "synapse.server.HomeServer", listeners: Iterable[ListenerCon
     refresh_certificate(hs)
 
     # Start the tracer
-    synapse.logging.opentracing.init_tracer(  # type: ignore[attr-defined] # noqa
-        hs
-    )
+    synapse.logging.opentracing.init_tracer(hs)  # type: ignore[attr-defined] # noqa
 
     # It is now safe to start your Synapse.
     hs.start_listening(listeners)
@@ -370,8 +368,7 @@ def setup_sentry(hs):
 
 
 def setup_sdnotify(hs):
-    """Adds process state hooks to tell systemd what we are up to.
-    """
+    """Adds process state hooks to tell systemd what we are up to."""
 
     # Tell systemd our state, if we're using it. This will silently fail if
     # we're not using systemd.
@@ -405,8 +402,7 @@ def install_dns_limiter(reactor, max_dns_requests_in_flight=100):
 
 
 class _LimitedHostnameResolver:
-    """Wraps a IHostnameResolver, limiting the number of in-flight DNS lookups.
-    """
+    """Wraps a IHostnameResolver, limiting the number of in-flight DNS lookups."""
 
     def __init__(self, resolver, max_dns_requests_in_flight):
         self._resolver = resolver

synapse/app/generic_worker.py

@@ -421,8 +421,7 @@ class GenericWorkerPresence(BasePresenceHandler):
         ]
 
     async def set_state(self, target_user, state, ignore_status_msg=False):
-        """Set the presence state of the user.
-        """
+        """Set the presence state of the user."""
         presence = state["presence"]
 
         valid_presence = (

synapse/appservice/__init__.py

@@ -166,7 +166,10 @@ class ApplicationService:
 
     @cached(num_args=1, cache_context=True)
     async def matches_user_in_member_list(
-        self, room_id: str, store: "DataStore", cache_context: _CacheContext,
+        self,
+        room_id: str,
+        store: "DataStore",
+        cache_context: _CacheContext,
     ) -> bool:
         """Check if this service is interested a room based upon it's membership
 

synapse/appservice/api.py

@@ -76,9 +76,6 @@ def _is_valid_3pe_result(r, field):
     fields = r["fields"]
     if not isinstance(fields, dict):
         return False
-    for k in fields.keys():
-        if not isinstance(fields[k], str):
-            return False
 
     return True
 
@@ -230,7 +227,9 @@ class ApplicationServiceApi(SimpleHttpClient):
 
         try:
             await self.put_json(
-                uri=uri, json_body=body, args={"access_token": service.hs_token},
+                uri=uri,
+                json_body=body,
+                args={"access_token": service.hs_token},
             )
             sent_transactions_counter.labels(service.id).inc()
             sent_events_counter.labels(service.id).inc(len(events))

synapse/appservice/scheduler.py

@@ -68,7 +68,7 @@ MAX_EPHEMERAL_EVENTS_PER_TRANSACTION = 100
 
 
 class ApplicationServiceScheduler:
-    """ Public facing API for this module. Does the required DI to tie the
+    """Public facing API for this module. Does the required DI to tie the
     components together. This also serves as the "event_pool", which in this
     case is a simple array.
     """

synapse/config/_base.py

@@ -224,7 +224,9 @@ class Config:
         return self.read_templates([filename])[0]
 
     def read_templates(
-        self, filenames: List[str], custom_template_directory: Optional[str] = None,
+        self,
+        filenames: List[str],
+        custom_template_directory: Optional[str] = None,
     ) -> List[jinja2.Template]:
         """Load a list of template files from disk using the given variables.
 
@@ -264,7 +266,10 @@ class Config:
 
         # TODO: switch to synapse.util.templates.build_jinja_env
         loader = jinja2.FileSystemLoader(search_directories)
-        env = jinja2.Environment(loader=loader, autoescape=jinja2.select_autoescape(),)
+        env = jinja2.Environment(
+            loader=loader,
+            autoescape=jinja2.select_autoescape(),
+        )
 
         # Update the environment with our custom filters
         env.filters.update(
@@ -825,8 +830,7 @@ class ShardedWorkerHandlingConfig:
     instances = attr.ib(type=List[str])
 
     def should_handle(self, instance_name: str, key: str) -> bool:
-        """Whether this instance is responsible for handling the given key.
-        """
+        """Whether this instance is responsible for handling the given key."""
         # If multiple instances are not defined we always return true
         if not self.instances or len(self.instances) == 1:
             return True

synapse/config/auth.py

@@ -18,8 +18,7 @@ from ._base import Config
 
 
 class AuthConfig(Config):
-    """Password and login configuration
-    """
+    """Password and login configuration"""
 
     section = "auth"
 
@@ -98,7 +97,7 @@ class AuthConfig(Config):
             # session to be active.
             #
             # This defaults to 0, meaning the user is queried for their credentials
-            # before every action, but this can be overridden to alow a single
+            # before every action, but this can be overridden to allow a single
             # validation to be re-used.  This weakens the protections afforded by
             # the user-interactive authentication process, by allowing for multiple
             # (and potentially different) operations to use the same validation session.

synapse/config/cas.py

@@ -13,7 +13,12 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+from typing import Any, List
+
+from synapse.config.sso import SsoAttributeRequirement
+
 from ._base import Config, ConfigError
+from ._util import validate_config
 
 
 class CasConfig(Config):
@@ -40,12 +45,16 @@ class CasConfig(Config):
             # TODO Update this to a _synapse URL.
             self.cas_service_url = public_baseurl + "_matrix/client/r0/login/cas/ticket"
             self.cas_displayname_attribute = cas_config.get("displayname_attribute")
-            self.cas_required_attributes = cas_config.get("required_attributes") or {}
+            required_attributes = cas_config.get("required_attributes") or {}
+            self.cas_required_attributes = _parsed_required_attributes_def(
+                required_attributes
+            )
+
         else:
             self.cas_server_url = None
             self.cas_service_url = None
             self.cas_displayname_attribute = None
-            self.cas_required_attributes = {}
+            self.cas_required_attributes = []
 
     def generate_config_section(self, config_dir_path, server_name, **kwargs):
         return """\
@@ -77,3 +86,22 @@ class CasConfig(Config):
           #  userGroup: "staff"
           #  department: None
         """
+
+
+# CAS uses a legacy required attributes mapping, not the one provided by
+# SsoAttributeRequirement.
+REQUIRED_ATTRIBUTES_SCHEMA = {
+    "type": "object",
+    "additionalProperties": {"anyOf": [{"type": "string"}, {"type": "null"}]},
+}
+
+
+def _parsed_required_attributes_def(
+    required_attributes: Any,
+) -> List[SsoAttributeRequirement]:
+    validate_config(
+        REQUIRED_ATTRIBUTES_SCHEMA,
+        required_attributes,
+        config_path=("cas_config", "required_attributes"),
+    )
+    return [SsoAttributeRequirement(k, v) for k, v in required_attributes.items()]

synapse/config/database.py

@@ -207,8 +207,7 @@ class DatabaseConfig(Config):
         )
 
     def get_single_database(self) -> DatabaseConnectionConfig:
-        """Returns the database if there is only one, useful for e.g. tests
-        """
+        """Returns the database if there is only one, useful for e.g. tests"""
         if not self.databases:
             raise Exception("More than one database exists")
 

synapse/config/emailconfig.py

@@ -289,7 +289,8 @@ class EmailConfig(Config):
                 self.email_notif_template_html,
                 self.email_notif_template_text,
             ) = self.read_templates(
-                [notif_template_html, notif_template_text], template_dir,
+                [notif_template_html, notif_template_text],
+                template_dir,
             )
 
             self.email_notif_for_new_users = email_config.get(
@@ -311,7 +312,8 @@ class EmailConfig(Config):
                 self.account_validity_template_html,
                 self.account_validity_template_text,
             ) = self.read_templates(
-                [expiry_template_html, expiry_template_text], template_dir,
+                [expiry_template_html, expiry_template_text],
+                template_dir,
             )
 
         subjects_config = email_config.get("subjects", {})

synapse/config/logger.py

@@ -162,7 +162,10 @@ class LoggingConfig(Config):
         )
 
         logging_group.add_argument(
-            "-f", "--log-file", dest="log_file", help=argparse.SUPPRESS,
+            "-f",
+            "--log-file",
+            dest="log_file",
+            help=argparse.SUPPRESS,
         )
 
     def generate_files(self, config, config_dir_path):

synapse/config/oidc_config.py

@@ -201,9 +201,9 @@ class OIDCConfig(Config):
           #  user_mapping_provider:
           #    config:
           #      subject_claim: "id"
-          #      localpart_template: "{{ user.login }}"
-          #      display_name_template: "{{ user.name }}"
-          #      email_template: "{{ user.email }}"
+          #      localpart_template: "{{{{ user.login }}}}"
+          #      display_name_template: "{{{{ user.name }}}}"
+          #      email_template: "{{{{ user.email }}}}"
 
           # For use with Keycloak
           #
@@ -230,8 +230,8 @@ class OIDCConfig(Config):
           #  user_mapping_provider:
           #    config:
           #      subject_claim: "id"
-          #      localpart_template: "{{ user.login }}"
-          #      display_name_template: "{{ user.name }}"
+          #      localpart_template: "{{{{ user.login }}}}"
+          #      display_name_template: "{{{{ user.name }}}}"
         """.format(
             mapping_provider=DEFAULT_USER_MAPPING_PROVIDER
         )
@@ -355,9 +355,10 @@ def _parse_oidc_config_dict(
     ump_config.setdefault("module", DEFAULT_USER_MAPPING_PROVIDER)
     ump_config.setdefault("config", {})
 
-    (user_mapping_provider_class, user_mapping_provider_config,) = load_module(
-        ump_config, config_path + ("user_mapping_provider",)
-    )
+    (
+        user_mapping_provider_class,
+        user_mapping_provider_config,
+    ) = load_module(ump_config, config_path + ("user_mapping_provider",))
 
     # Ensure loaded user mapping module has defined all necessary methods
     required_methods = [
@@ -372,7 +373,11 @@ def _parse_oidc_config_dict(
     if missing_methods:
         raise ConfigError(
             "Class %s is missing required "
-            "methods: %s" % (user_mapping_provider_class, ", ".join(missing_methods),),
+            "methods: %s"
+            % (
+                user_mapping_provider_class,
+                ", ".join(missing_methods),
+            ),
             config_path + ("user_mapping_provider", "module"),
         )
 

synapse/config/registration.py

@@ -391,6 +391,8 @@ class RegistrationConfig(Config):
         # By default, any room aliases included in this list will be created
         # as a publicly joinable room when the first user registers for the
         # homeserver. This behaviour can be customised with the settings below.
+        # If the room already exists, make certain it is a publicly joinable
+        # room. The join rule of the room must be set to 'public'.
         #
         #auto_join_rooms:
         #  - "#example:example.com"

synapse/config/repository.py

@@ -17,9 +17,7 @@ import os
 from collections import namedtuple
 from typing import Dict, List
 
-from netaddr import IPSet
-
-from synapse.config.server import DEFAULT_IP_RANGE_BLACKLIST
+from synapse.config.server import DEFAULT_IP_RANGE_BLACKLIST, generate_ip_set
 from synapse.python_dependencies import DependencyException, check_requirements
 from synapse.util.module_loader import load_module
 
@@ -54,7 +52,7 @@ MediaStorageProviderConfig = namedtuple(
 
 
 def parse_thumbnail_requirements(thumbnail_sizes):
-    """ Takes a list of dictionaries with "width", "height", and "method" keys
+    """Takes a list of dictionaries with "width", "height", and "method" keys
     and creates a map from image media types to the thumbnail size, thumbnailing
     method, and thumbnail media type to precalculate
 
@@ -187,16 +185,17 @@ class ContentRepositoryConfig(Config):
                     "to work"
                 )
 
-            self.url_preview_ip_range_blacklist = IPSet(
-                config["url_preview_ip_range_blacklist"]
-            )
-
             # we always blacklist '0.0.0.0' and '::', which are supposed to be
             # unroutable addresses.
-            self.url_preview_ip_range_blacklist.update(["0.0.0.0", "::"])
+            self.url_preview_ip_range_blacklist = generate_ip_set(
+                config["url_preview_ip_range_blacklist"],
+                ["0.0.0.0", "::"],
+                config_path=("url_preview_ip_range_blacklist",),
+            )
 
-            self.url_preview_ip_range_whitelist = IPSet(
-                config.get("url_preview_ip_range_whitelist", ())
+            self.url_preview_ip_range_whitelist = generate_ip_set(
+                config.get("url_preview_ip_range_whitelist", ()),
+                config_path=("url_preview_ip_range_whitelist",),
             )
 
             self.url_preview_url_blacklist = config.get("url_preview_url_blacklist", ())

synapse/config/room_directory.py

@@ -123,7 +123,7 @@ class RoomDirectoryConfig(Config):
             alias (str)
 
         Returns:
-            boolean: True if user is allowed to crate the alias
+            boolean: True if user is allowed to create the alias
         """
         for rule in self._alias_creation_rules:
             if rule.matches(user_id, room_id, [alias]):

synapse/config/saml2_config.py

@@ -17,8 +17,7 @@
 import logging
 from typing import Any, List
 
-import attr
-
+from synapse.config.sso import SsoAttributeRequirement
 from synapse.python_dependencies import DependencyException, check_requirements
 from synapse.util.module_loader import load_module, load_python_module
 
@@ -398,32 +397,18 @@ class SAML2Config(Config):
         }
 
 
-@attr.s(frozen=True)
-class SamlAttributeRequirement:
-    """Object describing a single requirement for SAML attributes."""
-
-    attribute = attr.ib(type=str)
-    value = attr.ib(type=str)
-
-    JSON_SCHEMA = {
-        "type": "object",
-        "properties": {"attribute": {"type": "string"}, "value": {"type": "string"}},
-        "required": ["attribute", "value"],
-    }
-
-
 ATTRIBUTE_REQUIREMENTS_SCHEMA = {
     "type": "array",
-    "items": SamlAttributeRequirement.JSON_SCHEMA,
+    "items": SsoAttributeRequirement.JSON_SCHEMA,
 }
 
 
 def _parse_attribute_requirements_def(
     attribute_requirements: Any,
-) -> List[SamlAttributeRequirement]:
+) -> List[SsoAttributeRequirement]:
     validate_config(
         ATTRIBUTE_REQUIREMENTS_SCHEMA,
         attribute_requirements,
-        config_path=["saml2_config", "attribute_requirements"],
+        config_path=("saml2_config", "attribute_requirements"),
     )
-    return [SamlAttributeRequirement(**x) for x in attribute_requirements]
+    return [SsoAttributeRequirement(**x) for x in attribute_requirements]

synapse/config/server.py

@@ -15,6 +15,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import itertools
 import logging
 import os.path
 import re
@@ -23,7 +24,7 @@ from typing import Any, Dict, Iterable, List, Optional, Set
 
 import attr
 import yaml
-from netaddr import IPSet
+from netaddr import AddrFormatError, IPNetwork, IPSet
 
 from synapse.api.room_versions import KNOWN_ROOM_VERSIONS
 from synapse.util.stringutils import parse_and_validate_server_name
@@ -40,6 +41,71 @@ logger = logging.Logger(__name__)
 # in the list.
 DEFAULT_BIND_ADDRESSES = ["::", "0.0.0.0"]
 
+
+def _6to4(network: IPNetwork) -> IPNetwork:
+    """Convert an IPv4 network into a 6to4 IPv6 network per RFC 3056."""
+
+    # 6to4 networks consist of:
+    # * 2002 as the first 16 bits
+    # * The first IPv4 address in the network hex-encoded as the next 32 bits
+    # * The new prefix length needs to include the bits from the 2002 prefix.
+    hex_network = hex(network.first)[2:]
+    hex_network = ("0" * (8 - len(hex_network))) + hex_network
+    return IPNetwork(
+        "2002:%s:%s::/%d"
+        % (
+            hex_network[:4],
+            hex_network[4:],
+            16 + network.prefixlen,
+        )
+    )
+
+
+def generate_ip_set(
+    ip_addresses: Optional[Iterable[str]],
+    extra_addresses: Optional[Iterable[str]] = None,
+    config_path: Optional[Iterable[str]] = None,
+) -> IPSet:
+    """
+    Generate an IPSet from a list of IP addresses or CIDRs.
+
+    Additionally, for each IPv4 network in the list of IP addresses, also
+    includes the corresponding IPv6 networks.
+
+    This includes:
+
+    * IPv4-Compatible IPv6 Address (see RFC 4291, section 2.5.5.1)
+    * IPv4-Mapped IPv6 Address (see RFC 4291, section 2.5.5.2)
+    * 6to4 Address (see RFC 3056, section 2)
+
+    Args:
+        ip_addresses: An iterable of IP addresses or CIDRs.
+        extra_addresses: An iterable of IP addresses or CIDRs.
+        config_path: The path in the configuration for error messages.
+
+    Returns:
+        A new IP set.
+    """
+    result = IPSet()
+    for ip in itertools.chain(ip_addresses or (), extra_addresses or ()):
+        try:
+            network = IPNetwork(ip)
+        except AddrFormatError as e:
+            raise ConfigError(
+                "Invalid IP range provided: %s." % (ip,), config_path
+            ) from e
+        result.add(network)
+
+        # It is possible that these already exist in the set, but that's OK.
+        if ":" not in str(network):
+            result.add(IPNetwork(network).ipv6(ipv4_compatible=True))
+            result.add(IPNetwork(network).ipv6(ipv4_compatible=False))
+            result.add(_6to4(network))
+
+    return result
+
+
+# IP ranges that are considered private / unroutable / don't make sense.
 DEFAULT_IP_RANGE_BLACKLIST = [
     # Localhost
     "127.0.0.0/8",
@@ -53,6 +119,8 @@ DEFAULT_IP_RANGE_BLACKLIST = [
     "192.0.0.0/24",
     # Link-local networks.
     "169.254.0.0/16",
+    # Formerly used for 6to4 relay.
+    "192.88.99.0/24",
     # Testing networks.
     "198.18.0.0/15",
     "192.0.2.0/24",
@@ -66,6 +134,12 @@ DEFAULT_IP_RANGE_BLACKLIST = [
     "fe80::/10",
     # Unique local addresses.
     "fc00::/7",
+    # Testing networks.
+    "2001:db8::/32",
+    # Multicast.
+    "ff00::/8",
+    # Site-local addresses
+    "fec0::/10",
 ]
 
 DEFAULT_ROOM_VERSION = "6"
@@ -185,7 +259,8 @@ class ServerConfig(Config):
         # Whether to require sharing a room with a user to retrieve their
         # profile data
         self.limit_profile_requests_to_users_who_share_rooms = config.get(
-            "limit_profile_requests_to_users_who_share_rooms", False,
+            "limit_profile_requests_to_users_who_share_rooms",
+            False,
         )
 
         if "restrict_public_rooms_to_local_users" in config and (
@@ -290,17 +365,15 @@ class ServerConfig(Config):
         )
 
         # Attempt to create an IPSet from the given ranges
-        try:
-            self.ip_range_blacklist = IPSet(ip_range_blacklist)
-        except Exception as e:
-            raise ConfigError("Invalid range(s) provided in ip_range_blacklist.") from e
-        # Always blacklist 0.0.0.0, ::
-        self.ip_range_blacklist.update(["0.0.0.0", "::"])
 
-        try:
-            self.ip_range_whitelist = IPSet(config.get("ip_range_whitelist", ()))
-        except Exception as e:
-            raise ConfigError("Invalid range(s) provided in ip_range_whitelist.") from e
+        # Always blacklist 0.0.0.0, ::
+        self.ip_range_blacklist = generate_ip_set(
+            ip_range_blacklist, ["0.0.0.0", "::"], config_path=("ip_range_blacklist",)
+        )
+
+        self.ip_range_whitelist = generate_ip_set(
+            config.get("ip_range_whitelist", ()), config_path=("ip_range_whitelist",)
+        )
 
         # The federation_ip_range_blacklist is used for backwards-compatibility
         # and only applies to federation and identity servers. If it is not given,
@@ -308,14 +381,12 @@ class ServerConfig(Config):
         federation_ip_range_blacklist = config.get(
             "federation_ip_range_blacklist", ip_range_blacklist
         )
-        try:
-            self.federation_ip_range_blacklist = IPSet(federation_ip_range_blacklist)
-        except Exception as e:
-            raise ConfigError(
-                "Invalid range(s) provided in federation_ip_range_blacklist."
-            ) from e
         # Always blacklist 0.0.0.0, ::
-        self.federation_ip_range_blacklist.update(["0.0.0.0", "::"])
+        self.federation_ip_range_blacklist = generate_ip_set(
+            federation_ip_range_blacklist,
+            ["0.0.0.0", "::"],
+            config_path=("federation_ip_range_blacklist",),
+        )
 
         if self.public_baseurl is not None:
             if self.public_baseurl[-1] != "/":
@@ -549,7 +620,9 @@ class ServerConfig(Config):
         if manhole:
             self.listeners.append(
                 ListenerConfig(
-                    port=manhole, bind_addresses=["127.0.0.1"], type="manhole",
+                    port=manhole,
+                    bind_addresses=["127.0.0.1"],
+                    type="manhole",
                 )
             )
 
@@ -585,7 +658,8 @@ class ServerConfig(Config):
         # and letting the client know which email address is bound to an account and
         # which one isn't.
         self.request_token_inhibit_3pid_errors = config.get(
-            "request_token_inhibit_3pid_errors", False,
+            "request_token_inhibit_3pid_errors",
+            False,
         )
 
         # List of users trialing the new experimental default push rules. This setting is

synapse/config/sso.py

@@ -12,14 +12,30 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-from typing import Any, Dict
+from typing import Any, Dict, Optional
+
+import attr
 
 from ._base import Config
 
 
+@attr.s(frozen=True)
+class SsoAttributeRequirement:
+    """Object describing a single requirement for SSO attributes."""
+
+    attribute = attr.ib(type=str)
+    # If a value is not given, than the attribute must simply exist.
+    value = attr.ib(type=Optional[str])
+
+    JSON_SCHEMA = {
+        "type": "object",
+        "properties": {"attribute": {"type": "string"}, "value": {"type": "string"}},
+        "required": ["attribute", "value"],
+    }
+
+
 class SSOConfig(Config):
-    """SSO Configuration
-    """
+    """SSO Configuration"""
 
     section = "sso"
 

synapse/config/workers.py

@@ -33,8 +33,7 @@ def _instance_to_list_converter(obj: Union[str, List[str]]) -> List[str]:
 
 @attr.s
 class InstanceLocationConfig:
-    """The host and port to talk to an instance via HTTP replication.
-    """
+    """The host and port to talk to an instance via HTTP replication."""
 
     host = attr.ib(type=str)
     port = attr.ib(type=int)
@@ -54,13 +53,19 @@ class WriterLocations:
     )
     typing = attr.ib(default="master", type=str)
     to_device = attr.ib(
-        default=["master"], type=List[str], converter=_instance_to_list_converter,
+        default=["master"],
+        type=List[str],
+        converter=_instance_to_list_converter,
     )
     account_data = attr.ib(
-        default=["master"], type=List[str], converter=_instance_to_list_converter,
+        default=["master"],
+        type=List[str],
+        converter=_instance_to_list_converter,
     )
     receipts = attr.ib(
-        default=["master"], type=List[str], converter=_instance_to_list_converter,
+        default=["master"],
+        type=List[str],
+        converter=_instance_to_list_converter,
     )
 
 
@@ -107,7 +112,9 @@ class WorkerConfig(Config):
         if manhole:
             self.worker_listeners.append(
                 ListenerConfig(
-                    port=manhole, bind_addresses=["127.0.0.1"], type="manhole",
+                    port=manhole,
+                    bind_addresses=["127.0.0.1"],
+                    type="manhole",
                 )
             )
 

synapse/event_auth.py

@@ -42,7 +42,7 @@ def check(
     do_sig_check: bool = True,
     do_size_check: bool = True,
 ) -> None:
-    """ Checks if this event is correctly authed.
+    """Checks if this event is correctly authed.
 
     Args:
         room_version_obj: the version of the room
@@ -423,7 +423,9 @@ def _can_send_event(event: EventBase, auth_events: StateMap[EventBase]) -> bool:
 
 
 def check_redaction(
-    room_version_obj: RoomVersion, event: EventBase, auth_events: StateMap[EventBase],
+    room_version_obj: RoomVersion,
+    event: EventBase,
+    auth_events: StateMap[EventBase],
 ) -> bool:
     """Check whether the event sender is allowed to redact the target event.
 
@@ -459,7 +461,9 @@ def check_redaction(
 
 
 def _check_power_levels(
-    room_version_obj: RoomVersion, event: EventBase, auth_events: StateMap[EventBase],
+    room_version_obj: RoomVersion,
+    event: EventBase,
+    auth_events: StateMap[EventBase],
 ) -> None:
     user_list = event.content.get("users", {})
     # Validate users

synapse/events/builder.py

@@ -98,7 +98,9 @@ class EventBuilder:
         return self._state_key is not None
 
     async def build(
-        self, prev_event_ids: List[str], auth_event_ids: Optional[List[str]],
+        self,
+        prev_event_ids: List[str],
+        auth_event_ids: Optional[List[str]],
     ) -> EventBase:
         """Transform into a fully signed and hashed event
 

synapse/events/snapshot.py

@@ -341,8 +341,7 @@ def _encode_state_dict(state_dict):
 
 
 def _decode_state_dict(input):
-    """Decodes a state dict encoded using `_encode_state_dict` above
-    """
+    """Decodes a state dict encoded using `_encode_state_dict` above"""
     if input is None:
         return None
 

synapse/events/spamcheck.py

@@ -17,6 +17,8 @@
 import inspect
 from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union
 
+from synapse.rest.media.v1._base import FileInfo
+from synapse.rest.media.v1.media_storage import ReadableFileWrapper
 from synapse.spam_checker_api import RegistrationBehaviour
 from synapse.types import Collection
 from synapse.util.async_helpers import maybe_awaitable
@@ -214,3 +216,48 @@ class SpamChecker:
                     return behaviour
 
         return RegistrationBehaviour.ALLOW
+
+    async def check_media_file_for_spam(
+        self, file_wrapper: ReadableFileWrapper, file_info: FileInfo
+    ) -> bool:
+        """Checks if a piece of newly uploaded media should be blocked.
+
+        This will be called for local uploads, downloads of remote media, each
+        thumbnail generated for those, and web pages/images used for URL
+        previews.
+
+        Note that care should be taken to not do blocking IO operations in the
+        main thread. For example, to get the contents of a file a module
+        should do::
+
+            async def check_media_file_for_spam(
+                self, file: ReadableFileWrapper, file_info: FileInfo
+            ) -> bool:
+                buffer = BytesIO()
+                await file.write_chunks_to(buffer.write)
+
+                if buffer.getvalue() == b"Hello World":
+                    return True
+
+                return False
+
+
+        Args:
+            file: An object that allows reading the contents of the media.
+            file_info: Metadata about the file.
+
+        Returns:
+            True if the media should be blocked or False if it should be
+            allowed.
+        """
+
+        for spam_checker in self.spam_checkers:
+            # For backwards compatibility, only run if the method exists on the
+            # spam checker
+            checker = getattr(spam_checker, "check_media_file_for_spam", None)
+            if checker:
+                spam = await maybe_awaitable(checker(file_wrapper, file_info))
+                if spam:
+                    return True
+
+        return False

synapse/events/third_party_rules.py

@@ -40,7 +40,8 @@ class ThirdPartyEventRules:
 
         if module is not None:
             self.third_party_rules = module(
-                config=config, module_api=hs.get_module_api(),
+                config=config,
+                module_api=hs.get_module_api(),
             )
 
     async def check_event_allowed(

synapse/events/utils.py

@@ -34,7 +34,7 @@ SPLIT_FIELD_REGEX = re.compile(r"(?<!\\)\.")
 
 
 def prune_event(event: EventBase) -> EventBase:
-    """ Returns a pruned version of the given event, which removes all keys we
+    """Returns a pruned version of the given event, which removes all keys we
     don't know about or think could potentially be dodgy.
 
     This is used when we "redact" an event. We want to remove all fields that

synapse/federation/federation_client.py

@@ -750,7 +750,11 @@ class FederationClient(FederationBase):
         return resp[1]
 
     async def send_invite(
-        self, destination: str, room_id: str, event_id: str, pdu: EventBase,
+        self,
+        destination: str,
+        room_id: str,
+        event_id: str,
+        pdu: EventBase,
     ) -> EventBase:
         room_version = await self.store.get_room_version(room_id)
 

synapse/federation/federation_server.py

@@ -85,7 +85,8 @@ received_queries_counter = Counter(
 )
 
 pdu_process_time = Histogram(
-    "synapse_federation_server_pdu_process_time", "Time taken to process an event",
+    "synapse_federation_server_pdu_process_time",
+    "Time taken to process an event",
 )
 
 
@@ -204,7 +205,7 @@ class FederationServer(FederationBase):
     async def _handle_incoming_transaction(
         self, origin: str, transaction: Transaction, request_time: int
     ) -> Tuple[int, Dict[str, Any]]:
-        """ Process an incoming transaction and return the HTTP response
+        """Process an incoming transaction and return the HTTP response
 
         Args:
             origin: the server making the request
@@ -373,8 +374,7 @@ class FederationServer(FederationBase):
         return pdu_results
 
     async def _handle_edus_in_txn(self, origin: str, transaction: Transaction):
-        """Process the EDUs in a received transaction.
-        """
+        """Process the EDUs in a received transaction."""
 
         async def _process_edu(edu_dict):
             received_edus_counter.inc()
@@ -437,7 +437,10 @@ class FederationServer(FederationBase):
             raise AuthError(403, "Host not in room.")
 
         resp = await self._state_ids_resp_cache.wrap(
-            (room_id, event_id), self._on_state_ids_request_compute, room_id, event_id,
+            (room_id, event_id),
+            self._on_state_ids_request_compute,
+            room_id,
+            event_id,
         )
 
         return 200, resp
@@ -679,7 +682,7 @@ class FederationServer(FederationBase):
         )
 
     async def _handle_received_pdu(self, origin: str, pdu: EventBase) -> None:
-        """ Process a PDU received in a federation /send/ transaction.
+        """Process a PDU received in a federation /send/ transaction.
 
         If the event is invalid, then this method throws a FederationError.
         (The error will then be logged and sent back to the sender (which
@@ -906,13 +909,11 @@ class FederationHandlerRegistry:
         self.query_handlers[query_type] = handler
 
     def register_instance_for_edu(self, edu_type: str, instance_name: str):
-        """Register that the EDU handler is on a different instance than master.
-        """
+        """Register that the EDU handler is on a different instance than master."""
         self._edu_type_to_instance[edu_type] = [instance_name]
 
     def register_instances_for_edu(self, edu_type: str, instance_names: List[str]):
-        """Register that the EDU handler is on multiple instances.
-        """
+        """Register that the EDU handler is on multiple instances."""
         self._edu_type_to_instance[edu_type] = instance_names
 
     async def on_edu(self, edu_type: str, origin: str, content: dict):

synapse/federation/persistence.py

@@ -30,8 +30,7 @@ logger = logging.getLogger(__name__)
 
 
 class TransactionActions:
-    """ Defines persistence actions that relate to handling Transactions.
-    """
+    """Defines persistence actions that relate to handling Transactions."""
 
     def __init__(self, datastore):
         self.store = datastore
@@ -57,8 +56,7 @@ class TransactionActions:
     async def set_response(
         self, origin: str, transaction: Transaction, code: int, response: JsonDict
     ) -> None:
-        """Persist how we responded to a transaction.
-        """
+        """Persist how we responded to a transaction."""
         transaction_id = transaction.transaction_id  # type: ignore
         if not transaction_id:
             raise RuntimeError("Cannot persist a transaction with no transaction_id")

synapse/federation/send_queue.py

@@ -468,8 +468,7 @@ class KeyedEduRow(
 
 
 class EduRow(BaseFederationRow, namedtuple("EduRow", ("edu",))):  # Edu
-    """Streams EDUs that don't have keys. See KeyedEduRow
-    """
+    """Streams EDUs that don't have keys. See KeyedEduRow"""
 
     TypeId = "e"
 
@@ -519,7 +518,10 @@ def process_rows_for_federation(transaction_queue, rows):
     # them into the appropriate collection and then send them off.
 
     buff = ParsedFederationStreamData(
-        presence=[], presence_destinations=[], keyed_edus={}, edus={},
+        presence=[],
+        presence_destinations=[],
+        keyed_edus={},
+        edus={},
     )
 
     # Parse the rows in the stream and add to the buffer

synapse/federation/sender/__init__.py

@@ -328,7 +328,9 @@ class FederationSender:
         # to allow us to perform catch-up later on if the remote is unreachable
         # for a while.
         await self.store.store_destination_rooms_entries(
-            destinations, pdu.room_id, pdu.internal_metadata.stream_ordering,
+            destinations,
+            pdu.room_id,
+            pdu.internal_metadata.stream_ordering,
         )
 
         for destination in destinations:
@@ -475,7 +477,7 @@ class FederationSender:
         self, states: List[UserPresenceState], destinations: List[str]
     ) -> None:
         """Send the given presence states to the given destinations.
-            destinations (list[str])
+        destinations (list[str])
         """
 
         if not states or not self.hs.config.use_presence:
@@ -616,8 +618,8 @@ class FederationSender:
         last_processed = None  # type: Optional[str]
 
         while True:
-            destinations_to_wake = await self.store.get_catch_up_outstanding_destinations(
-                last_processed
+            destinations_to_wake = (
+                await self.store.get_catch_up_outstanding_destinations(last_processed)
             )
 
             if not destinations_to_wake:

synapse/federation/sender/per_destination_queue.py

@@ -85,7 +85,8 @@ class PerDestinationQueue:
             # processing. We have a guard in `attempt_new_transaction` that
             # ensure we don't start sending stuff.
             logger.error(
-                "Create a per destination queue for %s on wrong worker", destination,
+                "Create a per destination queue for %s on wrong worker",
+                destination,
             )
             self._should_send_on_this_instance = False
 
@@ -440,8 +441,10 @@ class PerDestinationQueue:
 
         if first_catch_up_check:
             # first catchup so get last_successful_stream_ordering from database
-            self._last_successful_stream_ordering = await self._store.get_destination_last_successful_stream_ordering(
-                self._destination
+            self._last_successful_stream_ordering = (
+                await self._store.get_destination_last_successful_stream_ordering(
+                    self._destination
+                )
             )
 
         if self._last_successful_stream_ordering is None:
@@ -457,7 +460,8 @@ class PerDestinationQueue:
         # get at most 50 catchup room/PDUs
         while True:
             event_ids = await self._store.get_catch_up_room_event_ids(
-                self._destination, self._last_successful_stream_ordering,
+                self._destination,
+                self._last_successful_stream_ordering,
             )
 
             if not event_ids:

synapse/federation/sender/transaction_manager.py

@@ -65,7 +65,10 @@ class TransactionManager:
 
     @measure_func("_send_new_transaction")
     async def send_new_transaction(
-        self, destination: str, pdus: List[EventBase], edus: List[Edu],
+        self,
+        destination: str,
+        pdus: List[EventBase],
+        edus: List[Edu],
     ) -> bool:
         """
         Args:

synapse/federation/transport/client.py

@@ -39,7 +39,7 @@ class TransportLayerClient:
 
     @log_function
     def get_room_state_ids(self, destination, room_id, event_id):
-        """ Requests all state for a given room from the given server at the
+        """Requests all state for a given room from the given server at the
         given event. Returns the state's event_id's
 
         Args:
@@ -63,7 +63,7 @@ class TransportLayerClient:
 
     @log_function
     def get_event(self, destination, event_id, timeout=None):
-        """ Requests the pdu with give id and origin from the given server.
+        """Requests the pdu with give id and origin from the given server.
 
         Args:
             destination (str): The host name of the remote homeserver we want
@@ -84,7 +84,7 @@ class TransportLayerClient:
 
     @log_function
     def backfill(self, destination, room_id, event_tuples, limit):
-        """ Requests `limit` previous PDUs in a given context before list of
+        """Requests `limit` previous PDUs in a given context before list of
         PDUs.
 
         Args:
@@ -118,7 +118,7 @@ class TransportLayerClient:
 
     @log_function
     async def send_transaction(self, transaction, json_data_callback=None):
-        """ Sends the given Transaction to its destination
+        """Sends the given Transaction to its destination
 
         Args:
             transaction (Transaction)
@@ -551,8 +551,7 @@ class TransportLayerClient:
 
     @log_function
     def get_group_profile(self, destination, group_id, requester_user_id):
-        """Get a group profile
-        """
+        """Get a group profile"""
         path = _create_v1_path("/groups/%s/profile", group_id)
 
         return self.client.get_json(
@@ -584,8 +583,7 @@ class TransportLayerClient:
 
     @log_function
     def get_group_summary(self, destination, group_id, requester_user_id):
-        """Get a group summary
-        """
+        """Get a group summary"""
         path = _create_v1_path("/groups/%s/summary", group_id)
 
         return self.client.get_json(
@@ -597,8 +595,7 @@ class TransportLayerClient:
 
     @log_function
     def get_rooms_in_group(self, destination, group_id, requester_user_id):
-        """Get all rooms in a group
-        """
+        """Get all rooms in a group"""
         path = _create_v1_path("/groups/%s/rooms", group_id)
 
         return self.client.get_json(
@@ -611,8 +608,7 @@ class TransportLayerClient:
     def add_room_to_group(
         self, destination, group_id, requester_user_id, room_id, content
     ):
-        """Add a room to a group
-        """
+        """Add a room to a group"""
         path = _create_v1_path("/groups/%s/room/%s", group_id, room_id)
 
         return self.client.post_json(
@@ -626,8 +622,7 @@ class TransportLayerClient:
     def update_room_in_group(
         self, destination, group_id, requester_user_id, room_id, config_key, content
     ):
-        """Update room in group
-        """
+        """Update room in group"""
         path = _create_v1_path(
             "/groups/%s/room/%s/config/%s", group_id, room_id, config_key
         )
@@ -641,8 +636,7 @@ class TransportLayerClient:
         )
 
     def remove_room_from_group(self, destination, group_id, requester_user_id, room_id):
-        """Remove a room from a group
-        """
+        """Remove a room from a group"""
         path = _create_v1_path("/groups/%s/room/%s", group_id, room_id)
 
         return self.client.delete_json(
@@ -654,8 +648,7 @@ class TransportLayerClient:
 
     @log_function
     def get_users_in_group(self, destination, group_id, requester_user_id):
-        """Get users in a group
-        """
+        """Get users in a group"""
         path = _create_v1_path("/groups/%s/users", group_id)
 
         return self.client.get_json(
@@ -667,8 +660,7 @@ class TransportLayerClient:
 
     @log_function
     def get_invited_users_in_group(self, destination, group_id, requester_user_id):
-        """Get users that have been invited to a group
-        """
+        """Get users that have been invited to a group"""
         path = _create_v1_path("/groups/%s/invited_users", group_id)
 
         return self.client.get_json(
@@ -680,8 +672,7 @@ class TransportLayerClient:
 
     @log_function
     def accept_group_invite(self, destination, group_id, user_id, content):
-        """Accept a group invite
-        """
+        """Accept a group invite"""
         path = _create_v1_path("/groups/%s/users/%s/accept_invite", group_id, user_id)
 
         return self.client.post_json(
@@ -690,8 +681,7 @@ class TransportLayerClient:
 
     @log_function
     def join_group(self, destination, group_id, user_id, content):
-        """Attempts to join a group
-        """
+        """Attempts to join a group"""
         path = _create_v1_path("/groups/%s/users/%s/join", group_id, user_id)
 
         return self.client.post_json(
@@ -702,8 +692,7 @@ class TransportLayerClient:
     def invite_to_group(
         self, destination, group_id, user_id, requester_user_id, content
     ):
-        """Invite a user to a group
-        """
+        """Invite a user to a group"""
         path = _create_v1_path("/groups/%s/users/%s/invite", group_id, user_id)
 
         return self.client.post_json(
@@ -730,8 +719,7 @@ class TransportLayerClient:
     def remove_user_from_group(
         self, destination, group_id, requester_user_id, user_id, content
     ):
-        """Remove a user from a group
-        """
+        """Remove a user from a group"""
         path = _create_v1_path("/groups/%s/users/%s/remove", group_id, user_id)
 
         return self.client.post_json(
@@ -772,8 +760,7 @@ class TransportLayerClient:
     def update_group_summary_room(
         self, destination, group_id, user_id, room_id, category_id, content
     ):
-        """Update a room entry in a group summary
-        """
+        """Update a room entry in a group summary"""
         if category_id:
             path = _create_v1_path(
                 "/groups/%s/summary/categories/%s/rooms/%s",
@@ -796,8 +783,7 @@ class TransportLayerClient:
     def delete_group_summary_room(
         self, destination, group_id, user_id, room_id, category_id
     ):
-        """Delete a room entry in a group summary
-        """
+        """Delete a room entry in a group summary"""
         if category_id:
             path = _create_v1_path(
                 "/groups/%s/summary/categories/%s/rooms/%s",
@@ -817,8 +803,7 @@ class TransportLayerClient:
 
     @log_function
     def get_group_categories(self, destination, group_id, requester_user_id):
-        """Get all categories in a group
-        """
+        """Get all categories in a group"""
         path = _create_v1_path("/groups/%s/categories", group_id)
 
         return self.client.get_json(
@@ -830,8 +815,7 @@ class TransportLayerClient:
 
     @log_function
     def get_group_category(self, destination, group_id, requester_user_id, category_id):
-        """Get category info in a group
-        """
+        """Get category info in a group"""
         path = _create_v1_path("/groups/%s/categories/%s", group_id, category_id)
 
         return self.client.get_json(
@@ -845,8 +829,7 @@ class TransportLayerClient:
     def update_group_category(
         self, destination, group_id, requester_user_id, category_id, content
     ):
-        """Update a category in a group
-        """
+        """Update a category in a group"""
         path = _create_v1_path("/groups/%s/categories/%s", group_id, category_id)
 
         return self.client.post_json(
@@ -861,8 +844,7 @@ class TransportLayerClient:
     def delete_group_category(
         self, destination, group_id, requester_user_id, category_id
     ):
-        """Delete a category in a group
-        """
+        """Delete a category in a group"""
         path = _create_v1_path("/groups/%s/categories/%s", group_id, category_id)
 
         return self.client.delete_json(
@@ -874,8 +856,7 @@ class TransportLayerClient:
 
     @log_function
     def get_group_roles(self, destination, group_id, requester_user_id):
-        """Get all roles in a group
-        """
+        """Get all roles in a group"""
         path = _create_v1_path("/groups/%s/roles", group_id)
 
         return self.client.get_json(
@@ -887,8 +868,7 @@ class TransportLayerClient:
 
     @log_function
     def get_group_role(self, destination, group_id, requester_user_id, role_id):
-        """Get a roles info
-        """
+        """Get a roles info"""
         path = _create_v1_path("/groups/%s/roles/%s", group_id, role_id)
 
         return self.client.get_json(
@@ -902,8 +882,7 @@ class TransportLayerClient:
     def update_group_role(
         self, destination, group_id, requester_user_id, role_id, content
     ):
-        """Update a role in a group
-        """
+        """Update a role in a group"""
         path = _create_v1_path("/groups/%s/roles/%s", group_id, role_id)
 
         return self.client.post_json(
@@ -916,8 +895,7 @@ class TransportLayerClient:
 
     @log_function
     def delete_group_role(self, destination, group_id, requester_user_id, role_id):
-        """Delete a role in a group
-        """
+        """Delete a role in a group"""
         path = _create_v1_path("/groups/%s/roles/%s", group_id, role_id)
 
         return self.client.delete_json(
@@ -931,8 +909,7 @@ class TransportLayerClient:
     def update_group_summary_user(
         self, destination, group_id, requester_user_id, user_id, role_id, content
     ):
-        """Update a users entry in a group
-        """
+        """Update a users entry in a group"""
         if role_id:
             path = _create_v1_path(
                 "/groups/%s/summary/roles/%s/users/%s", group_id, role_id, user_id
@@ -950,8 +927,7 @@ class TransportLayerClient:
 
     @log_function
     def set_group_join_policy(self, destination, group_id, requester_user_id, content):
-        """Sets the join policy for a group
-        """
+        """Sets the join policy for a group"""
         path = _create_v1_path("/groups/%s/settings/m.join_policy", group_id)
 
         return self.client.put_json(
@@ -966,8 +942,7 @@ class TransportLayerClient:
     def delete_group_summary_user(
         self, destination, group_id, requester_user_id, user_id, role_id
     ):
-        """Delete a users entry in a group
-        """
+        """Delete a users entry in a group"""
         if role_id:
             path = _create_v1_path(
                 "/groups/%s/summary/roles/%s/users/%s", group_id, role_id, user_id
@@ -983,8 +958,7 @@ class TransportLayerClient:
         )
 
     def bulk_get_publicised_groups(self, destination, user_ids):
-        """Get the groups a list of users are publicising
-        """
+        """Get the groups a list of users are publicising"""
 
         path = _create_v1_path("/get_groups_publicised")
 

synapse/federation/transport/server.py

@@ -364,7 +364,10 @@ class BaseFederationServlet:
                 continue
 
             server.register_paths(
-                method, (pattern,), self._wrap(code), self.__class__.__name__,
+                method,
+                (pattern,),
+                self._wrap(code),
+                self.__class__.__name__,
             )
 
 
@@ -381,7 +384,7 @@ class FederationSendServlet(BaseFederationServlet):
 
     # This is when someone is trying to send us a bunch of data.
     async def on_PUT(self, origin, content, query, transaction_id):
-        """ Called on PUT /send/<transaction_id>/
+        """Called on PUT /send/<transaction_id>/
 
         Args:
             request (twisted.web.http.Request): The HTTP request.
@@ -855,8 +858,7 @@ class FederationVersionServlet(BaseFederationServlet):
 
 
 class FederationGroupsProfileServlet(BaseFederationServlet):
-    """Get/set the basic profile of a group on behalf of a user
-    """
+    """Get/set the basic profile of a group on behalf of a user"""
 
     PATH = "/groups/(?P<group_id>[^/]*)/profile"
 
@@ -895,8 +897,7 @@ class FederationGroupsSummaryServlet(BaseFederationServlet):
 
 
 class FederationGroupsRoomsServlet(BaseFederationServlet):
-    """Get the rooms in a group on behalf of a user
-    """
+    """Get the rooms in a group on behalf of a user"""
 
     PATH = "/groups/(?P<group_id>[^/]*)/rooms"
 
@@ -911,8 +912,7 @@ class FederationGroupsRoomsServlet(BaseFederationServlet):
 
 
 class FederationGroupsAddRoomsServlet(BaseFederationServlet):
-    """Add/remove room from group
-    """
+    """Add/remove room from group"""
 
     PATH = "/groups/(?P<group_id>[^/]*)/room/(?P<room_id>[^/]*)"
 
@@ -940,8 +940,7 @@ class FederationGroupsAddRoomsServlet(BaseFederationServlet):
 
 
 class FederationGroupsAddRoomsConfigServlet(BaseFederationServlet):
-    """Update room config in group
-    """
+    """Update room config in group"""
 
     PATH = (
         "/groups/(?P<group_id>[^/]*)/room/(?P<room_id>[^/]*)"
@@ -961,8 +960,7 @@ class FederationGroupsAddRoomsConfigServlet(BaseFederationServlet):
 
 
 class FederationGroupsUsersServlet(BaseFederationServlet):
-    """Get the users in a group on behalf of a user
-    """
+    """Get the users in a group on behalf of a user"""
 
     PATH = "/groups/(?P<group_id>[^/]*)/users"
 
@@ -977,8 +975,7 @@ class FederationGroupsUsersServlet(BaseFederationServlet):
 
 
 class FederationGroupsInvitedUsersServlet(BaseFederationServlet):
-    """Get the users that have been invited to a group
-    """
+    """Get the users that have been invited to a group"""
 
     PATH = "/groups/(?P<group_id>[^/]*)/invited_users"
 
@@ -995,8 +992,7 @@ class FederationGroupsInvitedUsersServlet(BaseFederationServlet):
 
 
 class FederationGroupsInviteServlet(BaseFederationServlet):
-    """Ask a group server to invite someone to the group
-    """
+    """Ask a group server to invite someone to the group"""
 
     PATH = "/groups/(?P<group_id>[^/]*)/users/(?P<user_id>[^/]*)/invite"
 
@@ -1013,8 +1009,7 @@ class FederationGroupsInviteServlet(BaseFederationServlet):
 
 
 class FederationGroupsAcceptInviteServlet(BaseFederationServlet):
-    """Accept an invitation from the group server
-    """
+    """Accept an invitation from the group server"""
 
     PATH = "/groups/(?P<group_id>[^/]*)/users/(?P<user_id>[^/]*)/accept_invite"
 
@@ -1028,8 +1023,7 @@ class FederationGroupsAcceptInviteServlet(BaseFederationServlet):
 
 
 class FederationGroupsJoinServlet(BaseFederationServlet):
-    """Attempt to join a group
-    """
+    """Attempt to join a group"""
 
     PATH = "/groups/(?P<group_id>[^/]*)/users/(?P<user_id>[^/]*)/join"
 
@@ -1043,8 +1037,7 @@ class FederationGroupsJoinServlet(BaseFederationServlet):
 
 
 class FederationGroupsRemoveUserServlet(BaseFederationServlet):
-    """Leave or kick a user from the group
-    """
+    """Leave or kick a user from the group"""
 
     PATH = "/groups/(?P<group_id>[^/]*)/users/(?P<user_id>[^/]*)/remove"
 
@@ -1061,8 +1054,7 @@ class FederationGroupsRemoveUserServlet(BaseFederationServlet):
 
 
 class FederationGroupsLocalInviteServlet(BaseFederationServlet):
-    """A group server has invited a local user
-    """
+    """A group server has invited a local user"""
 
     PATH = "/groups/local/(?P<group_id>[^/]*)/users/(?P<user_id>[^/]*)/invite"
 
@@ -1076,8 +1068,7 @@ class FederationGroupsLocalInviteServlet(BaseFederationServlet):
 
 
 class FederationGroupsRemoveLocalUserServlet(BaseFederationServlet):
-    """A group server has removed a local user
-    """
+    """A group server has removed a local user"""
 
     PATH = "/groups/local/(?P<group_id>[^/]*)/users/(?P<user_id>[^/]*)/remove"
 
@@ -1093,8 +1084,7 @@ class FederationGroupsRemoveLocalUserServlet(BaseFederationServlet):
 
 
 class FederationGroupsRenewAttestaionServlet(BaseFederationServlet):
-    """A group or user's server renews their attestation
-    """
+    """A group or user's server renews their attestation"""
 
     PATH = "/groups/(?P<group_id>[^/]*)/renew_attestation/(?P<user_id>[^/]*)"
 
@@ -1156,8 +1146,7 @@ class FederationGroupsSummaryRoomsServlet(BaseFederationServlet):
 
 
 class FederationGroupsCategoriesServlet(BaseFederationServlet):
-    """Get all categories for a group
-    """
+    """Get all categories for a group"""
 
     PATH = "/groups/(?P<group_id>[^/]*)/categories/?"
 
@@ -1172,8 +1161,7 @@ class FederationGroupsCategoriesServlet(BaseFederationServlet):
 
 
 class FederationGroupsCategoryServlet(BaseFederationServlet):
-    """Add/remove/get a category in a group
-    """
+    """Add/remove/get a category in a group"""
 
     PATH = "/groups/(?P<group_id>[^/]*)/categories/(?P<category_id>[^/]+)"
 
@@ -1218,8 +1206,7 @@ class FederationGroupsCategoryServlet(BaseFederationServlet):
 
 
 class FederationGroupsRolesServlet(BaseFederationServlet):
-    """Get roles in a group
-    """
+    """Get roles in a group"""
 
     PATH = "/groups/(?P<group_id>[^/]*)/roles/?"
 
@@ -1234,8 +1221,7 @@ class FederationGroupsRolesServlet(BaseFederationServlet):
 
 
 class FederationGroupsRoleServlet(BaseFederationServlet):
-    """Add/remove/get a role in a group
-    """
+    """Add/remove/get a role in a group"""
 
     PATH = "/groups/(?P<group_id>[^/]*)/roles/(?P<role_id>[^/]+)"
 
@@ -1325,8 +1311,7 @@ class FederationGroupsSummaryUsersServlet(BaseFederationServlet):
 
 
 class FederationGroupsBulkPublicisedServlet(BaseFederationServlet):
-    """Get roles in a group
-    """
+    """Get roles in a group"""
 
     PATH = "/get_groups_publicised"
 
@@ -1339,8 +1324,7 @@ class FederationGroupsBulkPublicisedServlet(BaseFederationServlet):
 
 
 class FederationGroupsSettingJoinPolicyServlet(BaseFederationServlet):
-    """Sets whether a group is joinable without an invite or knock
-    """
+    """Sets whether a group is joinable without an invite or knock"""
 
     PATH = "/groups/(?P<group_id>[^/]*)/settings/m.join_policy"
 

synapse/federation/units.py

@@ -29,7 +29,7 @@ logger = logging.getLogger(__name__)
 
 @attr.s(slots=True)
 class Edu(JsonEncodedObject):
-    """ An Edu represents a piece of data sent from one homeserver to another.
+    """An Edu represents a piece of data sent from one homeserver to another.
 
     In comparison to Pdus, Edus are not persisted for a long time on disk, are
     not meaningful beyond a given pair of homeservers, and don't have an
@@ -63,7 +63,7 @@ class Edu(JsonEncodedObject):
 
 
 class Transaction(JsonEncodedObject):
-    """ A transaction is a list of Pdus and Edus to be sent to a remote home
+    """A transaction is a list of Pdus and Edus to be sent to a remote home
     server with some extra metadata.
 
     Example transaction::
@@ -99,7 +99,7 @@ class Transaction(JsonEncodedObject):
     ]
 
     def __init__(self, transaction_id=None, pdus=[], **kwargs):
-        """ If we include a list of pdus then we decode then as PDU's
+        """If we include a list of pdus then we decode then as PDU's
         automatically.
         """
 
@@ -111,7 +111,7 @@ class Transaction(JsonEncodedObject):
 
     @staticmethod
     def create_new(pdus, **kwargs):
-        """ Used to create a new transaction. Will auto fill out
+        """Used to create a new transaction. Will auto fill out
         transaction_id and origin_server_ts keys.
         """
         if "origin_server_ts" not in kwargs:

synapse/groups/attestations.py

@@ -61,8 +61,7 @@ UPDATE_ATTESTATION_TIME_MS = 1 * 24 * 60 * 60 * 1000
 
 
 class GroupAttestationSigning:
-    """Creates and verifies group attestations.
-    """
+    """Creates and verifies group attestations."""
 
     def __init__(self, hs):
         self.keyring = hs.get_keyring()
@@ -125,8 +124,7 @@ class GroupAttestationSigning:
 
 
 class GroupAttestionRenewer:
-    """Responsible for sending and receiving attestation updates.
-    """
+    """Responsible for sending and receiving attestation updates."""
 
     def __init__(self, hs):
         self.clock = hs.get_clock()
@@ -142,8 +140,7 @@ class GroupAttestionRenewer:
             )
 
     async def on_renew_attestation(self, group_id, user_id, content):
-        """When a remote updates an attestation
-        """
+        """When a remote updates an attestation"""
         attestation = content["attestation"]
 
         if not self.is_mine_id(group_id) and not self.is_mine_id(user_id):
@@ -161,8 +158,7 @@ class GroupAttestionRenewer:
         return run_as_background_process("renew_attestations", self._renew_attestations)
 
     async def _renew_attestations(self):
-        """Called periodically to check if we need to update any of our attestations
-        """
+        """Called periodically to check if we need to update any of our attestations"""
 
         now = self.clock.time_msec()
 

synapse/groups/groups_server.py

@@ -18,6 +18,7 @@
 import logging
 
 from synapse.api.errors import Codes, SynapseError
+from synapse.handlers.profile import MAX_AVATAR_URL_LEN, MAX_DISPLAYNAME_LEN
 from synapse.types import GroupID, RoomID, UserID, get_domain_from_id
 from synapse.util.async_helpers import concurrently_execute
 
@@ -32,6 +33,11 @@ logger = logging.getLogger(__name__)
 # TODO: Flairs
 
 
+# Note that the maximum lengths are somewhat arbitrary.
+MAX_SHORT_DESC_LEN = 1000
+MAX_LONG_DESC_LEN = 10000
+
+
 class GroupsServerWorkerHandler:
     def __init__(self, hs):
         self.hs = hs
@@ -159,16 +165,14 @@ class GroupsServerWorkerHandler:
         }
 
     async def get_group_categories(self, group_id, requester_user_id):
-        """Get all categories in a group (as seen by user)
-        """
+        """Get all categories in a group (as seen by user)"""
         await self.check_group_is_ours(group_id, requester_user_id, and_exists=True)
 
         categories = await self.store.get_group_categories(group_id=group_id)
         return {"categories": categories}
 
     async def get_group_category(self, group_id, requester_user_id, category_id):
-        """Get a specific category in a group (as seen by user)
-        """
+        """Get a specific category in a group (as seen by user)"""
         await self.check_group_is_ours(group_id, requester_user_id, and_exists=True)
 
         res = await self.store.get_group_category(
@@ -180,24 +184,21 @@ class GroupsServerWorkerHandler:
         return res
 
     async def get_group_roles(self, group_id, requester_user_id):
-        """Get all roles in a group (as seen by user)
-        """
+        """Get all roles in a group (as seen by user)"""
         await self.check_group_is_ours(group_id, requester_user_id, and_exists=True)
 
         roles = await self.store.get_group_roles(group_id=group_id)
         return {"roles": roles}
 
     async def get_group_role(self, group_id, requester_user_id, role_id):
-        """Get a specific role in a group (as seen by user)
-        """
+        """Get a specific role in a group (as seen by user)"""
         await self.check_group_is_ours(group_id, requester_user_id, and_exists=True)
 
         res = await self.store.get_group_role(group_id=group_id, role_id=role_id)
         return res
 
     async def get_group_profile(self, group_id, requester_user_id):
-        """Get the group profile as seen by requester_user_id
-        """
+        """Get the group profile as seen by requester_user_id"""
 
         await self.check_group_is_ours(group_id, requester_user_id)
 
@@ -344,8 +345,7 @@ class GroupsServerHandler(GroupsServerWorkerHandler):
     async def update_group_summary_room(
         self, group_id, requester_user_id, room_id, category_id, content
     ):
-        """Add/update a room to the group summary
-        """
+        """Add/update a room to the group summary"""
         await self.check_group_is_ours(
             group_id, requester_user_id, and_exists=True, and_is_admin=requester_user_id
         )
@@ -369,8 +369,7 @@ class GroupsServerHandler(GroupsServerWorkerHandler):
     async def delete_group_summary_room(
         self, group_id, requester_user_id, room_id, category_id
     ):
-        """Remove a room from the summary
-        """
+        """Remove a room from the summary"""
         await self.check_group_is_ours(
             group_id, requester_user_id, and_exists=True, and_is_admin=requester_user_id
         )
@@ -403,8 +402,7 @@ class GroupsServerHandler(GroupsServerWorkerHandler):
     async def update_group_category(
         self, group_id, requester_user_id, category_id, content
     ):
-        """Add/Update a group category
-        """
+        """Add/Update a group category"""
         await self.check_group_is_ours(
             group_id, requester_user_id, and_exists=True, and_is_admin=requester_user_id
         )
@@ -422,8 +420,7 @@ class GroupsServerHandler(GroupsServerWorkerHandler):
         return {}
 
     async def delete_group_category(self, group_id, requester_user_id, category_id):
-        """Delete a group category
-        """
+        """Delete a group category"""
         await self.check_group_is_ours(
             group_id, requester_user_id, and_exists=True, and_is_admin=requester_user_id
         )
@@ -435,8 +432,7 @@ class GroupsServerHandler(GroupsServerWorkerHandler):
         return {}
 
     async def update_group_role(self, group_id, requester_user_id, role_id, content):
-        """Add/update a role in a group
-        """
+        """Add/update a role in a group"""
         await self.check_group_is_ours(
             group_id, requester_user_id, and_exists=True, and_is_admin=requester_user_id
         )
@@ -452,8 +448,7 @@ class GroupsServerHandler(GroupsServerWorkerHandler):
         return {}
 
     async def delete_group_role(self, group_id, requester_user_id, role_id):
-        """Remove role from group
-        """
+        """Remove role from group"""
         await self.check_group_is_ours(
             group_id, requester_user_id, and_exists=True, and_is_admin=requester_user_id
         )
@@ -465,8 +460,7 @@ class GroupsServerHandler(GroupsServerWorkerHandler):
     async def update_group_summary_user(
         self, group_id, requester_user_id, user_id, role_id, content
     ):
-        """Add/update a users entry in the group summary
-        """
+        """Add/update a users entry in the group summary"""
         await self.check_group_is_ours(
             group_id, requester_user_id, and_exists=True, and_is_admin=requester_user_id
         )
@@ -488,8 +482,7 @@ class GroupsServerHandler(GroupsServerWorkerHandler):
     async def delete_group_summary_user(
         self, group_id, requester_user_id, user_id, role_id
     ):
-        """Remove a user from the group summary
-        """
+        """Remove a user from the group summary"""
         await self.check_group_is_ours(
             group_id, requester_user_id, and_exists=True, and_is_admin=requester_user_id
         )
@@ -501,25 +494,38 @@ class GroupsServerHandler(GroupsServerWorkerHandler):
         return {}
 
     async def update_group_profile(self, group_id, requester_user_id, content):
-        """Update the group profile
-        """
+        """Update the group profile"""
         await self.check_group_is_ours(
             group_id, requester_user_id, and_exists=True, and_is_admin=requester_user_id
         )
 
         profile = {}
-        for keyname in ("name", "avatar_url", "short_description", "long_description"):
+        for keyname, max_length in (
+            ("name", MAX_DISPLAYNAME_LEN),
+            ("avatar_url", MAX_AVATAR_URL_LEN),
+            ("short_description", MAX_SHORT_DESC_LEN),
+            ("long_description", MAX_LONG_DESC_LEN),
+        ):
             if keyname in content:
                 value = content[keyname]
                 if not isinstance(value, str):
-                    raise SynapseError(400, "%r value is not a string" % (keyname,))
+                    raise SynapseError(
+                        400,
+                        "%r value is not a string" % (keyname,),
+                        errcode=Codes.INVALID_PARAM,
+                    )
+                if len(value) > max_length:
+                    raise SynapseError(
+                        400,
+                        "Invalid %s parameter" % (keyname,),
+                        errcode=Codes.INVALID_PARAM,
+                    )
                 profile[keyname] = value
 
         await self.store.update_group_profile(group_id, profile)
 
     async def add_room_to_group(self, group_id, requester_user_id, room_id, content):
-        """Add room to group
-        """
+        """Add room to group"""
         RoomID.from_string(room_id)  # Ensure valid room id
 
         await self.check_group_is_ours(
@@ -535,8 +541,7 @@ class GroupsServerHandler(GroupsServerWorkerHandler):
     async def update_room_in_group(
         self, group_id, requester_user_id, room_id, config_key, content
     ):
-        """Update room in group
-        """
+        """Update room in group"""
         RoomID.from_string(room_id)  # Ensure valid room id
 
         await self.check_group_is_ours(
@@ -555,8 +560,7 @@ class GroupsServerHandler(GroupsServerWorkerHandler):
         return {}
 
     async def remove_room_from_group(self, group_id, requester_user_id, room_id):
-        """Remove room from group
-        """
+        """Remove room from group"""
         await self.check_group_is_ours(
             group_id, requester_user_id, and_exists=True, and_is_admin=requester_user_id
         )
@@ -566,8 +570,7 @@ class GroupsServerHandler(GroupsServerWorkerHandler):
         return {}
 
     async def invite_to_group(self, group_id, user_id, requester_user_id, content):
-        """Invite user to group
-        """
+        """Invite user to group"""
 
         group = await self.check_group_is_ours(
             group_id, requester_user_id, and_exists=True, and_is_admin=requester_user_id
@@ -703,8 +706,7 @@ class GroupsServerHandler(GroupsServerWorkerHandler):
         return {"state": "join", "attestation": local_attestation}
 
     async def knock(self, group_id, requester_user_id, content):
-        """A user requests becoming a member of the group
-        """
+        """A user requests becoming a member of the group"""
         await self.check_group_is_ours(group_id, requester_user_id, and_exists=True)
 
         raise NotImplementedError()
@@ -897,8 +899,7 @@ class GroupsServerHandler(GroupsServerWorkerHandler):
 
 
 def _parse_join_policy_from_contents(content):
-    """Given a content for a request, return the specified join policy or None
-    """
+    """Given a content for a request, return the specified join policy or None"""
 
     join_policy_dict = content.get("m.join_policy")
     if join_policy_dict:
@@ -908,8 +909,7 @@ def _parse_join_policy_from_contents(content):
 
 
 def _parse_join_policy_dict(join_policy_dict):
-    """Given a dict for the "m.join_policy" config return the join policy specified
-    """
+    """Given a dict for the "m.join_policy" config return the join policy specified"""
     join_policy_type = join_policy_dict.get("type")
     if not join_policy_type:
         return "invite"

synapse/handlers/admin.py

@@ -203,13 +203,11 @@ class AdminHandler(BaseHandler):
 
 
 class ExfiltrationWriter(metaclass=abc.ABCMeta):
-    """Interface used to specify how to write exported data.
-    """
+    """Interface used to specify how to write exported data."""
 
     @abc.abstractmethod
     def write_events(self, room_id: str, events: List[EventBase]) -> None:
-        """Write a batch of events for a room.
-        """
+        """Write a batch of events for a room."""
         raise NotImplementedError()
 
     @abc.abstractmethod

synapse/handlers/appservice.py

@@ -290,7 +290,9 @@ class ApplicationServicesHandler:
             if not interested:
                 continue
             presence_events, _ = await presence_source.get_new_events(
-                user=user, service=service, from_key=from_key,
+                user=user,
+                service=service,
+                from_key=from_key,
             )
             time_now = self.clock.time_msec()
             events.extend(

synapse/handlers/auth.py

@@ -120,7 +120,9 @@ def convert_client_dict_legacy_fields_to_identifier(
     # Ensure the identifier has a type
     if "type" not in identifier:
         raise SynapseError(
-            400, "'identifier' dict has no key 'type'", errcode=Codes.MISSING_PARAM,
+            400,
+            "'identifier' dict has no key 'type'",
+            errcode=Codes.MISSING_PARAM,
         )
 
     return identifier
@@ -351,7 +353,11 @@ class AuthHandler(BaseHandler):
 
         try:
             result, params, session_id = await self.check_ui_auth(
-                flows, request, request_body, description, get_new_session_data,
+                flows,
+                request,
+                request_body,
+                description,
+                get_new_session_data,
             )
         except LoginError:
             # Update the ratelimiter to say we failed (`can_do_action` doesn't raise).
@@ -379,8 +385,7 @@ class AuthHandler(BaseHandler):
         return params, session_id
 
     async def _get_available_ui_auth_types(self, user: UserID) -> Iterable[str]:
-        """Get a list of the authentication types this user can use
-        """
+        """Get a list of the authentication types this user can use"""
 
         ui_auth_types = set()
 
@@ -723,7 +728,9 @@ class AuthHandler(BaseHandler):
         }
 
     def _auth_dict_for_flows(
-        self, flows: List[List[str]], session_id: str,
+        self,
+        flows: List[List[str]],
+        session_id: str,
     ) -> Dict[str, Any]:
         public_flows = []
         for f in flows:
@@ -880,7 +887,9 @@ class AuthHandler(BaseHandler):
         return self._supported_login_types
 
     async def validate_login(
-        self, login_submission: Dict[str, Any], ratelimit: bool = False,
+        self,
+        login_submission: Dict[str, Any],
+        ratelimit: bool = False,
     ) -> Tuple[str, Optional[Callable[[Dict[str, str]], Awaitable[None]]]]:
         """Authenticates the user for the /login API
 
@@ -1023,7 +1032,9 @@ class AuthHandler(BaseHandler):
             raise
 
     async def _validate_userid_login(
-        self, username: str, login_submission: Dict[str, Any],
+        self,
+        username: str,
+        login_submission: Dict[str, Any],
     ) -> Tuple[str, Optional[Callable[[Dict[str, str]], Awaitable[None]]]]:
         """Helper for validate_login
 
@@ -1446,7 +1457,8 @@ class AuthHandler(BaseHandler):
         # is considered OK since the newest SSO attributes should be most valid.
         if extra_attributes:
             self._extra_attributes[registered_user_id] = SsoLoginExtraAttributes(
-                self._clock.time_msec(), extra_attributes,
+                self._clock.time_msec(),
+                extra_attributes,
             )
 
         # Create a login token
@@ -1472,10 +1484,22 @@ class AuthHandler(BaseHandler):
         # Remove the query parameters from the redirect URL to get a shorter version of
         # it. This is only to display a human-readable URL in the template, but not the
         # URL we redirect users to.
-        redirect_url_no_params = client_redirect_url.split("?")[0]
+        url_parts = urllib.parse.urlsplit(client_redirect_url)
+
+        if url_parts.scheme == "https":
+            # for an https uri, just show the netloc (ie, the hostname. Specifically,
+            # the bit between "//" and "/"; this includes any potential
+            # "username:password@" prefix.)
+            display_url = url_parts.netloc
+        else:
+            # for other uris, strip the query-params (including the login token) and
+            # fragment.
+            display_url = urllib.parse.urlunsplit(
+                (url_parts.scheme, url_parts.netloc, url_parts.path, "", "")
+            )
 
         html = self._sso_redirect_confirm_template.render(
-            display_url=redirect_url_no_params,
+            display_url=display_url,
             redirect_url=redirect_url,
             server_name=self._server_name,
             new_user=new_user,
@@ -1690,5 +1714,9 @@ class PasswordProvider:
         # This might return an awaitable, if it does block the log out
         # until it completes.
         await maybe_awaitable(
-            g(user_id=user_id, device_id=device_id, access_token=access_token,)
+            g(
+                user_id=user_id,
+                device_id=device_id,
+                access_token=access_token,
+            )
         )

synapse/handlers/cas_handler.py

@@ -14,7 +14,7 @@
 # limitations under the License.
 import logging
 import urllib.parse
-from typing import TYPE_CHECKING, Dict, Optional
+from typing import TYPE_CHECKING, Dict, List, Optional
 from xml.etree import ElementTree as ET
 
 import attr
@@ -33,8 +33,7 @@ logger = logging.getLogger(__name__)
 
 
 class CasError(Exception):
-    """Used to catch errors when validating the CAS ticket.
-    """
+    """Used to catch errors when validating the CAS ticket."""
 
     def __init__(self, error, error_description=None):
         self.error = error
@@ -49,7 +48,7 @@ class CasError(Exception):
 @attr.s(slots=True, frozen=True)
 class CasResponse:
     username = attr.ib(type=str)
-    attributes = attr.ib(type=Dict[str, Optional[str]])
+    attributes = attr.ib(type=Dict[str, List[Optional[str]]])
 
 
 class CasHandler:
@@ -100,7 +99,10 @@ class CasHandler:
         Returns:
             The URL to use as a "service" parameter.
         """
-        return "%s?%s" % (self._cas_service_url, urllib.parse.urlencode(args),)
+        return "%s?%s" % (
+            self._cas_service_url,
+            urllib.parse.urlencode(args),
+        )
 
     async def _validate_ticket(
         self, ticket: str, service_args: Dict[str, str]
@@ -169,7 +171,7 @@ class CasHandler:
 
         # Iterate through the nodes and pull out the user and any extra attributes.
         user = None
-        attributes = {}
+        attributes = {}  # type: Dict[str, List[Optional[str]]]
         for child in root[0]:
             if child.tag.endswith("user"):
                 user = child.text
@@ -182,7 +184,7 @@ class CasHandler:
                     tag = attribute.tag
                     if "}" in tag:
                         tag = tag.split("}")[1]
-                    attributes[tag] = attribute.text
+                    attributes.setdefault(tag, []).append(attribute.text)
 
         # Ensure a user was found.
         if user is None:
@@ -296,36 +298,20 @@ class CasHandler:
         # first check if we're doing a UIA
         if session:
             return await self._sso_handler.complete_sso_ui_auth_request(
-                self.idp_id, cas_response.username, session, request,
+                self.idp_id,
+                cas_response.username,
+                session,
+                request,
             )
 
         # otherwise, we're handling a login request.
 
         # Ensure that the attributes of the logged in user meet the required
         # attributes.
-        for required_attribute, required_value in self._cas_required_attributes.items():
-            # If required attribute was not in CAS Response - Forbidden
-            if required_attribute not in cas_response.attributes:
-                self._sso_handler.render_error(
-                    request,
-                    "unauthorised",
-                    "You are not authorised to log in here.",
-                    401,
-                )
-                return
-
-            # Also need to check value
-            if required_value is not None:
-                actual_value = cas_response.attributes[required_attribute]
-                # If required attribute value does not match expected - Forbidden
-                if required_value != actual_value:
-                    self._sso_handler.render_error(
-                        request,
-                        "unauthorised",
-                        "You are not authorised to log in here.",
-                        401,
-                    )
-                    return
+        if not self._sso_handler.check_required_attributes(
+            request, cas_response.attributes, self._cas_required_attributes
+        ):
+            return
 
         # Call the mapper to register/login the user
 
@@ -372,9 +358,10 @@ class CasHandler:
             if failures:
                 raise RuntimeError("CAS is not expected to de-duplicate Matrix IDs")
 
+            # Arbitrarily use the first attribute found.
             display_name = cas_response.attributes.get(
-                self._cas_displayname_attribute, None
-            )
+                self._cas_displayname_attribute, [None]
+            )[0]
 
             return UserAttributes(localpart=localpart, display_name=display_name)
 
@@ -384,7 +371,8 @@ class CasHandler:
             user_id = UserID(localpart, self._hostname).to_string()
 
             logger.debug(
-                "Looking for existing account based on mapped %s", user_id,
+                "Looking for existing account based on mapped %s",
+                user_id,
             )
 
             users = await self._store.get_users_by_id_case_insensitive(user_id)

synapse/handlers/deactivate_account.py

@@ -196,8 +196,7 @@ class DeactivateAccountHandler(BaseHandler):
             run_as_background_process("user_parter_loop", self._user_parter_loop)
 
     async def _user_parter_loop(self) -> None:
-        """Loop that parts deactivated users from rooms
-        """
+        """Loop that parts deactivated users from rooms"""
         self._user_parter_running = True
         logger.info("Starting user parter")
         try:
@@ -214,8 +213,7 @@ class DeactivateAccountHandler(BaseHandler):
             self._user_parter_running = False
 
     async def _part_user(self, user_id: str) -> None:
-        """Causes the given user_id to leave all the rooms they're joined to
-        """
+        """Causes the given user_id to leave all the rooms they're joined to"""
         user = UserID.from_string(user_id)
 
         rooms_for_user = await self.store.get_rooms_for_user(user_id)

synapse/handlers/device.py

@@ -86,7 +86,7 @@ class DeviceWorkerHandler(BaseHandler):
 
     @trace
     async def get_device(self, user_id: str, device_id: str) -> JsonDict:
-        """ Retrieve the given device
+        """Retrieve the given device
 
         Args:
             user_id: The user to get the device from
@@ -341,7 +341,7 @@ class DeviceHandler(DeviceWorkerHandler):
 
     @trace
     async def delete_device(self, user_id: str, device_id: str) -> None:
-        """ Delete the given device
+        """Delete the given device
 
         Args:
             user_id: The user to delete the device from.
@@ -386,7 +386,7 @@ class DeviceHandler(DeviceWorkerHandler):
         await self.delete_devices(user_id, device_ids)
 
     async def delete_devices(self, user_id: str, device_ids: List[str]) -> None:
-        """ Delete several devices
+        """Delete several devices
 
         Args:
             user_id: The user to delete devices from.
@@ -417,7 +417,7 @@ class DeviceHandler(DeviceWorkerHandler):
         await self.notify_device_update(user_id, device_ids)
 
     async def update_device(self, user_id: str, device_id: str, content: dict) -> None:
-        """ Update the given device
+        """Update the given device
 
         Args:
             user_id: The user to update devices of.
@@ -534,7 +534,9 @@ class DeviceHandler(DeviceWorkerHandler):
             device id of the dehydrated device
         """
         device_id = await self.check_device_registered(
-            user_id, None, initial_device_display_name,
+            user_id,
+            None,
+            initial_device_display_name,
         )
         old_device_id = await self.store.store_dehydrated_device(
             user_id, device_id, device_data
@@ -803,7 +805,8 @@ class DeviceListUpdater:
                 try:
                     # Try to resync the current user's devices list.
                     result = await self.user_device_resync(
-                        user_id=user_id, mark_failed_as_stale=False,
+                        user_id=user_id,
+                        mark_failed_as_stale=False,
                     )
 
                     # user_device_resync only returns a result if it managed to
@@ -813,14 +816,17 @@ class DeviceListUpdater:
                     # self.store.update_remote_device_list_cache).
                     if result:
                         logger.debug(
-                            "Successfully resynced the device list for %s", user_id,
+                            "Successfully resynced the device list for %s",
+                            user_id,
                         )
                 except Exception as e:
                     # If there was an issue resyncing this user, e.g. if the remote
                     # server sent a malformed result, just log the error instead of
                     # aborting all the subsequent resyncs.
                     logger.debug(
-                        "Could not resync the device list for %s: %s", user_id, e,
+                        "Could not resync the device list for %s: %s",
+                        user_id,
+                        e,
                     )
         finally:
             # Allow future calls to retry resyncinc out of sync device lists.
@@ -855,7 +861,9 @@ class DeviceListUpdater:
             return None
         except (RequestSendFailed, HttpResponseException) as e:
             logger.warning(
-                "Failed to handle device list update for %s: %s", user_id, e,
+                "Failed to handle device list update for %s: %s",
+                user_id,
+                e,
             )
 
             if mark_failed_as_stale:
@@ -931,7 +939,9 @@ class DeviceListUpdater:
 
         # Handle cross-signing keys.
         cross_signing_device_ids = await self.process_cross_signing_key_update(
-            user_id, master_key, self_signing_key,
+            user_id,
+            master_key,
+            self_signing_key,
         )
         device_ids = device_ids + cross_signing_device_ids