Return empty object on success rather than null.

Signed-off-by: Olivier Wilkinson (reivilibre) <olivier@librepush.net>

.buildkite/merge_base_branch.sh

@@ -2,7 +2,7 @@
 
 set -ex
 
-if [[ "$BUILDKITE_BRANCH" =~ ^(develop|master|dinsic|shhs-.*|release-.*)$ ]]; then
+if [[ "$BUILDKITE_BRANCH" =~ ^(develop|master|dinsic|shhs|release-.*)$ ]]; then
     echo "Not merging forward, as this is a release branch"
     exit 0
 fi

.buildkite/pipeline.yml

@@ -31,7 +31,7 @@ steps:
       - "python -m pip install tox"
       - "scripts-dev/check-newsfragment"
     label: ":newspaper: Newsfile"
-    branches: "!master !develop !release-* !shhs-v*"
+    branches: "!master !develop !release-*"
     plugins:
       - docker#v3.0.1:
           image: "python:3.6"
@@ -47,16 +47,17 @@ steps:
 
   - wait
 
+
   - command:
-      - "python -m pip install tox"
+      - "apt-get update && apt-get install -y python3.5 python3.5-dev python3-pip libxml2-dev libxslt-dev zlib1g-dev"
+      - "python3.5 -m pip install tox"
       - "tox -e py35-old,codecov"
     label: ":python: 3.5 / SQLite / Old Deps"
-    branches: "!shhs !shhs-*"
     env:
       TRIAL_FLAGS: "-j 2"
     plugins:
       - docker#v3.0.1:
-          image: "python:3.5"
+          image: "ubuntu:xenial"  # We use xenail to get an old sqlite and python
           propagate-environment: true
     retry:
       automatic:
@@ -69,7 +70,6 @@ steps:
       - "python -m pip install tox"
       - "tox -e py35,codecov"
     label: ":python: 3.5 / SQLite"
-    branches: "!shhs !shhs-*"
     env:
       TRIAL_FLAGS: "-j 2"
     plugins:
@@ -87,7 +87,6 @@ steps:
       - "python -m pip install tox"
       - "tox -e py36,codecov"
     label: ":python: 3.6 / SQLite"
-    branches: "!shhs !shhs-*"
     env:
       TRIAL_FLAGS: "-j 2"
     plugins:
@@ -119,9 +118,10 @@ steps:
           limit: 2
 
   - label: ":python: 3.5 / :postgres: 9.5"
-    branches: "!shhs !shhs-*"
+    agents:
+      queue: "medium"
     env:
-      TRIAL_FLAGS: "-j 4"
+      TRIAL_FLAGS: "-j 8"
     command:
       - "bash -c 'python -m pip install tox && python -m tox -e py35-postgres,codecov'"
     plugins:
@@ -137,9 +137,10 @@ steps:
           limit: 2
 
   - label: ":python: 3.7 / :postgres: 9.5"
-    branches: "!shhs !shhs-*"
+    agents:
+      queue: "medium"
     env:
-      TRIAL_FLAGS: "-j 4"
+      TRIAL_FLAGS: "-j 8"
     command:
       - "bash -c 'python -m pip install tox && python -m tox -e py37-postgres,codecov'"
     plugins:
@@ -155,8 +156,10 @@ steps:
           limit: 2
 
   - label: ":python: 3.7 / :postgres: 11"
+    agents:
+      queue: "medium"
     env:
-      TRIAL_FLAGS: "-j 4"
+      TRIAL_FLAGS: "-j 8"
     command:
       - "bash -c 'python -m pip install tox && python -m tox -e py37-postgres,codecov'"
     plugins:
@@ -173,7 +176,6 @@ steps:
 
 
   - label: "SyTest - :python: 3.5 / SQLite / Monolith"
-    branches: "!shhs !shhs-*"
     agents:
       queue: "medium"
     command:
@@ -214,14 +216,15 @@ steps:
           limit: 2
 
   - label: "SyTest - :python: 3.5 / :postgres: 9.6 / Workers"
-    branches: "!shhs !shhs-*"
     agents:
       queue: "medium"
     env:
       POSTGRES: "1"
       WORKERS: "1"
+      BLACKLIST: "synapse-blacklist-with-workers"
     command:
       - "bash .buildkite/merge_base_branch.sh"
+      - "bash -c 'cat /src/sytest-blacklist /src/.buildkite/worker-blacklist > /src/synapse-blacklist-with-workers'"
       - "bash /synapse_sytest.sh"
     plugins:
       - docker#v3.0.1:
@@ -229,22 +232,9 @@ steps:
           propagate-environment: true
           always-pull: true
           workdir: "/src"
-    soft_fail: true
     retry:
       automatic:
         - exit_status: -1
           limit: 2
         - exit_status: 2
           limit: 2
-
-  - wait
-
-  - label: ":docker: x86_64"
-    agents:
-      queue: "release"
-    branches: "shhs-*"
-    command:
-      - "docker build -f docker/Dockerfile --build-arg PYTHON_VERSION=3.7.4 . -t matrixdotorg/synapse:${BUILDKITE_TAG}"
-      - "docker save matrixdotorg/synapse:${BUILDKITE_TAG} | gzip -9 > docker.tar.gz"
-    artifact_paths:
-      - "docker.tar.gz"

.buildkite/worker-blacklist

@@ -0,0 +1,30 @@
+# This file serves as a blacklist for SyTest tests that we expect will fail in
+# Synapse when run under worker mode. For more details, see sytest-blacklist.
+
+Message history can be paginated
+
+Can re-join room if re-invited
+
+/upgrade creates a new room
+
+The only membership state included in an initial sync is for all the senders in the timeline
+
+Local device key changes get to remote servers
+
+If remote user leaves room we no longer receive device updates
+
+Forgotten room messages cannot be paginated
+
+Inbound federation can get public room list
+
+Members from the gap are included in gappy incr LL sync
+
+Leaves are present in non-gapped incremental syncs
+
+Old leaves are present in gapped incremental syncs
+
+User sees updates to presence from other users in the incremental sync.
+
+Gapped incremental syncs include all state changes
+
+Old members are included in gappy incr LL sync if they start speaking

.circleci/config.yml

@@ -0,0 +1,33 @@
+version: 2
+jobs:
+  dockerhubuploadrelease:
+    machine: true
+    steps:
+      - checkout
+      - run: docker build -f docker/Dockerfile --label gitsha1=${CIRCLE_SHA1} -t matrixdotorg/synapse:${CIRCLE_TAG} -t matrixdotorg/synapse:${CIRCLE_TAG}-py3 .
+      - run: docker login --username $DOCKER_HUB_USERNAME --password $DOCKER_HUB_PASSWORD
+      - run: docker push matrixdotorg/synapse:${CIRCLE_TAG}
+      - run: docker push matrixdotorg/synapse:${CIRCLE_TAG}-py3
+  dockerhubuploadlatest:
+    machine: true
+    steps:
+      - checkout
+      - run: docker build -f docker/Dockerfile --label gitsha1=${CIRCLE_SHA1} -t matrixdotorg/synapse:latest -t matrixdotorg/synapse:latest-py3 .
+      - run: docker login --username $DOCKER_HUB_USERNAME --password $DOCKER_HUB_PASSWORD
+      - run: docker push matrixdotorg/synapse:latest
+      - run: docker push matrixdotorg/synapse:latest-py3
+
+workflows:
+  version: 2
+  build:
+    jobs:
+      - dockerhubuploadrelease:
+          filters:
+            tags:
+              only: /v[0-9].[0-9]+.[0-9]+.*/
+            branches:
+              ignore: /.*/
+      - dockerhubuploadlatest:
+          filters:
+            branches:
+              only: master

.codecov.yml

@@ -1,5 +1,4 @@
-comment:
-  layout: "diff"
+comment: off
 
 coverage:
   status:

.gitignore

@@ -16,10 +16,10 @@ _trial_temp*/
 /*.log
 /*.log.config
 /*.pid
+/.python-version
 /*.signing.key
 /env/
 /homeserver*.yaml
-/logs
 /media_store/
 /uploads
 
@@ -38,3 +38,4 @@ _trial_temp*/
 /docs/build/
 /htmlcov
 /pip-wheel-metadata/
+

CHANGES.md

@@ -1,3 +1,102 @@
+Synapse 1.3.1 (2019-08-17)
+==========================
+
+Features
+--------
+
+- Drop hard dependency on `sdnotify` python package. ([\#5871](https://github.com/matrix-org/synapse/issues/5871))
+
+
+Bugfixes
+--------
+
+- Fix startup issue (hang on ACME provisioning) due to ordering of Twisted reactor startup. Thanks to @chrismoos for supplying the fix. ([\#5867](https://github.com/matrix-org/synapse/issues/5867))
+
+
+Synapse 1.3.0 (2019-08-15)
+==========================
+
+Bugfixes
+--------
+
+- Fix 500 Internal Server Error on `publicRooms` when the public room list was
+  cached. ([\#5851](https://github.com/matrix-org/synapse/issues/5851))
+
+
+Synapse 1.3.0rc1 (2019-08-13)
+==========================
+
+Features
+--------
+
+- Use `M_USER_DEACTIVATED` instead of `M_UNKNOWN` for errcode when a deactivated user attempts to login. ([\#5686](https://github.com/matrix-org/synapse/issues/5686))
+- Add sd_notify hooks to ease systemd integration and allows usage of Type=Notify. ([\#5732](https://github.com/matrix-org/synapse/issues/5732))
+- Synapse will no longer serve any media repo admin endpoints when `enable_media_repo` is set to False in the configuration. If a media repo worker is used, the admin APIs relating to the media repo will be served from it instead. ([\#5754](https://github.com/matrix-org/synapse/issues/5754), [\#5848](https://github.com/matrix-org/synapse/issues/5848))
+- Synapse can now be configured to not join remote rooms of a given "complexity" (currently, state events) over federation. This option can be used to prevent adverse performance on resource-constrained homeservers. ([\#5783](https://github.com/matrix-org/synapse/issues/5783))
+- Allow defining HTML templates to serve the user on account renewal attempt when using the account validity feature. ([\#5807](https://github.com/matrix-org/synapse/issues/5807))
+
+
+Bugfixes
+--------
+
+- Fix UISIs during homeserver outage. ([\#5693](https://github.com/matrix-org/synapse/issues/5693), [\#5789](https://github.com/matrix-org/synapse/issues/5789))
+- Fix stack overflow in server key lookup code. ([\#5724](https://github.com/matrix-org/synapse/issues/5724))
+- start.sh no longer uses deprecated cli option. ([\#5725](https://github.com/matrix-org/synapse/issues/5725))
+- Log when we receive an event receipt from an unexpected origin. ([\#5743](https://github.com/matrix-org/synapse/issues/5743))
+- Fix debian packaging scripts to correctly build sid packages. ([\#5775](https://github.com/matrix-org/synapse/issues/5775))
+- Correctly handle redactions of redactions. ([\#5788](https://github.com/matrix-org/synapse/issues/5788))
+- Return 404 instead of 403 when accessing /rooms/{roomId}/event/{eventId} for an event without the appropriate permissions. ([\#5798](https://github.com/matrix-org/synapse/issues/5798))
+- Fix check that tombstone is a state event in push rules. ([\#5804](https://github.com/matrix-org/synapse/issues/5804))
+- Fix error when trying to login as a deactivated user when using a worker to handle login. ([\#5806](https://github.com/matrix-org/synapse/issues/5806))
+- Fix bug where user `/sync` stream could get wedged in rare circumstances. ([\#5825](https://github.com/matrix-org/synapse/issues/5825))
+- The purge_remote_media.sh script was fixed. ([\#5839](https://github.com/matrix-org/synapse/issues/5839))
+
+
+Deprecations and Removals
+-------------------------
+
+- Synapse now no longer accepts the `-v`/`--verbose`, `-f`/`--log-file`, or `--log-config` command line flags, and removes the deprecated `verbose` and `log_file` configuration file options. Users of these options should migrate their options into the dedicated log configuration. ([\#5678](https://github.com/matrix-org/synapse/issues/5678), [\#5729](https://github.com/matrix-org/synapse/issues/5729))
+- Remove non-functional 'expire_access_token' setting. ([\#5782](https://github.com/matrix-org/synapse/issues/5782))
+
+
+Internal Changes
+----------------
+
+- Make Jaeger fully configurable. ([\#5694](https://github.com/matrix-org/synapse/issues/5694))
+- Add precautionary measures to prevent future abuse of `window.opener` in default welcome page. ([\#5695](https://github.com/matrix-org/synapse/issues/5695))
+- Reduce database IO usage by optimising queries for current membership. ([\#5706](https://github.com/matrix-org/synapse/issues/5706), [\#5738](https://github.com/matrix-org/synapse/issues/5738), [\#5746](https://github.com/matrix-org/synapse/issues/5746), [\#5752](https://github.com/matrix-org/synapse/issues/5752), [\#5770](https://github.com/matrix-org/synapse/issues/5770), [\#5774](https://github.com/matrix-org/synapse/issues/5774), [\#5792](https://github.com/matrix-org/synapse/issues/5792), [\#5793](https://github.com/matrix-org/synapse/issues/5793))
+- Improve caching when fetching `get_filtered_current_state_ids`. ([\#5713](https://github.com/matrix-org/synapse/issues/5713))
+- Don't accept opentracing data from clients. ([\#5715](https://github.com/matrix-org/synapse/issues/5715))
+- Speed up PostgreSQL unit tests in CI. ([\#5717](https://github.com/matrix-org/synapse/issues/5717))
+- Update the coding style document. ([\#5719](https://github.com/matrix-org/synapse/issues/5719))
+- Improve database query performance when recording retry intervals for remote hosts. ([\#5720](https://github.com/matrix-org/synapse/issues/5720))
+- Add a set of opentracing utils. ([\#5722](https://github.com/matrix-org/synapse/issues/5722))
+- Cache result of get_version_string to reduce overhead of `/version` federation requests. ([\#5730](https://github.com/matrix-org/synapse/issues/5730))
+- Return 'user_type' in admin API user endpoints results. ([\#5731](https://github.com/matrix-org/synapse/issues/5731))
+- Don't package the sytest test blacklist file. ([\#5733](https://github.com/matrix-org/synapse/issues/5733))
+- Replace uses of returnValue with plain return, as returnValue is not needed on Python 3. ([\#5736](https://github.com/matrix-org/synapse/issues/5736))
+- Blacklist some flakey tests in worker mode. ([\#5740](https://github.com/matrix-org/synapse/issues/5740))
+- Fix some error cases in the caching layer. ([\#5749](https://github.com/matrix-org/synapse/issues/5749))
+- Add a prometheus metric for pending cache lookups. ([\#5750](https://github.com/matrix-org/synapse/issues/5750))
+- Stop trying to fetch events with event_id=None. ([\#5753](https://github.com/matrix-org/synapse/issues/5753))
+- Convert RedactionTestCase to modern test style. ([\#5768](https://github.com/matrix-org/synapse/issues/5768))
+- Allow looping calls to be given arguments. ([\#5780](https://github.com/matrix-org/synapse/issues/5780))
+- Set the logs emitted when checking typing and presence timeouts to DEBUG level, not INFO. ([\#5785](https://github.com/matrix-org/synapse/issues/5785))
+- Remove DelayedCall debugging from the test suite, as it is no longer required in the vast majority of Synapse's tests. ([\#5787](https://github.com/matrix-org/synapse/issues/5787))
+- Remove some spurious exceptions from the logs where we failed to talk to a remote server. ([\#5790](https://github.com/matrix-org/synapse/issues/5790))
+- Improve performance when making `.well-known` requests by sharing the SSL options between requests. ([\#5794](https://github.com/matrix-org/synapse/issues/5794))
+- Disable codecov GitHub comments on PRs. ([\#5796](https://github.com/matrix-org/synapse/issues/5796))
+- Don't allow clients to send tombstone events that reference the room it's sent in. ([\#5801](https://github.com/matrix-org/synapse/issues/5801))
+- Deny redactions of events sent in a different room. ([\#5802](https://github.com/matrix-org/synapse/issues/5802))
+- Deny sending well known state types as non-state events. ([\#5805](https://github.com/matrix-org/synapse/issues/5805))
+- Handle incorrectly encoded query params correctly by returning a 400. ([\#5808](https://github.com/matrix-org/synapse/issues/5808))
+- Handle pusher being deleted during processing rather than logging an exception. ([\#5809](https://github.com/matrix-org/synapse/issues/5809))
+- Return 502 not 500 when failing to reach any remote server. ([\#5810](https://github.com/matrix-org/synapse/issues/5810))
+- Reduce global pauses in the events stream caused by expensive state resolution during persistence. ([\#5826](https://github.com/matrix-org/synapse/issues/5826))
+- Add a lower bound to well-known lookup cache time to avoid repeated lookups. ([\#5836](https://github.com/matrix-org/synapse/issues/5836))
+- Whitelist history visbility sytests in worker mode tests. ([\#5843](https://github.com/matrix-org/synapse/issues/5843))
+
+
 Synapse 1.2.1 (2019-07-26)
 ==========================
 
@@ -8,9 +107,9 @@ This release includes *four* security fixes:
 
 - Prevent an attack where a federated server could send redactions for arbitrary events in v1 and v2 rooms. ([\#5767](https://github.com/matrix-org/synapse/issues/5767))
 - Prevent a denial-of-service attack where cycles of redaction events would make Synapse spin infinitely. Thanks to `@lrizika:matrix.org` for identifying and responsibly disclosing this issue. ([0f2ecb961](https://github.com/matrix-org/synapse/commit/0f2ecb961))
-- Prevent an attack where users could be joined or parted from public rooms without their consent. Thanks to @Dylanger for identifying and responsibly disclosing this issue. ([\#5744](https://github.com/matrix-org/synapse/issues/5744))
+- Prevent an attack where users could be joined or parted from public rooms without their consent. Thanks to @dylangerdaly for identifying and responsibly disclosing this issue. ([\#5744](https://github.com/matrix-org/synapse/issues/5744))
 - Fix a vulnerability where a federated server could spoof read-receipts from
-  users on other servers. Thanks to @Dylanger for identifying this issue too. ([\#5743](https://github.com/matrix-org/synapse/issues/5743))
+  users on other servers. Thanks to @dylangerdaly for identifying this issue too. ([\#5743](https://github.com/matrix-org/synapse/issues/5743))
 
 Additionally, the following fix was in Synapse **1.2.0**, but was not correctly
 identified during the original release:

INSTALL.md

@@ -419,12 +419,11 @@ If Synapse is not configured with an SMTP server, password reset via email will
 
 ## Registering a user
 
-You will need at least one user on your server in order to use a Matrix
-client. Users can be registered either via a Matrix client, or via a
-commandline script.
+The easiest way to create a new user is to do so from a client like [Riot](https://riot.im).
 
-To get started, it is easiest to use the command line to register new
-users. This can be done as follows:
+Alternatively you can do so from the command line if you have installed via pip. 
+
+This can be done as follows:
 
 ```
 $ source ~/synapse/env/bin/activate

MANIFEST.in

@@ -7,7 +7,6 @@ include demo/README
 include demo/demo.tls.dh
 include demo/*.py
 include demo/*.sh
-include sytest-blacklist
 
 recursive-include synapse/storage/schema *.sql
 recursive-include synapse/storage/schema *.sql.postgres
@@ -34,6 +33,7 @@ exclude Dockerfile
 exclude .dockerignore
 exclude test_postgresql.sh
 exclude .editorconfig
+exclude sytest-blacklist
 
 include pyproject.toml
 recursive-include changelog.d *

changelog.d/5072.feature

@@ -1 +0,0 @@
-Synapse can now be configured to not join remote rooms of a given "complexity" (currently, state events). This option can be used to prevent adverse performance on resource-constrained homeservers.

changelog.d/5099.misc

@@ -1 +0,0 @@
-Python 2 has been removed from the CI.

changelog.d/5633.bugfix

@@ -0,0 +1 @@
+Don't create broken room when power_level_content_override.users does not contain creator_id.

changelog.d/5844.misc

@@ -0,0 +1 @@
+Retry well-known lookup before the cache expires, giving a grace period where the remote well-known can be down but we still use the old result.

changelog.d/5856.feature

@@ -0,0 +1 @@
+Add a tag recording a request's authenticated entity and corresponding servlet in opentracing.

changelog.d/5857.bugfix

@@ -0,0 +1 @@
+Fix database index so that different backup versions can have the same sessions.

changelog.d/5860.misc

@@ -0,0 +1 @@
+Remove log line for debugging issue #5407.

changelog.d/5863.bugfix

@@ -0,0 +1 @@
+Fix Synapse looking for config options `password_reset_failure_template` and `password_reset_success_template`, when they are actually `password_reset_template_failure_html`, `password_reset_template_success_html`.

changelog.d/5878.feature

@@ -0,0 +1 @@
+Add admin API endpoint for setting whether or not a user is a server administrator.

contrib/purge_api/purge_remote_media.sh

@@ -51,4 +51,4 @@ TOKEN=$(sql "SELECT token FROM access_tokens WHERE user_id='$ADMIN' ORDER BY id
 # finally start pruning media:
 ###############################################################################
 set -x # for debugging the generated string
-curl --header "Authorization: Bearer $TOKEN" -v POST "$API_URL/admin/purge_media_cache/?before_ts=$UNIX_TIMESTAMP"
+curl --header "Authorization: Bearer $TOKEN" -X POST "$API_URL/admin/purge_media_cache/?before_ts=$UNIX_TIMESTAMP"

contrib/systemd-with-workers/system/matrix-synapse-worker@.service

@@ -4,7 +4,8 @@ After=matrix-synapse.service
 BindsTo=matrix-synapse.service
 
 [Service]
-Type=simple
+Type=notify
+NotifyAccess=main
 User=matrix-synapse
 WorkingDirectory=/var/lib/matrix-synapse
 EnvironmentFile=/etc/default/matrix-synapse

contrib/systemd-with-workers/system/matrix-synapse.service

@@ -2,7 +2,8 @@
 Description=Synapse Matrix Homeserver
 
 [Service]
-Type=simple
+Type=notify
+NotifyAccess=main
 User=matrix-synapse
 WorkingDirectory=/var/lib/matrix-synapse
 EnvironmentFile=/etc/default/matrix-synapse

contrib/systemd/matrix-synapse.service

@@ -14,7 +14,9 @@
 Description=Synapse Matrix homeserver
 
 [Service]
-Type=simple
+Type=notify
+NotifyAccess=main
+ExecReload=/bin/kill -HUP $MAINPID
 Restart=on-abort
 
 User=synapse

debian/changelog

@@ -1,8 +1,18 @@
-matrix-synapse-py3 (1.2.1) stable; urgency=medium
+matrix-synapse-py3 (1.3.1) stable; urgency=medium
 
-  * New synapse release 1.2.1.
+  * New synapse release 1.3.1.
 
- -- Synapse Packaging team <packages@matrix.org>  Fri, 26 Jul 2019 11:32:47 +0100
+ -- Synapse Packaging team <packages@matrix.org>  Sat, 17 Aug 2019 09:15:49 +0100
+
+matrix-synapse-py3 (1.3.0) stable; urgency=medium
+
+  [ Andrew Morgan ]
+  * Remove libsqlite3-dev from required build dependencies.
+
+  [ Synapse Packaging team ]
+  * New synapse release 1.3.0.
+
+ -- Synapse Packaging team <packages@matrix.org>  Thu, 15 Aug 2019 12:04:23 +0100
 
 matrix-synapse-py3 (1.2.0) stable; urgency=medium
 

debian/control

@@ -15,7 +15,6 @@ Build-Depends:
  python3-setuptools,
  python3-pip,
  python3-venv,
- libsqlite3-dev,
  tar,
 Standards-Version: 3.9.8
 Homepage: https://github.com/matrix-org/synapse

demo/start.sh

@@ -29,7 +29,7 @@ for port in 8080 8081 8082; do
 
     if ! grep -F "Customisation made by demo/start.sh" -q  $DIR/etc/$port.config; then
         printf '\n\n# Customisation made by demo/start.sh\n' >> $DIR/etc/$port.config
-        
+
         echo 'enable_registration: true' >> $DIR/etc/$port.config
 
         # Warning, this heredoc depends on the interaction of tabs and spaces. Please don't
@@ -43,7 +43,7 @@ for port in 8080 8081 8082; do
 		    tls: true
 		    resources:
 		      - names: [client, federation]
-		
+
 		  - port: $port
 		    tls: false
 		    bind_addresses: ['::1', '127.0.0.1']
@@ -68,7 +68,7 @@ for port in 8080 8081 8082; do
 
         # Generate tls keys
         openssl req -x509 -newkey rsa:4096 -keyout $DIR/etc/localhost\:$https_port.tls.key -out $DIR/etc/localhost\:$https_port.tls.crt -days 365 -nodes -subj "/O=matrix"
-        
+
         # Ignore keys from the trusted keys server
         echo '# Ignore keys from the trusted keys server' >> $DIR/etc/$port.config
         echo 'trusted_key_servers:' >> $DIR/etc/$port.config
@@ -120,7 +120,6 @@ for port in 8080 8081 8082; do
     python3 -m synapse.app.homeserver \
         --config-path "$DIR/etc/$port.config" \
         -D \
-        -vv \
 
     popd
 done

docker/Dockerfile-dhvirtualenv

@@ -42,6 +42,11 @@ RUN cd dh-virtualenv-1.1 && dpkg-buildpackage -us -uc -b
 ###
 FROM ${distro}
 
+# Get the distro we want to pull from as a dynamic build variable
+# (We need to define it in each build stage)
+ARG distro=""
+ENV distro ${distro}
+
 # Install the build dependencies
 #
 # NB: keep this list in sync with the list of build-deps in debian/control

docker/build_debian.sh

@@ -4,7 +4,8 @@
 
 set -ex
 
-DIST=`lsb_release -c -s`
+# Get the codename from distro env
+DIST=`cut -d ':' -f2 <<< $distro`
 
 # we get a read-only copy of the source: make a writeable copy
 cp -aT /synapse/source /synapse/build

docs/admin_api/user_admin_api.rst

@@ -84,3 +84,23 @@ with a body of:
    }
 
 including an ``access_token`` of a server admin.
+
+
+Change whether a user is a server administrator or not
+======================================================
+
+Note that you cannot demote yourself.
+
+The api is::
+
+    PUT /_synapse/admin/v1/users/<user_id>/admin
+
+with a body of:
+
+.. code:: json
+
+    {
+        "admin": true
+    }
+
+including an ``access_token`` of a server admin.

docs/code_style.rst

@@ -1,4 +1,8 @@
-# Code Style
+Code Style
+==========
+
+Formatting tools
+----------------
 
 The Synapse codebase uses a number of code formatting tools in order to
 quickly and automatically check for formatting (and sometimes logical) errors
@@ -6,20 +10,20 @@ in code.
 
 The necessary tools are detailed below.
 
-## Formatting tools
+- **black**
 
-The Synapse codebase uses [black](https://pypi.org/project/black/) as an
-opinionated code formatter, ensuring all comitted code is properly
-formatted.
+  The Synapse codebase uses `black <https://pypi.org/project/black/>`_ as an
+  opinionated code formatter, ensuring all comitted code is properly
+  formatted.
 
-First install ``black`` with::
+  First install ``black`` with::
 
-  pip install --upgrade black
+    pip install --upgrade black
 
-Have ``black`` auto-format your code (it shouldn't change any
-functionality) with::
+  Have ``black`` auto-format your code (it shouldn't change any functionality)
+  with::
 
-  black . --exclude="\.tox|build|env"
+    black . --exclude="\.tox|build|env"
 
 - **flake8**
 
@@ -54,17 +58,16 @@ functionality is supported in your editor for a more convenient development
 workflow. It is not, however, recommended to run ``flake8`` on save as it
 takes a while and is very resource intensive.
 
-## General rules
+General rules
+-------------
 
 - **Naming**:
 
   - Use camel case for class and type names
   - Use underscores for functions and variables.
 
-- Use double quotes ``"foo"`` rather than single quotes ``'foo'``.
-
-- **Comments**: should follow the `google code style
-  <http://google.github.io/styleguide/pyguide.html?showone=Comments#Comments>`_.
+- **Docstrings**: should follow the `google code style
+  <https://google.github.io/styleguide/pyguide.html#38-comments-and-docstrings>`_.
   This is so that we can generate documentation with `sphinx
   <http://sphinxcontrib-napoleon.readthedocs.org/en/latest/>`_. See the
   `examples
@@ -73,6 +76,8 @@ takes a while and is very resource intensive.
 
 - **Imports**:
 
+  - Imports should be sorted by ``isort`` as described above.
+
   - Prefer to import classes and functions rather than packages or modules.
 
     Example::
@@ -92,25 +97,84 @@ takes a while and is very resource intensive.
     This goes against the advice in the Google style guide, but it means that
     errors in the name are caught early (at import time).
 
-  - Multiple imports from the same package can be combined onto one line::
-
-      from synapse.types import GroupID, RoomID, UserID
-
-    An effort should be made to keep the individual imports in alphabetical
-    order.
-
-    If the list becomes long, wrap it with parentheses and split it over
-    multiple lines.
-
-  - As per `PEP-8 <https://www.python.org/dev/peps/pep-0008/#imports>`_,
-    imports should be grouped in the following order, with a blank line between
-    each group:
-
-    1. standard library imports
-    2. related third party imports
-    3. local application/library specific imports
-
-  - Imports within each group should be sorted alphabetically by module name.
-
   - Avoid wildcard imports (``from synapse.types import *``) and relative
     imports (``from .types import UserID``).
+
+Configuration file format
+-------------------------
+
+The `sample configuration file <./sample_config.yaml>`_ acts as a reference to
+Synapse's configuration options for server administrators. Remember that many
+readers will be unfamiliar with YAML and server administration in general, so
+that it is important that the file be as easy to understand as possible, which
+includes following a consistent format.
+
+Some guidelines follow:
+
+* Sections should be separated with a heading consisting of a single line
+  prefixed and suffixed with ``##``. There should be **two** blank lines
+  before the section header, and **one** after.
+
+* Each option should be listed in the file with the following format:
+
+  * A comment describing the setting. Each line of this comment should be
+    prefixed with a hash (``#``) and a space.
+
+    The comment should describe the default behaviour (ie, what happens if
+    the setting is omitted), as well as what the effect will be if the
+    setting is changed.
+
+    Often, the comment end with something like "uncomment the
+    following to \<do action>".
+
+  * A line consisting of only ``#``.
+
+  * A commented-out example setting, prefixed with only ``#``.
+
+    For boolean (on/off) options, convention is that this example should be
+    the *opposite* to the default (so the comment will end with "Uncomment
+    the following to enable [or disable] \<feature\>." For other options,
+    the example should give some non-default value which is likely to be
+    useful to the reader.
+
+* There should be a blank line between each option.
+
+* Where several settings are grouped into a single dict, *avoid* the
+  convention where the whole block is commented out, resulting in comment
+  lines starting ``# #``, as this is hard to read and confusing to
+  edit. Instead, leave the top-level config option uncommented, and follow
+  the conventions above for sub-options. Ensure that your code correctly
+  handles the top-level option being set to ``None`` (as it will be if no
+  sub-options are enabled).
+
+* Lines should be wrapped at 80 characters.
+
+Example::
+
+    ## Frobnication ##
+
+    # The frobnicator will ensure that all requests are fully frobnicated.
+    # To enable it, uncomment the following.
+    #
+    #frobnicator_enabled: true
+
+    # By default, the frobnicator will frobnicate with the default frobber.
+    # The following will make it use an alternative frobber.
+    #
+    #frobincator_frobber: special_frobber
+
+    # Settings for the frobber
+    #
+    frobber:
+       # frobbing speed. Defaults to 1.
+       #
+       #speed: 10
+
+       # frobbing distance. Defaults to 1000.
+       #
+       #distance: 100
+
+Note that the sample configuration is generated from the synapse code and is
+maintained by a script, ``scripts-dev/generate_sample_config``. Making sure
+that the output from this script matches the desired format is left as an
+exercise for the reader!

docs/log_contexts.rst

@@ -148,7 +148,7 @@ call any other functions.
         d = more_stuff()
         result = yield d            # also fine, of course
 
-        defer.returnValue(result)
+        return result
 
     def nonInlineCallbacksFun():
         logger.debug("just a wrapper really")

docs/sample_config.yaml

@@ -280,14 +280,20 @@ listeners:
 
 # Resource-constrained Homeserver Settings
 #
-# If limit_large_remote_room_joins is True, the room complexity will be
+# If limit_remote_rooms.enabled is True, the room complexity will be
 # checked before a user joins a new remote room. If it is above
-# limit_large_remote_room_complexity, it will disallow joining or
+# limit_remote_rooms.complexity, it will disallow joining or
 # instantly leave.
 #
+# limit_remote_rooms.complexity_error can be set to customise the text
+# displayed to the user when a room above the complexity threshold has
+# its join cancelled.
+#
 # Uncomment the below lines to enable:
-#limit_large_remote_room_joins: True
-#limit_large_remote_room_complexity: 1.0
+#limit_remote_rooms:
+#  enabled: True
+#  complexity: 1.0
+#  complexity_error: "This room is too complex."
 
 # Whether to require a user to be in the room to add an alias to it.
 # Defaults to 'true'.
@@ -559,6 +565,13 @@ log_config: "CONFDIR/SERVERNAME.log.config"
 
 
 
+## Media Store ##
+
+# Enable the media store service in the Synapse master. Uncomment the
+# following if you are using a separate media store worker.
+#
+#enable_media_repo: false
+
 # Directory where uploaded images and attachments are stored.
 #
 media_store_path: "DATADIR/media_store"
@@ -796,6 +809,16 @@ uploads_path: "DATADIR/uploads"
 #  period: 6w
 #  renew_at: 1w
 #  renew_email_subject: "Renew your %(app)s account"
+#  # Directory in which Synapse will try to find the HTML files to serve to the
+#  # user when trying to renew an account. Optional, defaults to
+#  # synapse/res/templates.
+#  template_dir: "res/templates"
+#  # HTML to be displayed to the user after they successfully renewed their
+#  # account. Optional.
+#  account_renewed_html_path: "account_renewed.html"
+#  # HTML to be displayed when the user tries to renew an account with an invalid
+#  # renewal token. Optional.
+#  invalid_token_html_path: "invalid_token.html"
 
 # Time that a user's session remains valid for, after they log in.
 #
@@ -936,10 +959,6 @@ uploads_path: "DATADIR/uploads"
 #
 # macaroon_secret_key: <PRIVATE STRING>
 
-# Used to enable access token expiration.
-#
-#expire_access_token: False
-
 # a secret which is used to calculate HMACs for form values, to stop
 # falsification of values. Must be specified for the User Consent
 # forms to work.
@@ -1441,3 +1460,19 @@ opentracing:
     #
     #homeserver_whitelist:
     #  - ".*"
+
+    # Jaeger can be configured to sample traces at different rates.
+    # All configuration options provided by Jaeger can be set here.
+    # Jaeger's configuration mostly related to trace sampling which
+    # is documented here:
+    # https://www.jaegertracing.io/docs/1.13/sampling/.
+    #
+    #jaeger_config:
+    #  sampler:
+    #    type: const
+    #    param: 1
+
+    #  Logging whether spans were started and reported
+    #
+    #  logging:
+    #    false

docs/workers.rst

@@ -206,6 +206,13 @@ Handles the media repository. It can handle all endpoints starting with::
 
     /_matrix/media/
 
+And the following regular expressions matching media-specific administration
+APIs::
+
+    ^/_synapse/admin/v1/purge_media_cache$
+    ^/_synapse/admin/v1/room/.*/media$
+    ^/_synapse/admin/v1/quarantine_media/.*$
+
 You should also set ``enable_media_repo: False`` in the shared configuration
 file to stop the main synapse running background jobs related to managing the
 media repository.

synapse/__init__.py

@@ -35,4 +35,4 @@ try:
 except ImportError:
     pass
 
-__version__ = "1.2.1"
+__version__ = "1.3.1"

synapse/api/auth.py

@@ -22,6 +22,7 @@ from netaddr import IPAddress
 
 from twisted.internet import defer
 
+import synapse.logging.opentracing as opentracing
 import synapse.types
 from synapse import event_auth
 from synapse.api.constants import EventTypes, JoinRules, Membership
@@ -128,7 +129,7 @@ class Auth(object):
             )
 
         self._check_joined_room(member, user_id, room_id)
-        defer.returnValue(member)
+        return member
 
     @defer.inlineCallbacks
     def check_user_was_in_room(self, room_id, user_id):
@@ -156,13 +157,13 @@ class Auth(object):
             if forgot:
                 raise AuthError(403, "User %s not in room %s" % (user_id, room_id))
 
-        defer.returnValue(member)
+        return member
 
     @defer.inlineCallbacks
     def check_host_in_room(self, room_id, host):
         with Measure(self.clock, "check_host_in_room"):
             latest_event_ids = yield self.store.is_host_joined(room_id, host)
-            defer.returnValue(latest_event_ids)
+            return latest_event_ids
 
     def _check_joined_room(self, member, user_id, room_id):
         if not member or member.membership != Membership.JOIN:
@@ -178,6 +179,7 @@ class Auth(object):
     def get_public_keys(self, invite_event):
         return event_auth.get_public_keys(invite_event)
 
+    @opentracing.trace
     @defer.inlineCallbacks
     def get_user_by_req(
         self, request, allow_guest=False, rights="access", allow_expired=False
@@ -209,6 +211,7 @@ class Auth(object):
             user_id, app_service = yield self._get_appservice_user_id(request)
             if user_id:
                 request.authenticated_entity = user_id
+                opentracing.set_tag("authenticated_entity", user_id)
 
                 if ip_addr and self.hs.config.track_appservice_user_ips:
                     yield self.store.insert_client_ip(
@@ -219,9 +222,7 @@ class Auth(object):
                         device_id="dummy-device",  # stubbed
                     )
 
-                defer.returnValue(
-                    synapse.types.create_requester(user_id, app_service=app_service)
-                )
+                return synapse.types.create_requester(user_id, app_service=app_service)
 
             user_info = yield self.get_user_by_access_token(access_token, rights)
             user = user_info["user"]
@@ -261,11 +262,10 @@ class Auth(object):
                 )
 
             request.authenticated_entity = user.to_string()
+            opentracing.set_tag("authenticated_entity", user.to_string())
 
-            defer.returnValue(
-                synapse.types.create_requester(
-                    user, token_id, is_guest, device_id, app_service=app_service
-                )
+            return synapse.types.create_requester(
+                user, token_id, is_guest, device_id, app_service=app_service
             )
         except KeyError:
             raise MissingClientTokenError()
@@ -276,25 +276,25 @@ class Auth(object):
             self.get_access_token_from_request(request)
         )
         if app_service is None:
-            defer.returnValue((None, None))
+            return (None, None)
 
         if app_service.ip_range_whitelist:
             ip_address = IPAddress(self.hs.get_ip_from_request(request))
             if ip_address not in app_service.ip_range_whitelist:
-                defer.returnValue((None, None))
+                return (None, None)
 
         if b"user_id" not in request.args:
-            defer.returnValue((app_service.sender, app_service))
+            return (app_service.sender, app_service)
 
         user_id = request.args[b"user_id"][0].decode("utf8")
         if app_service.sender == user_id:
-            defer.returnValue((app_service.sender, app_service))
+            return (app_service.sender, app_service)
 
         if not app_service.is_interested_in_user(user_id):
             raise AuthError(403, "Application service cannot masquerade as this user.")
         if not (yield self.store.get_user_by_id(user_id)):
             raise AuthError(403, "Application service has not registered this user")
-        defer.returnValue((user_id, app_service))
+        return (user_id, app_service)
 
     @defer.inlineCallbacks
     def get_user_by_access_token(self, token, rights="access"):
@@ -330,7 +330,7 @@ class Auth(object):
                         msg="Access token has expired", soft_logout=True
                     )
 
-                defer.returnValue(r)
+                return r
 
         # otherwise it needs to be a valid macaroon
         try:
@@ -378,7 +378,7 @@ class Auth(object):
                 }
             else:
                 raise RuntimeError("Unknown rights setting %s", rights)
-            defer.returnValue(ret)
+            return ret
         except (
             _InvalidMacaroonException,
             pymacaroons.exceptions.MacaroonException,
@@ -414,21 +414,16 @@ class Auth(object):
         try:
             user_id = self.get_user_id_from_macaroon(macaroon)
 
-            has_expiry = False
             guest = False
             for caveat in macaroon.caveats:
-                if caveat.caveat_id.startswith("time "):
-                    has_expiry = True
-                elif caveat.caveat_id == "guest = true":
+                if caveat.caveat_id == "guest = true":
                     guest = True
 
-            self.validate_macaroon(
-                macaroon, rights, self.hs.config.expire_access_token, user_id=user_id
-            )
+            self.validate_macaroon(macaroon, rights, user_id=user_id)
         except (pymacaroons.exceptions.MacaroonException, TypeError, ValueError):
             raise InvalidClientTokenError("Invalid macaroon passed.")
 
-        if not has_expiry and rights == "access":
+        if rights == "access":
             self.token_cache[token] = (user_id, guest)
 
         return user_id, guest
@@ -454,7 +449,7 @@ class Auth(object):
                 return caveat.caveat_id[len(user_prefix) :]
         raise InvalidClientTokenError("No user caveat in macaroon")
 
-    def validate_macaroon(self, macaroon, type_string, verify_expiry, user_id):
+    def validate_macaroon(self, macaroon, type_string, user_id):
         """
         validate that a Macaroon is understood by and was signed by this server.
 
@@ -462,7 +457,6 @@ class Auth(object):
             macaroon(pymacaroons.Macaroon): The macaroon to validate
             type_string(str): The kind of token required (e.g. "access",
                               "delete_pusher")
-            verify_expiry(bool): Whether to verify whether the macaroon has expired.
             user_id (str): The user_id required
         """
         v = pymacaroons.Verifier()
@@ -475,19 +469,7 @@ class Auth(object):
         v.satisfy_exact("type = " + type_string)
         v.satisfy_exact("user_id = %s" % user_id)
         v.satisfy_exact("guest = true")
-
-        # verify_expiry should really always be True, but there exist access
-        # tokens in the wild which expire when they should not, so we can't
-        # enforce expiry yet (so we have to allow any caveat starting with
-        # 'time < ' in access tokens).
-        #
-        # On the other hand, short-term login tokens (as used by CAS login, for
-        # example) have an expiry time which we do want to enforce.
-
-        if verify_expiry:
-            v.satisfy_general(self._verify_expiry)
-        else:
-            v.satisfy_general(lambda c: c.startswith("time < "))
+        v.satisfy_general(self._verify_expiry)
 
         # access_tokens include a nonce for uniqueness: any value is acceptable
         v.satisfy_general(lambda c: c.startswith("nonce = "))
@@ -506,7 +488,7 @@ class Auth(object):
     def _look_up_user_by_access_token(self, token):
         ret = yield self.store.get_user_by_access_token(token)
         if not ret:
-            defer.returnValue(None)
+            return None
 
         # we use ret.get() below because *lots* of unit tests stub out
         # get_user_by_access_token in a way where it only returns a couple of
@@ -518,7 +500,7 @@ class Auth(object):
             "device_id": ret.get("device_id"),
             "valid_until_ms": ret.get("valid_until_ms"),
         }
-        defer.returnValue(user_info)
+        return user_info
 
     def get_appservice_by_req(self, request):
         token = self.get_access_token_from_request(request)
@@ -543,7 +525,7 @@ class Auth(object):
     @defer.inlineCallbacks
     def compute_auth_events(self, event, current_state_ids, for_verification=False):
         if event.type == EventTypes.Create:
-            defer.returnValue([])
+            return []
 
         auth_ids = []
 
@@ -604,7 +586,7 @@ class Auth(object):
             if member_event.content["membership"] == Membership.JOIN:
                 auth_ids.append(member_event.event_id)
 
-        defer.returnValue(auth_ids)
+        return auth_ids
 
     @defer.inlineCallbacks
     def check_can_change_room_list(self, room_id, user):
@@ -618,7 +600,7 @@ class Auth(object):
 
         is_admin = yield self.is_server_admin(user)
         if is_admin:
-            defer.returnValue(True)
+            return True
 
         user_id = user.to_string()
         yield self.check_joined_room(room_id, user_id)
@@ -712,7 +694,7 @@ class Auth(object):
             #  * The user is a guest user, and has joined the room
             # else it will throw.
             member_event = yield self.check_user_was_in_room(room_id, user_id)
-            defer.returnValue((member_event.membership, member_event.event_id))
+            return (member_event.membership, member_event.event_id)
         except AuthError:
             visibility = yield self.state.get_current_state(
                 room_id, EventTypes.RoomHistoryVisibility, ""
@@ -721,7 +703,7 @@ class Auth(object):
                 visibility
                 and visibility.content["history_visibility"] == "world_readable"
             ):
-                defer.returnValue((Membership.JOIN, None))
+                return (Membership.JOIN, None)
                 return
             raise AuthError(
                 403, "Guest access not allowed", errcode=Codes.GUEST_ACCESS_FORBIDDEN

synapse/api/errors.py

@@ -61,6 +61,7 @@ class Codes(object):
     INCOMPATIBLE_ROOM_VERSION = "M_INCOMPATIBLE_ROOM_VERSION"
     WRONG_ROOM_KEYS_VERSION = "M_WRONG_ROOM_KEYS_VERSION"
     EXPIRED_ACCOUNT = "ORG_MATRIX_EXPIRED_ACCOUNT"
+    USER_DEACTIVATED = "M_USER_DEACTIVATED"
 
 
 class CodeMessageException(RuntimeError):
@@ -151,7 +152,7 @@ class UserDeactivatedError(SynapseError):
             msg (str): The human-readable error message
         """
         super(UserDeactivatedError, self).__init__(
-            code=http_client.FORBIDDEN, msg=msg, errcode=Codes.UNKNOWN
+            code=http_client.FORBIDDEN, msg=msg, errcode=Codes.USER_DEACTIVATED
         )
 
 

synapse/api/filtering.py

@@ -132,7 +132,7 @@ class Filtering(object):
     @defer.inlineCallbacks
     def get_user_filter(self, user_localpart, filter_id):
         result = yield self.store.get_user_filter(user_localpart, filter_id)
-        defer.returnValue(FilterCollection(result))
+        return FilterCollection(result)
 
     def add_user_filter(self, user_localpart, user_filter):
         self.check_valid_filter(user_filter)

synapse/app/_base.py

@@ -15,7 +15,9 @@
 
 import gc
 import logging
+import os
 import signal
+import socket
 import sys
 import traceback
 
@@ -242,9 +244,15 @@ def start(hs, listeners=None):
         if hasattr(signal, "SIGHUP"):
 
             def handle_sighup(*args, **kwargs):
+                # Tell systemd our state, if we're using it. This will silently fail if
+                # we're not using systemd.
+                sdnotify(b"RELOADING=1")
+
                 for i in _sighup_callbacks:
                     i(hs)
 
+                sdnotify(b"READY=1")
+
             signal.signal(signal.SIGHUP, handle_sighup)
 
             register_sighup(refresh_certificate)
@@ -260,6 +268,7 @@ def start(hs, listeners=None):
         hs.get_datastore().start_profiling()
 
         setup_sentry(hs)
+        setup_sdnotify(hs)
     except Exception:
         traceback.print_exc(file=sys.stderr)
         reactor = hs.get_reactor()
@@ -292,6 +301,21 @@ def setup_sentry(hs):
         scope.set_tag("worker_name", name)
 
 
+def setup_sdnotify(hs):
+    """Adds process state hooks to tell systemd what we are up to.
+    """
+
+    # Tell systemd our state, if we're using it. This will silently fail if
+    # we're not using systemd.
+    hs.get_reactor().addSystemEventTrigger(
+        "after", "startup", sdnotify, b"READY=1\nMAINPID=%i" % (os.getpid(),)
+    )
+
+    hs.get_reactor().addSystemEventTrigger(
+        "before", "shutdown", sdnotify, b"STOPPING=1"
+    )
+
+
 def install_dns_limiter(reactor, max_dns_requests_in_flight=100):
     """Replaces the resolver with one that limits the number of in flight DNS
     requests.
@@ -385,3 +409,35 @@ class _DeferredResolutionReceiver(object):
     def resolutionComplete(self):
         self._deferred.callback(())
         self._receiver.resolutionComplete()
+
+
+sdnotify_sockaddr = os.getenv("NOTIFY_SOCKET")
+
+
+def sdnotify(state):
+    """
+    Send a notification to systemd, if the NOTIFY_SOCKET env var is set.
+
+    This function is based on the sdnotify python package, but since it's only a few
+    lines of code, it's easier to duplicate it here than to add a dependency on a
+    package which many OSes don't include as a matter of principle.
+
+    Args:
+        state (bytes): notification to send
+    """
+    if not isinstance(state, bytes):
+        raise TypeError("sdnotify should be called with a bytes")
+    if not sdnotify_sockaddr:
+        return
+    addr = sdnotify_sockaddr
+    if addr[0] == "@":
+        addr = "\0" + addr[1:]
+
+    try:
+        with socket.socket(socket.AF_UNIX, socket.SOCK_DGRAM) as sock:
+            sock.connect(addr)
+            sock.sendall(state)
+    except Exception as e:
+        # this is a bit surprising, since we don't expect to have a NOTIFY_SOCKET
+        # unless systemd is expecting us to notify it.
+        logger.warning("Unable to send notification to systemd: %s", e)

synapse/app/appservice.py

@@ -168,7 +168,9 @@ def start(config_options):
     )
 
     ps.setup()
-    reactor.callWhenRunning(_base.start, ps, config.worker_listeners)
+    reactor.addSystemEventTrigger(
+        "before", "startup", _base.start, ps, config.worker_listeners
+    )
 
     _base.start_worker_reactor("synapse-appservice", config)
 

synapse/app/client_reader.py

@@ -194,7 +194,9 @@ def start(config_options):
     )
 
     ss.setup()
-    reactor.callWhenRunning(_base.start, ss, config.worker_listeners)
+    reactor.addSystemEventTrigger(
+        "before", "startup", _base.start, ss, config.worker_listeners
+    )
 
     _base.start_worker_reactor("synapse-client-reader", config)
 

synapse/app/event_creator.py

@@ -193,7 +193,9 @@ def start(config_options):
     )
 
     ss.setup()
-    reactor.callWhenRunning(_base.start, ss, config.worker_listeners)
+    reactor.addSystemEventTrigger(
+        "before", "startup", _base.start, ss, config.worker_listeners
+    )
 
     _base.start_worker_reactor("synapse-event-creator", config)
 

synapse/app/federation_reader.py

@@ -175,7 +175,9 @@ def start(config_options):
     )
 
     ss.setup()
-    reactor.callWhenRunning(_base.start, ss, config.worker_listeners)
+    reactor.addSystemEventTrigger(
+        "before", "startup", _base.start, ss, config.worker_listeners
+    )
 
     _base.start_worker_reactor("synapse-federation-reader", config)
 

synapse/app/federation_sender.py

@@ -198,7 +198,9 @@ def start(config_options):
     )
 
     ss.setup()
-    reactor.callWhenRunning(_base.start, ss, config.worker_listeners)
+    reactor.addSystemEventTrigger(
+        "before", "startup", _base.start, ss, config.worker_listeners
+    )
 
     _base.start_worker_reactor("synapse-federation-sender", config)
 

synapse/app/frontend_proxy.py

@@ -70,12 +70,12 @@ class PresenceStatusStubServlet(RestServlet):
         except HttpResponseException as e:
             raise e.to_synapse_error()
 
-        defer.returnValue((200, result))
+        return (200, result)
 
     @defer.inlineCallbacks
     def on_PUT(self, request, user_id):
         yield self.auth.get_user_by_req(request)
-        defer.returnValue((200, {}))
+        return (200, {})
 
 
 class KeyUploadServlet(RestServlet):
@@ -126,11 +126,11 @@ class KeyUploadServlet(RestServlet):
                 self.main_uri + request.uri.decode("ascii"), body, headers=headers
             )
 
-            defer.returnValue((200, result))
+            return (200, result)
         else:
             # Just interested in counts.
             result = yield self.store.count_e2e_one_time_keys(user_id, device_id)
-            defer.returnValue((200, {"one_time_key_counts": result}))
+            return (200, {"one_time_key_counts": result})
 
 
 class FrontendProxySlavedStore(
@@ -247,7 +247,9 @@ def start(config_options):
     )
 
     ss.setup()
-    reactor.callWhenRunning(_base.start, ss, config.worker_listeners)
+    reactor.addSystemEventTrigger(
+        "before", "startup", _base.start, ss, config.worker_listeners
+    )
 
     _base.start_worker_reactor("synapse-frontend-proxy", config)
 

synapse/app/homeserver.py

@@ -406,7 +406,7 @@ def setup(config_options):
         if provision:
             yield acme.provision_certificate()
 
-        defer.returnValue(provision)
+        return provision
 
     @defer.inlineCallbacks
     def reprovision_acme():

synapse/app/media_repository.py

@@ -26,6 +26,7 @@ from synapse.app import _base
 from synapse.config._base import ConfigError
 from synapse.config.homeserver import HomeServerConfig
 from synapse.config.logger import setup_logging
+from synapse.http.server import JsonResource
 from synapse.http.site import SynapseSite
 from synapse.logging.context import LoggingContext
 from synapse.metrics import METRICS_PREFIX, MetricsResource, RegistryProxy
@@ -35,6 +36,7 @@ from synapse.replication.slave.storage.client_ips import SlavedClientIpStore
 from synapse.replication.slave.storage.registration import SlavedRegistrationStore
 from synapse.replication.slave.storage.transactions import SlavedTransactionStore
 from synapse.replication.tcp.client import ReplicationClientHandler
+from synapse.rest.admin import register_servlets_for_media_repo
 from synapse.rest.media.v0.content_repository import ContentRepoResource
 from synapse.server import HomeServer
 from synapse.storage.engines import create_engine
@@ -71,6 +73,12 @@ class MediaRepositoryServer(HomeServer):
                     resources[METRICS_PREFIX] = MetricsResource(RegistryProxy)
                 elif name == "media":
                     media_repo = self.get_media_repository_resource()
+
+                    # We need to serve the admin servlets for media on the
+                    # worker.
+                    admin_resource = JsonResource(self, canonical_json=False)
+                    register_servlets_for_media_repo(self, admin_resource)
+
                     resources.update(
                         {
                             MEDIA_PREFIX: media_repo,
@@ -78,6 +86,7 @@ class MediaRepositoryServer(HomeServer):
                             CONTENT_REPO_PREFIX: ContentRepoResource(
                                 self, self.config.uploads_path
                             ),
+                            "/_synapse/admin": admin_resource,
                         }
                     )
 
@@ -161,7 +170,9 @@ def start(config_options):
     )
 
     ss.setup()
-    reactor.callWhenRunning(_base.start, ss, config.worker_listeners)
+    reactor.addSystemEventTrigger(
+        "before", "startup", _base.start, ss, config.worker_listeners
+    )
 
     _base.start_worker_reactor("synapse-media-repository", config)
 

synapse/app/pusher.py

@@ -216,7 +216,7 @@ def start(config_options):
         _base.start(ps, config.worker_listeners)
         ps.get_pusherpool().start()
 
-    reactor.callWhenRunning(start)
+    reactor.addSystemEventTrigger("before", "startup", start)
 
     _base.start_worker_reactor("synapse-pusher", config)
 

synapse/app/synchrotron.py

@@ -451,7 +451,9 @@ def start(config_options):
     )
 
     ss.setup()
-    reactor.callWhenRunning(_base.start, ss, config.worker_listeners)
+    reactor.addSystemEventTrigger(
+        "before", "startup", _base.start, ss, config.worker_listeners
+    )
 
     _base.start_worker_reactor("synapse-synchrotron", config)
 

synapse/app/user_dir.py

@@ -224,7 +224,9 @@ def start(config_options):
     )
 
     ss.setup()
-    reactor.callWhenRunning(_base.start, ss, config.worker_listeners)
+    reactor.addSystemEventTrigger(
+        "before", "startup", _base.start, ss, config.worker_listeners
+    )
 
     _base.start_worker_reactor("synapse-user-dir", config)
 

synapse/appservice/__init__.py

@@ -175,21 +175,21 @@ class ApplicationService(object):
     @defer.inlineCallbacks
     def _matches_user(self, event, store):
         if not event:
-            defer.returnValue(False)
+            return False
 
         if self.is_interested_in_user(event.sender):
-            defer.returnValue(True)
+            return True
         # also check m.room.member state key
         if event.type == EventTypes.Member and self.is_interested_in_user(
             event.state_key
         ):
-            defer.returnValue(True)
+            return True
 
         if not store:
-            defer.returnValue(False)
+            return False
 
         does_match = yield self._matches_user_in_member_list(event.room_id, store)
-        defer.returnValue(does_match)
+        return does_match
 
     @cachedInlineCallbacks(num_args=1, cache_context=True)
     def _matches_user_in_member_list(self, room_id, store, cache_context):
@@ -200,8 +200,8 @@ class ApplicationService(object):
         # check joined member events
         for user_id in member_list:
             if self.is_interested_in_user(user_id):
-                defer.returnValue(True)
-        defer.returnValue(False)
+                return True
+        return False
 
     def _matches_room_id(self, event):
         if hasattr(event, "room_id"):
@@ -211,13 +211,13 @@ class ApplicationService(object):
     @defer.inlineCallbacks
     def _matches_aliases(self, event, store):
         if not store or not event:
-            defer.returnValue(False)
+            return False
 
         alias_list = yield store.get_aliases_for_room(event.room_id)
         for alias in alias_list:
             if self.is_interested_in_alias(alias):
-                defer.returnValue(True)
-        defer.returnValue(False)
+                return True
+        return False
 
     @defer.inlineCallbacks
     def is_interested(self, event, store=None):
@@ -231,15 +231,15 @@ class ApplicationService(object):
         """
         # Do cheap checks first
         if self._matches_room_id(event):
-            defer.returnValue(True)
+            return True
 
         if (yield self._matches_aliases(event, store)):
-            defer.returnValue(True)
+            return True
 
         if (yield self._matches_user(event, store)):
-            defer.returnValue(True)
+            return True
 
-        defer.returnValue(False)
+        return False
 
     def is_interested_in_user(self, user_id):
         return (

synapse/appservice/api.py

@@ -97,40 +97,40 @@ class ApplicationServiceApi(SimpleHttpClient):
     @defer.inlineCallbacks
     def query_user(self, service, user_id):
         if service.url is None:
-            defer.returnValue(False)
+            return False
         uri = service.url + ("/users/%s" % urllib.parse.quote(user_id))
         response = None
         try:
             response = yield self.get_json(uri, {"access_token": service.hs_token})
             if response is not None:  # just an empty json object
-                defer.returnValue(True)
+                return True
         except CodeMessageException as e:
             if e.code == 404:
-                defer.returnValue(False)
+                return False
                 return
             logger.warning("query_user to %s received %s", uri, e.code)
         except Exception as ex:
             logger.warning("query_user to %s threw exception %s", uri, ex)
-        defer.returnValue(False)
+        return False
 
     @defer.inlineCallbacks
     def query_alias(self, service, alias):
         if service.url is None:
-            defer.returnValue(False)
+            return False
         uri = service.url + ("/rooms/%s" % urllib.parse.quote(alias))
         response = None
         try:
             response = yield self.get_json(uri, {"access_token": service.hs_token})
             if response is not None:  # just an empty json object
-                defer.returnValue(True)
+                return True
         except CodeMessageException as e:
             logger.warning("query_alias to %s received %s", uri, e.code)
             if e.code == 404:
-                defer.returnValue(False)
+                return False
                 return
         except Exception as ex:
             logger.warning("query_alias to %s threw exception %s", uri, ex)
-        defer.returnValue(False)
+        return False
 
     @defer.inlineCallbacks
     def query_3pe(self, service, kind, protocol, fields):
@@ -141,7 +141,7 @@ class ApplicationServiceApi(SimpleHttpClient):
         else:
             raise ValueError("Unrecognised 'kind' argument %r to query_3pe()", kind)
         if service.url is None:
-            defer.returnValue([])
+            return []
 
         uri = "%s%s/thirdparty/%s/%s" % (
             service.url,
@@ -155,7 +155,7 @@ class ApplicationServiceApi(SimpleHttpClient):
                 logger.warning(
                     "query_3pe to %s returned an invalid response %r", uri, response
                 )
-                defer.returnValue([])
+                return []
 
             ret = []
             for r in response:
@@ -166,14 +166,14 @@ class ApplicationServiceApi(SimpleHttpClient):
                         "query_3pe to %s returned an invalid result %r", uri, r
                     )
 
-            defer.returnValue(ret)
+            return ret
         except Exception as ex:
             logger.warning("query_3pe to %s threw exception %s", uri, ex)
-            defer.returnValue([])
+            return []
 
     def get_3pe_protocol(self, service, protocol):
         if service.url is None:
-            defer.returnValue({})
+            return {}
 
         @defer.inlineCallbacks
         def _get():
@@ -189,7 +189,7 @@ class ApplicationServiceApi(SimpleHttpClient):
                     logger.warning(
                         "query_3pe_protocol to %s did not return a" " valid result", uri
                     )
-                    defer.returnValue(None)
+                    return None
 
                 for instance in info.get("instances", []):
                     network_id = instance.get("network_id", None)
@@ -198,10 +198,10 @@ class ApplicationServiceApi(SimpleHttpClient):
                             service.id, network_id
                         ).to_string()
 
-                defer.returnValue(info)
+                return info
             except Exception as ex:
                 logger.warning("query_3pe_protocol to %s threw exception %s", uri, ex)
-                defer.returnValue(None)
+                return None
 
         key = (service.id, protocol)
         return self.protocol_meta_cache.wrap(key, _get)
@@ -209,7 +209,7 @@ class ApplicationServiceApi(SimpleHttpClient):
     @defer.inlineCallbacks
     def push_bulk(self, service, events, txn_id=None):
         if service.url is None:
-            defer.returnValue(True)
+            return True
 
         events = self._serialize(events)
 
@@ -229,14 +229,14 @@ class ApplicationServiceApi(SimpleHttpClient):
             )
             sent_transactions_counter.labels(service.id).inc()
             sent_events_counter.labels(service.id).inc(len(events))
-            defer.returnValue(True)
+            return True
             return
         except CodeMessageException as e:
             logger.warning("push_bulk to %s received %s", uri, e.code)
         except Exception as ex:
             logger.warning("push_bulk to %s threw exception %s", uri, ex)
         failed_transactions_counter.labels(service.id).inc()
-        defer.returnValue(False)
+        return False
 
     def _serialize(self, events):
         time_now = self.clock.time_msec()

synapse/appservice/scheduler.py

@@ -193,7 +193,7 @@ class _TransactionController(object):
     @defer.inlineCallbacks
     def _is_service_up(self, service):
         state = yield self.store.get_appservice_state(service)
-        defer.returnValue(state == ApplicationServiceState.UP or state is None)
+        return state == ApplicationServiceState.UP or state is None
 
 
 class _Recoverer(object):
@@ -208,7 +208,7 @@ class _Recoverer(object):
                 r.service.id,
             )
             r.recover()
-        defer.returnValue(recoverers)
+        return recoverers
 
     def __init__(self, clock, store, as_api, service, callback):
         self.clock = clock

synapse/config/emailconfig.py

@@ -132,21 +132,21 @@ class EmailConfig(Config):
             self.email_password_reset_template_text = email_config.get(
                 "password_reset_template_text", "password_reset.txt"
             )
-            self.email_password_reset_failure_template = email_config.get(
-                "password_reset_failure_template", "password_reset_failure.html"
+            self.email_password_reset_template_failure_html = email_config.get(
+                "password_reset_template_failure_html", "password_reset_failure.html"
             )
             # This template does not support any replaceable variables, so we will
             # read it from the disk once during setup
-            email_password_reset_success_template = email_config.get(
-                "password_reset_success_template", "password_reset_success.html"
+            email_password_reset_template_success_html = email_config.get(
+                "password_reset_template_success_html", "password_reset_success.html"
             )
 
             # Check templates exist
             for f in [
                 self.email_password_reset_template_html,
                 self.email_password_reset_template_text,
-                self.email_password_reset_failure_template,
-                email_password_reset_success_template,
+                self.email_password_reset_template_failure_html,
+                email_password_reset_template_success_html,
             ]:
                 p = os.path.join(self.email_template_dir, f)
                 if not os.path.isfile(p):
@@ -154,9 +154,9 @@ class EmailConfig(Config):
 
             # Retrieve content of web templates
             filepath = os.path.join(
-                self.email_template_dir, email_password_reset_success_template
+                self.email_template_dir, email_password_reset_template_success_html
             )
-            self.email_password_reset_success_html_content = self.read_file(
+            self.email_password_reset_template_success_html_content = self.read_file(
                 filepath, "email.password_reset_template_success_html"
             )
 

synapse/config/key.py

@@ -116,8 +116,6 @@ class KeyConfig(Config):
             seed = bytes(self.signing_key[0])
             self.macaroon_secret_key = hashlib.sha256(seed).digest()
 
-        self.expire_access_token = config.get("expire_access_token", False)
-
         # a secret which is used to calculate HMACs for form values, to stop
         # falsification of values
         self.form_secret = config.get("form_secret", None)
@@ -144,10 +142,6 @@ class KeyConfig(Config):
         #
         %(macaroon_secret_key)s
 
-        # Used to enable access token expiration.
-        #
-        #expire_access_token: False
-
         # a secret which is used to calculate HMACs for form values, to stop
         # falsification of values. Must be specified for the User Consent
         # forms to work.

synapse/config/logger.py

@@ -12,6 +12,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+
 import logging
 import logging.config
 import os
@@ -75,10 +76,8 @@ root:
 
 class LoggingConfig(Config):
     def read_config(self, config, **kwargs):
-        self.verbosity = config.get("verbose", 0)
-        self.no_redirect_stdio = config.get("no_redirect_stdio", False)
         self.log_config = self.abspath(config.get("log_config"))
-        self.log_file = self.abspath(config.get("log_file"))
+        self.no_redirect_stdio = config.get("no_redirect_stdio", False)
 
     def generate_config_section(self, config_dir_path, server_name, **kwargs):
         log_config = os.path.join(config_dir_path, server_name + ".log.config")
@@ -94,38 +93,12 @@ class LoggingConfig(Config):
         )
 
     def read_arguments(self, args):
-        if args.verbose is not None:
-            self.verbosity = args.verbose
         if args.no_redirect_stdio is not None:
             self.no_redirect_stdio = args.no_redirect_stdio
-        if args.log_config is not None:
-            self.log_config = args.log_config
-        if args.log_file is not None:
-            self.log_file = args.log_file
 
     @staticmethod
     def add_arguments(parser):
         logging_group = parser.add_argument_group("logging")
-        logging_group.add_argument(
-            "-v",
-            "--verbose",
-            dest="verbose",
-            action="count",
-            help="The verbosity level. Specify multiple times to increase "
-            "verbosity. (Ignored if --log-config is specified.)",
-        )
-        logging_group.add_argument(
-            "-f",
-            "--log-file",
-            dest="log_file",
-            help="File to log to. (Ignored if --log-config is specified.)",
-        )
-        logging_group.add_argument(
-            "--log-config",
-            dest="log_config",
-            default=None,
-            help="Python logging config file",
-        )
         logging_group.add_argument(
             "-n",
             "--no-redirect-stdio",
@@ -153,58 +126,29 @@ def setup_logging(config, use_worker_options=False):
         config (LoggingConfig | synapse.config.workers.WorkerConfig):
             configuration data
 
-        use_worker_options (bool): True to use 'worker_log_config' and
-            'worker_log_file' options instead of 'log_config' and 'log_file'.
+        use_worker_options (bool): True to use the 'worker_log_config' option
+            instead of 'log_config'.
 
         register_sighup (func | None): Function to call to register a
             sighup handler.
     """
     log_config = config.worker_log_config if use_worker_options else config.log_config
-    log_file = config.worker_log_file if use_worker_options else config.log_file
-
-    log_format = (
-        "%(asctime)s - %(name)s - %(lineno)d - %(levelname)s - %(request)s"
-        " - %(message)s"
-    )
 
     if log_config is None:
-        # We don't have a logfile, so fall back to the 'verbosity' param from
-        # the config or cmdline. (Note that we generate a log config for new
-        # installs, so this will be an unusual case)
-        level = logging.INFO
-        level_for_storage = logging.INFO
-        if config.verbosity:
-            level = logging.DEBUG
-            if config.verbosity > 1:
-                level_for_storage = logging.DEBUG
+        log_format = (
+            "%(asctime)s - %(name)s - %(lineno)d - %(levelname)s - %(request)s"
+            " - %(message)s"
+        )
 
         logger = logging.getLogger("")
-        logger.setLevel(level)
-
-        logging.getLogger("synapse.storage.SQL").setLevel(level_for_storage)
+        logger.setLevel(logging.INFO)
+        logging.getLogger("synapse.storage.SQL").setLevel(logging.INFO)
 
         formatter = logging.Formatter(log_format)
-        if log_file:
-            # TODO: Customisable file size / backup count
-            handler = logging.handlers.RotatingFileHandler(
-                log_file, maxBytes=(1000 * 1000 * 100), backupCount=3, encoding="utf8"
-            )
-
-            def sighup(signum, stack):
-                logger.info("Closing log file due to SIGHUP")
-                handler.doRollover()
-                logger.info("Opened new log file due to SIGHUP")
-
-        else:
-            handler = logging.StreamHandler()
-
-            def sighup(*args):
-                pass
 
+        handler = logging.StreamHandler()
         handler.setFormatter(formatter)
-
         handler.addFilter(LoggingContextFilter(request=""))
-
         logger.addHandler(handler)
     else:
 
@@ -218,8 +162,7 @@ def setup_logging(config, use_worker_options=False):
             logging.info("Reloaded log config from %s due to SIGHUP", log_config)
 
         load_log_config()
-
-    appbase.register_sighup(sighup)
+        appbase.register_sighup(sighup)
 
     # make sure that the first thing we log is a thing we can grep backwards
     # for

synapse/config/registration.py

@@ -13,8 +13,11 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import os
 from distutils.util import strtobool
 
+import pkg_resources
+
 from synapse.config._base import Config, ConfigError
 from synapse.types import RoomAlias
 from synapse.util.stringutils import random_string_with_symbols
@@ -41,8 +44,36 @@ class AccountValidityConfig(Config):
 
             self.startup_job_max_delta = self.period * 10.0 / 100.0
 
-        if self.renew_by_email_enabled and "public_baseurl" not in synapse_config:
-            raise ConfigError("Can't send renewal emails without 'public_baseurl'")
+        if self.renew_by_email_enabled:
+            if "public_baseurl" not in synapse_config:
+                raise ConfigError("Can't send renewal emails without 'public_baseurl'")
+
+        template_dir = config.get("template_dir")
+
+        if not template_dir:
+            template_dir = pkg_resources.resource_filename("synapse", "res/templates")
+
+        if "account_renewed_html_path" in config:
+            file_path = os.path.join(template_dir, config["account_renewed_html_path"])
+
+            self.account_renewed_html_content = self.read_file(
+                file_path, "account_validity.account_renewed_html_path"
+            )
+        else:
+            self.account_renewed_html_content = (
+                "<html><body>Your account has been successfully renewed.</body><html>"
+            )
+
+        if "invalid_token_html_path" in config:
+            file_path = os.path.join(template_dir, config["invalid_token_html_path"])
+
+            self.invalid_token_html_content = self.read_file(
+                file_path, "account_validity.invalid_token_html_path"
+            )
+        else:
+            self.invalid_token_html_content = (
+                "<html><body>Invalid renewal token.</body><html>"
+            )
 
 
 class RegistrationConfig(Config):
@@ -145,6 +176,16 @@ class RegistrationConfig(Config):
         #  period: 6w
         #  renew_at: 1w
         #  renew_email_subject: "Renew your %%(app)s account"
+        #  # Directory in which Synapse will try to find the HTML files to serve to the
+        #  # user when trying to renew an account. Optional, defaults to
+        #  # synapse/res/templates.
+        #  template_dir: "res/templates"
+        #  # HTML to be displayed to the user after they successfully renewed their
+        #  # account. Optional.
+        #  account_renewed_html_path: "account_renewed.html"
+        #  # HTML to be displayed when the user tries to renew an account with an invalid
+        #  # renewal token. Optional.
+        #  invalid_token_html_path: "invalid_token.html"
 
         # Time that a user's session remains valid for, after they log in.
         #

synapse/config/repository.py

@@ -12,6 +12,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+
 import os
 from collections import namedtuple
 
@@ -87,22 +88,25 @@ def parse_thumbnail_requirements(thumbnail_sizes):
 
 class ContentRepositoryConfig(Config):
     def read_config(self, config, **kwargs):
-        self.enable_media_repo = config.get("enable_media_repo", True)
+
+        # Only enable the media repo if either the media repo is enabled or the
+        # current worker app is the media repo.
+        if (
+            self.enable_media_repo is False
+            and config.get("worker_app") != "synapse.app.media_repository"
+        ):
+            self.can_load_media_repo = False
+            return
+        else:
+            self.can_load_media_repo = True
 
         self.max_upload_size = self.parse_size(config.get("max_upload_size", "10M"))
         self.max_image_pixels = self.parse_size(config.get("max_image_pixels", "32M"))
         self.max_spider_size = self.parse_size(config.get("max_spider_size", "10M"))
 
-        if self.enable_media_repo:
-            self.media_store_path = self.ensure_directory(
-                config.get("media_store_path", "media_store")
-            )
-            self.uploads_path = self.ensure_directory(
-                config.get("uploads_path", "uploads")
-            )
-        else:
-            self.media_store_path = None
-            self.uploads_path = None
+        self.media_store_path = self.ensure_directory(
+            config.get("media_store_path", "media_store")
+        )
 
         backup_media_store_path = config.get("backup_media_store_path")
 
@@ -159,6 +163,7 @@ class ContentRepositoryConfig(Config):
                 (provider_class, parsed_config, wrapper_config)
             )
 
+        self.uploads_path = self.ensure_directory(config.get("uploads_path", "uploads"))
         self.dynamic_thumbnails = config.get("dynamic_thumbnails", False)
         self.thumbnail_requirements = parse_thumbnail_requirements(
             config.get("thumbnail_sizes", DEFAULT_THUMBNAIL_SIZES)
@@ -210,6 +215,13 @@ class ContentRepositoryConfig(Config):
 
         return (
             r"""
+        ## Media Store ##
+
+        # Enable the media store service in the Synapse master. Uncomment the
+        # following if you are using a separate media store worker.
+        #
+        #enable_media_repo: false
+
         # Directory where uploaded images and attachments are stored.
         #
         media_store_path: "%(media_store)s"

synapse/config/server.py

@@ -18,6 +18,7 @@
 import logging
 import os.path
 
+import attr
 from netaddr import IPSet
 
 from synapse.api.room_versions import KNOWN_ROOM_VERSIONS
@@ -38,6 +39,12 @@ DEFAULT_BIND_ADDRESSES = ["::", "0.0.0.0"]
 
 DEFAULT_ROOM_VERSION = "4"
 
+ROOM_COMPLEXITY_TOO_GREAT = (
+    "Your homeserver is unable to join rooms this large or complex. "
+    "Please speak to your server administrator, or upgrade your instance "
+    "to join this room."
+)
+
 
 class ServerConfig(Config):
     def read_config(self, config, **kwargs):
@@ -247,10 +254,21 @@ class ServerConfig(Config):
 
         self.gc_thresholds = read_gc_thresholds(config.get("gc_thresholds", None))
 
-        # Resource-constrained Homeserver Configuration
-        self.limit_large_room_joins = config.get("limit_large_remote_room_joins", False)
-        self.limit_large_room_complexity = config.get(
-            "limit_large_remote_room_complexity", 1.0
+        @attr.s
+        class LimitRemoteRoomsConfig(object):
+            enabled = attr.ib(
+                validator=attr.validators.instance_of(bool), default=False
+            )
+            complexity = attr.ib(
+                validator=attr.validators.instance_of((int, float)), default=1.0
+            )
+            complexity_error = attr.ib(
+                validator=attr.validators.instance_of(str),
+                default=ROOM_COMPLEXITY_TOO_GREAT,
+            )
+
+        self.limit_remote_rooms = LimitRemoteRoomsConfig(
+            **config.get("limit_remote_rooms", {})
         )
 
         bind_port = config.get("bind_port")
@@ -625,14 +643,20 @@ class ServerConfig(Config):
 
         # Resource-constrained Homeserver Settings
         #
-        # If limit_large_remote_room_joins is True, the room complexity will be
+        # If limit_remote_rooms.enabled is True, the room complexity will be
         # checked before a user joins a new remote room. If it is above
-        # limit_large_remote_room_complexity, it will disallow joining or
+        # limit_remote_rooms.complexity, it will disallow joining or
         # instantly leave.
         #
+        # limit_remote_rooms.complexity_error can be set to customise the text
+        # displayed to the user when a room above the complexity threshold has
+        # its join cancelled.
+        #
         # Uncomment the below lines to enable:
-        #limit_large_remote_room_joins: True
-        #limit_large_remote_room_complexity: 1.0
+        #limit_remote_rooms:
+        #  enabled: True
+        #  complexity: 1.0
+        #  complexity_error: "This room is too complex."
 
         # Whether to require a user to be in the room to add an alias to it.
         # Defaults to 'true'.

synapse/config/tracer.py

@@ -23,6 +23,12 @@ class TracerConfig(Config):
             opentracing_config = {}
 
         self.opentracer_enabled = opentracing_config.get("enabled", False)
+
+        self.jaeger_config = opentracing_config.get(
+            "jaeger_config",
+            {"sampler": {"type": "const", "param": 1}, "logging": False},
+        )
+
         if not self.opentracer_enabled:
             return
 
@@ -56,4 +62,20 @@ class TracerConfig(Config):
             #
             #homeserver_whitelist:
             #  - ".*"
+
+            # Jaeger can be configured to sample traces at different rates.
+            # All configuration options provided by Jaeger can be set here.
+            # Jaeger's configuration mostly related to trace sampling which
+            # is documented here:
+            # https://www.jaegertracing.io/docs/1.13/sampling/.
+            #
+            #jaeger_config:
+            #  sampler:
+            #    type: const
+            #    param: 1
+
+            #  Logging whether spans were started and reported
+            #
+            #  logging:
+            #    false
         """

synapse/config/workers.py

@@ -31,7 +31,6 @@ class WorkerConfig(Config):
         self.worker_listeners = config.get("worker_listeners", [])
         self.worker_daemonize = config.get("worker_daemonize")
         self.worker_pid_file = config.get("worker_pid_file")
-        self.worker_log_file = config.get("worker_log_file")
         self.worker_log_config = config.get("worker_log_config")
 
         # The host used to connect to the main synapse
@@ -78,9 +77,5 @@ class WorkerConfig(Config):
 
         if args.daemonize is not None:
             self.worker_daemonize = args.daemonize
-        if args.log_config is not None:
-            self.worker_log_config = args.log_config
-        if args.log_file is not None:
-            self.worker_log_file = args.log_file
         if args.manhole is not None:
             self.worker_manhole = args.worker_manhole

synapse/crypto/context_factory.py

@@ -31,6 +31,7 @@ from twisted.internet.ssl import (
     platformTrust,
 )
 from twisted.python.failure import Failure
+from twisted.web.iweb import IPolicyForHTTPS
 
 logger = logging.getLogger(__name__)
 
@@ -74,6 +75,7 @@ class ServerContextFactory(ContextFactory):
         return self._context
 
 
+@implementer(IPolicyForHTTPS)
 class ClientTLSOptionsFactory(object):
     """Factory for Twisted SSLClientConnectionCreators that are used to make connections
     to remote servers for federation.
@@ -146,6 +148,12 @@ class ClientTLSOptionsFactory(object):
             f = Failure()
             tls_protocol.failVerification(f)
 
+    def creatorForNetloc(self, hostname, port):
+        """Implements the IPolicyForHTTPS interace so that this can be passed
+        directly to agents.
+        """
+        return self.get_options(hostname)
+
 
 @implementer(IOpenSSLClientConnectionCreator)
 class SSLClientConnectionCreator(object):

synapse/crypto/keyring.py

@@ -238,27 +238,9 @@ class Keyring(object):
         """
 
         try:
-            # create a deferred for each server we're going to look up the keys
-            # for; we'll resolve them once we have completed our lookups.
-            # These will be passed into wait_for_previous_lookups to block
-            # any other lookups until we have finished.
-            # The deferreds are called with no logcontext.
-            server_to_deferred = {
-                rq.server_name: defer.Deferred() for rq in verify_requests
-            }
+            ctx = LoggingContext.current_context()
 
-            # We want to wait for any previous lookups to complete before
-            # proceeding.
-            yield self.wait_for_previous_lookups(server_to_deferred)
-
-            # Actually start fetching keys.
-            self._get_server_verify_keys(verify_requests)
-
-            # When we've finished fetching all the keys for a given server_name,
-            # resolve the deferred passed to `wait_for_previous_lookups` so that
-            # any lookups waiting will proceed.
-            #
-            # map from server name to a set of request ids
+            # map from server name to a set of outstanding request ids
             server_to_request_ids = {}
 
             for verify_request in verify_requests:
@@ -266,40 +248,61 @@ class Keyring(object):
                 request_id = id(verify_request)
                 server_to_request_ids.setdefault(server_name, set()).add(request_id)
 
-            def remove_deferreds(res, verify_request):
+            # Wait for any previous lookups to complete before proceeding.
+            yield self.wait_for_previous_lookups(server_to_request_ids.keys())
+
+            # take out a lock on each of the servers by sticking a Deferred in
+            # key_downloads
+            for server_name in server_to_request_ids.keys():
+                self.key_downloads[server_name] = defer.Deferred()
+                logger.debug("Got key lookup lock on %s", server_name)
+
+            # When we've finished fetching all the keys for a given server_name,
+            # drop the lock by resolving the deferred in key_downloads.
+            def drop_server_lock(server_name):
+                d = self.key_downloads.pop(server_name)
+                d.callback(None)
+
+            def lookup_done(res, verify_request):
                 server_name = verify_request.server_name
-                request_id = id(verify_request)
-                server_to_request_ids[server_name].discard(request_id)
-                if not server_to_request_ids[server_name]:
-                    d = server_to_deferred.pop(server_name, None)
-                    if d:
-                        d.callback(None)
+                server_requests = server_to_request_ids[server_name]
+                server_requests.remove(id(verify_request))
+
+                # if there are no more requests for this server, we can drop the lock.
+                if not server_requests:
+                    with PreserveLoggingContext(ctx):
+                        logger.debug("Releasing key lookup lock on %s", server_name)
+
+                    # ... but not immediately, as that can cause stack explosions if
+                    # we get a long queue of lookups.
+                    self.clock.call_later(0, drop_server_lock, server_name)
+
                 return res
 
             for verify_request in verify_requests:
-                verify_request.key_ready.addBoth(remove_deferreds, verify_request)
+                verify_request.key_ready.addBoth(lookup_done, verify_request)
+
+            # Actually start fetching keys.
+            self._get_server_verify_keys(verify_requests)
         except Exception:
             logger.exception("Error starting key lookups")
 
     @defer.inlineCallbacks
-    def wait_for_previous_lookups(self, server_to_deferred):
+    def wait_for_previous_lookups(self, server_names):
         """Waits for any previous key lookups for the given servers to finish.
 
         Args:
-            server_to_deferred (dict[str, Deferred]): server_name to deferred which gets
-                resolved once we've finished looking up keys for that server.
-                The Deferreds should be regular twisted ones which call their
-                callbacks with no logcontext.
+            server_names (Iterable[str]): list of servers which we want to look up
 
-        Returns: a Deferred which resolves once all key lookups for the given
-            servers have completed. Follows the synapse rules of logcontext
-            preservation.
+        Returns:
+            Deferred[None]: resolves once all key lookups for the given servers have
+                completed. Follows the synapse rules of logcontext preservation.
         """
         loop_count = 1
         while True:
             wait_on = [
                 (server_name, self.key_downloads[server_name])
-                for server_name in server_to_deferred.keys()
+                for server_name in server_names
                 if server_name in self.key_downloads
             ]
             if not wait_on:
@@ -314,19 +317,6 @@ class Keyring(object):
 
             loop_count += 1
 
-        ctx = LoggingContext.current_context()
-
-        def rm(r, server_name_):
-            with PreserveLoggingContext(ctx):
-                logger.debug("Releasing key lookup lock on %s", server_name_)
-                self.key_downloads.pop(server_name_, None)
-            return r
-
-        for server_name, deferred in server_to_deferred.items():
-            logger.debug("Got key lookup lock on %s", server_name)
-            self.key_downloads[server_name] = deferred
-            deferred.addBoth(rm, server_name)
-
     def _get_server_verify_keys(self, verify_requests):
         """Tries to find at least one key for each verify request
 
@@ -472,7 +462,7 @@ class StoreKeyFetcher(KeyFetcher):
         keys = {}
         for (server_name, key_id), key in res.items():
             keys.setdefault(server_name, {})[key_id] = key
-        defer.returnValue(keys)
+        return keys
 
 
 class BaseV2KeyFetcher(object):
@@ -576,7 +566,7 @@ class BaseV2KeyFetcher(object):
             ).addErrback(unwrapFirstError)
         )
 
-        defer.returnValue(verify_keys)
+        return verify_keys
 
 
 class PerspectivesKeyFetcher(BaseV2KeyFetcher):
@@ -598,7 +588,7 @@ class PerspectivesKeyFetcher(BaseV2KeyFetcher):
                 result = yield self.get_server_verify_key_v2_indirect(
                     keys_to_fetch, key_server
                 )
-                defer.returnValue(result)
+                return result
             except KeyLookupError as e:
                 logger.warning(
                     "Key lookup failed from %r: %s", key_server.server_name, e
@@ -611,7 +601,7 @@ class PerspectivesKeyFetcher(BaseV2KeyFetcher):
                     str(e),
                 )
 
-            defer.returnValue({})
+            return {}
 
         results = yield make_deferred_yieldable(
             defer.gatherResults(
@@ -625,7 +615,7 @@ class PerspectivesKeyFetcher(BaseV2KeyFetcher):
             for server_name, keys in result.items():
                 union_of_keys.setdefault(server_name, {}).update(keys)
 
-        defer.returnValue(union_of_keys)
+        return union_of_keys
 
     @defer.inlineCallbacks
     def get_server_verify_key_v2_indirect(self, keys_to_fetch, key_server):
@@ -711,7 +701,7 @@ class PerspectivesKeyFetcher(BaseV2KeyFetcher):
             perspective_name, time_now_ms, added_keys
         )
 
-        defer.returnValue(keys)
+        return keys
 
     def _validate_perspectives_response(self, key_server, response):
         """Optionally check the signature on the result of a /key/query request
@@ -853,7 +843,7 @@ class ServerKeyFetcher(BaseV2KeyFetcher):
             )
             keys.update(response_keys)
 
-        defer.returnValue(keys)
+        return keys
 
 
 @defer.inlineCallbacks

synapse/events/builder.py

@@ -144,15 +144,13 @@ class EventBuilder(object):
         if self._origin_server_ts is not None:
             event_dict["origin_server_ts"] = self._origin_server_ts
 
-        defer.returnValue(
-            create_local_event_from_event_dict(
-                clock=self._clock,
-                hostname=self._hostname,
-                signing_key=self._signing_key,
-                format_version=self.format_version,
-                event_dict=event_dict,
-                internal_metadata_dict=self.internal_metadata.get_dict(),
-            )
+        return create_local_event_from_event_dict(
+            clock=self._clock,
+            hostname=self._hostname,
+            signing_key=self._signing_key,
+            format_version=self.format_version,
+            event_dict=event_dict,
+            internal_metadata_dict=self.internal_metadata.get_dict(),
         )
 
 

synapse/events/snapshot.py

@@ -133,19 +133,17 @@ class EventContext(object):
         else:
             prev_state_id = None
 
-        defer.returnValue(
-            {
-                "prev_state_id": prev_state_id,
-                "event_type": event.type,
-                "event_state_key": event.state_key if event.is_state() else None,
-                "state_group": self.state_group,
-                "rejected": self.rejected,
-                "prev_group": self.prev_group,
-                "delta_ids": _encode_state_dict(self.delta_ids),
-                "prev_state_events": self.prev_state_events,
-                "app_service_id": self.app_service.id if self.app_service else None,
-            }
-        )
+        return {
+            "prev_state_id": prev_state_id,
+            "event_type": event.type,
+            "event_state_key": event.state_key if event.is_state() else None,
+            "state_group": self.state_group,
+            "rejected": self.rejected,
+            "prev_group": self.prev_group,
+            "delta_ids": _encode_state_dict(self.delta_ids),
+            "prev_state_events": self.prev_state_events,
+            "app_service_id": self.app_service.id if self.app_service else None,
+        }
 
     @staticmethod
     def deserialize(store, input):
@@ -202,7 +200,7 @@ class EventContext(object):
 
         yield make_deferred_yieldable(self._fetching_state_deferred)
 
-        defer.returnValue(self._current_state_ids)
+        return self._current_state_ids
 
     @defer.inlineCallbacks
     def get_prev_state_ids(self, store):
@@ -222,7 +220,7 @@ class EventContext(object):
 
         yield make_deferred_yieldable(self._fetching_state_deferred)
 
-        defer.returnValue(self._prev_state_ids)
+        return self._prev_state_ids
 
     def get_cached_current_state_ids(self):
         """Gets the current state IDs if we have them already cached.

synapse/events/third_party_rules.py

@@ -51,7 +51,7 @@ class ThirdPartyEventRules(object):
             defer.Deferred[bool]: True if the event should be allowed, False if not.
         """
         if self.third_party_rules is None:
-            defer.returnValue(True)
+            return True
 
         prev_state_ids = yield context.get_prev_state_ids(self.store)
 
@@ -61,7 +61,7 @@ class ThirdPartyEventRules(object):
             state_events[key] = yield self.store.get_event(event_id, allow_none=True)
 
         ret = yield self.third_party_rules.check_event_allowed(event, state_events)
-        defer.returnValue(ret)
+        return ret
 
     @defer.inlineCallbacks
     def on_create_room(self, requester, config, is_requester_admin):
@@ -98,7 +98,7 @@ class ThirdPartyEventRules(object):
         """
 
         if self.third_party_rules is None:
-            defer.returnValue(True)
+            return True
 
         state_ids = yield self.store.get_filtered_current_state_ids(room_id)
         room_state_events = yield self.store.get_events(state_ids.values())
@@ -110,4 +110,4 @@ class ThirdPartyEventRules(object):
         ret = yield self.third_party_rules.check_threepid_can_be_invited(
             medium, address, state_events
         )
-        defer.returnValue(ret)
+        return ret

synapse/events/utils.py

@@ -360,7 +360,7 @@ class EventClientSerializer(object):
         """
         # To handle the case of presence events and the like
         if not isinstance(event, EventBase):
-            defer.returnValue(event)
+            return event
 
         event_id = event.event_id
         serialized_event = serialize_event(event, time_now, **kwargs)
@@ -406,7 +406,7 @@ class EventClientSerializer(object):
                     "sender": edit.sender,
                 }
 
-        defer.returnValue(serialized_event)
+        return serialized_event
 
     def serialize_events(self, events, time_now, **kwargs):
         """Serializes multiple events.

synapse/events/validator.py

@@ -95,10 +95,10 @@ class EventValidator(object):
 
         elif event.type == EventTypes.Topic:
             self._ensure_strings(event.content, ["topic"])
-
+            self._ensure_state_event(event)
         elif event.type == EventTypes.Name:
             self._ensure_strings(event.content, ["name"])
-
+            self._ensure_state_event(event)
         elif event.type == EventTypes.Member:
             if "membership" not in event.content:
                 raise SynapseError(400, "Content has not membership key")
@@ -106,9 +106,25 @@ class EventValidator(object):
             if event.content["membership"] not in Membership.LIST:
                 raise SynapseError(400, "Invalid membership key")
 
+            self._ensure_state_event(event)
+        elif event.type == EventTypes.Tombstone:
+            if "replacement_room" not in event.content:
+                raise SynapseError(400, "Content has no replacement_room key")
+
+            if event.content["replacement_room"] == event.room_id:
+                raise SynapseError(
+                    400, "Tombstone cannot reference the room it was sent in"
+                )
+
+            self._ensure_state_event(event)
+
     def _ensure_strings(self, d, keys):
         for s in keys:
             if s not in d:
                 raise SynapseError(400, "'%s' not in content" % (s,))
             if not isinstance(d[s], string_types):
                 raise SynapseError(400, "'%s' not a string type" % (s,))
+
+    def _ensure_state_event(self, event):
+        if not event.is_state():
+            raise SynapseError(400, "'%s' must be state events" % (event.type,))

synapse/federation/federation_base.py

@@ -106,7 +106,7 @@ class FederationBase(object):
                     "Failed to find copy of %s with valid signature", pdu.event_id
                 )
 
-            defer.returnValue(res)
+            return res
 
         handle = preserve_fn(handle_check_result)
         deferreds2 = [handle(pdu, deferred) for pdu, deferred in zip(pdus, deferreds)]
@@ -116,9 +116,9 @@ class FederationBase(object):
         ).addErrback(unwrapFirstError)
 
         if include_none:
-            defer.returnValue(valid_pdus)
+            return valid_pdus
         else:
-            defer.returnValue([p for p in valid_pdus if p])
+            return [p for p in valid_pdus if p]
 
     def _check_sigs_and_hash(self, room_version, pdu):
         return make_deferred_yieldable(

synapse/federation/federation_client.py

@@ -213,7 +213,7 @@ class FederationClient(FederationBase):
             ).addErrback(unwrapFirstError)
         )
 
-        defer.returnValue(pdus)
+        return pdus
 
     @defer.inlineCallbacks
     @log_function
@@ -245,7 +245,7 @@ class FederationClient(FederationBase):
 
         ev = self._get_pdu_cache.get(event_id)
         if ev:
-            defer.returnValue(ev)
+            return ev
 
         pdu_attempts = self.pdu_destination_tried.setdefault(event_id, {})
 
@@ -307,7 +307,7 @@ class FederationClient(FederationBase):
         if signed_pdu:
             self._get_pdu_cache[event_id] = signed_pdu
 
-        defer.returnValue(signed_pdu)
+        return signed_pdu
 
     @defer.inlineCallbacks
     @log_function
@@ -355,7 +355,7 @@ class FederationClient(FederationBase):
 
             auth_chain.sort(key=lambda e: e.depth)
 
-            defer.returnValue((pdus, auth_chain))
+            return (pdus, auth_chain)
         except HttpResponseException as e:
             if e.code == 400 or e.code == 404:
                 logger.info("Failed to use get_room_state_ids API, falling back")
@@ -404,7 +404,7 @@ class FederationClient(FederationBase):
 
         signed_auth.sort(key=lambda e: e.depth)
 
-        defer.returnValue((signed_pdus, signed_auth))
+        return (signed_pdus, signed_auth)
 
     @defer.inlineCallbacks
     def get_events_from_store_or_dest(self, destination, room_id, event_ids):
@@ -429,7 +429,7 @@ class FederationClient(FederationBase):
             missing_events.discard(k)
 
         if not missing_events:
-            defer.returnValue((signed_events, failed_to_fetch))
+            return (signed_events, failed_to_fetch)
 
         logger.debug(
             "Fetching unknown state/auth events %s for room %s",
@@ -465,7 +465,7 @@ class FederationClient(FederationBase):
             # We removed all events we successfully fetched from `batch`
             failed_to_fetch.update(batch)
 
-        defer.returnValue((signed_events, failed_to_fetch))
+        return (signed_events, failed_to_fetch)
 
     @defer.inlineCallbacks
     @log_function
@@ -485,7 +485,7 @@ class FederationClient(FederationBase):
 
         signed_auth.sort(key=lambda e: e.depth)
 
-        defer.returnValue(signed_auth)
+        return signed_auth
 
     @defer.inlineCallbacks
     def _try_destination_list(self, description, destinations, callback):
@@ -511,9 +511,8 @@ class FederationClient(FederationBase):
             The [Deferred] result of callback, if it succeeds
 
         Raises:
-            SynapseError if the chosen remote server returns a 300/400 code.
-
-            RuntimeError if no servers were reachable.
+            SynapseError if the chosen remote server returns a 300/400 code, or
+            no servers were reachable.
         """
         for destination in destinations:
             if destination == self.server_name:
@@ -521,7 +520,7 @@ class FederationClient(FederationBase):
 
             try:
                 res = yield callback(destination)
-                defer.returnValue(res)
+                return res
             except InvalidResponseError as e:
                 logger.warn("Failed to %s via %s: %s", description, destination, e)
             except HttpResponseException as e:
@@ -538,7 +537,7 @@ class FederationClient(FederationBase):
             except Exception:
                 logger.warn("Failed to %s via %s", description, destination, exc_info=1)
 
-        raise RuntimeError("Failed to %s via any server" % (description,))
+        raise SynapseError(502, "Failed to %s via any server" % (description,))
 
     def make_membership_event(
         self, destinations, room_id, user_id, membership, content, params
@@ -615,7 +614,7 @@ class FederationClient(FederationBase):
                 event_dict=pdu_dict,
             )
 
-            defer.returnValue((destination, ev, event_format))
+            return (destination, ev, event_format)
 
         return self._try_destination_list(
             "make_" + membership, destinations, send_request
@@ -728,13 +727,11 @@ class FederationClient(FederationBase):
 
             check_authchain_validity(signed_auth)
 
-            defer.returnValue(
-                {
-                    "state": signed_state,
-                    "auth_chain": signed_auth,
-                    "origin": destination,
-                }
-            )
+            return {
+                "state": signed_state,
+                "auth_chain": signed_auth,
+                "origin": destination,
+            }
 
         return self._try_destination_list("send_join", destinations, send_request)
 
@@ -758,7 +755,7 @@ class FederationClient(FederationBase):
 
         # FIXME: We should handle signature failures more gracefully.
 
-        defer.returnValue(pdu)
+        return pdu
 
     @defer.inlineCallbacks
     def _do_send_invite(self, destination, pdu, room_version):
@@ -786,7 +783,7 @@ class FederationClient(FederationBase):
                     "invite_room_state": pdu.unsigned.get("invite_room_state", []),
                 },
             )
-            defer.returnValue(content)
+            return content
         except HttpResponseException as e:
             if e.code in [400, 404]:
                 err = e.to_synapse_error()
@@ -821,7 +818,7 @@ class FederationClient(FederationBase):
             event_id=pdu.event_id,
             content=pdu.get_pdu_json(time_now),
         )
-        defer.returnValue(content)
+        return content
 
     def send_leave(self, destinations, pdu):
         """Sends a leave event to one of a list of homeservers.
@@ -856,7 +853,7 @@ class FederationClient(FederationBase):
             )
 
             logger.debug("Got content: %s", content)
-            defer.returnValue(None)
+            return None
 
         return self._try_destination_list("send_leave", destinations, send_request)
 
@@ -917,7 +914,7 @@ class FederationClient(FederationBase):
             "missing": content.get("missing", []),
         }
 
-        defer.returnValue(ret)
+        return ret
 
     @defer.inlineCallbacks
     def get_missing_events(
@@ -974,7 +971,7 @@ class FederationClient(FederationBase):
             # get_missing_events
             signed_events = []
 
-        defer.returnValue(signed_events)
+        return signed_events
 
     @defer.inlineCallbacks
     def forward_third_party_invite(self, destinations, room_id, event_dict):
@@ -986,7 +983,7 @@ class FederationClient(FederationBase):
                 yield self.transport_layer.exchange_third_party_invite(
                     destination=destination, room_id=room_id, event_dict=event_dict
                 )
-                defer.returnValue(None)
+                return None
             except CodeMessageException:
                 raise
             except Exception as e:

synapse/federation/federation_server.py

@@ -99,7 +99,7 @@ class FederationServer(FederationBase):
 
             res = self._transaction_from_pdus(pdus).get_dict()
 
-        defer.returnValue((200, res))
+        return (200, res)
 
     @defer.inlineCallbacks
     @log_function
@@ -126,7 +126,7 @@ class FederationServer(FederationBase):
                 origin, transaction, request_time
             )
 
-        defer.returnValue(result)
+        return result
 
     @defer.inlineCallbacks
     def _handle_incoming_transaction(self, origin, transaction, request_time):
@@ -147,8 +147,7 @@ class FederationServer(FederationBase):
                 "[%s] We've already responded to this request",
                 transaction.transaction_id,
             )
-            defer.returnValue(response)
-            return
+            return response
 
         logger.debug("[%s] Transaction is new", transaction.transaction_id)
 
@@ -163,7 +162,7 @@ class FederationServer(FederationBase):
             yield self.transaction_actions.set_response(
                 origin, transaction, 400, response
             )
-            defer.returnValue((400, response))
+            return (400, response)
 
         received_pdus_counter.inc(len(transaction.pdus))
 
@@ -265,7 +264,7 @@ class FederationServer(FederationBase):
         logger.debug("Returning: %s", str(response))
 
         yield self.transaction_actions.set_response(origin, transaction, 200, response)
-        defer.returnValue((200, response))
+        return (200, response)
 
     @defer.inlineCallbacks
     def received_edu(self, origin, edu_type, content):
@@ -298,7 +297,7 @@ class FederationServer(FederationBase):
                 event_id,
             )
 
-        defer.returnValue((200, resp))
+        return (200, resp)
 
     @defer.inlineCallbacks
     def on_state_ids_request(self, origin, room_id, event_id):
@@ -315,9 +314,7 @@ class FederationServer(FederationBase):
         state_ids = yield self.handler.get_state_ids_for_pdu(room_id, event_id)
         auth_chain_ids = yield self.store.get_auth_chain_ids(state_ids)
 
-        defer.returnValue(
-            (200, {"pdu_ids": state_ids, "auth_chain_ids": auth_chain_ids})
-        )
+        return (200, {"pdu_ids": state_ids, "auth_chain_ids": auth_chain_ids})
 
     @defer.inlineCallbacks
     def _on_context_state_request_compute(self, room_id, event_id):
@@ -336,12 +333,10 @@ class FederationServer(FederationBase):
                     )
                 )
 
-        defer.returnValue(
-            {
-                "pdus": [pdu.get_pdu_json() for pdu in pdus],
-                "auth_chain": [pdu.get_pdu_json() for pdu in auth_chain],
-            }
-        )
+        return {
+            "pdus": [pdu.get_pdu_json() for pdu in pdus],
+            "auth_chain": [pdu.get_pdu_json() for pdu in auth_chain],
+        }
 
     @defer.inlineCallbacks
     @log_function
@@ -349,15 +344,15 @@ class FederationServer(FederationBase):
         pdu = yield self.handler.get_persisted_pdu(origin, event_id)
 
         if pdu:
-            defer.returnValue((200, self._transaction_from_pdus([pdu]).get_dict()))
+            return (200, self._transaction_from_pdus([pdu]).get_dict())
         else:
-            defer.returnValue((404, ""))
+            return (404, "")
 
     @defer.inlineCallbacks
     def on_query_request(self, query_type, args):
         received_queries_counter.labels(query_type).inc()
         resp = yield self.registry.on_query(query_type, args)
-        defer.returnValue((200, resp))
+        return (200, resp)
 
     @defer.inlineCallbacks
     def on_make_join_request(self, origin, room_id, user_id, supported_versions):
@@ -371,9 +366,7 @@ class FederationServer(FederationBase):
 
         pdu = yield self.handler.on_make_join_request(origin, room_id, user_id)
         time_now = self._clock.time_msec()
-        defer.returnValue(
-            {"event": pdu.get_pdu_json(time_now), "room_version": room_version}
-        )
+        return {"event": pdu.get_pdu_json(time_now), "room_version": room_version}
 
     @defer.inlineCallbacks
     def on_invite_request(self, origin, content, room_version):
@@ -391,7 +384,7 @@ class FederationServer(FederationBase):
         yield self.check_server_matches_acl(origin_host, pdu.room_id)
         ret_pdu = yield self.handler.on_invite_request(origin, pdu)
         time_now = self._clock.time_msec()
-        defer.returnValue({"event": ret_pdu.get_pdu_json(time_now)})
+        return {"event": ret_pdu.get_pdu_json(time_now)}
 
     @defer.inlineCallbacks
     def on_send_join_request(self, origin, content, room_id):
@@ -407,16 +400,14 @@ class FederationServer(FederationBase):
         logger.debug("on_send_join_request: pdu sigs: %s", pdu.signatures)
         res_pdus = yield self.handler.on_send_join_request(origin, pdu)
         time_now = self._clock.time_msec()
-        defer.returnValue(
-            (
-                200,
-                {
-                    "state": [p.get_pdu_json(time_now) for p in res_pdus["state"]],
-                    "auth_chain": [
-                        p.get_pdu_json(time_now) for p in res_pdus["auth_chain"]
-                    ],
-                },
-            )
+        return (
+            200,
+            {
+                "state": [p.get_pdu_json(time_now) for p in res_pdus["state"]],
+                "auth_chain": [
+                    p.get_pdu_json(time_now) for p in res_pdus["auth_chain"]
+                ],
+            },
         )
 
     @defer.inlineCallbacks
@@ -428,9 +419,7 @@ class FederationServer(FederationBase):
         room_version = yield self.store.get_room_version(room_id)
 
         time_now = self._clock.time_msec()
-        defer.returnValue(
-            {"event": pdu.get_pdu_json(time_now), "room_version": room_version}
-        )
+        return {"event": pdu.get_pdu_json(time_now), "room_version": room_version}
 
     @defer.inlineCallbacks
     def on_send_leave_request(self, origin, content, room_id):
@@ -445,7 +434,7 @@ class FederationServer(FederationBase):
 
         logger.debug("on_send_leave_request: pdu sigs: %s", pdu.signatures)
         yield self.handler.on_send_leave_request(origin, pdu)
-        defer.returnValue((200, {}))
+        return (200, {})
 
     @defer.inlineCallbacks
     def on_event_auth(self, origin, room_id, event_id):
@@ -456,7 +445,7 @@ class FederationServer(FederationBase):
             time_now = self._clock.time_msec()
             auth_pdus = yield self.handler.on_event_auth(event_id)
             res = {"auth_chain": [a.get_pdu_json(time_now) for a in auth_pdus]}
-        defer.returnValue((200, res))
+        return (200, res)
 
     @defer.inlineCallbacks
     def on_query_auth_request(self, origin, content, room_id, event_id):
@@ -509,7 +498,7 @@ class FederationServer(FederationBase):
                 "missing": ret.get("missing", []),
             }
 
-        defer.returnValue((200, send_content))
+        return (200, send_content)
 
     @log_function
     def on_query_client_keys(self, origin, content):
@@ -548,7 +537,7 @@ class FederationServer(FederationBase):
             ),
         )
 
-        defer.returnValue({"one_time_keys": json_result})
+        return {"one_time_keys": json_result}
 
     @defer.inlineCallbacks
     @log_function
@@ -580,9 +569,7 @@ class FederationServer(FederationBase):
 
             time_now = self._clock.time_msec()
 
-        defer.returnValue(
-            {"events": [ev.get_pdu_json(time_now) for ev in missing_events]}
-        )
+        return {"events": [ev.get_pdu_json(time_now) for ev in missing_events]}
 
     @log_function
     def on_openid_userinfo(self, token):
@@ -676,14 +663,14 @@ class FederationServer(FederationBase):
         ret = yield self.handler.exchange_third_party_invite(
             sender_user_id, target_user_id, room_id, signed
         )
-        defer.returnValue(ret)
+        return ret
 
     @defer.inlineCallbacks
     def on_exchange_third_party_invite_request(self, origin, room_id, event_dict):
         ret = yield self.handler.on_exchange_third_party_invite_request(
             origin, room_id, event_dict
         )
-        defer.returnValue(ret)
+        return ret
 
     @defer.inlineCallbacks
     def check_server_matches_acl(self, server_name, room_id):

synapse/federation/sender/per_destination_queue.py

@@ -374,7 +374,7 @@ class PerDestinationQueue(object):
 
         assert len(edus) <= limit, "get_devices_by_remote returned too many EDUs"
 
-        defer.returnValue((edus, now_stream_id))
+        return (edus, now_stream_id)
 
     @defer.inlineCallbacks
     def _get_to_device_message_edus(self, limit):
@@ -393,4 +393,4 @@ class PerDestinationQueue(object):
             for content in contents
         ]
 
-        defer.returnValue((edus, stream_id))
+        return (edus, stream_id)

synapse/federation/sender/transaction_manager.py

@@ -133,4 +133,4 @@ class TransactionManager(object):
                 )
             success = False
 
-        defer.returnValue(success)
+        return success

synapse/federation/transport/client.py

@@ -187,7 +187,7 @@ class TransportLayerClient(object):
             try_trailing_slash_on_400=True,
         )
 
-        defer.returnValue(response)
+        return response
 
     @defer.inlineCallbacks
     @log_function
@@ -205,7 +205,7 @@ class TransportLayerClient(object):
             ignore_backoff=ignore_backoff,
         )
 
-        defer.returnValue(content)
+        return content
 
     @defer.inlineCallbacks
     @log_function
@@ -263,7 +263,7 @@ class TransportLayerClient(object):
             ignore_backoff=ignore_backoff,
         )
 
-        defer.returnValue(content)
+        return content
 
     @defer.inlineCallbacks
     @log_function
@@ -274,7 +274,7 @@ class TransportLayerClient(object):
             destination=destination, path=path, data=content
         )
 
-        defer.returnValue(response)
+        return response
 
     @defer.inlineCallbacks
     @log_function
@@ -292,7 +292,7 @@ class TransportLayerClient(object):
             ignore_backoff=True,
         )
 
-        defer.returnValue(response)
+        return response
 
     @defer.inlineCallbacks
     @log_function
@@ -303,7 +303,7 @@ class TransportLayerClient(object):
             destination=destination, path=path, data=content, ignore_backoff=True
         )
 
-        defer.returnValue(response)
+        return response
 
     @defer.inlineCallbacks
     @log_function
@@ -314,7 +314,7 @@ class TransportLayerClient(object):
             destination=destination, path=path, data=content, ignore_backoff=True
         )
 
-        defer.returnValue(response)
+        return response
 
     @defer.inlineCallbacks
     @log_function
@@ -343,7 +343,7 @@ class TransportLayerClient(object):
             destination=remote_server, path=path, args=args, ignore_backoff=True
         )
 
-        defer.returnValue(response)
+        return response
 
     @defer.inlineCallbacks
     @log_function
@@ -354,7 +354,7 @@ class TransportLayerClient(object):
             destination=destination, path=path, data=event_dict
         )
 
-        defer.returnValue(response)
+        return response
 
     @defer.inlineCallbacks
     @log_function
@@ -363,7 +363,7 @@ class TransportLayerClient(object):
 
         content = yield self.client.get_json(destination=destination, path=path)
 
-        defer.returnValue(content)
+        return content
 
     @defer.inlineCallbacks
     @log_function
@@ -374,7 +374,7 @@ class TransportLayerClient(object):
             destination=destination, path=path, data=content
         )
 
-        defer.returnValue(content)
+        return content
 
     @defer.inlineCallbacks
     @log_function
@@ -406,7 +406,7 @@ class TransportLayerClient(object):
         content = yield self.client.post_json(
             destination=destination, path=path, data=query_content, timeout=timeout
         )
-        defer.returnValue(content)
+        return content
 
     @defer.inlineCallbacks
     @log_function
@@ -430,7 +430,7 @@ class TransportLayerClient(object):
         content = yield self.client.get_json(
             destination=destination, path=path, timeout=timeout
         )
-        defer.returnValue(content)
+        return content
 
     @defer.inlineCallbacks
     @log_function
@@ -464,7 +464,7 @@ class TransportLayerClient(object):
         content = yield self.client.post_json(
             destination=destination, path=path, data=query_content, timeout=timeout
         )
-        defer.returnValue(content)
+        return content
 
     @defer.inlineCallbacks
     @log_function
@@ -492,7 +492,7 @@ class TransportLayerClient(object):
             timeout=timeout,
         )
 
-        defer.returnValue(content)
+        return content
 
     @log_function
     def get_group_profile(self, destination, group_id, requester_user_id):

synapse/federation/transport/server.py

@@ -19,8 +19,9 @@ import functools
 import logging
 import re
 
+from twisted.internet.defer import maybeDeferred
+
 import synapse
-import synapse.logging.opentracing as opentracing
 from synapse.api.errors import Codes, FederationDeniedError, SynapseError
 from synapse.api.room_versions import RoomVersions
 from synapse.api.urls import (
@@ -37,6 +38,7 @@ from synapse.http.servlet import (
     parse_string_from_args,
 )
 from synapse.logging.context import run_in_background
+from synapse.logging.opentracing import start_active_span_from_context, tags
 from synapse.types import ThirdPartyInstanceID, get_domain_from_id
 from synapse.util.ratelimitutils import FederationRateLimiter
 from synapse.util.versionstring import get_version_string
@@ -287,16 +289,17 @@ class BaseFederationServlet(object):
                 raise
 
             # Start an opentracing span
-            with opentracing.start_active_span_from_context(
+            with start_active_span_from_context(
                 request.requestHeaders,
                 "incoming-federation-request",
                 tags={
                     "request_id": request.get_request_id(),
-                    opentracing.tags.SPAN_KIND: opentracing.tags.SPAN_KIND_RPC_SERVER,
-                    opentracing.tags.HTTP_METHOD: request.get_method(),
-                    opentracing.tags.HTTP_URL: request.get_redacted_uri(),
-                    opentracing.tags.PEER_HOST_IPV6: request.getClientIP(),
+                    tags.SPAN_KIND: tags.SPAN_KIND_RPC_SERVER,
+                    tags.HTTP_METHOD: request.get_method(),
+                    tags.HTTP_URL: request.get_redacted_uri(),
+                    tags.PEER_HOST_IPV6: request.getClientIP(),
                     "authenticated_entity": origin,
+                    "servlet_name": request.request_metrics.name,
                 },
             ):
                 if origin:
@@ -745,8 +748,12 @@ class PublicRoomList(BaseFederationServlet):
         else:
             network_tuple = ThirdPartyInstanceID(None, None)
 
-        data = await self.handler.get_local_public_room_list(
-            limit, since_token, network_tuple=network_tuple, from_federation=True
+        data = await maybeDeferred(
+            self.handler.get_local_public_room_list,
+            limit,
+            since_token,
+            network_tuple=network_tuple,
+            from_federation=True,
         )
         return 200, data
 

synapse/groups/attestations.py

@@ -157,7 +157,7 @@ class GroupAttestionRenewer(object):
 
         yield self.store.update_remote_attestion(group_id, user_id, attestation)
 
-        defer.returnValue({})
+        return {}
 
     def _start_renew_attestations(self):
         return run_as_background_process("renew_attestations", self._renew_attestations)

synapse/groups/groups_server.py

@@ -85,7 +85,7 @@ class GroupsServerHandler(object):
             if not is_admin:
                 raise SynapseError(403, "User is not admin in group")
 
-        defer.returnValue(group)
+        return group
 
     @defer.inlineCallbacks
     def get_group_summary(self, group_id, requester_user_id):
@@ -151,22 +151,20 @@ class GroupsServerHandler(object):
             group_id, requester_user_id
         )
 
-        defer.returnValue(
-            {
-                "profile": profile,
-                "users_section": {
-                    "users": users,
-                    "roles": roles,
-                    "total_user_count_estimate": 0,  # TODO
-                },
-                "rooms_section": {
-                    "rooms": rooms,
-                    "categories": categories,
-                    "total_room_count_estimate": 0,  # TODO
-                },
-                "user": membership_info,
-            }
-        )
+        return {
+            "profile": profile,
+            "users_section": {
+                "users": users,
+                "roles": roles,
+                "total_user_count_estimate": 0,  # TODO
+            },
+            "rooms_section": {
+                "rooms": rooms,
+                "categories": categories,
+                "total_room_count_estimate": 0,  # TODO
+            },
+            "user": membership_info,
+        }
 
     @defer.inlineCallbacks
     def update_group_summary_room(
@@ -192,7 +190,7 @@ class GroupsServerHandler(object):
             is_public=is_public,
         )
 
-        defer.returnValue({})
+        return {}
 
     @defer.inlineCallbacks
     def delete_group_summary_room(
@@ -208,7 +206,7 @@ class GroupsServerHandler(object):
             group_id=group_id, room_id=room_id, category_id=category_id
         )
 
-        defer.returnValue({})
+        return {}
 
     @defer.inlineCallbacks
     def set_group_join_policy(self, group_id, requester_user_id, content):
@@ -228,7 +226,7 @@ class GroupsServerHandler(object):
 
         yield self.store.set_group_join_policy(group_id, join_policy=join_policy)
 
-        defer.returnValue({})
+        return {}
 
     @defer.inlineCallbacks
     def get_group_categories(self, group_id, requester_user_id):
@@ -237,7 +235,7 @@ class GroupsServerHandler(object):
         yield self.check_group_is_ours(group_id, requester_user_id, and_exists=True)
 
         categories = yield self.store.get_group_categories(group_id=group_id)
-        defer.returnValue({"categories": categories})
+        return {"categories": categories}
 
     @defer.inlineCallbacks
     def get_group_category(self, group_id, requester_user_id, category_id):
@@ -249,7 +247,7 @@ class GroupsServerHandler(object):
             group_id=group_id, category_id=category_id
         )
 
-        defer.returnValue(res)
+        return res
 
     @defer.inlineCallbacks
     def update_group_category(self, group_id, requester_user_id, category_id, content):
@@ -269,7 +267,7 @@ class GroupsServerHandler(object):
             profile=profile,
         )
 
-        defer.returnValue({})
+        return {}
 
     @defer.inlineCallbacks
     def delete_group_category(self, group_id, requester_user_id, category_id):
@@ -283,7 +281,7 @@ class GroupsServerHandler(object):
             group_id=group_id, category_id=category_id
         )
 
-        defer.returnValue({})
+        return {}
 
     @defer.inlineCallbacks
     def get_group_roles(self, group_id, requester_user_id):
@@ -292,7 +290,7 @@ class GroupsServerHandler(object):
         yield self.check_group_is_ours(group_id, requester_user_id, and_exists=True)
 
         roles = yield self.store.get_group_roles(group_id=group_id)
-        defer.returnValue({"roles": roles})
+        return {"roles": roles}
 
     @defer.inlineCallbacks
     def get_group_role(self, group_id, requester_user_id, role_id):
@@ -301,7 +299,7 @@ class GroupsServerHandler(object):
         yield self.check_group_is_ours(group_id, requester_user_id, and_exists=True)
 
         res = yield self.store.get_group_role(group_id=group_id, role_id=role_id)
-        defer.returnValue(res)
+        return res
 
     @defer.inlineCallbacks
     def update_group_role(self, group_id, requester_user_id, role_id, content):
@@ -319,7 +317,7 @@ class GroupsServerHandler(object):
             group_id=group_id, role_id=role_id, is_public=is_public, profile=profile
         )
 
-        defer.returnValue({})
+        return {}
 
     @defer.inlineCallbacks
     def delete_group_role(self, group_id, requester_user_id, role_id):
@@ -331,7 +329,7 @@ class GroupsServerHandler(object):
 
         yield self.store.remove_group_role(group_id=group_id, role_id=role_id)
 
-        defer.returnValue({})
+        return {}
 
     @defer.inlineCallbacks
     def update_group_summary_user(
@@ -355,7 +353,7 @@ class GroupsServerHandler(object):
             is_public=is_public,
         )
 
-        defer.returnValue({})
+        return {}
 
     @defer.inlineCallbacks
     def delete_group_summary_user(self, group_id, requester_user_id, user_id, role_id):
@@ -369,7 +367,7 @@ class GroupsServerHandler(object):
             group_id=group_id, user_id=user_id, role_id=role_id
         )
 
-        defer.returnValue({})
+        return {}
 
     @defer.inlineCallbacks
     def get_group_profile(self, group_id, requester_user_id):
@@ -391,7 +389,7 @@ class GroupsServerHandler(object):
             group_description = {key: group[key] for key in cols}
             group_description["is_openly_joinable"] = group["join_policy"] == "open"
 
-            defer.returnValue(group_description)
+            return group_description
         else:
             raise SynapseError(404, "Unknown group")
 
@@ -461,9 +459,7 @@ class GroupsServerHandler(object):
 
         # TODO: If admin add lists of users whose attestations have timed out
 
-        defer.returnValue(
-            {"chunk": chunk, "total_user_count_estimate": len(user_results)}
-        )
+        return {"chunk": chunk, "total_user_count_estimate": len(user_results)}
 
     @defer.inlineCallbacks
     def get_invited_users_in_group(self, group_id, requester_user_id):
@@ -494,9 +490,7 @@ class GroupsServerHandler(object):
                 logger.warn("Error getting profile for %s: %s", user_id, e)
             user_profiles.append(user_profile)
 
-        defer.returnValue(
-            {"chunk": user_profiles, "total_user_count_estimate": len(invited_users)}
-        )
+        return {"chunk": user_profiles, "total_user_count_estimate": len(invited_users)}
 
     @defer.inlineCallbacks
     def get_rooms_in_group(self, group_id, requester_user_id):
@@ -533,9 +527,7 @@ class GroupsServerHandler(object):
 
         chunk.sort(key=lambda e: -e["num_joined_members"])
 
-        defer.returnValue(
-            {"chunk": chunk, "total_room_count_estimate": len(room_results)}
-        )
+        return {"chunk": chunk, "total_room_count_estimate": len(room_results)}
 
     @defer.inlineCallbacks
     def add_room_to_group(self, group_id, requester_user_id, room_id, content):
@@ -551,7 +543,7 @@ class GroupsServerHandler(object):
 
         yield self.store.add_room_to_group(group_id, room_id, is_public=is_public)
 
-        defer.returnValue({})
+        return {}
 
     @defer.inlineCallbacks
     def update_room_in_group(
@@ -574,7 +566,7 @@ class GroupsServerHandler(object):
         else:
             raise SynapseError(400, "Uknown config option")
 
-        defer.returnValue({})
+        return {}
 
     @defer.inlineCallbacks
     def remove_room_from_group(self, group_id, requester_user_id, room_id):
@@ -586,7 +578,7 @@ class GroupsServerHandler(object):
 
         yield self.store.remove_room_from_group(group_id, room_id)
 
-        defer.returnValue({})
+        return {}
 
     @defer.inlineCallbacks
     def invite_to_group(self, group_id, user_id, requester_user_id, content):
@@ -644,9 +636,9 @@ class GroupsServerHandler(object):
             )
         elif res["state"] == "invite":
             yield self.store.add_group_invite(group_id, user_id)
-            defer.returnValue({"state": "invite"})
+            return {"state": "invite"}
         elif res["state"] == "reject":
-            defer.returnValue({"state": "reject"})
+            return {"state": "reject"}
         else:
             raise SynapseError(502, "Unknown state returned by HS")
 
@@ -679,7 +671,7 @@ class GroupsServerHandler(object):
             remote_attestation=remote_attestation,
         )
 
-        defer.returnValue(local_attestation)
+        return local_attestation
 
     @defer.inlineCallbacks
     def accept_invite(self, group_id, requester_user_id, content):
@@ -699,7 +691,7 @@ class GroupsServerHandler(object):
 
         local_attestation = yield self._add_user(group_id, requester_user_id, content)
 
-        defer.returnValue({"state": "join", "attestation": local_attestation})
+        return {"state": "join", "attestation": local_attestation}
 
     @defer.inlineCallbacks
     def join_group(self, group_id, requester_user_id, content):
@@ -716,7 +708,7 @@ class GroupsServerHandler(object):
 
         local_attestation = yield self._add_user(group_id, requester_user_id, content)
 
-        defer.returnValue({"state": "join", "attestation": local_attestation})
+        return {"state": "join", "attestation": local_attestation}
 
     @defer.inlineCallbacks
     def knock(self, group_id, requester_user_id, content):
@@ -769,7 +761,7 @@ class GroupsServerHandler(object):
         if not self.hs.is_mine_id(user_id):
             yield self.store.maybe_delete_remote_profile_cache(user_id)
 
-        defer.returnValue({})
+        return {}
 
     @defer.inlineCallbacks
     def create_group(self, group_id, requester_user_id, content):
@@ -845,7 +837,7 @@ class GroupsServerHandler(object):
                 avatar_url=user_profile.get("avatar_url"),
             )
 
-        defer.returnValue({"group_id": group_id})
+        return {"group_id": group_id}
 
     @defer.inlineCallbacks
     def delete_group(self, group_id, requester_user_id):

synapse/handlers/account_data.py

@@ -51,8 +51,8 @@ class AccountDataEventSource(object):
                     {"type": account_data_type, "content": content, "room_id": room_id}
                 )
 
-        defer.returnValue((results, current_stream_id))
+        return (results, current_stream_id)
 
     @defer.inlineCallbacks
     def get_pagination_rows(self, user, config, key):
-        defer.returnValue(([], config.to_id))
+        return ([], config.to_id)

synapse/handlers/account_validity.py

@@ -193,7 +193,7 @@ class AccountValidityHandler(object):
             if threepid["medium"] == "email":
                 addresses.append(threepid["address"])
 
-        defer.returnValue(addresses)
+        return addresses
 
     @defer.inlineCallbacks
     def _get_renewal_token(self, user_id):
@@ -214,7 +214,7 @@ class AccountValidityHandler(object):
             try:
                 renewal_token = stringutils.random_string(32)
                 yield self.store.set_renewal_token_for_user(user_id, renewal_token)
-                defer.returnValue(renewal_token)
+                return renewal_token
             except StoreError:
                 attempts += 1
         raise StoreError(500, "Couldn't generate a unique string as refresh string.")
@@ -226,11 +226,19 @@ class AccountValidityHandler(object):
 
         Args:
             renewal_token (str): Token sent with the renewal request.
+        Returns:
+            bool: Whether the provided token is valid.
         """
-        user_id = yield self.store.get_user_from_renewal_token(renewal_token)
+        try:
+            user_id = yield self.store.get_user_from_renewal_token(renewal_token)
+        except StoreError:
+            defer.returnValue(False)
+
         logger.debug("Renewing an account for user %s", user_id)
         yield self.renew_account_for_user(user_id)
 
+        defer.returnValue(True)
+
     @defer.inlineCallbacks
     def renew_account_for_user(self, user_id, expiration_ts=None, email_sent=False):
         """Renews the account attached to a given user by pushing back the
@@ -254,4 +262,4 @@ class AccountValidityHandler(object):
             user_id=user_id, expiration_ts=expiration_ts, email_sent=email_sent
         )
 
-        defer.returnValue(expiration_ts)
+        return expiration_ts

synapse/handlers/acme.py

@@ -100,4 +100,4 @@ class AcmeHandler(object):
             logger.exception("Failed saving!")
             raise
 
-        defer.returnValue(True)
+        return True

synapse/handlers/admin.py

@@ -49,7 +49,7 @@ class AdminHandler(BaseHandler):
             "devices": {"": {"sessions": [{"connections": connections}]}},
         }
 
-        defer.returnValue(ret)
+        return ret
 
     @defer.inlineCallbacks
     def get_users(self):
@@ -61,7 +61,7 @@ class AdminHandler(BaseHandler):
         """
         ret = yield self.store.get_users()
 
-        defer.returnValue(ret)
+        return ret
 
     @defer.inlineCallbacks
     def get_users_paginate(self, order, start, limit):
@@ -78,7 +78,7 @@ class AdminHandler(BaseHandler):
         """
         ret = yield self.store.get_users_paginate(order, start, limit)
 
-        defer.returnValue(ret)
+        return ret
 
     @defer.inlineCallbacks
     def search_users(self, term):
@@ -92,7 +92,17 @@ class AdminHandler(BaseHandler):
         """
         ret = yield self.store.search_users(term)
 
-        defer.returnValue(ret)
+        return ret
+
+    def set_user_server_admin(self, user, admin):
+        """
+        Set the admin bit on a user.
+
+        Args:
+            user_id (UserID): the (necessarily local) user to manipulate
+            admin (bool): whether or not the user should be an admin of this server
+        """
+        return self.store.set_server_admin(user, admin)
 
     @defer.inlineCallbacks
     def export_user_data(self, user_id, writer):
@@ -225,7 +235,7 @@ class AdminHandler(BaseHandler):
                 state = yield self.store.get_state_for_event(event_id)
                 writer.write_state(room_id, event_id, state)
 
-        defer.returnValue(writer.finished())
+        return writer.finished()
 
 
 class ExfiltrationWriter(object):

synapse/handlers/appservice.py

@@ -167,8 +167,8 @@ class ApplicationServicesHandler(object):
         for user_service in user_query_services:
             is_known_user = yield self.appservice_api.query_user(user_service, user_id)
             if is_known_user:
-                defer.returnValue(True)
-        defer.returnValue(False)
+                return True
+        return False
 
     @defer.inlineCallbacks
     def query_room_alias_exists(self, room_alias):
@@ -192,7 +192,7 @@ class ApplicationServicesHandler(object):
             if is_known_alias:
                 # the alias exists now so don't query more ASes.
                 result = yield self.store.get_association_from_room_alias(room_alias)
-                defer.returnValue(result)
+                return result
 
     @defer.inlineCallbacks
     def query_3pe(self, kind, protocol, fields):
@@ -215,7 +215,7 @@ class ApplicationServicesHandler(object):
             if success:
                 ret.extend(result)
 
-        defer.returnValue(ret)
+        return ret
 
     @defer.inlineCallbacks
     def get_3pe_protocols(self, only_protocol=None):
@@ -254,7 +254,7 @@ class ApplicationServicesHandler(object):
         for p in protocols.keys():
             protocols[p] = _merge_instances(protocols[p])
 
-        defer.returnValue(protocols)
+        return protocols
 
     @defer.inlineCallbacks
     def _get_services_for_event(self, event):
@@ -276,7 +276,7 @@ class ApplicationServicesHandler(object):
             if (yield s.is_interested(event, self.store)):
                 interested_list.append(s)
 
-        defer.returnValue(interested_list)
+        return interested_list
 
     def _get_services_for_user(self, user_id):
         services = self.store.get_app_services()
@@ -293,23 +293,23 @@ class ApplicationServicesHandler(object):
         if not self.is_mine_id(user_id):
             # we don't know if they are unknown or not since it isn't one of our
             # users. We can't poke ASes.
-            defer.returnValue(False)
+            return False
             return
 
         user_info = yield self.store.get_user_by_id(user_id)
         if user_info:
-            defer.returnValue(False)
+            return False
             return
 
         # user not found; could be the AS though, so check.
         services = self.store.get_app_services()
         service_list = [s for s in services if s.sender == user_id]
-        defer.returnValue(len(service_list) == 0)
+        return len(service_list) == 0
 
     @defer.inlineCallbacks
     def _check_user_exists(self, user_id):
         unknown_user = yield self._is_unknown_user(user_id)
         if unknown_user:
             exists = yield self.query_user_exists(user_id)
-            defer.returnValue(exists)
-        defer.returnValue(True)
+            return exists
+        return True

synapse/handlers/auth.py

@@ -155,7 +155,7 @@ class AuthHandler(BaseHandler):
         if user_id != requester.user.to_string():
             raise AuthError(403, "Invalid auth")
 
-        defer.returnValue(params)
+        return params
 
     @defer.inlineCallbacks
     def check_auth(self, flows, clientdict, clientip, password_servlet=False):
@@ -280,7 +280,7 @@ class AuthHandler(BaseHandler):
                     creds,
                     list(clientdict),
                 )
-                defer.returnValue((creds, clientdict, session["id"]))
+                return (creds, clientdict, session["id"])
 
         ret = self._auth_dict_for_flows(flows, session)
         ret["completed"] = list(creds)
@@ -307,8 +307,8 @@ class AuthHandler(BaseHandler):
         if result:
             creds[stagetype] = result
             self._save_session(sess)
-            defer.returnValue(True)
-        defer.returnValue(False)
+            return True
+        return False
 
     def get_session_id(self, clientdict):
         """
@@ -379,7 +379,7 @@ class AuthHandler(BaseHandler):
             res = yield checker(
                 authdict, clientip=clientip, password_servlet=password_servlet
             )
-            defer.returnValue(res)
+            return res
 
         # build a v1-login-style dict out of the authdict and fall back to the
         # v1 code
@@ -389,7 +389,7 @@ class AuthHandler(BaseHandler):
             raise SynapseError(400, "", Codes.MISSING_PARAM)
 
         (canonical_id, callback) = yield self.validate_login(user_id, authdict)
-        defer.returnValue(canonical_id)
+        return canonical_id
 
     @defer.inlineCallbacks
     def _check_recaptcha(self, authdict, clientip, **kwargs):
@@ -433,7 +433,7 @@ class AuthHandler(BaseHandler):
                 resp_body.get("hostname"),
             )
             if resp_body["success"]:
-                defer.returnValue(True)
+                return True
         raise LoginError(401, "", errcode=Codes.UNAUTHORIZED)
 
     def _check_email_identity(self, authdict, **kwargs):
@@ -502,7 +502,7 @@ class AuthHandler(BaseHandler):
 
         threepid["threepid_creds"] = authdict["threepid_creds"]
 
-        defer.returnValue(threepid)
+        return threepid
 
     def _get_params_recaptcha(self):
         return {"public_key": self.hs.config.recaptcha_public_key}
@@ -606,7 +606,7 @@ class AuthHandler(BaseHandler):
                 yield self.store.delete_access_token(access_token)
                 raise StoreError(400, "Login raced against device deletion")
 
-        defer.returnValue(access_token)
+        return access_token
 
     @defer.inlineCallbacks
     def check_user_exists(self, user_id):
@@ -629,8 +629,8 @@ class AuthHandler(BaseHandler):
         self.ratelimit_login_per_account(user_id)
         res = yield self._find_user_id_and_pwd_hash(user_id)
         if res is not None:
-            defer.returnValue(res[0])
-        defer.returnValue(None)
+            return res[0]
+        return None
 
     @defer.inlineCallbacks
     def _find_user_id_and_pwd_hash(self, user_id):
@@ -661,7 +661,7 @@ class AuthHandler(BaseHandler):
                 user_id,
                 user_infos.keys(),
             )
-        defer.returnValue(result)
+        return result
 
     def get_supported_login_types(self):
         """Get a the login types supported for the /login API
@@ -722,7 +722,7 @@ class AuthHandler(BaseHandler):
                 known_login_type = True
                 is_valid = yield provider.check_password(qualified_user_id, password)
                 if is_valid:
-                    defer.returnValue((qualified_user_id, None))
+                    return (qualified_user_id, None)
 
             if not hasattr(provider, "get_supported_login_types") or not hasattr(
                 provider, "check_auth"
@@ -756,7 +756,7 @@ class AuthHandler(BaseHandler):
             if result:
                 if isinstance(result, str):
                     result = (result, None)
-                defer.returnValue(result)
+                return result
 
         if login_type == LoginType.PASSWORD and self.hs.config.password_localdb_enabled:
             known_login_type = True
@@ -766,7 +766,7 @@ class AuthHandler(BaseHandler):
             )
 
             if canonical_user_id:
-                defer.returnValue((canonical_user_id, None))
+                return (canonical_user_id, None)
 
         if not known_login_type:
             raise SynapseError(400, "Unknown login type %s" % login_type)
@@ -814,9 +814,9 @@ class AuthHandler(BaseHandler):
                     if isinstance(result, str):
                         # If it's a str, set callback function to None
                         result = (result, None)
-                    defer.returnValue(result)
+                    return result
 
-        defer.returnValue((None, None))
+        return (None, None)
 
     @defer.inlineCallbacks
     def _check_local_password(self, user_id, password):
@@ -838,7 +838,7 @@ class AuthHandler(BaseHandler):
         """
         lookupres = yield self._find_user_id_and_pwd_hash(user_id)
         if not lookupres:
-            defer.returnValue(None)
+            return None
         (user_id, password_hash) = lookupres
 
         # If the password hash is None, the account has likely been deactivated
@@ -850,8 +850,8 @@ class AuthHandler(BaseHandler):
         result = yield self.validate_hash(password, password_hash)
         if not result:
             logger.warn("Failed password login for user %s", user_id)
-            defer.returnValue(None)
-        defer.returnValue(user_id)
+            return None
+        return user_id
 
     @defer.inlineCallbacks
     def validate_short_term_login_token_and_get_user_id(self, login_token):
@@ -860,12 +860,12 @@ class AuthHandler(BaseHandler):
         try:
             macaroon = pymacaroons.Macaroon.deserialize(login_token)
             user_id = auth_api.get_user_id_from_macaroon(macaroon)
-            auth_api.validate_macaroon(macaroon, "login", True, user_id)
+            auth_api.validate_macaroon(macaroon, "login", user_id)
         except Exception:
             raise AuthError(403, "Invalid token", errcode=Codes.FORBIDDEN)
         self.ratelimit_login_per_account(user_id)
         yield self.auth.check_auth_blocking(user_id)
-        defer.returnValue(user_id)
+        return user_id
 
     @defer.inlineCallbacks
     def delete_access_token(self, access_token):
@@ -976,7 +976,7 @@ class AuthHandler(BaseHandler):
         )
 
         yield self.store.user_delete_threepid(user_id, medium, address)
-        defer.returnValue(result)
+        return result
 
     def _save_session(self, session):
         # TODO: Persistent storage

synapse/handlers/deactivate_account.py

@@ -125,7 +125,7 @@ class DeactivateAccountHandler(BaseHandler):
         # Mark the user as deactivated.
         yield self.store.set_user_deactivated_status(user_id, True)
 
-        defer.returnValue(identity_server_supports_unbinding)
+        return identity_server_supports_unbinding
 
     def _start_user_parting(self):
         """

synapse/handlers/device.py

@@ -64,7 +64,7 @@ class DeviceWorkerHandler(BaseHandler):
         for device in devices:
             _update_device_from_client_ips(device, ips)
 
-        defer.returnValue(devices)
+        return devices
 
     @defer.inlineCallbacks
     def get_device(self, user_id, device_id):
@@ -85,7 +85,7 @@ class DeviceWorkerHandler(BaseHandler):
             raise errors.NotFoundError
         ips = yield self.store.get_last_client_ip_by_device(user_id, device_id)
         _update_device_from_client_ips(device, ips)
-        defer.returnValue(device)
+        return device
 
     @measure_func("device.get_user_ids_changed")
     @defer.inlineCallbacks
@@ -200,9 +200,7 @@ class DeviceWorkerHandler(BaseHandler):
             possibly_joined = []
             possibly_left = []
 
-        defer.returnValue(
-            {"changed": list(possibly_joined), "left": list(possibly_left)}
-        )
+        return {"changed": list(possibly_joined), "left": list(possibly_left)}
 
 
 class DeviceHandler(DeviceWorkerHandler):
@@ -211,12 +209,12 @@ class DeviceHandler(DeviceWorkerHandler):
 
         self.federation_sender = hs.get_federation_sender()
 
-        self._edu_updater = DeviceListEduUpdater(hs, self)
+        self.device_list_updater = DeviceListUpdater(hs, self)
 
         federation_registry = hs.get_federation_registry()
 
         federation_registry.register_edu_handler(
-            "m.device_list_update", self._edu_updater.incoming_device_list_update
+            "m.device_list_update", self.device_list_updater.incoming_device_list_update
         )
         federation_registry.register_query_handler(
             "user_devices", self.on_federation_query_user_devices
@@ -250,7 +248,7 @@ class DeviceHandler(DeviceWorkerHandler):
             )
             if new_device:
                 yield self.notify_device_update(user_id, [device_id])
-            defer.returnValue(device_id)
+            return device_id
 
         # if the device id is not specified, we'll autogen one, but loop a few
         # times in case of a clash.
@@ -264,7 +262,7 @@ class DeviceHandler(DeviceWorkerHandler):
             )
             if new_device:
                 yield self.notify_device_update(user_id, [device_id])
-                defer.returnValue(device_id)
+                return device_id
             attempts += 1
 
         raise errors.StoreError(500, "Couldn't generate a device ID.")
@@ -411,9 +409,7 @@ class DeviceHandler(DeviceWorkerHandler):
     @defer.inlineCallbacks
     def on_federation_query_user_devices(self, user_id):
         stream_id, devices = yield self.store.get_devices_with_keys_by_user(user_id)
-        defer.returnValue(
-            {"user_id": user_id, "stream_id": stream_id, "devices": devices}
-        )
+        return {"user_id": user_id, "stream_id": stream_id, "devices": devices}
 
     @defer.inlineCallbacks
     def user_left_room(self, user, room_id):
@@ -430,7 +426,7 @@ def _update_device_from_client_ips(device, client_ips):
     device.update({"last_seen_ts": ip.get("last_seen"), "last_seen_ip": ip.get("ip")})
 
 
-class DeviceListEduUpdater(object):
+class DeviceListUpdater(object):
     "Handles incoming device list updates from federation and updates the DB"
 
     def __init__(self, hs, device_handler):
@@ -523,75 +519,7 @@ class DeviceListEduUpdater(object):
             logger.debug("Need to re-sync devices for %r? %r", user_id, resync)
 
             if resync:
-                # Fetch all devices for the user.
-                origin = get_domain_from_id(user_id)
-                try:
-                    result = yield self.federation.query_user_devices(origin, user_id)
-                except (
-                    NotRetryingDestination,
-                    RequestSendFailed,
-                    HttpResponseException,
-                ):
-                    # TODO: Remember that we are now out of sync and try again
-                    # later
-                    logger.warn("Failed to handle device list update for %s", user_id)
-                    # We abort on exceptions rather than accepting the update
-                    # as otherwise synapse will 'forget' that its device list
-                    # is out of date. If we bail then we will retry the resync
-                    # next time we get a device list update for this user_id.
-                    # This makes it more likely that the device lists will
-                    # eventually become consistent.
-                    return
-                except FederationDeniedError as e:
-                    logger.info(e)
-                    return
-                except Exception:
-                    # TODO: Remember that we are now out of sync and try again
-                    # later
-                    logger.exception(
-                        "Failed to handle device list update for %s", user_id
-                    )
-                    return
-
-                stream_id = result["stream_id"]
-                devices = result["devices"]
-
-                # If the remote server has more than ~1000 devices for this user
-                # we assume that something is going horribly wrong (e.g. a bot
-                # that logs in and creates a new device every time it tries to
-                # send a message).  Maintaining lots of devices per user in the
-                # cache can cause serious performance issues as if this request
-                # takes more than 60s to complete, internal replication from the
-                # inbound federation worker to the synapse master may time out
-                # causing the inbound federation to fail and causing the remote
-                # server to retry, causing a DoS.  So in this scenario we give
-                # up on storing the total list of devices and only handle the
-                # delta instead.
-                if len(devices) > 1000:
-                    logger.warn(
-                        "Ignoring device list snapshot for %s as it has >1K devs (%d)",
-                        user_id,
-                        len(devices),
-                    )
-                    devices = []
-
-                for device in devices:
-                    logger.debug(
-                        "Handling resync update %r/%r, ID: %r",
-                        user_id,
-                        device["device_id"],
-                        stream_id,
-                    )
-
-                yield self.store.update_remote_device_list_cache(
-                    user_id, devices, stream_id
-                )
-                device_ids = [device["device_id"] for device in devices]
-                yield self.device_handler.notify_device_update(user_id, device_ids)
-
-                # We clobber the seen updates since we've re-synced from a given
-                # point.
-                self._seen_updates[user_id] = set([stream_id])
+                yield self.user_device_resync(user_id)
             else:
                 # Simply update the single device, since we know that is the only
                 # change (because of the single prev_id matching the current cache)
@@ -623,7 +551,7 @@ class DeviceListEduUpdater(object):
         for _, stream_id, prev_ids, _ in updates:
             if not prev_ids:
                 # We always do a resync if there are no previous IDs
-                defer.returnValue(True)
+                return True
 
             for prev_id in prev_ids:
                 if prev_id == extremity:
@@ -633,8 +561,82 @@ class DeviceListEduUpdater(object):
                 elif prev_id in stream_id_in_updates:
                     continue
                 else:
-                    defer.returnValue(True)
+                    return True
 
             stream_id_in_updates.add(stream_id)
 
-        defer.returnValue(False)
+        return False
+
+    @defer.inlineCallbacks
+    def user_device_resync(self, user_id):
+        """Fetches all devices for a user and updates the device cache with them.
+
+        Args:
+            user_id (str): The user's id whose device_list will be updated.
+        Returns:
+            Deferred[dict]: a dict with device info as under the "devices" in the result of this
+            request:
+            https://matrix.org/docs/spec/server_server/r0.1.2#get-matrix-federation-v1-user-devices-userid
+        """
+        # Fetch all devices for the user.
+        origin = get_domain_from_id(user_id)
+        try:
+            result = yield self.federation.query_user_devices(origin, user_id)
+        except (NotRetryingDestination, RequestSendFailed, HttpResponseException):
+            # TODO: Remember that we are now out of sync and try again
+            # later
+            logger.warn("Failed to handle device list update for %s", user_id)
+            # We abort on exceptions rather than accepting the update
+            # as otherwise synapse will 'forget' that its device list
+            # is out of date. If we bail then we will retry the resync
+            # next time we get a device list update for this user_id.
+            # This makes it more likely that the device lists will
+            # eventually become consistent.
+            return
+        except FederationDeniedError as e:
+            logger.info(e)
+            return
+        except Exception:
+            # TODO: Remember that we are now out of sync and try again
+            # later
+            logger.exception("Failed to handle device list update for %s", user_id)
+            return
+        stream_id = result["stream_id"]
+        devices = result["devices"]
+
+        # If the remote server has more than ~1000 devices for this user
+        # we assume that something is going horribly wrong (e.g. a bot
+        # that logs in and creates a new device every time it tries to
+        # send a message).  Maintaining lots of devices per user in the
+        # cache can cause serious performance issues as if this request
+        # takes more than 60s to complete, internal replication from the
+        # inbound federation worker to the synapse master may time out
+        # causing the inbound federation to fail and causing the remote
+        # server to retry, causing a DoS.  So in this scenario we give
+        # up on storing the total list of devices and only handle the
+        # delta instead.
+        if len(devices) > 1000:
+            logger.warn(
+                "Ignoring device list snapshot for %s as it has >1K devs (%d)",
+                user_id,
+                len(devices),
+            )
+            devices = []
+
+        for device in devices:
+            logger.debug(
+                "Handling resync update %r/%r, ID: %r",
+                user_id,
+                device["device_id"],
+                stream_id,
+            )
+
+        yield self.store.update_remote_device_list_cache(user_id, devices, stream_id)
+        device_ids = [device["device_id"] for device in devices]
+        yield self.device_handler.notify_device_update(user_id, device_ids)
+
+        # We clobber the seen updates since we've re-synced from a given
+        # point.
+        self._seen_updates[user_id] = set([stream_id])
+
+        defer.returnValue(result)

synapse/handlers/directory.py

@@ -210,7 +210,7 @@ class DirectoryHandler(BaseHandler):
         except AuthError as e:
             logger.info("Failed to update alias events: %s", e)
 
-        defer.returnValue(room_id)
+        return room_id
 
     @defer.inlineCallbacks
     def delete_appservice_association(self, service, room_alias):
@@ -229,7 +229,7 @@ class DirectoryHandler(BaseHandler):
 
         room_id = yield self.store.delete_room_alias(room_alias)
 
-        defer.returnValue(room_id)
+        return room_id
 
     @defer.inlineCallbacks
     def get_association(self, room_alias):
@@ -277,8 +277,7 @@ class DirectoryHandler(BaseHandler):
         else:
             servers = list(servers)
 
-        defer.returnValue({"room_id": room_id, "servers": servers})
-        return
+        return {"room_id": room_id, "servers": servers}
 
     @defer.inlineCallbacks
     def on_directory_query(self, args):
@@ -289,7 +288,7 @@ class DirectoryHandler(BaseHandler):
         result = yield self.get_association_from_room_alias(room_alias)
 
         if result is not None:
-            defer.returnValue({"room_id": result.room_id, "servers": result.servers})
+            return {"room_id": result.room_id, "servers": result.servers}
         else:
             raise SynapseError(
                 404,
@@ -342,7 +341,7 @@ class DirectoryHandler(BaseHandler):
             # Query AS to see if it exists
             as_handler = self.appservice_handler
             result = yield as_handler.query_room_alias_exists(room_alias)
-        defer.returnValue(result)
+        return result
 
     def can_modify_alias(self, alias, user_id=None):
         # Any application service "interested" in an alias they are regexing on
@@ -369,10 +368,10 @@ class DirectoryHandler(BaseHandler):
         creator = yield self.store.get_room_alias_creator(alias.to_string())
 
         if creator is not None and creator == user_id:
-            defer.returnValue(True)
+            return True
 
         is_admin = yield self.auth.is_server_admin(UserID.from_string(user_id))
-        defer.returnValue(is_admin)
+        return is_admin
 
     @defer.inlineCallbacks
     def edit_published_room_list(self, requester, room_id, visibility):

synapse/handlers/e2e_keys.py

@@ -25,6 +25,7 @@ from twisted.internet import defer
 from synapse.api.errors import CodeMessageException, SynapseError
 from synapse.logging.context import make_deferred_yieldable, run_in_background
 from synapse.types import UserID, get_domain_from_id
+from synapse.util import unwrapFirstError
 from synapse.util.retryutils import NotRetryingDestination
 
 logger = logging.getLogger(__name__)
@@ -65,6 +66,7 @@ class E2eKeysHandler(object):
             }
         }
         """
+
         device_keys_query = query_body.get("device_keys", {})
 
         # separate users by domain.
@@ -121,7 +123,56 @@ class E2eKeysHandler(object):
         # Now fetch any devices that we don't have in our cache
         @defer.inlineCallbacks
         def do_remote_query(destination):
+            """This is called when we are querying the device list of a user on
+            a remote homeserver and their device list is not in the device list
+            cache. If we share a room with this user and we're not querying for
+            specific user we will update the cache
+            with their device list."""
+
             destination_query = remote_queries_not_in_cache[destination]
+
+            # We first consider whether we wish to update the device list cache with
+            # the users device list. We want to track a user's devices when the
+            # authenticated user shares a room with the queried user and the query
+            # has not specified a particular device.
+            # If we update the cache for the queried user we remove them from further
+            # queries. We use the more efficient batched query_client_keys for all
+            # remaining users
+            user_ids_updated = []
+            for (user_id, device_list) in destination_query.items():
+                if user_id in user_ids_updated:
+                    continue
+
+                if device_list:
+                    continue
+
+                room_ids = yield self.store.get_rooms_for_user(user_id)
+                if not room_ids:
+                    continue
+
+                # We've decided we're sharing a room with this user and should
+                # probably be tracking their device lists. However, we haven't
+                # done an initial sync on the device list so we do it now.
+                try:
+                    user_devices = yield self.device_handler.device_list_updater.user_device_resync(
+                        user_id
+                    )
+                    user_devices = user_devices["devices"]
+                    for device in user_devices:
+                        results[user_id] = {device["device_id"]: device["keys"]}
+                    user_ids_updated.append(user_id)
+                except Exception as e:
+                    failures[destination] = _exception_to_failure(e)
+
+            if len(destination_query) == len(user_ids_updated):
+                # We've updated all the users in the query and we do not need to
+                # make any further remote calls.
+                return
+
+            # Remove all the users from the query which we have updated
+            for user_id in user_ids_updated:
+                destination_query.pop(user_id)
+
             try:
                 remote_result = yield self.federation.query_client_keys(
                     destination, {"device_keys": destination_query}, timeout=timeout
@@ -132,7 +183,8 @@ class E2eKeysHandler(object):
                         results[user_id] = keys
 
             except Exception as e:
-                failures[destination] = _exception_to_failure(e)
+                failure = _exception_to_failure(e)
+                failures[destination] = failure
 
         yield make_deferred_yieldable(
             defer.gatherResults(
@@ -141,10 +193,10 @@ class E2eKeysHandler(object):
                     for destination in remote_queries_not_in_cache
                 ],
                 consumeErrors=True,
-            )
+            ).addErrback(unwrapFirstError)
         )
 
-        defer.returnValue({"device_keys": results, "failures": failures})
+        return {"device_keys": results, "failures": failures}
 
     @defer.inlineCallbacks
     def query_local_devices(self, query):
@@ -189,7 +241,7 @@ class E2eKeysHandler(object):
                     r["unsigned"]["device_display_name"] = display_name
                 result_dict[user_id][device_id] = r
 
-        defer.returnValue(result_dict)
+        return result_dict
 
     @defer.inlineCallbacks
     def on_federation_query_client_keys(self, query_body):
@@ -197,7 +249,7 @@ class E2eKeysHandler(object):
         """
         device_keys_query = query_body.get("device_keys", {})
         res = yield self.query_local_devices(device_keys_query)
-        defer.returnValue({"device_keys": res})
+        return {"device_keys": res}
 
     @defer.inlineCallbacks
     def claim_one_time_keys(self, query, timeout):
@@ -234,8 +286,10 @@ class E2eKeysHandler(object):
                 for user_id, keys in remote_result["one_time_keys"].items():
                     if user_id in device_keys:
                         json_result[user_id] = keys
+
             except Exception as e:
-                failures[destination] = _exception_to_failure(e)
+                failure = _exception_to_failure(e)
+                failures[destination] = failure
 
         yield make_deferred_yieldable(
             defer.gatherResults(
@@ -259,10 +313,11 @@ class E2eKeysHandler(object):
             ),
         )
 
-        defer.returnValue({"one_time_keys": json_result, "failures": failures})
+        return {"one_time_keys": json_result, "failures": failures}
 
     @defer.inlineCallbacks
     def upload_keys_for_user(self, user_id, device_id, keys):
+
         time_now = self.clock.time_msec()
 
         # TODO: Validate the JSON to make sure it has the right keys.
@@ -297,7 +352,7 @@ class E2eKeysHandler(object):
 
         result = yield self.store.count_e2e_one_time_keys(user_id, device_id)
 
-        defer.returnValue({"one_time_key_counts": result})
+        return {"one_time_key_counts": result}
 
     @defer.inlineCallbacks
     def _upload_one_time_keys_for_user(

synapse/handlers/e2e_room_keys.py

@@ -84,7 +84,7 @@ class E2eRoomKeysHandler(object):
                 user_id, version, room_id, session_id
             )
 
-            defer.returnValue(results)
+            return results
 
     @defer.inlineCallbacks
     def delete_room_keys(self, user_id, version, room_id=None, session_id=None):
@@ -262,7 +262,7 @@ class E2eRoomKeysHandler(object):
             new_version = yield self.store.create_e2e_room_keys_version(
                 user_id, version_info
             )
-            defer.returnValue(new_version)
+            return new_version
 
     @defer.inlineCallbacks
     def get_version_info(self, user_id, version=None):
@@ -292,7 +292,7 @@ class E2eRoomKeysHandler(object):
                     raise NotFoundError("Unknown backup version")
                 else:
                     raise
-            defer.returnValue(res)
+            return res
 
     @defer.inlineCallbacks
     def delete_version(self, user_id, version=None):
@@ -350,4 +350,4 @@ class E2eRoomKeysHandler(object):
                 user_id, version, version_info
             )
 
-            defer.returnValue({})
+            return {}

synapse/handlers/events.py

@@ -143,7 +143,7 @@ class EventStreamHandler(BaseHandler):
                 "end": tokens[1].to_string(),
             }
 
-            defer.returnValue(chunk)
+            return chunk
 
 
 class EventHandler(BaseHandler):
@@ -166,7 +166,7 @@ class EventHandler(BaseHandler):
         event = yield self.store.get_event(event_id, check_room_id=room_id)
 
         if not event:
-            defer.returnValue(None)
+            return None
             return
 
         users = yield self.store.get_users_in_room(event.room_id)
@@ -179,4 +179,4 @@ class EventHandler(BaseHandler):
         if not filtered:
             raise AuthError(403, "You don't have permission to access that event.")
 
-        defer.returnValue(event)
+        return event

synapse/handlers/federation.py

@@ -210,7 +210,7 @@ class FederationHandler(BaseHandler):
                 event_id,
                 origin,
             )
-            defer.returnValue(None)
+            return None
 
         state = None
         auth_chain = []
@@ -676,7 +676,7 @@ class FederationHandler(BaseHandler):
         events = [e for e in events if e.event_id not in seen_events]
 
         if not events:
-            defer.returnValue([])
+            return []
 
         event_map = {e.event_id: e for e in events}
 
@@ -838,7 +838,7 @@ class FederationHandler(BaseHandler):
             # TODO: We can probably do something more clever here.
             yield self._handle_new_event(dest, event, backfilled=True)
 
-        defer.returnValue(events)
+        return events
 
     @defer.inlineCallbacks
     def maybe_backfill(self, room_id, current_depth):
@@ -894,7 +894,7 @@ class FederationHandler(BaseHandler):
         )
 
         if not filtered_extremities:
-            defer.returnValue(False)
+            return False
 
         # Check if we reached a point where we should start backfilling.
         sorted_extremeties_tuple = sorted(extremities.items(), key=lambda e: -int(e[1]))
@@ -965,7 +965,7 @@ class FederationHandler(BaseHandler):
                     # If this succeeded then we probably already have the
                     # appropriate stuff.
                     # TODO: We can probably do something more intelligent here.
-                    defer.returnValue(True)
+                    return True
                 except SynapseError as e:
                     logger.info("Failed to backfill from %s because %s", dom, e)
                     continue
@@ -978,6 +978,9 @@ class FederationHandler(BaseHandler):
                 except NotRetryingDestination as e:
                     logger.info(str(e))
                     continue
+                except RequestSendFailed as e:
+                    logger.info("Falied to get backfill from %s because %s", dom, e)
+                    continue
                 except FederationDeniedError as e:
                     logger.info(e)
                     continue
@@ -985,11 +988,11 @@ class FederationHandler(BaseHandler):
                     logger.exception("Failed to backfill from %s because %s", dom, e)
                     continue
 
-            defer.returnValue(False)
+            return False
 
         success = yield try_backfill(likely_domains)
         if success:
-            defer.returnValue(True)
+            return True
 
         # Huh, well *those* domains didn't work out. Lets try some domains
         # from the time.
@@ -1031,11 +1034,11 @@ class FederationHandler(BaseHandler):
                 [dom for dom, _ in likely_domains if dom not in tried_domains]
             )
             if success:
-                defer.returnValue(True)
+                return True
 
             tried_domains.update(dom for dom, _ in likely_domains)
 
-        defer.returnValue(False)
+        return False
 
     def _sanity_check_event(self, ev):
         """
@@ -1082,7 +1085,7 @@ class FederationHandler(BaseHandler):
             pdu=event,
         )
 
-        defer.returnValue(pdu)
+        return pdu
 
     @defer.inlineCallbacks
     def on_event_auth(self, event_id):
@@ -1090,7 +1093,7 @@ class FederationHandler(BaseHandler):
         auth = yield self.store.get_auth_chain(
             [auth_id for auth_id in event.auth_event_ids()], include_given=True
         )
-        defer.returnValue([e for e in auth])
+        return [e for e in auth]
 
     @log_function
     @defer.inlineCallbacks
@@ -1177,7 +1180,7 @@ class FederationHandler(BaseHandler):
 
             run_in_background(self._handle_queued_pdus, room_queue)
 
-        defer.returnValue(True)
+        return True
 
     @defer.inlineCallbacks
     def _handle_queued_pdus(self, room_queue):
@@ -1264,7 +1267,7 @@ class FederationHandler(BaseHandler):
             room_version, event, context, do_sig_check=False
         )
 
-        defer.returnValue(event)
+        return event
 
     @defer.inlineCallbacks
     @log_function
@@ -1325,7 +1328,7 @@ class FederationHandler(BaseHandler):
 
         state = yield self.store.get_events(list(prev_state_ids.values()))
 
-        defer.returnValue({"state": list(state.values()), "auth_chain": auth_chain})
+        return {"state": list(state.values()), "auth_chain": auth_chain}
 
     @defer.inlineCallbacks
     def on_invite_request(self, origin, pdu):
@@ -1381,7 +1384,7 @@ class FederationHandler(BaseHandler):
         context = yield self.state_handler.compute_event_context(event)
         yield self.persist_events_and_notify([(event, context)])
 
-        defer.returnValue(event)
+        return event
 
     @defer.inlineCallbacks
     def do_remotely_reject_invite(self, target_hosts, room_id, user_id):
@@ -1406,7 +1409,7 @@ class FederationHandler(BaseHandler):
         context = yield self.state_handler.compute_event_context(event)
         yield self.persist_events_and_notify([(event, context)])
 
-        defer.returnValue(event)
+        return event
 
     @defer.inlineCallbacks
     def _make_and_verify_event(
@@ -1424,7 +1427,7 @@ class FederationHandler(BaseHandler):
         assert event.user_id == user_id
         assert event.state_key == user_id
         assert event.room_id == room_id
-        defer.returnValue((origin, event, format_ver))
+        return (origin, event, format_ver)
 
     @defer.inlineCallbacks
     @log_function
@@ -1484,7 +1487,7 @@ class FederationHandler(BaseHandler):
             logger.warn("Failed to create new leave %r because %s", event, e)
             raise e
 
-        defer.returnValue(event)
+        return event
 
     @defer.inlineCallbacks
     @log_function
@@ -1517,7 +1520,7 @@ class FederationHandler(BaseHandler):
             event.signatures,
         )
 
-        defer.returnValue(None)
+        return None
 
     @defer.inlineCallbacks
     def get_state_for_pdu(self, room_id, event_id):
@@ -1545,9 +1548,9 @@ class FederationHandler(BaseHandler):
                     del results[(event.type, event.state_key)]
 
             res = list(results.values())
-            defer.returnValue(res)
+            return res
         else:
-            defer.returnValue([])
+            return []
 
     @defer.inlineCallbacks
     def get_state_ids_for_pdu(self, room_id, event_id):
@@ -1572,9 +1575,9 @@ class FederationHandler(BaseHandler):
                 else:
                     results.pop((event.type, event.state_key), None)
 
-            defer.returnValue(list(results.values()))
+            return list(results.values())
         else:
-            defer.returnValue([])
+            return []
 
     @defer.inlineCallbacks
     @log_function
@@ -1587,7 +1590,7 @@ class FederationHandler(BaseHandler):
 
         events = yield filter_events_for_server(self.store, origin, events)
 
-        defer.returnValue(events)
+        return events
 
     @defer.inlineCallbacks
     @log_function
@@ -1617,9 +1620,9 @@ class FederationHandler(BaseHandler):
 
             events = yield filter_events_for_server(self.store, origin, [event])
             event = events[0]
-            defer.returnValue(event)
+            return event
         else:
-            defer.returnValue(None)
+            return None
 
     def get_min_depth_for_context(self, context):
         return self.store.get_min_depth(context)
@@ -1651,7 +1654,7 @@ class FederationHandler(BaseHandler):
                     self.store.remove_push_actions_from_staging, event.event_id
                 )
 
-        defer.returnValue(context)
+        return context
 
     @defer.inlineCallbacks
     def _handle_new_events(self, origin, event_infos, backfilled=False):
@@ -1674,7 +1677,7 @@ class FederationHandler(BaseHandler):
                     auth_events=ev_info.get("auth_events"),
                     backfilled=backfilled,
                 )
-            defer.returnValue(res)
+            return res
 
         contexts = yield make_deferred_yieldable(
             defer.gatherResults(
@@ -1833,7 +1836,7 @@ class FederationHandler(BaseHandler):
         if event.type == EventTypes.GuestAccess and not context.rejected:
             yield self.maybe_kick_guest_users(event)
 
-        defer.returnValue(context)
+        return context
 
     @defer.inlineCallbacks
     def _check_for_soft_fail(self, event, state, backfilled):
@@ -1952,7 +1955,7 @@ class FederationHandler(BaseHandler):
 
         logger.debug("on_query_auth returning: %s", ret)
 
-        defer.returnValue(ret)
+        return ret
 
     @defer.inlineCallbacks
     def on_get_missing_events(
@@ -1975,7 +1978,7 @@ class FederationHandler(BaseHandler):
             self.store, origin, missing_events
         )
 
-        defer.returnValue(missing_events)
+        return missing_events
 
     @defer.inlineCallbacks
     @log_function
@@ -2451,16 +2454,14 @@ class FederationHandler(BaseHandler):
 
         logger.debug("construct_auth_difference returning")
 
-        defer.returnValue(
-            {
-                "auth_chain": local_auth,
-                "rejects": {
-                    e.event_id: {"reason": reason_map[e.event_id], "proof": None}
-                    for e in base_remote_rejected
-                },
-                "missing": [e.event_id for e in missing_locals],
-            }
-        )
+        return {
+            "auth_chain": local_auth,
+            "rejects": {
+                e.event_id: {"reason": reason_map[e.event_id], "proof": None}
+                for e in base_remote_rejected
+            },
+            "missing": [e.event_id for e in missing_locals],
+        }
 
     @defer.inlineCallbacks
     @log_function
@@ -2608,7 +2609,7 @@ class FederationHandler(BaseHandler):
             builder=builder
         )
         EventValidator().validate_new(event)
-        defer.returnValue((event, context))
+        return (event, context)
 
     @defer.inlineCallbacks
     def _check_signature(self, event, context):

synapse/handlers/groups_local.py

@@ -126,9 +126,12 @@ class GroupsLocalHandler(object):
                 group_id, requester_user_id
             )
         else:
-            res = yield self.transport_client.get_group_summary(
-                get_domain_from_id(group_id), group_id, requester_user_id
-            )
+            try:
+                res = yield self.transport_client.get_group_summary(
+                    get_domain_from_id(group_id), group_id, requester_user_id
+                )
+            except RequestSendFailed:
+                raise SynapseError(502, "Failed to contact group server")
 
             group_server_name = get_domain_from_id(group_id)
 
@@ -162,7 +165,7 @@ class GroupsLocalHandler(object):
 
         res.setdefault("user", {})["is_publicised"] = is_publicised
 
-        defer.returnValue(res)
+        return res
 
     @defer.inlineCallbacks
     def create_group(self, group_id, user_id, content):
@@ -183,9 +186,12 @@ class GroupsLocalHandler(object):
 
             content["user_profile"] = yield self.profile_handler.get_profile(user_id)
 
-            res = yield self.transport_client.create_group(
-                get_domain_from_id(group_id), group_id, user_id, content
-            )
+            try:
+                res = yield self.transport_client.create_group(
+                    get_domain_from_id(group_id), group_id, user_id, content
+                )
+            except RequestSendFailed:
+                raise SynapseError(502, "Failed to contact group server")
 
             remote_attestation = res["attestation"]
             yield self.attestations.verify_attestation(
@@ -207,7 +213,7 @@ class GroupsLocalHandler(object):
         )
         self.notifier.on_new_event("groups_key", token, users=[user_id])
 
-        defer.returnValue(res)
+        return res
 
     @defer.inlineCallbacks
     def get_users_in_group(self, group_id, requester_user_id):
@@ -217,13 +223,16 @@ class GroupsLocalHandler(object):
             res = yield self.groups_server_handler.get_users_in_group(
                 group_id, requester_user_id
             )
-            defer.returnValue(res)
+            return res
 
         group_server_name = get_domain_from_id(group_id)
 
-        res = yield self.transport_client.get_users_in_group(
-            get_domain_from_id(group_id), group_id, requester_user_id
-        )
+        try:
+            res = yield self.transport_client.get_users_in_group(
+                get_domain_from_id(group_id), group_id, requester_user_id
+            )
+        except RequestSendFailed:
+            raise SynapseError(502, "Failed to contact group server")
 
         chunk = res["chunk"]
         valid_entries = []
@@ -244,7 +253,7 @@ class GroupsLocalHandler(object):
 
         res["chunk"] = valid_entries
 
-        defer.returnValue(res)
+        return res
 
     @defer.inlineCallbacks
     def join_group(self, group_id, user_id, content):
@@ -258,9 +267,12 @@ class GroupsLocalHandler(object):
             local_attestation = self.attestations.create_attestation(group_id, user_id)
             content["attestation"] = local_attestation
 
-            res = yield self.transport_client.join_group(
-                get_domain_from_id(group_id), group_id, user_id, content
-            )
+            try:
+                res = yield self.transport_client.join_group(
+                    get_domain_from_id(group_id), group_id, user_id, content
+                )
+            except RequestSendFailed:
+                raise SynapseError(502, "Failed to contact group server")
 
             remote_attestation = res["attestation"]
 
@@ -285,7 +297,7 @@ class GroupsLocalHandler(object):
         )
         self.notifier.on_new_event("groups_key", token, users=[user_id])
 
-        defer.returnValue({})
+        return {}
 
     @defer.inlineCallbacks
     def accept_invite(self, group_id, user_id, content):
@@ -299,9 +311,12 @@ class GroupsLocalHandler(object):
             local_attestation = self.attestations.create_attestation(group_id, user_id)
             content["attestation"] = local_attestation
 
-            res = yield self.transport_client.accept_group_invite(
-                get_domain_from_id(group_id), group_id, user_id, content
-            )
+            try:
+                res = yield self.transport_client.accept_group_invite(
+                    get_domain_from_id(group_id), group_id, user_id, content
+                )
+            except RequestSendFailed:
+                raise SynapseError(502, "Failed to contact group server")
 
             remote_attestation = res["attestation"]
 
@@ -326,7 +341,7 @@ class GroupsLocalHandler(object):
         )
         self.notifier.on_new_event("groups_key", token, users=[user_id])
 
-        defer.returnValue({})
+        return {}
 
     @defer.inlineCallbacks
     def invite(self, group_id, user_id, requester_user_id, config):
@@ -338,15 +353,18 @@ class GroupsLocalHandler(object):
                 group_id, user_id, requester_user_id, content
             )
         else:
-            res = yield self.transport_client.invite_to_group(
-                get_domain_from_id(group_id),
-                group_id,
-                user_id,
-                requester_user_id,
-                content,
-            )
+            try:
+                res = yield self.transport_client.invite_to_group(
+                    get_domain_from_id(group_id),
+                    group_id,
+                    user_id,
+                    requester_user_id,
+                    content,
+                )
+            except RequestSendFailed:
+                raise SynapseError(502, "Failed to contact group server")
 
-        defer.returnValue(res)
+        return res
 
     @defer.inlineCallbacks
     def on_invite(self, group_id, user_id, content):
@@ -377,7 +395,7 @@ class GroupsLocalHandler(object):
             logger.warn("No profile for user %s: %s", user_id, e)
             user_profile = {}
 
-        defer.returnValue({"state": "invite", "user_profile": user_profile})
+        return {"state": "invite", "user_profile": user_profile}
 
     @defer.inlineCallbacks
     def remove_user_from_group(self, group_id, user_id, requester_user_id, content):
@@ -398,15 +416,18 @@ class GroupsLocalHandler(object):
             )
         else:
             content["requester_user_id"] = requester_user_id
-            res = yield self.transport_client.remove_user_from_group(
-                get_domain_from_id(group_id),
-                group_id,
-                requester_user_id,
-                user_id,
-                content,
-            )
+            try:
+                res = yield self.transport_client.remove_user_from_group(
+                    get_domain_from_id(group_id),
+                    group_id,
+                    requester_user_id,
+                    user_id,
+                    content,
+                )
+            except RequestSendFailed:
+                raise SynapseError(502, "Failed to contact group server")
 
-        defer.returnValue(res)
+        return res
 
     @defer.inlineCallbacks
     def user_removed_from_group(self, group_id, user_id, content):
@@ -421,7 +442,7 @@ class GroupsLocalHandler(object):
     @defer.inlineCallbacks
     def get_joined_groups(self, user_id):
         group_ids = yield self.store.get_joined_groups(user_id)
-        defer.returnValue({"groups": group_ids})
+        return {"groups": group_ids}
 
     @defer.inlineCallbacks
     def get_publicised_groups_for_user(self, user_id):
@@ -433,14 +454,18 @@ class GroupsLocalHandler(object):
             for app_service in self.store.get_app_services():
                 result.extend(app_service.get_groups_for_user(user_id))
 
-            defer.returnValue({"groups": result})
+            return {"groups": result}
         else:
-            bulk_result = yield self.transport_client.bulk_get_publicised_groups(
-                get_domain_from_id(user_id), [user_id]
-            )
+            try:
+                bulk_result = yield self.transport_client.bulk_get_publicised_groups(
+                    get_domain_from_id(user_id), [user_id]
+                )
+            except RequestSendFailed:
+                raise SynapseError(502, "Failed to contact group server")
+
             result = bulk_result.get("users", {}).get(user_id)
             # TODO: Verify attestations
-            defer.returnValue({"groups": result})
+            return {"groups": result}
 
     @defer.inlineCallbacks
     def bulk_get_publicised_groups(self, user_ids, proxy=True):
@@ -475,4 +500,4 @@ class GroupsLocalHandler(object):
             for app_service in self.store.get_app_services():
                 results[uid].extend(app_service.get_groups_for_user(uid))
 
-        defer.returnValue({"users": results})
+        return {"users": results}

synapse/handlers/identity.py

@@ -82,7 +82,7 @@ class IdentityHandler(BaseHandler):
                 "%s is not a trusted ID server: rejecting 3pid " + "credentials",
                 id_server,
             )
-            defer.returnValue(None)
+            return None
 
         try:
             data = yield self.http_client.get_json(
@@ -95,8 +95,8 @@ class IdentityHandler(BaseHandler):
             raise e.to_synapse_error()
 
         if "medium" in data:
-            defer.returnValue(data)
-        defer.returnValue(None)
+            return data
+        return None
 
     @defer.inlineCallbacks
     def bind_threepid(self, creds, mxid):
@@ -133,7 +133,7 @@ class IdentityHandler(BaseHandler):
             )
         except CodeMessageException as e:
             data = json.loads(e.msg)  # XXX WAT?
-        defer.returnValue(data)
+        return data
 
     @defer.inlineCallbacks
     def try_unbind_threepid(self, mxid, threepid):
@@ -161,7 +161,7 @@ class IdentityHandler(BaseHandler):
 
         # We don't know where to unbind, so we don't have a choice but to return
         if not id_servers:
-            defer.returnValue(False)
+            return False
 
         changed = True
         for id_server in id_servers:
@@ -169,7 +169,7 @@ class IdentityHandler(BaseHandler):
                 mxid, threepid, id_server
             )
 
-        defer.returnValue(changed)
+        return changed
 
     @defer.inlineCallbacks
     def try_unbind_threepid_with_id_server(self, mxid, threepid, id_server):
@@ -224,7 +224,7 @@ class IdentityHandler(BaseHandler):
             id_server=id_server,
         )
 
-        defer.returnValue(changed)
+        return changed
 
     @defer.inlineCallbacks
     def requestEmailToken(
@@ -250,7 +250,7 @@ class IdentityHandler(BaseHandler):
                 % (id_server, "/_matrix/identity/api/v1/validate/email/requestToken"),
                 params,
             )
-            defer.returnValue(data)
+            return data
         except HttpResponseException as e:
             logger.info("Proxied requestToken failed: %r", e)
             raise e.to_synapse_error()
@@ -278,7 +278,7 @@ class IdentityHandler(BaseHandler):
                 % (id_server, "/_matrix/identity/api/v1/validate/msisdn/requestToken"),
                 params,
             )
-            defer.returnValue(data)
+            return data
         except HttpResponseException as e:
             logger.info("Proxied requestToken failed: %r", e)
             raise e.to_synapse_error()

synapse/handlers/initial_sync.py

@@ -250,7 +250,7 @@ class InitialSyncHandler(BaseHandler):
             "end": now_token.to_string(),
         }
 
-        defer.returnValue(ret)
+        return ret
 
     @defer.inlineCallbacks
     def room_initial_sync(self, requester, room_id, pagin_config=None):
@@ -301,7 +301,7 @@ class InitialSyncHandler(BaseHandler):
 
         result["account_data"] = account_data_events
 
-        defer.returnValue(result)
+        return result
 
     @defer.inlineCallbacks
     def _room_initial_sync_parted(
@@ -330,28 +330,24 @@ class InitialSyncHandler(BaseHandler):
 
         time_now = self.clock.time_msec()
 
-        defer.returnValue(
-            {
-                "membership": membership,
-                "room_id": room_id,
-                "messages": {
-                    "chunk": (
-                        yield self._event_serializer.serialize_events(
-                            messages, time_now
-                        )
-                    ),
-                    "start": start_token.to_string(),
-                    "end": end_token.to_string(),
-                },
-                "state": (
-                    yield self._event_serializer.serialize_events(
-                        room_state.values(), time_now
-                    )
+        return {
+            "membership": membership,
+            "room_id": room_id,
+            "messages": {
+                "chunk": (
+                    yield self._event_serializer.serialize_events(messages, time_now)
                 ),
-                "presence": [],
-                "receipts": [],
-            }
-        )
+                "start": start_token.to_string(),
+                "end": end_token.to_string(),
+            },
+            "state": (
+                yield self._event_serializer.serialize_events(
+                    room_state.values(), time_now
+                )
+            ),
+            "presence": [],
+            "receipts": [],
+        }
 
     @defer.inlineCallbacks
     def _room_initial_sync_joined(
@@ -384,13 +380,13 @@ class InitialSyncHandler(BaseHandler):
         def get_presence():
             # If presence is disabled, return an empty list
             if not self.hs.config.use_presence:
-                defer.returnValue([])
+                return []
 
             states = yield presence_handler.get_states(
                 [m.user_id for m in room_members], as_event=True
             )
 
-            defer.returnValue(states)
+            return states
 
         @defer.inlineCallbacks
         def get_receipts():
@@ -399,7 +395,7 @@ class InitialSyncHandler(BaseHandler):
             )
             if not receipts:
                 receipts = []
-            defer.returnValue(receipts)
+            return receipts
 
         presence, receipts, (messages, token) = yield make_deferred_yieldable(
             defer.gatherResults(
@@ -442,7 +438,7 @@ class InitialSyncHandler(BaseHandler):
         if not is_peeking:
             ret["membership"] = membership
 
-        defer.returnValue(ret)
+        return ret
 
     @defer.inlineCallbacks
     def _check_in_room_or_world_readable(self, room_id, user_id):
@@ -453,7 +449,7 @@ class InitialSyncHandler(BaseHandler):
             #  * The user is a guest user, and has joined the room
             # else it will throw.
             member_event = yield self.auth.check_user_was_in_room(room_id, user_id)
-            defer.returnValue((member_event.membership, member_event.event_id))
+            return (member_event.membership, member_event.event_id)
             return
         except AuthError:
             visibility = yield self.state_handler.get_current_state(
@@ -463,7 +459,7 @@ class InitialSyncHandler(BaseHandler):
                 visibility
                 and visibility.content["history_visibility"] == "world_readable"
             ):
-                defer.returnValue((Membership.JOIN, None))
+                return (Membership.JOIN, None)
                 return
             raise AuthError(
                 403, "Guest access not allowed", errcode=Codes.GUEST_ACCESS_FORBIDDEN

synapse/handlers/message.py

@@ -87,7 +87,7 @@ class MessageHandler(object):
             )
             data = room_state[membership_event_id].get(key)
 
-        defer.returnValue(data)
+        return data
 
     @defer.inlineCallbacks
     def get_state_events(
@@ -174,7 +174,7 @@ class MessageHandler(object):
             # events, as clients won't use them.
             bundle_aggregations=False,
         )
-        defer.returnValue(events)
+        return events
 
     @defer.inlineCallbacks
     def get_joined_members(self, requester, room_id):
@@ -213,15 +213,13 @@ class MessageHandler(object):
                 # Loop fell through, AS has no interested users in room
                 raise AuthError(403, "Appservice not in room")
 
-        defer.returnValue(
-            {
-                user_id: {
-                    "avatar_url": profile.avatar_url,
-                    "display_name": profile.display_name,
-                }
-                for user_id, profile in iteritems(users_with_profile)
+        return {
+            user_id: {
+                "avatar_url": profile.avatar_url,
+                "display_name": profile.display_name,
             }
-        )
+            for user_id, profile in iteritems(users_with_profile)
+        }
 
 
 class EventCreationHandler(object):
@@ -380,7 +378,11 @@ class EventCreationHandler(object):
             # tolerate them in event_auth.check().
             prev_state_ids = yield context.get_prev_state_ids(self.store)
             prev_event_id = prev_state_ids.get((EventTypes.Member, event.sender))
-            prev_event = yield self.store.get_event(prev_event_id, allow_none=True)
+            prev_event = (
+                yield self.store.get_event(prev_event_id, allow_none=True)
+                if prev_event_id
+                else None
+            )
             if not prev_event or prev_event.membership != Membership.JOIN:
                 logger.warning(
                     (
@@ -398,7 +400,7 @@ class EventCreationHandler(object):
 
         self.validator.validate_new(event)
 
-        defer.returnValue((event, context))
+        return (event, context)
 
     def _is_exempt_from_privacy_policy(self, builder, requester):
         """"Determine if an event to be sent is exempt from having to consent
@@ -425,9 +427,9 @@ class EventCreationHandler(object):
     @defer.inlineCallbacks
     def _is_server_notices_room(self, room_id):
         if self.config.server_notices_mxid is None:
-            defer.returnValue(False)
+            return False
         user_ids = yield self.store.get_users_in_room(room_id)
-        defer.returnValue(self.config.server_notices_mxid in user_ids)
+        return self.config.server_notices_mxid in user_ids
 
     @defer.inlineCallbacks
     def assert_accepted_privacy_policy(self, requester):
@@ -507,7 +509,7 @@ class EventCreationHandler(object):
                     event.event_id,
                     prev_state.event_id,
                 )
-                defer.returnValue(prev_state)
+                return prev_state
 
         yield self.handle_new_client_event(
             requester=requester, event=event, context=context, ratelimit=ratelimit
@@ -523,6 +525,8 @@ class EventCreationHandler(object):
         """
         prev_state_ids = yield context.get_prev_state_ids(self.store)
         prev_event_id = prev_state_ids.get((event.type, event.state_key))
+        if not prev_event_id:
+            return
         prev_event = yield self.store.get_event(prev_event_id, allow_none=True)
         if not prev_event:
             return
@@ -531,7 +535,7 @@ class EventCreationHandler(object):
             prev_content = encode_canonical_json(prev_event.content)
             next_content = encode_canonical_json(event.content)
             if prev_content == next_content:
-                defer.returnValue(prev_event)
+                return prev_event
         return
 
     @defer.inlineCallbacks
@@ -563,7 +567,7 @@ class EventCreationHandler(object):
             yield self.send_nonmember_event(
                 requester, event, context, ratelimit=ratelimit
             )
-        defer.returnValue(event)
+        return event
 
     @measure_func("create_new_client_event")
     @defer.inlineCallbacks
@@ -626,7 +630,7 @@ class EventCreationHandler(object):
 
         logger.debug("Created event %s", event.event_id)
 
-        defer.returnValue((event, context))
+        return (event, context)
 
     @measure_func("handle_new_client_event")
     @defer.inlineCallbacks
@@ -791,7 +795,6 @@ class EventCreationHandler(object):
                 get_prev_content=False,
                 allow_rejected=False,
                 allow_none=True,
-                check_room_id=event.room_id,
             )
 
             # we can make some additional checks now if we have the original event.
@@ -799,6 +802,9 @@ class EventCreationHandler(object):
                 if original_event.type == EventTypes.Create:
                     raise AuthError(403, "Redacting create events is not permitted")
 
+                if original_event.room_id != event.room_id:
+                    raise SynapseError(400, "Cannot redact event from a different room")
+
             prev_state_ids = yield context.get_prev_state_ids(self.store)
             auth_events_ids = yield self.auth.compute_auth_events(
                 event, prev_state_ids, for_verification=True

synapse/handlers/pagination.py

@@ -242,13 +242,11 @@ class PaginationHandler(object):
             )
 
         if not events:
-            defer.returnValue(
-                {
-                    "chunk": [],
-                    "start": pagin_config.from_token.to_string(),
-                    "end": next_token.to_string(),
-                }
-            )
+            return {
+                "chunk": [],
+                "start": pagin_config.from_token.to_string(),
+                "end": next_token.to_string(),
+            }
 
         state = None
         if event_filter and event_filter.lazy_load_members() and len(events) > 0:
@@ -286,4 +284,4 @@ class PaginationHandler(object):
                 )
             )
 
-        defer.returnValue(chunk)
+        return chunk

synapse/handlers/presence.py

@@ -333,7 +333,7 @@ class PresenceHandler(object):
         """Checks the presence of users that have timed out and updates as
         appropriate.
         """
-        logger.info("Handling presence timeouts")
+        logger.debug("Handling presence timeouts")
         now = self.clock.time_msec()
 
         # Fetch the list of users that *may* have timed out. Things may have
@@ -461,7 +461,7 @@ class PresenceHandler(object):
                 if affect_presence:
                     run_in_background(_end)
 
-        defer.returnValue(_user_syncing())
+        return _user_syncing()
 
     def get_currently_syncing_users(self):
         """Get the set of user ids that are currently syncing on this HS.
@@ -556,7 +556,7 @@ class PresenceHandler(object):
         """Get the current presence state for a user.
         """
         res = yield self.current_state_for_users([user_id])
-        defer.returnValue(res[user_id])
+        return res[user_id]
 
     @defer.inlineCallbacks
     def current_state_for_users(self, user_ids):
@@ -585,7 +585,7 @@ class PresenceHandler(object):
                 states.update(new)
                 self.user_to_current_state.update(new)
 
-        defer.returnValue(states)
+        return states
 
     @defer.inlineCallbacks
     def _persist_and_notify(self, states):
@@ -681,7 +681,7 @@ class PresenceHandler(object):
     def get_state(self, target_user, as_event=False):
         results = yield self.get_states([target_user.to_string()], as_event=as_event)
 
-        defer.returnValue(results[0])
+        return results[0]
 
     @defer.inlineCallbacks
     def get_states(self, target_user_ids, as_event=False):
@@ -703,17 +703,15 @@ class PresenceHandler(object):
 
         now = self.clock.time_msec()
         if as_event:
-            defer.returnValue(
-                [
-                    {
-                        "type": "m.presence",
-                        "content": format_user_presence_state(state, now),
-                    }
-                    for state in updates
-                ]
-            )
+            return [
+                {
+                    "type": "m.presence",
+                    "content": format_user_presence_state(state, now),
+                }
+                for state in updates
+            ]
         else:
-            defer.returnValue(updates)
+            return updates
 
     @defer.inlineCallbacks
     def set_state(self, target_user, state, ignore_status_msg=False):
@@ -757,9 +755,9 @@ class PresenceHandler(object):
         )
 
         if observer_room_ids & observed_room_ids:
-            defer.returnValue(True)
+            return True
 
-        defer.returnValue(False)
+        return False
 
     @defer.inlineCallbacks
     def get_all_presence_updates(self, last_id, current_id):
@@ -778,7 +776,7 @@ class PresenceHandler(object):
         # TODO(markjh): replicate the unpersisted changes.
         # This could use the in-memory stores for recent changes.
         rows = yield self.store.get_all_presence_updates(last_id, current_id)
-        defer.returnValue(rows)
+        return rows
 
     def notify_new_event(self):
         """Called when new events have happened. Handles users and servers
@@ -1034,7 +1032,7 @@ class PresenceEventSource(object):
                 #
                 # Hence this guard where we just return nothing so that the sync
                 # doesn't return. C.f. #5503.
-                defer.returnValue(([], max_token))
+                return ([], max_token)
 
             presence = self.get_presence_handler()
             stream_change_cache = self.store.presence_stream_cache
@@ -1068,17 +1066,11 @@ class PresenceEventSource(object):
             updates = yield presence.current_state_for_users(user_ids_changed)
 
         if include_offline:
-            defer.returnValue((list(updates.values()), max_token))
+            return (list(updates.values()), max_token)
         else:
-            defer.returnValue(
-                (
-                    [
-                        s
-                        for s in itervalues(updates)
-                        if s.state != PresenceState.OFFLINE
-                    ],
-                    max_token,
-                )
+            return (
+                [s for s in itervalues(updates) if s.state != PresenceState.OFFLINE],
+                max_token,
             )
 
     def get_current_key(self):
@@ -1107,7 +1099,7 @@ class PresenceEventSource(object):
             )
             users_interested_in.update(user_ids)
 
-        defer.returnValue(users_interested_in)
+        return users_interested_in
 
 
 def handle_timeouts(user_states, is_mine_fn, syncing_user_ids, now):
@@ -1287,7 +1279,7 @@ def get_interested_parties(store, states):
         # Always notify self
         users_to_states.setdefault(state.user_id, []).append(state)
 
-    defer.returnValue((room_ids_to_states, users_to_states))
+    return (room_ids_to_states, users_to_states)
 
 
 @defer.inlineCallbacks
@@ -1321,4 +1313,4 @@ def get_interested_remotes(store, states, state_handler):
         host = get_domain_from_id(user_id)
         hosts_and_states.append(([host], states))
 
-    defer.returnValue(hosts_and_states)
+    return hosts_and_states

synapse/handlers/profile.py

@@ -73,7 +73,7 @@ class BaseProfileHandler(BaseHandler):
                     raise SynapseError(404, "Profile was not found", Codes.NOT_FOUND)
                 raise
 
-            defer.returnValue({"displayname": displayname, "avatar_url": avatar_url})
+            return {"displayname": displayname, "avatar_url": avatar_url}
         else:
             try:
                 result = yield self.federation.make_query(
@@ -82,7 +82,7 @@ class BaseProfileHandler(BaseHandler):
                     args={"user_id": user_id},
                     ignore_backoff=True,
                 )
-                defer.returnValue(result)
+                return result
             except RequestSendFailed as e:
                 raise_from(SynapseError(502, "Failed to fetch profile"), e)
             except HttpResponseException as e:
@@ -108,10 +108,10 @@ class BaseProfileHandler(BaseHandler):
                     raise SynapseError(404, "Profile was not found", Codes.NOT_FOUND)
                 raise
 
-            defer.returnValue({"displayname": displayname, "avatar_url": avatar_url})
+            return {"displayname": displayname, "avatar_url": avatar_url}
         else:
             profile = yield self.store.get_from_remote_profile_cache(user_id)
-            defer.returnValue(profile or {})
+            return profile or {}
 
     @defer.inlineCallbacks
     def get_displayname(self, target_user):
@@ -125,7 +125,7 @@ class BaseProfileHandler(BaseHandler):
                     raise SynapseError(404, "Profile was not found", Codes.NOT_FOUND)
                 raise
 
-            defer.returnValue(displayname)
+            return displayname
         else:
             try:
                 result = yield self.federation.make_query(
@@ -139,7 +139,7 @@ class BaseProfileHandler(BaseHandler):
             except HttpResponseException as e:
                 raise e.to_synapse_error()
 
-            defer.returnValue(result["displayname"])
+            return result["displayname"]
 
     @defer.inlineCallbacks
     def set_displayname(self, target_user, requester, new_displayname, by_admin=False):
@@ -186,7 +186,7 @@ class BaseProfileHandler(BaseHandler):
                 if e.code == 404:
                     raise SynapseError(404, "Profile was not found", Codes.NOT_FOUND)
                 raise
-            defer.returnValue(avatar_url)
+            return avatar_url
         else:
             try:
                 result = yield self.federation.make_query(
@@ -200,7 +200,7 @@ class BaseProfileHandler(BaseHandler):
             except HttpResponseException as e:
                 raise e.to_synapse_error()
 
-            defer.returnValue(result["avatar_url"])
+            return result["avatar_url"]
 
     @defer.inlineCallbacks
     def set_avatar_url(self, target_user, requester, new_avatar_url, by_admin=False):
@@ -251,7 +251,7 @@ class BaseProfileHandler(BaseHandler):
                 raise SynapseError(404, "Profile was not found", Codes.NOT_FOUND)
             raise
 
-        defer.returnValue(response)
+        return response
 
     @defer.inlineCallbacks
     def _update_join_states(self, requester, target_user):

synapse/handlers/receipts.py

@@ -93,7 +93,7 @@ class ReceiptsHandler(BaseHandler):
 
         if min_batch_id is None:
             # no new receipts
-            defer.returnValue(False)
+            return False
 
         affected_room_ids = list(set([r.room_id for r in receipts]))
 
@@ -103,7 +103,7 @@ class ReceiptsHandler(BaseHandler):
             min_batch_id, max_batch_id, affected_room_ids
         )
 
-        defer.returnValue(True)
+        return True
 
     @defer.inlineCallbacks
     def received_client_receipt(self, room_id, receipt_type, user_id, event_id):
@@ -133,9 +133,9 @@ class ReceiptsHandler(BaseHandler):
         )
 
         if not result:
-            defer.returnValue([])
+            return []
 
-        defer.returnValue(result)
+        return result
 
 
 class ReceiptEventSource(object):
@@ -148,13 +148,13 @@ class ReceiptEventSource(object):
         to_key = yield self.get_current_key()
 
         if from_key == to_key:
-            defer.returnValue(([], to_key))
+            return ([], to_key)
 
         events = yield self.store.get_linearized_receipts_for_rooms(
             room_ids, from_key=from_key, to_key=to_key
         )
 
-        defer.returnValue((events, to_key))
+        return (events, to_key)
 
     def get_current_key(self, direction="f"):
         return self.store.get_max_receipt_stream_id()
@@ -173,4 +173,4 @@ class ReceiptEventSource(object):
             room_ids, from_key=from_key, to_key=to_key
         )
 
-        defer.returnValue((events, to_key))
+        return (events, to_key)

synapse/handlers/register.py

@@ -265,7 +265,7 @@ class RegistrationHandler(BaseHandler):
             # Bind email to new account
             yield self._register_email_threepid(user_id, threepid_dict, None, False)
 
-        defer.returnValue(user_id)
+        return user_id
 
     @defer.inlineCallbacks
     def _auto_join_rooms(self, user_id):
@@ -360,7 +360,7 @@ class RegistrationHandler(BaseHandler):
             appservice_id=service_id,
             create_profile_with_displayname=user.localpart,
         )
-        defer.returnValue(user_id)
+        return user_id
 
     @defer.inlineCallbacks
     def check_recaptcha(self, ip, private_key, challenge, response):
@@ -461,7 +461,7 @@ class RegistrationHandler(BaseHandler):
 
         id = self._next_generated_user_id
         self._next_generated_user_id += 1
-        defer.returnValue(str(id))
+        return str(id)
 
     @defer.inlineCallbacks
     def _validate_captcha(self, ip_addr, private_key, challenge, response):
@@ -481,7 +481,7 @@ class RegistrationHandler(BaseHandler):
             "error_url": "http://www.recaptcha.net/recaptcha/api/challenge?"
             + "error=%s" % lines[1],
         }
-        defer.returnValue(json)
+        return json
 
     @defer.inlineCallbacks
     def _submit_captcha(self, ip_addr, private_key, challenge, response):
@@ -497,7 +497,7 @@ class RegistrationHandler(BaseHandler):
                 "response": response,
             },
         )
-        defer.returnValue(data)
+        return data
 
     @defer.inlineCallbacks
     def _join_user_to_room(self, requester, room_identifier):
@@ -622,7 +622,7 @@ class RegistrationHandler(BaseHandler):
                 initial_display_name=initial_display_name,
                 is_guest=is_guest,
             )
-            defer.returnValue((r["device_id"], r["access_token"]))
+            return (r["device_id"], r["access_token"])
 
         valid_until_ms = None
         if self.session_lifetime is not None:
@@ -645,7 +645,7 @@ class RegistrationHandler(BaseHandler):
                 user_id, device_id=device_id, valid_until_ms=valid_until_ms
             )
 
-        defer.returnValue((device_id, access_token))
+        return (device_id, access_token)
 
     @defer.inlineCallbacks
     def post_registration_actions(
@@ -798,7 +798,7 @@ class RegistrationHandler(BaseHandler):
             if ex.errcode == Codes.MISSING_PARAM:
                 # This will only happen if the ID server returns a malformed response
                 logger.info("Can't add incomplete 3pid")
-                defer.returnValue(None)
+                return None
             raise
 
         yield self._auth_handler.add_threepid(

synapse/handlers/room.py

@@ -128,7 +128,7 @@ class RoomCreationHandler(BaseHandler):
             old_room_id,
             new_version,  # args for _upgrade_room
         )
-        defer.returnValue(ret)
+        return ret
 
     @defer.inlineCallbacks
     def _upgrade_room(self, requester, old_room_id, new_version):
@@ -193,7 +193,7 @@ class RoomCreationHandler(BaseHandler):
             requester, old_room_id, new_room_id, old_room_state
         )
 
-        defer.returnValue(new_room_id)
+        return new_room_id
 
     @defer.inlineCallbacks
     def _update_upgraded_room_pls(
@@ -560,6 +560,18 @@ class RoomCreationHandler(BaseHandler):
 
         yield self.event_creation_handler.assert_accepted_privacy_policy(requester)
 
+        power_level_content_override = config.get("power_level_content_override")
+        if (
+            power_level_content_override
+            and "users" in power_level_content_override
+            and user_id not in power_level_content_override["users"]
+        ):
+            raise SynapseError(
+                400,
+                "Not a valid power_level_content_override: 'users' did not contain %s"
+                % (user_id,),
+            )
+
         invite_3pid_list = config.get("invite_3pid", [])
 
         visibility = config.get("visibility", None)
@@ -604,7 +616,7 @@ class RoomCreationHandler(BaseHandler):
             initial_state=initial_state,
             creation_content=creation_content,
             room_alias=room_alias,
-            power_level_content_override=config.get("power_level_content_override"),
+            power_level_content_override=power_level_content_override,
             creator_join_profile=creator_join_profile,
         )
 
@@ -671,7 +683,7 @@ class RoomCreationHandler(BaseHandler):
             result["room_alias"] = room_alias.to_string()
             yield directory_handler.send_room_alias_update_event(requester, room_id)
 
-        defer.returnValue(result)
+        return result
 
     @defer.inlineCallbacks
     def _send_events_for_new_room(
@@ -796,7 +808,7 @@ class RoomCreationHandler(BaseHandler):
                     room_creator_user_id=creator_id,
                     is_public=is_public,
                 )
-                defer.returnValue(gen_room_id)
+                return gen_room_id
             except StoreError:
                 attempts += 1
         raise StoreError(500, "Couldn't generate a room ID.")
@@ -839,7 +851,7 @@ class RoomContextHandler(object):
             event_id, get_prev_content=True, allow_none=True
         )
         if not event:
-            defer.returnValue(None)
+            return None
             return
 
         filtered = yield (filter_evts([event]))
@@ -890,7 +902,7 @@ class RoomContextHandler(object):
 
         results["end"] = token.copy_and_replace("room_key", results["end"]).to_string()
 
-        defer.returnValue(results)
+        return results
 
 
 class RoomEventSource(object):
@@ -941,7 +953,7 @@ class RoomEventSource(object):
             else:
                 end_key = to_key
 
-        defer.returnValue((events, end_key))
+        return (events, end_key)
 
     def get_current_key(self):
         return self.store.get_room_events_max_id()
@@ -959,4 +971,4 @@ class RoomEventSource(object):
             limit=config.limit,
         )
 
-        defer.returnValue((events, next_key))
+        return (events, next_key)

synapse/handlers/room_list.py

@@ -325,7 +325,7 @@ class RoomListHandler(BaseHandler):
                     current_limit=since_token.current_limit - 1,
                 ).to_token()
 
-        defer.returnValue(results)
+        return results
 
     @defer.inlineCallbacks
     def _append_room_entry_to_chunk(
@@ -420,7 +420,7 @@ class RoomListHandler(BaseHandler):
         if join_rules_event:
             join_rule = join_rules_event.content.get("join_rule", None)
             if not allow_private and join_rule and join_rule != JoinRules.PUBLIC:
-                defer.returnValue(None)
+                return None
 
         # Return whether this room is open to federation users or not
         create_event = current_state.get((EventTypes.Create, ""))
@@ -469,7 +469,7 @@ class RoomListHandler(BaseHandler):
             if avatar_url:
                 result["avatar_url"] = avatar_url
 
-        defer.returnValue(result)
+        return result
 
     @defer.inlineCallbacks
     def get_remote_public_room_list(
@@ -482,7 +482,7 @@ class RoomListHandler(BaseHandler):
         third_party_instance_id=None,
     ):
         if not self.enable_room_list_search:
-            defer.returnValue({"chunk": [], "total_room_count_estimate": 0})
+            return {"chunk": [], "total_room_count_estimate": 0}
 
         if search_filter:
             # We currently don't support searching across federation, so we have
@@ -507,7 +507,7 @@ class RoomListHandler(BaseHandler):
                 ]
             }
 
-        defer.returnValue(res)
+        return res
 
     def _get_remote_list_cached(
         self,

synapse/handlers/room_member.py

@@ -38,11 +38,6 @@ from ._base import BaseHandler
 logger = logging.getLogger(__name__)
 
 id_server_scheme = "https://"
-ROOM_COMPLEXITY_TOO_GREAT = (
-    "Your homeserver is unable to join rooms this large or complex. "
-    "Please speak to your server administrator, or upgrade your instance "
-    "to join this room."
-)
 
 
 class RoomMemberHandler(object):
@@ -195,7 +190,7 @@ class RoomMemberHandler(object):
         )
         if duplicate is not None:
             # Discard the new event since this membership change is a no-op.
-            defer.returnValue(duplicate)
+            return duplicate
 
         yield self.event_creation_handler.handle_new_client_event(
             requester, event, context, extra_users=[target], ratelimit=ratelimit
@@ -237,7 +232,7 @@ class RoomMemberHandler(object):
                 if prev_member_event.membership == Membership.JOIN:
                     yield self._user_left_room(target, room_id)
 
-        defer.returnValue(event)
+        return event
 
     @defer.inlineCallbacks
     def copy_room_tags_and_direct_to_room(self, old_room_id, new_room_id, user_id):
@@ -307,7 +302,7 @@ class RoomMemberHandler(object):
                 require_consent=require_consent,
             )
 
-        defer.returnValue(result)
+        return result
 
     @defer.inlineCallbacks
     def _update_membership(
@@ -427,7 +422,7 @@ class RoomMemberHandler(object):
                 same_membership = old_membership == effective_membership_state
                 same_sender = requester.user.to_string() == old_state.sender
                 if same_sender and same_membership and same_content:
-                    defer.returnValue(old_state)
+                    return old_state
 
             if old_membership in ["ban", "leave"] and action == "kick":
                 raise AuthError(403, "The target user is not in the room")
@@ -477,7 +472,7 @@ class RoomMemberHandler(object):
                 ret = yield self._remote_join(
                     requester, remote_room_hosts, room_id, target, content
                 )
-                defer.returnValue(ret)
+                return ret
 
         elif effective_membership_state == Membership.LEAVE:
             if not is_host_in_room:
@@ -499,7 +494,7 @@ class RoomMemberHandler(object):
                     res = yield self._remote_reject_invite(
                         requester, remote_room_hosts, room_id, target
                     )
-                    defer.returnValue(res)
+                    return res
 
         res = yield self._local_membership_update(
             requester=requester,
@@ -512,7 +507,7 @@ class RoomMemberHandler(object):
             content=content,
             require_consent=require_consent,
         )
-        defer.returnValue(res)
+        return res
 
     @defer.inlineCallbacks
     def send_membership_event(
@@ -600,11 +595,11 @@ class RoomMemberHandler(object):
         """
         guest_access_id = current_state_ids.get((EventTypes.GuestAccess, ""), None)
         if not guest_access_id:
-            defer.returnValue(False)
+            return False
 
         guest_access = yield self.store.get_event(guest_access_id)
 
-        defer.returnValue(
+        return (
             guest_access
             and guest_access.content
             and "guest_access" in guest_access.content
@@ -639,7 +634,7 @@ class RoomMemberHandler(object):
             servers.remove(room_alias.domain)
         servers.insert(0, room_alias.domain)
 
-        defer.returnValue((RoomID.from_string(room_id), servers))
+        return (RoomID.from_string(room_id), servers)
 
     @defer.inlineCallbacks
     def _get_inviter(self, user_id, room_id):
@@ -647,7 +642,7 @@ class RoomMemberHandler(object):
             user_id=user_id, room_id=room_id
         )
         if invite:
-            defer.returnValue(UserID.from_string(invite.sender))
+            return UserID.from_string(invite.sender)
 
     @defer.inlineCallbacks
     def do_3pid_invite(
@@ -712,11 +707,11 @@ class RoomMemberHandler(object):
                 if "signatures" not in data:
                     raise AuthError(401, "No signatures on 3pid binding")
                 yield self._verify_any_signature(data, id_server)
-                defer.returnValue(data["mxid"])
+                return data["mxid"]
 
         except IOError as e:
             logger.warn("Error from identity server lookup: %s" % (e,))
-            defer.returnValue(None)
+            return None
 
     @defer.inlineCallbacks
     def _verify_any_signature(self, data, server_hostname):
@@ -908,7 +903,7 @@ class RoomMemberHandler(object):
         if not public_keys:
             public_keys.append(fallback_public_key)
         display_name = data["display_name"]
-        defer.returnValue((token, public_keys, fallback_public_key, display_name))
+        return (token, public_keys, fallback_public_key, display_name)
 
     @defer.inlineCallbacks
     def _is_host_in_room(self, current_state_ids):
@@ -917,7 +912,7 @@ class RoomMemberHandler(object):
         create_event_id = current_state_ids.get(("m.room.create", ""))
         if len(current_state_ids) == 1 and create_event_id:
             # We can only get here if we're in the process of creating the room
-            defer.returnValue(True)
+            return True
 
         for etype, state_key in current_state_ids:
             if etype != EventTypes.Member or not self.hs.is_mine_id(state_key):
@@ -929,16 +924,16 @@ class RoomMemberHandler(object):
                 continue
 
             if event.membership == Membership.JOIN:
-                defer.returnValue(True)
+                return True
 
-        defer.returnValue(False)
+        return False
 
     @defer.inlineCallbacks
     def _is_server_notice_room(self, room_id):
         if self._server_notices_mxid is None:
-            defer.returnValue(False)
+            return False
         user_ids = yield self.store.get_users_in_room(room_id)
-        defer.returnValue(self._server_notices_mxid in user_ids)
+        return self._server_notices_mxid in user_ids
 
 
 class RoomMemberMasterHandler(RoomMemberHandler):
@@ -961,7 +956,7 @@ class RoomMemberMasterHandler(RoomMemberHandler):
         Returns: bool of whether the complexity is too great, or None
             if unable to be fetched
         """
-        max_complexity = self.hs.config.limit_large_room_complexity
+        max_complexity = self.hs.config.limit_remote_rooms.complexity
         complexity = yield self.federation_handler.get_room_complexity(
             remote_room_hosts, room_id
         )
@@ -982,7 +977,7 @@ class RoomMemberMasterHandler(RoomMemberHandler):
 
         Returns: bool
         """
-        max_complexity = self.hs.config.limit_large_room_complexity
+        max_complexity = self.hs.config.limit_remote_rooms.complexity
         complexity = yield self.store.get_room_complexity(room_id)
 
         if complexity["v1"] > max_complexity:
@@ -1004,7 +999,7 @@ class RoomMemberMasterHandler(RoomMemberHandler):
         if len(remote_room_hosts) == 0:
             raise SynapseError(404, "No known servers")
 
-        if self.hs.config.limit_large_room_joins:
+        if self.hs.config.limit_remote_rooms.enabled:
             # Fetch the room complexity
             too_complex = yield self._is_remote_room_too_complex(
                 room_id, remote_room_hosts
@@ -1012,7 +1007,7 @@ class RoomMemberMasterHandler(RoomMemberHandler):
             if too_complex is True:
                 raise SynapseError(
                     code=400,
-                    msg=ROOM_COMPLEXITY_TOO_GREAT,
+                    msg=self.hs.config.limit_remote_rooms.complexity_error,
                     errcode=Codes.RESOURCE_LIMIT_EXCEEDED,
                 )
 
@@ -1027,7 +1022,7 @@ class RoomMemberMasterHandler(RoomMemberHandler):
 
         # Check the room we just joined wasn't too large, if we didn't fetch the
         # complexity of it before.
-        if self.hs.config.limit_large_room_joins:
+        if self.hs.config.limit_remote_rooms.enabled:
             if too_complex is False:
                 # We checked, and we're under the limit.
                 return
@@ -1046,7 +1041,7 @@ class RoomMemberMasterHandler(RoomMemberHandler):
             )
             raise SynapseError(
                 code=400,
-                msg=ROOM_COMPLEXITY_TOO_GREAT,
+                msg=self.hs.config.limit_remote_rooms.complexity_error,
                 errcode=Codes.RESOURCE_LIMIT_EXCEEDED,
             )
 
@@ -1059,7 +1054,7 @@ class RoomMemberMasterHandler(RoomMemberHandler):
             ret = yield fed_handler.do_remotely_reject_invite(
                 remote_room_hosts, room_id, target.to_string()
             )
-            defer.returnValue(ret)
+            return ret
         except Exception as e:
             # if we were unable to reject the exception, just mark
             # it as rejected on our end and plough ahead.
@@ -1070,7 +1065,7 @@ class RoomMemberMasterHandler(RoomMemberHandler):
             logger.warn("Failed to reject invite: %s", e)
 
             yield self.store.locally_reject_invite(target.to_string(), room_id)
-            defer.returnValue({})
+            return {}
 
     def _user_joined_room(self, target, room_id):
         """Implements RoomMemberHandler._user_joined_room