Merge branch 'shhs' of ssh://github.com/matrix-org/synapse into shhs

Synapse 1.2.1 (2019-07-26)
==========================

Security update
---------------

This release includes *four* security fixes:

- Prevent an attack where a federated server could send redactions for arbitrary events in v1 and v2 rooms. ([\#5767](https://github.com/matrix-org/synapse/issues/5767))
- Prevent a denial-of-service attack where cycles of redaction events would make Synapse spin infinitely. Thanks to `@lrizika:matrix.org` for identifying and responsibly disclosing this issue. ([0f2ecb961](https://github.com/matrix-org/synapse/commit/0f2ecb961))
- Prevent an attack where users could be joined or parted from public rooms without their consent. Thanks to @Dylanger for identifying and responsibly disclosing this issue. ([\#5744](https://github.com/matrix-org/synapse/issues/5744))
- Fix a vulnerability where a federated server could spoof read-receipts from
  users on other servers. Thanks to @Dylanger for identifying this issue too. ([\#5743](https://github.com/matrix-org/synapse/issues/5743))

Additionally, the following fix was in Synapse **1.2.0**, but was not correctly
identified during the original release:

- It was possible for a room moderator to send a redaction for an `m.room.create` event, which would downgrade the room to version 1. Thanks to `/dev/ponies` for identifying and responsibly disclosing this issue! ([\#5701](https://github.com/matrix-org/synapse/issues/5701))

.buildkite/merge_base_branch.sh

@@ -2,7 +2,7 @@
 
 set -ex
 
-if [[ "$BUILDKITE_BRANCH" =~ ^(develop|master|dinsic|shhs|release-.*)$ ]]; then
+if [[ "$BUILDKITE_BRANCH" =~ ^(develop|master|dinsic|shhs-.*|release-.*)$ ]]; then
     echo "Not merging forward, as this is a release branch"
     exit 0
 fi

.buildkite/pipeline.yml

@@ -31,7 +31,7 @@ steps:
       - "python -m pip install tox"
       - "scripts-dev/check-newsfragment"
     label: ":newspaper: Newsfile"
-    branches: "!master !develop !release-*"
+    branches: "!master !develop !release-* !shhs-v*"
     plugins:
       - docker#v3.0.1:
           image: "python:3.6"
@@ -47,17 +47,16 @@ steps:
 
   - wait
 
-
   - command:
-      - "apt-get update && apt-get install -y python3.5 python3.5-dev python3-pip libxml2-dev libxslt-dev zlib1g-dev"
-      - "python3.5 -m pip install tox"
+      - "python -m pip install tox"
       - "tox -e py35-old,codecov"
     label: ":python: 3.5 / SQLite / Old Deps"
+    branches: "!shhs !shhs-*"
     env:
       TRIAL_FLAGS: "-j 2"
     plugins:
       - docker#v3.0.1:
-          image: "ubuntu:xenial"  # We use xenail to get an old sqlite and python
+          image: "python:3.5"
           propagate-environment: true
     retry:
       automatic:
@@ -70,6 +69,7 @@ steps:
       - "python -m pip install tox"
       - "tox -e py35,codecov"
     label: ":python: 3.5 / SQLite"
+    branches: "!shhs !shhs-*"
     env:
       TRIAL_FLAGS: "-j 2"
     plugins:
@@ -87,6 +87,7 @@ steps:
       - "python -m pip install tox"
       - "tox -e py36,codecov"
     label: ":python: 3.6 / SQLite"
+    branches: "!shhs !shhs-*"
     env:
       TRIAL_FLAGS: "-j 2"
     plugins:
@@ -118,10 +119,9 @@ steps:
           limit: 2
 
   - label: ":python: 3.5 / :postgres: 9.5"
-    agents:
-      queue: "medium"
+    branches: "!shhs !shhs-*"
     env:
-      TRIAL_FLAGS: "-j 8"
+      TRIAL_FLAGS: "-j 4"
     command:
       - "bash -c 'python -m pip install tox && python -m tox -e py35-postgres,codecov'"
     plugins:
@@ -137,10 +137,9 @@ steps:
           limit: 2
 
   - label: ":python: 3.7 / :postgres: 9.5"
-    agents:
-      queue: "medium"
+    branches: "!shhs !shhs-*"
     env:
-      TRIAL_FLAGS: "-j 8"
+      TRIAL_FLAGS: "-j 4"
     command:
       - "bash -c 'python -m pip install tox && python -m tox -e py37-postgres,codecov'"
     plugins:
@@ -156,10 +155,8 @@ steps:
           limit: 2
 
   - label: ":python: 3.7 / :postgres: 11"
-    agents:
-      queue: "medium"
     env:
-      TRIAL_FLAGS: "-j 8"
+      TRIAL_FLAGS: "-j 4"
     command:
       - "bash -c 'python -m pip install tox && python -m tox -e py37-postgres,codecov'"
     plugins:
@@ -176,6 +173,7 @@ steps:
 
 
   - label: "SyTest - :python: 3.5 / SQLite / Monolith"
+    branches: "!shhs !shhs-*"
     agents:
       queue: "medium"
     command:
@@ -216,15 +214,14 @@ steps:
           limit: 2
 
   - label: "SyTest - :python: 3.5 / :postgres: 9.6 / Workers"
+    branches: "!shhs !shhs-*"
     agents:
       queue: "medium"
     env:
       POSTGRES: "1"
       WORKERS: "1"
-      BLACKLIST: "synapse-blacklist-with-workers"
     command:
       - "bash .buildkite/merge_base_branch.sh"
-      - "bash -c 'cat /src/sytest-blacklist /src/.buildkite/worker-blacklist > /src/synapse-blacklist-with-workers'"
       - "bash /synapse_sytest.sh"
     plugins:
       - docker#v3.0.1:
@@ -232,9 +229,22 @@ steps:
           propagate-environment: true
           always-pull: true
           workdir: "/src"
+    soft_fail: true
     retry:
       automatic:
         - exit_status: -1
           limit: 2
         - exit_status: 2
           limit: 2
+
+  - wait
+
+  - label: ":docker: x86_64"
+    agents:
+      queue: "release"
+    branches: "shhs-*"
+    command:
+      - "docker build -f docker/Dockerfile --build-arg PYTHON_VERSION=3.7.4 . -t matrixdotorg/synapse:${BUILDKITE_TAG}"
+      - "docker save matrixdotorg/synapse:${BUILDKITE_TAG} | gzip -9 > docker.tar.gz"
+    artifact_paths:
+      - "docker.tar.gz"

.buildkite/worker-blacklist

@@ -1,30 +0,0 @@
-# This file serves as a blacklist for SyTest tests that we expect will fail in
-# Synapse when run under worker mode. For more details, see sytest-blacklist.
-
-Message history can be paginated
-
-Can re-join room if re-invited
-
-/upgrade creates a new room
-
-The only membership state included in an initial sync is for all the senders in the timeline
-
-Local device key changes get to remote servers
-
-If remote user leaves room we no longer receive device updates
-
-Forgotten room messages cannot be paginated
-
-Inbound federation can get public room list
-
-Members from the gap are included in gappy incr LL sync
-
-Leaves are present in non-gapped incremental syncs
-
-Old leaves are present in gapped incremental syncs
-
-User sees updates to presence from other users in the incremental sync.
-
-Gapped incremental syncs include all state changes
-
-Old members are included in gappy incr LL sync if they start speaking

.circleci/config.yml

@@ -1,33 +0,0 @@
-version: 2
-jobs:
-  dockerhubuploadrelease:
-    machine: true
-    steps:
-      - checkout
-      - run: docker build -f docker/Dockerfile --label gitsha1=${CIRCLE_SHA1} -t matrixdotorg/synapse:${CIRCLE_TAG} -t matrixdotorg/synapse:${CIRCLE_TAG}-py3 .
-      - run: docker login --username $DOCKER_HUB_USERNAME --password $DOCKER_HUB_PASSWORD
-      - run: docker push matrixdotorg/synapse:${CIRCLE_TAG}
-      - run: docker push matrixdotorg/synapse:${CIRCLE_TAG}-py3
-  dockerhubuploadlatest:
-    machine: true
-    steps:
-      - checkout
-      - run: docker build -f docker/Dockerfile --label gitsha1=${CIRCLE_SHA1} -t matrixdotorg/synapse:latest -t matrixdotorg/synapse:latest-py3 .
-      - run: docker login --username $DOCKER_HUB_USERNAME --password $DOCKER_HUB_PASSWORD
-      - run: docker push matrixdotorg/synapse:latest
-      - run: docker push matrixdotorg/synapse:latest-py3
-
-workflows:
-  version: 2
-  build:
-    jobs:
-      - dockerhubuploadrelease:
-          filters:
-            tags:
-              only: /v[0-9].[0-9]+.[0-9]+.*/
-            branches:
-              ignore: /.*/
-      - dockerhubuploadlatest:
-          filters:
-            branches:
-              only: master

.codecov.yml

@@ -1,4 +1,5 @@
-comment: off
+comment:
+  layout: "diff"
 
 coverage:
   status:

.gitignore

@@ -16,10 +16,10 @@ _trial_temp*/
 /*.log
 /*.log.config
 /*.pid
-/.python-version
 /*.signing.key
 /env/
 /homeserver*.yaml
+/logs
 /media_store/
 /uploads
 
@@ -38,4 +38,3 @@ _trial_temp*/
 /docs/build/
 /htmlcov
 /pip-wheel-metadata/
-

CHANGES.md

@@ -1,102 +1,3 @@
-Synapse 1.3.1 (2019-08-17)
-==========================
-
-Features
---------
-
-- Drop hard dependency on `sdnotify` python package. ([\#5871](https://github.com/matrix-org/synapse/issues/5871))
-
-
-Bugfixes
---------
-
-- Fix startup issue (hang on ACME provisioning) due to ordering of Twisted reactor startup. Thanks to @chrismoos for supplying the fix. ([\#5867](https://github.com/matrix-org/synapse/issues/5867))
-
-
-Synapse 1.3.0 (2019-08-15)
-==========================
-
-Bugfixes
---------
-
-- Fix 500 Internal Server Error on `publicRooms` when the public room list was
-  cached. ([\#5851](https://github.com/matrix-org/synapse/issues/5851))
-
-
-Synapse 1.3.0rc1 (2019-08-13)
-==========================
-
-Features
---------
-
-- Use `M_USER_DEACTIVATED` instead of `M_UNKNOWN` for errcode when a deactivated user attempts to login. ([\#5686](https://github.com/matrix-org/synapse/issues/5686))
-- Add sd_notify hooks to ease systemd integration and allows usage of Type=Notify. ([\#5732](https://github.com/matrix-org/synapse/issues/5732))
-- Synapse will no longer serve any media repo admin endpoints when `enable_media_repo` is set to False in the configuration. If a media repo worker is used, the admin APIs relating to the media repo will be served from it instead. ([\#5754](https://github.com/matrix-org/synapse/issues/5754), [\#5848](https://github.com/matrix-org/synapse/issues/5848))
-- Synapse can now be configured to not join remote rooms of a given "complexity" (currently, state events) over federation. This option can be used to prevent adverse performance on resource-constrained homeservers. ([\#5783](https://github.com/matrix-org/synapse/issues/5783))
-- Allow defining HTML templates to serve the user on account renewal attempt when using the account validity feature. ([\#5807](https://github.com/matrix-org/synapse/issues/5807))
-
-
-Bugfixes
---------
-
-- Fix UISIs during homeserver outage. ([\#5693](https://github.com/matrix-org/synapse/issues/5693), [\#5789](https://github.com/matrix-org/synapse/issues/5789))
-- Fix stack overflow in server key lookup code. ([\#5724](https://github.com/matrix-org/synapse/issues/5724))
-- start.sh no longer uses deprecated cli option. ([\#5725](https://github.com/matrix-org/synapse/issues/5725))
-- Log when we receive an event receipt from an unexpected origin. ([\#5743](https://github.com/matrix-org/synapse/issues/5743))
-- Fix debian packaging scripts to correctly build sid packages. ([\#5775](https://github.com/matrix-org/synapse/issues/5775))
-- Correctly handle redactions of redactions. ([\#5788](https://github.com/matrix-org/synapse/issues/5788))
-- Return 404 instead of 403 when accessing /rooms/{roomId}/event/{eventId} for an event without the appropriate permissions. ([\#5798](https://github.com/matrix-org/synapse/issues/5798))
-- Fix check that tombstone is a state event in push rules. ([\#5804](https://github.com/matrix-org/synapse/issues/5804))
-- Fix error when trying to login as a deactivated user when using a worker to handle login. ([\#5806](https://github.com/matrix-org/synapse/issues/5806))
-- Fix bug where user `/sync` stream could get wedged in rare circumstances. ([\#5825](https://github.com/matrix-org/synapse/issues/5825))
-- The purge_remote_media.sh script was fixed. ([\#5839](https://github.com/matrix-org/synapse/issues/5839))
-
-
-Deprecations and Removals
--------------------------
-
-- Synapse now no longer accepts the `-v`/`--verbose`, `-f`/`--log-file`, or `--log-config` command line flags, and removes the deprecated `verbose` and `log_file` configuration file options. Users of these options should migrate their options into the dedicated log configuration. ([\#5678](https://github.com/matrix-org/synapse/issues/5678), [\#5729](https://github.com/matrix-org/synapse/issues/5729))
-- Remove non-functional 'expire_access_token' setting. ([\#5782](https://github.com/matrix-org/synapse/issues/5782))
-
-
-Internal Changes
-----------------
-
-- Make Jaeger fully configurable. ([\#5694](https://github.com/matrix-org/synapse/issues/5694))
-- Add precautionary measures to prevent future abuse of `window.opener` in default welcome page. ([\#5695](https://github.com/matrix-org/synapse/issues/5695))
-- Reduce database IO usage by optimising queries for current membership. ([\#5706](https://github.com/matrix-org/synapse/issues/5706), [\#5738](https://github.com/matrix-org/synapse/issues/5738), [\#5746](https://github.com/matrix-org/synapse/issues/5746), [\#5752](https://github.com/matrix-org/synapse/issues/5752), [\#5770](https://github.com/matrix-org/synapse/issues/5770), [\#5774](https://github.com/matrix-org/synapse/issues/5774), [\#5792](https://github.com/matrix-org/synapse/issues/5792), [\#5793](https://github.com/matrix-org/synapse/issues/5793))
-- Improve caching when fetching `get_filtered_current_state_ids`. ([\#5713](https://github.com/matrix-org/synapse/issues/5713))
-- Don't accept opentracing data from clients. ([\#5715](https://github.com/matrix-org/synapse/issues/5715))
-- Speed up PostgreSQL unit tests in CI. ([\#5717](https://github.com/matrix-org/synapse/issues/5717))
-- Update the coding style document. ([\#5719](https://github.com/matrix-org/synapse/issues/5719))
-- Improve database query performance when recording retry intervals for remote hosts. ([\#5720](https://github.com/matrix-org/synapse/issues/5720))
-- Add a set of opentracing utils. ([\#5722](https://github.com/matrix-org/synapse/issues/5722))
-- Cache result of get_version_string to reduce overhead of `/version` federation requests. ([\#5730](https://github.com/matrix-org/synapse/issues/5730))
-- Return 'user_type' in admin API user endpoints results. ([\#5731](https://github.com/matrix-org/synapse/issues/5731))
-- Don't package the sytest test blacklist file. ([\#5733](https://github.com/matrix-org/synapse/issues/5733))
-- Replace uses of returnValue with plain return, as returnValue is not needed on Python 3. ([\#5736](https://github.com/matrix-org/synapse/issues/5736))
-- Blacklist some flakey tests in worker mode. ([\#5740](https://github.com/matrix-org/synapse/issues/5740))
-- Fix some error cases in the caching layer. ([\#5749](https://github.com/matrix-org/synapse/issues/5749))
-- Add a prometheus metric for pending cache lookups. ([\#5750](https://github.com/matrix-org/synapse/issues/5750))
-- Stop trying to fetch events with event_id=None. ([\#5753](https://github.com/matrix-org/synapse/issues/5753))
-- Convert RedactionTestCase to modern test style. ([\#5768](https://github.com/matrix-org/synapse/issues/5768))
-- Allow looping calls to be given arguments. ([\#5780](https://github.com/matrix-org/synapse/issues/5780))
-- Set the logs emitted when checking typing and presence timeouts to DEBUG level, not INFO. ([\#5785](https://github.com/matrix-org/synapse/issues/5785))
-- Remove DelayedCall debugging from the test suite, as it is no longer required in the vast majority of Synapse's tests. ([\#5787](https://github.com/matrix-org/synapse/issues/5787))
-- Remove some spurious exceptions from the logs where we failed to talk to a remote server. ([\#5790](https://github.com/matrix-org/synapse/issues/5790))
-- Improve performance when making `.well-known` requests by sharing the SSL options between requests. ([\#5794](https://github.com/matrix-org/synapse/issues/5794))
-- Disable codecov GitHub comments on PRs. ([\#5796](https://github.com/matrix-org/synapse/issues/5796))
-- Don't allow clients to send tombstone events that reference the room it's sent in. ([\#5801](https://github.com/matrix-org/synapse/issues/5801))
-- Deny redactions of events sent in a different room. ([\#5802](https://github.com/matrix-org/synapse/issues/5802))
-- Deny sending well known state types as non-state events. ([\#5805](https://github.com/matrix-org/synapse/issues/5805))
-- Handle incorrectly encoded query params correctly by returning a 400. ([\#5808](https://github.com/matrix-org/synapse/issues/5808))
-- Handle pusher being deleted during processing rather than logging an exception. ([\#5809](https://github.com/matrix-org/synapse/issues/5809))
-- Return 502 not 500 when failing to reach any remote server. ([\#5810](https://github.com/matrix-org/synapse/issues/5810))
-- Reduce global pauses in the events stream caused by expensive state resolution during persistence. ([\#5826](https://github.com/matrix-org/synapse/issues/5826))
-- Add a lower bound to well-known lookup cache time to avoid repeated lookups. ([\#5836](https://github.com/matrix-org/synapse/issues/5836))
-- Whitelist history visbility sytests in worker mode tests. ([\#5843](https://github.com/matrix-org/synapse/issues/5843))
-
-
 Synapse 1.2.1 (2019-07-26)
 ==========================
 
@@ -107,9 +8,9 @@ This release includes *four* security fixes:
 
 - Prevent an attack where a federated server could send redactions for arbitrary events in v1 and v2 rooms. ([\#5767](https://github.com/matrix-org/synapse/issues/5767))
 - Prevent a denial-of-service attack where cycles of redaction events would make Synapse spin infinitely. Thanks to `@lrizika:matrix.org` for identifying and responsibly disclosing this issue. ([0f2ecb961](https://github.com/matrix-org/synapse/commit/0f2ecb961))
-- Prevent an attack where users could be joined or parted from public rooms without their consent. Thanks to @dylangerdaly for identifying and responsibly disclosing this issue. ([\#5744](https://github.com/matrix-org/synapse/issues/5744))
+- Prevent an attack where users could be joined or parted from public rooms without their consent. Thanks to @Dylanger for identifying and responsibly disclosing this issue. ([\#5744](https://github.com/matrix-org/synapse/issues/5744))
 - Fix a vulnerability where a federated server could spoof read-receipts from
-  users on other servers. Thanks to @dylangerdaly for identifying this issue too. ([\#5743](https://github.com/matrix-org/synapse/issues/5743))
+  users on other servers. Thanks to @Dylanger for identifying this issue too. ([\#5743](https://github.com/matrix-org/synapse/issues/5743))
 
 Additionally, the following fix was in Synapse **1.2.0**, but was not correctly
 identified during the original release:

INSTALL.md

@@ -419,11 +419,12 @@ If Synapse is not configured with an SMTP server, password reset via email will
 
 ## Registering a user
 
-The easiest way to create a new user is to do so from a client like [Riot](https://riot.im).
+You will need at least one user on your server in order to use a Matrix
+client. Users can be registered either via a Matrix client, or via a
+commandline script.
 
-Alternatively you can do so from the command line if you have installed via pip. 
-
-This can be done as follows:
+To get started, it is easiest to use the command line to register new
+users. This can be done as follows:
 
 ```
 $ source ~/synapse/env/bin/activate

MANIFEST.in

@@ -7,6 +7,7 @@ include demo/README
 include demo/demo.tls.dh
 include demo/*.py
 include demo/*.sh
+include sytest-blacklist
 
 recursive-include synapse/storage/schema *.sql
 recursive-include synapse/storage/schema *.sql.postgres
@@ -33,7 +34,6 @@ exclude Dockerfile
 exclude .dockerignore
 exclude test_postgresql.sh
 exclude .editorconfig
-exclude sytest-blacklist
 
 include pyproject.toml
 recursive-include changelog.d *

changelog.d/5072.feature

@@ -0,0 +1 @@
+Synapse can now be configured to not join remote rooms of a given "complexity" (currently, state events). This option can be used to prevent adverse performance on resource-constrained homeservers.

changelog.d/5099.misc

@@ -0,0 +1 @@
+Python 2 has been removed from the CI.

changelog.d/5878.feature

@@ -1 +0,0 @@
-Add admin API endpoint for setting whether or not a user is a server administrator.

changelog.d/5914.feature

@@ -1 +0,0 @@
-Add admin API endpoint for getting whether or not a user is a server administrator.

changelog.d/6015.feature

@@ -1 +0,0 @@
-Add config option to increase ratelimits for room admins redacting messages.

contrib/purge_api/purge_remote_media.sh

@@ -51,4 +51,4 @@ TOKEN=$(sql "SELECT token FROM access_tokens WHERE user_id='$ADMIN' ORDER BY id
 # finally start pruning media:
 ###############################################################################
 set -x # for debugging the generated string
-curl --header "Authorization: Bearer $TOKEN" -X POST "$API_URL/admin/purge_media_cache/?before_ts=$UNIX_TIMESTAMP"
+curl --header "Authorization: Bearer $TOKEN" -v POST "$API_URL/admin/purge_media_cache/?before_ts=$UNIX_TIMESTAMP"

contrib/systemd-with-workers/system/matrix-synapse-worker@.service

@@ -4,8 +4,7 @@ After=matrix-synapse.service
 BindsTo=matrix-synapse.service
 
 [Service]
-Type=notify
-NotifyAccess=main
+Type=simple
 User=matrix-synapse
 WorkingDirectory=/var/lib/matrix-synapse
 EnvironmentFile=/etc/default/matrix-synapse

contrib/systemd-with-workers/system/matrix-synapse.service

@@ -2,8 +2,7 @@
 Description=Synapse Matrix Homeserver
 
 [Service]
-Type=notify
-NotifyAccess=main
+Type=simple
 User=matrix-synapse
 WorkingDirectory=/var/lib/matrix-synapse
 EnvironmentFile=/etc/default/matrix-synapse

contrib/systemd/matrix-synapse.service

@@ -14,9 +14,7 @@
 Description=Synapse Matrix homeserver
 
 [Service]
-Type=notify
-NotifyAccess=main
-ExecReload=/bin/kill -HUP $MAINPID
+Type=simple
 Restart=on-abort
 
 User=synapse

debian/changelog

@@ -1,18 +1,8 @@
-matrix-synapse-py3 (1.3.1) stable; urgency=medium
+matrix-synapse-py3 (1.2.1) stable; urgency=medium
 
-  * New synapse release 1.3.1.
+  * New synapse release 1.2.1.
 
- -- Synapse Packaging team <packages@matrix.org>  Sat, 17 Aug 2019 09:15:49 +0100
-
-matrix-synapse-py3 (1.3.0) stable; urgency=medium
-
-  [ Andrew Morgan ]
-  * Remove libsqlite3-dev from required build dependencies.
-
-  [ Synapse Packaging team ]
-  * New synapse release 1.3.0.
-
- -- Synapse Packaging team <packages@matrix.org>  Thu, 15 Aug 2019 12:04:23 +0100
+ -- Synapse Packaging team <packages@matrix.org>  Fri, 26 Jul 2019 11:32:47 +0100
 
 matrix-synapse-py3 (1.2.0) stable; urgency=medium
 

debian/control

@@ -15,6 +15,7 @@ Build-Depends:
  python3-setuptools,
  python3-pip,
  python3-venv,
+ libsqlite3-dev,
  tar,
 Standards-Version: 3.9.8
 Homepage: https://github.com/matrix-org/synapse

demo/start.sh

@@ -29,7 +29,7 @@ for port in 8080 8081 8082; do
 
     if ! grep -F "Customisation made by demo/start.sh" -q  $DIR/etc/$port.config; then
         printf '\n\n# Customisation made by demo/start.sh\n' >> $DIR/etc/$port.config
-
+        
         echo 'enable_registration: true' >> $DIR/etc/$port.config
 
         # Warning, this heredoc depends on the interaction of tabs and spaces. Please don't
@@ -43,7 +43,7 @@ for port in 8080 8081 8082; do
 		    tls: true
 		    resources:
 		      - names: [client, federation]
-
+		
 		  - port: $port
 		    tls: false
 		    bind_addresses: ['::1', '127.0.0.1']
@@ -68,7 +68,7 @@ for port in 8080 8081 8082; do
 
         # Generate tls keys
         openssl req -x509 -newkey rsa:4096 -keyout $DIR/etc/localhost\:$https_port.tls.key -out $DIR/etc/localhost\:$https_port.tls.crt -days 365 -nodes -subj "/O=matrix"
-
+        
         # Ignore keys from the trusted keys server
         echo '# Ignore keys from the trusted keys server' >> $DIR/etc/$port.config
         echo 'trusted_key_servers:' >> $DIR/etc/$port.config
@@ -120,6 +120,7 @@ for port in 8080 8081 8082; do
     python3 -m synapse.app.homeserver \
         --config-path "$DIR/etc/$port.config" \
         -D \
+        -vv \
 
     popd
 done

docker/Dockerfile-dhvirtualenv

@@ -42,11 +42,6 @@ RUN cd dh-virtualenv-1.1 && dpkg-buildpackage -us -uc -b
 ###
 FROM ${distro}
 
-# Get the distro we want to pull from as a dynamic build variable
-# (We need to define it in each build stage)
-ARG distro=""
-ENV distro ${distro}
-
 # Install the build dependencies
 #
 # NB: keep this list in sync with the list of build-deps in debian/control

docker/build_debian.sh

@@ -4,8 +4,7 @@
 
 set -ex
 
-# Get the codename from distro env
-DIST=`cut -d ':' -f2 <<< $distro`
+DIST=`lsb_release -c -s`
 
 # we get a read-only copy of the source: make a writeable copy
 cp -aT /synapse/source /synapse/build

docs/admin_api/user_admin_api.rst

@@ -84,42 +84,3 @@ with a body of:
    }
 
 including an ``access_token`` of a server admin.
-
-
-Get whether a user is a server administrator or not
-===================================================
-
-
-The api is::
-
-    GET /_synapse/admin/v1/users/<user_id>/admin
-
-including an ``access_token`` of a server admin.
-
-A response body like the following is returned:
-
-.. code:: json
-
-    {
-        "admin": true
-    }
-
-
-Change whether a user is a server administrator or not
-======================================================
-
-Note that you cannot demote yourself.
-
-The api is::
-
-    PUT /_synapse/admin/v1/users/<user_id>/admin
-
-with a body of:
-
-.. code:: json
-
-    {
-        "admin": true
-    }
-
-including an ``access_token`` of a server admin.

docs/code_style.rst

@@ -1,8 +1,4 @@
-Code Style
-==========
-
-Formatting tools
-----------------
+# Code Style
 
 The Synapse codebase uses a number of code formatting tools in order to
 quickly and automatically check for formatting (and sometimes logical) errors
@@ -10,20 +6,20 @@ in code.
 
 The necessary tools are detailed below.
 
-- **black**
+## Formatting tools
 
-  The Synapse codebase uses `black <https://pypi.org/project/black/>`_ as an
-  opinionated code formatter, ensuring all comitted code is properly
-  formatted.
+The Synapse codebase uses [black](https://pypi.org/project/black/) as an
+opinionated code formatter, ensuring all comitted code is properly
+formatted.
 
-  First install ``black`` with::
+First install ``black`` with::
 
-    pip install --upgrade black
+  pip install --upgrade black
 
-  Have ``black`` auto-format your code (it shouldn't change any functionality)
-  with::
+Have ``black`` auto-format your code (it shouldn't change any
+functionality) with::
 
-    black . --exclude="\.tox|build|env"
+  black . --exclude="\.tox|build|env"
 
 - **flake8**
 
@@ -58,16 +54,17 @@ functionality is supported in your editor for a more convenient development
 workflow. It is not, however, recommended to run ``flake8`` on save as it
 takes a while and is very resource intensive.
 
-General rules
--------------
+## General rules
 
 - **Naming**:
 
   - Use camel case for class and type names
   - Use underscores for functions and variables.
 
-- **Docstrings**: should follow the `google code style
-  <https://google.github.io/styleguide/pyguide.html#38-comments-and-docstrings>`_.
+- Use double quotes ``"foo"`` rather than single quotes ``'foo'``.
+
+- **Comments**: should follow the `google code style
+  <http://google.github.io/styleguide/pyguide.html?showone=Comments#Comments>`_.
   This is so that we can generate documentation with `sphinx
   <http://sphinxcontrib-napoleon.readthedocs.org/en/latest/>`_. See the
   `examples
@@ -76,8 +73,6 @@ General rules
 
 - **Imports**:
 
-  - Imports should be sorted by ``isort`` as described above.
-
   - Prefer to import classes and functions rather than packages or modules.
 
     Example::
@@ -97,84 +92,25 @@ General rules
     This goes against the advice in the Google style guide, but it means that
     errors in the name are caught early (at import time).
 
+  - Multiple imports from the same package can be combined onto one line::
+
+      from synapse.types import GroupID, RoomID, UserID
+
+    An effort should be made to keep the individual imports in alphabetical
+    order.
+
+    If the list becomes long, wrap it with parentheses and split it over
+    multiple lines.
+
+  - As per `PEP-8 <https://www.python.org/dev/peps/pep-0008/#imports>`_,
+    imports should be grouped in the following order, with a blank line between
+    each group:
+
+    1. standard library imports
+    2. related third party imports
+    3. local application/library specific imports
+
+  - Imports within each group should be sorted alphabetically by module name.
+
   - Avoid wildcard imports (``from synapse.types import *``) and relative
     imports (``from .types import UserID``).
-
-Configuration file format
--------------------------
-
-The `sample configuration file <./sample_config.yaml>`_ acts as a reference to
-Synapse's configuration options for server administrators. Remember that many
-readers will be unfamiliar with YAML and server administration in general, so
-that it is important that the file be as easy to understand as possible, which
-includes following a consistent format.
-
-Some guidelines follow:
-
-* Sections should be separated with a heading consisting of a single line
-  prefixed and suffixed with ``##``. There should be **two** blank lines
-  before the section header, and **one** after.
-
-* Each option should be listed in the file with the following format:
-
-  * A comment describing the setting. Each line of this comment should be
-    prefixed with a hash (``#``) and a space.
-
-    The comment should describe the default behaviour (ie, what happens if
-    the setting is omitted), as well as what the effect will be if the
-    setting is changed.
-
-    Often, the comment end with something like "uncomment the
-    following to \<do action>".
-
-  * A line consisting of only ``#``.
-
-  * A commented-out example setting, prefixed with only ``#``.
-
-    For boolean (on/off) options, convention is that this example should be
-    the *opposite* to the default (so the comment will end with "Uncomment
-    the following to enable [or disable] \<feature\>." For other options,
-    the example should give some non-default value which is likely to be
-    useful to the reader.
-
-* There should be a blank line between each option.
-
-* Where several settings are grouped into a single dict, *avoid* the
-  convention where the whole block is commented out, resulting in comment
-  lines starting ``# #``, as this is hard to read and confusing to
-  edit. Instead, leave the top-level config option uncommented, and follow
-  the conventions above for sub-options. Ensure that your code correctly
-  handles the top-level option being set to ``None`` (as it will be if no
-  sub-options are enabled).
-
-* Lines should be wrapped at 80 characters.
-
-Example::
-
-    ## Frobnication ##
-
-    # The frobnicator will ensure that all requests are fully frobnicated.
-    # To enable it, uncomment the following.
-    #
-    #frobnicator_enabled: true
-
-    # By default, the frobnicator will frobnicate with the default frobber.
-    # The following will make it use an alternative frobber.
-    #
-    #frobincator_frobber: special_frobber
-
-    # Settings for the frobber
-    #
-    frobber:
-       # frobbing speed. Defaults to 1.
-       #
-       #speed: 10
-
-       # frobbing distance. Defaults to 1000.
-       #
-       #distance: 100
-
-Note that the sample configuration is generated from the synapse code and is
-maintained by a script, ``scripts-dev/generate_sample_config``. Making sure
-that the output from this script matches the desired format is left as an
-exercise for the reader!

docs/log_contexts.rst

@@ -148,7 +148,7 @@ call any other functions.
         d = more_stuff()
         result = yield d            # also fine, of course
 
-        return result
+        defer.returnValue(result)
 
     def nonInlineCallbacksFun():
         logger.debug("just a wrapper really")

docs/sample_config.yaml

@@ -280,20 +280,14 @@ listeners:
 
 # Resource-constrained Homeserver Settings
 #
-# If limit_remote_rooms.enabled is True, the room complexity will be
+# If limit_large_remote_room_joins is True, the room complexity will be
 # checked before a user joins a new remote room. If it is above
-# limit_remote_rooms.complexity, it will disallow joining or
+# limit_large_remote_room_complexity, it will disallow joining or
 # instantly leave.
 #
-# limit_remote_rooms.complexity_error can be set to customise the text
-# displayed to the user when a room above the complexity threshold has
-# its join cancelled.
-#
 # Uncomment the below lines to enable:
-#limit_remote_rooms:
-#  enabled: True
-#  complexity: 1.0
-#  complexity_error: "This room is too complex."
+#limit_large_remote_room_joins: True
+#limit_large_remote_room_complexity: 1.0
 
 # Whether to require a user to be in the room to add an alias to it.
 # Defaults to 'true'.
@@ -510,9 +504,6 @@ log_config: "CONFDIR/SERVERNAME.log.config"
 #   - one for login that ratelimits login requests based on the account the
 #     client is attempting to log into, based on the amount of failed login
 #     attempts for this account.
-#   - one for ratelimiting redactions by room admins. If this is not explicitly
-#     set then it uses the same ratelimiting as per rc_message. This is useful
-#     to allow room admins to deal with abuse quickly.
 #
 # The defaults are as shown below.
 #
@@ -534,10 +525,6 @@ log_config: "CONFDIR/SERVERNAME.log.config"
 #  failed_attempts:
 #    per_second: 0.17
 #    burst_count: 3
-#
-#rc_admin_redaction:
-#  per_second: 1
-#  burst_count: 50
 
 
 # Ratelimiting settings for incoming federation
@@ -572,13 +559,6 @@ log_config: "CONFDIR/SERVERNAME.log.config"
 
 
 
-## Media Store ##
-
-# Enable the media store service in the Synapse master. Uncomment the
-# following if you are using a separate media store worker.
-#
-#enable_media_repo: false
-
 # Directory where uploaded images and attachments are stored.
 #
 media_store_path: "DATADIR/media_store"
@@ -816,16 +796,6 @@ uploads_path: "DATADIR/uploads"
 #  period: 6w
 #  renew_at: 1w
 #  renew_email_subject: "Renew your %(app)s account"
-#  # Directory in which Synapse will try to find the HTML files to serve to the
-#  # user when trying to renew an account. Optional, defaults to
-#  # synapse/res/templates.
-#  template_dir: "res/templates"
-#  # HTML to be displayed to the user after they successfully renewed their
-#  # account. Optional.
-#  account_renewed_html_path: "account_renewed.html"
-#  # HTML to be displayed when the user tries to renew an account with an invalid
-#  # renewal token. Optional.
-#  invalid_token_html_path: "invalid_token.html"
 
 # Time that a user's session remains valid for, after they log in.
 #
@@ -966,6 +936,10 @@ uploads_path: "DATADIR/uploads"
 #
 # macaroon_secret_key: <PRIVATE STRING>
 
+# Used to enable access token expiration.
+#
+#expire_access_token: False
+
 # a secret which is used to calculate HMACs for form values, to stop
 # falsification of values. Must be specified for the User Consent
 # forms to work.
@@ -1467,19 +1441,3 @@ opentracing:
     #
     #homeserver_whitelist:
     #  - ".*"
-
-    # Jaeger can be configured to sample traces at different rates.
-    # All configuration options provided by Jaeger can be set here.
-    # Jaeger's configuration mostly related to trace sampling which
-    # is documented here:
-    # https://www.jaegertracing.io/docs/1.13/sampling/.
-    #
-    #jaeger_config:
-    #  sampler:
-    #    type: const
-    #    param: 1
-
-    #  Logging whether spans were started and reported
-    #
-    #  logging:
-    #    false

docs/workers.rst

@@ -206,13 +206,6 @@ Handles the media repository. It can handle all endpoints starting with::
 
     /_matrix/media/
 
-And the following regular expressions matching media-specific administration
-APIs::
-
-    ^/_synapse/admin/v1/purge_media_cache$
-    ^/_synapse/admin/v1/room/.*/media$
-    ^/_synapse/admin/v1/quarantine_media/.*$
-
 You should also set ``enable_media_repo: False`` in the shared configuration
 file to stop the main synapse running background jobs related to managing the
 media repository.

synapse/__init__.py

@@ -35,4 +35,4 @@ try:
 except ImportError:
     pass
 
-__version__ = "1.3.2-alpha.2+modular"
+__version__ = "1.2.1"

synapse/api/auth.py

@@ -128,7 +128,7 @@ class Auth(object):
             )
 
         self._check_joined_room(member, user_id, room_id)
-        return member
+        defer.returnValue(member)
 
     @defer.inlineCallbacks
     def check_user_was_in_room(self, room_id, user_id):
@@ -156,13 +156,13 @@ class Auth(object):
             if forgot:
                 raise AuthError(403, "User %s not in room %s" % (user_id, room_id))
 
-        return member
+        defer.returnValue(member)
 
     @defer.inlineCallbacks
     def check_host_in_room(self, room_id, host):
         with Measure(self.clock, "check_host_in_room"):
             latest_event_ids = yield self.store.is_host_joined(room_id, host)
-            return latest_event_ids
+            defer.returnValue(latest_event_ids)
 
     def _check_joined_room(self, member, user_id, room_id):
         if not member or member.membership != Membership.JOIN:
@@ -219,7 +219,9 @@ class Auth(object):
                         device_id="dummy-device",  # stubbed
                     )
 
-                return synapse.types.create_requester(user_id, app_service=app_service)
+                defer.returnValue(
+                    synapse.types.create_requester(user_id, app_service=app_service)
+                )
 
             user_info = yield self.get_user_by_access_token(access_token, rights)
             user = user_info["user"]
@@ -260,8 +262,10 @@ class Auth(object):
 
             request.authenticated_entity = user.to_string()
 
-            return synapse.types.create_requester(
-                user, token_id, is_guest, device_id, app_service=app_service
+            defer.returnValue(
+                synapse.types.create_requester(
+                    user, token_id, is_guest, device_id, app_service=app_service
+                )
             )
         except KeyError:
             raise MissingClientTokenError()
@@ -272,25 +276,25 @@ class Auth(object):
             self.get_access_token_from_request(request)
         )
         if app_service is None:
-            return (None, None)
+            defer.returnValue((None, None))
 
         if app_service.ip_range_whitelist:
             ip_address = IPAddress(self.hs.get_ip_from_request(request))
             if ip_address not in app_service.ip_range_whitelist:
-                return (None, None)
+                defer.returnValue((None, None))
 
         if b"user_id" not in request.args:
-            return (app_service.sender, app_service)
+            defer.returnValue((app_service.sender, app_service))
 
         user_id = request.args[b"user_id"][0].decode("utf8")
         if app_service.sender == user_id:
-            return (app_service.sender, app_service)
+            defer.returnValue((app_service.sender, app_service))
 
         if not app_service.is_interested_in_user(user_id):
             raise AuthError(403, "Application service cannot masquerade as this user.")
         if not (yield self.store.get_user_by_id(user_id)):
             raise AuthError(403, "Application service has not registered this user")
-        return (user_id, app_service)
+        defer.returnValue((user_id, app_service))
 
     @defer.inlineCallbacks
     def get_user_by_access_token(self, token, rights="access"):
@@ -326,7 +330,7 @@ class Auth(object):
                         msg="Access token has expired", soft_logout=True
                     )
 
-                return r
+                defer.returnValue(r)
 
         # otherwise it needs to be a valid macaroon
         try:
@@ -374,7 +378,7 @@ class Auth(object):
                 }
             else:
                 raise RuntimeError("Unknown rights setting %s", rights)
-            return ret
+            defer.returnValue(ret)
         except (
             _InvalidMacaroonException,
             pymacaroons.exceptions.MacaroonException,
@@ -410,16 +414,21 @@ class Auth(object):
         try:
             user_id = self.get_user_id_from_macaroon(macaroon)
 
+            has_expiry = False
             guest = False
             for caveat in macaroon.caveats:
-                if caveat.caveat_id == "guest = true":
+                if caveat.caveat_id.startswith("time "):
+                    has_expiry = True
+                elif caveat.caveat_id == "guest = true":
                     guest = True
 
-            self.validate_macaroon(macaroon, rights, user_id=user_id)
+            self.validate_macaroon(
+                macaroon, rights, self.hs.config.expire_access_token, user_id=user_id
+            )
         except (pymacaroons.exceptions.MacaroonException, TypeError, ValueError):
             raise InvalidClientTokenError("Invalid macaroon passed.")
 
-        if rights == "access":
+        if not has_expiry and rights == "access":
             self.token_cache[token] = (user_id, guest)
 
         return user_id, guest
@@ -445,7 +454,7 @@ class Auth(object):
                 return caveat.caveat_id[len(user_prefix) :]
         raise InvalidClientTokenError("No user caveat in macaroon")
 
-    def validate_macaroon(self, macaroon, type_string, user_id):
+    def validate_macaroon(self, macaroon, type_string, verify_expiry, user_id):
         """
         validate that a Macaroon is understood by and was signed by this server.
 
@@ -453,6 +462,7 @@ class Auth(object):
             macaroon(pymacaroons.Macaroon): The macaroon to validate
             type_string(str): The kind of token required (e.g. "access",
                               "delete_pusher")
+            verify_expiry(bool): Whether to verify whether the macaroon has expired.
             user_id (str): The user_id required
         """
         v = pymacaroons.Verifier()
@@ -465,7 +475,19 @@ class Auth(object):
         v.satisfy_exact("type = " + type_string)
         v.satisfy_exact("user_id = %s" % user_id)
         v.satisfy_exact("guest = true")
-        v.satisfy_general(self._verify_expiry)
+
+        # verify_expiry should really always be True, but there exist access
+        # tokens in the wild which expire when they should not, so we can't
+        # enforce expiry yet (so we have to allow any caveat starting with
+        # 'time < ' in access tokens).
+        #
+        # On the other hand, short-term login tokens (as used by CAS login, for
+        # example) have an expiry time which we do want to enforce.
+
+        if verify_expiry:
+            v.satisfy_general(self._verify_expiry)
+        else:
+            v.satisfy_general(lambda c: c.startswith("time < "))
 
         # access_tokens include a nonce for uniqueness: any value is acceptable
         v.satisfy_general(lambda c: c.startswith("nonce = "))
@@ -484,7 +506,7 @@ class Auth(object):
     def _look_up_user_by_access_token(self, token):
         ret = yield self.store.get_user_by_access_token(token)
         if not ret:
-            return None
+            defer.returnValue(None)
 
         # we use ret.get() below because *lots* of unit tests stub out
         # get_user_by_access_token in a way where it only returns a couple of
@@ -496,7 +518,7 @@ class Auth(object):
             "device_id": ret.get("device_id"),
             "valid_until_ms": ret.get("valid_until_ms"),
         }
-        return user_info
+        defer.returnValue(user_info)
 
     def get_appservice_by_req(self, request):
         token = self.get_access_token_from_request(request)
@@ -521,7 +543,7 @@ class Auth(object):
     @defer.inlineCallbacks
     def compute_auth_events(self, event, current_state_ids, for_verification=False):
         if event.type == EventTypes.Create:
-            return []
+            defer.returnValue([])
 
         auth_ids = []
 
@@ -582,7 +604,7 @@ class Auth(object):
             if member_event.content["membership"] == Membership.JOIN:
                 auth_ids.append(member_event.event_id)
 
-        return auth_ids
+        defer.returnValue(auth_ids)
 
     @defer.inlineCallbacks
     def check_can_change_room_list(self, room_id, user):
@@ -596,7 +618,7 @@ class Auth(object):
 
         is_admin = yield self.is_server_admin(user)
         if is_admin:
-            return True
+            defer.returnValue(True)
 
         user_id = user.to_string()
         yield self.check_joined_room(room_id, user_id)
@@ -690,7 +712,7 @@ class Auth(object):
             #  * The user is a guest user, and has joined the room
             # else it will throw.
             member_event = yield self.check_user_was_in_room(room_id, user_id)
-            return (member_event.membership, member_event.event_id)
+            defer.returnValue((member_event.membership, member_event.event_id))
         except AuthError:
             visibility = yield self.state.get_current_state(
                 room_id, EventTypes.RoomHistoryVisibility, ""
@@ -699,7 +721,7 @@ class Auth(object):
                 visibility
                 and visibility.content["history_visibility"] == "world_readable"
             ):
-                return (Membership.JOIN, None)
+                defer.returnValue((Membership.JOIN, None))
                 return
             raise AuthError(
                 403, "Guest access not allowed", errcode=Codes.GUEST_ACCESS_FORBIDDEN

synapse/api/errors.py

@@ -61,7 +61,6 @@ class Codes(object):
     INCOMPATIBLE_ROOM_VERSION = "M_INCOMPATIBLE_ROOM_VERSION"
     WRONG_ROOM_KEYS_VERSION = "M_WRONG_ROOM_KEYS_VERSION"
     EXPIRED_ACCOUNT = "ORG_MATRIX_EXPIRED_ACCOUNT"
-    USER_DEACTIVATED = "M_USER_DEACTIVATED"
 
 
 class CodeMessageException(RuntimeError):
@@ -152,7 +151,7 @@ class UserDeactivatedError(SynapseError):
             msg (str): The human-readable error message
         """
         super(UserDeactivatedError, self).__init__(
-            code=http_client.FORBIDDEN, msg=msg, errcode=Codes.USER_DEACTIVATED
+            code=http_client.FORBIDDEN, msg=msg, errcode=Codes.UNKNOWN
         )
 
 

synapse/api/filtering.py

@@ -132,7 +132,7 @@ class Filtering(object):
     @defer.inlineCallbacks
     def get_user_filter(self, user_localpart, filter_id):
         result = yield self.store.get_user_filter(user_localpart, filter_id)
-        return FilterCollection(result)
+        defer.returnValue(FilterCollection(result))
 
     def add_user_filter(self, user_localpart, user_filter):
         self.check_valid_filter(user_filter)

synapse/app/_base.py

@@ -15,9 +15,7 @@
 
 import gc
 import logging
-import os
 import signal
-import socket
 import sys
 import traceback
 
@@ -244,15 +242,9 @@ def start(hs, listeners=None):
         if hasattr(signal, "SIGHUP"):
 
             def handle_sighup(*args, **kwargs):
-                # Tell systemd our state, if we're using it. This will silently fail if
-                # we're not using systemd.
-                sdnotify(b"RELOADING=1")
-
                 for i in _sighup_callbacks:
                     i(hs)
 
-                sdnotify(b"READY=1")
-
             signal.signal(signal.SIGHUP, handle_sighup)
 
             register_sighup(refresh_certificate)
@@ -268,7 +260,6 @@ def start(hs, listeners=None):
         hs.get_datastore().start_profiling()
 
         setup_sentry(hs)
-        setup_sdnotify(hs)
     except Exception:
         traceback.print_exc(file=sys.stderr)
         reactor = hs.get_reactor()
@@ -301,21 +292,6 @@ def setup_sentry(hs):
         scope.set_tag("worker_name", name)
 
 
-def setup_sdnotify(hs):
-    """Adds process state hooks to tell systemd what we are up to.
-    """
-
-    # Tell systemd our state, if we're using it. This will silently fail if
-    # we're not using systemd.
-    hs.get_reactor().addSystemEventTrigger(
-        "after", "startup", sdnotify, b"READY=1\nMAINPID=%i" % (os.getpid(),)
-    )
-
-    hs.get_reactor().addSystemEventTrigger(
-        "before", "shutdown", sdnotify, b"STOPPING=1"
-    )
-
-
 def install_dns_limiter(reactor, max_dns_requests_in_flight=100):
     """Replaces the resolver with one that limits the number of in flight DNS
     requests.
@@ -409,35 +385,3 @@ class _DeferredResolutionReceiver(object):
     def resolutionComplete(self):
         self._deferred.callback(())
         self._receiver.resolutionComplete()
-
-
-sdnotify_sockaddr = os.getenv("NOTIFY_SOCKET")
-
-
-def sdnotify(state):
-    """
-    Send a notification to systemd, if the NOTIFY_SOCKET env var is set.
-
-    This function is based on the sdnotify python package, but since it's only a few
-    lines of code, it's easier to duplicate it here than to add a dependency on a
-    package which many OSes don't include as a matter of principle.
-
-    Args:
-        state (bytes): notification to send
-    """
-    if not isinstance(state, bytes):
-        raise TypeError("sdnotify should be called with a bytes")
-    if not sdnotify_sockaddr:
-        return
-    addr = sdnotify_sockaddr
-    if addr[0] == "@":
-        addr = "\0" + addr[1:]
-
-    try:
-        with socket.socket(socket.AF_UNIX, socket.SOCK_DGRAM) as sock:
-            sock.connect(addr)
-            sock.sendall(state)
-    except Exception as e:
-        # this is a bit surprising, since we don't expect to have a NOTIFY_SOCKET
-        # unless systemd is expecting us to notify it.
-        logger.warning("Unable to send notification to systemd: %s", e)

synapse/app/appservice.py

@@ -168,9 +168,7 @@ def start(config_options):
     )
 
     ps.setup()
-    reactor.addSystemEventTrigger(
-        "before", "startup", _base.start, ps, config.worker_listeners
-    )
+    reactor.callWhenRunning(_base.start, ps, config.worker_listeners)
 
     _base.start_worker_reactor("synapse-appservice", config)
 

synapse/app/client_reader.py

@@ -194,9 +194,7 @@ def start(config_options):
     )
 
     ss.setup()
-    reactor.addSystemEventTrigger(
-        "before", "startup", _base.start, ss, config.worker_listeners
-    )
+    reactor.callWhenRunning(_base.start, ss, config.worker_listeners)
 
     _base.start_worker_reactor("synapse-client-reader", config)
 

synapse/app/event_creator.py

@@ -193,9 +193,7 @@ def start(config_options):
     )
 
     ss.setup()
-    reactor.addSystemEventTrigger(
-        "before", "startup", _base.start, ss, config.worker_listeners
-    )
+    reactor.callWhenRunning(_base.start, ss, config.worker_listeners)
 
     _base.start_worker_reactor("synapse-event-creator", config)
 

synapse/app/federation_reader.py

@@ -175,9 +175,7 @@ def start(config_options):
     )
 
     ss.setup()
-    reactor.addSystemEventTrigger(
-        "before", "startup", _base.start, ss, config.worker_listeners
-    )
+    reactor.callWhenRunning(_base.start, ss, config.worker_listeners)
 
     _base.start_worker_reactor("synapse-federation-reader", config)
 

synapse/app/federation_sender.py

@@ -198,9 +198,7 @@ def start(config_options):
     )
 
     ss.setup()
-    reactor.addSystemEventTrigger(
-        "before", "startup", _base.start, ss, config.worker_listeners
-    )
+    reactor.callWhenRunning(_base.start, ss, config.worker_listeners)
 
     _base.start_worker_reactor("synapse-federation-sender", config)
 

synapse/app/frontend_proxy.py

@@ -70,12 +70,12 @@ class PresenceStatusStubServlet(RestServlet):
         except HttpResponseException as e:
             raise e.to_synapse_error()
 
-        return (200, result)
+        defer.returnValue((200, result))
 
     @defer.inlineCallbacks
     def on_PUT(self, request, user_id):
         yield self.auth.get_user_by_req(request)
-        return (200, {})
+        defer.returnValue((200, {}))
 
 
 class KeyUploadServlet(RestServlet):
@@ -126,11 +126,11 @@ class KeyUploadServlet(RestServlet):
                 self.main_uri + request.uri.decode("ascii"), body, headers=headers
             )
 
-            return (200, result)
+            defer.returnValue((200, result))
         else:
             # Just interested in counts.
             result = yield self.store.count_e2e_one_time_keys(user_id, device_id)
-            return (200, {"one_time_key_counts": result})
+            defer.returnValue((200, {"one_time_key_counts": result}))
 
 
 class FrontendProxySlavedStore(
@@ -247,9 +247,7 @@ def start(config_options):
     )
 
     ss.setup()
-    reactor.addSystemEventTrigger(
-        "before", "startup", _base.start, ss, config.worker_listeners
-    )
+    reactor.callWhenRunning(_base.start, ss, config.worker_listeners)
 
     _base.start_worker_reactor("synapse-frontend-proxy", config)
 

synapse/app/homeserver.py

@@ -406,7 +406,7 @@ def setup(config_options):
         if provision:
             yield acme.provision_certificate()
 
-        return provision
+        defer.returnValue(provision)
 
     @defer.inlineCallbacks
     def reprovision_acme():

synapse/app/media_repository.py

@@ -26,7 +26,6 @@ from synapse.app import _base
 from synapse.config._base import ConfigError
 from synapse.config.homeserver import HomeServerConfig
 from synapse.config.logger import setup_logging
-from synapse.http.server import JsonResource
 from synapse.http.site import SynapseSite
 from synapse.logging.context import LoggingContext
 from synapse.metrics import METRICS_PREFIX, MetricsResource, RegistryProxy
@@ -36,7 +35,6 @@ from synapse.replication.slave.storage.client_ips import SlavedClientIpStore
 from synapse.replication.slave.storage.registration import SlavedRegistrationStore
 from synapse.replication.slave.storage.transactions import SlavedTransactionStore
 from synapse.replication.tcp.client import ReplicationClientHandler
-from synapse.rest.admin import register_servlets_for_media_repo
 from synapse.rest.media.v0.content_repository import ContentRepoResource
 from synapse.server import HomeServer
 from synapse.storage.engines import create_engine
@@ -73,12 +71,6 @@ class MediaRepositoryServer(HomeServer):
                     resources[METRICS_PREFIX] = MetricsResource(RegistryProxy)
                 elif name == "media":
                     media_repo = self.get_media_repository_resource()
-
-                    # We need to serve the admin servlets for media on the
-                    # worker.
-                    admin_resource = JsonResource(self, canonical_json=False)
-                    register_servlets_for_media_repo(self, admin_resource)
-
                     resources.update(
                         {
                             MEDIA_PREFIX: media_repo,
@@ -86,7 +78,6 @@ class MediaRepositoryServer(HomeServer):
                             CONTENT_REPO_PREFIX: ContentRepoResource(
                                 self, self.config.uploads_path
                             ),
-                            "/_synapse/admin": admin_resource,
                         }
                     )
 
@@ -170,9 +161,7 @@ def start(config_options):
     )
 
     ss.setup()
-    reactor.addSystemEventTrigger(
-        "before", "startup", _base.start, ss, config.worker_listeners
-    )
+    reactor.callWhenRunning(_base.start, ss, config.worker_listeners)
 
     _base.start_worker_reactor("synapse-media-repository", config)
 

synapse/app/pusher.py

@@ -216,7 +216,7 @@ def start(config_options):
         _base.start(ps, config.worker_listeners)
         ps.get_pusherpool().start()
 
-    reactor.addSystemEventTrigger("before", "startup", start)
+    reactor.callWhenRunning(start)
 
     _base.start_worker_reactor("synapse-pusher", config)
 

synapse/app/synchrotron.py

@@ -451,9 +451,7 @@ def start(config_options):
     )
 
     ss.setup()
-    reactor.addSystemEventTrigger(
-        "before", "startup", _base.start, ss, config.worker_listeners
-    )
+    reactor.callWhenRunning(_base.start, ss, config.worker_listeners)
 
     _base.start_worker_reactor("synapse-synchrotron", config)
 

synapse/app/user_dir.py

@@ -224,9 +224,7 @@ def start(config_options):
     )
 
     ss.setup()
-    reactor.addSystemEventTrigger(
-        "before", "startup", _base.start, ss, config.worker_listeners
-    )
+    reactor.callWhenRunning(_base.start, ss, config.worker_listeners)
 
     _base.start_worker_reactor("synapse-user-dir", config)
 

synapse/appservice/__init__.py

@@ -175,21 +175,21 @@ class ApplicationService(object):
     @defer.inlineCallbacks
     def _matches_user(self, event, store):
         if not event:
-            return False
+            defer.returnValue(False)
 
         if self.is_interested_in_user(event.sender):
-            return True
+            defer.returnValue(True)
         # also check m.room.member state key
         if event.type == EventTypes.Member and self.is_interested_in_user(
             event.state_key
         ):
-            return True
+            defer.returnValue(True)
 
         if not store:
-            return False
+            defer.returnValue(False)
 
         does_match = yield self._matches_user_in_member_list(event.room_id, store)
-        return does_match
+        defer.returnValue(does_match)
 
     @cachedInlineCallbacks(num_args=1, cache_context=True)
     def _matches_user_in_member_list(self, room_id, store, cache_context):
@@ -200,8 +200,8 @@ class ApplicationService(object):
         # check joined member events
         for user_id in member_list:
             if self.is_interested_in_user(user_id):
-                return True
-        return False
+                defer.returnValue(True)
+        defer.returnValue(False)
 
     def _matches_room_id(self, event):
         if hasattr(event, "room_id"):
@@ -211,13 +211,13 @@ class ApplicationService(object):
     @defer.inlineCallbacks
     def _matches_aliases(self, event, store):
         if not store or not event:
-            return False
+            defer.returnValue(False)
 
         alias_list = yield store.get_aliases_for_room(event.room_id)
         for alias in alias_list:
             if self.is_interested_in_alias(alias):
-                return True
-        return False
+                defer.returnValue(True)
+        defer.returnValue(False)
 
     @defer.inlineCallbacks
     def is_interested(self, event, store=None):
@@ -231,15 +231,15 @@ class ApplicationService(object):
         """
         # Do cheap checks first
         if self._matches_room_id(event):
-            return True
+            defer.returnValue(True)
 
         if (yield self._matches_aliases(event, store)):
-            return True
+            defer.returnValue(True)
 
         if (yield self._matches_user(event, store)):
-            return True
+            defer.returnValue(True)
 
-        return False
+        defer.returnValue(False)
 
     def is_interested_in_user(self, user_id):
         return (

synapse/appservice/api.py

@@ -97,40 +97,40 @@ class ApplicationServiceApi(SimpleHttpClient):
     @defer.inlineCallbacks
     def query_user(self, service, user_id):
         if service.url is None:
-            return False
+            defer.returnValue(False)
         uri = service.url + ("/users/%s" % urllib.parse.quote(user_id))
         response = None
         try:
             response = yield self.get_json(uri, {"access_token": service.hs_token})
             if response is not None:  # just an empty json object
-                return True
+                defer.returnValue(True)
         except CodeMessageException as e:
             if e.code == 404:
-                return False
+                defer.returnValue(False)
                 return
             logger.warning("query_user to %s received %s", uri, e.code)
         except Exception as ex:
             logger.warning("query_user to %s threw exception %s", uri, ex)
-        return False
+        defer.returnValue(False)
 
     @defer.inlineCallbacks
     def query_alias(self, service, alias):
         if service.url is None:
-            return False
+            defer.returnValue(False)
         uri = service.url + ("/rooms/%s" % urllib.parse.quote(alias))
         response = None
         try:
             response = yield self.get_json(uri, {"access_token": service.hs_token})
             if response is not None:  # just an empty json object
-                return True
+                defer.returnValue(True)
         except CodeMessageException as e:
             logger.warning("query_alias to %s received %s", uri, e.code)
             if e.code == 404:
-                return False
+                defer.returnValue(False)
                 return
         except Exception as ex:
             logger.warning("query_alias to %s threw exception %s", uri, ex)
-        return False
+        defer.returnValue(False)
 
     @defer.inlineCallbacks
     def query_3pe(self, service, kind, protocol, fields):
@@ -141,7 +141,7 @@ class ApplicationServiceApi(SimpleHttpClient):
         else:
             raise ValueError("Unrecognised 'kind' argument %r to query_3pe()", kind)
         if service.url is None:
-            return []
+            defer.returnValue([])
 
         uri = "%s%s/thirdparty/%s/%s" % (
             service.url,
@@ -155,7 +155,7 @@ class ApplicationServiceApi(SimpleHttpClient):
                 logger.warning(
                     "query_3pe to %s returned an invalid response %r", uri, response
                 )
-                return []
+                defer.returnValue([])
 
             ret = []
             for r in response:
@@ -166,14 +166,14 @@ class ApplicationServiceApi(SimpleHttpClient):
                         "query_3pe to %s returned an invalid result %r", uri, r
                     )
 
-            return ret
+            defer.returnValue(ret)
         except Exception as ex:
             logger.warning("query_3pe to %s threw exception %s", uri, ex)
-            return []
+            defer.returnValue([])
 
     def get_3pe_protocol(self, service, protocol):
         if service.url is None:
-            return {}
+            defer.returnValue({})
 
         @defer.inlineCallbacks
         def _get():
@@ -189,7 +189,7 @@ class ApplicationServiceApi(SimpleHttpClient):
                     logger.warning(
                         "query_3pe_protocol to %s did not return a" " valid result", uri
                     )
-                    return None
+                    defer.returnValue(None)
 
                 for instance in info.get("instances", []):
                     network_id = instance.get("network_id", None)
@@ -198,10 +198,10 @@ class ApplicationServiceApi(SimpleHttpClient):
                             service.id, network_id
                         ).to_string()
 
-                return info
+                defer.returnValue(info)
             except Exception as ex:
                 logger.warning("query_3pe_protocol to %s threw exception %s", uri, ex)
-                return None
+                defer.returnValue(None)
 
         key = (service.id, protocol)
         return self.protocol_meta_cache.wrap(key, _get)
@@ -209,7 +209,7 @@ class ApplicationServiceApi(SimpleHttpClient):
     @defer.inlineCallbacks
     def push_bulk(self, service, events, txn_id=None):
         if service.url is None:
-            return True
+            defer.returnValue(True)
 
         events = self._serialize(events)
 
@@ -229,14 +229,14 @@ class ApplicationServiceApi(SimpleHttpClient):
             )
             sent_transactions_counter.labels(service.id).inc()
             sent_events_counter.labels(service.id).inc(len(events))
-            return True
+            defer.returnValue(True)
             return
         except CodeMessageException as e:
             logger.warning("push_bulk to %s received %s", uri, e.code)
         except Exception as ex:
             logger.warning("push_bulk to %s threw exception %s", uri, ex)
         failed_transactions_counter.labels(service.id).inc()
-        return False
+        defer.returnValue(False)
 
     def _serialize(self, events):
         time_now = self.clock.time_msec()

synapse/appservice/scheduler.py

@@ -193,7 +193,7 @@ class _TransactionController(object):
     @defer.inlineCallbacks
     def _is_service_up(self, service):
         state = yield self.store.get_appservice_state(service)
-        return state == ApplicationServiceState.UP or state is None
+        defer.returnValue(state == ApplicationServiceState.UP or state is None)
 
 
 class _Recoverer(object):
@@ -208,7 +208,7 @@ class _Recoverer(object):
                 r.service.id,
             )
             r.recover()
-        return recoverers
+        defer.returnValue(recoverers)
 
     def __init__(self, clock, store, as_api, service, callback):
         self.clock = clock

synapse/config/key.py

@@ -116,6 +116,8 @@ class KeyConfig(Config):
             seed = bytes(self.signing_key[0])
             self.macaroon_secret_key = hashlib.sha256(seed).digest()
 
+        self.expire_access_token = config.get("expire_access_token", False)
+
         # a secret which is used to calculate HMACs for form values, to stop
         # falsification of values
         self.form_secret = config.get("form_secret", None)
@@ -142,6 +144,10 @@ class KeyConfig(Config):
         #
         %(macaroon_secret_key)s
 
+        # Used to enable access token expiration.
+        #
+        #expire_access_token: False
+
         # a secret which is used to calculate HMACs for form values, to stop
         # falsification of values. Must be specified for the User Consent
         # forms to work.

synapse/config/logger.py

@@ -12,7 +12,6 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-
 import logging
 import logging.config
 import os
@@ -76,8 +75,10 @@ root:
 
 class LoggingConfig(Config):
     def read_config(self, config, **kwargs):
-        self.log_config = self.abspath(config.get("log_config"))
+        self.verbosity = config.get("verbose", 0)
         self.no_redirect_stdio = config.get("no_redirect_stdio", False)
+        self.log_config = self.abspath(config.get("log_config"))
+        self.log_file = self.abspath(config.get("log_file"))
 
     def generate_config_section(self, config_dir_path, server_name, **kwargs):
         log_config = os.path.join(config_dir_path, server_name + ".log.config")
@@ -93,12 +94,38 @@ class LoggingConfig(Config):
         )
 
     def read_arguments(self, args):
+        if args.verbose is not None:
+            self.verbosity = args.verbose
         if args.no_redirect_stdio is not None:
             self.no_redirect_stdio = args.no_redirect_stdio
+        if args.log_config is not None:
+            self.log_config = args.log_config
+        if args.log_file is not None:
+            self.log_file = args.log_file
 
     @staticmethod
     def add_arguments(parser):
         logging_group = parser.add_argument_group("logging")
+        logging_group.add_argument(
+            "-v",
+            "--verbose",
+            dest="verbose",
+            action="count",
+            help="The verbosity level. Specify multiple times to increase "
+            "verbosity. (Ignored if --log-config is specified.)",
+        )
+        logging_group.add_argument(
+            "-f",
+            "--log-file",
+            dest="log_file",
+            help="File to log to. (Ignored if --log-config is specified.)",
+        )
+        logging_group.add_argument(
+            "--log-config",
+            dest="log_config",
+            default=None,
+            help="Python logging config file",
+        )
         logging_group.add_argument(
             "-n",
             "--no-redirect-stdio",
@@ -126,29 +153,58 @@ def setup_logging(config, use_worker_options=False):
         config (LoggingConfig | synapse.config.workers.WorkerConfig):
             configuration data
 
-        use_worker_options (bool): True to use the 'worker_log_config' option
-            instead of 'log_config'.
+        use_worker_options (bool): True to use 'worker_log_config' and
+            'worker_log_file' options instead of 'log_config' and 'log_file'.
 
         register_sighup (func | None): Function to call to register a
             sighup handler.
     """
     log_config = config.worker_log_config if use_worker_options else config.log_config
+    log_file = config.worker_log_file if use_worker_options else config.log_file
+
+    log_format = (
+        "%(asctime)s - %(name)s - %(lineno)d - %(levelname)s - %(request)s"
+        " - %(message)s"
+    )
 
     if log_config is None:
-        log_format = (
-            "%(asctime)s - %(name)s - %(lineno)d - %(levelname)s - %(request)s"
-            " - %(message)s"
-        )
+        # We don't have a logfile, so fall back to the 'verbosity' param from
+        # the config or cmdline. (Note that we generate a log config for new
+        # installs, so this will be an unusual case)
+        level = logging.INFO
+        level_for_storage = logging.INFO
+        if config.verbosity:
+            level = logging.DEBUG
+            if config.verbosity > 1:
+                level_for_storage = logging.DEBUG
 
         logger = logging.getLogger("")
-        logger.setLevel(logging.INFO)
-        logging.getLogger("synapse.storage.SQL").setLevel(logging.INFO)
+        logger.setLevel(level)
+
+        logging.getLogger("synapse.storage.SQL").setLevel(level_for_storage)
 
         formatter = logging.Formatter(log_format)
+        if log_file:
+            # TODO: Customisable file size / backup count
+            handler = logging.handlers.RotatingFileHandler(
+                log_file, maxBytes=(1000 * 1000 * 100), backupCount=3, encoding="utf8"
+            )
+
+            def sighup(signum, stack):
+                logger.info("Closing log file due to SIGHUP")
+                handler.doRollover()
+                logger.info("Opened new log file due to SIGHUP")
+
+        else:
+            handler = logging.StreamHandler()
+
+            def sighup(*args):
+                pass
 
-        handler = logging.StreamHandler()
         handler.setFormatter(formatter)
+
         handler.addFilter(LoggingContextFilter(request=""))
+
         logger.addHandler(handler)
     else:
 
@@ -162,7 +218,8 @@ def setup_logging(config, use_worker_options=False):
             logging.info("Reloaded log config from %s due to SIGHUP", log_config)
 
         load_log_config()
-        appbase.register_sighup(sighup)
+
+    appbase.register_sighup(sighup)
 
     # make sure that the first thing we log is a thing we can grep backwards
     # for

synapse/config/ratelimiting.py

@@ -80,12 +80,6 @@ class RatelimitConfig(Config):
             "federation_rr_transactions_per_room_per_second", 50
         )
 
-        rc_admin_redaction = config.get("rc_admin_redaction")
-        if rc_admin_redaction:
-            self.rc_admin_redaction = RateLimitConfig(rc_admin_redaction)
-        else:
-            self.rc_admin_redaction = None
-
     def generate_config_section(self, **kwargs):
         return """\
         ## Ratelimiting ##
@@ -108,9 +102,6 @@ class RatelimitConfig(Config):
         #   - one for login that ratelimits login requests based on the account the
         #     client is attempting to log into, based on the amount of failed login
         #     attempts for this account.
-        #   - one for ratelimiting redactions by room admins. If this is not explicitly
-        #     set then it uses the same ratelimiting as per rc_message. This is useful
-        #     to allow room admins to deal with abuse quickly.
         #
         # The defaults are as shown below.
         #
@@ -132,10 +123,6 @@ class RatelimitConfig(Config):
         #  failed_attempts:
         #    per_second: 0.17
         #    burst_count: 3
-        #
-        #rc_admin_redaction:
-        #  per_second: 1
-        #  burst_count: 50
 
 
         # Ratelimiting settings for incoming federation

synapse/config/registration.py

@@ -13,11 +13,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import os
 from distutils.util import strtobool
 
-import pkg_resources
-
 from synapse.config._base import Config, ConfigError
 from synapse.types import RoomAlias
 from synapse.util.stringutils import random_string_with_symbols
@@ -44,36 +41,8 @@ class AccountValidityConfig(Config):
 
             self.startup_job_max_delta = self.period * 10.0 / 100.0
 
-        if self.renew_by_email_enabled:
-            if "public_baseurl" not in synapse_config:
-                raise ConfigError("Can't send renewal emails without 'public_baseurl'")
-
-        template_dir = config.get("template_dir")
-
-        if not template_dir:
-            template_dir = pkg_resources.resource_filename("synapse", "res/templates")
-
-        if "account_renewed_html_path" in config:
-            file_path = os.path.join(template_dir, config["account_renewed_html_path"])
-
-            self.account_renewed_html_content = self.read_file(
-                file_path, "account_validity.account_renewed_html_path"
-            )
-        else:
-            self.account_renewed_html_content = (
-                "<html><body>Your account has been successfully renewed.</body><html>"
-            )
-
-        if "invalid_token_html_path" in config:
-            file_path = os.path.join(template_dir, config["invalid_token_html_path"])
-
-            self.invalid_token_html_content = self.read_file(
-                file_path, "account_validity.invalid_token_html_path"
-            )
-        else:
-            self.invalid_token_html_content = (
-                "<html><body>Invalid renewal token.</body><html>"
-            )
+        if self.renew_by_email_enabled and "public_baseurl" not in synapse_config:
+            raise ConfigError("Can't send renewal emails without 'public_baseurl'")
 
 
 class RegistrationConfig(Config):
@@ -176,16 +145,6 @@ class RegistrationConfig(Config):
         #  period: 6w
         #  renew_at: 1w
         #  renew_email_subject: "Renew your %%(app)s account"
-        #  # Directory in which Synapse will try to find the HTML files to serve to the
-        #  # user when trying to renew an account. Optional, defaults to
-        #  # synapse/res/templates.
-        #  template_dir: "res/templates"
-        #  # HTML to be displayed to the user after they successfully renewed their
-        #  # account. Optional.
-        #  account_renewed_html_path: "account_renewed.html"
-        #  # HTML to be displayed when the user tries to renew an account with an invalid
-        #  # renewal token. Optional.
-        #  invalid_token_html_path: "invalid_token.html"
 
         # Time that a user's session remains valid for, after they log in.
         #

synapse/config/repository.py

@@ -12,7 +12,6 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-
 import os
 from collections import namedtuple
 
@@ -88,25 +87,22 @@ def parse_thumbnail_requirements(thumbnail_sizes):
 
 class ContentRepositoryConfig(Config):
     def read_config(self, config, **kwargs):
-
-        # Only enable the media repo if either the media repo is enabled or the
-        # current worker app is the media repo.
-        if (
-            self.enable_media_repo is False
-            and config.get("worker_app") != "synapse.app.media_repository"
-        ):
-            self.can_load_media_repo = False
-            return
-        else:
-            self.can_load_media_repo = True
+        self.enable_media_repo = config.get("enable_media_repo", True)
 
         self.max_upload_size = self.parse_size(config.get("max_upload_size", "10M"))
         self.max_image_pixels = self.parse_size(config.get("max_image_pixels", "32M"))
         self.max_spider_size = self.parse_size(config.get("max_spider_size", "10M"))
 
-        self.media_store_path = self.ensure_directory(
-            config.get("media_store_path", "media_store")
-        )
+        if self.enable_media_repo:
+            self.media_store_path = self.ensure_directory(
+                config.get("media_store_path", "media_store")
+            )
+            self.uploads_path = self.ensure_directory(
+                config.get("uploads_path", "uploads")
+            )
+        else:
+            self.media_store_path = None
+            self.uploads_path = None
 
         backup_media_store_path = config.get("backup_media_store_path")
 
@@ -163,7 +159,6 @@ class ContentRepositoryConfig(Config):
                 (provider_class, parsed_config, wrapper_config)
             )
 
-        self.uploads_path = self.ensure_directory(config.get("uploads_path", "uploads"))
         self.dynamic_thumbnails = config.get("dynamic_thumbnails", False)
         self.thumbnail_requirements = parse_thumbnail_requirements(
             config.get("thumbnail_sizes", DEFAULT_THUMBNAIL_SIZES)
@@ -215,13 +210,6 @@ class ContentRepositoryConfig(Config):
 
         return (
             r"""
-        ## Media Store ##
-
-        # Enable the media store service in the Synapse master. Uncomment the
-        # following if you are using a separate media store worker.
-        #
-        #enable_media_repo: false
-
         # Directory where uploaded images and attachments are stored.
         #
         media_store_path: "%(media_store)s"

synapse/config/server.py

@@ -18,7 +18,6 @@
 import logging
 import os.path
 
-import attr
 from netaddr import IPSet
 
 from synapse.api.room_versions import KNOWN_ROOM_VERSIONS
@@ -39,12 +38,6 @@ DEFAULT_BIND_ADDRESSES = ["::", "0.0.0.0"]
 
 DEFAULT_ROOM_VERSION = "4"
 
-ROOM_COMPLEXITY_TOO_GREAT = (
-    "Your homeserver is unable to join rooms this large or complex. "
-    "Please speak to your server administrator, or upgrade your instance "
-    "to join this room."
-)
-
 
 class ServerConfig(Config):
     def read_config(self, config, **kwargs):
@@ -254,21 +247,10 @@ class ServerConfig(Config):
 
         self.gc_thresholds = read_gc_thresholds(config.get("gc_thresholds", None))
 
-        @attr.s
-        class LimitRemoteRoomsConfig(object):
-            enabled = attr.ib(
-                validator=attr.validators.instance_of(bool), default=False
-            )
-            complexity = attr.ib(
-                validator=attr.validators.instance_of((int, float)), default=1.0
-            )
-            complexity_error = attr.ib(
-                validator=attr.validators.instance_of(str),
-                default=ROOM_COMPLEXITY_TOO_GREAT,
-            )
-
-        self.limit_remote_rooms = LimitRemoteRoomsConfig(
-            **config.get("limit_remote_rooms", {})
+        # Resource-constrained Homeserver Configuration
+        self.limit_large_room_joins = config.get("limit_large_remote_room_joins", False)
+        self.limit_large_room_complexity = config.get(
+            "limit_large_remote_room_complexity", 1.0
         )
 
         bind_port = config.get("bind_port")
@@ -643,20 +625,14 @@ class ServerConfig(Config):
 
         # Resource-constrained Homeserver Settings
         #
-        # If limit_remote_rooms.enabled is True, the room complexity will be
+        # If limit_large_remote_room_joins is True, the room complexity will be
         # checked before a user joins a new remote room. If it is above
-        # limit_remote_rooms.complexity, it will disallow joining or
+        # limit_large_remote_room_complexity, it will disallow joining or
         # instantly leave.
         #
-        # limit_remote_rooms.complexity_error can be set to customise the text
-        # displayed to the user when a room above the complexity threshold has
-        # its join cancelled.
-        #
         # Uncomment the below lines to enable:
-        #limit_remote_rooms:
-        #  enabled: True
-        #  complexity: 1.0
-        #  complexity_error: "This room is too complex."
+        #limit_large_remote_room_joins: True
+        #limit_large_remote_room_complexity: 1.0
 
         # Whether to require a user to be in the room to add an alias to it.
         # Defaults to 'true'.

synapse/config/tracer.py

@@ -23,12 +23,6 @@ class TracerConfig(Config):
             opentracing_config = {}
 
         self.opentracer_enabled = opentracing_config.get("enabled", False)
-
-        self.jaeger_config = opentracing_config.get(
-            "jaeger_config",
-            {"sampler": {"type": "const", "param": 1}, "logging": False},
-        )
-
         if not self.opentracer_enabled:
             return
 
@@ -62,20 +56,4 @@ class TracerConfig(Config):
             #
             #homeserver_whitelist:
             #  - ".*"
-
-            # Jaeger can be configured to sample traces at different rates.
-            # All configuration options provided by Jaeger can be set here.
-            # Jaeger's configuration mostly related to trace sampling which
-            # is documented here:
-            # https://www.jaegertracing.io/docs/1.13/sampling/.
-            #
-            #jaeger_config:
-            #  sampler:
-            #    type: const
-            #    param: 1
-
-            #  Logging whether spans were started and reported
-            #
-            #  logging:
-            #    false
         """

synapse/config/workers.py

@@ -31,6 +31,7 @@ class WorkerConfig(Config):
         self.worker_listeners = config.get("worker_listeners", [])
         self.worker_daemonize = config.get("worker_daemonize")
         self.worker_pid_file = config.get("worker_pid_file")
+        self.worker_log_file = config.get("worker_log_file")
         self.worker_log_config = config.get("worker_log_config")
 
         # The host used to connect to the main synapse
@@ -77,5 +78,9 @@ class WorkerConfig(Config):
 
         if args.daemonize is not None:
             self.worker_daemonize = args.daemonize
+        if args.log_config is not None:
+            self.worker_log_config = args.log_config
+        if args.log_file is not None:
+            self.worker_log_file = args.log_file
         if args.manhole is not None:
             self.worker_manhole = args.worker_manhole

synapse/crypto/context_factory.py

@@ -31,7 +31,6 @@ from twisted.internet.ssl import (
     platformTrust,
 )
 from twisted.python.failure import Failure
-from twisted.web.iweb import IPolicyForHTTPS
 
 logger = logging.getLogger(__name__)
 
@@ -75,7 +74,6 @@ class ServerContextFactory(ContextFactory):
         return self._context
 
 
-@implementer(IPolicyForHTTPS)
 class ClientTLSOptionsFactory(object):
     """Factory for Twisted SSLClientConnectionCreators that are used to make connections
     to remote servers for federation.
@@ -148,12 +146,6 @@ class ClientTLSOptionsFactory(object):
             f = Failure()
             tls_protocol.failVerification(f)
 
-    def creatorForNetloc(self, hostname, port):
-        """Implements the IPolicyForHTTPS interace so that this can be passed
-        directly to agents.
-        """
-        return self.get_options(hostname)
-
 
 @implementer(IOpenSSLClientConnectionCreator)
 class SSLClientConnectionCreator(object):

synapse/crypto/keyring.py

@@ -238,9 +238,27 @@ class Keyring(object):
         """
 
         try:
-            ctx = LoggingContext.current_context()
+            # create a deferred for each server we're going to look up the keys
+            # for; we'll resolve them once we have completed our lookups.
+            # These will be passed into wait_for_previous_lookups to block
+            # any other lookups until we have finished.
+            # The deferreds are called with no logcontext.
+            server_to_deferred = {
+                rq.server_name: defer.Deferred() for rq in verify_requests
+            }
 
-            # map from server name to a set of outstanding request ids
+            # We want to wait for any previous lookups to complete before
+            # proceeding.
+            yield self.wait_for_previous_lookups(server_to_deferred)
+
+            # Actually start fetching keys.
+            self._get_server_verify_keys(verify_requests)
+
+            # When we've finished fetching all the keys for a given server_name,
+            # resolve the deferred passed to `wait_for_previous_lookups` so that
+            # any lookups waiting will proceed.
+            #
+            # map from server name to a set of request ids
             server_to_request_ids = {}
 
             for verify_request in verify_requests:
@@ -248,61 +266,40 @@ class Keyring(object):
                 request_id = id(verify_request)
                 server_to_request_ids.setdefault(server_name, set()).add(request_id)
 
-            # Wait for any previous lookups to complete before proceeding.
-            yield self.wait_for_previous_lookups(server_to_request_ids.keys())
-
-            # take out a lock on each of the servers by sticking a Deferred in
-            # key_downloads
-            for server_name in server_to_request_ids.keys():
-                self.key_downloads[server_name] = defer.Deferred()
-                logger.debug("Got key lookup lock on %s", server_name)
-
-            # When we've finished fetching all the keys for a given server_name,
-            # drop the lock by resolving the deferred in key_downloads.
-            def drop_server_lock(server_name):
-                d = self.key_downloads.pop(server_name)
-                d.callback(None)
-
-            def lookup_done(res, verify_request):
+            def remove_deferreds(res, verify_request):
                 server_name = verify_request.server_name
-                server_requests = server_to_request_ids[server_name]
-                server_requests.remove(id(verify_request))
-
-                # if there are no more requests for this server, we can drop the lock.
-                if not server_requests:
-                    with PreserveLoggingContext(ctx):
-                        logger.debug("Releasing key lookup lock on %s", server_name)
-
-                    # ... but not immediately, as that can cause stack explosions if
-                    # we get a long queue of lookups.
-                    self.clock.call_later(0, drop_server_lock, server_name)
-
+                request_id = id(verify_request)
+                server_to_request_ids[server_name].discard(request_id)
+                if not server_to_request_ids[server_name]:
+                    d = server_to_deferred.pop(server_name, None)
+                    if d:
+                        d.callback(None)
                 return res
 
             for verify_request in verify_requests:
-                verify_request.key_ready.addBoth(lookup_done, verify_request)
-
-            # Actually start fetching keys.
-            self._get_server_verify_keys(verify_requests)
+                verify_request.key_ready.addBoth(remove_deferreds, verify_request)
         except Exception:
             logger.exception("Error starting key lookups")
 
     @defer.inlineCallbacks
-    def wait_for_previous_lookups(self, server_names):
+    def wait_for_previous_lookups(self, server_to_deferred):
         """Waits for any previous key lookups for the given servers to finish.
 
         Args:
-            server_names (Iterable[str]): list of servers which we want to look up
+            server_to_deferred (dict[str, Deferred]): server_name to deferred which gets
+                resolved once we've finished looking up keys for that server.
+                The Deferreds should be regular twisted ones which call their
+                callbacks with no logcontext.
 
-        Returns:
-            Deferred[None]: resolves once all key lookups for the given servers have
-                completed. Follows the synapse rules of logcontext preservation.
+        Returns: a Deferred which resolves once all key lookups for the given
+            servers have completed. Follows the synapse rules of logcontext
+            preservation.
         """
         loop_count = 1
         while True:
             wait_on = [
                 (server_name, self.key_downloads[server_name])
-                for server_name in server_names
+                for server_name in server_to_deferred.keys()
                 if server_name in self.key_downloads
             ]
             if not wait_on:
@@ -317,6 +314,19 @@ class Keyring(object):
 
             loop_count += 1
 
+        ctx = LoggingContext.current_context()
+
+        def rm(r, server_name_):
+            with PreserveLoggingContext(ctx):
+                logger.debug("Releasing key lookup lock on %s", server_name_)
+                self.key_downloads.pop(server_name_, None)
+            return r
+
+        for server_name, deferred in server_to_deferred.items():
+            logger.debug("Got key lookup lock on %s", server_name)
+            self.key_downloads[server_name] = deferred
+            deferred.addBoth(rm, server_name)
+
     def _get_server_verify_keys(self, verify_requests):
         """Tries to find at least one key for each verify request
 
@@ -462,7 +472,7 @@ class StoreKeyFetcher(KeyFetcher):
         keys = {}
         for (server_name, key_id), key in res.items():
             keys.setdefault(server_name, {})[key_id] = key
-        return keys
+        defer.returnValue(keys)
 
 
 class BaseV2KeyFetcher(object):
@@ -566,7 +576,7 @@ class BaseV2KeyFetcher(object):
             ).addErrback(unwrapFirstError)
         )
 
-        return verify_keys
+        defer.returnValue(verify_keys)
 
 
 class PerspectivesKeyFetcher(BaseV2KeyFetcher):
@@ -588,7 +598,7 @@ class PerspectivesKeyFetcher(BaseV2KeyFetcher):
                 result = yield self.get_server_verify_key_v2_indirect(
                     keys_to_fetch, key_server
                 )
-                return result
+                defer.returnValue(result)
             except KeyLookupError as e:
                 logger.warning(
                     "Key lookup failed from %r: %s", key_server.server_name, e
@@ -601,7 +611,7 @@ class PerspectivesKeyFetcher(BaseV2KeyFetcher):
                     str(e),
                 )
 
-            return {}
+            defer.returnValue({})
 
         results = yield make_deferred_yieldable(
             defer.gatherResults(
@@ -615,7 +625,7 @@ class PerspectivesKeyFetcher(BaseV2KeyFetcher):
             for server_name, keys in result.items():
                 union_of_keys.setdefault(server_name, {}).update(keys)
 
-        return union_of_keys
+        defer.returnValue(union_of_keys)
 
     @defer.inlineCallbacks
     def get_server_verify_key_v2_indirect(self, keys_to_fetch, key_server):
@@ -701,7 +711,7 @@ class PerspectivesKeyFetcher(BaseV2KeyFetcher):
             perspective_name, time_now_ms, added_keys
         )
 
-        return keys
+        defer.returnValue(keys)
 
     def _validate_perspectives_response(self, key_server, response):
         """Optionally check the signature on the result of a /key/query request
@@ -843,7 +853,7 @@ class ServerKeyFetcher(BaseV2KeyFetcher):
             )
             keys.update(response_keys)
 
-        return keys
+        defer.returnValue(keys)
 
 
 @defer.inlineCallbacks

synapse/events/builder.py

@@ -144,13 +144,15 @@ class EventBuilder(object):
         if self._origin_server_ts is not None:
             event_dict["origin_server_ts"] = self._origin_server_ts
 
-        return create_local_event_from_event_dict(
-            clock=self._clock,
-            hostname=self._hostname,
-            signing_key=self._signing_key,
-            format_version=self.format_version,
-            event_dict=event_dict,
-            internal_metadata_dict=self.internal_metadata.get_dict(),
+        defer.returnValue(
+            create_local_event_from_event_dict(
+                clock=self._clock,
+                hostname=self._hostname,
+                signing_key=self._signing_key,
+                format_version=self.format_version,
+                event_dict=event_dict,
+                internal_metadata_dict=self.internal_metadata.get_dict(),
+            )
         )
 
 

synapse/events/snapshot.py

@@ -133,17 +133,19 @@ class EventContext(object):
         else:
             prev_state_id = None
 
-        return {
-            "prev_state_id": prev_state_id,
-            "event_type": event.type,
-            "event_state_key": event.state_key if event.is_state() else None,
-            "state_group": self.state_group,
-            "rejected": self.rejected,
-            "prev_group": self.prev_group,
-            "delta_ids": _encode_state_dict(self.delta_ids),
-            "prev_state_events": self.prev_state_events,
-            "app_service_id": self.app_service.id if self.app_service else None,
-        }
+        defer.returnValue(
+            {
+                "prev_state_id": prev_state_id,
+                "event_type": event.type,
+                "event_state_key": event.state_key if event.is_state() else None,
+                "state_group": self.state_group,
+                "rejected": self.rejected,
+                "prev_group": self.prev_group,
+                "delta_ids": _encode_state_dict(self.delta_ids),
+                "prev_state_events": self.prev_state_events,
+                "app_service_id": self.app_service.id if self.app_service else None,
+            }
+        )
 
     @staticmethod
     def deserialize(store, input):
@@ -200,7 +202,7 @@ class EventContext(object):
 
         yield make_deferred_yieldable(self._fetching_state_deferred)
 
-        return self._current_state_ids
+        defer.returnValue(self._current_state_ids)
 
     @defer.inlineCallbacks
     def get_prev_state_ids(self, store):
@@ -220,7 +222,7 @@ class EventContext(object):
 
         yield make_deferred_yieldable(self._fetching_state_deferred)
 
-        return self._prev_state_ids
+        defer.returnValue(self._prev_state_ids)
 
     def get_cached_current_state_ids(self):
         """Gets the current state IDs if we have them already cached.

synapse/events/third_party_rules.py

@@ -51,7 +51,7 @@ class ThirdPartyEventRules(object):
             defer.Deferred[bool]: True if the event should be allowed, False if not.
         """
         if self.third_party_rules is None:
-            return True
+            defer.returnValue(True)
 
         prev_state_ids = yield context.get_prev_state_ids(self.store)
 
@@ -61,7 +61,7 @@ class ThirdPartyEventRules(object):
             state_events[key] = yield self.store.get_event(event_id, allow_none=True)
 
         ret = yield self.third_party_rules.check_event_allowed(event, state_events)
-        return ret
+        defer.returnValue(ret)
 
     @defer.inlineCallbacks
     def on_create_room(self, requester, config, is_requester_admin):
@@ -98,7 +98,7 @@ class ThirdPartyEventRules(object):
         """
 
         if self.third_party_rules is None:
-            return True
+            defer.returnValue(True)
 
         state_ids = yield self.store.get_filtered_current_state_ids(room_id)
         room_state_events = yield self.store.get_events(state_ids.values())
@@ -110,4 +110,4 @@ class ThirdPartyEventRules(object):
         ret = yield self.third_party_rules.check_threepid_can_be_invited(
             medium, address, state_events
         )
-        return ret
+        defer.returnValue(ret)

synapse/events/utils.py

@@ -360,7 +360,7 @@ class EventClientSerializer(object):
         """
         # To handle the case of presence events and the like
         if not isinstance(event, EventBase):
-            return event
+            defer.returnValue(event)
 
         event_id = event.event_id
         serialized_event = serialize_event(event, time_now, **kwargs)
@@ -406,7 +406,7 @@ class EventClientSerializer(object):
                     "sender": edit.sender,
                 }
 
-        return serialized_event
+        defer.returnValue(serialized_event)
 
     def serialize_events(self, events, time_now, **kwargs):
         """Serializes multiple events.

synapse/events/validator.py

@@ -95,10 +95,10 @@ class EventValidator(object):
 
         elif event.type == EventTypes.Topic:
             self._ensure_strings(event.content, ["topic"])
-            self._ensure_state_event(event)
+
         elif event.type == EventTypes.Name:
             self._ensure_strings(event.content, ["name"])
-            self._ensure_state_event(event)
+
         elif event.type == EventTypes.Member:
             if "membership" not in event.content:
                 raise SynapseError(400, "Content has not membership key")
@@ -106,25 +106,9 @@ class EventValidator(object):
             if event.content["membership"] not in Membership.LIST:
                 raise SynapseError(400, "Invalid membership key")
 
-            self._ensure_state_event(event)
-        elif event.type == EventTypes.Tombstone:
-            if "replacement_room" not in event.content:
-                raise SynapseError(400, "Content has no replacement_room key")
-
-            if event.content["replacement_room"] == event.room_id:
-                raise SynapseError(
-                    400, "Tombstone cannot reference the room it was sent in"
-                )
-
-            self._ensure_state_event(event)
-
     def _ensure_strings(self, d, keys):
         for s in keys:
             if s not in d:
                 raise SynapseError(400, "'%s' not in content" % (s,))
             if not isinstance(d[s], string_types):
                 raise SynapseError(400, "'%s' not a string type" % (s,))
-
-    def _ensure_state_event(self, event):
-        if not event.is_state():
-            raise SynapseError(400, "'%s' must be state events" % (event.type,))

synapse/federation/federation_base.py

@@ -106,7 +106,7 @@ class FederationBase(object):
                     "Failed to find copy of %s with valid signature", pdu.event_id
                 )
 
-            return res
+            defer.returnValue(res)
 
         handle = preserve_fn(handle_check_result)
         deferreds2 = [handle(pdu, deferred) for pdu, deferred in zip(pdus, deferreds)]
@@ -116,9 +116,9 @@ class FederationBase(object):
         ).addErrback(unwrapFirstError)
 
         if include_none:
-            return valid_pdus
+            defer.returnValue(valid_pdus)
         else:
-            return [p for p in valid_pdus if p]
+            defer.returnValue([p for p in valid_pdus if p])
 
     def _check_sigs_and_hash(self, room_version, pdu):
         return make_deferred_yieldable(

synapse/federation/federation_client.py

@@ -213,7 +213,7 @@ class FederationClient(FederationBase):
             ).addErrback(unwrapFirstError)
         )
 
-        return pdus
+        defer.returnValue(pdus)
 
     @defer.inlineCallbacks
     @log_function
@@ -245,7 +245,7 @@ class FederationClient(FederationBase):
 
         ev = self._get_pdu_cache.get(event_id)
         if ev:
-            return ev
+            defer.returnValue(ev)
 
         pdu_attempts = self.pdu_destination_tried.setdefault(event_id, {})
 
@@ -307,7 +307,7 @@ class FederationClient(FederationBase):
         if signed_pdu:
             self._get_pdu_cache[event_id] = signed_pdu
 
-        return signed_pdu
+        defer.returnValue(signed_pdu)
 
     @defer.inlineCallbacks
     @log_function
@@ -355,7 +355,7 @@ class FederationClient(FederationBase):
 
             auth_chain.sort(key=lambda e: e.depth)
 
-            return (pdus, auth_chain)
+            defer.returnValue((pdus, auth_chain))
         except HttpResponseException as e:
             if e.code == 400 or e.code == 404:
                 logger.info("Failed to use get_room_state_ids API, falling back")
@@ -404,7 +404,7 @@ class FederationClient(FederationBase):
 
         signed_auth.sort(key=lambda e: e.depth)
 
-        return (signed_pdus, signed_auth)
+        defer.returnValue((signed_pdus, signed_auth))
 
     @defer.inlineCallbacks
     def get_events_from_store_or_dest(self, destination, room_id, event_ids):
@@ -429,7 +429,7 @@ class FederationClient(FederationBase):
             missing_events.discard(k)
 
         if not missing_events:
-            return (signed_events, failed_to_fetch)
+            defer.returnValue((signed_events, failed_to_fetch))
 
         logger.debug(
             "Fetching unknown state/auth events %s for room %s",
@@ -465,7 +465,7 @@ class FederationClient(FederationBase):
             # We removed all events we successfully fetched from `batch`
             failed_to_fetch.update(batch)
 
-        return (signed_events, failed_to_fetch)
+        defer.returnValue((signed_events, failed_to_fetch))
 
     @defer.inlineCallbacks
     @log_function
@@ -485,7 +485,7 @@ class FederationClient(FederationBase):
 
         signed_auth.sort(key=lambda e: e.depth)
 
-        return signed_auth
+        defer.returnValue(signed_auth)
 
     @defer.inlineCallbacks
     def _try_destination_list(self, description, destinations, callback):
@@ -511,8 +511,9 @@ class FederationClient(FederationBase):
             The [Deferred] result of callback, if it succeeds
 
         Raises:
-            SynapseError if the chosen remote server returns a 300/400 code, or
-            no servers were reachable.
+            SynapseError if the chosen remote server returns a 300/400 code.
+
+            RuntimeError if no servers were reachable.
         """
         for destination in destinations:
             if destination == self.server_name:
@@ -520,7 +521,7 @@ class FederationClient(FederationBase):
 
             try:
                 res = yield callback(destination)
-                return res
+                defer.returnValue(res)
             except InvalidResponseError as e:
                 logger.warn("Failed to %s via %s: %s", description, destination, e)
             except HttpResponseException as e:
@@ -537,7 +538,7 @@ class FederationClient(FederationBase):
             except Exception:
                 logger.warn("Failed to %s via %s", description, destination, exc_info=1)
 
-        raise SynapseError(502, "Failed to %s via any server" % (description,))
+        raise RuntimeError("Failed to %s via any server" % (description,))
 
     def make_membership_event(
         self, destinations, room_id, user_id, membership, content, params
@@ -614,7 +615,7 @@ class FederationClient(FederationBase):
                 event_dict=pdu_dict,
             )
 
-            return (destination, ev, event_format)
+            defer.returnValue((destination, ev, event_format))
 
         return self._try_destination_list(
             "make_" + membership, destinations, send_request
@@ -727,11 +728,13 @@ class FederationClient(FederationBase):
 
             check_authchain_validity(signed_auth)
 
-            return {
-                "state": signed_state,
-                "auth_chain": signed_auth,
-                "origin": destination,
-            }
+            defer.returnValue(
+                {
+                    "state": signed_state,
+                    "auth_chain": signed_auth,
+                    "origin": destination,
+                }
+            )
 
         return self._try_destination_list("send_join", destinations, send_request)
 
@@ -755,7 +758,7 @@ class FederationClient(FederationBase):
 
         # FIXME: We should handle signature failures more gracefully.
 
-        return pdu
+        defer.returnValue(pdu)
 
     @defer.inlineCallbacks
     def _do_send_invite(self, destination, pdu, room_version):
@@ -783,7 +786,7 @@ class FederationClient(FederationBase):
                     "invite_room_state": pdu.unsigned.get("invite_room_state", []),
                 },
             )
-            return content
+            defer.returnValue(content)
         except HttpResponseException as e:
             if e.code in [400, 404]:
                 err = e.to_synapse_error()
@@ -818,7 +821,7 @@ class FederationClient(FederationBase):
             event_id=pdu.event_id,
             content=pdu.get_pdu_json(time_now),
         )
-        return content
+        defer.returnValue(content)
 
     def send_leave(self, destinations, pdu):
         """Sends a leave event to one of a list of homeservers.
@@ -853,7 +856,7 @@ class FederationClient(FederationBase):
             )
 
             logger.debug("Got content: %s", content)
-            return None
+            defer.returnValue(None)
 
         return self._try_destination_list("send_leave", destinations, send_request)
 
@@ -914,7 +917,7 @@ class FederationClient(FederationBase):
             "missing": content.get("missing", []),
         }
 
-        return ret
+        defer.returnValue(ret)
 
     @defer.inlineCallbacks
     def get_missing_events(
@@ -971,7 +974,7 @@ class FederationClient(FederationBase):
             # get_missing_events
             signed_events = []
 
-        return signed_events
+        defer.returnValue(signed_events)
 
     @defer.inlineCallbacks
     def forward_third_party_invite(self, destinations, room_id, event_dict):
@@ -983,7 +986,7 @@ class FederationClient(FederationBase):
                 yield self.transport_layer.exchange_third_party_invite(
                     destination=destination, room_id=room_id, event_dict=event_dict
                 )
-                return None
+                defer.returnValue(None)
             except CodeMessageException:
                 raise
             except Exception as e:

synapse/federation/federation_server.py

@@ -99,7 +99,7 @@ class FederationServer(FederationBase):
 
             res = self._transaction_from_pdus(pdus).get_dict()
 
-        return (200, res)
+        defer.returnValue((200, res))
 
     @defer.inlineCallbacks
     @log_function
@@ -126,7 +126,7 @@ class FederationServer(FederationBase):
                 origin, transaction, request_time
             )
 
-        return result
+        defer.returnValue(result)
 
     @defer.inlineCallbacks
     def _handle_incoming_transaction(self, origin, transaction, request_time):
@@ -147,7 +147,8 @@ class FederationServer(FederationBase):
                 "[%s] We've already responded to this request",
                 transaction.transaction_id,
             )
-            return response
+            defer.returnValue(response)
+            return
 
         logger.debug("[%s] Transaction is new", transaction.transaction_id)
 
@@ -162,7 +163,7 @@ class FederationServer(FederationBase):
             yield self.transaction_actions.set_response(
                 origin, transaction, 400, response
             )
-            return (400, response)
+            defer.returnValue((400, response))
 
         received_pdus_counter.inc(len(transaction.pdus))
 
@@ -264,7 +265,7 @@ class FederationServer(FederationBase):
         logger.debug("Returning: %s", str(response))
 
         yield self.transaction_actions.set_response(origin, transaction, 200, response)
-        return (200, response)
+        defer.returnValue((200, response))
 
     @defer.inlineCallbacks
     def received_edu(self, origin, edu_type, content):
@@ -297,7 +298,7 @@ class FederationServer(FederationBase):
                 event_id,
             )
 
-        return (200, resp)
+        defer.returnValue((200, resp))
 
     @defer.inlineCallbacks
     def on_state_ids_request(self, origin, room_id, event_id):
@@ -314,7 +315,9 @@ class FederationServer(FederationBase):
         state_ids = yield self.handler.get_state_ids_for_pdu(room_id, event_id)
         auth_chain_ids = yield self.store.get_auth_chain_ids(state_ids)
 
-        return (200, {"pdu_ids": state_ids, "auth_chain_ids": auth_chain_ids})
+        defer.returnValue(
+            (200, {"pdu_ids": state_ids, "auth_chain_ids": auth_chain_ids})
+        )
 
     @defer.inlineCallbacks
     def _on_context_state_request_compute(self, room_id, event_id):
@@ -333,10 +336,12 @@ class FederationServer(FederationBase):
                     )
                 )
 
-        return {
-            "pdus": [pdu.get_pdu_json() for pdu in pdus],
-            "auth_chain": [pdu.get_pdu_json() for pdu in auth_chain],
-        }
+        defer.returnValue(
+            {
+                "pdus": [pdu.get_pdu_json() for pdu in pdus],
+                "auth_chain": [pdu.get_pdu_json() for pdu in auth_chain],
+            }
+        )
 
     @defer.inlineCallbacks
     @log_function
@@ -344,15 +349,15 @@ class FederationServer(FederationBase):
         pdu = yield self.handler.get_persisted_pdu(origin, event_id)
 
         if pdu:
-            return (200, self._transaction_from_pdus([pdu]).get_dict())
+            defer.returnValue((200, self._transaction_from_pdus([pdu]).get_dict()))
         else:
-            return (404, "")
+            defer.returnValue((404, ""))
 
     @defer.inlineCallbacks
     def on_query_request(self, query_type, args):
         received_queries_counter.labels(query_type).inc()
         resp = yield self.registry.on_query(query_type, args)
-        return (200, resp)
+        defer.returnValue((200, resp))
 
     @defer.inlineCallbacks
     def on_make_join_request(self, origin, room_id, user_id, supported_versions):
@@ -366,7 +371,9 @@ class FederationServer(FederationBase):
 
         pdu = yield self.handler.on_make_join_request(origin, room_id, user_id)
         time_now = self._clock.time_msec()
-        return {"event": pdu.get_pdu_json(time_now), "room_version": room_version}
+        defer.returnValue(
+            {"event": pdu.get_pdu_json(time_now), "room_version": room_version}
+        )
 
     @defer.inlineCallbacks
     def on_invite_request(self, origin, content, room_version):
@@ -384,7 +391,7 @@ class FederationServer(FederationBase):
         yield self.check_server_matches_acl(origin_host, pdu.room_id)
         ret_pdu = yield self.handler.on_invite_request(origin, pdu)
         time_now = self._clock.time_msec()
-        return {"event": ret_pdu.get_pdu_json(time_now)}
+        defer.returnValue({"event": ret_pdu.get_pdu_json(time_now)})
 
     @defer.inlineCallbacks
     def on_send_join_request(self, origin, content, room_id):
@@ -400,14 +407,16 @@ class FederationServer(FederationBase):
         logger.debug("on_send_join_request: pdu sigs: %s", pdu.signatures)
         res_pdus = yield self.handler.on_send_join_request(origin, pdu)
         time_now = self._clock.time_msec()
-        return (
-            200,
-            {
-                "state": [p.get_pdu_json(time_now) for p in res_pdus["state"]],
-                "auth_chain": [
-                    p.get_pdu_json(time_now) for p in res_pdus["auth_chain"]
-                ],
-            },
+        defer.returnValue(
+            (
+                200,
+                {
+                    "state": [p.get_pdu_json(time_now) for p in res_pdus["state"]],
+                    "auth_chain": [
+                        p.get_pdu_json(time_now) for p in res_pdus["auth_chain"]
+                    ],
+                },
+            )
         )
 
     @defer.inlineCallbacks
@@ -419,7 +428,9 @@ class FederationServer(FederationBase):
         room_version = yield self.store.get_room_version(room_id)
 
         time_now = self._clock.time_msec()
-        return {"event": pdu.get_pdu_json(time_now), "room_version": room_version}
+        defer.returnValue(
+            {"event": pdu.get_pdu_json(time_now), "room_version": room_version}
+        )
 
     @defer.inlineCallbacks
     def on_send_leave_request(self, origin, content, room_id):
@@ -434,7 +445,7 @@ class FederationServer(FederationBase):
 
         logger.debug("on_send_leave_request: pdu sigs: %s", pdu.signatures)
         yield self.handler.on_send_leave_request(origin, pdu)
-        return (200, {})
+        defer.returnValue((200, {}))
 
     @defer.inlineCallbacks
     def on_event_auth(self, origin, room_id, event_id):
@@ -445,7 +456,7 @@ class FederationServer(FederationBase):
             time_now = self._clock.time_msec()
             auth_pdus = yield self.handler.on_event_auth(event_id)
             res = {"auth_chain": [a.get_pdu_json(time_now) for a in auth_pdus]}
-        return (200, res)
+        defer.returnValue((200, res))
 
     @defer.inlineCallbacks
     def on_query_auth_request(self, origin, content, room_id, event_id):
@@ -498,7 +509,7 @@ class FederationServer(FederationBase):
                 "missing": ret.get("missing", []),
             }
 
-        return (200, send_content)
+        defer.returnValue((200, send_content))
 
     @log_function
     def on_query_client_keys(self, origin, content):
@@ -537,7 +548,7 @@ class FederationServer(FederationBase):
             ),
         )
 
-        return {"one_time_keys": json_result}
+        defer.returnValue({"one_time_keys": json_result})
 
     @defer.inlineCallbacks
     @log_function
@@ -569,7 +580,9 @@ class FederationServer(FederationBase):
 
             time_now = self._clock.time_msec()
 
-        return {"events": [ev.get_pdu_json(time_now) for ev in missing_events]}
+        defer.returnValue(
+            {"events": [ev.get_pdu_json(time_now) for ev in missing_events]}
+        )
 
     @log_function
     def on_openid_userinfo(self, token):
@@ -663,14 +676,14 @@ class FederationServer(FederationBase):
         ret = yield self.handler.exchange_third_party_invite(
             sender_user_id, target_user_id, room_id, signed
         )
-        return ret
+        defer.returnValue(ret)
 
     @defer.inlineCallbacks
     def on_exchange_third_party_invite_request(self, origin, room_id, event_dict):
         ret = yield self.handler.on_exchange_third_party_invite_request(
             origin, room_id, event_dict
         )
-        return ret
+        defer.returnValue(ret)
 
     @defer.inlineCallbacks
     def check_server_matches_acl(self, server_name, room_id):

synapse/federation/sender/per_destination_queue.py

@@ -374,7 +374,7 @@ class PerDestinationQueue(object):
 
         assert len(edus) <= limit, "get_devices_by_remote returned too many EDUs"
 
-        return (edus, now_stream_id)
+        defer.returnValue((edus, now_stream_id))
 
     @defer.inlineCallbacks
     def _get_to_device_message_edus(self, limit):
@@ -393,4 +393,4 @@ class PerDestinationQueue(object):
             for content in contents
         ]
 
-        return (edus, stream_id)
+        defer.returnValue((edus, stream_id))

synapse/federation/sender/transaction_manager.py

@@ -133,4 +133,4 @@ class TransactionManager(object):
                 )
             success = False
 
-        return success
+        defer.returnValue(success)

synapse/federation/transport/client.py

@@ -187,7 +187,7 @@ class TransportLayerClient(object):
             try_trailing_slash_on_400=True,
         )
 
-        return response
+        defer.returnValue(response)
 
     @defer.inlineCallbacks
     @log_function
@@ -205,7 +205,7 @@ class TransportLayerClient(object):
             ignore_backoff=ignore_backoff,
         )
 
-        return content
+        defer.returnValue(content)
 
     @defer.inlineCallbacks
     @log_function
@@ -263,7 +263,7 @@ class TransportLayerClient(object):
             ignore_backoff=ignore_backoff,
         )
 
-        return content
+        defer.returnValue(content)
 
     @defer.inlineCallbacks
     @log_function
@@ -274,7 +274,7 @@ class TransportLayerClient(object):
             destination=destination, path=path, data=content
         )
 
-        return response
+        defer.returnValue(response)
 
     @defer.inlineCallbacks
     @log_function
@@ -292,7 +292,7 @@ class TransportLayerClient(object):
             ignore_backoff=True,
         )
 
-        return response
+        defer.returnValue(response)
 
     @defer.inlineCallbacks
     @log_function
@@ -303,7 +303,7 @@ class TransportLayerClient(object):
             destination=destination, path=path, data=content, ignore_backoff=True
         )
 
-        return response
+        defer.returnValue(response)
 
     @defer.inlineCallbacks
     @log_function
@@ -314,7 +314,7 @@ class TransportLayerClient(object):
             destination=destination, path=path, data=content, ignore_backoff=True
         )
 
-        return response
+        defer.returnValue(response)
 
     @defer.inlineCallbacks
     @log_function
@@ -343,7 +343,7 @@ class TransportLayerClient(object):
             destination=remote_server, path=path, args=args, ignore_backoff=True
         )
 
-        return response
+        defer.returnValue(response)
 
     @defer.inlineCallbacks
     @log_function
@@ -354,7 +354,7 @@ class TransportLayerClient(object):
             destination=destination, path=path, data=event_dict
         )
 
-        return response
+        defer.returnValue(response)
 
     @defer.inlineCallbacks
     @log_function
@@ -363,7 +363,7 @@ class TransportLayerClient(object):
 
         content = yield self.client.get_json(destination=destination, path=path)
 
-        return content
+        defer.returnValue(content)
 
     @defer.inlineCallbacks
     @log_function
@@ -374,7 +374,7 @@ class TransportLayerClient(object):
             destination=destination, path=path, data=content
         )
 
-        return content
+        defer.returnValue(content)
 
     @defer.inlineCallbacks
     @log_function
@@ -406,7 +406,7 @@ class TransportLayerClient(object):
         content = yield self.client.post_json(
             destination=destination, path=path, data=query_content, timeout=timeout
         )
-        return content
+        defer.returnValue(content)
 
     @defer.inlineCallbacks
     @log_function
@@ -430,7 +430,7 @@ class TransportLayerClient(object):
         content = yield self.client.get_json(
             destination=destination, path=path, timeout=timeout
         )
-        return content
+        defer.returnValue(content)
 
     @defer.inlineCallbacks
     @log_function
@@ -464,7 +464,7 @@ class TransportLayerClient(object):
         content = yield self.client.post_json(
             destination=destination, path=path, data=query_content, timeout=timeout
         )
-        return content
+        defer.returnValue(content)
 
     @defer.inlineCallbacks
     @log_function
@@ -492,7 +492,7 @@ class TransportLayerClient(object):
             timeout=timeout,
         )
 
-        return content
+        defer.returnValue(content)
 
     @log_function
     def get_group_profile(self, destination, group_id, requester_user_id):

synapse/federation/transport/server.py

@@ -19,8 +19,6 @@ import functools
 import logging
 import re
 
-from twisted.internet.defer import maybeDeferred
-
 import synapse
 import synapse.logging.opentracing as opentracing
 from synapse.api.errors import Codes, FederationDeniedError, SynapseError
@@ -747,12 +745,8 @@ class PublicRoomList(BaseFederationServlet):
         else:
             network_tuple = ThirdPartyInstanceID(None, None)
 
-        data = await maybeDeferred(
-            self.handler.get_local_public_room_list,
-            limit,
-            since_token,
-            network_tuple=network_tuple,
-            from_federation=True,
+        data = await self.handler.get_local_public_room_list(
+            limit, since_token, network_tuple=network_tuple, from_federation=True
         )
         return 200, data
 

synapse/groups/attestations.py

@@ -157,7 +157,7 @@ class GroupAttestionRenewer(object):
 
         yield self.store.update_remote_attestion(group_id, user_id, attestation)
 
-        return {}
+        defer.returnValue({})
 
     def _start_renew_attestations(self):
         return run_as_background_process("renew_attestations", self._renew_attestations)

synapse/groups/groups_server.py

@@ -85,7 +85,7 @@ class GroupsServerHandler(object):
             if not is_admin:
                 raise SynapseError(403, "User is not admin in group")
 
-        return group
+        defer.returnValue(group)
 
     @defer.inlineCallbacks
     def get_group_summary(self, group_id, requester_user_id):
@@ -151,20 +151,22 @@ class GroupsServerHandler(object):
             group_id, requester_user_id
         )
 
-        return {
-            "profile": profile,
-            "users_section": {
-                "users": users,
-                "roles": roles,
-                "total_user_count_estimate": 0,  # TODO
-            },
-            "rooms_section": {
-                "rooms": rooms,
-                "categories": categories,
-                "total_room_count_estimate": 0,  # TODO
-            },
-            "user": membership_info,
-        }
+        defer.returnValue(
+            {
+                "profile": profile,
+                "users_section": {
+                    "users": users,
+                    "roles": roles,
+                    "total_user_count_estimate": 0,  # TODO
+                },
+                "rooms_section": {
+                    "rooms": rooms,
+                    "categories": categories,
+                    "total_room_count_estimate": 0,  # TODO
+                },
+                "user": membership_info,
+            }
+        )
 
     @defer.inlineCallbacks
     def update_group_summary_room(
@@ -190,7 +192,7 @@ class GroupsServerHandler(object):
             is_public=is_public,
         )
 
-        return {}
+        defer.returnValue({})
 
     @defer.inlineCallbacks
     def delete_group_summary_room(
@@ -206,7 +208,7 @@ class GroupsServerHandler(object):
             group_id=group_id, room_id=room_id, category_id=category_id
         )
 
-        return {}
+        defer.returnValue({})
 
     @defer.inlineCallbacks
     def set_group_join_policy(self, group_id, requester_user_id, content):
@@ -226,7 +228,7 @@ class GroupsServerHandler(object):
 
         yield self.store.set_group_join_policy(group_id, join_policy=join_policy)
 
-        return {}
+        defer.returnValue({})
 
     @defer.inlineCallbacks
     def get_group_categories(self, group_id, requester_user_id):
@@ -235,7 +237,7 @@ class GroupsServerHandler(object):
         yield self.check_group_is_ours(group_id, requester_user_id, and_exists=True)
 
         categories = yield self.store.get_group_categories(group_id=group_id)
-        return {"categories": categories}
+        defer.returnValue({"categories": categories})
 
     @defer.inlineCallbacks
     def get_group_category(self, group_id, requester_user_id, category_id):
@@ -247,7 +249,7 @@ class GroupsServerHandler(object):
             group_id=group_id, category_id=category_id
         )
 
-        return res
+        defer.returnValue(res)
 
     @defer.inlineCallbacks
     def update_group_category(self, group_id, requester_user_id, category_id, content):
@@ -267,7 +269,7 @@ class GroupsServerHandler(object):
             profile=profile,
         )
 
-        return {}
+        defer.returnValue({})
 
     @defer.inlineCallbacks
     def delete_group_category(self, group_id, requester_user_id, category_id):
@@ -281,7 +283,7 @@ class GroupsServerHandler(object):
             group_id=group_id, category_id=category_id
         )
 
-        return {}
+        defer.returnValue({})
 
     @defer.inlineCallbacks
     def get_group_roles(self, group_id, requester_user_id):
@@ -290,7 +292,7 @@ class GroupsServerHandler(object):
         yield self.check_group_is_ours(group_id, requester_user_id, and_exists=True)
 
         roles = yield self.store.get_group_roles(group_id=group_id)
-        return {"roles": roles}
+        defer.returnValue({"roles": roles})
 
     @defer.inlineCallbacks
     def get_group_role(self, group_id, requester_user_id, role_id):
@@ -299,7 +301,7 @@ class GroupsServerHandler(object):
         yield self.check_group_is_ours(group_id, requester_user_id, and_exists=True)
 
         res = yield self.store.get_group_role(group_id=group_id, role_id=role_id)
-        return res
+        defer.returnValue(res)
 
     @defer.inlineCallbacks
     def update_group_role(self, group_id, requester_user_id, role_id, content):
@@ -317,7 +319,7 @@ class GroupsServerHandler(object):
             group_id=group_id, role_id=role_id, is_public=is_public, profile=profile
         )
 
-        return {}
+        defer.returnValue({})
 
     @defer.inlineCallbacks
     def delete_group_role(self, group_id, requester_user_id, role_id):
@@ -329,7 +331,7 @@ class GroupsServerHandler(object):
 
         yield self.store.remove_group_role(group_id=group_id, role_id=role_id)
 
-        return {}
+        defer.returnValue({})
 
     @defer.inlineCallbacks
     def update_group_summary_user(
@@ -353,7 +355,7 @@ class GroupsServerHandler(object):
             is_public=is_public,
         )
 
-        return {}
+        defer.returnValue({})
 
     @defer.inlineCallbacks
     def delete_group_summary_user(self, group_id, requester_user_id, user_id, role_id):
@@ -367,7 +369,7 @@ class GroupsServerHandler(object):
             group_id=group_id, user_id=user_id, role_id=role_id
         )
 
-        return {}
+        defer.returnValue({})
 
     @defer.inlineCallbacks
     def get_group_profile(self, group_id, requester_user_id):
@@ -389,7 +391,7 @@ class GroupsServerHandler(object):
             group_description = {key: group[key] for key in cols}
             group_description["is_openly_joinable"] = group["join_policy"] == "open"
 
-            return group_description
+            defer.returnValue(group_description)
         else:
             raise SynapseError(404, "Unknown group")
 
@@ -459,7 +461,9 @@ class GroupsServerHandler(object):
 
         # TODO: If admin add lists of users whose attestations have timed out
 
-        return {"chunk": chunk, "total_user_count_estimate": len(user_results)}
+        defer.returnValue(
+            {"chunk": chunk, "total_user_count_estimate": len(user_results)}
+        )
 
     @defer.inlineCallbacks
     def get_invited_users_in_group(self, group_id, requester_user_id):
@@ -490,7 +494,9 @@ class GroupsServerHandler(object):
                 logger.warn("Error getting profile for %s: %s", user_id, e)
             user_profiles.append(user_profile)
 
-        return {"chunk": user_profiles, "total_user_count_estimate": len(invited_users)}
+        defer.returnValue(
+            {"chunk": user_profiles, "total_user_count_estimate": len(invited_users)}
+        )
 
     @defer.inlineCallbacks
     def get_rooms_in_group(self, group_id, requester_user_id):
@@ -527,7 +533,9 @@ class GroupsServerHandler(object):
 
         chunk.sort(key=lambda e: -e["num_joined_members"])
 
-        return {"chunk": chunk, "total_room_count_estimate": len(room_results)}
+        defer.returnValue(
+            {"chunk": chunk, "total_room_count_estimate": len(room_results)}
+        )
 
     @defer.inlineCallbacks
     def add_room_to_group(self, group_id, requester_user_id, room_id, content):
@@ -543,7 +551,7 @@ class GroupsServerHandler(object):
 
         yield self.store.add_room_to_group(group_id, room_id, is_public=is_public)
 
-        return {}
+        defer.returnValue({})
 
     @defer.inlineCallbacks
     def update_room_in_group(
@@ -566,7 +574,7 @@ class GroupsServerHandler(object):
         else:
             raise SynapseError(400, "Uknown config option")
 
-        return {}
+        defer.returnValue({})
 
     @defer.inlineCallbacks
     def remove_room_from_group(self, group_id, requester_user_id, room_id):
@@ -578,7 +586,7 @@ class GroupsServerHandler(object):
 
         yield self.store.remove_room_from_group(group_id, room_id)
 
-        return {}
+        defer.returnValue({})
 
     @defer.inlineCallbacks
     def invite_to_group(self, group_id, user_id, requester_user_id, content):
@@ -636,9 +644,9 @@ class GroupsServerHandler(object):
             )
         elif res["state"] == "invite":
             yield self.store.add_group_invite(group_id, user_id)
-            return {"state": "invite"}
+            defer.returnValue({"state": "invite"})
         elif res["state"] == "reject":
-            return {"state": "reject"}
+            defer.returnValue({"state": "reject"})
         else:
             raise SynapseError(502, "Unknown state returned by HS")
 
@@ -671,7 +679,7 @@ class GroupsServerHandler(object):
             remote_attestation=remote_attestation,
         )
 
-        return local_attestation
+        defer.returnValue(local_attestation)
 
     @defer.inlineCallbacks
     def accept_invite(self, group_id, requester_user_id, content):
@@ -691,7 +699,7 @@ class GroupsServerHandler(object):
 
         local_attestation = yield self._add_user(group_id, requester_user_id, content)
 
-        return {"state": "join", "attestation": local_attestation}
+        defer.returnValue({"state": "join", "attestation": local_attestation})
 
     @defer.inlineCallbacks
     def join_group(self, group_id, requester_user_id, content):
@@ -708,7 +716,7 @@ class GroupsServerHandler(object):
 
         local_attestation = yield self._add_user(group_id, requester_user_id, content)
 
-        return {"state": "join", "attestation": local_attestation}
+        defer.returnValue({"state": "join", "attestation": local_attestation})
 
     @defer.inlineCallbacks
     def knock(self, group_id, requester_user_id, content):
@@ -761,7 +769,7 @@ class GroupsServerHandler(object):
         if not self.hs.is_mine_id(user_id):
             yield self.store.maybe_delete_remote_profile_cache(user_id)
 
-        return {}
+        defer.returnValue({})
 
     @defer.inlineCallbacks
     def create_group(self, group_id, requester_user_id, content):
@@ -837,7 +845,7 @@ class GroupsServerHandler(object):
                 avatar_url=user_profile.get("avatar_url"),
             )
 
-        return {"group_id": group_id}
+        defer.returnValue({"group_id": group_id})
 
     @defer.inlineCallbacks
     def delete_group(self, group_id, requester_user_id):

synapse/handlers/_base.py

@@ -45,7 +45,6 @@ class BaseHandler(object):
         self.state_handler = hs.get_state_handler()
         self.distributor = hs.get_distributor()
         self.ratelimiter = hs.get_ratelimiter()
-        self.admin_redaction_ratelimiter = hs.get_admin_redaction_ratelimiter()
         self.clock = hs.get_clock()
         self.hs = hs
 
@@ -54,7 +53,7 @@ class BaseHandler(object):
         self.event_builder_factory = hs.get_event_builder_factory()
 
     @defer.inlineCallbacks
-    def ratelimit(self, requester, update=True, is_admin_redaction=False):
+    def ratelimit(self, requester, update=True):
         """Ratelimits requests.
 
         Args:
@@ -63,9 +62,6 @@ class BaseHandler(object):
                 Set to False when doing multiple checks for one request (e.g.
                 to check up front if we would reject the request), and set to
                 True for the last call for a given request.
-            is_admin_redaction (bool): Whether this is a room admin/moderator
-                redacting an event. If so then we may apply different
-                ratelimits depending on config.
 
         Raises:
             LimitExceededError if the request should be ratelimited
@@ -94,33 +90,16 @@ class BaseHandler(object):
             messages_per_second = override.messages_per_second
             burst_count = override.burst_count
         else:
-            # We default to different values if this is an admin redaction and
-            # the config is set
-            if is_admin_redaction and self.hs.config.rc_admin_redaction:
-                messages_per_second = self.hs.config.rc_admin_redaction.per_second
-                burst_count = self.hs.config.rc_admin_redaction.burst_count
-            else:
-                messages_per_second = self.hs.config.rc_message.per_second
-                burst_count = self.hs.config.rc_message.burst_count
+            messages_per_second = self.hs.config.rc_message.per_second
+            burst_count = self.hs.config.rc_message.burst_count
 
-        if is_admin_redaction and self.hs.config.rc_admin_redaction:
-            # If we have separate config for admin redactions we use a separate
-            # ratelimiter
-            allowed, time_allowed = self.admin_redaction_ratelimiter.can_do_action(
-                user_id,
-                time_now,
-                rate_hz=messages_per_second,
-                burst_count=burst_count,
-                update=update,
-            )
-        else:
-            allowed, time_allowed = self.ratelimiter.can_do_action(
-                user_id,
-                time_now,
-                rate_hz=messages_per_second,
-                burst_count=burst_count,
-                update=update,
-            )
+        allowed, time_allowed = self.ratelimiter.can_do_action(
+            user_id,
+            time_now,
+            rate_hz=messages_per_second,
+            burst_count=burst_count,
+            update=update,
+        )
         if not allowed:
             raise LimitExceededError(
                 retry_after_ms=int(1000 * (time_allowed - time_now))

synapse/handlers/account_data.py

@@ -51,8 +51,8 @@ class AccountDataEventSource(object):
                     {"type": account_data_type, "content": content, "room_id": room_id}
                 )
 
-        return (results, current_stream_id)
+        defer.returnValue((results, current_stream_id))
 
     @defer.inlineCallbacks
     def get_pagination_rows(self, user, config, key):
-        return ([], config.to_id)
+        defer.returnValue(([], config.to_id))

synapse/handlers/account_validity.py

@@ -193,7 +193,7 @@ class AccountValidityHandler(object):
             if threepid["medium"] == "email":
                 addresses.append(threepid["address"])
 
-        return addresses
+        defer.returnValue(addresses)
 
     @defer.inlineCallbacks
     def _get_renewal_token(self, user_id):
@@ -214,7 +214,7 @@ class AccountValidityHandler(object):
             try:
                 renewal_token = stringutils.random_string(32)
                 yield self.store.set_renewal_token_for_user(user_id, renewal_token)
-                return renewal_token
+                defer.returnValue(renewal_token)
             except StoreError:
                 attempts += 1
         raise StoreError(500, "Couldn't generate a unique string as refresh string.")
@@ -226,19 +226,11 @@ class AccountValidityHandler(object):
 
         Args:
             renewal_token (str): Token sent with the renewal request.
-        Returns:
-            bool: Whether the provided token is valid.
         """
-        try:
-            user_id = yield self.store.get_user_from_renewal_token(renewal_token)
-        except StoreError:
-            defer.returnValue(False)
-
+        user_id = yield self.store.get_user_from_renewal_token(renewal_token)
         logger.debug("Renewing an account for user %s", user_id)
         yield self.renew_account_for_user(user_id)
 
-        defer.returnValue(True)
-
     @defer.inlineCallbacks
     def renew_account_for_user(self, user_id, expiration_ts=None, email_sent=False):
         """Renews the account attached to a given user by pushing back the
@@ -262,4 +254,4 @@ class AccountValidityHandler(object):
             user_id=user_id, expiration_ts=expiration_ts, email_sent=email_sent
         )
 
-        return expiration_ts
+        defer.returnValue(expiration_ts)

synapse/handlers/acme.py

@@ -100,4 +100,4 @@ class AcmeHandler(object):
             logger.exception("Failed saving!")
             raise
 
-        return True
+        defer.returnValue(True)

synapse/handlers/admin.py

@@ -49,7 +49,7 @@ class AdminHandler(BaseHandler):
             "devices": {"": {"sessions": [{"connections": connections}]}},
         }
 
-        return ret
+        defer.returnValue(ret)
 
     @defer.inlineCallbacks
     def get_users(self):
@@ -61,7 +61,7 @@ class AdminHandler(BaseHandler):
         """
         ret = yield self.store.get_users()
 
-        return ret
+        defer.returnValue(ret)
 
     @defer.inlineCallbacks
     def get_users_paginate(self, order, start, limit):
@@ -78,7 +78,7 @@ class AdminHandler(BaseHandler):
         """
         ret = yield self.store.get_users_paginate(order, start, limit)
 
-        return ret
+        defer.returnValue(ret)
 
     @defer.inlineCallbacks
     def search_users(self, term):
@@ -92,26 +92,7 @@ class AdminHandler(BaseHandler):
         """
         ret = yield self.store.search_users(term)
 
-        return ret
-
-    def get_user_server_admin(self, user):
-        """
-        Get the admin bit on a user.
-
-        Args:
-            user_id (UserID): the (necessarily local) user to manipulate
-        """
-        return self.store.is_server_admin(user)
-
-    def set_user_server_admin(self, user, admin):
-        """
-        Set the admin bit on a user.
-
-        Args:
-            user_id (UserID): the (necessarily local) user to manipulate
-            admin (bool): whether or not the user should be an admin of this server
-        """
-        return self.store.set_server_admin(user, admin)
+        defer.returnValue(ret)
 
     @defer.inlineCallbacks
     def export_user_data(self, user_id, writer):
@@ -244,7 +225,7 @@ class AdminHandler(BaseHandler):
                 state = yield self.store.get_state_for_event(event_id)
                 writer.write_state(room_id, event_id, state)
 
-        return writer.finished()
+        defer.returnValue(writer.finished())
 
 
 class ExfiltrationWriter(object):

synapse/handlers/appservice.py

@@ -167,8 +167,8 @@ class ApplicationServicesHandler(object):
         for user_service in user_query_services:
             is_known_user = yield self.appservice_api.query_user(user_service, user_id)
             if is_known_user:
-                return True
-        return False
+                defer.returnValue(True)
+        defer.returnValue(False)
 
     @defer.inlineCallbacks
     def query_room_alias_exists(self, room_alias):
@@ -192,7 +192,7 @@ class ApplicationServicesHandler(object):
             if is_known_alias:
                 # the alias exists now so don't query more ASes.
                 result = yield self.store.get_association_from_room_alias(room_alias)
-                return result
+                defer.returnValue(result)
 
     @defer.inlineCallbacks
     def query_3pe(self, kind, protocol, fields):
@@ -215,7 +215,7 @@ class ApplicationServicesHandler(object):
             if success:
                 ret.extend(result)
 
-        return ret
+        defer.returnValue(ret)
 
     @defer.inlineCallbacks
     def get_3pe_protocols(self, only_protocol=None):
@@ -254,7 +254,7 @@ class ApplicationServicesHandler(object):
         for p in protocols.keys():
             protocols[p] = _merge_instances(protocols[p])
 
-        return protocols
+        defer.returnValue(protocols)
 
     @defer.inlineCallbacks
     def _get_services_for_event(self, event):
@@ -276,7 +276,7 @@ class ApplicationServicesHandler(object):
             if (yield s.is_interested(event, self.store)):
                 interested_list.append(s)
 
-        return interested_list
+        defer.returnValue(interested_list)
 
     def _get_services_for_user(self, user_id):
         services = self.store.get_app_services()
@@ -293,23 +293,23 @@ class ApplicationServicesHandler(object):
         if not self.is_mine_id(user_id):
             # we don't know if they are unknown or not since it isn't one of our
             # users. We can't poke ASes.
-            return False
+            defer.returnValue(False)
             return
 
         user_info = yield self.store.get_user_by_id(user_id)
         if user_info:
-            return False
+            defer.returnValue(False)
             return
 
         # user not found; could be the AS though, so check.
         services = self.store.get_app_services()
         service_list = [s for s in services if s.sender == user_id]
-        return len(service_list) == 0
+        defer.returnValue(len(service_list) == 0)
 
     @defer.inlineCallbacks
     def _check_user_exists(self, user_id):
         unknown_user = yield self._is_unknown_user(user_id)
         if unknown_user:
             exists = yield self.query_user_exists(user_id)
-            return exists
-        return True
+            defer.returnValue(exists)
+        defer.returnValue(True)

synapse/handlers/auth.py

@@ -155,7 +155,7 @@ class AuthHandler(BaseHandler):
         if user_id != requester.user.to_string():
             raise AuthError(403, "Invalid auth")
 
-        return params
+        defer.returnValue(params)
 
     @defer.inlineCallbacks
     def check_auth(self, flows, clientdict, clientip, password_servlet=False):
@@ -280,7 +280,7 @@ class AuthHandler(BaseHandler):
                     creds,
                     list(clientdict),
                 )
-                return (creds, clientdict, session["id"])
+                defer.returnValue((creds, clientdict, session["id"]))
 
         ret = self._auth_dict_for_flows(flows, session)
         ret["completed"] = list(creds)
@@ -307,8 +307,8 @@ class AuthHandler(BaseHandler):
         if result:
             creds[stagetype] = result
             self._save_session(sess)
-            return True
-        return False
+            defer.returnValue(True)
+        defer.returnValue(False)
 
     def get_session_id(self, clientdict):
         """
@@ -379,7 +379,7 @@ class AuthHandler(BaseHandler):
             res = yield checker(
                 authdict, clientip=clientip, password_servlet=password_servlet
             )
-            return res
+            defer.returnValue(res)
 
         # build a v1-login-style dict out of the authdict and fall back to the
         # v1 code
@@ -389,7 +389,7 @@ class AuthHandler(BaseHandler):
             raise SynapseError(400, "", Codes.MISSING_PARAM)
 
         (canonical_id, callback) = yield self.validate_login(user_id, authdict)
-        return canonical_id
+        defer.returnValue(canonical_id)
 
     @defer.inlineCallbacks
     def _check_recaptcha(self, authdict, clientip, **kwargs):
@@ -433,7 +433,7 @@ class AuthHandler(BaseHandler):
                 resp_body.get("hostname"),
             )
             if resp_body["success"]:
-                return True
+                defer.returnValue(True)
         raise LoginError(401, "", errcode=Codes.UNAUTHORIZED)
 
     def _check_email_identity(self, authdict, **kwargs):
@@ -502,7 +502,7 @@ class AuthHandler(BaseHandler):
 
         threepid["threepid_creds"] = authdict["threepid_creds"]
 
-        return threepid
+        defer.returnValue(threepid)
 
     def _get_params_recaptcha(self):
         return {"public_key": self.hs.config.recaptcha_public_key}
@@ -606,7 +606,7 @@ class AuthHandler(BaseHandler):
                 yield self.store.delete_access_token(access_token)
                 raise StoreError(400, "Login raced against device deletion")
 
-        return access_token
+        defer.returnValue(access_token)
 
     @defer.inlineCallbacks
     def check_user_exists(self, user_id):
@@ -629,8 +629,8 @@ class AuthHandler(BaseHandler):
         self.ratelimit_login_per_account(user_id)
         res = yield self._find_user_id_and_pwd_hash(user_id)
         if res is not None:
-            return res[0]
-        return None
+            defer.returnValue(res[0])
+        defer.returnValue(None)
 
     @defer.inlineCallbacks
     def _find_user_id_and_pwd_hash(self, user_id):
@@ -661,7 +661,7 @@ class AuthHandler(BaseHandler):
                 user_id,
                 user_infos.keys(),
             )
-        return result
+        defer.returnValue(result)
 
     def get_supported_login_types(self):
         """Get a the login types supported for the /login API
@@ -722,7 +722,7 @@ class AuthHandler(BaseHandler):
                 known_login_type = True
                 is_valid = yield provider.check_password(qualified_user_id, password)
                 if is_valid:
-                    return (qualified_user_id, None)
+                    defer.returnValue((qualified_user_id, None))
 
             if not hasattr(provider, "get_supported_login_types") or not hasattr(
                 provider, "check_auth"
@@ -756,7 +756,7 @@ class AuthHandler(BaseHandler):
             if result:
                 if isinstance(result, str):
                     result = (result, None)
-                return result
+                defer.returnValue(result)
 
         if login_type == LoginType.PASSWORD and self.hs.config.password_localdb_enabled:
             known_login_type = True
@@ -766,7 +766,7 @@ class AuthHandler(BaseHandler):
             )
 
             if canonical_user_id:
-                return (canonical_user_id, None)
+                defer.returnValue((canonical_user_id, None))
 
         if not known_login_type:
             raise SynapseError(400, "Unknown login type %s" % login_type)
@@ -814,9 +814,9 @@ class AuthHandler(BaseHandler):
                     if isinstance(result, str):
                         # If it's a str, set callback function to None
                         result = (result, None)
-                    return result
+                    defer.returnValue(result)
 
-        return (None, None)
+        defer.returnValue((None, None))
 
     @defer.inlineCallbacks
     def _check_local_password(self, user_id, password):
@@ -838,7 +838,7 @@ class AuthHandler(BaseHandler):
         """
         lookupres = yield self._find_user_id_and_pwd_hash(user_id)
         if not lookupres:
-            return None
+            defer.returnValue(None)
         (user_id, password_hash) = lookupres
 
         # If the password hash is None, the account has likely been deactivated
@@ -850,8 +850,8 @@ class AuthHandler(BaseHandler):
         result = yield self.validate_hash(password, password_hash)
         if not result:
             logger.warn("Failed password login for user %s", user_id)
-            return None
-        return user_id
+            defer.returnValue(None)
+        defer.returnValue(user_id)
 
     @defer.inlineCallbacks
     def validate_short_term_login_token_and_get_user_id(self, login_token):
@@ -860,12 +860,12 @@ class AuthHandler(BaseHandler):
         try:
             macaroon = pymacaroons.Macaroon.deserialize(login_token)
             user_id = auth_api.get_user_id_from_macaroon(macaroon)
-            auth_api.validate_macaroon(macaroon, "login", user_id)
+            auth_api.validate_macaroon(macaroon, "login", True, user_id)
         except Exception:
             raise AuthError(403, "Invalid token", errcode=Codes.FORBIDDEN)
         self.ratelimit_login_per_account(user_id)
         yield self.auth.check_auth_blocking(user_id)
-        return user_id
+        defer.returnValue(user_id)
 
     @defer.inlineCallbacks
     def delete_access_token(self, access_token):
@@ -976,7 +976,7 @@ class AuthHandler(BaseHandler):
         )
 
         yield self.store.user_delete_threepid(user_id, medium, address)
-        return result
+        defer.returnValue(result)
 
     def _save_session(self, session):
         # TODO: Persistent storage

synapse/handlers/deactivate_account.py

@@ -125,7 +125,7 @@ class DeactivateAccountHandler(BaseHandler):
         # Mark the user as deactivated.
         yield self.store.set_user_deactivated_status(user_id, True)
 
-        return identity_server_supports_unbinding
+        defer.returnValue(identity_server_supports_unbinding)
 
     def _start_user_parting(self):
         """

synapse/handlers/device.py

@@ -64,7 +64,7 @@ class DeviceWorkerHandler(BaseHandler):
         for device in devices:
             _update_device_from_client_ips(device, ips)
 
-        return devices
+        defer.returnValue(devices)
 
     @defer.inlineCallbacks
     def get_device(self, user_id, device_id):
@@ -85,7 +85,7 @@ class DeviceWorkerHandler(BaseHandler):
             raise errors.NotFoundError
         ips = yield self.store.get_last_client_ip_by_device(user_id, device_id)
         _update_device_from_client_ips(device, ips)
-        return device
+        defer.returnValue(device)
 
     @measure_func("device.get_user_ids_changed")
     @defer.inlineCallbacks
@@ -200,7 +200,9 @@ class DeviceWorkerHandler(BaseHandler):
             possibly_joined = []
             possibly_left = []
 
-        return {"changed": list(possibly_joined), "left": list(possibly_left)}
+        defer.returnValue(
+            {"changed": list(possibly_joined), "left": list(possibly_left)}
+        )
 
 
 class DeviceHandler(DeviceWorkerHandler):
@@ -209,12 +211,12 @@ class DeviceHandler(DeviceWorkerHandler):
 
         self.federation_sender = hs.get_federation_sender()
 
-        self.device_list_updater = DeviceListUpdater(hs, self)
+        self._edu_updater = DeviceListEduUpdater(hs, self)
 
         federation_registry = hs.get_federation_registry()
 
         federation_registry.register_edu_handler(
-            "m.device_list_update", self.device_list_updater.incoming_device_list_update
+            "m.device_list_update", self._edu_updater.incoming_device_list_update
         )
         federation_registry.register_query_handler(
             "user_devices", self.on_federation_query_user_devices
@@ -248,7 +250,7 @@ class DeviceHandler(DeviceWorkerHandler):
             )
             if new_device:
                 yield self.notify_device_update(user_id, [device_id])
-            return device_id
+            defer.returnValue(device_id)
 
         # if the device id is not specified, we'll autogen one, but loop a few
         # times in case of a clash.
@@ -262,7 +264,7 @@ class DeviceHandler(DeviceWorkerHandler):
             )
             if new_device:
                 yield self.notify_device_update(user_id, [device_id])
-                return device_id
+                defer.returnValue(device_id)
             attempts += 1
 
         raise errors.StoreError(500, "Couldn't generate a device ID.")
@@ -409,7 +411,9 @@ class DeviceHandler(DeviceWorkerHandler):
     @defer.inlineCallbacks
     def on_federation_query_user_devices(self, user_id):
         stream_id, devices = yield self.store.get_devices_with_keys_by_user(user_id)
-        return {"user_id": user_id, "stream_id": stream_id, "devices": devices}
+        defer.returnValue(
+            {"user_id": user_id, "stream_id": stream_id, "devices": devices}
+        )
 
     @defer.inlineCallbacks
     def user_left_room(self, user, room_id):
@@ -426,7 +430,7 @@ def _update_device_from_client_ips(device, client_ips):
     device.update({"last_seen_ts": ip.get("last_seen"), "last_seen_ip": ip.get("ip")})
 
 
-class DeviceListUpdater(object):
+class DeviceListEduUpdater(object):
     "Handles incoming device list updates from federation and updates the DB"
 
     def __init__(self, hs, device_handler):
@@ -519,7 +523,75 @@ class DeviceListUpdater(object):
             logger.debug("Need to re-sync devices for %r? %r", user_id, resync)
 
             if resync:
-                yield self.user_device_resync(user_id)
+                # Fetch all devices for the user.
+                origin = get_domain_from_id(user_id)
+                try:
+                    result = yield self.federation.query_user_devices(origin, user_id)
+                except (
+                    NotRetryingDestination,
+                    RequestSendFailed,
+                    HttpResponseException,
+                ):
+                    # TODO: Remember that we are now out of sync and try again
+                    # later
+                    logger.warn("Failed to handle device list update for %s", user_id)
+                    # We abort on exceptions rather than accepting the update
+                    # as otherwise synapse will 'forget' that its device list
+                    # is out of date. If we bail then we will retry the resync
+                    # next time we get a device list update for this user_id.
+                    # This makes it more likely that the device lists will
+                    # eventually become consistent.
+                    return
+                except FederationDeniedError as e:
+                    logger.info(e)
+                    return
+                except Exception:
+                    # TODO: Remember that we are now out of sync and try again
+                    # later
+                    logger.exception(
+                        "Failed to handle device list update for %s", user_id
+                    )
+                    return
+
+                stream_id = result["stream_id"]
+                devices = result["devices"]
+
+                # If the remote server has more than ~1000 devices for this user
+                # we assume that something is going horribly wrong (e.g. a bot
+                # that logs in and creates a new device every time it tries to
+                # send a message).  Maintaining lots of devices per user in the
+                # cache can cause serious performance issues as if this request
+                # takes more than 60s to complete, internal replication from the
+                # inbound federation worker to the synapse master may time out
+                # causing the inbound federation to fail and causing the remote
+                # server to retry, causing a DoS.  So in this scenario we give
+                # up on storing the total list of devices and only handle the
+                # delta instead.
+                if len(devices) > 1000:
+                    logger.warn(
+                        "Ignoring device list snapshot for %s as it has >1K devs (%d)",
+                        user_id,
+                        len(devices),
+                    )
+                    devices = []
+
+                for device in devices:
+                    logger.debug(
+                        "Handling resync update %r/%r, ID: %r",
+                        user_id,
+                        device["device_id"],
+                        stream_id,
+                    )
+
+                yield self.store.update_remote_device_list_cache(
+                    user_id, devices, stream_id
+                )
+                device_ids = [device["device_id"] for device in devices]
+                yield self.device_handler.notify_device_update(user_id, device_ids)
+
+                # We clobber the seen updates since we've re-synced from a given
+                # point.
+                self._seen_updates[user_id] = set([stream_id])
             else:
                 # Simply update the single device, since we know that is the only
                 # change (because of the single prev_id matching the current cache)
@@ -551,7 +623,7 @@ class DeviceListUpdater(object):
         for _, stream_id, prev_ids, _ in updates:
             if not prev_ids:
                 # We always do a resync if there are no previous IDs
-                return True
+                defer.returnValue(True)
 
             for prev_id in prev_ids:
                 if prev_id == extremity:
@@ -561,82 +633,8 @@ class DeviceListUpdater(object):
                 elif prev_id in stream_id_in_updates:
                     continue
                 else:
-                    return True
+                    defer.returnValue(True)
 
             stream_id_in_updates.add(stream_id)
 
-        return False
-
-    @defer.inlineCallbacks
-    def user_device_resync(self, user_id):
-        """Fetches all devices for a user and updates the device cache with them.
-
-        Args:
-            user_id (str): The user's id whose device_list will be updated.
-        Returns:
-            Deferred[dict]: a dict with device info as under the "devices" in the result of this
-            request:
-            https://matrix.org/docs/spec/server_server/r0.1.2#get-matrix-federation-v1-user-devices-userid
-        """
-        # Fetch all devices for the user.
-        origin = get_domain_from_id(user_id)
-        try:
-            result = yield self.federation.query_user_devices(origin, user_id)
-        except (NotRetryingDestination, RequestSendFailed, HttpResponseException):
-            # TODO: Remember that we are now out of sync and try again
-            # later
-            logger.warn("Failed to handle device list update for %s", user_id)
-            # We abort on exceptions rather than accepting the update
-            # as otherwise synapse will 'forget' that its device list
-            # is out of date. If we bail then we will retry the resync
-            # next time we get a device list update for this user_id.
-            # This makes it more likely that the device lists will
-            # eventually become consistent.
-            return
-        except FederationDeniedError as e:
-            logger.info(e)
-            return
-        except Exception:
-            # TODO: Remember that we are now out of sync and try again
-            # later
-            logger.exception("Failed to handle device list update for %s", user_id)
-            return
-        stream_id = result["stream_id"]
-        devices = result["devices"]
-
-        # If the remote server has more than ~1000 devices for this user
-        # we assume that something is going horribly wrong (e.g. a bot
-        # that logs in and creates a new device every time it tries to
-        # send a message).  Maintaining lots of devices per user in the
-        # cache can cause serious performance issues as if this request
-        # takes more than 60s to complete, internal replication from the
-        # inbound federation worker to the synapse master may time out
-        # causing the inbound federation to fail and causing the remote
-        # server to retry, causing a DoS.  So in this scenario we give
-        # up on storing the total list of devices and only handle the
-        # delta instead.
-        if len(devices) > 1000:
-            logger.warn(
-                "Ignoring device list snapshot for %s as it has >1K devs (%d)",
-                user_id,
-                len(devices),
-            )
-            devices = []
-
-        for device in devices:
-            logger.debug(
-                "Handling resync update %r/%r, ID: %r",
-                user_id,
-                device["device_id"],
-                stream_id,
-            )
-
-        yield self.store.update_remote_device_list_cache(user_id, devices, stream_id)
-        device_ids = [device["device_id"] for device in devices]
-        yield self.device_handler.notify_device_update(user_id, device_ids)
-
-        # We clobber the seen updates since we've re-synced from a given
-        # point.
-        self._seen_updates[user_id] = set([stream_id])
-
-        defer.returnValue(result)
+        defer.returnValue(False)

synapse/handlers/directory.py

@@ -210,7 +210,7 @@ class DirectoryHandler(BaseHandler):
         except AuthError as e:
             logger.info("Failed to update alias events: %s", e)
 
-        return room_id
+        defer.returnValue(room_id)
 
     @defer.inlineCallbacks
     def delete_appservice_association(self, service, room_alias):
@@ -229,7 +229,7 @@ class DirectoryHandler(BaseHandler):
 
         room_id = yield self.store.delete_room_alias(room_alias)
 
-        return room_id
+        defer.returnValue(room_id)
 
     @defer.inlineCallbacks
     def get_association(self, room_alias):
@@ -277,7 +277,8 @@ class DirectoryHandler(BaseHandler):
         else:
             servers = list(servers)
 
-        return {"room_id": room_id, "servers": servers}
+        defer.returnValue({"room_id": room_id, "servers": servers})
+        return
 
     @defer.inlineCallbacks
     def on_directory_query(self, args):
@@ -288,7 +289,7 @@ class DirectoryHandler(BaseHandler):
         result = yield self.get_association_from_room_alias(room_alias)
 
         if result is not None:
-            return {"room_id": result.room_id, "servers": result.servers}
+            defer.returnValue({"room_id": result.room_id, "servers": result.servers})
         else:
             raise SynapseError(
                 404,
@@ -341,7 +342,7 @@ class DirectoryHandler(BaseHandler):
             # Query AS to see if it exists
             as_handler = self.appservice_handler
             result = yield as_handler.query_room_alias_exists(room_alias)
-        return result
+        defer.returnValue(result)
 
     def can_modify_alias(self, alias, user_id=None):
         # Any application service "interested" in an alias they are regexing on
@@ -368,10 +369,10 @@ class DirectoryHandler(BaseHandler):
         creator = yield self.store.get_room_alias_creator(alias.to_string())
 
         if creator is not None and creator == user_id:
-            return True
+            defer.returnValue(True)
 
         is_admin = yield self.auth.is_server_admin(UserID.from_string(user_id))
-        return is_admin
+        defer.returnValue(is_admin)
 
     @defer.inlineCallbacks
     def edit_published_room_list(self, requester, room_id, visibility):

synapse/handlers/e2e_keys.py

@@ -25,7 +25,6 @@ from twisted.internet import defer
 from synapse.api.errors import CodeMessageException, SynapseError
 from synapse.logging.context import make_deferred_yieldable, run_in_background
 from synapse.types import UserID, get_domain_from_id
-from synapse.util import unwrapFirstError
 from synapse.util.retryutils import NotRetryingDestination
 
 logger = logging.getLogger(__name__)
@@ -66,7 +65,6 @@ class E2eKeysHandler(object):
             }
         }
         """
-
         device_keys_query = query_body.get("device_keys", {})
 
         # separate users by domain.
@@ -123,56 +121,7 @@ class E2eKeysHandler(object):
         # Now fetch any devices that we don't have in our cache
         @defer.inlineCallbacks
         def do_remote_query(destination):
-            """This is called when we are querying the device list of a user on
-            a remote homeserver and their device list is not in the device list
-            cache. If we share a room with this user and we're not querying for
-            specific user we will update the cache
-            with their device list."""
-
             destination_query = remote_queries_not_in_cache[destination]
-
-            # We first consider whether we wish to update the device list cache with
-            # the users device list. We want to track a user's devices when the
-            # authenticated user shares a room with the queried user and the query
-            # has not specified a particular device.
-            # If we update the cache for the queried user we remove them from further
-            # queries. We use the more efficient batched query_client_keys for all
-            # remaining users
-            user_ids_updated = []
-            for (user_id, device_list) in destination_query.items():
-                if user_id in user_ids_updated:
-                    continue
-
-                if device_list:
-                    continue
-
-                room_ids = yield self.store.get_rooms_for_user(user_id)
-                if not room_ids:
-                    continue
-
-                # We've decided we're sharing a room with this user and should
-                # probably be tracking their device lists. However, we haven't
-                # done an initial sync on the device list so we do it now.
-                try:
-                    user_devices = yield self.device_handler.device_list_updater.user_device_resync(
-                        user_id
-                    )
-                    user_devices = user_devices["devices"]
-                    for device in user_devices:
-                        results[user_id] = {device["device_id"]: device["keys"]}
-                    user_ids_updated.append(user_id)
-                except Exception as e:
-                    failures[destination] = _exception_to_failure(e)
-
-            if len(destination_query) == len(user_ids_updated):
-                # We've updated all the users in the query and we do not need to
-                # make any further remote calls.
-                return
-
-            # Remove all the users from the query which we have updated
-            for user_id in user_ids_updated:
-                destination_query.pop(user_id)
-
             try:
                 remote_result = yield self.federation.query_client_keys(
                     destination, {"device_keys": destination_query}, timeout=timeout
@@ -183,8 +132,7 @@ class E2eKeysHandler(object):
                         results[user_id] = keys
 
             except Exception as e:
-                failure = _exception_to_failure(e)
-                failures[destination] = failure
+                failures[destination] = _exception_to_failure(e)
 
         yield make_deferred_yieldable(
             defer.gatherResults(
@@ -193,10 +141,10 @@ class E2eKeysHandler(object):
                     for destination in remote_queries_not_in_cache
                 ],
                 consumeErrors=True,
-            ).addErrback(unwrapFirstError)
+            )
         )
 
-        return {"device_keys": results, "failures": failures}
+        defer.returnValue({"device_keys": results, "failures": failures})
 
     @defer.inlineCallbacks
     def query_local_devices(self, query):
@@ -241,7 +189,7 @@ class E2eKeysHandler(object):
                     r["unsigned"]["device_display_name"] = display_name
                 result_dict[user_id][device_id] = r
 
-        return result_dict
+        defer.returnValue(result_dict)
 
     @defer.inlineCallbacks
     def on_federation_query_client_keys(self, query_body):
@@ -249,7 +197,7 @@ class E2eKeysHandler(object):
         """
         device_keys_query = query_body.get("device_keys", {})
         res = yield self.query_local_devices(device_keys_query)
-        return {"device_keys": res}
+        defer.returnValue({"device_keys": res})
 
     @defer.inlineCallbacks
     def claim_one_time_keys(self, query, timeout):
@@ -286,10 +234,8 @@ class E2eKeysHandler(object):
                 for user_id, keys in remote_result["one_time_keys"].items():
                     if user_id in device_keys:
                         json_result[user_id] = keys
-
             except Exception as e:
-                failure = _exception_to_failure(e)
-                failures[destination] = failure
+                failures[destination] = _exception_to_failure(e)
 
         yield make_deferred_yieldable(
             defer.gatherResults(
@@ -313,11 +259,10 @@ class E2eKeysHandler(object):
             ),
         )
 
-        return {"one_time_keys": json_result, "failures": failures}
+        defer.returnValue({"one_time_keys": json_result, "failures": failures})
 
     @defer.inlineCallbacks
     def upload_keys_for_user(self, user_id, device_id, keys):
-
         time_now = self.clock.time_msec()
 
         # TODO: Validate the JSON to make sure it has the right keys.
@@ -352,7 +297,7 @@ class E2eKeysHandler(object):
 
         result = yield self.store.count_e2e_one_time_keys(user_id, device_id)
 
-        return {"one_time_key_counts": result}
+        defer.returnValue({"one_time_key_counts": result})
 
     @defer.inlineCallbacks
     def _upload_one_time_keys_for_user(

synapse/handlers/e2e_room_keys.py

@@ -84,7 +84,7 @@ class E2eRoomKeysHandler(object):
                 user_id, version, room_id, session_id
             )
 
-            return results
+            defer.returnValue(results)
 
     @defer.inlineCallbacks
     def delete_room_keys(self, user_id, version, room_id=None, session_id=None):
@@ -262,7 +262,7 @@ class E2eRoomKeysHandler(object):
             new_version = yield self.store.create_e2e_room_keys_version(
                 user_id, version_info
             )
-            return new_version
+            defer.returnValue(new_version)
 
     @defer.inlineCallbacks
     def get_version_info(self, user_id, version=None):
@@ -292,7 +292,7 @@ class E2eRoomKeysHandler(object):
                     raise NotFoundError("Unknown backup version")
                 else:
                     raise
-            return res
+            defer.returnValue(res)
 
     @defer.inlineCallbacks
     def delete_version(self, user_id, version=None):
@@ -350,4 +350,4 @@ class E2eRoomKeysHandler(object):
                 user_id, version, version_info
             )
 
-            return {}
+            defer.returnValue({})

synapse/handlers/events.py

@@ -143,7 +143,7 @@ class EventStreamHandler(BaseHandler):
                 "end": tokens[1].to_string(),
             }
 
-            return chunk
+            defer.returnValue(chunk)
 
 
 class EventHandler(BaseHandler):
@@ -166,7 +166,7 @@ class EventHandler(BaseHandler):
         event = yield self.store.get_event(event_id, check_room_id=room_id)
 
         if not event:
-            return None
+            defer.returnValue(None)
             return
 
         users = yield self.store.get_users_in_room(event.room_id)
@@ -179,4 +179,4 @@ class EventHandler(BaseHandler):
         if not filtered:
             raise AuthError(403, "You don't have permission to access that event.")
 
-        return event
+        defer.returnValue(event)

synapse/handlers/federation.py

@@ -210,7 +210,7 @@ class FederationHandler(BaseHandler):
                 event_id,
                 origin,
             )
-            return None
+            defer.returnValue(None)
 
         state = None
         auth_chain = []
@@ -676,7 +676,7 @@ class FederationHandler(BaseHandler):
         events = [e for e in events if e.event_id not in seen_events]
 
         if not events:
-            return []
+            defer.returnValue([])
 
         event_map = {e.event_id: e for e in events}
 
@@ -838,7 +838,7 @@ class FederationHandler(BaseHandler):
             # TODO: We can probably do something more clever here.
             yield self._handle_new_event(dest, event, backfilled=True)
 
-        return events
+        defer.returnValue(events)
 
     @defer.inlineCallbacks
     def maybe_backfill(self, room_id, current_depth):
@@ -894,7 +894,7 @@ class FederationHandler(BaseHandler):
         )
 
         if not filtered_extremities:
-            return False
+            defer.returnValue(False)
 
         # Check if we reached a point where we should start backfilling.
         sorted_extremeties_tuple = sorted(extremities.items(), key=lambda e: -int(e[1]))
@@ -965,7 +965,7 @@ class FederationHandler(BaseHandler):
                     # If this succeeded then we probably already have the
                     # appropriate stuff.
                     # TODO: We can probably do something more intelligent here.
-                    return True
+                    defer.returnValue(True)
                 except SynapseError as e:
                     logger.info("Failed to backfill from %s because %s", dom, e)
                     continue
@@ -978,9 +978,6 @@ class FederationHandler(BaseHandler):
                 except NotRetryingDestination as e:
                     logger.info(str(e))
                     continue
-                except RequestSendFailed as e:
-                    logger.info("Falied to get backfill from %s because %s", dom, e)
-                    continue
                 except FederationDeniedError as e:
                     logger.info(e)
                     continue
@@ -988,11 +985,11 @@ class FederationHandler(BaseHandler):
                     logger.exception("Failed to backfill from %s because %s", dom, e)
                     continue
 
-            return False
+            defer.returnValue(False)
 
         success = yield try_backfill(likely_domains)
         if success:
-            return True
+            defer.returnValue(True)
 
         # Huh, well *those* domains didn't work out. Lets try some domains
         # from the time.
@@ -1034,11 +1031,11 @@ class FederationHandler(BaseHandler):
                 [dom for dom, _ in likely_domains if dom not in tried_domains]
             )
             if success:
-                return True
+                defer.returnValue(True)
 
             tried_domains.update(dom for dom, _ in likely_domains)
 
-        return False
+        defer.returnValue(False)
 
     def _sanity_check_event(self, ev):
         """
@@ -1085,7 +1082,7 @@ class FederationHandler(BaseHandler):
             pdu=event,
         )
 
-        return pdu
+        defer.returnValue(pdu)
 
     @defer.inlineCallbacks
     def on_event_auth(self, event_id):
@@ -1093,7 +1090,7 @@ class FederationHandler(BaseHandler):
         auth = yield self.store.get_auth_chain(
             [auth_id for auth_id in event.auth_event_ids()], include_given=True
         )
-        return [e for e in auth]
+        defer.returnValue([e for e in auth])
 
     @log_function
     @defer.inlineCallbacks
@@ -1180,7 +1177,7 @@ class FederationHandler(BaseHandler):
 
             run_in_background(self._handle_queued_pdus, room_queue)
 
-        return True
+        defer.returnValue(True)
 
     @defer.inlineCallbacks
     def _handle_queued_pdus(self, room_queue):
@@ -1267,7 +1264,7 @@ class FederationHandler(BaseHandler):
             room_version, event, context, do_sig_check=False
         )
 
-        return event
+        defer.returnValue(event)
 
     @defer.inlineCallbacks
     @log_function
@@ -1328,7 +1325,7 @@ class FederationHandler(BaseHandler):
 
         state = yield self.store.get_events(list(prev_state_ids.values()))
 
-        return {"state": list(state.values()), "auth_chain": auth_chain}
+        defer.returnValue({"state": list(state.values()), "auth_chain": auth_chain})
 
     @defer.inlineCallbacks
     def on_invite_request(self, origin, pdu):
@@ -1384,7 +1381,7 @@ class FederationHandler(BaseHandler):
         context = yield self.state_handler.compute_event_context(event)
         yield self.persist_events_and_notify([(event, context)])
 
-        return event
+        defer.returnValue(event)
 
     @defer.inlineCallbacks
     def do_remotely_reject_invite(self, target_hosts, room_id, user_id):
@@ -1409,7 +1406,7 @@ class FederationHandler(BaseHandler):
         context = yield self.state_handler.compute_event_context(event)
         yield self.persist_events_and_notify([(event, context)])
 
-        return event
+        defer.returnValue(event)
 
     @defer.inlineCallbacks
     def _make_and_verify_event(
@@ -1427,7 +1424,7 @@ class FederationHandler(BaseHandler):
         assert event.user_id == user_id
         assert event.state_key == user_id
         assert event.room_id == room_id
-        return (origin, event, format_ver)
+        defer.returnValue((origin, event, format_ver))
 
     @defer.inlineCallbacks
     @log_function
@@ -1487,7 +1484,7 @@ class FederationHandler(BaseHandler):
             logger.warn("Failed to create new leave %r because %s", event, e)
             raise e
 
-        return event
+        defer.returnValue(event)
 
     @defer.inlineCallbacks
     @log_function
@@ -1520,7 +1517,7 @@ class FederationHandler(BaseHandler):
             event.signatures,
         )
 
-        return None
+        defer.returnValue(None)
 
     @defer.inlineCallbacks
     def get_state_for_pdu(self, room_id, event_id):
@@ -1548,9 +1545,9 @@ class FederationHandler(BaseHandler):
                     del results[(event.type, event.state_key)]
 
             res = list(results.values())
-            return res
+            defer.returnValue(res)
         else:
-            return []
+            defer.returnValue([])
 
     @defer.inlineCallbacks
     def get_state_ids_for_pdu(self, room_id, event_id):
@@ -1575,9 +1572,9 @@ class FederationHandler(BaseHandler):
                 else:
                     results.pop((event.type, event.state_key), None)
 
-            return list(results.values())
+            defer.returnValue(list(results.values()))
         else:
-            return []
+            defer.returnValue([])
 
     @defer.inlineCallbacks
     @log_function
@@ -1590,7 +1587,7 @@ class FederationHandler(BaseHandler):
 
         events = yield filter_events_for_server(self.store, origin, events)
 
-        return events
+        defer.returnValue(events)
 
     @defer.inlineCallbacks
     @log_function
@@ -1620,9 +1617,9 @@ class FederationHandler(BaseHandler):
 
             events = yield filter_events_for_server(self.store, origin, [event])
             event = events[0]
-            return event
+            defer.returnValue(event)
         else:
-            return None
+            defer.returnValue(None)
 
     def get_min_depth_for_context(self, context):
         return self.store.get_min_depth(context)
@@ -1654,7 +1651,7 @@ class FederationHandler(BaseHandler):
                     self.store.remove_push_actions_from_staging, event.event_id
                 )
 
-        return context
+        defer.returnValue(context)
 
     @defer.inlineCallbacks
     def _handle_new_events(self, origin, event_infos, backfilled=False):
@@ -1677,7 +1674,7 @@ class FederationHandler(BaseHandler):
                     auth_events=ev_info.get("auth_events"),
                     backfilled=backfilled,
                 )
-            return res
+            defer.returnValue(res)
 
         contexts = yield make_deferred_yieldable(
             defer.gatherResults(
@@ -1836,7 +1833,7 @@ class FederationHandler(BaseHandler):
         if event.type == EventTypes.GuestAccess and not context.rejected:
             yield self.maybe_kick_guest_users(event)
 
-        return context
+        defer.returnValue(context)
 
     @defer.inlineCallbacks
     def _check_for_soft_fail(self, event, state, backfilled):
@@ -1955,7 +1952,7 @@ class FederationHandler(BaseHandler):
 
         logger.debug("on_query_auth returning: %s", ret)
 
-        return ret
+        defer.returnValue(ret)
 
     @defer.inlineCallbacks
     def on_get_missing_events(
@@ -1978,7 +1975,7 @@ class FederationHandler(BaseHandler):
             self.store, origin, missing_events
         )
 
-        return missing_events
+        defer.returnValue(missing_events)
 
     @defer.inlineCallbacks
     @log_function
@@ -2454,14 +2451,16 @@ class FederationHandler(BaseHandler):
 
         logger.debug("construct_auth_difference returning")
 
-        return {
-            "auth_chain": local_auth,
-            "rejects": {
-                e.event_id: {"reason": reason_map[e.event_id], "proof": None}
-                for e in base_remote_rejected
-            },
-            "missing": [e.event_id for e in missing_locals],
-        }
+        defer.returnValue(
+            {
+                "auth_chain": local_auth,
+                "rejects": {
+                    e.event_id: {"reason": reason_map[e.event_id], "proof": None}
+                    for e in base_remote_rejected
+                },
+                "missing": [e.event_id for e in missing_locals],
+            }
+        )
 
     @defer.inlineCallbacks
     @log_function
@@ -2609,7 +2608,7 @@ class FederationHandler(BaseHandler):
             builder=builder
         )
         EventValidator().validate_new(event)
-        return (event, context)
+        defer.returnValue((event, context))
 
     @defer.inlineCallbacks
     def _check_signature(self, event, context):

synapse/handlers/groups_local.py

@@ -126,12 +126,9 @@ class GroupsLocalHandler(object):
                 group_id, requester_user_id
             )
         else:
-            try:
-                res = yield self.transport_client.get_group_summary(
-                    get_domain_from_id(group_id), group_id, requester_user_id
-                )
-            except RequestSendFailed:
-                raise SynapseError(502, "Failed to contact group server")
+            res = yield self.transport_client.get_group_summary(
+                get_domain_from_id(group_id), group_id, requester_user_id
+            )
 
             group_server_name = get_domain_from_id(group_id)
 
@@ -165,7 +162,7 @@ class GroupsLocalHandler(object):
 
         res.setdefault("user", {})["is_publicised"] = is_publicised
 
-        return res
+        defer.returnValue(res)
 
     @defer.inlineCallbacks
     def create_group(self, group_id, user_id, content):
@@ -186,12 +183,9 @@ class GroupsLocalHandler(object):
 
             content["user_profile"] = yield self.profile_handler.get_profile(user_id)
 
-            try:
-                res = yield self.transport_client.create_group(
-                    get_domain_from_id(group_id), group_id, user_id, content
-                )
-            except RequestSendFailed:
-                raise SynapseError(502, "Failed to contact group server")
+            res = yield self.transport_client.create_group(
+                get_domain_from_id(group_id), group_id, user_id, content
+            )
 
             remote_attestation = res["attestation"]
             yield self.attestations.verify_attestation(
@@ -213,7 +207,7 @@ class GroupsLocalHandler(object):
         )
         self.notifier.on_new_event("groups_key", token, users=[user_id])
 
-        return res
+        defer.returnValue(res)
 
     @defer.inlineCallbacks
     def get_users_in_group(self, group_id, requester_user_id):
@@ -223,16 +217,13 @@ class GroupsLocalHandler(object):
             res = yield self.groups_server_handler.get_users_in_group(
                 group_id, requester_user_id
             )
-            return res
+            defer.returnValue(res)
 
         group_server_name = get_domain_from_id(group_id)
 
-        try:
-            res = yield self.transport_client.get_users_in_group(
-                get_domain_from_id(group_id), group_id, requester_user_id
-            )
-        except RequestSendFailed:
-            raise SynapseError(502, "Failed to contact group server")
+        res = yield self.transport_client.get_users_in_group(
+            get_domain_from_id(group_id), group_id, requester_user_id
+        )
 
         chunk = res["chunk"]
         valid_entries = []
@@ -253,7 +244,7 @@ class GroupsLocalHandler(object):
 
         res["chunk"] = valid_entries
 
-        return res
+        defer.returnValue(res)
 
     @defer.inlineCallbacks
     def join_group(self, group_id, user_id, content):
@@ -267,12 +258,9 @@ class GroupsLocalHandler(object):
             local_attestation = self.attestations.create_attestation(group_id, user_id)
             content["attestation"] = local_attestation
 
-            try:
-                res = yield self.transport_client.join_group(
-                    get_domain_from_id(group_id), group_id, user_id, content
-                )
-            except RequestSendFailed:
-                raise SynapseError(502, "Failed to contact group server")
+            res = yield self.transport_client.join_group(
+                get_domain_from_id(group_id), group_id, user_id, content
+            )
 
             remote_attestation = res["attestation"]
 
@@ -297,7 +285,7 @@ class GroupsLocalHandler(object):
         )
         self.notifier.on_new_event("groups_key", token, users=[user_id])
 
-        return {}
+        defer.returnValue({})
 
     @defer.inlineCallbacks
     def accept_invite(self, group_id, user_id, content):
@@ -311,12 +299,9 @@ class GroupsLocalHandler(object):
             local_attestation = self.attestations.create_attestation(group_id, user_id)
             content["attestation"] = local_attestation
 
-            try:
-                res = yield self.transport_client.accept_group_invite(
-                    get_domain_from_id(group_id), group_id, user_id, content
-                )
-            except RequestSendFailed:
-                raise SynapseError(502, "Failed to contact group server")
+            res = yield self.transport_client.accept_group_invite(
+                get_domain_from_id(group_id), group_id, user_id, content
+            )
 
             remote_attestation = res["attestation"]
 
@@ -341,7 +326,7 @@ class GroupsLocalHandler(object):
         )
         self.notifier.on_new_event("groups_key", token, users=[user_id])
 
-        return {}
+        defer.returnValue({})
 
     @defer.inlineCallbacks
     def invite(self, group_id, user_id, requester_user_id, config):
@@ -353,18 +338,15 @@ class GroupsLocalHandler(object):
                 group_id, user_id, requester_user_id, content
             )
         else:
-            try:
-                res = yield self.transport_client.invite_to_group(
-                    get_domain_from_id(group_id),
-                    group_id,
-                    user_id,
-                    requester_user_id,
-                    content,
-                )
-            except RequestSendFailed:
-                raise SynapseError(502, "Failed to contact group server")
+            res = yield self.transport_client.invite_to_group(
+                get_domain_from_id(group_id),
+                group_id,
+                user_id,
+                requester_user_id,
+                content,
+            )
 
-        return res
+        defer.returnValue(res)
 
     @defer.inlineCallbacks
     def on_invite(self, group_id, user_id, content):
@@ -395,7 +377,7 @@ class GroupsLocalHandler(object):
             logger.warn("No profile for user %s: %s", user_id, e)
             user_profile = {}
 
-        return {"state": "invite", "user_profile": user_profile}
+        defer.returnValue({"state": "invite", "user_profile": user_profile})
 
     @defer.inlineCallbacks
     def remove_user_from_group(self, group_id, user_id, requester_user_id, content):
@@ -416,18 +398,15 @@ class GroupsLocalHandler(object):
             )
         else:
             content["requester_user_id"] = requester_user_id
-            try:
-                res = yield self.transport_client.remove_user_from_group(
-                    get_domain_from_id(group_id),
-                    group_id,
-                    requester_user_id,
-                    user_id,
-                    content,
-                )
-            except RequestSendFailed:
-                raise SynapseError(502, "Failed to contact group server")
+            res = yield self.transport_client.remove_user_from_group(
+                get_domain_from_id(group_id),
+                group_id,
+                requester_user_id,
+                user_id,
+                content,
+            )
 
-        return res
+        defer.returnValue(res)
 
     @defer.inlineCallbacks
     def user_removed_from_group(self, group_id, user_id, content):
@@ -442,7 +421,7 @@ class GroupsLocalHandler(object):
     @defer.inlineCallbacks
     def get_joined_groups(self, user_id):
         group_ids = yield self.store.get_joined_groups(user_id)
-        return {"groups": group_ids}
+        defer.returnValue({"groups": group_ids})
 
     @defer.inlineCallbacks
     def get_publicised_groups_for_user(self, user_id):
@@ -454,18 +433,14 @@ class GroupsLocalHandler(object):
             for app_service in self.store.get_app_services():
                 result.extend(app_service.get_groups_for_user(user_id))
 
-            return {"groups": result}
+            defer.returnValue({"groups": result})
         else:
-            try:
-                bulk_result = yield self.transport_client.bulk_get_publicised_groups(
-                    get_domain_from_id(user_id), [user_id]
-                )
-            except RequestSendFailed:
-                raise SynapseError(502, "Failed to contact group server")
-
+            bulk_result = yield self.transport_client.bulk_get_publicised_groups(
+                get_domain_from_id(user_id), [user_id]
+            )
             result = bulk_result.get("users", {}).get(user_id)
             # TODO: Verify attestations
-            return {"groups": result}
+            defer.returnValue({"groups": result})
 
     @defer.inlineCallbacks
     def bulk_get_publicised_groups(self, user_ids, proxy=True):
@@ -500,4 +475,4 @@ class GroupsLocalHandler(object):
             for app_service in self.store.get_app_services():
                 results[uid].extend(app_service.get_groups_for_user(uid))
 
-        return {"users": results}
+        defer.returnValue({"users": results})

synapse/handlers/identity.py

@@ -82,7 +82,7 @@ class IdentityHandler(BaseHandler):
                 "%s is not a trusted ID server: rejecting 3pid " + "credentials",
                 id_server,
             )
-            return None
+            defer.returnValue(None)
 
         try:
             data = yield self.http_client.get_json(
@@ -95,8 +95,8 @@ class IdentityHandler(BaseHandler):
             raise e.to_synapse_error()
 
         if "medium" in data:
-            return data
-        return None
+            defer.returnValue(data)
+        defer.returnValue(None)
 
     @defer.inlineCallbacks
     def bind_threepid(self, creds, mxid):
@@ -133,7 +133,7 @@ class IdentityHandler(BaseHandler):
             )
         except CodeMessageException as e:
             data = json.loads(e.msg)  # XXX WAT?
-        return data
+        defer.returnValue(data)
 
     @defer.inlineCallbacks
     def try_unbind_threepid(self, mxid, threepid):
@@ -161,7 +161,7 @@ class IdentityHandler(BaseHandler):
 
         # We don't know where to unbind, so we don't have a choice but to return
         if not id_servers:
-            return False
+            defer.returnValue(False)
 
         changed = True
         for id_server in id_servers:
@@ -169,7 +169,7 @@ class IdentityHandler(BaseHandler):
                 mxid, threepid, id_server
             )
 
-        return changed
+        defer.returnValue(changed)
 
     @defer.inlineCallbacks
     def try_unbind_threepid_with_id_server(self, mxid, threepid, id_server):
@@ -224,7 +224,7 @@ class IdentityHandler(BaseHandler):
             id_server=id_server,
         )
 
-        return changed
+        defer.returnValue(changed)
 
     @defer.inlineCallbacks
     def requestEmailToken(
@@ -250,7 +250,7 @@ class IdentityHandler(BaseHandler):
                 % (id_server, "/_matrix/identity/api/v1/validate/email/requestToken"),
                 params,
             )
-            return data
+            defer.returnValue(data)
         except HttpResponseException as e:
             logger.info("Proxied requestToken failed: %r", e)
             raise e.to_synapse_error()
@@ -278,7 +278,7 @@ class IdentityHandler(BaseHandler):
                 % (id_server, "/_matrix/identity/api/v1/validate/msisdn/requestToken"),
                 params,
             )
-            return data
+            defer.returnValue(data)
         except HttpResponseException as e:
             logger.info("Proxied requestToken failed: %r", e)
             raise e.to_synapse_error()

synapse/handlers/initial_sync.py

@@ -250,7 +250,7 @@ class InitialSyncHandler(BaseHandler):
             "end": now_token.to_string(),
         }
 
-        return ret
+        defer.returnValue(ret)
 
     @defer.inlineCallbacks
     def room_initial_sync(self, requester, room_id, pagin_config=None):
@@ -301,7 +301,7 @@ class InitialSyncHandler(BaseHandler):
 
         result["account_data"] = account_data_events
 
-        return result
+        defer.returnValue(result)
 
     @defer.inlineCallbacks
     def _room_initial_sync_parted(
@@ -330,24 +330,28 @@ class InitialSyncHandler(BaseHandler):
 
         time_now = self.clock.time_msec()
 
-        return {
-            "membership": membership,
-            "room_id": room_id,
-            "messages": {
-                "chunk": (
-                    yield self._event_serializer.serialize_events(messages, time_now)
+        defer.returnValue(
+            {
+                "membership": membership,
+                "room_id": room_id,
+                "messages": {
+                    "chunk": (
+                        yield self._event_serializer.serialize_events(
+                            messages, time_now
+                        )
+                    ),
+                    "start": start_token.to_string(),
+                    "end": end_token.to_string(),
+                },
+                "state": (
+                    yield self._event_serializer.serialize_events(
+                        room_state.values(), time_now
+                    )
                 ),
-                "start": start_token.to_string(),
-                "end": end_token.to_string(),
-            },
-            "state": (
-                yield self._event_serializer.serialize_events(
-                    room_state.values(), time_now
-                )
-            ),
-            "presence": [],
-            "receipts": [],
-        }
+                "presence": [],
+                "receipts": [],
+            }
+        )
 
     @defer.inlineCallbacks
     def _room_initial_sync_joined(
@@ -380,13 +384,13 @@ class InitialSyncHandler(BaseHandler):
         def get_presence():
             # If presence is disabled, return an empty list
             if not self.hs.config.use_presence:
-                return []
+                defer.returnValue([])
 
             states = yield presence_handler.get_states(
                 [m.user_id for m in room_members], as_event=True
             )
 
-            return states
+            defer.returnValue(states)
 
         @defer.inlineCallbacks
         def get_receipts():
@@ -395,7 +399,7 @@ class InitialSyncHandler(BaseHandler):
             )
             if not receipts:
                 receipts = []
-            return receipts
+            defer.returnValue(receipts)
 
         presence, receipts, (messages, token) = yield make_deferred_yieldable(
             defer.gatherResults(
@@ -438,7 +442,7 @@ class InitialSyncHandler(BaseHandler):
         if not is_peeking:
             ret["membership"] = membership
 
-        return ret
+        defer.returnValue(ret)
 
     @defer.inlineCallbacks
     def _check_in_room_or_world_readable(self, room_id, user_id):
@@ -449,7 +453,7 @@ class InitialSyncHandler(BaseHandler):
             #  * The user is a guest user, and has joined the room
             # else it will throw.
             member_event = yield self.auth.check_user_was_in_room(room_id, user_id)
-            return (member_event.membership, member_event.event_id)
+            defer.returnValue((member_event.membership, member_event.event_id))
             return
         except AuthError:
             visibility = yield self.state_handler.get_current_state(
@@ -459,7 +463,7 @@ class InitialSyncHandler(BaseHandler):
                 visibility
                 and visibility.content["history_visibility"] == "world_readable"
             ):
-                return (Membership.JOIN, None)
+                defer.returnValue((Membership.JOIN, None))
                 return
             raise AuthError(
                 403, "Guest access not allowed", errcode=Codes.GUEST_ACCESS_FORBIDDEN

synapse/handlers/message.py

@@ -87,7 +87,7 @@ class MessageHandler(object):
             )
             data = room_state[membership_event_id].get(key)
 
-        return data
+        defer.returnValue(data)
 
     @defer.inlineCallbacks
     def get_state_events(
@@ -174,7 +174,7 @@ class MessageHandler(object):
             # events, as clients won't use them.
             bundle_aggregations=False,
         )
-        return events
+        defer.returnValue(events)
 
     @defer.inlineCallbacks
     def get_joined_members(self, requester, room_id):
@@ -213,13 +213,15 @@ class MessageHandler(object):
                 # Loop fell through, AS has no interested users in room
                 raise AuthError(403, "Appservice not in room")
 
-        return {
-            user_id: {
-                "avatar_url": profile.avatar_url,
-                "display_name": profile.display_name,
+        defer.returnValue(
+            {
+                user_id: {
+                    "avatar_url": profile.avatar_url,
+                    "display_name": profile.display_name,
+                }
+                for user_id, profile in iteritems(users_with_profile)
             }
-            for user_id, profile in iteritems(users_with_profile)
-        }
+        )
 
 
 class EventCreationHandler(object):
@@ -378,11 +380,7 @@ class EventCreationHandler(object):
             # tolerate them in event_auth.check().
             prev_state_ids = yield context.get_prev_state_ids(self.store)
             prev_event_id = prev_state_ids.get((EventTypes.Member, event.sender))
-            prev_event = (
-                yield self.store.get_event(prev_event_id, allow_none=True)
-                if prev_event_id
-                else None
-            )
+            prev_event = yield self.store.get_event(prev_event_id, allow_none=True)
             if not prev_event or prev_event.membership != Membership.JOIN:
                 logger.warning(
                     (
@@ -400,7 +398,7 @@ class EventCreationHandler(object):
 
         self.validator.validate_new(event)
 
-        return (event, context)
+        defer.returnValue((event, context))
 
     def _is_exempt_from_privacy_policy(self, builder, requester):
         """"Determine if an event to be sent is exempt from having to consent
@@ -427,9 +425,9 @@ class EventCreationHandler(object):
     @defer.inlineCallbacks
     def _is_server_notices_room(self, room_id):
         if self.config.server_notices_mxid is None:
-            return False
+            defer.returnValue(False)
         user_ids = yield self.store.get_users_in_room(room_id)
-        return self.config.server_notices_mxid in user_ids
+        defer.returnValue(self.config.server_notices_mxid in user_ids)
 
     @defer.inlineCallbacks
     def assert_accepted_privacy_policy(self, requester):
@@ -509,7 +507,7 @@ class EventCreationHandler(object):
                     event.event_id,
                     prev_state.event_id,
                 )
-                return prev_state
+                defer.returnValue(prev_state)
 
         yield self.handle_new_client_event(
             requester=requester, event=event, context=context, ratelimit=ratelimit
@@ -525,8 +523,6 @@ class EventCreationHandler(object):
         """
         prev_state_ids = yield context.get_prev_state_ids(self.store)
         prev_event_id = prev_state_ids.get((event.type, event.state_key))
-        if not prev_event_id:
-            return
         prev_event = yield self.store.get_event(prev_event_id, allow_none=True)
         if not prev_event:
             return
@@ -535,7 +531,7 @@ class EventCreationHandler(object):
             prev_content = encode_canonical_json(prev_event.content)
             next_content = encode_canonical_json(event.content)
             if prev_content == next_content:
-                return prev_event
+                defer.returnValue(prev_event)
         return
 
     @defer.inlineCallbacks
@@ -567,7 +563,7 @@ class EventCreationHandler(object):
             yield self.send_nonmember_event(
                 requester, event, context, ratelimit=ratelimit
             )
-        return event
+        defer.returnValue(event)
 
     @measure_func("create_new_client_event")
     @defer.inlineCallbacks
@@ -630,7 +626,7 @@ class EventCreationHandler(object):
 
         logger.debug("Created event %s", event.event_id)
 
-        return (event, context)
+        defer.returnValue((event, context))
 
     @measure_func("handle_new_client_event")
     @defer.inlineCallbacks
@@ -726,27 +722,7 @@ class EventCreationHandler(object):
         assert not self.config.worker_app
 
         if ratelimit:
-            # We check if this is a room admin redacting an event so that we
-            # can apply different ratelimiting. We do this by simply checking
-            # it's not a self-redaction (to avoid having to look up whether the
-            # user is actually admin or not).
-            is_admin_redaction = False
-            if event.type == EventTypes.Redaction:
-                original_event = yield self.store.get_event(
-                    event.redacts,
-                    check_redacted=False,
-                    get_prev_content=False,
-                    allow_rejected=False,
-                    allow_none=True,
-                )
-
-                is_admin_redaction = (
-                    original_event and event.sender != original_event.sender
-                )
-
-            yield self.base_handler.ratelimit(
-                requester, is_admin_redaction=is_admin_redaction
-            )
+            yield self.base_handler.ratelimit(requester)
 
         yield self.base_handler.maybe_kick_guest_users(event, context)
 
@@ -815,6 +791,7 @@ class EventCreationHandler(object):
                 get_prev_content=False,
                 allow_rejected=False,
                 allow_none=True,
+                check_room_id=event.room_id,
             )
 
             # we can make some additional checks now if we have the original event.
@@ -822,9 +799,6 @@ class EventCreationHandler(object):
                 if original_event.type == EventTypes.Create:
                     raise AuthError(403, "Redacting create events is not permitted")
 
-                if original_event.room_id != event.room_id:
-                    raise SynapseError(400, "Cannot redact event from a different room")
-
             prev_state_ids = yield context.get_prev_state_ids(self.store)
             auth_events_ids = yield self.auth.compute_auth_events(
                 event, prev_state_ids, for_verification=True

synapse/handlers/pagination.py

@@ -242,11 +242,13 @@ class PaginationHandler(object):
             )
 
         if not events:
-            return {
-                "chunk": [],
-                "start": pagin_config.from_token.to_string(),
-                "end": next_token.to_string(),
-            }
+            defer.returnValue(
+                {
+                    "chunk": [],
+                    "start": pagin_config.from_token.to_string(),
+                    "end": next_token.to_string(),
+                }
+            )
 
         state = None
         if event_filter and event_filter.lazy_load_members() and len(events) > 0:
@@ -284,4 +286,4 @@ class PaginationHandler(object):
                 )
             )
 
-        return chunk
+        defer.returnValue(chunk)

synapse/handlers/presence.py

@@ -333,7 +333,7 @@ class PresenceHandler(object):
         """Checks the presence of users that have timed out and updates as
         appropriate.
         """
-        logger.debug("Handling presence timeouts")
+        logger.info("Handling presence timeouts")
         now = self.clock.time_msec()
 
         # Fetch the list of users that *may* have timed out. Things may have
@@ -461,7 +461,7 @@ class PresenceHandler(object):
                 if affect_presence:
                     run_in_background(_end)
 
-        return _user_syncing()
+        defer.returnValue(_user_syncing())
 
     def get_currently_syncing_users(self):
         """Get the set of user ids that are currently syncing on this HS.
@@ -556,7 +556,7 @@ class PresenceHandler(object):
         """Get the current presence state for a user.
         """
         res = yield self.current_state_for_users([user_id])
-        return res[user_id]
+        defer.returnValue(res[user_id])
 
     @defer.inlineCallbacks
     def current_state_for_users(self, user_ids):
@@ -585,7 +585,7 @@ class PresenceHandler(object):
                 states.update(new)
                 self.user_to_current_state.update(new)
 
-        return states
+        defer.returnValue(states)
 
     @defer.inlineCallbacks
     def _persist_and_notify(self, states):
@@ -681,7 +681,7 @@ class PresenceHandler(object):
     def get_state(self, target_user, as_event=False):
         results = yield self.get_states([target_user.to_string()], as_event=as_event)
 
-        return results[0]
+        defer.returnValue(results[0])
 
     @defer.inlineCallbacks
     def get_states(self, target_user_ids, as_event=False):
@@ -703,15 +703,17 @@ class PresenceHandler(object):
 
         now = self.clock.time_msec()
         if as_event:
-            return [
-                {
-                    "type": "m.presence",
-                    "content": format_user_presence_state(state, now),
-                }
-                for state in updates
-            ]
+            defer.returnValue(
+                [
+                    {
+                        "type": "m.presence",
+                        "content": format_user_presence_state(state, now),
+                    }
+                    for state in updates
+                ]
+            )
         else:
-            return updates
+            defer.returnValue(updates)
 
     @defer.inlineCallbacks
     def set_state(self, target_user, state, ignore_status_msg=False):
@@ -755,9 +757,9 @@ class PresenceHandler(object):
         )
 
         if observer_room_ids & observed_room_ids:
-            return True
+            defer.returnValue(True)
 
-        return False
+        defer.returnValue(False)
 
     @defer.inlineCallbacks
     def get_all_presence_updates(self, last_id, current_id):
@@ -776,7 +778,7 @@ class PresenceHandler(object):
         # TODO(markjh): replicate the unpersisted changes.
         # This could use the in-memory stores for recent changes.
         rows = yield self.store.get_all_presence_updates(last_id, current_id)
-        return rows
+        defer.returnValue(rows)
 
     def notify_new_event(self):
         """Called when new events have happened. Handles users and servers
@@ -1032,7 +1034,7 @@ class PresenceEventSource(object):
                 #
                 # Hence this guard where we just return nothing so that the sync
                 # doesn't return. C.f. #5503.
-                return ([], max_token)
+                defer.returnValue(([], max_token))
 
             presence = self.get_presence_handler()
             stream_change_cache = self.store.presence_stream_cache
@@ -1066,11 +1068,17 @@ class PresenceEventSource(object):
             updates = yield presence.current_state_for_users(user_ids_changed)
 
         if include_offline:
-            return (list(updates.values()), max_token)
+            defer.returnValue((list(updates.values()), max_token))
         else:
-            return (
-                [s for s in itervalues(updates) if s.state != PresenceState.OFFLINE],
-                max_token,
+            defer.returnValue(
+                (
+                    [
+                        s
+                        for s in itervalues(updates)
+                        if s.state != PresenceState.OFFLINE
+                    ],
+                    max_token,
+                )
             )
 
     def get_current_key(self):
@@ -1099,7 +1107,7 @@ class PresenceEventSource(object):
             )
             users_interested_in.update(user_ids)
 
-        return users_interested_in
+        defer.returnValue(users_interested_in)
 
 
 def handle_timeouts(user_states, is_mine_fn, syncing_user_ids, now):
@@ -1279,7 +1287,7 @@ def get_interested_parties(store, states):
         # Always notify self
         users_to_states.setdefault(state.user_id, []).append(state)
 
-    return (room_ids_to_states, users_to_states)
+    defer.returnValue((room_ids_to_states, users_to_states))
 
 
 @defer.inlineCallbacks
@@ -1313,4 +1321,4 @@ def get_interested_remotes(store, states, state_handler):
         host = get_domain_from_id(user_id)
         hosts_and_states.append(([host], states))
 
-    return hosts_and_states
+    defer.returnValue(hosts_and_states)

synapse/handlers/profile.py

@@ -73,7 +73,7 @@ class BaseProfileHandler(BaseHandler):
                     raise SynapseError(404, "Profile was not found", Codes.NOT_FOUND)
                 raise
 
-            return {"displayname": displayname, "avatar_url": avatar_url}
+            defer.returnValue({"displayname": displayname, "avatar_url": avatar_url})
         else:
             try:
                 result = yield self.federation.make_query(
@@ -82,7 +82,7 @@ class BaseProfileHandler(BaseHandler):
                     args={"user_id": user_id},
                     ignore_backoff=True,
                 )
-                return result
+                defer.returnValue(result)
             except RequestSendFailed as e:
                 raise_from(SynapseError(502, "Failed to fetch profile"), e)
             except HttpResponseException as e:
@@ -108,10 +108,10 @@ class BaseProfileHandler(BaseHandler):
                     raise SynapseError(404, "Profile was not found", Codes.NOT_FOUND)
                 raise
 
-            return {"displayname": displayname, "avatar_url": avatar_url}
+            defer.returnValue({"displayname": displayname, "avatar_url": avatar_url})
         else:
             profile = yield self.store.get_from_remote_profile_cache(user_id)
-            return profile or {}
+            defer.returnValue(profile or {})
 
     @defer.inlineCallbacks
     def get_displayname(self, target_user):
@@ -125,7 +125,7 @@ class BaseProfileHandler(BaseHandler):
                     raise SynapseError(404, "Profile was not found", Codes.NOT_FOUND)
                 raise
 
-            return displayname
+            defer.returnValue(displayname)
         else:
             try:
                 result = yield self.federation.make_query(
@@ -139,7 +139,7 @@ class BaseProfileHandler(BaseHandler):
             except HttpResponseException as e:
                 raise e.to_synapse_error()
 
-            return result["displayname"]
+            defer.returnValue(result["displayname"])
 
     @defer.inlineCallbacks
     def set_displayname(self, target_user, requester, new_displayname, by_admin=False):
@@ -186,7 +186,7 @@ class BaseProfileHandler(BaseHandler):
                 if e.code == 404:
                     raise SynapseError(404, "Profile was not found", Codes.NOT_FOUND)
                 raise
-            return avatar_url
+            defer.returnValue(avatar_url)
         else:
             try:
                 result = yield self.federation.make_query(
@@ -200,7 +200,7 @@ class BaseProfileHandler(BaseHandler):
             except HttpResponseException as e:
                 raise e.to_synapse_error()
 
-            return result["avatar_url"]
+            defer.returnValue(result["avatar_url"])
 
     @defer.inlineCallbacks
     def set_avatar_url(self, target_user, requester, new_avatar_url, by_admin=False):
@@ -251,7 +251,7 @@ class BaseProfileHandler(BaseHandler):
                 raise SynapseError(404, "Profile was not found", Codes.NOT_FOUND)
             raise
 
-        return response
+        defer.returnValue(response)
 
     @defer.inlineCallbacks
     def _update_join_states(self, requester, target_user):

synapse/handlers/receipts.py

@@ -93,7 +93,7 @@ class ReceiptsHandler(BaseHandler):
 
         if min_batch_id is None:
             # no new receipts
-            return False
+            defer.returnValue(False)
 
         affected_room_ids = list(set([r.room_id for r in receipts]))
 
@@ -103,7 +103,7 @@ class ReceiptsHandler(BaseHandler):
             min_batch_id, max_batch_id, affected_room_ids
         )
 
-        return True
+        defer.returnValue(True)
 
     @defer.inlineCallbacks
     def received_client_receipt(self, room_id, receipt_type, user_id, event_id):
@@ -133,9 +133,9 @@ class ReceiptsHandler(BaseHandler):
         )
 
         if not result:
-            return []
+            defer.returnValue([])
 
-        return result
+        defer.returnValue(result)
 
 
 class ReceiptEventSource(object):
@@ -148,13 +148,13 @@ class ReceiptEventSource(object):
         to_key = yield self.get_current_key()
 
         if from_key == to_key:
-            return ([], to_key)
+            defer.returnValue(([], to_key))
 
         events = yield self.store.get_linearized_receipts_for_rooms(
             room_ids, from_key=from_key, to_key=to_key
         )
 
-        return (events, to_key)
+        defer.returnValue((events, to_key))
 
     def get_current_key(self, direction="f"):
         return self.store.get_max_receipt_stream_id()
@@ -173,4 +173,4 @@ class ReceiptEventSource(object):
             room_ids, from_key=from_key, to_key=to_key
         )
 
-        return (events, to_key)
+        defer.returnValue((events, to_key))

synapse/handlers/register.py

@@ -265,7 +265,7 @@ class RegistrationHandler(BaseHandler):
             # Bind email to new account
             yield self._register_email_threepid(user_id, threepid_dict, None, False)
 
-        return user_id
+        defer.returnValue(user_id)
 
     @defer.inlineCallbacks
     def _auto_join_rooms(self, user_id):
@@ -360,7 +360,7 @@ class RegistrationHandler(BaseHandler):
             appservice_id=service_id,
             create_profile_with_displayname=user.localpart,
         )
-        return user_id
+        defer.returnValue(user_id)
 
     @defer.inlineCallbacks
     def check_recaptcha(self, ip, private_key, challenge, response):
@@ -461,7 +461,7 @@ class RegistrationHandler(BaseHandler):
 
         id = self._next_generated_user_id
         self._next_generated_user_id += 1
-        return str(id)
+        defer.returnValue(str(id))
 
     @defer.inlineCallbacks
     def _validate_captcha(self, ip_addr, private_key, challenge, response):
@@ -481,7 +481,7 @@ class RegistrationHandler(BaseHandler):
             "error_url": "http://www.recaptcha.net/recaptcha/api/challenge?"
             + "error=%s" % lines[1],
         }
-        return json
+        defer.returnValue(json)
 
     @defer.inlineCallbacks
     def _submit_captcha(self, ip_addr, private_key, challenge, response):
@@ -497,7 +497,7 @@ class RegistrationHandler(BaseHandler):
                 "response": response,
             },
         )
-        return data
+        defer.returnValue(data)
 
     @defer.inlineCallbacks
     def _join_user_to_room(self, requester, room_identifier):
@@ -622,7 +622,7 @@ class RegistrationHandler(BaseHandler):
                 initial_display_name=initial_display_name,
                 is_guest=is_guest,
             )
-            return (r["device_id"], r["access_token"])
+            defer.returnValue((r["device_id"], r["access_token"]))
 
         valid_until_ms = None
         if self.session_lifetime is not None:
@@ -645,7 +645,7 @@ class RegistrationHandler(BaseHandler):
                 user_id, device_id=device_id, valid_until_ms=valid_until_ms
             )
 
-        return (device_id, access_token)
+        defer.returnValue((device_id, access_token))
 
     @defer.inlineCallbacks
     def post_registration_actions(
@@ -798,7 +798,7 @@ class RegistrationHandler(BaseHandler):
             if ex.errcode == Codes.MISSING_PARAM:
                 # This will only happen if the ID server returns a malformed response
                 logger.info("Can't add incomplete 3pid")
-                return None
+                defer.returnValue(None)
             raise
 
         yield self._auth_handler.add_threepid(

synapse/handlers/room.py

@@ -128,7 +128,7 @@ class RoomCreationHandler(BaseHandler):
             old_room_id,
             new_version,  # args for _upgrade_room
         )
-        return ret
+        defer.returnValue(ret)
 
     @defer.inlineCallbacks
     def _upgrade_room(self, requester, old_room_id, new_version):
@@ -193,7 +193,7 @@ class RoomCreationHandler(BaseHandler):
             requester, old_room_id, new_room_id, old_room_state
         )
 
-        return new_room_id
+        defer.returnValue(new_room_id)
 
     @defer.inlineCallbacks
     def _update_upgraded_room_pls(
@@ -671,7 +671,7 @@ class RoomCreationHandler(BaseHandler):
             result["room_alias"] = room_alias.to_string()
             yield directory_handler.send_room_alias_update_event(requester, room_id)
 
-        return result
+        defer.returnValue(result)
 
     @defer.inlineCallbacks
     def _send_events_for_new_room(
@@ -796,7 +796,7 @@ class RoomCreationHandler(BaseHandler):
                     room_creator_user_id=creator_id,
                     is_public=is_public,
                 )
-                return gen_room_id
+                defer.returnValue(gen_room_id)
             except StoreError:
                 attempts += 1
         raise StoreError(500, "Couldn't generate a room ID.")
@@ -839,7 +839,7 @@ class RoomContextHandler(object):
             event_id, get_prev_content=True, allow_none=True
         )
         if not event:
-            return None
+            defer.returnValue(None)
             return
 
         filtered = yield (filter_evts([event]))
@@ -890,7 +890,7 @@ class RoomContextHandler(object):
 
         results["end"] = token.copy_and_replace("room_key", results["end"]).to_string()
 
-        return results
+        defer.returnValue(results)
 
 
 class RoomEventSource(object):
@@ -941,7 +941,7 @@ class RoomEventSource(object):
             else:
                 end_key = to_key
 
-        return (events, end_key)
+        defer.returnValue((events, end_key))
 
     def get_current_key(self):
         return self.store.get_room_events_max_id()
@@ -959,4 +959,4 @@ class RoomEventSource(object):
             limit=config.limit,
         )
 
-        return (events, next_key)
+        defer.returnValue((events, next_key))

synapse/handlers/room_list.py

@@ -325,7 +325,7 @@ class RoomListHandler(BaseHandler):
                     current_limit=since_token.current_limit - 1,
                 ).to_token()
 
-        return results
+        defer.returnValue(results)
 
     @defer.inlineCallbacks
     def _append_room_entry_to_chunk(
@@ -420,7 +420,7 @@ class RoomListHandler(BaseHandler):
         if join_rules_event:
             join_rule = join_rules_event.content.get("join_rule", None)
             if not allow_private and join_rule and join_rule != JoinRules.PUBLIC:
-                return None
+                defer.returnValue(None)
 
         # Return whether this room is open to federation users or not
         create_event = current_state.get((EventTypes.Create, ""))
@@ -469,7 +469,7 @@ class RoomListHandler(BaseHandler):
             if avatar_url:
                 result["avatar_url"] = avatar_url
 
-        return result
+        defer.returnValue(result)
 
     @defer.inlineCallbacks
     def get_remote_public_room_list(
@@ -482,7 +482,7 @@ class RoomListHandler(BaseHandler):
         third_party_instance_id=None,
     ):
         if not self.enable_room_list_search:
-            return {"chunk": [], "total_room_count_estimate": 0}
+            defer.returnValue({"chunk": [], "total_room_count_estimate": 0})
 
         if search_filter:
             # We currently don't support searching across federation, so we have
@@ -507,7 +507,7 @@ class RoomListHandler(BaseHandler):
                 ]
             }
 
-        return res
+        defer.returnValue(res)
 
     def _get_remote_list_cached(
         self,

synapse/handlers/room_member.py

@@ -38,6 +38,11 @@ from ._base import BaseHandler
 logger = logging.getLogger(__name__)
 
 id_server_scheme = "https://"
+ROOM_COMPLEXITY_TOO_GREAT = (
+    "Your homeserver is unable to join rooms this large or complex. "
+    "Please speak to your server administrator, or upgrade your instance "
+    "to join this room."
+)
 
 
 class RoomMemberHandler(object):
@@ -190,7 +195,7 @@ class RoomMemberHandler(object):
         )
         if duplicate is not None:
             # Discard the new event since this membership change is a no-op.
-            return duplicate
+            defer.returnValue(duplicate)
 
         yield self.event_creation_handler.handle_new_client_event(
             requester, event, context, extra_users=[target], ratelimit=ratelimit
@@ -232,7 +237,7 @@ class RoomMemberHandler(object):
                 if prev_member_event.membership == Membership.JOIN:
                     yield self._user_left_room(target, room_id)
 
-        return event
+        defer.returnValue(event)
 
     @defer.inlineCallbacks
     def copy_room_tags_and_direct_to_room(self, old_room_id, new_room_id, user_id):
@@ -302,7 +307,7 @@ class RoomMemberHandler(object):
                 require_consent=require_consent,
             )
 
-        return result
+        defer.returnValue(result)
 
     @defer.inlineCallbacks
     def _update_membership(
@@ -422,7 +427,7 @@ class RoomMemberHandler(object):
                 same_membership = old_membership == effective_membership_state
                 same_sender = requester.user.to_string() == old_state.sender
                 if same_sender and same_membership and same_content:
-                    return old_state
+                    defer.returnValue(old_state)
 
             if old_membership in ["ban", "leave"] and action == "kick":
                 raise AuthError(403, "The target user is not in the room")
@@ -472,7 +477,7 @@ class RoomMemberHandler(object):
                 ret = yield self._remote_join(
                     requester, remote_room_hosts, room_id, target, content
                 )
-                return ret
+                defer.returnValue(ret)
 
         elif effective_membership_state == Membership.LEAVE:
             if not is_host_in_room:
@@ -494,7 +499,7 @@ class RoomMemberHandler(object):
                     res = yield self._remote_reject_invite(
                         requester, remote_room_hosts, room_id, target
                     )
-                    return res
+                    defer.returnValue(res)
 
         res = yield self._local_membership_update(
             requester=requester,
@@ -507,7 +512,7 @@ class RoomMemberHandler(object):
             content=content,
             require_consent=require_consent,
         )
-        return res
+        defer.returnValue(res)
 
     @defer.inlineCallbacks
     def send_membership_event(
@@ -595,11 +600,11 @@ class RoomMemberHandler(object):
         """
         guest_access_id = current_state_ids.get((EventTypes.GuestAccess, ""), None)
         if not guest_access_id:
-            return False
+            defer.returnValue(False)
 
         guest_access = yield self.store.get_event(guest_access_id)
 
-        return (
+        defer.returnValue(
             guest_access
             and guest_access.content
             and "guest_access" in guest_access.content
@@ -634,7 +639,7 @@ class RoomMemberHandler(object):
             servers.remove(room_alias.domain)
         servers.insert(0, room_alias.domain)
 
-        return (RoomID.from_string(room_id), servers)
+        defer.returnValue((RoomID.from_string(room_id), servers))
 
     @defer.inlineCallbacks
     def _get_inviter(self, user_id, room_id):
@@ -642,7 +647,7 @@ class RoomMemberHandler(object):
             user_id=user_id, room_id=room_id
         )
         if invite:
-            return UserID.from_string(invite.sender)
+            defer.returnValue(UserID.from_string(invite.sender))
 
     @defer.inlineCallbacks
     def do_3pid_invite(
@@ -707,11 +712,11 @@ class RoomMemberHandler(object):
                 if "signatures" not in data:
                     raise AuthError(401, "No signatures on 3pid binding")
                 yield self._verify_any_signature(data, id_server)
-                return data["mxid"]
+                defer.returnValue(data["mxid"])
 
         except IOError as e:
             logger.warn("Error from identity server lookup: %s" % (e,))
-            return None
+            defer.returnValue(None)
 
     @defer.inlineCallbacks
     def _verify_any_signature(self, data, server_hostname):
@@ -903,7 +908,7 @@ class RoomMemberHandler(object):
         if not public_keys:
             public_keys.append(fallback_public_key)
         display_name = data["display_name"]
-        return (token, public_keys, fallback_public_key, display_name)
+        defer.returnValue((token, public_keys, fallback_public_key, display_name))
 
     @defer.inlineCallbacks
     def _is_host_in_room(self, current_state_ids):
@@ -912,7 +917,7 @@ class RoomMemberHandler(object):
         create_event_id = current_state_ids.get(("m.room.create", ""))
         if len(current_state_ids) == 1 and create_event_id:
             # We can only get here if we're in the process of creating the room
-            return True
+            defer.returnValue(True)
 
         for etype, state_key in current_state_ids:
             if etype != EventTypes.Member or not self.hs.is_mine_id(state_key):
@@ -924,16 +929,16 @@ class RoomMemberHandler(object):
                 continue
 
             if event.membership == Membership.JOIN:
-                return True
+                defer.returnValue(True)
 
-        return False
+        defer.returnValue(False)
 
     @defer.inlineCallbacks
     def _is_server_notice_room(self, room_id):
         if self._server_notices_mxid is None:
-            return False
+            defer.returnValue(False)
         user_ids = yield self.store.get_users_in_room(room_id)
-        return self._server_notices_mxid in user_ids
+        defer.returnValue(self._server_notices_mxid in user_ids)
 
 
 class RoomMemberMasterHandler(RoomMemberHandler):
@@ -956,7 +961,7 @@ class RoomMemberMasterHandler(RoomMemberHandler):
         Returns: bool of whether the complexity is too great, or None
             if unable to be fetched
         """
-        max_complexity = self.hs.config.limit_remote_rooms.complexity
+        max_complexity = self.hs.config.limit_large_room_complexity
         complexity = yield self.federation_handler.get_room_complexity(
             remote_room_hosts, room_id
         )
@@ -977,7 +982,7 @@ class RoomMemberMasterHandler(RoomMemberHandler):
 
         Returns: bool
         """
-        max_complexity = self.hs.config.limit_remote_rooms.complexity
+        max_complexity = self.hs.config.limit_large_room_complexity
         complexity = yield self.store.get_room_complexity(room_id)
 
         if complexity["v1"] > max_complexity:
@@ -999,7 +1004,7 @@ class RoomMemberMasterHandler(RoomMemberHandler):
         if len(remote_room_hosts) == 0:
             raise SynapseError(404, "No known servers")
 
-        if self.hs.config.limit_remote_rooms.enabled:
+        if self.hs.config.limit_large_room_joins:
             # Fetch the room complexity
             too_complex = yield self._is_remote_room_too_complex(
                 room_id, remote_room_hosts
@@ -1007,7 +1012,7 @@ class RoomMemberMasterHandler(RoomMemberHandler):
             if too_complex is True:
                 raise SynapseError(
                     code=400,
-                    msg=self.hs.config.limit_remote_rooms.complexity_error,
+                    msg=ROOM_COMPLEXITY_TOO_GREAT,
                     errcode=Codes.RESOURCE_LIMIT_EXCEEDED,
                 )
 
@@ -1022,7 +1027,7 @@ class RoomMemberMasterHandler(RoomMemberHandler):
 
         # Check the room we just joined wasn't too large, if we didn't fetch the
         # complexity of it before.
-        if self.hs.config.limit_remote_rooms.enabled:
+        if self.hs.config.limit_large_room_joins:
             if too_complex is False:
                 # We checked, and we're under the limit.
                 return
@@ -1041,7 +1046,7 @@ class RoomMemberMasterHandler(RoomMemberHandler):
             )
             raise SynapseError(
                 code=400,
-                msg=self.hs.config.limit_remote_rooms.complexity_error,
+                msg=ROOM_COMPLEXITY_TOO_GREAT,
                 errcode=Codes.RESOURCE_LIMIT_EXCEEDED,
             )
 
@@ -1054,7 +1059,7 @@ class RoomMemberMasterHandler(RoomMemberHandler):
             ret = yield fed_handler.do_remotely_reject_invite(
                 remote_room_hosts, room_id, target.to_string()
             )
-            return ret
+            defer.returnValue(ret)
         except Exception as e:
             # if we were unable to reject the exception, just mark
             # it as rejected on our end and plough ahead.
@@ -1065,7 +1070,7 @@ class RoomMemberMasterHandler(RoomMemberHandler):
             logger.warn("Failed to reject invite: %s", e)
 
             yield self.store.locally_reject_invite(target.to_string(), room_id)
-            return {}
+            defer.returnValue({})
 
     def _user_joined_room(self, target, room_id):
         """Implements RoomMemberHandler._user_joined_room

synapse/handlers/room_member_worker.py

@@ -53,7 +53,7 @@ class RoomMemberWorkerHandler(RoomMemberHandler):
 
         yield self._user_joined_room(user, room_id)
 
-        return ret
+        defer.returnValue(ret)
 
     def _remote_reject_invite(self, requester, remote_room_hosts, room_id, target):
         """Implements RoomMemberHandler._remote_reject_invite

synapse/handlers/search.py

@@ -69,7 +69,7 @@ class SearchHandler(BaseHandler):
             # Scan through the old room for further predecessors
             room_id = predecessor["room_id"]
 
-        return historical_room_ids
+        defer.returnValue(historical_room_ids)
 
     @defer.inlineCallbacks
     def search(self, user, content, batch=None):
@@ -186,11 +186,13 @@ class SearchHandler(BaseHandler):
             room_ids.intersection_update({batch_group_key})
 
         if not room_ids:
-            return {
-                "search_categories": {
-                    "room_events": {"results": [], "count": 0, "highlights": []}
+            defer.returnValue(
+                {
+                    "search_categories": {
+                        "room_events": {"results": [], "count": 0, "highlights": []}
+                    }
                 }
-            }
+            )
 
         rank_map = {}  # event_id -> rank of event
         allowed_events = []
@@ -453,4 +455,4 @@ class SearchHandler(BaseHandler):
         if global_next_batch:
             rooms_cat_res["next_batch"] = global_next_batch
 
-        return {"search_categories": {"room_events": rooms_cat_res}}
+        defer.returnValue({"search_categories": {"room_events": rooms_cat_res}})

synapse/handlers/state_deltas.py

@@ -48,7 +48,7 @@ class StateDeltasHandler(object):
 
         if not event and not prev_event:
             logger.debug("Neither event exists: %r %r", prev_event_id, event_id)
-            return None
+            defer.returnValue(None)
 
         prev_value = None
         value = None
@@ -62,8 +62,8 @@ class StateDeltasHandler(object):
         logger.debug("prev_value: %r -> value: %r", prev_value, value)
 
         if value == public_value and prev_value != public_value:
-            return True
+            defer.returnValue(True)
         elif value != public_value and prev_value == public_value:
-            return False
+            defer.returnValue(False)
         else:
-            return None
+            defer.returnValue(None)