Fix errors

.buildkite/pipeline.yml

@@ -49,15 +49,14 @@ steps:
 
 
   - command:
-      - "apt-get update && apt-get install -y python3.5 python3.5-dev python3-pip libxml2-dev libxslt-dev zlib1g-dev"
-      - "python3.5 -m pip install tox"
+      - "python -m pip install tox"
       - "tox -e py35-old,codecov"
     label: ":python: 3.5 / SQLite / Old Deps"
     env:
       TRIAL_FLAGS: "-j 2"
     plugins:
       - docker#v3.0.1:
-          image: "ubuntu:xenial"  # We use xenail to get an old sqlite and python
+          image: "python:3.5"
           propagate-environment: true
     retry:
       automatic:
@@ -118,10 +117,8 @@ steps:
           limit: 2
 
   - label: ":python: 3.5 / :postgres: 9.5"
-    agents:
-      queue: "medium"
     env:
-      TRIAL_FLAGS: "-j 8"
+      TRIAL_FLAGS: "-j 4"
     command:
       - "bash -c 'python -m pip install tox && python -m tox -e py35-postgres,codecov'"
     plugins:
@@ -137,10 +134,8 @@ steps:
           limit: 2
 
   - label: ":python: 3.7 / :postgres: 9.5"
-    agents:
-      queue: "medium"
     env:
-      TRIAL_FLAGS: "-j 8"
+      TRIAL_FLAGS: "-j 4"
     command:
       - "bash -c 'python -m pip install tox && python -m tox -e py37-postgres,codecov'"
     plugins:
@@ -156,10 +151,8 @@ steps:
           limit: 2
 
   - label: ":python: 3.7 / :postgres: 11"
-    agents:
-      queue: "medium"
     env:
-      TRIAL_FLAGS: "-j 8"
+      TRIAL_FLAGS: "-j 4"
     command:
       - "bash -c 'python -m pip install tox && python -m tox -e py37-postgres,codecov'"
     plugins:
@@ -180,13 +173,11 @@ steps:
       queue: "medium"
     command:
       - "bash .buildkite/merge_base_branch.sh"
-      - "bash /synapse_sytest.sh"
+      - "bash .buildkite/synapse_sytest.sh"
     plugins:
       - docker#v3.0.1:
           image: "matrixdotorg/sytest-synapse:py35"
           propagate-environment: true
-          always-pull: true
-          workdir: "/src"
     retry:
       automatic:
         - exit_status: -1
@@ -201,13 +192,11 @@ steps:
       POSTGRES: "1"
     command:
       - "bash .buildkite/merge_base_branch.sh"
-      - "bash /synapse_sytest.sh"
+      - "bash .buildkite/synapse_sytest.sh"
     plugins:
       - docker#v3.0.1:
           image: "matrixdotorg/sytest-synapse:py35"
           propagate-environment: true
-          always-pull: true
-          workdir: "/src"
     retry:
       automatic:
         - exit_status: -1
@@ -221,17 +210,14 @@ steps:
     env:
       POSTGRES: "1"
       WORKERS: "1"
-      BLACKLIST: "synapse-blacklist-with-workers"
     command:
       - "bash .buildkite/merge_base_branch.sh"
-      - "bash -c 'cat /src/sytest-blacklist /src/.buildkite/worker-blacklist > /src/synapse-blacklist-with-workers'"
-      - "bash /synapse_sytest.sh"
+      - "bash .buildkite/synapse_sytest.sh"
     plugins:
       - docker#v3.0.1:
           image: "matrixdotorg/sytest-synapse:py35"
           propagate-environment: true
-          always-pull: true
-          workdir: "/src"
+    soft_fail: true
     retry:
       automatic:
         - exit_status: -1

.buildkite/synapse_sytest.sh

@@ -0,0 +1,145 @@
+#!/bin/bash
+#
+# Fetch sytest, and then run the tests for synapse. The entrypoint for the
+# sytest-synapse docker images.
+
+set -ex
+
+if [ -n "$BUILDKITE" ]
+then
+    SYNAPSE_DIR=`pwd`
+else
+    SYNAPSE_DIR="/src"
+fi
+
+# Attempt to find a sytest to use.
+# If /sytest exists, it means that a SyTest checkout has been mounted into the Docker image.
+if [ -d "/sytest" ]; then
+    # If the user has mounted in a SyTest checkout, use that.
+    echo "Using local sytests..."
+
+    # create ourselves a working directory and dos2unix some scripts therein
+    mkdir -p /work/jenkins
+    for i in install-deps.pl run-tests.pl tap-to-junit-xml.pl jenkins/prep_sytest_for_postgres.sh; do
+        dos2unix -n "/sytest/$i" "/work/$i"
+    done
+    ln -sf /sytest/tests /work
+    ln -sf /sytest/keys /work
+    SYTEST_LIB="/sytest/lib"
+else
+    if [ -n "BUILDKITE_BRANCH" ]
+    then
+        branch_name=$BUILDKITE_BRANCH
+    else
+        # Otherwise, try and find out what the branch that the Synapse checkout is using. Fall back to develop if it's not a branch.
+        branch_name="$(git --git-dir=/src/.git symbolic-ref HEAD 2>/dev/null)" || branch_name="develop"
+    fi
+
+    # Try and fetch the branch
+    echo "Trying to get same-named sytest branch..."
+    wget -q https://github.com/matrix-org/sytest/archive/$branch_name.tar.gz -O sytest.tar.gz || {
+        # Probably a 404, fall back to develop
+        echo "Using develop instead..."
+        wget -q https://github.com/matrix-org/sytest/archive/develop.tar.gz -O sytest.tar.gz
+    }
+
+    mkdir -p /work
+    tar -C /work --strip-components=1 -xf sytest.tar.gz
+    SYTEST_LIB="/work/lib"
+fi
+
+cd /work
+
+# PostgreSQL setup
+if [ -n "$POSTGRES" ]
+then
+    export PGUSER=postgres
+    export POSTGRES_DB_1=pg1
+    export POSTGRES_DB_2=pg2
+
+    # Start the database
+    su -c 'eatmydata /usr/lib/postgresql/9.6/bin/pg_ctl -w -D /var/lib/postgresql/data start' postgres
+
+    # Use the Jenkins script to write out the configuration for a PostgreSQL using Synapse
+    jenkins/prep_sytest_for_postgres.sh
+
+    # Make the test databases for the two Synapse servers that will be spun up
+    su -c 'psql -c "CREATE DATABASE pg1;"' postgres
+    su -c 'psql -c "CREATE DATABASE pg2;"' postgres
+
+fi
+
+if [ -n "$OFFLINE" ]; then
+    # if we're in offline mode, just put synapse into the virtualenv, and
+    # hope that the deps are up-to-date.
+    #
+    # (`pip install -e` likes to reinstall setuptools even if it's already installed,
+    # so we just run setup.py explicitly.)
+    #
+    (cd $SYNAPSE_DIR && /venv/bin/python setup.py -q develop)
+else
+    # We've already created the virtualenv, but lets double check we have all
+    # deps.
+    /venv/bin/pip install -q --upgrade --no-cache-dir -e $SYNAPSE_DIR
+    /venv/bin/pip install -q --upgrade --no-cache-dir \
+        lxml psycopg2 coverage codecov tap.py
+
+    # Make sure all Perl deps are installed -- this is done in the docker build
+    # so will only install packages added since the last Docker build
+    ./install-deps.pl
+fi
+
+
+# Run the tests
+>&2 echo "+++ Running tests"
+
+RUN_TESTS=(
+    perl -I "$SYTEST_LIB" ./run-tests.pl --python=/venv/bin/python --synapse-directory=$SYNAPSE_DIR --coverage -O tap --all
+)
+
+TEST_STATUS=0
+
+if [ -n "$WORKERS" ]; then
+    RUN_TESTS+=(-I Synapse::ViaHaproxy --dendron-binary=/pydron.py)
+else
+    RUN_TESTS+=(-I Synapse)
+fi
+
+"${RUN_TESTS[@]}" "$@" > results.tap || TEST_STATUS=$?
+
+if [ $TEST_STATUS -ne 0 ]; then
+    >&2 echo -e "run-tests \e[31mFAILED\e[0m: exit code $TEST_STATUS"
+else
+    >&2 echo -e "run-tests \e[32mPASSED\e[0m"
+fi
+
+>&2 echo "--- Copying assets"
+
+# Copy out the logs
+mkdir -p /logs
+cp results.tap /logs/results.tap
+rsync --ignore-missing-args  --min-size=1B -av server-0 server-1 /logs --include "*/" --include="*.log.*" --include="*.log" --exclude="*"
+
+# Upload coverage to codecov and upload files, if running on Buildkite
+if [ -n "$BUILDKITE" ]
+then
+    /venv/bin/coverage combine || true
+    /venv/bin/coverage xml || true
+    /venv/bin/codecov -X gcov -f coverage.xml
+
+    wget -O buildkite.tar.gz https://github.com/buildkite/agent/releases/download/v3.13.0/buildkite-agent-linux-amd64-3.13.0.tar.gz
+    tar xvf buildkite.tar.gz
+    chmod +x ./buildkite-agent
+
+    # Upload the files
+    ./buildkite-agent artifact upload "/logs/**/*.log*"
+    ./buildkite-agent artifact upload "/logs/results.tap"
+
+    if [ $TEST_STATUS -ne 0 ]; then
+        # Annotate, if failure
+        /venv/bin/python $SYNAPSE_DIR/.buildkite/format_tap.py /logs/results.tap "$BUILDKITE_LABEL" | ./buildkite-agent annotate --style="error" --context="$BUILDKITE_LABEL"
+    fi
+fi
+
+
+exit $TEST_STATUS

.buildkite/worker-blacklist

@@ -1,34 +0,0 @@
-# This file serves as a blacklist for SyTest tests that we expect will fail in
-# Synapse when run under worker mode. For more details, see sytest-blacklist.
-
-Message history can be paginated
-
-m.room.history_visibility == "world_readable" allows/forbids appropriately for Guest users
-
-m.room.history_visibility == "world_readable" allows/forbids appropriately for Real users
-
-Can re-join room if re-invited
-
-/upgrade creates a new room
-
-The only membership state included in an initial sync is for all the senders in the timeline
-
-Local device key changes get to remote servers
-
-If remote user leaves room we no longer receive device updates
-
-Forgotten room messages cannot be paginated
-
-Inbound federation can get public room list
-
-Members from the gap are included in gappy incr LL sync
-
-Leaves are present in non-gapped incremental syncs
-
-Old leaves are present in gapped incremental syncs
-
-User sees updates to presence from other users in the incremental sync.
-
-Gapped incremental syncs include all state changes
-
-Old members are included in gappy incr LL sync if they start speaking

.codecov.yml

@@ -1,4 +1,5 @@
-comment: off
+comment:
+  layout: "diff"
 
 coverage:
   status:

CHANGES.md

@@ -1,138 +1,3 @@
-Synapse 1.2.1 (2019-07-26)
-==========================
-
-Security update
----------------
-
-This release includes *four* security fixes:
-
-- Prevent an attack where a federated server could send redactions for arbitrary events in v1 and v2 rooms. ([\#5767](https://github.com/matrix-org/synapse/issues/5767))
-- Prevent a denial-of-service attack where cycles of redaction events would make Synapse spin infinitely. Thanks to `@lrizika:matrix.org` for identifying and responsibly disclosing this issue. ([0f2ecb961](https://github.com/matrix-org/synapse/commit/0f2ecb961))
-- Prevent an attack where users could be joined or parted from public rooms without their consent. Thanks to @dylangerdaly for identifying and responsibly disclosing this issue. ([\#5744](https://github.com/matrix-org/synapse/issues/5744))
-- Fix a vulnerability where a federated server could spoof read-receipts from
-  users on other servers. Thanks to @dylangerdaly for identifying this issue too. ([\#5743](https://github.com/matrix-org/synapse/issues/5743))
-
-Additionally, the following fix was in Synapse **1.2.0**, but was not correctly
-identified during the original release:
-
-- It was possible for a room moderator to send a redaction for an `m.room.create` event, which would downgrade the room to version 1. Thanks to `/dev/ponies` for identifying and responsibly disclosing this issue! ([\#5701](https://github.com/matrix-org/synapse/issues/5701))
-
-Synapse 1.2.0 (2019-07-25)
-==========================
-
-No significant changes.
-
-
-Synapse 1.2.0rc2 (2019-07-24)
-=============================
-
-Bugfixes
---------
-
-- Fix a regression introduced in v1.2.0rc1 which led to incorrect labels on some prometheus metrics. ([\#5734](https://github.com/matrix-org/synapse/issues/5734))
-
-
-Synapse 1.2.0rc1 (2019-07-22)
-=============================
-
-Security fixes
---------------
-
-This update included a security fix which was initially incorrectly flagged as
-a regular bug fix.
-
-- It was possible for a room moderator to send a redaction for an `m.room.create` event, which would downgrade the room to version 1. Thanks to `/dev/ponies` for identifying and responsibly disclosing this issue! ([\#5701](https://github.com/matrix-org/synapse/issues/5701))
-
-Features
---------
-
-- Add support for opentracing. ([\#5544](https://github.com/matrix-org/synapse/issues/5544), [\#5712](https://github.com/matrix-org/synapse/issues/5712))
-- Add ability to pull all locally stored events out of synapse that a particular user can see. ([\#5589](https://github.com/matrix-org/synapse/issues/5589))
-- Add a basic admin command app to allow server operators to run Synapse admin commands separately from the main production instance. ([\#5597](https://github.com/matrix-org/synapse/issues/5597))
-- Add `sender` and `origin_server_ts` fields to `m.replace`. ([\#5613](https://github.com/matrix-org/synapse/issues/5613))
-- Add default push rule to ignore reactions. ([\#5623](https://github.com/matrix-org/synapse/issues/5623))
-- Include the original event when asking for its relations. ([\#5626](https://github.com/matrix-org/synapse/issues/5626))
-- Implement `session_lifetime` configuration option, after which access tokens will expire. ([\#5660](https://github.com/matrix-org/synapse/issues/5660))
-- Return "This account has been deactivated" when a deactivated user tries to login. ([\#5674](https://github.com/matrix-org/synapse/issues/5674))
-- Enable aggregations support by default ([\#5714](https://github.com/matrix-org/synapse/issues/5714))
-
-
-Bugfixes
---------
-
-- Fix 'utime went backwards' errors on daemonization. ([\#5609](https://github.com/matrix-org/synapse/issues/5609))
-- Various minor fixes to the federation request rate limiter. ([\#5621](https://github.com/matrix-org/synapse/issues/5621))
-- Forbid viewing relations on an event once it has been redacted. ([\#5629](https://github.com/matrix-org/synapse/issues/5629))
-- Fix requests to the `/store_invite` endpoint of identity servers being sent in the wrong format. ([\#5638](https://github.com/matrix-org/synapse/issues/5638))
-- Fix newly-registered users not being able to lookup their own profile without joining a room. ([\#5644](https://github.com/matrix-org/synapse/issues/5644))
-- Fix bug in #5626 that prevented the original_event field from actually having the contents of the original event in a call to `/relations`. ([\#5654](https://github.com/matrix-org/synapse/issues/5654))
-- Fix 3PID bind requests being sent to identity servers as `application/x-form-www-urlencoded` data, which is deprecated. ([\#5658](https://github.com/matrix-org/synapse/issues/5658))
-- Fix some problems with authenticating redactions in recent room versions. ([\#5699](https://github.com/matrix-org/synapse/issues/5699), [\#5700](https://github.com/matrix-org/synapse/issues/5700), [\#5707](https://github.com/matrix-org/synapse/issues/5707))
-
-
-Updates to the Docker image
----------------------------
-
-- Base Docker image on a newer Alpine Linux version (3.8 -> 3.10). ([\#5619](https://github.com/matrix-org/synapse/issues/5619))
-- Add missing space in default logging file format generated by the Docker image. ([\#5620](https://github.com/matrix-org/synapse/issues/5620))
-
-
-Improved Documentation
-----------------------
-
-- Add information about nginx normalisation to reverse_proxy.rst. Contributed by @skalarproduktraum - thanks! ([\#5397](https://github.com/matrix-org/synapse/issues/5397))
-- --no-pep517 should be --no-use-pep517 in the documentation to setup the development environment. ([\#5651](https://github.com/matrix-org/synapse/issues/5651))
-- Improvements to Postgres setup instructions. Contributed by @Lrizika - thanks! ([\#5661](https://github.com/matrix-org/synapse/issues/5661))
-- Minor tweaks to postgres documentation. ([\#5675](https://github.com/matrix-org/synapse/issues/5675))
-
-
-Deprecations and Removals
--------------------------
-
-- Remove support for the `invite_3pid_guest` configuration setting. ([\#5625](https://github.com/matrix-org/synapse/issues/5625))
-
-
-Internal Changes
-----------------
-
-- Move logging code out of `synapse.util` and into `synapse.logging`. ([\#5606](https://github.com/matrix-org/synapse/issues/5606), [\#5617](https://github.com/matrix-org/synapse/issues/5617))
-- Add a blacklist file to the repo to blacklist certain sytests from failing CI. ([\#5611](https://github.com/matrix-org/synapse/issues/5611))
-- Make runtime errors surrounding password reset emails much clearer. ([\#5616](https://github.com/matrix-org/synapse/issues/5616))
-- Remove dead code for persiting outgoing federation transactions. ([\#5622](https://github.com/matrix-org/synapse/issues/5622))
-- Add `lint.sh` to the scripts-dev folder which will run all linting steps required by CI. ([\#5627](https://github.com/matrix-org/synapse/issues/5627))
-- Move RegistrationHandler.get_or_create_user to test code. ([\#5628](https://github.com/matrix-org/synapse/issues/5628))
-- Add some more common python virtual-environment paths to the black exclusion list. ([\#5630](https://github.com/matrix-org/synapse/issues/5630))
-- Some counter metrics exposed over Prometheus have been renamed, with the old names preserved for backwards compatibility and deprecated. See `docs/metrics-howto.rst` for details. ([\#5636](https://github.com/matrix-org/synapse/issues/5636))
-- Unblacklist some user_directory sytests. ([\#5637](https://github.com/matrix-org/synapse/issues/5637))
-- Factor out some redundant code in the login implementation. ([\#5639](https://github.com/matrix-org/synapse/issues/5639))
-- Update ModuleApi to avoid register(generate_token=True). ([\#5640](https://github.com/matrix-org/synapse/issues/5640))
-- Remove access-token support from `RegistrationHandler.register`, and rename it. ([\#5641](https://github.com/matrix-org/synapse/issues/5641))
-- Remove access-token support from `RegistrationStore.register`, and rename it. ([\#5642](https://github.com/matrix-org/synapse/issues/5642))
-- Improve logging for auto-join when a new user is created. ([\#5643](https://github.com/matrix-org/synapse/issues/5643))
-- Remove unused and unnecessary check for FederationDeniedError in _exception_to_failure. ([\#5645](https://github.com/matrix-org/synapse/issues/5645))
-- Fix a small typo in a code comment. ([\#5655](https://github.com/matrix-org/synapse/issues/5655))
-- Clean up exception handling around client access tokens. ([\#5656](https://github.com/matrix-org/synapse/issues/5656))
-- Add a mechanism for per-test homeserver configuration in the unit tests. ([\#5657](https://github.com/matrix-org/synapse/issues/5657))
-- Inline issue_access_token. ([\#5659](https://github.com/matrix-org/synapse/issues/5659))
-- Update the sytest BuildKite configuration to checkout Synapse in `/src`. ([\#5664](https://github.com/matrix-org/synapse/issues/5664))
-- Add a `docker` type to the towncrier configuration. ([\#5673](https://github.com/matrix-org/synapse/issues/5673))
-- Convert `synapse.federation.transport.server` to `async`. Might improve some stack traces. ([\#5689](https://github.com/matrix-org/synapse/issues/5689))
-- Documentation for opentracing. ([\#5703](https://github.com/matrix-org/synapse/issues/5703))
-
-
-Synapse 1.1.0 (2019-07-04)
-==========================
-
-As of v1.1.0, Synapse no longer supports Python 2, nor Postgres version 9.4.
-See the [upgrade notes](UPGRADE.rst#upgrading-to-v110) for more details.
-
-This release also deprecates the use of environment variables to configure the
-docker image. See the [docker README](https://github.com/matrix-org/synapse/blob/release-v1.1.0/docker/README.md#legacy-dynamic-configuration-file-support)
-for more details.
-
-No changes since 1.1.0rc2.
-
-
 Synapse 1.1.0rc2 (2019-07-03)
 =============================
 
@@ -140,7 +5,7 @@ Bugfixes
 --------
 
 - Fix regression in 1.1rc1 where OPTIONS requests to the media repo would fail. ([\#5593](https://github.com/matrix-org/synapse/issues/5593))
-- Removed the `SYNAPSE_SMTP_*` docker container environment variables. Using these environment variables prevented the docker container from starting in Synapse v1.0, even though they didn't actually allow any functionality anyway. ([\#5596](https://github.com/matrix-org/synapse/issues/5596))
+- Removed the `SYNAPSE_SMTP_*` docker container environment variables. Using these environment variables prevented the docker container from starting in Synapse v1.0, even though they didn't actually allow any functionality anyway. Users are advised to remove `SYNAPSE_SMTP_HOST`, `SYNAPSE_SMTP_PORT`, `SYNAPSE_SMTP_USER`, `SYNAPSE_SMTP_PASSWORD` and `SYNAPSE_SMTP_FROM` environment variables from their docker run commands. ([\#5596](https://github.com/matrix-org/synapse/issues/5596))
 - Fix a number of "Starting txn from sentinel context" warnings. ([\#5605](https://github.com/matrix-org/synapse/issues/5605))
 
 

CONTRIBUTING.rst

@@ -30,10 +30,11 @@ use github's pull request workflow to review the contribution, and either ask
 you to make any refinements needed or merge it and make them ourselves. The
 changes will then land on master when we next do a release.
 
-We use `Buildkite <https://buildkite.com/matrix-dot-org/synapse>`_ for
-continuous integration.  Buildkite builds need to be authorised by a
-maintainer. If your change breaks the build, this will be shown in GitHub, so
-please keep an eye on the pull request for feedback.
+We use `CircleCI <https://circleci.com/gh/matrix-org>`_ and `Buildkite
+<https://buildkite.com/matrix-dot-org/synapse>`_ for continuous integration.
+Buildkite builds need to be authorised by a maintainer. If your change breaks
+the build, this will be shown in GitHub, so please keep an eye on the pull
+request for feedback.
 
 To run unit tests in a local development environment, you can use:
 
@@ -69,21 +70,13 @@ All changes, even minor ones, need a corresponding changelog / newsfragment
 entry. These are managed by Towncrier
 (https://github.com/hawkowl/towncrier).
 
-To create a changelog entry, make a new file in the ``changelog.d`` file named
-in the format of ``PRnumber.type``. The type can be one of the following:
+To create a changelog entry, make a new file in the ``changelog.d``
+file named in the format of ``PRnumber.type``. The type can be
+one of ``feature``, ``bugfix``, ``removal`` (also used for
+deprecations), or ``misc`` (for internal-only changes).
 
-* ``feature``.
-* ``bugfix``.
-* ``docker`` (for updates to the Docker image).
-* ``doc`` (for updates to the documentation).
-* ``removal`` (also used for deprecations).
-* ``misc`` (for internal-only changes).
-
-The content of the file is your changelog entry, which should be a short
-description of your change in the same style as the rest of our `changelog
-<https://github.com/matrix-org/synapse/blob/master/CHANGES.md>`_. The file can
-contain Markdown formatting, and should end with a full stop ('.') for
-consistency.
+The content of the file is your changelog entry, which can contain Markdown
+formatting. The entry should end with a full stop ('.') for consistency.
 
 Adding credits to the changelog is encouraged, we value your
 contributions and would like to have you shouted out in the release notes!

MANIFEST.in

@@ -33,7 +33,6 @@ exclude Dockerfile
 exclude .dockerignore
 exclude test_postgresql.sh
 exclude .editorconfig
-exclude sytest-blacklist
 
 include pyproject.toml
 recursive-include changelog.d *

README.rst

@@ -272,7 +272,7 @@ to install using pip and a virtualenv::
 
     virtualenv -p python3 env
     source env/bin/activate
-    python -m pip install --no-use-pep517 -e .[all]
+    python -m pip install --no-pep-517 -e .[all]
 
 This will run a process of downloading and installing all the needed
 dependencies into a virtual env.

UPGRADE.rst

@@ -49,13 +49,6 @@ returned by the Client-Server API:
     # configured on port 443.
     curl -kv https://<host.name>/_matrix/client/versions 2>&1 | grep "Server:"
 
-Upgrading to v1.2.0
-===================
-
-Some counter metrics have been renamed, with the old names deprecated. See
-`the metrics documentation <docs/metrics-howto.rst#renaming-of-metrics--deprecation-of-old-names-in-12>`_
-for details.
-
 Upgrading to v1.1.0
 ===================
 

changelog.d/5606.misc

@@ -0,0 +1 @@
+Move logging code out of `synapse.util` and into `synapse.logging`.

changelog.d/5609.bugfix

@@ -0,0 +1 @@
+Fix 'utime went backwards' errors on daemonization.

changelog.d/5678.removal

@@ -1 +0,0 @@
-Synapse now no longer accepts the `-v`/`--verbose`, `-f`/`--log-file`, or `--log-config` command line flags, and removes the deprecated `verbose` and `log_file` configuration file options. Users of these options should migrate their options into the dedicated log configuration.

changelog.d/5693.bugfix

@@ -1 +0,0 @@
-Fix UISIs during homeserver outage.

changelog.d/5694.misc

@@ -1 +0,0 @@
-Make Jaeger fully configurable.

changelog.d/5695.misc

@@ -1 +0,0 @@
-Add precautionary measures to prevent future abuse of `window.opener` in default welcome page.

changelog.d/5706.misc

@@ -1 +0,0 @@
-Reduce database IO usage by optimising queries for current membership.

changelog.d/5713.misc

@@ -1 +0,0 @@
-Improve caching when fetching `get_filtered_current_state_ids`.

changelog.d/5715.misc

@@ -1 +0,0 @@
-Don't accept opentracing data from clients.

changelog.d/5717.misc

@@ -1 +0,0 @@
-Speed up PostgreSQL unit tests in CI.

changelog.d/5719.misc

@@ -1 +0,0 @@
-Update the coding style document.

changelog.d/5720.misc

@@ -1 +0,0 @@
-Improve database query performance when recording retry intervals for remote hosts.

changelog.d/5722.misc

@@ -1 +0,0 @@
-Add a set of opentracing utils.

changelog.d/5724.bugfix

@@ -1 +0,0 @@
-Fix stack overflow in server key lookup code.

changelog.d/5725.bugfix

@@ -1 +0,0 @@
-start.sh no longer uses deprecated cli option.

changelog.d/5729.removal

@@ -1 +0,0 @@
- Synapse now no longer accepts the `-v`/`--verbose`, `-f`/`--log-file`, or `--log-config` command line flags, and removes the deprecated `verbose` and `log_file` configuration file options. Users of these options should migrate their options into the dedicated log configuration.

changelog.d/5730.misc

@@ -1 +0,0 @@
-Cache result of get_version_string to reduce overhead of `/version` federation requests.

changelog.d/5731.misc

@@ -1 +0,0 @@
-Return 'user_type' in admin API user endpoints results.

changelog.d/5732.feature

@@ -1 +0,0 @@
-Add sd_notify hooks to ease systemd integration and allows usage of Type=Notify.

changelog.d/5733.misc

@@ -1 +0,0 @@
-Don't package the sytest test blacklist file.

changelog.d/5736.misc

@@ -1 +0,0 @@
-Replace uses of returnValue with plain return, as returnValue is not needed on Python 3.

changelog.d/5738.misc

@@ -1 +0,0 @@
-Reduce database IO usage by optimising queries for current membership.

changelog.d/5740.misc

@@ -1 +0,0 @@
-Blacklist some flakey tests in worker mode.

changelog.d/5743.bugfix

@@ -1 +0,0 @@
-Log when we receive an event receipt from an unexpected origin.

changelog.d/5746.misc

@@ -1 +0,0 @@
-Reduce database IO usage by optimising queries for current membership.

changelog.d/5749.misc

@@ -1 +0,0 @@
-Fix some error cases in the caching layer.

changelog.d/5750.misc

@@ -1 +0,0 @@
-Add a prometheus metric for pending cache lookups.

changelog.d/5752.misc

@@ -1 +0,0 @@
-Reduce database IO usage by optimising queries for current membership.

changelog.d/5753.misc

@@ -1 +0,0 @@
-Stop trying to fetch events with event_id=None.

changelog.d/5768.misc

@@ -1 +0,0 @@
-Convert RedactionTestCase to modern test style.

changelog.d/5770.misc

@@ -1 +0,0 @@
-Reduce database IO usage by optimising queries for current membership.

changelog.d/5774.misc

@@ -1 +0,0 @@
-Reduce database IO usage by optimising queries for current membership.

changelog.d/5775.bugfix

@@ -1 +0,0 @@
-Fix debian packaging scripts to correctly build sid packages.

changelog.d/5780.misc

@@ -1 +0,0 @@
-Allow looping calls to be given arguments.

changelog.d/5782.removal

@@ -1 +0,0 @@
-Remove non-functional 'expire_access_token' setting.

changelog.d/5783.feature

@@ -1 +0,0 @@
-Synapse can now be configured to not join remote rooms of a given "complexity" (currently, state events) over federation. This option can be used to prevent adverse performance on resource-constrained homeservers.

changelog.d/5785.misc

@@ -1 +0,0 @@
-Set the logs emitted when checking typing and presence timeouts to DEBUG level, not INFO.

changelog.d/5787.misc

@@ -1 +0,0 @@
-Remove DelayedCall debugging from the test suite, as it is no longer required in the vast majority of Synapse's tests.

changelog.d/5789.bugfix

@@ -1 +0,0 @@
-Fix UISIs during homeserver outage.

changelog.d/5792.misc

@@ -1 +0,0 @@
-Reduce database IO usage by optimising queries for current membership.

changelog.d/5793.misc

@@ -1 +0,0 @@
-Reduce database IO usage by optimising queries for current membership.

changelog.d/5794.misc

@@ -1 +0,0 @@
-Improve performance when making `.well-known` requests by sharing the SSL options between requests.

changelog.d/5796.misc

@@ -1 +0,0 @@
-Disable codecov GitHub comments on PRs.

changelog.d/5798.bugfix

@@ -1 +0,0 @@
-Return 404 instead of 403 when accessing /rooms/{roomId}/event/{eventId} for an event without the appropriate permissions.

contrib/systemd-with-workers/system/matrix-synapse-worker@.service

@@ -4,8 +4,7 @@ After=matrix-synapse.service
 BindsTo=matrix-synapse.service
 
 [Service]
-Type=notify
-NotifyAccess=main
+Type=simple
 User=matrix-synapse
 WorkingDirectory=/var/lib/matrix-synapse
 EnvironmentFile=/etc/default/matrix-synapse

contrib/systemd-with-workers/system/matrix-synapse.service

@@ -2,8 +2,7 @@
 Description=Synapse Matrix Homeserver
 
 [Service]
-Type=notify
-NotifyAccess=main
+Type=simple
 User=matrix-synapse
 WorkingDirectory=/var/lib/matrix-synapse
 EnvironmentFile=/etc/default/matrix-synapse

contrib/systemd/matrix-synapse.service

@@ -14,9 +14,7 @@
 Description=Synapse Matrix homeserver
 
 [Service]
-Type=notify
-NotifyAccess=main
-ExecReload=/bin/kill -HUP $MAINPID
+Type=simple
 Restart=on-abort
 
 User=synapse

debian/changelog

@@ -1,31 +1,12 @@
-matrix-synapse-py3 (1.2.1) stable; urgency=medium
-
-  * New synapse release 1.2.1.
-
- -- Synapse Packaging team <packages@matrix.org>  Fri, 26 Jul 2019 11:32:47 +0100
-
-matrix-synapse-py3 (1.2.0) stable; urgency=medium
-
-  [ Amber Brown ]
-  * Update logging config defaults to match API changes in Synapse.
-
-  [ Richard van der Hoff ]
-  * Add Recommends and Depends for some libraries which you probably want.
-
-  [ Synapse Packaging team ]
-  * New synapse release 1.2.0.
-
- -- Synapse Packaging team <packages@matrix.org>  Thu, 25 Jul 2019 14:10:07 +0100
-
-matrix-synapse-py3 (1.1.0) stable; urgency=medium
+matrix-synapse-py3 (1.0.0+nmu1) UNRELEASED; urgency=medium
 
   [ Silke Hofstra ]
   * Include systemd-python to allow logging to the systemd journal.
 
-  [ Synapse Packaging team ]
-  * New synapse release 1.1.0.
+  [ Amber Brown ]
+  * Update logging config defaults to match API changes in Synapse.
 
- -- Synapse Packaging team <packages@matrix.org>  Thu, 04 Jul 2019 11:43:41 +0100
+ -- Silke Hofstra <silke@slxh.eu>  Wed, 29 May 2019 09:45:29 +0200
 
 matrix-synapse-py3 (1.0.0) stable; urgency=medium
 

debian/control

@@ -2,20 +2,16 @@ Source: matrix-synapse-py3
 Section: contrib/python
 Priority: extra
 Maintainer: Synapse Packaging team <packages@matrix.org>
-# keep this list in sync with the build dependencies in docker/Dockerfile-dhvirtualenv.
 Build-Depends:
  debhelper (>= 9),
  dh-systemd,
  dh-virtualenv (>= 1.1),
- libsystemd-dev,
- libpq-dev,
  lsb-release,
  python3-dev,
  python3,
  python3-setuptools,
  python3-pip,
  python3-venv,
- libsqlite3-dev,
  tar,
 Standards-Version: 3.9.8
 Homepage: https://github.com/matrix-org/synapse
@@ -32,12 +28,9 @@ Depends:
  debconf,
  python3-distutils|libpython3-stdlib (<< 3.6),
  ${misc:Depends},
- ${shlibs:Depends},
  ${synapse:pydepends},
 # some of our scripts use perl, but none of them are important,
 # so we put perl:Depends in Suggests rather than Depends.
-Recommends:
- ${shlibs1:Recommends},
 Suggests:
  sqlite3,
  ${perl:Depends},

debian/rules

@@ -3,29 +3,15 @@
 # Build Debian package using https://github.com/spotify/dh-virtualenv
 #
 
-# assume we only have one package
-PACKAGE_NAME:=`dh_listpackages`
-
 override_dh_systemd_enable:
 	dh_systemd_enable --name=matrix-synapse
 
 override_dh_installinit:
 	dh_installinit --name=matrix-synapse
 
-# we don't really want to strip the symbols from our object files.
 override_dh_strip:
 
 override_dh_shlibdeps:
-        # make the postgres package's dependencies a recommendation
-        # rather than a hard dependency.
-	find debian/$(PACKAGE_NAME)/ -path '*/site-packages/psycopg2/*.so' | \
-	    xargs dpkg-shlibdeps -Tdebian/$(PACKAGE_NAME).substvars \
-	        -pshlibs1 -dRecommends
-
-        # all the other dependencies can be normal 'Depends' requirements,
-        # except for PIL's, which is self-contained and which confuses
-        # dpkg-shlibdeps.
-	dh_shlibdeps -X site-packages/PIL/.libs -X site-packages/psycopg2
 
 override_dh_virtualenv:
 	./debian/build_virtualenv

demo/start.sh

@@ -29,7 +29,7 @@ for port in 8080 8081 8082; do
 
     if ! grep -F "Customisation made by demo/start.sh" -q  $DIR/etc/$port.config; then
         printf '\n\n# Customisation made by demo/start.sh\n' >> $DIR/etc/$port.config
-
+        
         echo 'enable_registration: true' >> $DIR/etc/$port.config
 
         # Warning, this heredoc depends on the interaction of tabs and spaces. Please don't
@@ -43,7 +43,7 @@ for port in 8080 8081 8082; do
 		    tls: true
 		    resources:
 		      - names: [client, federation]
-
+		
 		  - port: $port
 		    tls: false
 		    bind_addresses: ['::1', '127.0.0.1']
@@ -68,7 +68,7 @@ for port in 8080 8081 8082; do
 
         # Generate tls keys
         openssl req -x509 -newkey rsa:4096 -keyout $DIR/etc/localhost\:$https_port.tls.key -out $DIR/etc/localhost\:$https_port.tls.crt -days 365 -nodes -subj "/O=matrix"
-
+        
         # Ignore keys from the trusted keys server
         echo '# Ignore keys from the trusted keys server' >> $DIR/etc/$port.config
         echo 'trusted_key_servers:' >> $DIR/etc/$port.config
@@ -120,6 +120,7 @@ for port in 8080 8081 8082; do
     python3 -m synapse.app.homeserver \
         --config-path "$DIR/etc/$port.config" \
         -D \
+        -vv \
 
     popd
 done

docker/Dockerfile

@@ -16,7 +16,7 @@ ARG PYTHON_VERSION=3.7
 ###
 ### Stage 0: builder
 ###
-FROM docker.io/python:${PYTHON_VERSION}-alpine3.10 as builder
+FROM docker.io/python:${PYTHON_VERSION}-alpine3.8 as builder
 
 # install the OS build deps
 
@@ -55,7 +55,7 @@ RUN pip install --prefix="/install" --no-warn-script-location \
 ### Stage 1: runtime
 ###
 
-FROM docker.io/python:${PYTHON_VERSION}-alpine3.10
+FROM docker.io/python:${PYTHON_VERSION}-alpine3.8
 
 # xmlsec is required for saml support
 RUN apk add --no-cache --virtual .runtime_deps \

docker/Dockerfile-dhvirtualenv

@@ -42,15 +42,7 @@ RUN cd dh-virtualenv-1.1 && dpkg-buildpackage -us -uc -b
 ###
 FROM ${distro}
 
-# Get the distro we want to pull from as a dynamic build variable
-# (We need to define it in each build stage)
-ARG distro=""
-ENV distro ${distro}
-
 # Install the build dependencies
-#
-# NB: keep this list in sync with the list of build-deps in debian/control
-# TODO: it would be nice to do that automatically.
 RUN apt-get update -qq -o Acquire::Languages=none \
     && env DEBIAN_FRONTEND=noninteractive apt-get install \
         -yqq --no-install-recommends -o Dpkg::Options::=--force-unsafe-io \

docker/build_debian.sh

@@ -4,8 +4,7 @@
 
 set -ex
 
-# Get the codename from distro env
-DIST=`cut -d ':' -f2 <<< $distro`
+DIST=`lsb_release -c -s`
 
 # we get a read-only copy of the source: make a writeable copy
 cp -aT /synapse/source /synapse/build

docker/conf/log.config

@@ -2,7 +2,7 @@ version: 1
 
 formatters:
   precise:
-   format: '%(asctime)s - %(name)s - %(lineno)d - %(levelname)s - %(request)s - %(message)s'
+   format: '%(asctime)s - %(name)s - %(lineno)d - %(levelname)s - %(request)s- %(message)s'
 
 filters:
   context:

docs/code_style.rst

@@ -1,8 +1,4 @@
-Code Style
-==========
-
-Formatting tools
-----------------
+# Code Style
 
 The Synapse codebase uses a number of code formatting tools in order to
 quickly and automatically check for formatting (and sometimes logical) errors
@@ -10,20 +6,20 @@ in code.
 
 The necessary tools are detailed below.
 
-- **black**
+## Formatting tools
 
-  The Synapse codebase uses `black <https://pypi.org/project/black/>`_ as an
-  opinionated code formatter, ensuring all comitted code is properly
-  formatted.
+The Synapse codebase uses [black](https://pypi.org/project/black/) as an
+opinionated code formatter, ensuring all comitted code is properly
+formatted.
 
-  First install ``black`` with::
+First install ``black`` with::
 
-    pip install --upgrade black
+  pip install --upgrade black
 
-  Have ``black`` auto-format your code (it shouldn't change any functionality)
-  with::
+Have ``black`` auto-format your code (it shouldn't change any
+functionality) with::
 
-    black . --exclude="\.tox|build|env"
+  black . --exclude="\.tox|build|env"
 
 - **flake8**
 
@@ -58,16 +54,17 @@ functionality is supported in your editor for a more convenient development
 workflow. It is not, however, recommended to run ``flake8`` on save as it
 takes a while and is very resource intensive.
 
-General rules
--------------
+## General rules
 
 - **Naming**:
 
   - Use camel case for class and type names
   - Use underscores for functions and variables.
 
-- **Docstrings**: should follow the `google code style
-  <https://google.github.io/styleguide/pyguide.html#38-comments-and-docstrings>`_.
+- Use double quotes ``"foo"`` rather than single quotes ``'foo'``.
+
+- **Comments**: should follow the `google code style
+  <http://google.github.io/styleguide/pyguide.html?showone=Comments#Comments>`_.
   This is so that we can generate documentation with `sphinx
   <http://sphinxcontrib-napoleon.readthedocs.org/en/latest/>`_. See the
   `examples
@@ -76,8 +73,6 @@ General rules
 
 - **Imports**:
 
-  - Imports should be sorted by ``isort`` as described above.
-
   - Prefer to import classes and functions rather than packages or modules.
 
     Example::
@@ -97,84 +92,25 @@ General rules
     This goes against the advice in the Google style guide, but it means that
     errors in the name are caught early (at import time).
 
+  - Multiple imports from the same package can be combined onto one line::
+
+      from synapse.types import GroupID, RoomID, UserID
+
+    An effort should be made to keep the individual imports in alphabetical
+    order.
+
+    If the list becomes long, wrap it with parentheses and split it over
+    multiple lines.
+
+  - As per `PEP-8 <https://www.python.org/dev/peps/pep-0008/#imports>`_,
+    imports should be grouped in the following order, with a blank line between
+    each group:
+
+    1. standard library imports
+    2. related third party imports
+    3. local application/library specific imports
+
+  - Imports within each group should be sorted alphabetically by module name.
+
   - Avoid wildcard imports (``from synapse.types import *``) and relative
     imports (``from .types import UserID``).
-
-Configuration file format
--------------------------
-
-The `sample configuration file <./sample_config.yaml>`_ acts as a reference to
-Synapse's configuration options for server administrators. Remember that many
-readers will be unfamiliar with YAML and server administration in general, so
-that it is important that the file be as easy to understand as possible, which
-includes following a consistent format.
-
-Some guidelines follow:
-
-* Sections should be separated with a heading consisting of a single line
-  prefixed and suffixed with ``##``. There should be **two** blank lines
-  before the section header, and **one** after.
-
-* Each option should be listed in the file with the following format:
-
-  * A comment describing the setting. Each line of this comment should be
-    prefixed with a hash (``#``) and a space.
-
-    The comment should describe the default behaviour (ie, what happens if
-    the setting is omitted), as well as what the effect will be if the
-    setting is changed.
-
-    Often, the comment end with something like "uncomment the
-    following to \<do action>".
-
-  * A line consisting of only ``#``.
-
-  * A commented-out example setting, prefixed with only ``#``.
-
-    For boolean (on/off) options, convention is that this example should be
-    the *opposite* to the default (so the comment will end with "Uncomment
-    the following to enable [or disable] \<feature\>." For other options,
-    the example should give some non-default value which is likely to be
-    useful to the reader.
-
-* There should be a blank line between each option.
-
-* Where several settings are grouped into a single dict, *avoid* the
-  convention where the whole block is commented out, resulting in comment
-  lines starting ``# #``, as this is hard to read and confusing to
-  edit. Instead, leave the top-level config option uncommented, and follow
-  the conventions above for sub-options. Ensure that your code correctly
-  handles the top-level option being set to ``None`` (as it will be if no
-  sub-options are enabled).
-
-* Lines should be wrapped at 80 characters.
-
-Example::
-
-    ## Frobnication ##
-
-    # The frobnicator will ensure that all requests are fully frobnicated.
-    # To enable it, uncomment the following.
-    #
-    #frobnicator_enabled: true
-
-    # By default, the frobnicator will frobnicate with the default frobber.
-    # The following will make it use an alternative frobber.
-    #
-    #frobincator_frobber: special_frobber
-
-    # Settings for the frobber
-    #
-    frobber:
-       # frobbing speed. Defaults to 1.
-       #
-       #speed: 10
-
-       # frobbing distance. Defaults to 1000.
-       #
-       #distance: 100
-
-Note that the sample configuration is generated from the synapse code and is
-maintained by a script, ``scripts-dev/generate_sample_config``. Making sure
-that the output from this script matches the desired format is left as an
-exercise for the reader!

docs/log_contexts.rst

@@ -148,7 +148,7 @@ call any other functions.
         d = more_stuff()
         result = yield d            # also fine, of course
 
-        return result
+        defer.returnValue(result)
 
     def nonInlineCallbacksFun():
         logger.debug("just a wrapper really")

docs/metrics-howto.rst

@@ -59,108 +59,6 @@ How to monitor Synapse metrics using Prometheus
    Restart Prometheus.
 
 
-Renaming of metrics & deprecation of old names in 1.2
------------------------------------------------------
-
-Synapse 1.2 updates the Prometheus metrics to match the naming convention of the
-upstream ``prometheus_client``. The old names are considered deprecated and will
-be removed in a future version of Synapse.
-
-+-----------------------------------------------------------------------------+-----------------------------------------------------------------------+
-|                                  New Name                                   |                               Old Name                                |
-+=============================================================================+=======================================================================+
-| python_gc_objects_collected_total                                           | python_gc_objects_collected                                           |
-+-----------------------------------------------------------------------------+-----------------------------------------------------------------------+
-| python_gc_objects_uncollectable_total                                       | python_gc_objects_uncollectable                                       |
-+-----------------------------------------------------------------------------+-----------------------------------------------------------------------+
-| python_gc_collections_total                                                 | python_gc_collections                                                 |
-+-----------------------------------------------------------------------------+-----------------------------------------------------------------------+
-| process_cpu_seconds_total                                                   | process_cpu_seconds                                                   |
-+-----------------------------------------------------------------------------+-----------------------------------------------------------------------+
-| synapse_federation_client_sent_transactions_total                           | synapse_federation_client_sent_transactions                           |
-+-----------------------------------------------------------------------------+-----------------------------------------------------------------------+
-| synapse_federation_client_events_processed_total                            | synapse_federation_client_events_processed                            |
-+-----------------------------------------------------------------------------+-----------------------------------------------------------------------+
-| synapse_event_processing_loop_count_total                                   | synapse_event_processing_loop_count                                   |
-+-----------------------------------------------------------------------------+-----------------------------------------------------------------------+
-| synapse_event_processing_loop_room_count_total                              | synapse_event_processing_loop_room_count                              |
-+-----------------------------------------------------------------------------+-----------------------------------------------------------------------+
-| synapse_util_metrics_block_count_total                                      | synapse_util_metrics_block_count                                      |
-+-----------------------------------------------------------------------------+-----------------------------------------------------------------------+
-| synapse_util_metrics_block_time_seconds_total                               | synapse_util_metrics_block_time_seconds                               |
-+-----------------------------------------------------------------------------+-----------------------------------------------------------------------+
-| synapse_util_metrics_block_ru_utime_seconds_total                           | synapse_util_metrics_block_ru_utime_seconds                           |
-+-----------------------------------------------------------------------------+-----------------------------------------------------------------------+
-| synapse_util_metrics_block_ru_stime_seconds_total                           | synapse_util_metrics_block_ru_stime_seconds                           |
-+-----------------------------------------------------------------------------+-----------------------------------------------------------------------+
-| synapse_util_metrics_block_db_txn_count_total                               | synapse_util_metrics_block_db_txn_count                               |
-+-----------------------------------------------------------------------------+-----------------------------------------------------------------------+
-| synapse_util_metrics_block_db_txn_duration_seconds_total                    | synapse_util_metrics_block_db_txn_duration_seconds                    |
-+-----------------------------------------------------------------------------+-----------------------------------------------------------------------+
-| synapse_util_metrics_block_db_sched_duration_seconds_total                  | synapse_util_metrics_block_db_sched_duration_seconds                  |
-+-----------------------------------------------------------------------------+-----------------------------------------------------------------------+
-| synapse_background_process_start_count_total                                | synapse_background_process_start_count                                |
-+-----------------------------------------------------------------------------+-----------------------------------------------------------------------+
-| synapse_background_process_ru_utime_seconds_total                           | synapse_background_process_ru_utime_seconds                           |
-+-----------------------------------------------------------------------------+-----------------------------------------------------------------------+
-| synapse_background_process_ru_stime_seconds_total                           | synapse_background_process_ru_stime_seconds                           |
-+-----------------------------------------------------------------------------+-----------------------------------------------------------------------+
-| synapse_background_process_db_txn_count_total                               | synapse_background_process_db_txn_count                               |
-+-----------------------------------------------------------------------------+-----------------------------------------------------------------------+
-| synapse_background_process_db_txn_duration_seconds_total                    | synapse_background_process_db_txn_duration_seconds                    |
-+-----------------------------------------------------------------------------+-----------------------------------------------------------------------+
-| synapse_background_process_db_sched_duration_seconds_total                  | synapse_background_process_db_sched_duration_seconds                  |
-+-----------------------------------------------------------------------------+-----------------------------------------------------------------------+
-| synapse_storage_events_persisted_events_total                               | synapse_storage_events_persisted_events                               |
-+-----------------------------------------------------------------------------+-----------------------------------------------------------------------+
-| synapse_storage_events_persisted_events_sep_total                           | synapse_storage_events_persisted_events_sep                           |
-+-----------------------------------------------------------------------------+-----------------------------------------------------------------------+
-| synapse_storage_events_state_delta_total                                    | synapse_storage_events_state_delta                                    |
-+-----------------------------------------------------------------------------+-----------------------------------------------------------------------+
-| synapse_storage_events_state_delta_single_event_total                       | synapse_storage_events_state_delta_single_event                       |
-+-----------------------------------------------------------------------------+-----------------------------------------------------------------------+
-| synapse_storage_events_state_delta_reuse_delta_total                        | synapse_storage_events_state_delta_reuse_delta                        |
-+-----------------------------------------------------------------------------+-----------------------------------------------------------------------+
-| synapse_federation_server_received_pdus_total                               | synapse_federation_server_received_pdus                               |
-+-----------------------------------------------------------------------------+-----------------------------------------------------------------------+
-| synapse_federation_server_received_edus_total                               | synapse_federation_server_received_edus                               |
-+-----------------------------------------------------------------------------+-----------------------------------------------------------------------+
-| synapse_handler_presence_notified_presence_total                            | synapse_handler_presence_notified_presence                            |
-+-----------------------------------------------------------------------------+-----------------------------------------------------------------------+
-| synapse_handler_presence_federation_presence_out_total                      | synapse_handler_presence_federation_presence_out                      |
-+-----------------------------------------------------------------------------+-----------------------------------------------------------------------+
-| synapse_handler_presence_presence_updates_total                             | synapse_handler_presence_presence_updates                             |
-+-----------------------------------------------------------------------------+-----------------------------------------------------------------------+
-| synapse_handler_presence_timers_fired_total                                 | synapse_handler_presence_timers_fired                                 |
-+-----------------------------------------------------------------------------+-----------------------------------------------------------------------+
-| synapse_handler_presence_federation_presence_total                          | synapse_handler_presence_federation_presence                          |
-+-----------------------------------------------------------------------------+-----------------------------------------------------------------------+
-| synapse_handler_presence_bump_active_time_total                             | synapse_handler_presence_bump_active_time                             |
-+-----------------------------------------------------------------------------+-----------------------------------------------------------------------+
-| synapse_federation_client_sent_edus_total                                   | synapse_federation_client_sent_edus                                   |
-+-----------------------------------------------------------------------------+-----------------------------------------------------------------------+
-| synapse_federation_client_sent_pdu_destinations_count_total                 | synapse_federation_client_sent_pdu_destinations:count                 |
-+-----------------------------------------------------------------------------+-----------------------------------------------------------------------+
-| synapse_federation_client_sent_pdu_destinations_total                       | synapse_federation_client_sent_pdu_destinations:total                 |
-+-----------------------------------------------------------------------------+-----------------------------------------------------------------------+
-| synapse_handlers_appservice_events_processed_total                          | synapse_handlers_appservice_events_processed                          |
-+-----------------------------------------------------------------------------+-----------------------------------------------------------------------+
-| synapse_notifier_notified_events_total                                      | synapse_notifier_notified_events                                      |
-+-----------------------------------------------------------------------------+-----------------------------------------------------------------------+
-| synapse_push_bulk_push_rule_evaluator_push_rules_invalidation_counter_total | synapse_push_bulk_push_rule_evaluator_push_rules_invalidation_counter |
-+-----------------------------------------------------------------------------+-----------------------------------------------------------------------+
-| synapse_push_bulk_push_rule_evaluator_push_rules_state_size_counter_total   | synapse_push_bulk_push_rule_evaluator_push_rules_state_size_counter   |
-+-----------------------------------------------------------------------------+-----------------------------------------------------------------------+
-| synapse_http_httppusher_http_pushes_processed_total                         | synapse_http_httppusher_http_pushes_processed                         |
-+-----------------------------------------------------------------------------+-----------------------------------------------------------------------+
-| synapse_http_httppusher_http_pushes_failed_total                            | synapse_http_httppusher_http_pushes_failed                            |
-+-----------------------------------------------------------------------------+-----------------------------------------------------------------------+
-| synapse_http_httppusher_badge_updates_processed_total                       | synapse_http_httppusher_badge_updates_processed                       |
-+-----------------------------------------------------------------------------+-----------------------------------------------------------------------+
-| synapse_http_httppusher_badge_updates_failed_total                          | synapse_http_httppusher_badge_updates_failed                          |
-+-----------------------------------------------------------------------------+-----------------------------------------------------------------------+
-
-
 Removal of deprecated metrics & time based counters becoming histograms in 0.31.0
 ---------------------------------------------------------------------------------
 

docs/opentracing.rst

@@ -1,100 +0,0 @@
-===========
-OpenTracing
-===========
-
-Background
-----------
-
-OpenTracing is a semi-standard being adopted by a number of distributed tracing
-platforms. It is a common api for facilitating vendor-agnostic tracing
-instrumentation. That is, we can use the OpenTracing api and select one of a
-number of tracer implementations to do the heavy lifting in the background.
-Our current selected implementation is Jaeger.
-
-OpenTracing is a tool which gives an insight into the causal relationship of
-work done in and between servers. The servers each track events and report them
-to a centralised server - in Synapse's case: Jaeger. The basic unit used to
-represent events is the span. The span roughly represents a single piece of work
-that was done and the time at which it occurred. A span can have child spans,
-meaning that the work of the child had to be completed for the parent span to
-complete, or it can have follow-on spans which represent work that is undertaken
-as a result of the parent but is not depended on by the parent to in order to
-finish.
-
-Since this is undertaken in a distributed environment a request to another
-server, such as an RPC or a simple GET, can be considered a span (a unit or
-work) for the local server. This causal link is what OpenTracing aims to
-capture and visualise. In order to do this metadata about the local server's
-span, i.e the 'span context', needs to be included with the request to the
-remote.
-
-It is up to the remote server to decide what it does with the spans
-it creates. This is called the sampling policy and it can be configured
-through Jaeger's settings.
-
-For OpenTracing concepts see 
-https://opentracing.io/docs/overview/what-is-tracing/.
-
-For more information about Jaeger's implementation see
-https://www.jaegertracing.io/docs/
-
-=====================
-Seting up OpenTracing
-=====================
-
-To receive OpenTracing spans, start up a Jaeger server. This can be done
-using docker like so:
-
-.. code-block:: bash
-
-   docker run -d --name jaeger
-     -p 6831:6831/udp \
-     -p 6832:6832/udp \
-     -p 5778:5778 \
-     -p 16686:16686 \
-     -p 14268:14268 \
-     jaegertracing/all-in-one:1.13
-
-Latest documentation is probably at
-https://www.jaegertracing.io/docs/1.13/getting-started/
-
-
-Enable OpenTracing in Synapse
------------------------------
-
-OpenTracing is not enabled by default. It must be enabled in the homeserver
-config by uncommenting the config options under ``opentracing`` as shown in
-the `sample config <./sample_config.yaml>`_. For example:
-
-.. code-block:: yaml
-
-  opentracing:
-    tracer_enabled: true
-    homeserver_whitelist:
-      - "mytrustedhomeserver.org"
-      - "*.myotherhomeservers.com"
-
-Homeserver whitelisting
------------------------
-
-The homeserver whitelist is configured using regular expressions. A list of regular
-expressions can be given and their union will be compared when propagating any
-spans contexts to another homeserver. 
-
-Though it's mostly safe to send and receive span contexts to and from
-untrusted users since span contexts are usually opaque ids it can lead to
-two problems, namely:
-
-- If the span context is marked as sampled by the sending homeserver the receiver will
-  sample it. Therefore two homeservers with wildly different sampling policies
-  could incur higher sampling counts than intended.
-- Sending servers can attach arbitrary data to spans, known as 'baggage'. For safety this has been disabled in Synapse
-  but that doesn't prevent another server sending you baggage which will be logged
-  to OpenTracing's logs.
-
-==================
-Configuring Jaeger
-==================
-
-Sampling strategies can be set as in this document:
-https://www.jaegertracing.io/docs/1.13/sampling/

docs/postgres.rst

@@ -11,9 +11,7 @@ a postgres database.
 
 * If you are using the `matrix.org debian/ubuntu
   packages <../INSTALL.md#matrixorg-packages>`_,
-  the necessary python library will already be installed, but you will need to
-  ensure the low-level postgres library is installed, which you can do with
-  ``apt install libpq5``.
+  the necessary libraries will already be installed.
 
 * For other pre-built packages, please consult the documentation from the
   relevant package.
@@ -36,14 +34,9 @@ Assuming your PostgreSQL database user is called ``postgres``, create a user
    su - postgres
    createuser --pwprompt synapse_user
 
-Before you can authenticate with the ``synapse_user``, you must create a
-database that it can access. To create a database, first connect to the database
-with your database user::
-
-   su - postgres
-   psql
-
-and then run::
+The PostgreSQL database used *must* have the correct encoding set, otherwise it
+would not be able to store UTF8 strings. To create a database with the correct
+encoding use, e.g.::
 
    CREATE DATABASE synapse
     ENCODING 'UTF8'
@@ -53,13 +46,7 @@ and then run::
     OWNER synapse_user;
 
 This would create an appropriate database named ``synapse`` owned by the
-``synapse_user`` user (which must already have been created as above).
-
-Note that the PostgreSQL database *must* have the correct encoding set (as
-shown above), otherwise it will not be able to store UTF8 strings.
-
-You may need to enable password authentication so ``synapse_user`` can connect
-to the database. See https://www.postgresql.org/docs/11/auth-pg-hba-conf.html.
+``synapse_user`` user (which must already exist).
 
 Tuning Postgres
 ===============

docs/reverse_proxy.rst

@@ -48,8 +48,6 @@ Let's assume that we expect clients to connect to our server at
               proxy_set_header X-Forwarded-For $remote_addr;
           }
       }
-      
-  Do not add a `/` after the port in `proxy_pass`, otherwise nginx will canonicalise/normalise the URI.
 
 * Caddy::
 

docs/sample_config.yaml

@@ -278,23 +278,6 @@ listeners:
 # Used by phonehome stats to group together related servers.
 #server_context: context
 
-# Resource-constrained Homeserver Settings
-#
-# If limit_remote_rooms.enabled is True, the room complexity will be
-# checked before a user joins a new remote room. If it is above
-# limit_remote_rooms.complexity, it will disallow joining or
-# instantly leave.
-#
-# limit_remote_rooms.complexity_error can be set to customise the text
-# displayed to the user when a room above the complexity threshold has
-# its join cancelled.
-#
-# Uncomment the below lines to enable:
-#limit_remote_rooms:
-#  enabled: True
-#  complexity: 1.0
-#  complexity_error: "This room is too complex."
-
 # Whether to require a user to be in the room to add an alias to it.
 # Defaults to 'true'.
 #
@@ -803,17 +786,6 @@ uploads_path: "DATADIR/uploads"
 #  renew_at: 1w
 #  renew_email_subject: "Renew your %(app)s account"
 
-# Time that a user's session remains valid for, after they log in.
-#
-# Note that this is not currently compatible with guest logins.
-#
-# Note also that this is calculated at login time: changes are not applied
-# retrospectively to users who have already logged in.
-#
-# By default, this is infinite.
-#
-#session_lifetime: 24h
-
 # The user must provide all of the below types of 3PID when registering.
 #
 #registrations_require_3pid:
@@ -942,6 +914,10 @@ uploads_path: "DATADIR/uploads"
 #
 # macaroon_secret_key: <PRIVATE STRING>
 
+# Used to enable access token expiration.
+#
+#expire_access_token: False
+
 # a secret which is used to calculate HMACs for form values, to stop
 # falsification of values. Must be specified for the User Consent
 # forms to work.
@@ -1419,43 +1395,3 @@ password_config:
 #  module: "my_custom_project.SuperRulesSet"
 #  config:
 #    example_option: 'things'
-
-
-## Opentracing ##
-
-# These settings enable opentracing, which implements distributed tracing.
-# This allows you to observe the causal chains of events across servers
-# including requests, key lookups etc., across any server running
-# synapse or any other other services which supports opentracing
-# (specifically those implemented with Jaeger).
-#
-opentracing:
-    # tracing is disabled by default. Uncomment the following line to enable it.
-    #
-    #enabled: true
-
-    # The list of homeservers we wish to send and receive span contexts and span baggage.
-    # See docs/opentracing.rst
-    # This is a list of regexes which are matched against the server_name of the
-    # homeserver.
-    #
-    # By defult, it is empty, so no servers are matched.
-    #
-    #homeserver_whitelist:
-    #  - ".*"
-
-    # Jaeger can be configured to sample traces at different rates.
-    # All configuration options provided by Jaeger can be set here.
-    # Jaeger's configuration mostly related to trace sampling which
-    # is documented here:
-    # https://www.jaegertracing.io/docs/1.13/sampling/.
-    #
-    #jaeger_config:
-    #  sampler:
-    #    type: const
-    #    param: 1
-
-    #  Logging whether spans were started and reported
-    #
-    #  logging:
-    #    false

pyproject.toml

@@ -14,11 +14,6 @@
         name = "Bugfixes"
         showcontent = true
 
-    [[tool.towncrier.type]]
-        directory = "docker"
-        name = "Updates to the Docker image"
-        showcontent = true
-
     [[tool.towncrier.type]]
         directory = "doc"
         name = "Improved Documentation"
@@ -44,8 +39,6 @@ exclude = '''
     | \.git          # root of the project
     | \.tox
     | \.venv
-    | \.env
-    | env
     | _build
     | _trial_temp.*
     | build

scripts-dev/lint.sh

@@ -1,12 +0,0 @@
-#!/bin/sh
-#
-# Runs linting scripts over the local Synapse checkout
-# isort - sorts import statements
-# flake8 - lints and finds mistakes
-# black - opinionated code formatter
-
-set -e
-
-isort -y -rc synapse tests scripts-dev scripts
-flake8 synapse tests
-python3 -m black synapse tests scripts-dev scripts

synapse/__init__.py

@@ -35,4 +35,4 @@ try:
 except ImportError:
     pass
 
-__version__ = "1.2.1"
+__version__ = "1.1.0rc2"

synapse/api/auth.py

@@ -25,13 +25,7 @@ from twisted.internet import defer
 import synapse.types
 from synapse import event_auth
 from synapse.api.constants import EventTypes, JoinRules, Membership
-from synapse.api.errors import (
-    AuthError,
-    Codes,
-    InvalidClientTokenError,
-    MissingClientTokenError,
-    ResourceLimitError,
-)
+from synapse.api.errors import AuthError, Codes, ResourceLimitError
 from synapse.config.server import is_threepid_reserved
 from synapse.types import UserID
 from synapse.util.caches import CACHE_SIZE_FACTOR, register_cache
@@ -69,6 +63,7 @@ class Auth(object):
         self.clock = hs.get_clock()
         self.store = hs.get_datastore()
         self.state = hs.get_state_handler()
+        self.TOKEN_NOT_FOUND_HTTP_STATUS = 401
 
         self.token_cache = LruCache(CACHE_SIZE_FACTOR * 10000)
         register_cache("cache", "token_cache", self.token_cache)
@@ -128,7 +123,7 @@ class Auth(object):
             )
 
         self._check_joined_room(member, user_id, room_id)
-        return member
+        defer.returnValue(member)
 
     @defer.inlineCallbacks
     def check_user_was_in_room(self, room_id, user_id):
@@ -156,13 +151,13 @@ class Auth(object):
             if forgot:
                 raise AuthError(403, "User %s not in room %s" % (user_id, room_id))
 
-        return member
+        defer.returnValue(member)
 
     @defer.inlineCallbacks
     def check_host_in_room(self, room_id, host):
         with Measure(self.clock, "check_host_in_room"):
             latest_event_ids = yield self.store.is_host_joined(room_id, host)
-            return latest_event_ids
+            defer.returnValue(latest_event_ids)
 
     def _check_joined_room(self, member, user_id, room_id):
         if not member or member.membership != Membership.JOIN:
@@ -194,17 +189,18 @@ class Auth(object):
         Returns:
             defer.Deferred: resolves to a ``synapse.types.Requester`` object
         Raises:
-            InvalidClientCredentialsError if no user by that token exists or the token
-                is invalid.
-            AuthError if access is denied for the user in the access token
+            AuthError if no user by that token exists or the token is invalid.
         """
+        # Can optionally look elsewhere in the request (e.g. headers)
         try:
             ip_addr = self.hs.get_ip_from_request(request)
             user_agent = request.requestHeaders.getRawHeaders(
                 b"User-Agent", default=[b""]
             )[0].decode("ascii", "surrogateescape")
 
-            access_token = self.get_access_token_from_request(request)
+            access_token = self.get_access_token_from_request(
+                request, self.TOKEN_NOT_FOUND_HTTP_STATUS
+            )
 
             user_id, app_service = yield self._get_appservice_user_id(request)
             if user_id:
@@ -219,7 +215,9 @@ class Auth(object):
                         device_id="dummy-device",  # stubbed
                     )
 
-                return synapse.types.create_requester(user_id, app_service=app_service)
+                defer.returnValue(
+                    synapse.types.create_requester(user_id, app_service=app_service)
+                )
 
             user_info = yield self.get_user_by_access_token(access_token, rights)
             user = user_info["user"]
@@ -260,37 +258,45 @@ class Auth(object):
 
             request.authenticated_entity = user.to_string()
 
-            return synapse.types.create_requester(
-                user, token_id, is_guest, device_id, app_service=app_service
+            defer.returnValue(
+                synapse.types.create_requester(
+                    user, token_id, is_guest, device_id, app_service=app_service
+                )
             )
         except KeyError:
-            raise MissingClientTokenError()
+            raise AuthError(
+                self.TOKEN_NOT_FOUND_HTTP_STATUS,
+                "Missing access token.",
+                errcode=Codes.MISSING_TOKEN,
+            )
 
     @defer.inlineCallbacks
     def _get_appservice_user_id(self, request):
         app_service = self.store.get_app_service_by_token(
-            self.get_access_token_from_request(request)
+            self.get_access_token_from_request(
+                request, self.TOKEN_NOT_FOUND_HTTP_STATUS
+            )
         )
         if app_service is None:
-            return (None, None)
+            defer.returnValue((None, None))
 
         if app_service.ip_range_whitelist:
             ip_address = IPAddress(self.hs.get_ip_from_request(request))
             if ip_address not in app_service.ip_range_whitelist:
-                return (None, None)
+                defer.returnValue((None, None))
 
         if b"user_id" not in request.args:
-            return (app_service.sender, app_service)
+            defer.returnValue((app_service.sender, app_service))
 
         user_id = request.args[b"user_id"][0].decode("utf8")
         if app_service.sender == user_id:
-            return (app_service.sender, app_service)
+            defer.returnValue((app_service.sender, app_service))
 
         if not app_service.is_interested_in_user(user_id):
             raise AuthError(403, "Application service cannot masquerade as this user.")
         if not (yield self.store.get_user_by_id(user_id)):
             raise AuthError(403, "Application service has not registered this user")
-        return (user_id, app_service)
+        defer.returnValue((user_id, app_service))
 
     @defer.inlineCallbacks
     def get_user_by_access_token(self, token, rights="access"):
@@ -307,26 +313,14 @@ class Auth(object):
                `token_id` (int|None): access token id. May be None if guest
                `device_id` (str|None): device corresponding to access token
         Raises:
-            InvalidClientCredentialsError if no user by that token exists or the token
-                is invalid.
+            AuthError if no user by that token exists or the token is invalid.
         """
 
         if rights == "access":
             # first look in the database
             r = yield self._look_up_user_by_access_token(token)
             if r:
-                valid_until_ms = r["valid_until_ms"]
-                if (
-                    valid_until_ms is not None
-                    and valid_until_ms < self.clock.time_msec()
-                ):
-                    # there was a valid access token, but it has expired.
-                    # soft-logout the user.
-                    raise InvalidClientTokenError(
-                        msg="Access token has expired", soft_logout=True
-                    )
-
-                return r
+                defer.returnValue(r)
 
         # otherwise it needs to be a valid macaroon
         try:
@@ -337,7 +331,11 @@ class Auth(object):
                 if not guest:
                     # non-guest access tokens must be in the database
                     logger.warning("Unrecognised access token - not in store.")
-                    raise InvalidClientTokenError()
+                    raise AuthError(
+                        self.TOKEN_NOT_FOUND_HTTP_STATUS,
+                        "Unrecognised access token.",
+                        errcode=Codes.UNKNOWN_TOKEN,
+                    )
 
                 # Guest access tokens are not stored in the database (there can
                 # only be one access token per guest, anyway).
@@ -352,10 +350,16 @@ class Auth(object):
                 # guest tokens.
                 stored_user = yield self.store.get_user_by_id(user_id)
                 if not stored_user:
-                    raise InvalidClientTokenError("Unknown user_id %s" % user_id)
+                    raise AuthError(
+                        self.TOKEN_NOT_FOUND_HTTP_STATUS,
+                        "Unknown user_id %s" % user_id,
+                        errcode=Codes.UNKNOWN_TOKEN,
+                    )
                 if not stored_user["is_guest"]:
-                    raise InvalidClientTokenError(
-                        "Guest access token used for regular user"
+                    raise AuthError(
+                        self.TOKEN_NOT_FOUND_HTTP_STATUS,
+                        "Guest access token used for regular user",
+                        errcode=Codes.UNKNOWN_TOKEN,
                     )
                 ret = {
                     "user": user,
@@ -374,7 +378,7 @@ class Auth(object):
                 }
             else:
                 raise RuntimeError("Unknown rights setting %s", rights)
-            return ret
+            defer.returnValue(ret)
         except (
             _InvalidMacaroonException,
             pymacaroons.exceptions.MacaroonException,
@@ -382,7 +386,11 @@ class Auth(object):
             ValueError,
         ) as e:
             logger.warning("Invalid macaroon in auth: %s %s", type(e), e)
-            raise InvalidClientTokenError("Invalid macaroon passed.")
+            raise AuthError(
+                self.TOKEN_NOT_FOUND_HTTP_STATUS,
+                "Invalid macaroon passed.",
+                errcode=Codes.UNKNOWN_TOKEN,
+            )
 
     def _parse_and_validate_macaroon(self, token, rights="access"):
         """Takes a macaroon and tries to parse and validate it. This is cached
@@ -410,16 +418,25 @@ class Auth(object):
         try:
             user_id = self.get_user_id_from_macaroon(macaroon)
 
+            has_expiry = False
             guest = False
             for caveat in macaroon.caveats:
-                if caveat.caveat_id == "guest = true":
+                if caveat.caveat_id.startswith("time "):
+                    has_expiry = True
+                elif caveat.caveat_id == "guest = true":
                     guest = True
 
-            self.validate_macaroon(macaroon, rights, user_id=user_id)
+            self.validate_macaroon(
+                macaroon, rights, self.hs.config.expire_access_token, user_id=user_id
+            )
         except (pymacaroons.exceptions.MacaroonException, TypeError, ValueError):
-            raise InvalidClientTokenError("Invalid macaroon passed.")
+            raise AuthError(
+                self.TOKEN_NOT_FOUND_HTTP_STATUS,
+                "Invalid macaroon passed.",
+                errcode=Codes.UNKNOWN_TOKEN,
+            )
 
-        if rights == "access":
+        if not has_expiry and rights == "access":
             self.token_cache[token] = (user_id, guest)
 
         return user_id, guest
@@ -436,16 +453,19 @@ class Auth(object):
             (str) user id
 
         Raises:
-            InvalidClientCredentialsError if there is no user_id caveat in the
-                macaroon
+            AuthError if there is no user_id caveat in the macaroon
         """
         user_prefix = "user_id = "
         for caveat in macaroon.caveats:
             if caveat.caveat_id.startswith(user_prefix):
                 return caveat.caveat_id[len(user_prefix) :]
-        raise InvalidClientTokenError("No user caveat in macaroon")
+        raise AuthError(
+            self.TOKEN_NOT_FOUND_HTTP_STATUS,
+            "No user caveat in macaroon",
+            errcode=Codes.UNKNOWN_TOKEN,
+        )
 
-    def validate_macaroon(self, macaroon, type_string, user_id):
+    def validate_macaroon(self, macaroon, type_string, verify_expiry, user_id):
         """
         validate that a Macaroon is understood by and was signed by this server.
 
@@ -453,6 +473,7 @@ class Auth(object):
             macaroon(pymacaroons.Macaroon): The macaroon to validate
             type_string(str): The kind of token required (e.g. "access",
                               "delete_pusher")
+            verify_expiry(bool): Whether to verify whether the macaroon has expired.
             user_id (str): The user_id required
         """
         v = pymacaroons.Verifier()
@@ -465,7 +486,19 @@ class Auth(object):
         v.satisfy_exact("type = " + type_string)
         v.satisfy_exact("user_id = %s" % user_id)
         v.satisfy_exact("guest = true")
-        v.satisfy_general(self._verify_expiry)
+
+        # verify_expiry should really always be True, but there exist access
+        # tokens in the wild which expire when they should not, so we can't
+        # enforce expiry yet (so we have to allow any caveat starting with
+        # 'time < ' in access tokens).
+        #
+        # On the other hand, short-term login tokens (as used by CAS login, for
+        # example) have an expiry time which we do want to enforce.
+
+        if verify_expiry:
+            v.satisfy_general(self._verify_expiry)
+        else:
+            v.satisfy_general(lambda c: c.startswith("time < "))
 
         # access_tokens include a nonce for uniqueness: any value is acceptable
         v.satisfy_general(lambda c: c.startswith("nonce = "))
@@ -484,7 +517,16 @@ class Auth(object):
     def _look_up_user_by_access_token(self, token):
         ret = yield self.store.get_user_by_access_token(token)
         if not ret:
-            return None
+            defer.returnValue(None)
+
+        if ret.get("expired", False):
+            logger.warn("Doing soft logout on user")
+            raise AuthError(
+                401,
+                "Token soft logged out",
+                errcode=Codes.UNKNOWN_TOKEN,
+                soft_logout=True,
+            )
 
         # we use ret.get() below because *lots* of unit tests stub out
         # get_user_by_access_token in a way where it only returns a couple of
@@ -494,18 +536,26 @@ class Auth(object):
             "token_id": ret.get("token_id", None),
             "is_guest": False,
             "device_id": ret.get("device_id"),
-            "valid_until_ms": ret.get("valid_until_ms"),
         }
-        return user_info
+        defer.returnValue(user_info)
 
     def get_appservice_by_req(self, request):
-        token = self.get_access_token_from_request(request)
-        service = self.store.get_app_service_by_token(token)
-        if not service:
-            logger.warn("Unrecognised appservice access token.")
-            raise InvalidClientTokenError()
-        request.authenticated_entity = service.sender
-        return defer.succeed(service)
+        try:
+            token = self.get_access_token_from_request(
+                request, self.TOKEN_NOT_FOUND_HTTP_STATUS
+            )
+            service = self.store.get_app_service_by_token(token)
+            if not service:
+                logger.warn("Unrecognised appservice access token.")
+                raise AuthError(
+                    self.TOKEN_NOT_FOUND_HTTP_STATUS,
+                    "Unrecognised access token.",
+                    errcode=Codes.UNKNOWN_TOKEN,
+                )
+            request.authenticated_entity = service.sender
+            return defer.succeed(service)
+        except KeyError:
+            raise AuthError(self.TOKEN_NOT_FOUND_HTTP_STATUS, "Missing access token.")
 
     def is_server_admin(self, user):
         """ Check if the given user is a local server admin.
@@ -521,7 +571,7 @@ class Auth(object):
     @defer.inlineCallbacks
     def compute_auth_events(self, event, current_state_ids, for_verification=False):
         if event.type == EventTypes.Create:
-            return []
+            defer.returnValue([])
 
         auth_ids = []
 
@@ -582,7 +632,22 @@ class Auth(object):
             if member_event.content["membership"] == Membership.JOIN:
                 auth_ids.append(member_event.event_id)
 
-        return auth_ids
+        defer.returnValue(auth_ids)
+
+    def check_redaction(self, room_version, event, auth_events):
+        """Check whether the event sender is allowed to redact the target event.
+
+        Returns:
+            True if the the sender is allowed to redact the target event if the
+                target event was created by them.
+            False if the sender is allowed to redact the target event with no
+                further checks.
+
+        Raises:
+            AuthError if the event sender is definitely not allowed to redact
+                the target event.
+        """
+        return event_auth.check_redaction(room_version, event, auth_events)
 
     @defer.inlineCallbacks
     def check_can_change_room_list(self, room_id, user):
@@ -596,7 +661,7 @@ class Auth(object):
 
         is_admin = yield self.is_server_admin(user)
         if is_admin:
-            return True
+            defer.returnValue(True)
 
         user_id = user.to_string()
         yield self.check_joined_room(room_id, user_id)
@@ -636,16 +701,20 @@ class Auth(object):
         return bool(query_params) or bool(auth_headers)
 
     @staticmethod
-    def get_access_token_from_request(request):
+    def get_access_token_from_request(request, token_not_found_http_status=401):
         """Extracts the access_token from the request.
 
         Args:
             request: The http request.
+            token_not_found_http_status(int): The HTTP status code to set in the
+                AuthError if the token isn't found. This is used in some of the
+                legacy APIs to change the status code to 403 from the default of
+                401 since some of the old clients depended on auth errors returning
+                403.
         Returns:
             unicode: The access_token
         Raises:
-            MissingClientTokenError: If there isn't a single access_token in the
-                request
+            AuthError: If there isn't an access_token in the request.
         """
 
         auth_headers = request.requestHeaders.getRawHeaders(b"Authorization")
@@ -654,20 +723,34 @@ class Auth(object):
             # Try the get the access_token from a "Authorization: Bearer"
             # header
             if query_params is not None:
-                raise MissingClientTokenError(
-                    "Mixing Authorization headers and access_token query parameters."
+                raise AuthError(
+                    token_not_found_http_status,
+                    "Mixing Authorization headers and access_token query parameters.",
+                    errcode=Codes.MISSING_TOKEN,
                 )
             if len(auth_headers) > 1:
-                raise MissingClientTokenError("Too many Authorization headers.")
+                raise AuthError(
+                    token_not_found_http_status,
+                    "Too many Authorization headers.",
+                    errcode=Codes.MISSING_TOKEN,
+                )
             parts = auth_headers[0].split(b" ")
             if parts[0] == b"Bearer" and len(parts) == 2:
                 return parts[1].decode("ascii")
             else:
-                raise MissingClientTokenError("Invalid Authorization header.")
+                raise AuthError(
+                    token_not_found_http_status,
+                    "Invalid Authorization header.",
+                    errcode=Codes.MISSING_TOKEN,
+                )
         else:
             # Try to get the access_token from the query params.
             if not query_params:
-                raise MissingClientTokenError()
+                raise AuthError(
+                    token_not_found_http_status,
+                    "Missing access token.",
+                    errcode=Codes.MISSING_TOKEN,
+                )
 
             return query_params[0].decode("ascii")
 
@@ -690,7 +773,7 @@ class Auth(object):
             #  * The user is a guest user, and has joined the room
             # else it will throw.
             member_event = yield self.check_user_was_in_room(room_id, user_id)
-            return (member_event.membership, member_event.event_id)
+            defer.returnValue((member_event.membership, member_event.event_id))
         except AuthError:
             visibility = yield self.state.get_current_state(
                 room_id, EventTypes.RoomHistoryVisibility, ""
@@ -699,7 +782,7 @@ class Auth(object):
                 visibility
                 and visibility.content["history_visibility"] == "world_readable"
             ):
-                return (Membership.JOIN, None)
+                defer.returnValue((Membership.JOIN, None))
                 return
             raise AuthError(
                 403, "Guest access not allowed", errcode=Codes.GUEST_ACCESS_FORBIDDEN

synapse/api/errors.py

@@ -85,7 +85,7 @@ class SynapseError(CodeMessageException):
         errcode (str): Matrix error code e.g 'M_FORBIDDEN'
     """
 
-    def __init__(self, code, msg, errcode=Codes.UNKNOWN):
+    def __init__(self, code, msg, errcode=Codes.UNKNOWN, soft_logout=False):
         """Constructs a synapse error.
 
         Args:
@@ -95,9 +95,10 @@ class SynapseError(CodeMessageException):
         """
         super(SynapseError, self).__init__(code, msg)
         self.errcode = errcode
+        self.soft_logout = soft_logout
 
     def error_dict(self):
-        return cs_error(self.msg, self.errcode)
+        return cs_error(self.msg, self.errcode, self.soft_logout)
 
 
 class ProxiedRequestError(SynapseError):
@@ -139,22 +140,6 @@ class ConsentNotGivenError(SynapseError):
         return cs_error(self.msg, self.errcode, consent_uri=self._consent_uri)
 
 
-class UserDeactivatedError(SynapseError):
-    """The error returned to the client when the user attempted to access an
-    authenticated endpoint, but the account has been deactivated.
-    """
-
-    def __init__(self, msg):
-        """Constructs a UserDeactivatedError
-
-        Args:
-            msg (str): The human-readable error message
-        """
-        super(UserDeactivatedError, self).__init__(
-            code=http_client.FORBIDDEN, msg=msg, errcode=Codes.UNKNOWN
-        )
-
-
 class RegistrationError(SynapseError):
     """An error raised when a registration event fails."""
 
@@ -226,9 +211,7 @@ class NotFoundError(SynapseError):
 
 
 class AuthError(SynapseError):
-    """An error raised when there was a problem authorising an event, and at various
-    other poorly-defined times.
-    """
+    """An error raised when there was a problem authorising an event."""
 
     def __init__(self, *args, **kwargs):
         if "errcode" not in kwargs:
@@ -236,41 +219,6 @@ class AuthError(SynapseError):
         super(AuthError, self).__init__(*args, **kwargs)
 
 
-class InvalidClientCredentialsError(SynapseError):
-    """An error raised when there was a problem with the authorisation credentials
-    in a client request.
-
-    https://matrix.org/docs/spec/client_server/r0.5.0#using-access-tokens:
-
-    When credentials are required but missing or invalid, the HTTP call will
-    return with a status of 401 and the error code, M_MISSING_TOKEN or
-    M_UNKNOWN_TOKEN respectively.
-    """
-
-    def __init__(self, msg, errcode):
-        super().__init__(code=401, msg=msg, errcode=errcode)
-
-
-class MissingClientTokenError(InvalidClientCredentialsError):
-    """Raised when we couldn't find the access token in a request"""
-
-    def __init__(self, msg="Missing access token"):
-        super().__init__(msg=msg, errcode="M_MISSING_TOKEN")
-
-
-class InvalidClientTokenError(InvalidClientCredentialsError):
-    """Raised when we didn't understand the access token in a request"""
-
-    def __init__(self, msg="Unrecognised access token", soft_logout=False):
-        super().__init__(msg=msg, errcode="M_UNKNOWN_TOKEN")
-        self._soft_logout = soft_logout
-
-    def error_dict(self):
-        d = super().error_dict()
-        d["soft_logout"] = self._soft_logout
-        return d
-
-
 class ResourceLimitError(SynapseError):
     """
     Any error raised when there is a problem with resource usage.
@@ -436,7 +384,7 @@ class RequestSendFailed(RuntimeError):
         self.can_retry = can_retry
 
 
-def cs_error(msg, code=Codes.UNKNOWN, **kwargs):
+def cs_error(msg, code=Codes.UNKNOWN, soft_logout=False, **kwargs):
     """ Utility method for constructing an error response for client-server
     interactions.
 
@@ -447,7 +395,7 @@ def cs_error(msg, code=Codes.UNKNOWN, **kwargs):
     Returns:
         A dict representing the error response JSON.
     """
-    err = {"error": msg, "errcode": code}
+    err = {"error": msg, "errcode": code, "soft_logout": soft_logout}
     for key, value in iteritems(kwargs):
         err[key] = value
     return err

synapse/api/filtering.py

@@ -132,7 +132,7 @@ class Filtering(object):
     @defer.inlineCallbacks
     def get_user_filter(self, user_localpart, filter_id):
         result = yield self.store.get_user_filter(user_localpart, filter_id)
-        return FilterCollection(result)
+        defer.returnValue(FilterCollection(result))
 
     def add_user_filter(self, user_localpart, user_filter):
         self.check_valid_filter(user_filter)

synapse/app/_base.py

@@ -15,12 +15,10 @@
 
 import gc
 import logging
-import os
 import signal
 import sys
 import traceback
 
-import sdnotify
 from daemonize import Daemonize
 
 from twisted.internet import defer, error, reactor
@@ -50,7 +48,7 @@ def register_sighup(func):
     _sighup_callbacks.append(func)
 
 
-def start_worker_reactor(appname, config, run_command=reactor.run):
+def start_worker_reactor(appname, config):
     """ Run the reactor in the main process
 
     Daemonizes if necessary, and then configures some resources, before starting
@@ -59,7 +57,6 @@ def start_worker_reactor(appname, config, run_command=reactor.run):
     Args:
         appname (str): application name which will be sent to syslog
         config (synapse.config.Config): config object
-        run_command (Callable[]): callable that actually runs the reactor
     """
 
     logger = logging.getLogger(config.worker_app)
@@ -72,19 +69,11 @@ def start_worker_reactor(appname, config, run_command=reactor.run):
         daemonize=config.worker_daemonize,
         print_pidfile=config.print_pidfile,
         logger=logger,
-        run_command=run_command,
     )
 
 
 def start_reactor(
-    appname,
-    soft_file_limit,
-    gc_thresholds,
-    pid_file,
-    daemonize,
-    print_pidfile,
-    logger,
-    run_command=reactor.run,
+    appname, soft_file_limit, gc_thresholds, pid_file, daemonize, print_pidfile, logger
 ):
     """ Run the reactor in the main process
 
@@ -99,7 +88,6 @@ def start_reactor(
         daemonize (bool): true to run the reactor in a background process
         print_pidfile (bool): whether to print the pid file, if daemonize is True
         logger (logging.Logger): logger instance to pass to Daemonize
-        run_command (Callable[]): callable that actually runs the reactor
     """
 
     install_dns_limiter(reactor)
@@ -109,7 +97,7 @@ def start_reactor(
         change_resource_limit(soft_file_limit)
         if gc_thresholds:
             gc.set_threshold(*gc_thresholds)
-        run_command()
+        reactor.run()
 
     # make sure that we run the reactor with the sentinel log context,
     # otherwise other PreserveLoggingContext instances will get confused
@@ -151,7 +139,8 @@ def listen_metrics(bind_addresses, port):
     """
     Start Prometheus metrics server.
     """
-    from synapse.metrics import RegistryProxy, start_http_server
+    from synapse.metrics import RegistryProxy
+    from prometheus_client import start_http_server
 
     for host in bind_addresses:
         logger.info("Starting metrics listener on %s:%d", host, port)
@@ -244,16 +233,9 @@ def start(hs, listeners=None):
         if hasattr(signal, "SIGHUP"):
 
             def handle_sighup(*args, **kwargs):
-                # Tell systemd our state, if we're using it. This will silently fail if
-                # we're not using systemd.
-                sd_channel = sdnotify.SystemdNotifier()
-                sd_channel.notify("RELOADING=1")
-
                 for i in _sighup_callbacks:
                     i(hs)
 
-                sd_channel.notify("READY=1")
-
             signal.signal(signal.SIGHUP, handle_sighup)
 
             register_sighup(refresh_certificate)
@@ -261,15 +243,11 @@ def start(hs, listeners=None):
         # Load the certificate from disk.
         refresh_certificate(hs)
 
-        # Start the tracer
-        synapse.logging.opentracing.init_tracer(hs.config)
-
         # It is now safe to start your Synapse.
         hs.start_listening(listeners)
         hs.get_datastore().start_profiling()
 
         setup_sentry(hs)
-        setup_sdnotify(hs)
     except Exception:
         traceback.print_exc(file=sys.stderr)
         reactor = hs.get_reactor()
@@ -302,25 +280,6 @@ def setup_sentry(hs):
         scope.set_tag("worker_name", name)
 
 
-def setup_sdnotify(hs):
-    """Adds process state hooks to tell systemd what we are up to.
-    """
-
-    # Tell systemd our state, if we're using it. This will silently fail if
-    # we're not using systemd.
-    sd_channel = sdnotify.SystemdNotifier()
-
-    hs.get_reactor().addSystemEventTrigger(
-        "after",
-        "startup",
-        lambda: sd_channel.notify("READY=1\nMAINPID=%s" % (os.getpid())),
-    )
-
-    hs.get_reactor().addSystemEventTrigger(
-        "before", "shutdown", lambda: sd_channel.notify("STOPPING=1")
-    )
-
-
 def install_dns_limiter(reactor, max_dns_requests_in_flight=100):
     """Replaces the resolver with one that limits the number of in flight DNS
     requests.

synapse/app/admin_cmd.py

@@ -1,264 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-# Copyright 2019 Matrix.org Foundation C.I.C.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-import argparse
-import logging
-import os
-import sys
-import tempfile
-
-from canonicaljson import json
-
-from twisted.internet import defer, task
-
-import synapse
-from synapse.app import _base
-from synapse.config._base import ConfigError
-from synapse.config.homeserver import HomeServerConfig
-from synapse.config.logger import setup_logging
-from synapse.handlers.admin import ExfiltrationWriter
-from synapse.replication.slave.storage._base import BaseSlavedStore
-from synapse.replication.slave.storage.account_data import SlavedAccountDataStore
-from synapse.replication.slave.storage.appservice import SlavedApplicationServiceStore
-from synapse.replication.slave.storage.client_ips import SlavedClientIpStore
-from synapse.replication.slave.storage.deviceinbox import SlavedDeviceInboxStore
-from synapse.replication.slave.storage.devices import SlavedDeviceStore
-from synapse.replication.slave.storage.events import SlavedEventStore
-from synapse.replication.slave.storage.filtering import SlavedFilteringStore
-from synapse.replication.slave.storage.groups import SlavedGroupServerStore
-from synapse.replication.slave.storage.presence import SlavedPresenceStore
-from synapse.replication.slave.storage.push_rule import SlavedPushRuleStore
-from synapse.replication.slave.storage.receipts import SlavedReceiptsStore
-from synapse.replication.slave.storage.registration import SlavedRegistrationStore
-from synapse.replication.slave.storage.room import RoomStore
-from synapse.replication.tcp.client import ReplicationClientHandler
-from synapse.server import HomeServer
-from synapse.storage.engines import create_engine
-from synapse.util.logcontext import LoggingContext
-from synapse.util.versionstring import get_version_string
-
-logger = logging.getLogger("synapse.app.admin_cmd")
-
-
-class AdminCmdSlavedStore(
-    SlavedReceiptsStore,
-    SlavedAccountDataStore,
-    SlavedApplicationServiceStore,
-    SlavedRegistrationStore,
-    SlavedFilteringStore,
-    SlavedPresenceStore,
-    SlavedGroupServerStore,
-    SlavedDeviceInboxStore,
-    SlavedDeviceStore,
-    SlavedPushRuleStore,
-    SlavedEventStore,
-    SlavedClientIpStore,
-    RoomStore,
-    BaseSlavedStore,
-):
-    pass
-
-
-class AdminCmdServer(HomeServer):
-    DATASTORE_CLASS = AdminCmdSlavedStore
-
-    def _listen_http(self, listener_config):
-        pass
-
-    def start_listening(self, listeners):
-        pass
-
-    def build_tcp_replication(self):
-        return AdminCmdReplicationHandler(self)
-
-
-class AdminCmdReplicationHandler(ReplicationClientHandler):
-    @defer.inlineCallbacks
-    def on_rdata(self, stream_name, token, rows):
-        pass
-
-    def get_streams_to_replicate(self):
-        return {}
-
-
-@defer.inlineCallbacks
-def export_data_command(hs, args):
-    """Export data for a user.
-
-    Args:
-        hs (HomeServer)
-        args (argparse.Namespace)
-    """
-
-    user_id = args.user_id
-    directory = args.output_directory
-
-    res = yield hs.get_handlers().admin_handler.export_user_data(
-        user_id, FileExfiltrationWriter(user_id, directory=directory)
-    )
-    print(res)
-
-
-class FileExfiltrationWriter(ExfiltrationWriter):
-    """An ExfiltrationWriter that writes the users data to a directory.
-    Returns the directory location on completion.
-
-    Note: This writes to disk on the main reactor thread.
-
-    Args:
-        user_id (str): The user whose data is being exfiltrated.
-        directory (str|None): The directory to write the data to, if None then
-            will write to a temporary directory.
-    """
-
-    def __init__(self, user_id, directory=None):
-        self.user_id = user_id
-
-        if directory:
-            self.base_directory = directory
-        else:
-            self.base_directory = tempfile.mkdtemp(
-                prefix="synapse-exfiltrate__%s__" % (user_id,)
-            )
-
-        os.makedirs(self.base_directory, exist_ok=True)
-        if list(os.listdir(self.base_directory)):
-            raise Exception("Directory must be empty")
-
-    def write_events(self, room_id, events):
-        room_directory = os.path.join(self.base_directory, "rooms", room_id)
-        os.makedirs(room_directory, exist_ok=True)
-        events_file = os.path.join(room_directory, "events")
-
-        with open(events_file, "a") as f:
-            for event in events:
-                print(json.dumps(event.get_pdu_json()), file=f)
-
-    def write_state(self, room_id, event_id, state):
-        room_directory = os.path.join(self.base_directory, "rooms", room_id)
-        state_directory = os.path.join(room_directory, "state")
-        os.makedirs(state_directory, exist_ok=True)
-
-        event_file = os.path.join(state_directory, event_id)
-
-        with open(event_file, "a") as f:
-            for event in state.values():
-                print(json.dumps(event.get_pdu_json()), file=f)
-
-    def write_invite(self, room_id, event, state):
-        self.write_events(room_id, [event])
-
-        # We write the invite state somewhere else as they aren't full events
-        # and are only a subset of the state at the event.
-        room_directory = os.path.join(self.base_directory, "rooms", room_id)
-        os.makedirs(room_directory, exist_ok=True)
-
-        invite_state = os.path.join(room_directory, "invite_state")
-
-        with open(invite_state, "a") as f:
-            for event in state.values():
-                print(json.dumps(event), file=f)
-
-    def finished(self):
-        return self.base_directory
-
-
-def start(config_options):
-    parser = argparse.ArgumentParser(description="Synapse Admin Command")
-    HomeServerConfig.add_arguments_to_parser(parser)
-
-    subparser = parser.add_subparsers(
-        title="Admin Commands",
-        required=True,
-        dest="command",
-        metavar="<admin_command>",
-        help="The admin command to perform.",
-    )
-    export_data_parser = subparser.add_parser(
-        "export-data", help="Export all data for a user"
-    )
-    export_data_parser.add_argument("user_id", help="User to extra data from")
-    export_data_parser.add_argument(
-        "--output-directory",
-        action="store",
-        metavar="DIRECTORY",
-        required=False,
-        help="The directory to store the exported data in. Must be empty. Defaults"
-        " to creating a temp directory.",
-    )
-    export_data_parser.set_defaults(func=export_data_command)
-
-    try:
-        config, args = HomeServerConfig.load_config_with_parser(parser, config_options)
-    except ConfigError as e:
-        sys.stderr.write("\n" + str(e) + "\n")
-        sys.exit(1)
-
-    if config.worker_app is not None:
-        assert config.worker_app == "synapse.app.admin_cmd"
-
-    # Update the config with some basic overrides so that don't have to specify
-    # a full worker config.
-    config.worker_app = "synapse.app.admin_cmd"
-
-    if (
-        not config.worker_daemonize
-        and not config.worker_log_file
-        and not config.worker_log_config
-    ):
-        # Since we're meant to be run as a "command" let's not redirect stdio
-        # unless we've actually set log config.
-        config.no_redirect_stdio = True
-
-    # Explicitly disable background processes
-    config.update_user_directory = False
-    config.start_pushers = False
-    config.send_federation = False
-
-    setup_logging(config, use_worker_options=True)
-
-    synapse.events.USE_FROZEN_DICTS = config.use_frozen_dicts
-
-    database_engine = create_engine(config.database_config)
-
-    ss = AdminCmdServer(
-        config.server_name,
-        db_config=config.database_config,
-        config=config,
-        version_string="Synapse/" + get_version_string(synapse),
-        database_engine=database_engine,
-    )
-
-    ss.setup()
-
-    # We use task.react as the basic run command as it correctly handles tearing
-    # down the reactor when the deferreds resolve and setting the return value.
-    # We also make sure that `_base.start` gets run before we actually run the
-    # command.
-
-    @defer.inlineCallbacks
-    def run(_reactor):
-        with LoggingContext("command"):
-            yield _base.start(ss, [])
-            yield args.func(ss, args)
-
-    _base.start_worker_reactor(
-        "synapse-admin-cmd", config, run_command=lambda: task.react(run)
-    )
-
-
-if __name__ == "__main__":
-    with LoggingContext("main"):
-        start(sys.argv[1:])

synapse/app/appservice.py

@@ -27,7 +27,8 @@ from synapse.config.homeserver import HomeServerConfig
 from synapse.config.logger import setup_logging
 from synapse.http.site import SynapseSite
 from synapse.logging.context import LoggingContext, run_in_background
-from synapse.metrics import METRICS_PREFIX, MetricsResource, RegistryProxy
+from synapse.metrics import RegistryProxy
+from synapse.metrics.resource import METRICS_PREFIX, MetricsResource
 from synapse.replication.slave.storage.appservice import SlavedApplicationServiceStore
 from synapse.replication.slave.storage.directory import DirectoryStore
 from synapse.replication.slave.storage.events import SlavedEventStore
@@ -168,9 +169,7 @@ def start(config_options):
     )
 
     ps.setup()
-    reactor.addSystemEventTrigger(
-        "before", "startup", _base.start, ps, config.worker_listeners
-    )
+    reactor.callWhenRunning(_base.start, ps, config.worker_listeners)
 
     _base.start_worker_reactor("synapse-appservice", config)
 

synapse/app/client_reader.py

@@ -28,7 +28,8 @@ from synapse.config.logger import setup_logging
 from synapse.http.server import JsonResource
 from synapse.http.site import SynapseSite
 from synapse.logging.context import LoggingContext
-from synapse.metrics import METRICS_PREFIX, MetricsResource, RegistryProxy
+from synapse.metrics import RegistryProxy
+from synapse.metrics.resource import METRICS_PREFIX, MetricsResource
 from synapse.replication.slave.storage._base import BaseSlavedStore
 from synapse.replication.slave.storage.account_data import SlavedAccountDataStore
 from synapse.replication.slave.storage.appservice import SlavedApplicationServiceStore
@@ -194,9 +195,7 @@ def start(config_options):
     )
 
     ss.setup()
-    reactor.addSystemEventTrigger(
-        "before", "startup", _base.start, ss, config.worker_listeners
-    )
+    reactor.callWhenRunning(_base.start, ss, config.worker_listeners)
 
     _base.start_worker_reactor("synapse-client-reader", config)
 

synapse/app/event_creator.py

@@ -28,7 +28,8 @@ from synapse.config.logger import setup_logging
 from synapse.http.server import JsonResource
 from synapse.http.site import SynapseSite
 from synapse.logging.context import LoggingContext
-from synapse.metrics import METRICS_PREFIX, MetricsResource, RegistryProxy
+from synapse.metrics import RegistryProxy
+from synapse.metrics.resource import METRICS_PREFIX, MetricsResource
 from synapse.replication.slave.storage._base import BaseSlavedStore
 from synapse.replication.slave.storage.account_data import SlavedAccountDataStore
 from synapse.replication.slave.storage.appservice import SlavedApplicationServiceStore
@@ -193,9 +194,7 @@ def start(config_options):
     )
 
     ss.setup()
-    reactor.addSystemEventTrigger(
-        "before", "startup", _base.start, ss, config.worker_listeners
-    )
+    reactor.callWhenRunning(_base.start, ss, config.worker_listeners)
 
     _base.start_worker_reactor("synapse-event-creator", config)
 

synapse/app/federation_reader.py

@@ -29,7 +29,8 @@ from synapse.config.logger import setup_logging
 from synapse.federation.transport.server import TransportLayerServer
 from synapse.http.site import SynapseSite
 from synapse.logging.context import LoggingContext
-from synapse.metrics import METRICS_PREFIX, MetricsResource, RegistryProxy
+from synapse.metrics import RegistryProxy
+from synapse.metrics.resource import METRICS_PREFIX, MetricsResource
 from synapse.replication.slave.storage._base import BaseSlavedStore
 from synapse.replication.slave.storage.account_data import SlavedAccountDataStore
 from synapse.replication.slave.storage.appservice import SlavedApplicationServiceStore
@@ -175,9 +176,7 @@ def start(config_options):
     )
 
     ss.setup()
-    reactor.addSystemEventTrigger(
-        "before", "startup", _base.start, ss, config.worker_listeners
-    )
+    reactor.callWhenRunning(_base.start, ss, config.worker_listeners)
 
     _base.start_worker_reactor("synapse-federation-reader", config)
 

synapse/app/federation_sender.py

@@ -28,8 +28,9 @@ from synapse.config.logger import setup_logging
 from synapse.federation import send_queue
 from synapse.http.site import SynapseSite
 from synapse.logging.context import LoggingContext, run_in_background
-from synapse.metrics import METRICS_PREFIX, MetricsResource, RegistryProxy
+from synapse.metrics import RegistryProxy
 from synapse.metrics.background_process_metrics import run_as_background_process
+from synapse.metrics.resource import METRICS_PREFIX, MetricsResource
 from synapse.replication.slave.storage.deviceinbox import SlavedDeviceInboxStore
 from synapse.replication.slave.storage.devices import SlavedDeviceStore
 from synapse.replication.slave.storage.events import SlavedEventStore
@@ -198,9 +199,7 @@ def start(config_options):
     )
 
     ss.setup()
-    reactor.addSystemEventTrigger(
-        "before", "startup", _base.start, ss, config.worker_listeners
-    )
+    reactor.callWhenRunning(_base.start, ss, config.worker_listeners)
 
     _base.start_worker_reactor("synapse-federation-sender", config)
 

synapse/app/frontend_proxy.py

@@ -30,7 +30,8 @@ from synapse.http.server import JsonResource
 from synapse.http.servlet import RestServlet, parse_json_object_from_request
 from synapse.http.site import SynapseSite
 from synapse.logging.context import LoggingContext
-from synapse.metrics import METRICS_PREFIX, MetricsResource, RegistryProxy
+from synapse.metrics import RegistryProxy
+from synapse.metrics.resource import METRICS_PREFIX, MetricsResource
 from synapse.replication.slave.storage._base import BaseSlavedStore
 from synapse.replication.slave.storage.appservice import SlavedApplicationServiceStore
 from synapse.replication.slave.storage.client_ips import SlavedClientIpStore
@@ -70,12 +71,12 @@ class PresenceStatusStubServlet(RestServlet):
         except HttpResponseException as e:
             raise e.to_synapse_error()
 
-        return (200, result)
+        defer.returnValue((200, result))
 
     @defer.inlineCallbacks
     def on_PUT(self, request, user_id):
         yield self.auth.get_user_by_req(request)
-        return (200, {})
+        defer.returnValue((200, {}))
 
 
 class KeyUploadServlet(RestServlet):
@@ -126,11 +127,11 @@ class KeyUploadServlet(RestServlet):
                 self.main_uri + request.uri.decode("ascii"), body, headers=headers
             )
 
-            return (200, result)
+            defer.returnValue((200, result))
         else:
             # Just interested in counts.
             result = yield self.store.count_e2e_one_time_keys(user_id, device_id)
-            return (200, {"one_time_key_counts": result})
+            defer.returnValue((200, {"one_time_key_counts": result}))
 
 
 class FrontendProxySlavedStore(
@@ -247,9 +248,7 @@ def start(config_options):
     )
 
     ss.setup()
-    reactor.addSystemEventTrigger(
-        "before", "startup", _base.start, ss, config.worker_listeners
-    )
+    reactor.callWhenRunning(_base.start, ss, config.worker_listeners)
 
     _base.start_worker_reactor("synapse-frontend-proxy", config)
 

synapse/app/homeserver.py

@@ -55,8 +55,9 @@ from synapse.http.additional_resource import AdditionalResource
 from synapse.http.server import RootRedirect
 from synapse.http.site import SynapseSite
 from synapse.logging.context import LoggingContext
-from synapse.metrics import METRICS_PREFIX, MetricsResource, RegistryProxy
+from synapse.metrics import RegistryProxy
 from synapse.metrics.background_process_metrics import run_as_background_process
+from synapse.metrics.resource import METRICS_PREFIX, MetricsResource
 from synapse.module_api import ModuleApi
 from synapse.python_dependencies import check_requirements
 from synapse.replication.http import REPLICATION_PREFIX, ReplicationRestResource
@@ -406,7 +407,7 @@ def setup(config_options):
         if provision:
             yield acme.provision_certificate()
 
-        return provision
+        defer.returnValue(provision)
 
     @defer.inlineCallbacks
     def reprovision_acme():
@@ -447,7 +448,7 @@ def setup(config_options):
                 reactor.stop()
             sys.exit(1)
 
-    reactor.addSystemEventTrigger("before", "startup", start)
+    reactor.callWhenRunning(start)
 
     return hs
 

synapse/app/media_repository.py

@@ -28,7 +28,8 @@ from synapse.config.homeserver import HomeServerConfig
 from synapse.config.logger import setup_logging
 from synapse.http.site import SynapseSite
 from synapse.logging.context import LoggingContext
-from synapse.metrics import METRICS_PREFIX, MetricsResource, RegistryProxy
+from synapse.metrics import RegistryProxy
+from synapse.metrics.resource import METRICS_PREFIX, MetricsResource
 from synapse.replication.slave.storage._base import BaseSlavedStore
 from synapse.replication.slave.storage.appservice import SlavedApplicationServiceStore
 from synapse.replication.slave.storage.client_ips import SlavedClientIpStore
@@ -161,9 +162,7 @@ def start(config_options):
     )
 
     ss.setup()
-    reactor.addSystemEventTrigger(
-        "before", "startup", _base.start, ss, config.worker_listeners
-    )
+    reactor.callWhenRunning(_base.start, ss, config.worker_listeners)
 
     _base.start_worker_reactor("synapse-media-repository", config)
 

synapse/app/pusher.py

@@ -27,7 +27,8 @@ from synapse.config.homeserver import HomeServerConfig
 from synapse.config.logger import setup_logging
 from synapse.http.site import SynapseSite
 from synapse.logging.context import LoggingContext, run_in_background
-from synapse.metrics import METRICS_PREFIX, MetricsResource, RegistryProxy
+from synapse.metrics import RegistryProxy
+from synapse.metrics.resource import METRICS_PREFIX, MetricsResource
 from synapse.replication.slave.storage._base import __func__
 from synapse.replication.slave.storage.account_data import SlavedAccountDataStore
 from synapse.replication.slave.storage.events import SlavedEventStore
@@ -216,7 +217,7 @@ def start(config_options):
         _base.start(ps, config.worker_listeners)
         ps.get_pusherpool().start()
 
-    reactor.addSystemEventTrigger("before", "startup", start)
+    reactor.callWhenRunning(start)
 
     _base.start_worker_reactor("synapse-pusher", config)
 

synapse/app/synchrotron.py

@@ -32,7 +32,8 @@ from synapse.handlers.presence import PresenceHandler, get_interested_parties
 from synapse.http.server import JsonResource
 from synapse.http.site import SynapseSite
 from synapse.logging.context import LoggingContext, run_in_background
-from synapse.metrics import METRICS_PREFIX, MetricsResource, RegistryProxy
+from synapse.metrics import RegistryProxy
+from synapse.metrics.resource import METRICS_PREFIX, MetricsResource
 from synapse.replication.slave.storage._base import BaseSlavedStore, __func__
 from synapse.replication.slave.storage.account_data import SlavedAccountDataStore
 from synapse.replication.slave.storage.appservice import SlavedApplicationServiceStore
@@ -451,9 +452,7 @@ def start(config_options):
     )
 
     ss.setup()
-    reactor.addSystemEventTrigger(
-        "before", "startup", _base.start, ss, config.worker_listeners
-    )
+    reactor.callWhenRunning(_base.start, ss, config.worker_listeners)
 
     _base.start_worker_reactor("synapse-synchrotron", config)
 

synapse/app/user_dir.py

@@ -29,7 +29,8 @@ from synapse.config.logger import setup_logging
 from synapse.http.server import JsonResource
 from synapse.http.site import SynapseSite
 from synapse.logging.context import LoggingContext, run_in_background
-from synapse.metrics import METRICS_PREFIX, MetricsResource, RegistryProxy
+from synapse.metrics import RegistryProxy
+from synapse.metrics.resource import METRICS_PREFIX, MetricsResource
 from synapse.replication.slave.storage._base import BaseSlavedStore
 from synapse.replication.slave.storage.appservice import SlavedApplicationServiceStore
 from synapse.replication.slave.storage.client_ips import SlavedClientIpStore
@@ -224,9 +225,7 @@ def start(config_options):
     )
 
     ss.setup()
-    reactor.addSystemEventTrigger(
-        "before", "startup", _base.start, ss, config.worker_listeners
-    )
+    reactor.callWhenRunning(_base.start, ss, config.worker_listeners)
 
     _base.start_worker_reactor("synapse-user-dir", config)
 

synapse/appservice/__init__.py

@@ -175,21 +175,21 @@ class ApplicationService(object):
     @defer.inlineCallbacks
     def _matches_user(self, event, store):
         if not event:
-            return False
+            defer.returnValue(False)
 
         if self.is_interested_in_user(event.sender):
-            return True
+            defer.returnValue(True)
         # also check m.room.member state key
         if event.type == EventTypes.Member and self.is_interested_in_user(
             event.state_key
         ):
-            return True
+            defer.returnValue(True)
 
         if not store:
-            return False
+            defer.returnValue(False)
 
         does_match = yield self._matches_user_in_member_list(event.room_id, store)
-        return does_match
+        defer.returnValue(does_match)
 
     @cachedInlineCallbacks(num_args=1, cache_context=True)
     def _matches_user_in_member_list(self, room_id, store, cache_context):
@@ -200,8 +200,8 @@ class ApplicationService(object):
         # check joined member events
         for user_id in member_list:
             if self.is_interested_in_user(user_id):
-                return True
-        return False
+                defer.returnValue(True)
+        defer.returnValue(False)
 
     def _matches_room_id(self, event):
         if hasattr(event, "room_id"):
@@ -211,13 +211,13 @@ class ApplicationService(object):
     @defer.inlineCallbacks
     def _matches_aliases(self, event, store):
         if not store or not event:
-            return False
+            defer.returnValue(False)
 
         alias_list = yield store.get_aliases_for_room(event.room_id)
         for alias in alias_list:
             if self.is_interested_in_alias(alias):
-                return True
-        return False
+                defer.returnValue(True)
+        defer.returnValue(False)
 
     @defer.inlineCallbacks
     def is_interested(self, event, store=None):
@@ -231,15 +231,15 @@ class ApplicationService(object):
         """
         # Do cheap checks first
         if self._matches_room_id(event):
-            return True
+            defer.returnValue(True)
 
         if (yield self._matches_aliases(event, store)):
-            return True
+            defer.returnValue(True)
 
         if (yield self._matches_user(event, store)):
-            return True
+            defer.returnValue(True)
 
-        return False
+        defer.returnValue(False)
 
     def is_interested_in_user(self, user_id):
         return (

synapse/appservice/api.py

@@ -97,40 +97,40 @@ class ApplicationServiceApi(SimpleHttpClient):
     @defer.inlineCallbacks
     def query_user(self, service, user_id):
         if service.url is None:
-            return False
+            defer.returnValue(False)
         uri = service.url + ("/users/%s" % urllib.parse.quote(user_id))
         response = None
         try:
             response = yield self.get_json(uri, {"access_token": service.hs_token})
             if response is not None:  # just an empty json object
-                return True
+                defer.returnValue(True)
         except CodeMessageException as e:
             if e.code == 404:
-                return False
+                defer.returnValue(False)
                 return
             logger.warning("query_user to %s received %s", uri, e.code)
         except Exception as ex:
             logger.warning("query_user to %s threw exception %s", uri, ex)
-        return False
+        defer.returnValue(False)
 
     @defer.inlineCallbacks
     def query_alias(self, service, alias):
         if service.url is None:
-            return False
+            defer.returnValue(False)
         uri = service.url + ("/rooms/%s" % urllib.parse.quote(alias))
         response = None
         try:
             response = yield self.get_json(uri, {"access_token": service.hs_token})
             if response is not None:  # just an empty json object
-                return True
+                defer.returnValue(True)
         except CodeMessageException as e:
             logger.warning("query_alias to %s received %s", uri, e.code)
             if e.code == 404:
-                return False
+                defer.returnValue(False)
                 return
         except Exception as ex:
             logger.warning("query_alias to %s threw exception %s", uri, ex)
-        return False
+        defer.returnValue(False)
 
     @defer.inlineCallbacks
     def query_3pe(self, service, kind, protocol, fields):
@@ -141,7 +141,7 @@ class ApplicationServiceApi(SimpleHttpClient):
         else:
             raise ValueError("Unrecognised 'kind' argument %r to query_3pe()", kind)
         if service.url is None:
-            return []
+            defer.returnValue([])
 
         uri = "%s%s/thirdparty/%s/%s" % (
             service.url,
@@ -155,7 +155,7 @@ class ApplicationServiceApi(SimpleHttpClient):
                 logger.warning(
                     "query_3pe to %s returned an invalid response %r", uri, response
                 )
-                return []
+                defer.returnValue([])
 
             ret = []
             for r in response:
@@ -166,14 +166,14 @@ class ApplicationServiceApi(SimpleHttpClient):
                         "query_3pe to %s returned an invalid result %r", uri, r
                     )
 
-            return ret
+            defer.returnValue(ret)
         except Exception as ex:
             logger.warning("query_3pe to %s threw exception %s", uri, ex)
-            return []
+            defer.returnValue([])
 
     def get_3pe_protocol(self, service, protocol):
         if service.url is None:
-            return {}
+            defer.returnValue({})
 
         @defer.inlineCallbacks
         def _get():
@@ -189,7 +189,7 @@ class ApplicationServiceApi(SimpleHttpClient):
                     logger.warning(
                         "query_3pe_protocol to %s did not return a" " valid result", uri
                     )
-                    return None
+                    defer.returnValue(None)
 
                 for instance in info.get("instances", []):
                     network_id = instance.get("network_id", None)
@@ -198,10 +198,10 @@ class ApplicationServiceApi(SimpleHttpClient):
                             service.id, network_id
                         ).to_string()
 
-                return info
+                defer.returnValue(info)
             except Exception as ex:
                 logger.warning("query_3pe_protocol to %s threw exception %s", uri, ex)
-                return None
+                defer.returnValue(None)
 
         key = (service.id, protocol)
         return self.protocol_meta_cache.wrap(key, _get)
@@ -209,7 +209,7 @@ class ApplicationServiceApi(SimpleHttpClient):
     @defer.inlineCallbacks
     def push_bulk(self, service, events, txn_id=None):
         if service.url is None:
-            return True
+            defer.returnValue(True)
 
         events = self._serialize(events)
 
@@ -229,14 +229,14 @@ class ApplicationServiceApi(SimpleHttpClient):
             )
             sent_transactions_counter.labels(service.id).inc()
             sent_events_counter.labels(service.id).inc(len(events))
-            return True
+            defer.returnValue(True)
             return
         except CodeMessageException as e:
             logger.warning("push_bulk to %s received %s", uri, e.code)
         except Exception as ex:
             logger.warning("push_bulk to %s threw exception %s", uri, ex)
         failed_transactions_counter.labels(service.id).inc()
-        return False
+        defer.returnValue(False)
 
     def _serialize(self, events):
         time_now = self.clock.time_msec()

synapse/appservice/scheduler.py

@@ -193,7 +193,7 @@ class _TransactionController(object):
     @defer.inlineCallbacks
     def _is_service_up(self, service):
         state = yield self.store.get_appservice_state(service)
-        return state == ApplicationServiceState.UP or state is None
+        defer.returnValue(state == ApplicationServiceState.UP or state is None)
 
 
 class _Recoverer(object):
@@ -208,7 +208,7 @@ class _Recoverer(object):
                 r.service.id,
             )
             r.recover()
-        return recoverers
+        defer.returnValue(recoverers)
 
     def __init__(self, clock, store, as_api, service, callback):
         self.clock = clock

synapse/config/_base.py

@@ -137,42 +137,12 @@ class Config(object):
             return file_stream.read()
 
     def invoke_all(self, name, *args, **kargs):
-        """Invoke all instance methods with the given name and arguments in the
-        class's MRO.
-
-        Args:
-            name (str): Name of function to invoke
-            *args
-            **kwargs
-
-        Returns:
-            list: The list of the return values from each method called
-        """
         results = []
         for cls in type(self).mro():
             if name in cls.__dict__:
                 results.append(getattr(cls, name)(self, *args, **kargs))
         return results
 
-    @classmethod
-    def invoke_all_static(cls, name, *args, **kargs):
-        """Invoke all static methods with the given name and arguments in the
-        class's MRO.
-
-        Args:
-            name (str): Name of function to invoke
-            *args
-            **kwargs
-
-        Returns:
-            list: The list of the return values from each method called
-        """
-        results = []
-        for c in cls.mro():
-            if name in c.__dict__:
-                results.append(getattr(c, name)(*args, **kargs))
-        return results
-
     def generate_config(
         self,
         config_dir_path,
@@ -232,23 +202,6 @@ class Config(object):
         Returns: Config object.
         """
         config_parser = argparse.ArgumentParser(description=description)
-        cls.add_arguments_to_parser(config_parser)
-        obj, _ = cls.load_config_with_parser(config_parser, argv)
-
-        return obj
-
-    @classmethod
-    def add_arguments_to_parser(cls, config_parser):
-        """Adds all the config flags to an ArgumentParser.
-
-        Doesn't support config-file-generation: used by the worker apps.
-
-        Used for workers where we want to add extra flags/subcommands.
-
-        Args:
-            config_parser (ArgumentParser): App description
-        """
-
         config_parser.add_argument(
             "-c",
             "--config-path",
@@ -266,34 +219,16 @@ class Config(object):
             " Defaults to the directory containing the last config file",
         )
 
-        cls.invoke_all_static("add_arguments", config_parser)
-
-    @classmethod
-    def load_config_with_parser(cls, parser, argv):
-        """Parse the commandline and config files with the given parser
-
-        Doesn't support config-file-generation: used by the worker apps.
-
-        Used for workers where we want to add extra flags/subcommands.
-
-        Args:
-            parser (ArgumentParser)
-            argv (list[str])
-
-        Returns:
-            tuple[HomeServerConfig, argparse.Namespace]: Returns the parsed
-            config object and the parsed argparse.Namespace object from
-            `parser.parse_args(..)`
-        """
-
         obj = cls()
 
-        config_args = parser.parse_args(argv)
+        obj.invoke_all("add_arguments", config_parser)
+
+        config_args = config_parser.parse_args(argv)
 
         config_files = find_config_files(search_paths=config_args.config_path)
 
         if not config_files:
-            parser.error("Must supply a config file.")
+            config_parser.error("Must supply a config file.")
 
         if config_args.keys_directory:
             config_dir_path = config_args.keys_directory
@@ -309,7 +244,7 @@ class Config(object):
 
         obj.invoke_all("read_arguments", config_args)
 
-        return obj, config_args
+        return obj
 
     @classmethod
     def load_or_generate_config(cls, description, argv):
@@ -466,7 +401,7 @@ class Config(object):
             formatter_class=argparse.RawDescriptionHelpFormatter,
         )
 
-        obj.invoke_all_static("add_arguments", parser)
+        obj.invoke_all("add_arguments", parser)
         args = parser.parse_args(remaining_args)
 
         config_dict = read_config_files(config_files)

synapse/config/database.py

@@ -69,8 +69,7 @@ class DatabaseConfig(Config):
             if database_path is not None:
                 self.database_config["args"]["database"] = database_path
 
-    @staticmethod
-    def add_arguments(parser):
+    def add_arguments(self, parser):
         db_group = parser.add_argument_group("database")
         db_group.add_argument(
             "-d",

synapse/config/emailconfig.py

@@ -112,17 +112,13 @@ class EmailConfig(Config):
             missing = []
             for k in required:
                 if k not in email_config:
-                    missing.append("email." + k)
-
-            if config.get("public_baseurl") is None:
-                missing.append("public_base_url")
+                    missing.append(k)
 
             if len(missing) > 0:
                 raise RuntimeError(
-                    "Password resets emails are configured to be sent from "
-                    "this homeserver due to a partial 'email' block. "
-                    "However, the following required keys are missing: %s"
-                    % (", ".join(missing),)
+                    "email.password_reset_behaviour is set to 'local' "
+                    "but required keys are missing: %s"
+                    % (", ".join(["email." + k for k in missing]),)
                 )
 
             # Templates for password reset emails
@@ -160,6 +156,13 @@ class EmailConfig(Config):
                 filepath, "email.password_reset_template_success_html"
             )
 
+            if config.get("public_baseurl") is None:
+                raise RuntimeError(
+                    "email.password_reset_behaviour is set to 'local' but no "
+                    "public_baseurl is set. This is necessary to generate password "
+                    "reset links"
+                )
+
         if self.email_enable_notifs:
             required = [
                 "smtp_host",

synapse/config/homeserver.py

@@ -40,7 +40,6 @@ from .spam_checker import SpamCheckerConfig
 from .stats import StatsConfig
 from .third_party_event_rules import ThirdPartyRulesConfig
 from .tls import TlsConfig
-from .tracer import TracerConfig
 from .user_directory import UserDirectoryConfig
 from .voip import VoipConfig
 from .workers import WorkerConfig
@@ -76,6 +75,5 @@ class HomeServerConfig(
     ServerNoticesConfig,
     RoomDirectoryConfig,
     ThirdPartyRulesConfig,
-    TracerConfig,
 ):
     pass

synapse/config/key.py

@@ -116,6 +116,8 @@ class KeyConfig(Config):
             seed = bytes(self.signing_key[0])
             self.macaroon_secret_key = hashlib.sha256(seed).digest()
 
+        self.expire_access_token = config.get("expire_access_token", False)
+
         # a secret which is used to calculate HMACs for form values, to stop
         # falsification of values
         self.form_secret = config.get("form_secret", None)
@@ -142,6 +144,10 @@ class KeyConfig(Config):
         #
         %(macaroon_secret_key)s
 
+        # Used to enable access token expiration.
+        #
+        #expire_access_token: False
+
         # a secret which is used to calculate HMACs for form values, to stop
         # falsification of values. Must be specified for the User Consent
         # forms to work.

synapse/config/logger.py

@@ -12,7 +12,6 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-
 import logging
 import logging.config
 import os
@@ -76,8 +75,10 @@ root:
 
 class LoggingConfig(Config):
     def read_config(self, config, **kwargs):
-        self.log_config = self.abspath(config.get("log_config"))
+        self.verbosity = config.get("verbose", 0)
         self.no_redirect_stdio = config.get("no_redirect_stdio", False)
+        self.log_config = self.abspath(config.get("log_config"))
+        self.log_file = self.abspath(config.get("log_file"))
 
     def generate_config_section(self, config_dir_path, server_name, **kwargs):
         log_config = os.path.join(config_dir_path, server_name + ".log.config")
@@ -93,12 +94,37 @@ class LoggingConfig(Config):
         )
 
     def read_arguments(self, args):
+        if args.verbose is not None:
+            self.verbosity = args.verbose
         if args.no_redirect_stdio is not None:
             self.no_redirect_stdio = args.no_redirect_stdio
+        if args.log_config is not None:
+            self.log_config = args.log_config
+        if args.log_file is not None:
+            self.log_file = args.log_file
 
-    @staticmethod
-    def add_arguments(parser):
+    def add_arguments(cls, parser):
         logging_group = parser.add_argument_group("logging")
+        logging_group.add_argument(
+            "-v",
+            "--verbose",
+            dest="verbose",
+            action="count",
+            help="The verbosity level. Specify multiple times to increase "
+            "verbosity. (Ignored if --log-config is specified.)",
+        )
+        logging_group.add_argument(
+            "-f",
+            "--log-file",
+            dest="log_file",
+            help="File to log to. (Ignored if --log-config is specified.)",
+        )
+        logging_group.add_argument(
+            "--log-config",
+            dest="log_config",
+            default=None,
+            help="Python logging config file",
+        )
         logging_group.add_argument(
             "-n",
             "--no-redirect-stdio",
@@ -126,29 +152,58 @@ def setup_logging(config, use_worker_options=False):
         config (LoggingConfig | synapse.config.workers.WorkerConfig):
             configuration data
 
-        use_worker_options (bool): True to use the 'worker_log_config' option
-            instead of 'log_config'.
+        use_worker_options (bool): True to use 'worker_log_config' and
+            'worker_log_file' options instead of 'log_config' and 'log_file'.
 
         register_sighup (func | None): Function to call to register a
             sighup handler.
     """
     log_config = config.worker_log_config if use_worker_options else config.log_config
+    log_file = config.worker_log_file if use_worker_options else config.log_file
+
+    log_format = (
+        "%(asctime)s - %(name)s - %(lineno)d - %(levelname)s - %(request)s"
+        " - %(message)s"
+    )
 
     if log_config is None:
-        log_format = (
-            "%(asctime)s - %(name)s - %(lineno)d - %(levelname)s - %(request)s"
-            " - %(message)s"
-        )
+        # We don't have a logfile, so fall back to the 'verbosity' param from
+        # the config or cmdline. (Note that we generate a log config for new
+        # installs, so this will be an unusual case)
+        level = logging.INFO
+        level_for_storage = logging.INFO
+        if config.verbosity:
+            level = logging.DEBUG
+            if config.verbosity > 1:
+                level_for_storage = logging.DEBUG
 
         logger = logging.getLogger("")
-        logger.setLevel(logging.INFO)
-        logging.getLogger("synapse.storage.SQL").setLevel(logging.INFO)
+        logger.setLevel(level)
+
+        logging.getLogger("synapse.storage.SQL").setLevel(level_for_storage)
 
         formatter = logging.Formatter(log_format)
+        if log_file:
+            # TODO: Customisable file size / backup count
+            handler = logging.handlers.RotatingFileHandler(
+                log_file, maxBytes=(1000 * 1000 * 100), backupCount=3, encoding="utf8"
+            )
+
+            def sighup(signum, stack):
+                logger.info("Closing log file due to SIGHUP")
+                handler.doRollover()
+                logger.info("Opened new log file due to SIGHUP")
+
+        else:
+            handler = logging.StreamHandler()
+
+            def sighup(*args):
+                pass
 
-        handler = logging.StreamHandler()
         handler.setFormatter(formatter)
+
         handler.addFilter(LoggingContextFilter(request=""))
+
         logger.addHandler(handler)
     else:
 
@@ -162,7 +217,8 @@ def setup_logging(config, use_worker_options=False):
             logging.info("Reloaded log config from %s due to SIGHUP", log_config)
 
         load_log_config()
-        appbase.register_sighup(sighup)
+
+    appbase.register_sighup(sighup)
 
     # make sure that the first thing we log is a thing we can grep backwards
     # for

synapse/config/ratelimiting.py

@@ -23,7 +23,7 @@ class RateLimitConfig(object):
 
 class FederationRateLimitConfig(object):
     _items_and_default = {
-        "window_size": 1000,
+        "window_size": 10000,
         "sleep_limit": 10,
         "sleep_delay": 500,
         "reject_limit": 50,
@@ -54,7 +54,7 @@ class RatelimitConfig(Config):
 
         # Load the new-style federation config, if it exists. Otherwise, fall
         # back to the old method.
-        if "rc_federation" in config:
+        if "federation_rc" in config:
             self.rc_federation = FederationRateLimitConfig(**config["rc_federation"])
         else:
             self.rc_federation = FederationRateLimitConfig(