change other 404 to a SynapseError

Share SSL options for well-known requests

.buildkite/pipeline.yml

@@ -5,8 +5,8 @@ steps:
 
   - command:
       - "python -m pip install tox"
-      - "tox -e pep8"
-    label: "\U0001F9F9 PEP-8"
+      - "tox -e check_codestyle"
+    label: "\U0001F9F9 Check Style"
     plugins:
       - docker#v3.0.1:
           image: "python:3.6"
@@ -49,14 +49,15 @@ steps:
 
 
   - command:
-      - "python -m pip install tox"
+      - "apt-get update && apt-get install -y python3.5 python3.5-dev python3-pip libxml2-dev libxslt-dev zlib1g-dev"
+      - "python3.5 -m pip install tox"
       - "tox -e py35-old,codecov"
     label: ":python: 3.5 / SQLite / Old Deps"
     env:
       TRIAL_FLAGS: "-j 2"
     plugins:
       - docker#v3.0.1:
-          image: "python:3.5"
+          image: "ubuntu:xenial"  # We use xenail to get an old sqlite and python
           propagate-environment: true
     retry:
       automatic:
@@ -117,8 +118,10 @@ steps:
           limit: 2
 
   - label: ":python: 3.5 / :postgres: 9.5"
+    agents:
+      queue: "medium"
     env:
-      TRIAL_FLAGS: "-j 4"
+      TRIAL_FLAGS: "-j 8"
     command:
       - "bash -c 'python -m pip install tox && python -m tox -e py35-postgres,codecov'"
     plugins:
@@ -134,8 +137,10 @@ steps:
           limit: 2
 
   - label: ":python: 3.7 / :postgres: 9.5"
+    agents:
+      queue: "medium"
     env:
-      TRIAL_FLAGS: "-j 4"
+      TRIAL_FLAGS: "-j 8"
     command:
       - "bash -c 'python -m pip install tox && python -m tox -e py37-postgres,codecov'"
     plugins:
@@ -151,8 +156,10 @@ steps:
           limit: 2
 
   - label: ":python: 3.7 / :postgres: 11"
+    agents:
+      queue: "medium"
     env:
-      TRIAL_FLAGS: "-j 4"
+      TRIAL_FLAGS: "-j 8"
     command:
       - "bash -c 'python -m pip install tox && python -m tox -e py37-postgres,codecov'"
     plugins:
@@ -173,11 +180,13 @@ steps:
       queue: "medium"
     command:
       - "bash .buildkite/merge_base_branch.sh"
-      - "bash .buildkite/synapse_sytest.sh"
+      - "bash /synapse_sytest.sh"
     plugins:
       - docker#v3.0.1:
           image: "matrixdotorg/sytest-synapse:py35"
           propagate-environment: true
+          always-pull: true
+          workdir: "/src"
     retry:
       automatic:
         - exit_status: -1
@@ -192,11 +201,13 @@ steps:
       POSTGRES: "1"
     command:
       - "bash .buildkite/merge_base_branch.sh"
-      - "bash .buildkite/synapse_sytest.sh"
+      - "bash /synapse_sytest.sh"
     plugins:
       - docker#v3.0.1:
           image: "matrixdotorg/sytest-synapse:py35"
           propagate-environment: true
+          always-pull: true
+          workdir: "/src"
     retry:
       automatic:
         - exit_status: -1
@@ -210,14 +221,17 @@ steps:
     env:
       POSTGRES: "1"
       WORKERS: "1"
+      BLACKLIST: "synapse-blacklist-with-workers"
     command:
       - "bash .buildkite/merge_base_branch.sh"
-      - "bash .buildkite/synapse_sytest.sh"
+      - "bash -c 'cat /src/sytest-blacklist /src/.buildkite/worker-blacklist > /src/synapse-blacklist-with-workers'"
+      - "bash /synapse_sytest.sh"
     plugins:
       - docker#v3.0.1:
           image: "matrixdotorg/sytest-synapse:py35"
           propagate-environment: true
-    soft_fail: true
+          always-pull: true
+          workdir: "/src"
     retry:
       automatic:
         - exit_status: -1

.buildkite/synapse_sytest.sh

@@ -1,145 +0,0 @@
-#!/bin/bash
-#
-# Fetch sytest, and then run the tests for synapse. The entrypoint for the
-# sytest-synapse docker images.
-
-set -ex
-
-if [ -n "$BUILDKITE" ]
-then
-    SYNAPSE_DIR=`pwd`
-else
-    SYNAPSE_DIR="/src"
-fi
-
-# Attempt to find a sytest to use.
-# If /sytest exists, it means that a SyTest checkout has been mounted into the Docker image.
-if [ -d "/sytest" ]; then
-    # If the user has mounted in a SyTest checkout, use that.
-    echo "Using local sytests..."
-
-    # create ourselves a working directory and dos2unix some scripts therein
-    mkdir -p /work/jenkins
-    for i in install-deps.pl run-tests.pl tap-to-junit-xml.pl jenkins/prep_sytest_for_postgres.sh; do
-        dos2unix -n "/sytest/$i" "/work/$i"
-    done
-    ln -sf /sytest/tests /work
-    ln -sf /sytest/keys /work
-    SYTEST_LIB="/sytest/lib"
-else
-    if [ -n "BUILDKITE_BRANCH" ]
-    then
-        branch_name=$BUILDKITE_BRANCH
-    else
-        # Otherwise, try and find out what the branch that the Synapse checkout is using. Fall back to develop if it's not a branch.
-        branch_name="$(git --git-dir=/src/.git symbolic-ref HEAD 2>/dev/null)" || branch_name="develop"
-    fi
-
-    # Try and fetch the branch
-    echo "Trying to get same-named sytest branch..."
-    wget -q https://github.com/matrix-org/sytest/archive/$branch_name.tar.gz -O sytest.tar.gz || {
-        # Probably a 404, fall back to develop
-        echo "Using develop instead..."
-        wget -q https://github.com/matrix-org/sytest/archive/develop.tar.gz -O sytest.tar.gz
-    }
-
-    mkdir -p /work
-    tar -C /work --strip-components=1 -xf sytest.tar.gz
-    SYTEST_LIB="/work/lib"
-fi
-
-cd /work
-
-# PostgreSQL setup
-if [ -n "$POSTGRES" ]
-then
-    export PGUSER=postgres
-    export POSTGRES_DB_1=pg1
-    export POSTGRES_DB_2=pg2
-
-    # Start the database
-    su -c 'eatmydata /usr/lib/postgresql/9.6/bin/pg_ctl -w -D /var/lib/postgresql/data start' postgres
-
-    # Use the Jenkins script to write out the configuration for a PostgreSQL using Synapse
-    jenkins/prep_sytest_for_postgres.sh
-
-    # Make the test databases for the two Synapse servers that will be spun up
-    su -c 'psql -c "CREATE DATABASE pg1;"' postgres
-    su -c 'psql -c "CREATE DATABASE pg2;"' postgres
-
-fi
-
-if [ -n "$OFFLINE" ]; then
-    # if we're in offline mode, just put synapse into the virtualenv, and
-    # hope that the deps are up-to-date.
-    #
-    # (`pip install -e` likes to reinstall setuptools even if it's already installed,
-    # so we just run setup.py explicitly.)
-    #
-    (cd $SYNAPSE_DIR && /venv/bin/python setup.py -q develop)
-else
-    # We've already created the virtualenv, but lets double check we have all
-    # deps.
-    /venv/bin/pip install -q --upgrade --no-cache-dir -e $SYNAPSE_DIR
-    /venv/bin/pip install -q --upgrade --no-cache-dir \
-        lxml psycopg2 coverage codecov tap.py
-
-    # Make sure all Perl deps are installed -- this is done in the docker build
-    # so will only install packages added since the last Docker build
-    ./install-deps.pl
-fi
-
-
-# Run the tests
->&2 echo "+++ Running tests"
-
-RUN_TESTS=(
-    perl -I "$SYTEST_LIB" ./run-tests.pl --python=/venv/bin/python --synapse-directory=$SYNAPSE_DIR --coverage -O tap --all
-)
-
-TEST_STATUS=0
-
-if [ -n "$WORKERS" ]; then
-    RUN_TESTS+=(-I Synapse::ViaHaproxy --dendron-binary=/pydron.py)
-else
-    RUN_TESTS+=(-I Synapse)
-fi
-
-"${RUN_TESTS[@]}" "$@" > results.tap || TEST_STATUS=$?
-
-if [ $TEST_STATUS -ne 0 ]; then
-    >&2 echo -e "run-tests \e[31mFAILED\e[0m: exit code $TEST_STATUS"
-else
-    >&2 echo -e "run-tests \e[32mPASSED\e[0m"
-fi
-
->&2 echo "--- Copying assets"
-
-# Copy out the logs
-mkdir -p /logs
-cp results.tap /logs/results.tap
-rsync --ignore-missing-args  --min-size=1B -av server-0 server-1 /logs --include "*/" --include="*.log.*" --include="*.log" --exclude="*"
-
-# Upload coverage to codecov and upload files, if running on Buildkite
-if [ -n "$BUILDKITE" ]
-then
-    /venv/bin/coverage combine || true
-    /venv/bin/coverage xml || true
-    /venv/bin/codecov -X gcov -f coverage.xml
-
-    wget -O buildkite.tar.gz https://github.com/buildkite/agent/releases/download/v3.13.0/buildkite-agent-linux-amd64-3.13.0.tar.gz
-    tar xvf buildkite.tar.gz
-    chmod +x ./buildkite-agent
-
-    # Upload the files
-    ./buildkite-agent artifact upload "/logs/**/*.log*"
-    ./buildkite-agent artifact upload "/logs/results.tap"
-
-    if [ $TEST_STATUS -ne 0 ]; then
-        # Annotate, if failure
-        /venv/bin/python $SYNAPSE_DIR/.buildkite/format_tap.py /logs/results.tap "$BUILDKITE_LABEL" | ./buildkite-agent annotate --style="error" --context="$BUILDKITE_LABEL"
-    fi
-fi
-
-
-exit $TEST_STATUS

.buildkite/worker-blacklist

@@ -0,0 +1,34 @@
+# This file serves as a blacklist for SyTest tests that we expect will fail in
+# Synapse when run under worker mode. For more details, see sytest-blacklist.
+
+Message history can be paginated
+
+m.room.history_visibility == "world_readable" allows/forbids appropriately for Guest users
+
+m.room.history_visibility == "world_readable" allows/forbids appropriately for Real users
+
+Can re-join room if re-invited
+
+/upgrade creates a new room
+
+The only membership state included in an initial sync is for all the senders in the timeline
+
+Local device key changes get to remote servers
+
+If remote user leaves room we no longer receive device updates
+
+Forgotten room messages cannot be paginated
+
+Inbound federation can get public room list
+
+Members from the gap are included in gappy incr LL sync
+
+Leaves are present in non-gapped incremental syncs
+
+Old leaves are present in gapped incremental syncs
+
+User sees updates to presence from other users in the incremental sync.
+
+Gapped incremental syncs include all state changes
+
+Old members are included in gappy incr LL sync if they start speaking

.circleci/config.yml

@@ -4,22 +4,15 @@ jobs:
     machine: true
     steps:
       - checkout
-      - run: docker build -f docker/Dockerfile --label gitsha1=${CIRCLE_SHA1} -t matrixdotorg/synapse:${CIRCLE_TAG} -t matrixdotorg/synapse:${CIRCLE_TAG}-py3 --build-arg PYTHON_VERSION=3.6 .
+      - run: docker build -f docker/Dockerfile --label gitsha1=${CIRCLE_SHA1} -t matrixdotorg/synapse:${CIRCLE_TAG} -t matrixdotorg/synapse:${CIRCLE_TAG}-py3 .
       - run: docker login --username $DOCKER_HUB_USERNAME --password $DOCKER_HUB_PASSWORD
       - run: docker push matrixdotorg/synapse:${CIRCLE_TAG}
       - run: docker push matrixdotorg/synapse:${CIRCLE_TAG}-py3
-  dockerhubuploadreleaseshhs:
-    machine: true
-    steps:
-      - checkout
-      - run: docker build -f docker/Dockerfile --label gitsha1=${CIRCLE_SHA1} -t matrixdotorg/synapse:${CIRCLE_TAG} --build-arg PYTHON_VERSION=3.7 .
-      - run: docker login --username $DOCKER_HUB_USERNAME --password $DOCKER_HUB_PASSWORD
-      - run: docker push matrixdotorg/synapse:${CIRCLE_TAG}
   dockerhubuploadlatest:
     machine: true
     steps:
       - checkout
-      - run: docker build -f docker/Dockerfile --label gitsha1=${CIRCLE_SHA1} -t matrixdotorg/synapse:latest -t matrixdotorg/synapse:latest-py3 --build-arg PYTHON_VERSION=3.6 .
+      - run: docker build -f docker/Dockerfile --label gitsha1=${CIRCLE_SHA1} -t matrixdotorg/synapse:latest -t matrixdotorg/synapse:latest-py3 .
       - run: docker login --username $DOCKER_HUB_USERNAME --password $DOCKER_HUB_PASSWORD
       - run: docker push matrixdotorg/synapse:latest
       - run: docker push matrixdotorg/synapse:latest-py3
@@ -34,12 +27,6 @@ workflows:
               only: /v[0-9].[0-9]+.[0-9]+.*/
             branches:
               ignore: /.*/
-      - dockerhubuploadreleaseshhs:
-          filters:
-            tags:
-              only: /shhs-v[0-9]+.[0-9]+.*/
-            branches:
-              ignore: /.*/
       - dockerhubuploadlatest:
           filters:
             branches:

.codecov.yml

@@ -1,5 +1,4 @@
-comment:
-  layout: "diff"
+comment: off
 
 coverage:
   status:

.dockerignore

@@ -1,9 +1,13 @@
-Dockerfile
-.travis.yml
-.gitignore
-demo/etc
-tox.ini
-.git/*
-.tox/*
-debian/matrix-synapse/
-debian/matrix-synapse-*/
+# ignore everything by default
+*
+
+# things to include
+!docker
+!scripts
+!synapse
+!MANIFEST.in
+!README.rst
+!setup.py
+!synctl
+
+**/__pycache__

.github/ISSUE_TEMPLATE/SUPPORT_REQUEST.md

@@ -4,6 +4,7 @@ about: I need support for Synapse
 
 ---
 
-# Please ask for support in [**#matrix:matrix.org**](https://matrix.to/#/#matrix:matrix.org)
+Please don't file github issues asking for support.
 
-## Don't file an issue as a support request.
+Instead, please join [`#synapse:matrix.org`](https://matrix.to/#/#synapse:matrix.org)
+(from a matrix.org account if necessary), and ask there.

.github/SUPPORT.md

@@ -1,3 +1,3 @@
-[**#matrix:matrix.org**](https://matrix.to/#/#matrix:matrix.org) is the official support room for Matrix, and can be accessed by any client from https://matrix.org/docs/projects/try-matrix-now.html 
-
-It can also be access via IRC bridge at irc://irc.freenode.net/matrix or on the web here: https://webchat.freenode.net/?channels=matrix
+[**#synapse:matrix.org**](https://matrix.to/#/#synapse:matrix.org) is the official support room for
+Synapse, and can be accessed by any client from https://matrix.org/docs/projects/try-matrix-now.html.
+Please ask for support there, rather than filing github issues.

.gitignore

@@ -19,7 +19,6 @@ _trial_temp*/
 /*.signing.key
 /env/
 /homeserver*.yaml
-/logs
 /media_store/
 /uploads
 
@@ -38,3 +37,4 @@ _trial_temp*/
 /docs/build/
 /htmlcov
 /pip-wheel-metadata/
+

AUTHORS.rst

@@ -72,3 +72,6 @@ Jason Robinson <jasonr at matrix.org>
 
 Joseph Weston <joseph at weston.cloud>
  + Add admin API for querying HS version
+
+Benjamin Saunders <ben.e.saunders at gmail dot com>
+ * Documentation improvements

CHANGES.md

@@ -1,3 +1,259 @@
+Synapse 1.2.1 (2019-07-26)
+==========================
+
+Security update
+---------------
+
+This release includes *four* security fixes:
+
+- Prevent an attack where a federated server could send redactions for arbitrary events in v1 and v2 rooms. ([\#5767](https://github.com/matrix-org/synapse/issues/5767))
+- Prevent a denial-of-service attack where cycles of redaction events would make Synapse spin infinitely. Thanks to `@lrizika:matrix.org` for identifying and responsibly disclosing this issue. ([0f2ecb961](https://github.com/matrix-org/synapse/commit/0f2ecb961))
+- Prevent an attack where users could be joined or parted from public rooms without their consent. Thanks to @dylangerdaly for identifying and responsibly disclosing this issue. ([\#5744](https://github.com/matrix-org/synapse/issues/5744))
+- Fix a vulnerability where a federated server could spoof read-receipts from
+  users on other servers. Thanks to @dylangerdaly for identifying this issue too. ([\#5743](https://github.com/matrix-org/synapse/issues/5743))
+
+Additionally, the following fix was in Synapse **1.2.0**, but was not correctly
+identified during the original release:
+
+- It was possible for a room moderator to send a redaction for an `m.room.create` event, which would downgrade the room to version 1. Thanks to `/dev/ponies` for identifying and responsibly disclosing this issue! ([\#5701](https://github.com/matrix-org/synapse/issues/5701))
+
+Synapse 1.2.0 (2019-07-25)
+==========================
+
+No significant changes.
+
+
+Synapse 1.2.0rc2 (2019-07-24)
+=============================
+
+Bugfixes
+--------
+
+- Fix a regression introduced in v1.2.0rc1 which led to incorrect labels on some prometheus metrics. ([\#5734](https://github.com/matrix-org/synapse/issues/5734))
+
+
+Synapse 1.2.0rc1 (2019-07-22)
+=============================
+
+Security fixes
+--------------
+
+This update included a security fix which was initially incorrectly flagged as
+a regular bug fix.
+
+- It was possible for a room moderator to send a redaction for an `m.room.create` event, which would downgrade the room to version 1. Thanks to `/dev/ponies` for identifying and responsibly disclosing this issue! ([\#5701](https://github.com/matrix-org/synapse/issues/5701))
+
+Features
+--------
+
+- Add support for opentracing. ([\#5544](https://github.com/matrix-org/synapse/issues/5544), [\#5712](https://github.com/matrix-org/synapse/issues/5712))
+- Add ability to pull all locally stored events out of synapse that a particular user can see. ([\#5589](https://github.com/matrix-org/synapse/issues/5589))
+- Add a basic admin command app to allow server operators to run Synapse admin commands separately from the main production instance. ([\#5597](https://github.com/matrix-org/synapse/issues/5597))
+- Add `sender` and `origin_server_ts` fields to `m.replace`. ([\#5613](https://github.com/matrix-org/synapse/issues/5613))
+- Add default push rule to ignore reactions. ([\#5623](https://github.com/matrix-org/synapse/issues/5623))
+- Include the original event when asking for its relations. ([\#5626](https://github.com/matrix-org/synapse/issues/5626))
+- Implement `session_lifetime` configuration option, after which access tokens will expire. ([\#5660](https://github.com/matrix-org/synapse/issues/5660))
+- Return "This account has been deactivated" when a deactivated user tries to login. ([\#5674](https://github.com/matrix-org/synapse/issues/5674))
+- Enable aggregations support by default ([\#5714](https://github.com/matrix-org/synapse/issues/5714))
+
+
+Bugfixes
+--------
+
+- Fix 'utime went backwards' errors on daemonization. ([\#5609](https://github.com/matrix-org/synapse/issues/5609))
+- Various minor fixes to the federation request rate limiter. ([\#5621](https://github.com/matrix-org/synapse/issues/5621))
+- Forbid viewing relations on an event once it has been redacted. ([\#5629](https://github.com/matrix-org/synapse/issues/5629))
+- Fix requests to the `/store_invite` endpoint of identity servers being sent in the wrong format. ([\#5638](https://github.com/matrix-org/synapse/issues/5638))
+- Fix newly-registered users not being able to lookup their own profile without joining a room. ([\#5644](https://github.com/matrix-org/synapse/issues/5644))
+- Fix bug in #5626 that prevented the original_event field from actually having the contents of the original event in a call to `/relations`. ([\#5654](https://github.com/matrix-org/synapse/issues/5654))
+- Fix 3PID bind requests being sent to identity servers as `application/x-form-www-urlencoded` data, which is deprecated. ([\#5658](https://github.com/matrix-org/synapse/issues/5658))
+- Fix some problems with authenticating redactions in recent room versions. ([\#5699](https://github.com/matrix-org/synapse/issues/5699), [\#5700](https://github.com/matrix-org/synapse/issues/5700), [\#5707](https://github.com/matrix-org/synapse/issues/5707))
+
+
+Updates to the Docker image
+---------------------------
+
+- Base Docker image on a newer Alpine Linux version (3.8 -> 3.10). ([\#5619](https://github.com/matrix-org/synapse/issues/5619))
+- Add missing space in default logging file format generated by the Docker image. ([\#5620](https://github.com/matrix-org/synapse/issues/5620))
+
+
+Improved Documentation
+----------------------
+
+- Add information about nginx normalisation to reverse_proxy.rst. Contributed by @skalarproduktraum - thanks! ([\#5397](https://github.com/matrix-org/synapse/issues/5397))
+- --no-pep517 should be --no-use-pep517 in the documentation to setup the development environment. ([\#5651](https://github.com/matrix-org/synapse/issues/5651))
+- Improvements to Postgres setup instructions. Contributed by @Lrizika - thanks! ([\#5661](https://github.com/matrix-org/synapse/issues/5661))
+- Minor tweaks to postgres documentation. ([\#5675](https://github.com/matrix-org/synapse/issues/5675))
+
+
+Deprecations and Removals
+-------------------------
+
+- Remove support for the `invite_3pid_guest` configuration setting. ([\#5625](https://github.com/matrix-org/synapse/issues/5625))
+
+
+Internal Changes
+----------------
+
+- Move logging code out of `synapse.util` and into `synapse.logging`. ([\#5606](https://github.com/matrix-org/synapse/issues/5606), [\#5617](https://github.com/matrix-org/synapse/issues/5617))
+- Add a blacklist file to the repo to blacklist certain sytests from failing CI. ([\#5611](https://github.com/matrix-org/synapse/issues/5611))
+- Make runtime errors surrounding password reset emails much clearer. ([\#5616](https://github.com/matrix-org/synapse/issues/5616))
+- Remove dead code for persiting outgoing federation transactions. ([\#5622](https://github.com/matrix-org/synapse/issues/5622))
+- Add `lint.sh` to the scripts-dev folder which will run all linting steps required by CI. ([\#5627](https://github.com/matrix-org/synapse/issues/5627))
+- Move RegistrationHandler.get_or_create_user to test code. ([\#5628](https://github.com/matrix-org/synapse/issues/5628))
+- Add some more common python virtual-environment paths to the black exclusion list. ([\#5630](https://github.com/matrix-org/synapse/issues/5630))
+- Some counter metrics exposed over Prometheus have been renamed, with the old names preserved for backwards compatibility and deprecated. See `docs/metrics-howto.rst` for details. ([\#5636](https://github.com/matrix-org/synapse/issues/5636))
+- Unblacklist some user_directory sytests. ([\#5637](https://github.com/matrix-org/synapse/issues/5637))
+- Factor out some redundant code in the login implementation. ([\#5639](https://github.com/matrix-org/synapse/issues/5639))
+- Update ModuleApi to avoid register(generate_token=True). ([\#5640](https://github.com/matrix-org/synapse/issues/5640))
+- Remove access-token support from `RegistrationHandler.register`, and rename it. ([\#5641](https://github.com/matrix-org/synapse/issues/5641))
+- Remove access-token support from `RegistrationStore.register`, and rename it. ([\#5642](https://github.com/matrix-org/synapse/issues/5642))
+- Improve logging for auto-join when a new user is created. ([\#5643](https://github.com/matrix-org/synapse/issues/5643))
+- Remove unused and unnecessary check for FederationDeniedError in _exception_to_failure. ([\#5645](https://github.com/matrix-org/synapse/issues/5645))
+- Fix a small typo in a code comment. ([\#5655](https://github.com/matrix-org/synapse/issues/5655))
+- Clean up exception handling around client access tokens. ([\#5656](https://github.com/matrix-org/synapse/issues/5656))
+- Add a mechanism for per-test homeserver configuration in the unit tests. ([\#5657](https://github.com/matrix-org/synapse/issues/5657))
+- Inline issue_access_token. ([\#5659](https://github.com/matrix-org/synapse/issues/5659))
+- Update the sytest BuildKite configuration to checkout Synapse in `/src`. ([\#5664](https://github.com/matrix-org/synapse/issues/5664))
+- Add a `docker` type to the towncrier configuration. ([\#5673](https://github.com/matrix-org/synapse/issues/5673))
+- Convert `synapse.federation.transport.server` to `async`. Might improve some stack traces. ([\#5689](https://github.com/matrix-org/synapse/issues/5689))
+- Documentation for opentracing. ([\#5703](https://github.com/matrix-org/synapse/issues/5703))
+
+
+Synapse 1.1.0 (2019-07-04)
+==========================
+
+As of v1.1.0, Synapse no longer supports Python 2, nor Postgres version 9.4.
+See the [upgrade notes](UPGRADE.rst#upgrading-to-v110) for more details.
+
+This release also deprecates the use of environment variables to configure the
+docker image. See the [docker README](https://github.com/matrix-org/synapse/blob/release-v1.1.0/docker/README.md#legacy-dynamic-configuration-file-support)
+for more details.
+
+No changes since 1.1.0rc2.
+
+
+Synapse 1.1.0rc2 (2019-07-03)
+=============================
+
+Bugfixes
+--------
+
+- Fix regression in 1.1rc1 where OPTIONS requests to the media repo would fail. ([\#5593](https://github.com/matrix-org/synapse/issues/5593))
+- Removed the `SYNAPSE_SMTP_*` docker container environment variables. Using these environment variables prevented the docker container from starting in Synapse v1.0, even though they didn't actually allow any functionality anyway. ([\#5596](https://github.com/matrix-org/synapse/issues/5596))
+- Fix a number of "Starting txn from sentinel context" warnings. ([\#5605](https://github.com/matrix-org/synapse/issues/5605))
+
+
+Internal Changes
+----------------
+
+- Update github templates. ([\#5552](https://github.com/matrix-org/synapse/issues/5552))
+
+
+Synapse 1.1.0rc1 (2019-07-02)
+=============================
+
+As of v1.1.0, Synapse no longer supports Python 2, nor Postgres version 9.4.
+See the [upgrade notes](UPGRADE.rst#upgrading-to-v110) for more details.
+
+Features
+--------
+
+- Added possibilty to disable local password authentication. Contributed by Daniel Hoffend. ([\#5092](https://github.com/matrix-org/synapse/issues/5092))
+- Add monthly active users to phonehome stats. ([\#5252](https://github.com/matrix-org/synapse/issues/5252))
+- Allow expired user to trigger renewal email sending manually. ([\#5363](https://github.com/matrix-org/synapse/issues/5363))
+- Statistics on forward extremities per room are now exposed via Prometheus. ([\#5384](https://github.com/matrix-org/synapse/issues/5384), [\#5458](https://github.com/matrix-org/synapse/issues/5458), [\#5461](https://github.com/matrix-org/synapse/issues/5461))
+- Add --no-daemonize option to run synapse in the foreground, per issue #4130. Contributed by Soham Gumaste. ([\#5412](https://github.com/matrix-org/synapse/issues/5412), [\#5587](https://github.com/matrix-org/synapse/issues/5587))
+- Fully support SAML2 authentication. Contributed by [Alexander Trost](https://github.com/galexrt) - thank you! ([\#5422](https://github.com/matrix-org/synapse/issues/5422))
+- Allow server admins to define implementations of extra rules for allowing or denying incoming events. ([\#5440](https://github.com/matrix-org/synapse/issues/5440), [\#5474](https://github.com/matrix-org/synapse/issues/5474), [\#5477](https://github.com/matrix-org/synapse/issues/5477))
+- Add support for handling pagination APIs on client reader worker. ([\#5505](https://github.com/matrix-org/synapse/issues/5505), [\#5513](https://github.com/matrix-org/synapse/issues/5513), [\#5531](https://github.com/matrix-org/synapse/issues/5531))
+- Improve help and cmdline option names for --generate-config options. ([\#5512](https://github.com/matrix-org/synapse/issues/5512))
+- Allow configuration of the path used for ACME account keys. ([\#5516](https://github.com/matrix-org/synapse/issues/5516), [\#5521](https://github.com/matrix-org/synapse/issues/5521), [\#5522](https://github.com/matrix-org/synapse/issues/5522))
+- Add --data-dir and --open-private-ports options. ([\#5524](https://github.com/matrix-org/synapse/issues/5524))
+- Split public rooms directory auth config in two settings, in order to manage client auth independently from the federation part of it. Obsoletes the "restrict_public_rooms_to_local_users" configuration setting. If "restrict_public_rooms_to_local_users" is set in the config, Synapse will act as if both new options are enabled, i.e. require authentication through the client API and deny federation requests. ([\#5534](https://github.com/matrix-org/synapse/issues/5534))
+- The minimum TLS version used for outgoing federation requests can now be set with `federation_client_minimum_tls_version`. ([\#5550](https://github.com/matrix-org/synapse/issues/5550))
+- Optimise devices changed query to not pull unnecessary rows from the database, reducing database load. ([\#5559](https://github.com/matrix-org/synapse/issues/5559))
+- Add new metrics for number of forward extremities being persisted and number of state groups involved in resolution. ([\#5476](https://github.com/matrix-org/synapse/issues/5476))
+
+Bugfixes
+--------
+
+- Fix bug processing incoming events over federation if call to `/get_missing_events` fails. ([\#5042](https://github.com/matrix-org/synapse/issues/5042))
+- Prevent more than one room upgrade happening simultaneously on the same room. ([\#5051](https://github.com/matrix-org/synapse/issues/5051))
+- Fix a bug where running synapse_port_db would cause the account validity feature to fail because it didn't set the type of the email_sent column to boolean. ([\#5325](https://github.com/matrix-org/synapse/issues/5325))
+- Warn about disabling email-based password resets when a reset occurs, and remove warning when someone attempts a phone-based reset. ([\#5387](https://github.com/matrix-org/synapse/issues/5387))
+- Fix email notifications for unnamed rooms with multiple people. ([\#5388](https://github.com/matrix-org/synapse/issues/5388))
+- Fix exceptions in federation reader worker caused by attempting to renew attestations, which should only happen on master worker. ([\#5389](https://github.com/matrix-org/synapse/issues/5389))
+- Fix handling of failures fetching remote content to not log failures as exceptions. ([\#5390](https://github.com/matrix-org/synapse/issues/5390))
+- Fix a bug where deactivated users could receive renewal emails if the account validity feature is on. ([\#5394](https://github.com/matrix-org/synapse/issues/5394))
+- Fix missing invite state after exchanging 3PID invites over federaton. ([\#5464](https://github.com/matrix-org/synapse/issues/5464))
+- Fix intermittent exceptions on Apple hardware. Also fix bug that caused database activity times to be under-reported in log lines. ([\#5498](https://github.com/matrix-org/synapse/issues/5498))
+- Fix logging error when a tampered event is detected. ([\#5500](https://github.com/matrix-org/synapse/issues/5500))
+- Fix bug where clients could tight loop calling `/sync` for a period. ([\#5507](https://github.com/matrix-org/synapse/issues/5507))
+- Fix bug with `jinja2` preventing Synapse from starting. Users who had this problem should now simply need to run `pip install matrix-synapse`. ([\#5514](https://github.com/matrix-org/synapse/issues/5514))
+- Fix a regression where homeservers on private IP addresses were incorrectly blacklisted. ([\#5523](https://github.com/matrix-org/synapse/issues/5523))
+- Fixed m.login.jwt using unregistred user_id and added pyjwt>=1.6.4 as jwt conditional dependencies. Contributed by Pau Rodriguez-Estivill. ([\#5555](https://github.com/matrix-org/synapse/issues/5555), [\#5586](https://github.com/matrix-org/synapse/issues/5586))
+- Fix a bug that would cause invited users to receive several emails for a single 3PID invite in case the inviter is rate limited. ([\#5576](https://github.com/matrix-org/synapse/issues/5576))
+
+
+Updates to the Docker image
+---------------------------
+- Add ability to change Docker containers [timezone](https://en.wikipedia.org/wiki/List_of_tz_database_time_zones) with the `TZ` variable. ([\#5383](https://github.com/matrix-org/synapse/issues/5383))
+- Update docker image to use Python 3.7. ([\#5546](https://github.com/matrix-org/synapse/issues/5546))
+- Deprecate the use of environment variables for configuration, and make the use of a static configuration the default. ([\#5561](https://github.com/matrix-org/synapse/issues/5561), [\#5562](https://github.com/matrix-org/synapse/issues/5562), [\#5566](https://github.com/matrix-org/synapse/issues/5566), [\#5567](https://github.com/matrix-org/synapse/issues/5567))
+- Increase default log level for docker image to INFO. It can still be changed by editing the generated log.config file. ([\#5547](https://github.com/matrix-org/synapse/issues/5547))
+- Send synapse logs to the docker logging system, by default. ([\#5565](https://github.com/matrix-org/synapse/issues/5565))
+- Open the non-TLS port by default. ([\#5568](https://github.com/matrix-org/synapse/issues/5568))
+- Fix failure to start under docker with SAML support enabled. ([\#5490](https://github.com/matrix-org/synapse/issues/5490))
+- Use a sensible location for data files when generating a config file. ([\#5563](https://github.com/matrix-org/synapse/issues/5563))
+
+
+Deprecations and Removals
+-------------------------
+
+- Python 2.7 is no longer a supported platform. Synapse now requires Python 3.5+ to run. ([\#5425](https://github.com/matrix-org/synapse/issues/5425))
+- PostgreSQL 9.4 is no longer supported. Synapse requires Postgres 9.5+ or above for Postgres support. ([\#5448](https://github.com/matrix-org/synapse/issues/5448))
+- Remove support for cpu_affinity setting. ([\#5525](https://github.com/matrix-org/synapse/issues/5525))
+
+
+Improved Documentation
+----------------------
+- Improve README section on performance troubleshooting. ([\#4276](https://github.com/matrix-org/synapse/issues/4276))
+- Add information about how to install and run `black` on the codebase to code_style.rst. ([\#5537](https://github.com/matrix-org/synapse/issues/5537))
+- Improve install docs on choosing server_name. ([\#5558](https://github.com/matrix-org/synapse/issues/5558))
+
+
+Internal Changes
+----------------
+
+- Add logging to 3pid invite signature verification. ([\#5015](https://github.com/matrix-org/synapse/issues/5015))
+- Update example haproxy config to a more compatible setup. ([\#5313](https://github.com/matrix-org/synapse/issues/5313))
+- Track deactivated accounts in the database. ([\#5378](https://github.com/matrix-org/synapse/issues/5378), [\#5465](https://github.com/matrix-org/synapse/issues/5465), [\#5493](https://github.com/matrix-org/synapse/issues/5493))
+- Clean up code for sending federation EDUs. ([\#5381](https://github.com/matrix-org/synapse/issues/5381))
+- Add a sponsor button to the repo. ([\#5382](https://github.com/matrix-org/synapse/issues/5382), [\#5386](https://github.com/matrix-org/synapse/issues/5386))
+- Don't log non-200 responses from federation queries as exceptions. ([\#5383](https://github.com/matrix-org/synapse/issues/5383))
+- Update Python syntax in contrib/ to Python 3. ([\#5446](https://github.com/matrix-org/synapse/issues/5446))
+- Update federation_client dev script to support `.well-known` and work with python3. ([\#5447](https://github.com/matrix-org/synapse/issues/5447))
+- SyTest has been moved to Buildkite. ([\#5459](https://github.com/matrix-org/synapse/issues/5459))
+- Demo script now uses python3. ([\#5460](https://github.com/matrix-org/synapse/issues/5460))
+- Synapse can now handle RestServlets that return coroutines. ([\#5475](https://github.com/matrix-org/synapse/issues/5475), [\#5585](https://github.com/matrix-org/synapse/issues/5585))
+- The demo servers talk to each other again. ([\#5478](https://github.com/matrix-org/synapse/issues/5478))
+- Add an EXPERIMENTAL config option to try and periodically clean up extremities by sending dummy events. ([\#5480](https://github.com/matrix-org/synapse/issues/5480))
+- Synapse's codebase is now formatted by `black`. ([\#5482](https://github.com/matrix-org/synapse/issues/5482))
+- Some cleanups and sanity-checking in the CPU and database metrics. ([\#5499](https://github.com/matrix-org/synapse/issues/5499))
+- Improve email notification logging. ([\#5502](https://github.com/matrix-org/synapse/issues/5502))
+- Fix "Unexpected entry in 'full_schemas'" log warning. ([\#5509](https://github.com/matrix-org/synapse/issues/5509))
+- Improve logging when generating config files. ([\#5510](https://github.com/matrix-org/synapse/issues/5510))
+- Refactor and clean up Config parser for maintainability. ([\#5511](https://github.com/matrix-org/synapse/issues/5511))
+- Make the config clearer in that email.template_dir is relative to the Synapse's root directory, not the `synapse/` folder within it. ([\#5543](https://github.com/matrix-org/synapse/issues/5543))
+- Update v1.0.0 release changelog to include more information about changes to password resets. ([\#5545](https://github.com/matrix-org/synapse/issues/5545))
+- Remove non-functioning check_event_hash.py dev script. ([\#5548](https://github.com/matrix-org/synapse/issues/5548))
+- Synapse will now only allow TLS v1.2 connections when serving federation, if it terminates TLS. As Synapse's allowed ciphers were only able to be used in TLSv1.2 before, this does not change behaviour. ([\#5550](https://github.com/matrix-org/synapse/issues/5550))
+- Logging when running GC collection on generation 0 is now at the DEBUG level, not INFO. ([\#5557](https://github.com/matrix-org/synapse/issues/5557))
+- Reduce the amount of stuff we send in the docker context. ([\#5564](https://github.com/matrix-org/synapse/issues/5564))
+- Point the reverse links in the Purge History contrib scripts at the intended location. ([\#5570](https://github.com/matrix-org/synapse/issues/5570))
+
+
 Synapse 1.0.0 (2019-06-11)
 ==========================
 
@@ -52,7 +308,11 @@ Features
 - Add a script to generate new signing-key files. ([\#5361](https://github.com/matrix-org/synapse/issues/5361))
 - Update upgrade and installation guides ahead of 1.0. ([\#5371](https://github.com/matrix-org/synapse/issues/5371))
 - Replace the `perspectives` configuration section with `trusted_key_servers`, and make validating the signatures on responses optional (since TLS will do this job for us). ([\#5374](https://github.com/matrix-org/synapse/issues/5374))
-- Add ability to perform password reset via email without trusting the identity server. ([\#5377](https://github.com/matrix-org/synapse/issues/5377))
+- Add ability to perform password reset via email without trusting the identity server. **As a result of this PR, password resets will now be disabled on the default configuration.**
+
+  Password reset emails are now sent from the homeserver by default, instead of the identity server. To enable this functionality, ensure `email` and `public_baseurl` config options are filled out.
+
+  If you would like to re-enable password resets being sent from the identity server (warning: this is dangerous! See [#5345](https://github.com/matrix-org/synapse/pull/5345)), set `email.trust_identity_server_for_password_resets` to true. ([\#5377](https://github.com/matrix-org/synapse/issues/5377))
 - Set default room version to v4. ([\#5379](https://github.com/matrix-org/synapse/issues/5379))
 
 

CONTRIBUTING.rst

@@ -30,11 +30,10 @@ use github's pull request workflow to review the contribution, and either ask
 you to make any refinements needed or merge it and make them ourselves. The
 changes will then land on master when we next do a release.
 
-We use `CircleCI <https://circleci.com/gh/matrix-org>`_ and `Buildkite
-<https://buildkite.com/matrix-dot-org/synapse>`_ for continuous integration.
-Buildkite builds need to be authorised by a maintainer. If your change breaks
-the build, this will be shown in GitHub, so please keep an eye on the pull
-request for feedback.
+We use `Buildkite <https://buildkite.com/matrix-dot-org/synapse>`_ for
+continuous integration.  Buildkite builds need to be authorised by a
+maintainer. If your change breaks the build, this will be shown in GitHub, so
+please keep an eye on the pull request for feedback.
 
 To run unit tests in a local development environment, you can use:
 
@@ -70,13 +69,21 @@ All changes, even minor ones, need a corresponding changelog / newsfragment
 entry. These are managed by Towncrier
 (https://github.com/hawkowl/towncrier).
 
-To create a changelog entry, make a new file in the ``changelog.d``
-file named in the format of ``PRnumber.type``. The type can be
-one of ``feature``, ``bugfix``, ``removal`` (also used for
-deprecations), or ``misc`` (for internal-only changes).
+To create a changelog entry, make a new file in the ``changelog.d`` file named
+in the format of ``PRnumber.type``. The type can be one of the following:
 
-The content of the file is your changelog entry, which can contain Markdown
-formatting. The entry should end with a full stop ('.') for consistency.
+* ``feature``.
+* ``bugfix``.
+* ``docker`` (for updates to the Docker image).
+* ``doc`` (for updates to the documentation).
+* ``removal`` (also used for deprecations).
+* ``misc`` (for internal-only changes).
+
+The content of the file is your changelog entry, which should be a short
+description of your change in the same style as the rest of our `changelog
+<https://github.com/matrix-org/synapse/blob/master/CHANGES.md>`_. The file can
+contain Markdown formatting, and should end with a full stop ('.') for
+consistency.
 
 Adding credits to the changelog is encouraged, we value your
 contributions and would like to have you shouted out in the release notes!

INSTALL.md

@@ -1,3 +1,4 @@
+- [Choosing your server name](#choosing-your-server-name)
 - [Installing Synapse](#installing-synapse)
   - [Installing from source](#installing-from-source)
     - [Platform-Specific Instructions](#platform-specific-instructions)
@@ -10,6 +11,22 @@
   - [Setting up a TURN server](#setting-up-a-turn-server)
   - [URL previews](#url-previews)
 
+# Choosing your server name
+
+It is important to choose the name for your server before you install Synapse,
+because it cannot be changed later.
+
+The server name determines the "domain" part of user-ids for users on your
+server: these will all be of the format `@user:my.domain.name`. It also
+determines how other matrix servers will reach yours for federation.
+
+For a test configuration, set this to the hostname of your server. For a more
+production-ready setup, you will probably want to specify your domain
+(`example.com`) rather than a matrix-specific hostname here (in the same way
+that your email address is probably `user@example.com` rather than
+`user@email.example.com`) - but doing so may require more advanced setup: see
+[Setting up Federation](docs/federate.md).
+
 # Installing Synapse
 
 ## Installing from source
@@ -64,16 +81,7 @@ python -m synapse.app.homeserver \
     --report-stats=[yes|no]
 ```
 
-... substituting an appropriate value for `--server-name`. The server name
-determines the "domain" part of user-ids for users on your server: these will
-all be of the format `@user:my.domain.name`. It also determines how other
-matrix servers will reach yours for Federation. For a test configuration,
-set this to the hostname of your server. For a more production-ready setup, you
-will probably want to specify your domain (`example.com`) rather than a
-matrix-specific hostname here (in the same way that your email address is
-probably `user@example.com` rather than `user@email.example.com`) - but
-doing so may require more advanced setup: see [Setting up Federation](docs/federate.md).
-Beware that the server name cannot be changed later.
+... substituting an appropriate value for `--server-name`.
 
 This command will generate you a config file that you can then customise, but it will
 also generate a set of keys for you. These keys will allow your Home Server to
@@ -86,9 +94,6 @@ different. See the
 [spec](https://matrix.org/docs/spec/server_server/latest.html#retrieving-server-keys)
 for more information on key management.)
 
-You will need to give Synapse a TLS certficate before it will start - see [TLS
-certificates](#tls-certificates).
-
 To actually run your new homeserver, pick a working directory for Synapse to
 run (e.g. `~/synapse`), and::
 

MANIFEST.in

@@ -33,6 +33,7 @@ exclude Dockerfile
 exclude .dockerignore
 exclude test_postgresql.sh
 exclude .editorconfig
+exclude sytest-blacklist
 
 include pyproject.toml
 recursive-include changelog.d *

README.rst

@@ -272,7 +272,7 @@ to install using pip and a virtualenv::
 
     virtualenv -p python3 env
     source env/bin/activate
-    python -m pip install --no-pep-517 -e .[all]
+    python -m pip install --no-use-pep517 -e .[all]
 
 This will run a process of downloading and installing all the needed
 dependencies into a virtual env.
@@ -340,8 +340,11 @@ log lines and looking for any 'Processed request' lines which take more than
 a few seconds to execute. Please let us know at #synapse:matrix.org if
 you see this failure mode so we can help debug it, however.
 
-Help!! Synapse eats all my RAM!
--------------------------------
+Help!! Synapse is slow and eats all my RAM/CPU!
+-----------------------------------------------
+
+First, ensure you are running the latest version of Synapse, using Python 3
+with a PostgreSQL database.
 
 Synapse's architecture is quite RAM hungry currently - we deliberately
 cache a lot of recent room data and metadata in RAM in order to speed up
@@ -352,14 +355,29 @@ variable. The default is 0.5, which can be decreased to reduce RAM usage
 in memory constrained enviroments, or increased if performance starts to
 degrade.
 
+However, degraded performance due to a low cache factor, common on
+machines with slow disks, often leads to explosions in memory use due
+backlogged requests. In this case, reducing the cache factor will make
+things worse. Instead, try increasing it drastically. 2.0 is a good
+starting value.
+
 Using `libjemalloc <http://jemalloc.net/>`_ can also yield a significant
-improvement in overall amount, and especially in terms of giving back RAM
-to the OS. To use it, the library must simply be put in the LD_PRELOAD
-environment variable when launching Synapse. On Debian, this can be done
-by installing the ``libjemalloc1`` package and adding this line to
-``/etc/default/matrix-synapse``::
+improvement in overall memory use, and especially in terms of giving back
+RAM to the OS. To use it, the library must simply be put in the
+LD_PRELOAD environment variable when launching Synapse. On Debian, this
+can be done by installing the ``libjemalloc1`` package and adding this
+line to ``/etc/default/matrix-synapse``::
 
     LD_PRELOAD=/usr/lib/x86_64-linux-gnu/libjemalloc.so.1
 
 This can make a significant difference on Python 2.7 - it's unclear how
 much of an improvement it provides on Python 3.x.
+
+If you're encountering high CPU use by the Synapse process itself, you
+may be affected by a bug with presence tracking that leads to a
+massive excess of outgoing federation requests (see `discussion
+<https://github.com/matrix-org/synapse/issues/3971>`_). If metrics
+indicate that your server is also issuing far more outgoing federation
+requests than can be accounted for by your users' activity, this is a
+likely cause. The misbehavior can be worked around by setting
+``use_presence: false`` in the Synapse config file.

UPGRADE.rst

@@ -49,16 +49,23 @@ returned by the Client-Server API:
     # configured on port 443.
     curl -kv https://<host.name>/_matrix/client/versions 2>&1 | grep "Server:"
 
-Upgrading to v1.1
-=================
+Upgrading to v1.2.0
+===================
 
-Synapse 1.1 removes support for older Python and PostgreSQL versions, as
+Some counter metrics have been renamed, with the old names deprecated. See
+`the metrics documentation <docs/metrics-howto.rst#renaming-of-metrics--deprecation-of-old-names-in-12>`_
+for details.
+
+Upgrading to v1.1.0
+===================
+
+Synapse v1.1.0 removes support for older Python and PostgreSQL versions, as
 outlined in `our deprecation notice <https://matrix.org/blog/2019/04/08/synapse-deprecating-postgres-9-4-and-python-2-x>`_.
 
 Minimum Python Version
 ----------------------
 
-Synapse v1.1 has a minimum Python requirement of Python 3.5. Python 3.6 or
+Synapse v1.1.0 has a minimum Python requirement of Python 3.5. Python 3.6 or
 Python 3.7 are recommended as they have improved internal string handling,
 significantly reducing memory usage.
 

changelog.d/5072.feature

@@ -1 +0,0 @@
-Synapse can now be configured to not join remote rooms of a given "complexity" (currently, state events). This option can be used to prevent adverse performance on resource-constrained homeservers.

changelog.d/5099.misc

@@ -1 +0,0 @@
-Python 2 has been removed from the CI.

changelog.d/5252.feature

@@ -1 +0,0 @@
-Add monthly active users to phonehome stats.

changelog.d/5325.bugfix

@@ -1 +0,0 @@
-Fix a bug where running synapse_port_db would cause the account validity feature to fail because it didn't set the type of the email_sent column to boolean.

changelog.d/5363.feature

@@ -1 +0,0 @@
-Allow expired user to trigger renewal email sending manually.

changelog.d/5378.misc

@@ -1 +0,0 @@
-Track deactivated accounts in the database.

changelog.d/5381.misc

@@ -1 +0,0 @@
-Clean up code for sending federation EDUs.

changelog.d/5382.misc

@@ -1 +0,0 @@
-Add a sponsor button to the repo.

changelog.d/5383.misc

@@ -1 +0,0 @@
-Don't log non-200 responses from federation queries as exceptions.

changelog.d/5384.feature

@@ -1 +0,0 @@
-Statistics on forward extremities per room are now exposed via Prometheus.

changelog.d/5386.misc

@@ -1 +0,0 @@
-Add a sponsor button to the repo.

changelog.d/5387.bugfix

@@ -1 +0,0 @@
-Warn about disabling email-based password resets when a reset occurs, and remove warning when someone attempts a phone-based reset.

changelog.d/5388.bugfix

@@ -1 +0,0 @@
-Fix email notifications for unnamed rooms with multiple people.

changelog.d/5389.bugfix

@@ -1 +0,0 @@
-Fix exceptions in federation reader worker caused by attempting to renew attestations, which should only happen on master worker.

changelog.d/5390.bugfix

@@ -1 +0,0 @@
-Fix handling of failures fetching remote content to not log failures as exceptions.

changelog.d/5394.bugfix

@@ -1 +0,0 @@
-Fix a bug where deactivated users could receive renewal emails if the account validity feature is on.

changelog.d/5412.feature

@@ -1 +0,0 @@
-Add --no-daemonize option to run synapse in the foreground, per issue #4130. Contributed by Soham Gumaste.

changelog.d/5425.removal

@@ -1 +0,0 @@
-Python 2.7 is no longer a supported platform. Synapse now requires Python 3.5+ to run.

changelog.d/5440.feature

@@ -1 +0,0 @@
-Allow server admins to define implementations of extra rules for allowing or denying incoming events.

changelog.d/5446.misc

@@ -1 +0,0 @@
-Update Python syntax in contrib/ to Python 3. 

changelog.d/5447.misc

@@ -1 +0,0 @@
-Update federation_client dev script to support `.well-known` and work with python3.

changelog.d/5448.removal

@@ -1 +0,0 @@
-PostgreSQL 9.4 is no longer supported. Synapse requires Postgres 9.5+ or above for Postgres support.

changelog.d/5458.feature

@@ -1 +0,0 @@
-Statistics on forward extremities per room are now exposed via Prometheus.

changelog.d/5459.misc

@@ -1 +0,0 @@
-SyTest has been moved to Buildkite.

changelog.d/5460.misc

@@ -1 +0,0 @@
-Demo script now uses python3.

changelog.d/5461.feature

@@ -1 +0,0 @@
-Statistics on forward extremities per room are now exposed via Prometheus.

changelog.d/5464.bugfix

@@ -1 +0,0 @@
-Fix missing invite state after exchanging 3PID invites over federaton.

changelog.d/5465.misc

@@ -1,2 +0,0 @@
-Track deactivated accounts in the database.
-

changelog.d/5474.feature

@@ -1 +0,0 @@
-Allow server admins to define implementations of extra rules for allowing or denying incoming events.

changelog.d/5477.feature

@@ -1 +0,0 @@
-Allow server admins to define implementations of extra rules for allowing or denying incoming events.

changelog.d/5478.misc

@@ -1 +0,0 @@
-The demo servers talk to each other again.

changelog.d/5678.removal

@@ -0,0 +1 @@
+Synapse now no longer accepts the `-v`/`--verbose`, `-f`/`--log-file`, or `--log-config` command line flags, and removes the deprecated `verbose` and `log_file` configuration file options. Users of these options should migrate their options into the dedicated log configuration.

changelog.d/5693.bugfix

@@ -0,0 +1 @@
+Fix UISIs during homeserver outage.

changelog.d/5694.misc

@@ -0,0 +1 @@
+Make Jaeger fully configurable.

changelog.d/5695.misc

@@ -0,0 +1 @@
+Add precautionary measures to prevent future abuse of `window.opener` in default welcome page.

changelog.d/5706.misc

@@ -0,0 +1 @@
+Reduce database IO usage by optimising queries for current membership.

changelog.d/5713.misc

@@ -0,0 +1 @@
+Improve caching when fetching `get_filtered_current_state_ids`.

changelog.d/5715.misc

@@ -0,0 +1 @@
+Don't accept opentracing data from clients.

changelog.d/5717.misc

@@ -0,0 +1 @@
+Speed up PostgreSQL unit tests in CI.

changelog.d/5719.misc

@@ -0,0 +1 @@
+Update the coding style document.

changelog.d/5720.misc

@@ -0,0 +1 @@
+Improve database query performance when recording retry intervals for remote hosts.

changelog.d/5722.misc

@@ -0,0 +1 @@
+Add a set of opentracing utils.

changelog.d/5724.bugfix

@@ -0,0 +1 @@
+Fix stack overflow in server key lookup code.

changelog.d/5725.bugfix

@@ -0,0 +1 @@
+start.sh no longer uses deprecated cli option.

changelog.d/5729.removal

@@ -0,0 +1 @@
+ Synapse now no longer accepts the `-v`/`--verbose`, `-f`/`--log-file`, or `--log-config` command line flags, and removes the deprecated `verbose` and `log_file` configuration file options. Users of these options should migrate their options into the dedicated log configuration.

changelog.d/5730.misc

@@ -0,0 +1 @@
+Cache result of get_version_string to reduce overhead of `/version` federation requests.

changelog.d/5731.misc

@@ -0,0 +1 @@
+Return 'user_type' in admin API user endpoints results.

changelog.d/5732.feature

@@ -0,0 +1 @@
+Add sd_notify hooks to ease systemd integration and allows usage of Type=Notify.

changelog.d/5733.misc

@@ -0,0 +1 @@
+Don't package the sytest test blacklist file.

changelog.d/5736.misc

@@ -0,0 +1 @@
+Replace uses of returnValue with plain return, as returnValue is not needed on Python 3.

changelog.d/5738.misc

@@ -0,0 +1 @@
+Reduce database IO usage by optimising queries for current membership.

changelog.d/5740.misc

@@ -0,0 +1 @@
+Blacklist some flakey tests in worker mode.

changelog.d/5743.bugfix

@@ -0,0 +1 @@
+Log when we receive an event receipt from an unexpected origin.

changelog.d/5746.misc

@@ -0,0 +1 @@
+Reduce database IO usage by optimising queries for current membership.

changelog.d/5749.misc

@@ -0,0 +1 @@
+Fix some error cases in the caching layer.

changelog.d/5750.misc

@@ -0,0 +1 @@
+Add a prometheus metric for pending cache lookups.

changelog.d/5752.misc

@@ -0,0 +1 @@
+Reduce database IO usage by optimising queries for current membership.

changelog.d/5753.misc

@@ -0,0 +1 @@
+Stop trying to fetch events with event_id=None.

changelog.d/5768.misc

@@ -0,0 +1 @@
+Convert RedactionTestCase to modern test style.

changelog.d/5770.misc

@@ -0,0 +1 @@
+Reduce database IO usage by optimising queries for current membership.

changelog.d/5774.misc

@@ -0,0 +1 @@
+Reduce database IO usage by optimising queries for current membership.

changelog.d/5775.bugfix

@@ -0,0 +1 @@
+Fix debian packaging scripts to correctly build sid packages.

changelog.d/5780.misc

@@ -0,0 +1 @@
+Allow looping calls to be given arguments.

changelog.d/5782.removal

@@ -0,0 +1 @@
+Remove non-functional 'expire_access_token' setting.

changelog.d/5783.feature

@@ -0,0 +1 @@
+Synapse can now be configured to not join remote rooms of a given "complexity" (currently, state events) over federation. This option can be used to prevent adverse performance on resource-constrained homeservers.

changelog.d/5785.misc

@@ -0,0 +1 @@
+Set the logs emitted when checking typing and presence timeouts to DEBUG level, not INFO.

changelog.d/5787.misc

@@ -0,0 +1 @@
+Remove DelayedCall debugging from the test suite, as it is no longer required in the vast majority of Synapse's tests.

changelog.d/5789.bugfix

@@ -0,0 +1 @@
+Fix UISIs during homeserver outage.

changelog.d/5792.misc

@@ -0,0 +1 @@
+Reduce database IO usage by optimising queries for current membership.

changelog.d/5793.misc

@@ -0,0 +1 @@
+Reduce database IO usage by optimising queries for current membership.

changelog.d/5794.misc

@@ -0,0 +1 @@
+Improve performance when making `.well-known` requests by sharing the SSL options between requests.

changelog.d/5796.misc

@@ -0,0 +1 @@
+Disable codecov GitHub comments on PRs.

changelog.d/5798.bugfix

@@ -0,0 +1 @@
+Return 404 instead of 403 when accessing /rooms/{roomId}/event/{eventId} for an event without the appropriate permissions.

contrib/cmdclient/console.py

@@ -37,9 +37,8 @@ from signedjson.sign import verify_signed_json, SignatureVerifyException
 
 CONFIG_JSON = "cmdclient_config.json"
 
-TRUSTED_ID_SERVERS = [
-    'localhost:8001'
-]
+TRUSTED_ID_SERVERS = ["localhost:8001"]
+
 
 class SynapseCmd(cmd.Cmd):
 
@@ -59,7 +58,7 @@ class SynapseCmd(cmd.Cmd):
             "token": token,
             "verbose": "on",
             "complete_usernames": "on",
-            "send_delivery_receipts": "on"
+            "send_delivery_receipts": "on",
         }
         self.path_prefix = "/_matrix/client/api/v1"
         self.event_stream_token = "END"
@@ -120,12 +119,11 @@ class SynapseCmd(cmd.Cmd):
             config_rules = [  # key, valid_values
                 ("verbose", ["on", "off"]),
                 ("complete_usernames", ["on", "off"]),
-                ("send_delivery_receipts", ["on", "off"])
+                ("send_delivery_receipts", ["on", "off"]),
             ]
             for key, valid_vals in config_rules:
                 if key == args["key"] and args["val"] not in valid_vals:
-                    print("%s value must be one of %s" % (args["key"],
-                                                          valid_vals))
+                    print("%s value must be one of %s" % (args["key"], valid_vals))
                     return
 
             # toggle the http client verbosity
@@ -159,16 +157,13 @@ class SynapseCmd(cmd.Cmd):
             else:
                 password = pwd
 
-        body = {
-            "type": "m.login.password"
-        }
+        body = {"type": "m.login.password"}
         if "userid" in args:
             body["user"] = args["userid"]
         if password:
             body["password"] = password
 
-        reactor.callFromThread(self._do_register, body,
-                               "noupdate" not in args)
+        reactor.callFromThread(self._do_register, body, "noupdate" not in args)
 
     @defer.inlineCallbacks
     def _do_register(self, data, update_config):
@@ -179,7 +174,9 @@ class SynapseCmd(cmd.Cmd):
 
         passwordFlow = None
         for flow in json_res["flows"]:
-            if flow["type"] == "m.login.recaptcha" or ("stages" in flow and "m.login.recaptcha" in flow["stages"]):
+            if flow["type"] == "m.login.recaptcha" or (
+                "stages" in flow and "m.login.recaptcha" in flow["stages"]
+            ):
                 print("Unable to register: Home server requires captcha.")
                 return
             if flow["type"] == "m.login.password" and "stages" not in flow:
@@ -202,9 +199,7 @@ class SynapseCmd(cmd.Cmd):
         """
         try:
             args = self._parse(line, ["user_id"], force_keys=True)
-            can_login = threads.blockingCallFromThread(
-                reactor,
-                self._check_can_login)
+            can_login = threads.blockingCallFromThread(reactor, self._check_can_login)
             if can_login:
                 p = getpass.getpass("Enter your password: ")
                 user = args["user_id"]
@@ -212,20 +207,16 @@ class SynapseCmd(cmd.Cmd):
                     domain = self._domain()
                     if domain:
                         user = "@" + user + ":" + domain
-                
+
                 reactor.callFromThread(self._do_login, user, p)
-                #print " got %s " % p
+                # print " got %s " % p
         except Exception as e:
             print(e)
 
     @defer.inlineCallbacks
     def _do_login(self, user, password):
         path = "/login"
-        data = {
-            "user": user,
-            "password": password,
-            "type": "m.login.password"
-        }
+        data = {"user": user, "password": password, "type": "m.login.password"}
         url = self._url() + path
         json_res = yield self.http_client.do_request("POST", url, data=data)
         print(json_res)
@@ -249,12 +240,13 @@ class SynapseCmd(cmd.Cmd):
             print("Failed to find any login flows.")
             defer.returnValue(False)
 
-        flow = json_res["flows"][0] # assume first is the one we want.
-        if ("type" not in flow or "m.login.password" != flow["type"] or
-                "stages" in flow):
+        flow = json_res["flows"][0]  # assume first is the one we want.
+        if "type" not in flow or "m.login.password" != flow["type"] or "stages" in flow:
             fallback_url = self._url() + "/login/fallback"
-            print ("Unable to login via the command line client. Please visit "
-                "%s to login." % fallback_url)
+            print(
+                "Unable to login via the command line client. Please visit "
+                "%s to login." % fallback_url
+            )
             defer.returnValue(False)
         defer.returnValue(True)
 
@@ -264,21 +256,33 @@ class SynapseCmd(cmd.Cmd):
         <clientSecret> A string of characters generated when requesting an email that you'll supply in subsequent calls to identify yourself
         <sendAttempt> The number of times the user has requested an email. Leave this the same between requests to retry the request at the transport level. Increment it to request that the email be sent again.
         """
-        args = self._parse(line, ['address', 'clientSecret', 'sendAttempt'])
+        args = self._parse(line, ["address", "clientSecret", "sendAttempt"])
 
-        postArgs = {'email': args['address'], 'clientSecret': args['clientSecret'], 'sendAttempt': args['sendAttempt']}
+        postArgs = {
+            "email": args["address"],
+            "clientSecret": args["clientSecret"],
+            "sendAttempt": args["sendAttempt"],
+        }
 
         reactor.callFromThread(self._do_emailrequest, postArgs)
 
     @defer.inlineCallbacks
     def _do_emailrequest(self, args):
-        url = self._identityServerUrl()+"/_matrix/identity/api/v1/validate/email/requestToken"
+        url = (
+            self._identityServerUrl()
+            + "/_matrix/identity/api/v1/validate/email/requestToken"
+        )
 
-        json_res = yield self.http_client.do_request("POST", url, data=urllib.urlencode(args), jsonreq=False,
-                                                     headers={'Content-Type': ['application/x-www-form-urlencoded']})
+        json_res = yield self.http_client.do_request(
+            "POST",
+            url,
+            data=urllib.urlencode(args),
+            jsonreq=False,
+            headers={"Content-Type": ["application/x-www-form-urlencoded"]},
+        )
         print(json_res)
-        if 'sid' in json_res:
-            print("Token sent. Your session ID is %s" % (json_res['sid']))
+        if "sid" in json_res:
+            print("Token sent. Your session ID is %s" % (json_res["sid"]))
 
     def do_emailvalidate(self, line):
         """Validate and associate a third party ID
@@ -286,18 +290,30 @@ class SynapseCmd(cmd.Cmd):
         <token> The token sent to your third party identifier address
         <clientSecret> The same clientSecret you supplied in requestToken
         """
-        args = self._parse(line, ['sid', 'token', 'clientSecret'])
+        args = self._parse(line, ["sid", "token", "clientSecret"])
 
-        postArgs = { 'sid' : args['sid'], 'token' : args['token'], 'clientSecret': args['clientSecret'] }
+        postArgs = {
+            "sid": args["sid"],
+            "token": args["token"],
+            "clientSecret": args["clientSecret"],
+        }
 
         reactor.callFromThread(self._do_emailvalidate, postArgs)
 
     @defer.inlineCallbacks
     def _do_emailvalidate(self, args):
-        url = self._identityServerUrl()+"/_matrix/identity/api/v1/validate/email/submitToken"
+        url = (
+            self._identityServerUrl()
+            + "/_matrix/identity/api/v1/validate/email/submitToken"
+        )
 
-        json_res = yield self.http_client.do_request("POST", url, data=urllib.urlencode(args), jsonreq=False,
-                                                     headers={'Content-Type': ['application/x-www-form-urlencoded']})
+        json_res = yield self.http_client.do_request(
+            "POST",
+            url,
+            data=urllib.urlencode(args),
+            jsonreq=False,
+            headers={"Content-Type": ["application/x-www-form-urlencoded"]},
+        )
         print(json_res)
 
     def do_3pidbind(self, line):
@@ -305,19 +321,24 @@ class SynapseCmd(cmd.Cmd):
         <sid> The session ID (sid) given to you in the response to requestToken
         <clientSecret> The same clientSecret you supplied in requestToken
         """
-        args = self._parse(line, ['sid', 'clientSecret'])
+        args = self._parse(line, ["sid", "clientSecret"])
 
-        postArgs = { 'sid' : args['sid'], 'clientSecret': args['clientSecret'] }
-        postArgs['mxid'] = self.config["user"]
+        postArgs = {"sid": args["sid"], "clientSecret": args["clientSecret"]}
+        postArgs["mxid"] = self.config["user"]
 
         reactor.callFromThread(self._do_3pidbind, postArgs)
 
     @defer.inlineCallbacks
     def _do_3pidbind(self, args):
-        url = self._identityServerUrl()+"/_matrix/identity/api/v1/3pid/bind"
+        url = self._identityServerUrl() + "/_matrix/identity/api/v1/3pid/bind"
 
-        json_res = yield self.http_client.do_request("POST", url, data=urllib.urlencode(args), jsonreq=False,
-                                                     headers={'Content-Type': ['application/x-www-form-urlencoded']})
+        json_res = yield self.http_client.do_request(
+            "POST",
+            url,
+            data=urllib.urlencode(args),
+            jsonreq=False,
+            headers={"Content-Type": ["application/x-www-form-urlencoded"]},
+        )
         print(json_res)
 
     def do_join(self, line):
@@ -356,9 +377,7 @@ class SynapseCmd(cmd.Cmd):
                 if "topic" not in args:
                     print("Must specify a new topic.")
                     return
-                body = {
-                    "topic": args["topic"]
-                }
+                body = {"topic": args["topic"]}
                 reactor.callFromThread(self._run_and_pprint, "PUT", path, body)
             elif args["action"].lower() == "get":
                 reactor.callFromThread(self._run_and_pprint, "GET", path)
@@ -378,45 +397,60 @@ class SynapseCmd(cmd.Cmd):
 
     @defer.inlineCallbacks
     def _do_invite(self, roomid, userstring):
-        if (not userstring.startswith('@') and
-                    self._is_on("complete_usernames")):
-            url = self._identityServerUrl()+"/_matrix/identity/api/v1/lookup"
+        if not userstring.startswith("@") and self._is_on("complete_usernames"):
+            url = self._identityServerUrl() + "/_matrix/identity/api/v1/lookup"
 
-            json_res = yield self.http_client.do_request("GET", url, qparams={'medium':'email','address':userstring})
+            json_res = yield self.http_client.do_request(
+                "GET", url, qparams={"medium": "email", "address": userstring}
+            )
 
             mxid = None
 
-            if 'mxid' in json_res and 'signatures' in json_res:
-                url = self._identityServerUrl()+"/_matrix/identity/api/v1/pubkey/ed25519"
+            if "mxid" in json_res and "signatures" in json_res:
+                url = (
+                    self._identityServerUrl()
+                    + "/_matrix/identity/api/v1/pubkey/ed25519"
+                )
 
                 pubKey = None
                 pubKeyObj = yield self.http_client.do_request("GET", url)
-                if 'public_key' in pubKeyObj:
-                    pubKey = nacl.signing.VerifyKey(pubKeyObj['public_key'], encoder=nacl.encoding.HexEncoder)
+                if "public_key" in pubKeyObj:
+                    pubKey = nacl.signing.VerifyKey(
+                        pubKeyObj["public_key"], encoder=nacl.encoding.HexEncoder
+                    )
                 else:
                     print("No public key found in pubkey response!")
 
                 sigValid = False
 
                 if pubKey:
-                    for signame in json_res['signatures']:
+                    for signame in json_res["signatures"]:
                         if signame not in TRUSTED_ID_SERVERS:
-                            print("Ignoring signature from untrusted server %s" % (signame))
+                            print(
+                                "Ignoring signature from untrusted server %s"
+                                % (signame)
+                            )
                         else:
                             try:
                                 verify_signed_json(json_res, signame, pubKey)
                                 sigValid = True
-                                print("Mapping %s -> %s correctly signed by %s" % (userstring, json_res['mxid'], signame))
+                                print(
+                                    "Mapping %s -> %s correctly signed by %s"
+                                    % (userstring, json_res["mxid"], signame)
+                                )
                                 break
                             except SignatureVerifyException as e:
                                 print("Invalid signature from %s" % (signame))
                                 print(e)
 
                 if sigValid:
-                    print("Resolved 3pid %s to %s" % (userstring, json_res['mxid']))
-                    mxid = json_res['mxid']
+                    print("Resolved 3pid %s to %s" % (userstring, json_res["mxid"]))
+                    mxid = json_res["mxid"]
                 else:
-                    print("Got association for %s but couldn't verify signature" % (userstring))
+                    print(
+                        "Got association for %s but couldn't verify signature"
+                        % (userstring)
+                    )
 
             if not mxid:
                 mxid = "@" + userstring + ":" + self._domain()
@@ -435,12 +469,11 @@ class SynapseCmd(cmd.Cmd):
         """Sends a message. "send <roomid> <body>" """
         args = self._parse(line, ["roomid", "body"])
         txn_id = "txn%s" % int(time.time())
-        path = "/rooms/%s/send/m.room.message/%s" % (urllib.quote(args["roomid"]),
-                                             txn_id)
-        body_json = {
-            "msgtype": "m.text",
-            "body": args["body"]
-        }
+        path = "/rooms/%s/send/m.room.message/%s" % (
+            urllib.quote(args["roomid"]),
+            txn_id,
+        )
+        body_json = {"msgtype": "m.text", "body": args["body"]}
         reactor.callFromThread(self._run_and_pprint, "PUT", path, body_json)
 
     def do_list(self, line):
@@ -472,8 +505,7 @@ class SynapseCmd(cmd.Cmd):
                     print("Bad query param: %s" % key_value)
                     return
 
-        reactor.callFromThread(self._run_and_pprint, "GET", path,
-                               query_params=qp)
+        reactor.callFromThread(self._run_and_pprint, "GET", path, query_params=qp)
 
     def do_create(self, line):
         """Creates a room.
@@ -513,8 +545,16 @@ class SynapseCmd(cmd.Cmd):
             return
 
         args["method"] = args["method"].upper()
-        valid_methods = ["PUT", "GET", "POST", "DELETE",
-                         "XPUT", "XGET", "XPOST", "XDELETE"]
+        valid_methods = [
+            "PUT",
+            "GET",
+            "POST",
+            "DELETE",
+            "XPUT",
+            "XGET",
+            "XPOST",
+            "XDELETE",
+        ]
         if args["method"] not in valid_methods:
             print("Unsupported method: %s" % args["method"])
             return
@@ -541,10 +581,13 @@ class SynapseCmd(cmd.Cmd):
             except:
                 pass
 
-        reactor.callFromThread(self._run_and_pprint, args["method"],
-                                                     args["path"],
-                                                     args["data"],
-                                                     query_params=qp)
+        reactor.callFromThread(
+            self._run_and_pprint,
+            args["method"],
+            args["path"],
+            args["data"],
+            query_params=qp,
+        )
 
     def do_stream(self, line):
         """Stream data from the server: "stream <longpoll timeout ms>" """
@@ -561,19 +604,22 @@ class SynapseCmd(cmd.Cmd):
     @defer.inlineCallbacks
     def _do_event_stream(self, timeout):
         res = yield self.http_client.get_json(
-                self._url() + "/events",
-                {
-                    "access_token": self._tok(),
-                    "timeout": str(timeout),
-                    "from": self.event_stream_token
-                })
+            self._url() + "/events",
+            {
+                "access_token": self._tok(),
+                "timeout": str(timeout),
+                "from": self.event_stream_token,
+            },
+        )
         print(json.dumps(res, indent=4))
 
         if "chunk" in res:
             for event in res["chunk"]:
-                if (event["type"] == "m.room.message" and
-                        self._is_on("send_delivery_receipts") and
-                        event["user_id"] != self._usr()):  # not sent by us
+                if (
+                    event["type"] == "m.room.message"
+                    and self._is_on("send_delivery_receipts")
+                    and event["user_id"] != self._usr()
+                ):  # not sent by us
                     self._send_receipt(event, "d")
 
         # update the position in the stram
@@ -581,18 +627,28 @@ class SynapseCmd(cmd.Cmd):
             self.event_stream_token = res["end"]
 
     def _send_receipt(self, event, feedback_type):
-        path = ("/rooms/%s/messages/%s/%s/feedback/%s/%s" %
-               (urllib.quote(event["room_id"]), event["user_id"], event["msg_id"],
-                self._usr(), feedback_type))
+        path = "/rooms/%s/messages/%s/%s/feedback/%s/%s" % (
+            urllib.quote(event["room_id"]),
+            event["user_id"],
+            event["msg_id"],
+            self._usr(),
+            feedback_type,
+        )
         data = {}
-        reactor.callFromThread(self._run_and_pprint, "PUT", path, data=data,
-                               alt_text="Sent receipt for %s" % event["msg_id"])
+        reactor.callFromThread(
+            self._run_and_pprint,
+            "PUT",
+            path,
+            data=data,
+            alt_text="Sent receipt for %s" % event["msg_id"],
+        )
 
     def _do_membership_change(self, roomid, membership, userid):
-        path = "/rooms/%s/state/m.room.member/%s" % (urllib.quote(roomid), urllib.quote(userid))
-        data = {
-            "membership": membership
-        }
+        path = "/rooms/%s/state/m.room.member/%s" % (
+            urllib.quote(roomid),
+            urllib.quote(userid),
+        )
+        data = {"membership": membership}
         reactor.callFromThread(self._run_and_pprint, "PUT", path, data=data)
 
     def do_displayname(self, line):
@@ -645,15 +701,20 @@ class SynapseCmd(cmd.Cmd):
         for i, arg in enumerate(line_args):
             for config_key in self.config:
                 if ("$" + config_key) in arg:
-                    arg = arg.replace("$" + config_key,
-                                      self.config[config_key])
+                    arg = arg.replace("$" + config_key, self.config[config_key])
             line_args[i] = arg
 
         return dict(zip(keys, line_args))
 
     @defer.inlineCallbacks
-    def _run_and_pprint(self, method, path, data=None,
-                        query_params={"access_token": None}, alt_text=None):
+    def _run_and_pprint(
+        self,
+        method,
+        path,
+        data=None,
+        query_params={"access_token": None},
+        alt_text=None,
+    ):
         """ Runs an HTTP request and pretty prints the output.
 
         Args:
@@ -666,9 +727,9 @@ class SynapseCmd(cmd.Cmd):
         if "access_token" in query_params:
             query_params["access_token"] = self._tok()
 
-        json_res = yield self.http_client.do_request(method, url,
-                                                    data=data,
-                                                    qparams=query_params)
+        json_res = yield self.http_client.do_request(
+            method, url, data=data, qparams=query_params
+        )
         if alt_text:
             print(alt_text)
         else:
@@ -676,7 +737,7 @@ class SynapseCmd(cmd.Cmd):
 
 
 def save_config(config):
-    with open(CONFIG_JSON, 'w') as out:
+    with open(CONFIG_JSON, "w") as out:
         json.dump(config, out)
 
 
@@ -700,7 +761,7 @@ def main(server_url, identity_server_url, username, token, config_path):
     global CONFIG_JSON
     CONFIG_JSON = config_path  # bit cheeky, but just overwrite the global
     try:
-        with open(config_path, 'r') as config:
+        with open(config_path, "r") as config:
             syn_cmd.config = json.load(config)
             try:
                 http_client.verbose = "on" == syn_cmd.config["verbose"]
@@ -717,23 +778,33 @@ def main(server_url, identity_server_url, username, token, config_path):
     reactor.run()
 
 
-if __name__ == '__main__':
+if __name__ == "__main__":
     parser = argparse.ArgumentParser("Starts a synapse client.")
     parser.add_argument(
-        "-s", "--server", dest="server", default="http://localhost:8008",
-        help="The URL of the home server to talk to.")
+        "-s",
+        "--server",
+        dest="server",
+        default="http://localhost:8008",
+        help="The URL of the home server to talk to.",
+    )
     parser.add_argument(
-        "-i", "--identity-server", dest="identityserver", default="http://localhost:8090",
-        help="The URL of the identity server to talk to.")
+        "-i",
+        "--identity-server",
+        dest="identityserver",
+        default="http://localhost:8090",
+        help="The URL of the identity server to talk to.",
+    )
     parser.add_argument(
-        "-u", "--username", dest="username",
-        help="Your username on the server.")
+        "-u", "--username", dest="username", help="Your username on the server."
+    )
+    parser.add_argument("-t", "--token", dest="token", help="Your access token.")
     parser.add_argument(
-        "-t", "--token", dest="token",
-        help="Your access token.")
-    parser.add_argument(
-        "-c", "--config", dest="config", default=CONFIG_JSON,
-        help="The location of the config.json file to read from.")
+        "-c",
+        "--config",
+        dest="config",
+        default=CONFIG_JSON,
+        help="The location of the config.json file to read from.",
+    )
     args = parser.parse_args()
 
     if not args.server:

contrib/cmdclient/http.py

@@ -73,9 +73,7 @@ class TwistedHttpClient(HttpClient):
     @defer.inlineCallbacks
     def put_json(self, url, data):
         response = yield self._create_put_request(
-            url,
-            data,
-            headers_dict={"Content-Type": ["application/json"]}
+            url, data, headers_dict={"Content-Type": ["application/json"]}
         )
         body = yield readBody(response)
         defer.returnValue((response.code, body))
@@ -95,40 +93,34 @@ class TwistedHttpClient(HttpClient):
         """
 
         if "Content-Type" not in headers_dict:
-            raise defer.error(
-                RuntimeError("Must include Content-Type header for PUTs"))
+            raise defer.error(RuntimeError("Must include Content-Type header for PUTs"))
 
         return self._create_request(
-            "PUT",
-            url,
-            producer=_JsonProducer(json_data),
-            headers_dict=headers_dict
+            "PUT", url, producer=_JsonProducer(json_data), headers_dict=headers_dict
         )
 
     def _create_get_request(self, url, headers_dict={}):
         """ Wrapper of _create_request to issue a GET request
         """
-        return self._create_request(
-            "GET",
-            url,
-            headers_dict=headers_dict
-        )
+        return self._create_request("GET", url, headers_dict=headers_dict)
 
     @defer.inlineCallbacks
-    def do_request(self, method, url, data=None, qparams=None, jsonreq=True, headers={}):
+    def do_request(
+        self, method, url, data=None, qparams=None, jsonreq=True, headers={}
+    ):
         if qparams:
             url = "%s?%s" % (url, urllib.urlencode(qparams, True))
 
         if jsonreq:
             prod = _JsonProducer(data)
-            headers['Content-Type'] = ["application/json"];
+            headers["Content-Type"] = ["application/json"]
         else:
             prod = _RawProducer(data)
 
         if method in ["POST", "PUT"]:
-            response = yield self._create_request(method, url,
-                    producer=prod,
-                    headers_dict=headers)
+            response = yield self._create_request(
+                method, url, producer=prod, headers_dict=headers
+            )
         else:
             response = yield self._create_request(method, url)
 
@@ -155,10 +147,7 @@ class TwistedHttpClient(HttpClient):
         while True:
             try:
                 response = yield self.agent.request(
-                    method,
-                    url.encode("UTF8"),
-                    Headers(headers_dict),
-                    producer
+                    method, url.encode("UTF8"), Headers(headers_dict), producer
                 )
                 break
             except Exception as e:
@@ -179,6 +168,7 @@ class TwistedHttpClient(HttpClient):
         reactor.callLater(seconds, d.callback, seconds)
         return d
 
+
 class _RawProducer(object):
     def __init__(self, data):
         self.data = data
@@ -195,9 +185,11 @@ class _RawProducer(object):
     def stopProducing(self):
         pass
 
+
 class _JsonProducer(object):
     """ Used by the twisted http client to create the HTTP body from json
     """
+
     def __init__(self, jsn):
         self.data = jsn
         self.body = json.dumps(jsn).encode("utf8")

contrib/docker/README.md

@@ -1,5 +1,9 @@
 # Synapse Docker
 
+FIXME: this is out-of-date as of
+https://github.com/matrix-org/synapse/issues/5518. Contributions to bring it up
+to date would be welcome.
+
 ### Automated configuration
 
 It is recommended that you use Docker Compose to run your containers, including

contrib/example_log_config.yaml

@@ -1,7 +1,7 @@
-# Example log_config file for synapse. To enable, point `log_config` to it in 
+# Example log_config file for synapse. To enable, point `log_config` to it in
 # `homeserver.yaml`, and restart synapse.
 #
-# This configuration will produce similar results to the defaults within 
+# This configuration will produce similar results to the defaults within
 # synapse, but can be edited to give more flexibility.
 
 version: 1
@@ -12,7 +12,7 @@ formatters:
 
 filters:
   context:
-    (): synapse.util.logcontext.LoggingContextFilter
+    (): synapse.logging.context.LoggingContextFilter
     request: ""
 
 handlers:
@@ -35,7 +35,7 @@ handlers:
 root:
     level: INFO
     handlers: [console] # to use file handler instead, switch to [file]
-    
+
 loggers:
     synapse:
         level: INFO

contrib/experiments/cursesio.py

@@ -19,13 +19,13 @@ from curses.ascii import isprint
 from twisted.internet import reactor
 
 
-class CursesStdIO():
+class CursesStdIO:
     def __init__(self, stdscr, callback=None):
         self.statusText = "Synapse test app -"
-        self.searchText = ''
+        self.searchText = ""
         self.stdscr = stdscr
 
-        self.logLine = ''
+        self.logLine = ""
 
         self.callback = callback
 
@@ -71,8 +71,7 @@ class CursesStdIO():
         i = 0
         index = len(self.lines) - 1
         while i < (self.rows - 3) and index >= 0:
-            self.stdscr.addstr(self.rows - 3 - i, 0, self.lines[index],
-                               curses.A_NORMAL)
+            self.stdscr.addstr(self.rows - 3 - i, 0, self.lines[index], curses.A_NORMAL)
             i = i + 1
             index = index - 1
 
@@ -85,15 +84,13 @@ class CursesStdIO():
             raise RuntimeError("TextTooLongError")
 
         self.stdscr.addstr(
-            self.rows - 2, 0,
-            text + ' ' * (self.cols - len(text)),
-            curses.A_STANDOUT)
+            self.rows - 2, 0, text + " " * (self.cols - len(text)), curses.A_STANDOUT
+        )
 
     def printLogLine(self, text):
         self.stdscr.addstr(
-            0, 0,
-            text + ' ' * (self.cols - len(text)),
-            curses.A_STANDOUT)
+            0, 0, text + " " * (self.cols - len(text)), curses.A_STANDOUT
+        )
 
     def doRead(self):
         """ Input is ready! """
@@ -105,7 +102,7 @@ class CursesStdIO():
 
         elif c == curses.KEY_ENTER or c == 10:
             text = self.searchText
-            self.searchText = ''
+            self.searchText = ""
 
             self.print_line(">> %s" % text)
 
@@ -122,11 +119,13 @@ class CursesStdIO():
                 return
             self.searchText = self.searchText + chr(c)
 
-        self.stdscr.addstr(self.rows - 1, 0,
-                           self.searchText + (' ' * (
-                           self.cols - len(self.searchText) - 2)))
+        self.stdscr.addstr(
+            self.rows - 1,
+            0,
+            self.searchText + (" " * (self.cols - len(self.searchText) - 2)),
+        )
 
-        self.paintStatus(self.statusText + ' %d' % len(self.searchText))
+        self.paintStatus(self.statusText + " %d" % len(self.searchText))
         self.stdscr.move(self.rows - 1, len(self.searchText))
         self.stdscr.refresh()
 
@@ -143,7 +142,6 @@ class CursesStdIO():
 
 
 class Callback(object):
-
     def __init__(self, stdio):
         self.stdio = stdio
 
@@ -152,7 +150,7 @@ class Callback(object):
 
 
 def main(stdscr):
-    screen = CursesStdIO(stdscr)   # create Screen object
+    screen = CursesStdIO(stdscr)  # create Screen object
 
     callback = Callback(screen)
 
@@ -164,5 +162,5 @@ def main(stdscr):
     screen.close()
 
 
-if __name__ == '__main__':
+if __name__ == "__main__":
     curses.wrapper(main)

contrib/experiments/test_messaging.py

@@ -28,9 +28,7 @@ Currently assumes the local address is localhost:<port>
 """
 
 
-from synapse.federation import (
-    ReplicationHandler
-)
+from synapse.federation import ReplicationHandler
 
 from synapse.federation.units import Pdu
 
@@ -38,7 +36,7 @@ from synapse.util import origin_from_ucid
 
 from synapse.app.homeserver import SynapseHomeServer
 
-#from synapse.util.logutils import log_function
+# from synapse.logging.utils import log_function
 
 from twisted.internet import reactor, defer
 from twisted.python import log
@@ -83,7 +81,7 @@ class InputOutput(object):
                 room_name, = m.groups()
                 self.print_line("%s joining %s" % (self.user, room_name))
                 self.server.join_room(room_name, self.user, self.user)
-                #self.print_line("OK.")
+                # self.print_line("OK.")
                 return
 
             m = re.match("^invite (\S+) (\S+)$", line)
@@ -92,7 +90,7 @@ class InputOutput(object):
                 room_name, invitee = m.groups()
                 self.print_line("%s invited to %s" % (invitee, room_name))
                 self.server.invite_to_room(room_name, self.user, invitee)
-                #self.print_line("OK.")
+                # self.print_line("OK.")
                 return
 
             m = re.match("^send (\S+) (.*)$", line)
@@ -101,7 +99,7 @@ class InputOutput(object):
                 room_name, body = m.groups()
                 self.print_line("%s send to %s" % (self.user, room_name))
                 self.server.send_message(room_name, self.user, body)
-                #self.print_line("OK.")
+                # self.print_line("OK.")
                 return
 
             m = re.match("^backfill (\S+)$", line)
@@ -125,7 +123,6 @@ class InputOutput(object):
 
 
 class IOLoggerHandler(logging.Handler):
-
     def __init__(self, io):
         logging.Handler.__init__(self)
         self.io = io
@@ -142,6 +139,7 @@ class Room(object):
     """ Used to store (in memory) the current membership state of a room, and
     which home servers we should send PDUs associated with the room to.
     """
+
     def __init__(self, room_name):
         self.room_name = room_name
         self.invited = set()
@@ -175,6 +173,7 @@ class HomeServer(ReplicationHandler):
     """ A very basic home server implentation that allows people to join a
     room and then invite other people.
     """
+
     def __init__(self, server_name, replication_layer, output):
         self.server_name = server_name
         self.replication_layer = replication_layer
@@ -197,26 +196,27 @@ class HomeServer(ReplicationHandler):
             elif pdu.content["membership"] == "invite":
                 self._on_invite(pdu.origin, pdu.context, pdu.state_key)
         else:
-            self.output.print_line("#%s (unrec) %s = %s" %
-                (pdu.context, pdu.pdu_type, json.dumps(pdu.content))
+            self.output.print_line(
+                "#%s (unrec) %s = %s"
+                % (pdu.context, pdu.pdu_type, json.dumps(pdu.content))
             )
 
-    #def on_state_change(self, pdu):
-        ##self.output.print_line("#%s (state) %s *** %s" %
-                ##(pdu.context, pdu.state_key, pdu.pdu_type)
-            ##)
+    # def on_state_change(self, pdu):
+    ##self.output.print_line("#%s (state) %s *** %s" %
+    ##(pdu.context, pdu.state_key, pdu.pdu_type)
+    ##)
 
-        #if "joinee" in pdu.content:
-            #self._on_join(pdu.context, pdu.content["joinee"])
-        #elif "invitee" in pdu.content:
-            #self._on_invite(pdu.origin, pdu.context, pdu.content["invitee"])
+    # if "joinee" in pdu.content:
+    # self._on_join(pdu.context, pdu.content["joinee"])
+    # elif "invitee" in pdu.content:
+    # self._on_invite(pdu.origin, pdu.context, pdu.content["invitee"])
 
     def _on_message(self, pdu):
         """ We received a message
         """
-        self.output.print_line("#%s %s %s" %
-                (pdu.context, pdu.content["sender"], pdu.content["body"])
-            )
+        self.output.print_line(
+            "#%s %s %s" % (pdu.context, pdu.content["sender"], pdu.content["body"])
+        )
 
     def _on_join(self, context, joinee):
         """ Someone has joined a room, either a remote user or a local user
@@ -224,9 +224,7 @@ class HomeServer(ReplicationHandler):
         room = self._get_or_create_room(context)
         room.add_participant(joinee)
 
-        self.output.print_line("#%s %s %s" %
-                (context, joinee, "*** JOINED")
-            )
+        self.output.print_line("#%s %s %s" % (context, joinee, "*** JOINED"))
 
     def _on_invite(self, origin, context, invitee):
         """ Someone has been invited
@@ -234,9 +232,7 @@ class HomeServer(ReplicationHandler):
         room = self._get_or_create_room(context)
         room.add_invited(invitee)
 
-        self.output.print_line("#%s %s %s" %
-                (context, invitee, "*** INVITED")
-            )
+        self.output.print_line("#%s %s %s" % (context, invitee, "*** INVITED"))
 
         if not room.have_got_metadata and origin is not self.server_name:
             logger.debug("Get room state")
@@ -272,14 +268,14 @@ class HomeServer(ReplicationHandler):
 
         try:
             pdu = Pdu.create_new(
-                    context=room_name,
-                    pdu_type="sy.room.member",
-                    is_state=True,
-                    state_key=joinee,
-                    content={"membership": "join"},
-                    origin=self.server_name,
-                    destinations=destinations,
-                )
+                context=room_name,
+                pdu_type="sy.room.member",
+                is_state=True,
+                state_key=joinee,
+                content={"membership": "join"},
+                origin=self.server_name,
+                destinations=destinations,
+            )
             yield self.replication_layer.send_pdu(pdu)
         except Exception as e:
             logger.exception(e)
@@ -318,21 +314,21 @@ class HomeServer(ReplicationHandler):
         return self.replication_layer.backfill(dest, room_name, limit)
 
     def _get_room_remote_servers(self, room_name):
-        return [i for i in self.joined_rooms.setdefault(room_name,).servers]
+        return [i for i in self.joined_rooms.setdefault(room_name).servers]
 
     def _get_or_create_room(self, room_name):
         return self.joined_rooms.setdefault(room_name, Room(room_name))
 
     def get_servers_for_context(self, context):
         return defer.succeed(
-                self.joined_rooms.setdefault(context, Room(context)).servers
-            )
+            self.joined_rooms.setdefault(context, Room(context)).servers
+        )
 
 
 def main(stdscr):
     parser = argparse.ArgumentParser()
-    parser.add_argument('user', type=str)
-    parser.add_argument('-v', '--verbose', action='count')
+    parser.add_argument("user", type=str)
+    parser.add_argument("-v", "--verbose", action="count")
     args = parser.parse_args()
 
     user = args.user
@@ -342,8 +338,9 @@ def main(stdscr):
 
     root_logger = logging.getLogger()
 
-    formatter = logging.Formatter('%(asctime)s - %(name)s - %(lineno)d - '
-            '%(levelname)s - %(message)s')
+    formatter = logging.Formatter(
+        "%(asctime)s - %(name)s - %(lineno)d - " "%(levelname)s - %(message)s"
+    )
     if not os.path.exists("logs"):
         os.makedirs("logs")
     fh = logging.FileHandler("logs/%s" % user)

contrib/graph/graph.py

@@ -1,4 +1,5 @@
 from __future__ import print_function
+
 # Copyright 2014-2016 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -58,9 +59,9 @@ def make_graph(pdus, room, filename_prefix):
         name = make_name(pdu.get("pdu_id"), pdu.get("origin"))
         pdu_map[name] = pdu
 
-        t = datetime.datetime.fromtimestamp(
-            float(pdu["ts"]) / 1000
-        ).strftime('%Y-%m-%d %H:%M:%S,%f')
+        t = datetime.datetime.fromtimestamp(float(pdu["ts"]) / 1000).strftime(
+            "%Y-%m-%d %H:%M:%S,%f"
+        )
 
         label = (
             "<"
@@ -80,11 +81,7 @@ def make_graph(pdus, room, filename_prefix):
             "depth": pdu.get("depth"),
         }
 
-        node = pydot.Node(
-            name=name,
-            label=label,
-            color=color_map[pdu.get("origin")]
-        )
+        node = pydot.Node(name=name, label=label, color=color_map[pdu.get("origin")])
         node_map[name] = node
         graph.add_node(node)
 
@@ -108,14 +105,13 @@ def make_graph(pdus, room, filename_prefix):
 
             if prev_state_name in node_map:
                 state_edge = pydot.Edge(
-                    node_map[start_name], node_map[prev_state_name],
-                    style='dotted'
+                    node_map[start_name], node_map[prev_state_name], style="dotted"
                 )
                 graph.add_edge(state_edge)
 
-    graph.write('%s.dot' % filename_prefix, format='raw', prog='dot')
-#    graph.write_png("%s.png" % filename_prefix, prog='dot')
-    graph.write_svg("%s.svg" % filename_prefix, prog='dot')
+    graph.write("%s.dot" % filename_prefix, format="raw", prog="dot")
+    #    graph.write_png("%s.png" % filename_prefix, prog='dot')
+    graph.write_svg("%s.svg" % filename_prefix, prog="dot")
 
 
 def get_pdus(host, room):
@@ -131,15 +127,14 @@ def get_pdus(host, room):
 if __name__ == "__main__":
     parser = argparse.ArgumentParser(
         description="Generate a PDU graph for a given room by talking "
-                    "to the given homeserver to get the list of PDUs. \n"
-                    "Requires pydot."
+        "to the given homeserver to get the list of PDUs. \n"
+        "Requires pydot."
     )
     parser.add_argument(
-        "-p", "--prefix", dest="prefix",
-        help="String to prefix output files with"
+        "-p", "--prefix", dest="prefix", help="String to prefix output files with"
     )
-    parser.add_argument('host')
-    parser.add_argument('room')
+    parser.add_argument("host")
+    parser.add_argument("room")
 
     args = parser.parse_args()
 

contrib/graph/graph2.py

@@ -36,10 +36,7 @@ def make_graph(db_name, room_id, file_prefix, limit):
     args = [room_id]
 
     if limit:
-        sql += (
-            " ORDER BY topological_ordering DESC, stream_ordering DESC "
-            "LIMIT ?"
-        )
+        sql += " ORDER BY topological_ordering DESC, stream_ordering DESC " "LIMIT ?"
 
         args.append(limit)
 
@@ -56,9 +53,8 @@ def make_graph(db_name, room_id, file_prefix, limit):
 
     for event in events:
         c = conn.execute(
-            "SELECT state_group FROM event_to_state_groups "
-            "WHERE event_id = ?",
-            (event.event_id,)
+            "SELECT state_group FROM event_to_state_groups " "WHERE event_id = ?",
+            (event.event_id,),
         )
 
         res = c.fetchone()
@@ -69,7 +65,7 @@ def make_graph(db_name, room_id, file_prefix, limit):
 
         t = datetime.datetime.fromtimestamp(
             float(event.origin_server_ts) / 1000
-        ).strftime('%Y-%m-%d %H:%M:%S,%f')
+        ).strftime("%Y-%m-%d %H:%M:%S,%f")
 
         content = json.dumps(unfreeze(event.get_dict()["content"]))
 
@@ -93,10 +89,7 @@ def make_graph(db_name, room_id, file_prefix, limit):
             "state_group": state_group,
         }
 
-        node = pydot.Node(
-            name=event.event_id,
-            label=label,
-        )
+        node = pydot.Node(name=event.event_id, label=label)
 
         node_map[event.event_id] = node
         graph.add_node(node)
@@ -106,10 +99,7 @@ def make_graph(db_name, room_id, file_prefix, limit):
             try:
                 end_node = node_map[prev_id]
             except:
-                end_node = pydot.Node(
-                    name=prev_id,
-                    label="<<b>%s</b>>" % (prev_id,),
-                )
+                end_node = pydot.Node(name=prev_id, label="<<b>%s</b>>" % (prev_id,))
 
                 node_map[prev_id] = end_node
                 graph.add_node(end_node)
@@ -121,36 +111,33 @@ def make_graph(db_name, room_id, file_prefix, limit):
         if len(event_ids) <= 1:
             continue
 
-        cluster = pydot.Cluster(
-            str(group),
-            label="<State Group: %s>" % (str(group),)
-        )
+        cluster = pydot.Cluster(str(group), label="<State Group: %s>" % (str(group),))
 
         for event_id in event_ids:
             cluster.add_node(node_map[event_id])
 
         graph.add_subgraph(cluster)
 
-    graph.write('%s.dot' % file_prefix, format='raw', prog='dot')
-    graph.write_svg("%s.svg" % file_prefix, prog='dot')
+    graph.write("%s.dot" % file_prefix, format="raw", prog="dot")
+    graph.write_svg("%s.svg" % file_prefix, prog="dot")
+
 
 if __name__ == "__main__":
     parser = argparse.ArgumentParser(
         description="Generate a PDU graph for a given room by talking "
-                    "to the given homeserver to get the list of PDUs. \n"
-                    "Requires pydot."
+        "to the given homeserver to get the list of PDUs. \n"
+        "Requires pydot."
     )
     parser.add_argument(
-        "-p", "--prefix", dest="prefix",
+        "-p",
+        "--prefix",
+        dest="prefix",
         help="String to prefix output files with",
-        default="graph_output"
+        default="graph_output",
     )
-    parser.add_argument(
-        "-l", "--limit",
-        help="Only retrieve the last N events.",
-    )
-    parser.add_argument('db')
-    parser.add_argument('room')
+    parser.add_argument("-l", "--limit", help="Only retrieve the last N events.")
+    parser.add_argument("db")
+    parser.add_argument("room")
 
     args = parser.parse_args()
 

contrib/graph/graph3.py

@@ -1,4 +1,5 @@
 from __future__ import print_function
+
 # Copyright 2016 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -42,7 +43,7 @@ def make_graph(file_name, room_id, file_prefix, limit):
     print("Sorted events")
 
     if limit:
-        events = events[-int(limit):]
+        events = events[-int(limit) :]
 
     node_map = {}
 
@@ -51,7 +52,7 @@ def make_graph(file_name, room_id, file_prefix, limit):
     for event in events:
         t = datetime.datetime.fromtimestamp(
             float(event.origin_server_ts) / 1000
-        ).strftime('%Y-%m-%d %H:%M:%S,%f')
+        ).strftime("%Y-%m-%d %H:%M:%S,%f")
 
         content = json.dumps(unfreeze(event.get_dict()["content"]), indent=4)
         content = content.replace("\n", "<br/>\n")
@@ -67,9 +68,10 @@ def make_graph(file_name, room_id, file_prefix, limit):
                 value = json.dumps(value)
 
             content.append(
-                "<b>%s</b>: %s," % (
-                    cgi.escape(key, quote=True).encode("ascii", 'xmlcharrefreplace'),
-                    cgi.escape(value, quote=True).encode("ascii", 'xmlcharrefreplace'),
+                "<b>%s</b>: %s,"
+                % (
+                    cgi.escape(key, quote=True).encode("ascii", "xmlcharrefreplace"),
+                    cgi.escape(value, quote=True).encode("ascii", "xmlcharrefreplace"),
                 )
             )
 
@@ -95,10 +97,7 @@ def make_graph(file_name, room_id, file_prefix, limit):
             "depth": event.depth,
         }
 
-        node = pydot.Node(
-            name=event.event_id,
-            label=label,
-        )
+        node = pydot.Node(name=event.event_id, label=label)
 
         node_map[event.event_id] = node
         graph.add_node(node)
@@ -110,10 +109,7 @@ def make_graph(file_name, room_id, file_prefix, limit):
             try:
                 end_node = node_map[prev_id]
             except:
-                end_node = pydot.Node(
-                    name=prev_id,
-                    label="<<b>%s</b>>" % (prev_id,),
-                )
+                end_node = pydot.Node(name=prev_id, label="<<b>%s</b>>" % (prev_id,))
 
                 node_map[prev_id] = end_node
                 graph.add_node(end_node)
@@ -123,31 +119,31 @@ def make_graph(file_name, room_id, file_prefix, limit):
 
     print("Created edges")
 
-    graph.write('%s.dot' % file_prefix, format='raw', prog='dot')
+    graph.write("%s.dot" % file_prefix, format="raw", prog="dot")
 
     print("Created Dot")
 
-    graph.write_svg("%s.svg" % file_prefix, prog='dot')
+    graph.write_svg("%s.svg" % file_prefix, prog="dot")
 
     print("Created svg")
 
+
 if __name__ == "__main__":
     parser = argparse.ArgumentParser(
         description="Generate a PDU graph for a given room by reading "
-                    "from a file with line deliminated events. \n"
-                    "Requires pydot."
+        "from a file with line deliminated events. \n"
+        "Requires pydot."
     )
     parser.add_argument(
-        "-p", "--prefix", dest="prefix",
+        "-p",
+        "--prefix",
+        dest="prefix",
         help="String to prefix output files with",
-        default="graph_output"
+        default="graph_output",
     )
-    parser.add_argument(
-        "-l", "--limit",
-        help="Only retrieve the last N events.",
-    )
-    parser.add_argument('event_file')
-    parser.add_argument('room')
+    parser.add_argument("-l", "--limit", help="Only retrieve the last N events.")
+    parser.add_argument("event_file")
+    parser.add_argument("room")
 
     args = parser.parse_args()
 

contrib/jitsimeetbridge/jitsimeetbridge.py

@@ -20,24 +20,25 @@ import urllib
 import subprocess
 import time
 
-#ACCESS_TOKEN="" #
+# ACCESS_TOKEN="" #
 
-MATRIXBASE = 'https://matrix.org/_matrix/client/api/v1/'
-MYUSERNAME = '@davetest:matrix.org'
+MATRIXBASE = "https://matrix.org/_matrix/client/api/v1/"
+MYUSERNAME = "@davetest:matrix.org"
 
-HTTPBIND = 'https://meet.jit.si/http-bind'
-#HTTPBIND = 'https://jitsi.vuc.me/http-bind'
-#ROOMNAME = "matrix"
+HTTPBIND = "https://meet.jit.si/http-bind"
+# HTTPBIND = 'https://jitsi.vuc.me/http-bind'
+# ROOMNAME = "matrix"
 ROOMNAME = "pibble"
 
-HOST="guest.jit.si"
-#HOST="jitsi.vuc.me"
+HOST = "guest.jit.si"
+# HOST="jitsi.vuc.me"
 
-TURNSERVER="turn.guest.jit.si"
-#TURNSERVER="turn.jitsi.vuc.me"
+TURNSERVER = "turn.guest.jit.si"
+# TURNSERVER="turn.jitsi.vuc.me"
+
+ROOMDOMAIN = "meet.jit.si"
+# ROOMDOMAIN="conference.jitsi.vuc.me"
 
-ROOMDOMAIN="meet.jit.si"
-#ROOMDOMAIN="conference.jitsi.vuc.me"
 
 class TrivialMatrixClient:
     def __init__(self, access_token):
@@ -46,38 +47,50 @@ class TrivialMatrixClient:
 
     def getEvent(self):
         while True:
-            url = MATRIXBASE+'events?access_token='+self.access_token+"&timeout=60000"
+            url = (
+                MATRIXBASE
+                + "events?access_token="
+                + self.access_token
+                + "&timeout=60000"
+            )
             if self.token:
-                url += "&from="+self.token
+                url += "&from=" + self.token
             req = grequests.get(url)
             resps = grequests.map([req])
             obj = json.loads(resps[0].content)
-            print("incoming from matrix",obj)
-            if 'end' not in obj:
+            print("incoming from matrix", obj)
+            if "end" not in obj:
                 continue
-            self.token = obj['end']
-            if len(obj['chunk']):
-                return obj['chunk'][0]
+            self.token = obj["end"]
+            if len(obj["chunk"]):
+                return obj["chunk"][0]
 
     def joinRoom(self, roomId):
-        url = MATRIXBASE+'rooms/'+roomId+'/join?access_token='+self.access_token
+        url = MATRIXBASE + "rooms/" + roomId + "/join?access_token=" + self.access_token
         print(url)
-        headers={ 'Content-Type': 'application/json' }
-        req = grequests.post(url, headers=headers, data='{}')
+        headers = {"Content-Type": "application/json"}
+        req = grequests.post(url, headers=headers, data="{}")
         resps = grequests.map([req])
         obj = json.loads(resps[0].content)
-        print("response: ",obj)
+        print("response: ", obj)
 
     def sendEvent(self, roomId, evType, event):
-        url = MATRIXBASE+'rooms/'+roomId+'/send/'+evType+'?access_token='+self.access_token
+        url = (
+            MATRIXBASE
+            + "rooms/"
+            + roomId
+            + "/send/"
+            + evType
+            + "?access_token="
+            + self.access_token
+        )
         print(url)
         print(json.dumps(event))
-        headers={ 'Content-Type': 'application/json' }
+        headers = {"Content-Type": "application/json"}
         req = grequests.post(url, headers=headers, data=json.dumps(event))
         resps = grequests.map([req])
         obj = json.loads(resps[0].content)
-        print("response: ",obj)
-
+        print("response: ", obj)
 
 
 xmppClients = {}
@@ -87,38 +100,39 @@ def matrixLoop():
     while True:
         ev = matrixCli.getEvent()
         print(ev)
-        if ev['type'] == 'm.room.member':
-            print('membership event')
-            if ev['membership'] == 'invite' and ev['state_key'] == MYUSERNAME:
-                roomId = ev['room_id']
+        if ev["type"] == "m.room.member":
+            print("membership event")
+            if ev["membership"] == "invite" and ev["state_key"] == MYUSERNAME:
+                roomId = ev["room_id"]
                 print("joining room %s" % (roomId))
                 matrixCli.joinRoom(roomId)
-        elif ev['type'] == 'm.room.message':
-            if ev['room_id'] in xmppClients:
+        elif ev["type"] == "m.room.message":
+            if ev["room_id"] in xmppClients:
                 print("already have a bridge for that user, ignoring")
                 continue
             print("got message, connecting")
-            xmppClients[ev['room_id']] = TrivialXmppClient(ev['room_id'], ev['user_id'])
-            gevent.spawn(xmppClients[ev['room_id']].xmppLoop)
-        elif ev['type'] == 'm.call.invite':
+            xmppClients[ev["room_id"]] = TrivialXmppClient(ev["room_id"], ev["user_id"])
+            gevent.spawn(xmppClients[ev["room_id"]].xmppLoop)
+        elif ev["type"] == "m.call.invite":
             print("Incoming call")
-            #sdp = ev['content']['offer']['sdp']
-            #print "sdp: %s" % (sdp)
-            #xmppClients[ev['room_id']] = TrivialXmppClient(ev['room_id'], ev['user_id'])
-            #gevent.spawn(xmppClients[ev['room_id']].xmppLoop)
-        elif ev['type'] == 'm.call.answer':
+            # sdp = ev['content']['offer']['sdp']
+            # print "sdp: %s" % (sdp)
+            # xmppClients[ev['room_id']] = TrivialXmppClient(ev['room_id'], ev['user_id'])
+            # gevent.spawn(xmppClients[ev['room_id']].xmppLoop)
+        elif ev["type"] == "m.call.answer":
             print("Call answered")
-            sdp = ev['content']['answer']['sdp']
-            if ev['room_id'] not in xmppClients:
+            sdp = ev["content"]["answer"]["sdp"]
+            if ev["room_id"] not in xmppClients:
                 print("We didn't have a call for that room")
                 continue
             # should probably check call ID too
-            xmppCli = xmppClients[ev['room_id']]
+            xmppCli = xmppClients[ev["room_id"]]
             xmppCli.sendAnswer(sdp)
-        elif ev['type'] == 'm.call.hangup':
-            if ev['room_id'] in xmppClients:
-                xmppClients[ev['room_id']].stop()
-                del xmppClients[ev['room_id']]
+        elif ev["type"] == "m.call.hangup":
+            if ev["room_id"] in xmppClients:
+                xmppClients[ev["room_id"]].stop()
+                del xmppClients[ev["room_id"]]
+
 
 class TrivialXmppClient:
     def __init__(self, matrixRoom, userId):
@@ -132,130 +146,155 @@ class TrivialXmppClient:
 
     def nextRid(self):
         self.rid += 1
-        return '%d' % (self.rid)
+        return "%d" % (self.rid)
 
     def sendIq(self, xml):
-        fullXml = "<body rid='%s' xmlns='http://jabber.org/protocol/httpbind' sid='%s'>%s</body>" % (self.nextRid(), self.sid, xml)
-        #print "\t>>>%s" % (fullXml)
+        fullXml = (
+            "<body rid='%s' xmlns='http://jabber.org/protocol/httpbind' sid='%s'>%s</body>"
+            % (self.nextRid(), self.sid, xml)
+        )
+        # print "\t>>>%s" % (fullXml)
         return self.xmppPoke(fullXml)
 
     def xmppPoke(self, xml):
-        headers = {'Content-Type': 'application/xml'}
+        headers = {"Content-Type": "application/xml"}
         req = grequests.post(HTTPBIND, verify=False, headers=headers, data=xml)
         resps = grequests.map([req])
         obj = BeautifulSoup(resps[0].content)
         return obj
 
     def sendAnswer(self, answer):
-        print("sdp from matrix client",answer)
-        p = subprocess.Popen(['node', 'unjingle/unjingle.js', '--sdp'], stdin=subprocess.PIPE, stdout=subprocess.PIPE)
+        print("sdp from matrix client", answer)
+        p = subprocess.Popen(
+            ["node", "unjingle/unjingle.js", "--sdp"],
+            stdin=subprocess.PIPE,
+            stdout=subprocess.PIPE,
+        )
         jingle, out_err = p.communicate(answer)
         jingle = jingle % {
-            'tojid': self.callfrom,
-            'action': 'session-accept',
-            'initiator': self.callfrom,
-            'responder': self.jid,
-            'sid': self.callsid
+            "tojid": self.callfrom,
+            "action": "session-accept",
+            "initiator": self.callfrom,
+            "responder": self.jid,
+            "sid": self.callsid,
         }
-        print("answer jingle from sdp",jingle)
+        print("answer jingle from sdp", jingle)
         res = self.sendIq(jingle)
-        print("reply from answer: ",res)
+        print("reply from answer: ", res)
 
         self.ssrcs = {}
         jingleSoup = BeautifulSoup(jingle)
-        for cont in jingleSoup.iq.jingle.findAll('content'):
+        for cont in jingleSoup.iq.jingle.findAll("content"):
             if cont.description:
-                self.ssrcs[cont['name']] = cont.description['ssrc']
-        print("my ssrcs:",self.ssrcs)
+                self.ssrcs[cont["name"]] = cont.description["ssrc"]
+        print("my ssrcs:", self.ssrcs)
 
-        gevent.joinall([
-                gevent.spawn(self.advertiseSsrcs)
-        ])
+        gevent.joinall([gevent.spawn(self.advertiseSsrcs)])
 
     def advertiseSsrcs(self):
         time.sleep(7)
         print("SSRC spammer started")
         while self.running:
-            ssrcMsg = "<presence to='%(tojid)s' xmlns='jabber:client'><x xmlns='http://jabber.org/protocol/muc'/><c xmlns='http://jabber.org/protocol/caps' hash='sha-1' node='http://jitsi.org/jitsimeet' ver='0WkSdhFnAUxrz4ImQQLdB80GFlE='/><nick xmlns='http://jabber.org/protocol/nick'>%(nick)s</nick><stats xmlns='http://jitsi.org/jitmeet/stats'><stat name='bitrate_download' value='175'/><stat name='bitrate_upload' value='176'/><stat name='packetLoss_total' value='0'/><stat name='packetLoss_download' value='0'/><stat name='packetLoss_upload' value='0'/></stats><media xmlns='http://estos.de/ns/mjs'><source type='audio' ssrc='%(assrc)s' direction='sendre'/><source type='video' ssrc='%(vssrc)s' direction='sendre'/></media></presence>" % { 'tojid': "%s@%s/%s" % (ROOMNAME, ROOMDOMAIN, self.shortJid), 'nick': self.userId, 'assrc': self.ssrcs['audio'], 'vssrc': self.ssrcs['video'] }
+            ssrcMsg = (
+                "<presence to='%(tojid)s' xmlns='jabber:client'><x xmlns='http://jabber.org/protocol/muc'/><c xmlns='http://jabber.org/protocol/caps' hash='sha-1' node='http://jitsi.org/jitsimeet' ver='0WkSdhFnAUxrz4ImQQLdB80GFlE='/><nick xmlns='http://jabber.org/protocol/nick'>%(nick)s</nick><stats xmlns='http://jitsi.org/jitmeet/stats'><stat name='bitrate_download' value='175'/><stat name='bitrate_upload' value='176'/><stat name='packetLoss_total' value='0'/><stat name='packetLoss_download' value='0'/><stat name='packetLoss_upload' value='0'/></stats><media xmlns='http://estos.de/ns/mjs'><source type='audio' ssrc='%(assrc)s' direction='sendre'/><source type='video' ssrc='%(vssrc)s' direction='sendre'/></media></presence>"
+                % {
+                    "tojid": "%s@%s/%s" % (ROOMNAME, ROOMDOMAIN, self.shortJid),
+                    "nick": self.userId,
+                    "assrc": self.ssrcs["audio"],
+                    "vssrc": self.ssrcs["video"],
+                }
+            )
             res = self.sendIq(ssrcMsg)
-            print("reply from ssrc announce: ",res)
+            print("reply from ssrc announce: ", res)
             time.sleep(10)
 
-
-
     def xmppLoop(self):
         self.matrixCallId = time.time()
-        res = self.xmppPoke("<body rid='%s' xmlns='http://jabber.org/protocol/httpbind' to='%s' xml:lang='en' wait='60' hold='1' content='text/xml; charset=utf-8' ver='1.6' xmpp:version='1.0' xmlns:xmpp='urn:xmpp:xbosh'/>" % (self.nextRid(), HOST))
+        res = self.xmppPoke(
+            "<body rid='%s' xmlns='http://jabber.org/protocol/httpbind' to='%s' xml:lang='en' wait='60' hold='1' content='text/xml; charset=utf-8' ver='1.6' xmpp:version='1.0' xmlns:xmpp='urn:xmpp:xbosh'/>"
+            % (self.nextRid(), HOST)
+        )
 
         print(res)
-        self.sid = res.body['sid']
+        self.sid = res.body["sid"]
         print("sid %s" % (self.sid))
 
-        res = self.sendIq("<auth xmlns='urn:ietf:params:xml:ns:xmpp-sasl' mechanism='ANONYMOUS'/>")
+        res = self.sendIq(
+            "<auth xmlns='urn:ietf:params:xml:ns:xmpp-sasl' mechanism='ANONYMOUS'/>"
+        )
 
-        res = self.xmppPoke("<body rid='%s' xmlns='http://jabber.org/protocol/httpbind' sid='%s' to='%s' xml:lang='en' xmpp:restart='true' xmlns:xmpp='urn:xmpp:xbosh'/>" % (self.nextRid(), self.sid, HOST))
+        res = self.xmppPoke(
+            "<body rid='%s' xmlns='http://jabber.org/protocol/httpbind' sid='%s' to='%s' xml:lang='en' xmpp:restart='true' xmlns:xmpp='urn:xmpp:xbosh'/>"
+            % (self.nextRid(), self.sid, HOST)
+        )
 
-        res = self.sendIq("<iq type='set' id='_bind_auth_2' xmlns='jabber:client'><bind xmlns='urn:ietf:params:xml:ns:xmpp-bind'/></iq>")
+        res = self.sendIq(
+            "<iq type='set' id='_bind_auth_2' xmlns='jabber:client'><bind xmlns='urn:ietf:params:xml:ns:xmpp-bind'/></iq>"
+        )
         print(res)
 
         self.jid = res.body.iq.bind.jid.string
         print("jid: %s" % (self.jid))
-        self.shortJid = self.jid.split('-')[0]
+        self.shortJid = self.jid.split("-")[0]
 
-        res = self.sendIq("<iq type='set' id='_session_auth_2' xmlns='jabber:client'><session xmlns='urn:ietf:params:xml:ns:xmpp-session'/></iq>")
+        res = self.sendIq(
+            "<iq type='set' id='_session_auth_2' xmlns='jabber:client'><session xmlns='urn:ietf:params:xml:ns:xmpp-session'/></iq>"
+        )
 
-        #randomthing = res.body.iq['to']
-        #whatsitpart = randomthing.split('-')[0]
+        # randomthing = res.body.iq['to']
+        # whatsitpart = randomthing.split('-')[0]
 
-        #print "other random bind thing: %s" % (randomthing)
+        # print "other random bind thing: %s" % (randomthing)
 
         # advertise preence to the jitsi room, with our nick
-        res = self.sendIq("<iq type='get' to='%s' xmlns='jabber:client' id='1:sendIQ'><services xmlns='urn:xmpp:extdisco:1'><service host='%s'/></services></iq><presence to='%s@%s/d98f6c40' xmlns='jabber:client'><x xmlns='http://jabber.org/protocol/muc'/><c xmlns='http://jabber.org/protocol/caps' hash='sha-1' node='http://jitsi.org/jitsimeet' ver='0WkSdhFnAUxrz4ImQQLdB80GFlE='/><nick xmlns='http://jabber.org/protocol/nick'>%s</nick></presence>" % (HOST, TURNSERVER, ROOMNAME, ROOMDOMAIN, self.userId))
-        self.muc = {'users': []}
-        for p in res.body.findAll('presence'):
+        res = self.sendIq(
+            "<iq type='get' to='%s' xmlns='jabber:client' id='1:sendIQ'><services xmlns='urn:xmpp:extdisco:1'><service host='%s'/></services></iq><presence to='%s@%s/d98f6c40' xmlns='jabber:client'><x xmlns='http://jabber.org/protocol/muc'/><c xmlns='http://jabber.org/protocol/caps' hash='sha-1' node='http://jitsi.org/jitsimeet' ver='0WkSdhFnAUxrz4ImQQLdB80GFlE='/><nick xmlns='http://jabber.org/protocol/nick'>%s</nick></presence>"
+            % (HOST, TURNSERVER, ROOMNAME, ROOMDOMAIN, self.userId)
+        )
+        self.muc = {"users": []}
+        for p in res.body.findAll("presence"):
             u = {}
-            u['shortJid'] = p['from'].split('/')[1]
+            u["shortJid"] = p["from"].split("/")[1]
             if p.c and p.c.nick:
-                u['nick'] = p.c.nick.string
-            self.muc['users'].append(u)
-        print("muc: ",self.muc)
+                u["nick"] = p.c.nick.string
+            self.muc["users"].append(u)
+        print("muc: ", self.muc)
 
         # wait for stuff
         while True:
             print("waiting...")
             res = self.sendIq("")
-            print("got from stream: ",res)
+            print("got from stream: ", res)
             if res.body.iq:
-                jingles = res.body.iq.findAll('jingle')
+                jingles = res.body.iq.findAll("jingle")
                 if len(jingles):
-                    self.callfrom = res.body.iq['from']
+                    self.callfrom = res.body.iq["from"]
                     self.handleInvite(jingles[0])
-            elif 'type' in res.body and res.body['type'] == 'terminate':
+            elif "type" in res.body and res.body["type"] == "terminate":
                 self.running = False
                 del xmppClients[self.matrixRoom]
                 return
 
     def handleInvite(self, jingle):
-        self.initiator = jingle['initiator']
-        self.callsid = jingle['sid']
-        p = subprocess.Popen(['node', 'unjingle/unjingle.js', '--jingle'], stdin=subprocess.PIPE, stdout=subprocess.PIPE)
-        print("raw jingle invite",str(jingle))
+        self.initiator = jingle["initiator"]
+        self.callsid = jingle["sid"]
+        p = subprocess.Popen(
+            ["node", "unjingle/unjingle.js", "--jingle"],
+            stdin=subprocess.PIPE,
+            stdout=subprocess.PIPE,
+        )
+        print("raw jingle invite", str(jingle))
         sdp, out_err = p.communicate(str(jingle))
-        print("transformed remote offer sdp",sdp)
+        print("transformed remote offer sdp", sdp)
         inviteEvent = {
-            'offer': {
-                'type': 'offer',
-                'sdp': sdp
-            },
-            'call_id': self.matrixCallId,
-            'version': 0,
-            'lifetime': 30000
+            "offer": {"type": "offer", "sdp": sdp},
+            "call_id": self.matrixCallId,
+            "version": 0,
+            "lifetime": 30000,
         }
-        matrixCli.sendEvent(self.matrixRoom, 'm.call.invite', inviteEvent)
+        matrixCli.sendEvent(self.matrixRoom, "m.call.invite", inviteEvent)
+
 
 matrixCli = TrivialMatrixClient(ACCESS_TOKEN)  # Undefined name
 
-gevent.joinall([
-    gevent.spawn(matrixLoop)
-])
-
+gevent.joinall([gevent.spawn(matrixLoop)])