Remove commas and None, to please pep8

* Rewrite more internal IS urls.

.buildkite/docker-compose.py27.pg94.yaml

@@ -0,0 +1,21 @@
+version: '3.1'
+
+services:
+
+  postgres:
+    image: postgres:9.4
+    environment:
+      POSTGRES_PASSWORD: postgres
+
+  testenv:
+    image: python:2.7
+    depends_on:
+      - postgres
+    env_file: .env
+    environment:
+      SYNAPSE_POSTGRES_HOST: postgres
+      SYNAPSE_POSTGRES_USER: postgres
+      SYNAPSE_POSTGRES_PASSWORD: postgres
+    working_dir: /app
+    volumes:
+      - ..:/app

.buildkite/docker-compose.py27.pg95.yaml

@@ -0,0 +1,21 @@
+version: '3.1'
+
+services:
+
+  postgres:
+    image: postgres:9.5
+    environment:
+      POSTGRES_PASSWORD: postgres
+
+  testenv:
+    image: python:2.7
+    depends_on:
+      - postgres
+    env_file: .env
+    environment:
+      SYNAPSE_POSTGRES_HOST: postgres
+      SYNAPSE_POSTGRES_USER: postgres
+      SYNAPSE_POSTGRES_PASSWORD: postgres
+    working_dir: /app
+    volumes:
+      - ..:/app

.buildkite/docker-compose.py35.pg94.yaml

@@ -0,0 +1,21 @@
+version: '3.1'
+
+services:
+
+  postgres:
+    image: postgres:9.4
+    environment:
+      POSTGRES_PASSWORD: postgres
+
+  testenv:
+    image: python:3.5
+    depends_on:
+      - postgres
+    env_file: .env
+    environment:
+      SYNAPSE_POSTGRES_HOST: postgres
+      SYNAPSE_POSTGRES_USER: postgres
+      SYNAPSE_POSTGRES_PASSWORD: postgres
+    working_dir: /app
+    volumes:
+      - ..:/app

.buildkite/format_tap.py

@@ -1,33 +0,0 @@
-import sys
-from tap.parser import Parser
-from tap.line import Result, Unknown, Diagnostic
-
-out = ["### TAP Output for " + sys.argv[2]]
-
-p = Parser()
-
-in_error = False
-
-for line in p.parse_file(sys.argv[1]):
-    if isinstance(line, Result):
-        if in_error:
-            out.append("")
-            out.append("</pre></code></details>")
-            out.append("")
-            out.append("----")
-            out.append("")
-        in_error = False
-
-        if not line.ok and not line.todo:
-            in_error = True
-
-            out.append("FAILURE Test #%d: ``%s``" % (line.number, line.description))
-            out.append("")
-            out.append("<details><summary>Show log</summary><code><pre>")
-
-    elif isinstance(line, Diagnostic) and in_error:
-        out.append(line.text)
-
-if out:
-    for line in out[:-3]:
-        print(line)

.buildkite/pipeline.yml

@@ -2,11 +2,10 @@ env:
   CODECOV_TOKEN: "2dd7eb9b-0eda-45fe-a47c-9b5ac040045f"
 
 steps:
-
   - command:
       - "python -m pip install tox"
-      - "tox -e check_codestyle"
-    label: "\U0001F9F9 Check Style"
+      - "tox -e pep8"
+    label: "\U0001F9F9 PEP-8"
     plugins:
       - docker#v3.0.1:
           image: "python:3.6"
@@ -27,15 +26,7 @@ steps:
       - docker#v3.0.1:
           image: "python:3.6"
 
-  - command:
-      - "python -m pip install tox"
-      - "scripts-dev/check-newsfragment"
-    label: ":newspaper: Newsfile"
-    branches: "!master !develop !release-*"
-    plugins:
-      - docker#v3.0.1:
-          image: "python:3.6"
-          propagate-environment: true
+  - wait
 
   - command:
       - "python -m pip install tox"
@@ -45,25 +36,16 @@ steps:
       - docker#v3.0.1:
           image: "python:3.6"
 
-  - wait
-
-
   - command:
       - "python -m pip install tox"
-      - "tox -e py35-old,codecov"
-    label: ":python: 3.5 / SQLite / Old Deps"
+      - "tox -e py27,codecov"
+    label: ":python: 2.7 / SQLite"
     env:
       TRIAL_FLAGS: "-j 2"
     plugins:
       - docker#v3.0.1:
-          image: "python:3.5"
+          image: "python:2.7"
           propagate-environment: true
-    retry:
-      automatic:
-        - exit_status: -1
-          limit: 2
-        - exit_status: 2
-          limit: 2
 
   - command:
       - "python -m pip install tox"
@@ -75,12 +57,6 @@ steps:
       - docker#v3.0.1:
           image: "python:3.5"
           propagate-environment: true
-    retry:
-      automatic:
-        - exit_status: -1
-          limit: 2
-        - exit_status: 2
-          limit: 2
 
   - command:
       - "python -m pip install tox"
@@ -92,12 +68,6 @@ steps:
       - docker#v3.0.1:
           image: "python:3.6"
           propagate-environment: true
-    retry:
-      automatic:
-        - exit_status: -1
-          limit: 2
-        - exit_status: 2
-          limit: 2
 
   - command:
       - "python -m pip install tox"
@@ -109,12 +79,50 @@ steps:
       - docker#v3.0.1:
           image: "python:3.7"
           propagate-environment: true
-    retry:
-      automatic:
-        - exit_status: -1
-          limit: 2
-        - exit_status: 2
-          limit: 2
+
+  - command:
+      - "python -m pip install tox"
+      - "tox -e py27-old,codecov"
+    label: ":python: 2.7 / SQLite / Old Deps"
+    env:
+      TRIAL_FLAGS: "-j 2"
+    plugins:
+      - docker#v3.0.1:
+          image: "python:2.7"
+          propagate-environment: true
+
+  - label: ":python: 2.7 / :postgres: 9.4"
+    env:
+      TRIAL_FLAGS: "-j 4"
+    command:
+      - "bash -c 'python -m pip install tox && python -m tox -e py27-postgres,codecov'"
+    plugins:
+      - docker-compose#v2.1.0:
+          run: testenv
+          config:
+            - .buildkite/docker-compose.py27.pg94.yaml
+
+  - label: ":python: 2.7 / :postgres: 9.5"
+    env:
+      TRIAL_FLAGS: "-j 4"
+    command:
+      - "bash -c 'python -m pip install tox && python -m tox -e py27-postgres,codecov'"
+    plugins:
+      - docker-compose#v2.1.0:
+          run: testenv
+          config:
+            - .buildkite/docker-compose.py27.pg95.yaml
+
+  - label: ":python: 3.5 / :postgres: 9.4"
+    env:
+      TRIAL_FLAGS: "-j 4"
+    command:
+      - "bash -c 'python -m pip install tox && python -m tox -e py35-postgres,codecov'"
+    plugins:
+      - docker-compose#v2.1.0:
+          run: testenv
+          config:
+            - .buildkite/docker-compose.py35.pg94.yaml
 
   - label: ":python: 3.5 / :postgres: 9.5"
     env:
@@ -126,12 +134,6 @@ steps:
           run: testenv
           config:
             - .buildkite/docker-compose.py35.pg95.yaml
-    retry:
-      automatic:
-        - exit_status: -1
-          limit: 2
-        - exit_status: 2
-          limit: 2
 
   - label: ":python: 3.7 / :postgres: 9.5"
     env:
@@ -143,12 +145,6 @@ steps:
           run: testenv
           config:
             - .buildkite/docker-compose.py37.pg95.yaml
-    retry:
-      automatic:
-        - exit_status: -1
-          limit: 2
-        - exit_status: 2
-          limit: 2
 
   - label: ":python: 3.7 / :postgres: 11"
     env:
@@ -160,67 +156,3 @@ steps:
           run: testenv
           config:
             - .buildkite/docker-compose.py37.pg11.yaml
-    retry:
-      automatic:
-        - exit_status: -1
-          limit: 2
-        - exit_status: 2
-          limit: 2
-
-
-  - label: "SyTest - :python: 3.5 / SQLite / Monolith"
-    agents:
-      queue: "medium"
-    command:
-      - "bash .buildkite/merge_base_branch.sh"
-      - "bash .buildkite/synapse_sytest.sh"
-    plugins:
-      - docker#v3.0.1:
-          image: "matrixdotorg/sytest-synapse:py35"
-          propagate-environment: true
-    retry:
-      automatic:
-        - exit_status: -1
-          limit: 2
-        - exit_status: 2
-          limit: 2
-
-  - label: "SyTest - :python: 3.5 / :postgres: 9.6 / Monolith"
-    agents:
-      queue: "medium"
-    env:
-      POSTGRES: "1"
-    command:
-      - "bash .buildkite/merge_base_branch.sh"
-      - "bash .buildkite/synapse_sytest.sh"
-    plugins:
-      - docker#v3.0.1:
-          image: "matrixdotorg/sytest-synapse:py35"
-          propagate-environment: true
-    retry:
-      automatic:
-        - exit_status: -1
-          limit: 2
-        - exit_status: 2
-          limit: 2
-
-  - label: "SyTest - :python: 3.5 / :postgres: 9.6 / Workers"
-    agents:
-      queue: "medium"
-    env:
-      POSTGRES: "1"
-      WORKERS: "1"
-    command:
-      - "bash .buildkite/merge_base_branch.sh"
-      - "bash .buildkite/synapse_sytest.sh"
-    plugins:
-      - docker#v3.0.1:
-          image: "matrixdotorg/sytest-synapse:py35"
-          propagate-environment: true
-    soft_fail: true
-    retry:
-      automatic:
-        - exit_status: -1
-          limit: 2
-        - exit_status: 2
-          limit: 2

.buildkite/synapse_sytest.sh

@@ -1,145 +0,0 @@
-#!/bin/bash
-#
-# Fetch sytest, and then run the tests for synapse. The entrypoint for the
-# sytest-synapse docker images.
-
-set -ex
-
-if [ -n "$BUILDKITE" ]
-then
-    SYNAPSE_DIR=`pwd`
-else
-    SYNAPSE_DIR="/src"
-fi
-
-# Attempt to find a sytest to use.
-# If /sytest exists, it means that a SyTest checkout has been mounted into the Docker image.
-if [ -d "/sytest" ]; then
-    # If the user has mounted in a SyTest checkout, use that.
-    echo "Using local sytests..."
-
-    # create ourselves a working directory and dos2unix some scripts therein
-    mkdir -p /work/jenkins
-    for i in install-deps.pl run-tests.pl tap-to-junit-xml.pl jenkins/prep_sytest_for_postgres.sh; do
-        dos2unix -n "/sytest/$i" "/work/$i"
-    done
-    ln -sf /sytest/tests /work
-    ln -sf /sytest/keys /work
-    SYTEST_LIB="/sytest/lib"
-else
-    if [ -n "BUILDKITE_BRANCH" ]
-    then
-        branch_name=$BUILDKITE_BRANCH
-    else
-        # Otherwise, try and find out what the branch that the Synapse checkout is using. Fall back to develop if it's not a branch.
-        branch_name="$(git --git-dir=/src/.git symbolic-ref HEAD 2>/dev/null)" || branch_name="develop"
-    fi
-
-    # Try and fetch the branch
-    echo "Trying to get same-named sytest branch..."
-    wget -q https://github.com/matrix-org/sytest/archive/$branch_name.tar.gz -O sytest.tar.gz || {
-        # Probably a 404, fall back to develop
-        echo "Using develop instead..."
-        wget -q https://github.com/matrix-org/sytest/archive/develop.tar.gz -O sytest.tar.gz
-    }
-
-    mkdir -p /work
-    tar -C /work --strip-components=1 -xf sytest.tar.gz
-    SYTEST_LIB="/work/lib"
-fi
-
-cd /work
-
-# PostgreSQL setup
-if [ -n "$POSTGRES" ]
-then
-    export PGUSER=postgres
-    export POSTGRES_DB_1=pg1
-    export POSTGRES_DB_2=pg2
-
-    # Start the database
-    su -c 'eatmydata /usr/lib/postgresql/9.6/bin/pg_ctl -w -D /var/lib/postgresql/data start' postgres
-
-    # Use the Jenkins script to write out the configuration for a PostgreSQL using Synapse
-    jenkins/prep_sytest_for_postgres.sh
-
-    # Make the test databases for the two Synapse servers that will be spun up
-    su -c 'psql -c "CREATE DATABASE pg1;"' postgres
-    su -c 'psql -c "CREATE DATABASE pg2;"' postgres
-
-fi
-
-if [ -n "$OFFLINE" ]; then
-    # if we're in offline mode, just put synapse into the virtualenv, and
-    # hope that the deps are up-to-date.
-    #
-    # (`pip install -e` likes to reinstall setuptools even if it's already installed,
-    # so we just run setup.py explicitly.)
-    #
-    (cd $SYNAPSE_DIR && /venv/bin/python setup.py -q develop)
-else
-    # We've already created the virtualenv, but lets double check we have all
-    # deps.
-    /venv/bin/pip install -q --upgrade --no-cache-dir -e $SYNAPSE_DIR
-    /venv/bin/pip install -q --upgrade --no-cache-dir \
-        lxml psycopg2 coverage codecov tap.py
-
-    # Make sure all Perl deps are installed -- this is done in the docker build
-    # so will only install packages added since the last Docker build
-    ./install-deps.pl
-fi
-
-
-# Run the tests
->&2 echo "+++ Running tests"
-
-RUN_TESTS=(
-    perl -I "$SYTEST_LIB" ./run-tests.pl --python=/venv/bin/python --synapse-directory=$SYNAPSE_DIR --coverage -O tap --all
-)
-
-TEST_STATUS=0
-
-if [ -n "$WORKERS" ]; then
-    RUN_TESTS+=(-I Synapse::ViaHaproxy --dendron-binary=/pydron.py)
-else
-    RUN_TESTS+=(-I Synapse)
-fi
-
-"${RUN_TESTS[@]}" "$@" > results.tap || TEST_STATUS=$?
-
-if [ $TEST_STATUS -ne 0 ]; then
-    >&2 echo -e "run-tests \e[31mFAILED\e[0m: exit code $TEST_STATUS"
-else
-    >&2 echo -e "run-tests \e[32mPASSED\e[0m"
-fi
-
->&2 echo "--- Copying assets"
-
-# Copy out the logs
-mkdir -p /logs
-cp results.tap /logs/results.tap
-rsync --ignore-missing-args  --min-size=1B -av server-0 server-1 /logs --include "*/" --include="*.log.*" --include="*.log" --exclude="*"
-
-# Upload coverage to codecov and upload files, if running on Buildkite
-if [ -n "$BUILDKITE" ]
-then
-    /venv/bin/coverage combine || true
-    /venv/bin/coverage xml || true
-    /venv/bin/codecov -X gcov -f coverage.xml
-
-    wget -O buildkite.tar.gz https://github.com/buildkite/agent/releases/download/v3.13.0/buildkite-agent-linux-amd64-3.13.0.tar.gz
-    tar xvf buildkite.tar.gz
-    chmod +x ./buildkite-agent
-
-    # Upload the files
-    ./buildkite-agent artifact upload "/logs/**/*.log*"
-    ./buildkite-agent artifact upload "/logs/results.tap"
-
-    if [ $TEST_STATUS -ne 0 ]; then
-        # Annotate, if failure
-        /venv/bin/python $SYNAPSE_DIR/.buildkite/format_tap.py /logs/results.tap "$BUILDKITE_LABEL" | ./buildkite-agent annotate --style="error" --context="$BUILDKITE_LABEL"
-    fi
-fi
-
-
-exit $TEST_STATUS

.circleci/config.yml

@@ -4,23 +4,160 @@ jobs:
     machine: true
     steps:
       - checkout
-      - run: docker build -f docker/Dockerfile --label gitsha1=${CIRCLE_SHA1} -t matrixdotorg/synapse:${CIRCLE_TAG} -t matrixdotorg/synapse:${CIRCLE_TAG}-py3 .
+      - run: docker build -f docker/Dockerfile --label gitsha1=${CIRCLE_SHA1} -t matrixdotorg/synapse:${CIRCLE_TAG}-py2 .
+      - run: docker build -f docker/Dockerfile --label gitsha1=${CIRCLE_SHA1} -t matrixdotorg/synapse:${CIRCLE_TAG} -t matrixdotorg/synapse:${CIRCLE_TAG}-py3 --build-arg PYTHON_VERSION=3.6 .
       - run: docker login --username $DOCKER_HUB_USERNAME --password $DOCKER_HUB_PASSWORD
       - run: docker push matrixdotorg/synapse:${CIRCLE_TAG}
+      - run: docker push matrixdotorg/synapse:${CIRCLE_TAG}-py2
       - run: docker push matrixdotorg/synapse:${CIRCLE_TAG}-py3
   dockerhubuploadlatest:
     machine: true
     steps:
       - checkout
-      - run: docker build -f docker/Dockerfile --label gitsha1=${CIRCLE_SHA1} -t matrixdotorg/synapse:latest -t matrixdotorg/synapse:latest-py3 .
+      - run: docker build -f docker/Dockerfile --label gitsha1=${CIRCLE_SHA1} -t matrixdotorg/synapse:latest-py2 .
+      - run: docker build -f docker/Dockerfile --label gitsha1=${CIRCLE_SHA1} -t matrixdotorg/synapse:latest -t matrixdotorg/synapse:latest-py3 --build-arg PYTHON_VERSION=3.6 .
       - run: docker login --username $DOCKER_HUB_USERNAME --password $DOCKER_HUB_PASSWORD
       - run: docker push matrixdotorg/synapse:latest
+      - run: docker push matrixdotorg/synapse:latest-py2
       - run: docker push matrixdotorg/synapse:latest-py3
+  sytestpy2:
+    docker:
+      - image: matrixdotorg/sytest-synapsepy2
+    working_directory: /src
+    steps:
+      - checkout
+      - run: /synapse_sytest.sh
+      - store_artifacts:
+          path: /logs
+          destination: logs
+      - store_test_results:
+          path: /logs
+  sytestpy2postgres:
+    docker:
+      - image: matrixdotorg/sytest-synapsepy2
+    working_directory: /src
+    steps:
+      - checkout
+      - run: POSTGRES=1 /synapse_sytest.sh
+      - store_artifacts:
+          path: /logs
+          destination: logs
+      - store_test_results:
+          path: /logs
+  sytestpy2merged:
+    docker:
+      - image: matrixdotorg/sytest-synapsepy2
+    working_directory: /src
+    steps:
+      - checkout
+      - run: bash .circleci/merge_base_branch.sh
+      - run: /synapse_sytest.sh
+      - store_artifacts:
+          path: /logs
+          destination: logs
+      - store_test_results:
+          path: /logs
+  sytestpy2postgresmerged:
+    docker:
+      - image: matrixdotorg/sytest-synapsepy2
+    working_directory: /src
+    steps:
+      - checkout
+      - run: bash .circleci/merge_base_branch.sh
+      - run: POSTGRES=1 /synapse_sytest.sh
+      - store_artifacts:
+          path: /logs
+          destination: logs
+      - store_test_results:
+          path: /logs
+
+  sytestpy3:
+    docker:
+      - image: matrixdotorg/sytest-synapsepy3
+    working_directory: /src
+    steps:
+      - checkout
+      - run: /synapse_sytest.sh
+      - store_artifacts:
+          path: /logs
+          destination: logs
+      - store_test_results:
+          path: /logs
+  sytestpy3postgres:
+    docker:
+      - image: matrixdotorg/sytest-synapsepy3
+    working_directory: /src
+    steps:
+      - checkout
+      - run: POSTGRES=1 /synapse_sytest.sh
+      - store_artifacts:
+          path: /logs
+          destination: logs
+      - store_test_results:
+          path: /logs
+  sytestpy3merged:
+    docker:
+      - image: matrixdotorg/sytest-synapsepy3
+    working_directory: /src
+    steps:
+      - checkout
+      - run: bash .circleci/merge_base_branch.sh
+      - run: /synapse_sytest.sh
+      - store_artifacts:
+          path: /logs
+          destination: logs
+      - store_test_results:
+          path: /logs
+  sytestpy3postgresmerged:
+    docker:
+      - image: matrixdotorg/sytest-synapsepy3
+    working_directory: /src
+    steps:
+      - checkout
+      - run: bash .circleci/merge_base_branch.sh
+      - run: POSTGRES=1 /synapse_sytest.sh
+      - store_artifacts:
+          path: /logs
+          destination: logs
+      - store_test_results:
+          path: /logs
 
 workflows:
   version: 2
   build:
     jobs:
+      - sytestpy2:
+          filters:
+            branches:
+              only: /develop|master|release-.*/
+      - sytestpy2postgres:
+          filters:
+            branches:
+              only: /develop|master|release-.*/
+      - sytestpy3:
+          filters:
+            branches:
+              only: /develop|master|release-.*/
+      - sytestpy3postgres:
+          filters:
+            branches:
+              only: /develop|master|release-.*/
+      - sytestpy2merged:
+          filters:
+            branches:
+              ignore: /develop|master|release-.*/
+      - sytestpy2postgresmerged:
+          filters:
+            branches:
+              ignore: /develop|master|release-.*/
+      - sytestpy3merged:
+          filters:
+            branches:
+              ignore: /develop|master|release-.*/
+      - sytestpy3postgresmerged:
+          filters:
+            branches:
+              ignore: /develop|master|release-.*/
       - dockerhubuploadrelease:
           filters:
             tags:

.circleci/merge_base_branch.sh

@@ -1,21 +1,22 @@
 #!/usr/bin/env bash
 
-set -ex
+set -e
 
-if [[ "$BUILDKITE_BRANCH" =~ ^(develop|master|dinsic|shhs|release-.*)$ ]]; then
-    echo "Not merging forward, as this is a release branch"
-    exit 0
-fi
+# CircleCI doesn't give CIRCLE_PR_NUMBER in the environment for non-forked PRs. Wonderful.
+# In this case, we just need to do some ~shell magic~ to strip it out of the PULL_REQUEST URL.
+echo 'export CIRCLE_PR_NUMBER="${CIRCLE_PR_NUMBER:-${CIRCLE_PULL_REQUEST##*/}}"' >> $BASH_ENV
+source $BASH_ENV
 
-if [[ -z $BUILDKITE_PULL_REQUEST_BASE_BRANCH ]]; then
-    echo "Not a pull request, or hasn't had a PR opened yet..."
+if [[ -z "${CIRCLE_PR_NUMBER}" ]]
+then
+    echo "Can't figure out what the PR number is! Assuming merge target is develop."
 
     # It probably hasn't had a PR opened yet. Since all PRs land on develop, we
     # can probably assume it's based on it and will be merged into it.
     GITBASE="develop"
 else
     # Get the reference, using the GitHub API
-    GITBASE=$BUILDKITE_PULL_REQUEST_BASE_BRANCH
+    GITBASE=`wget -O- https://api.github.com/repos/matrix-org/synapse/pulls/${CIRCLE_PR_NUMBER} | jq -r '.base.ref'`
 fi
 
 # Show what we are before

.dockerignore

@@ -1,13 +1,9 @@
-# ignore everything by default
-*
-
-# things to include
-!docker
-!scripts
-!synapse
-!MANIFEST.in
-!README.rst
-!setup.py
-!synctl
-
-**/__pycache__
+Dockerfile
+.travis.yml
+.gitignore
+demo/etc
+tox.ini
+.git/*
+.tox/*
+debian/matrix-synapse/
+debian/matrix-synapse-*/

.github/FUNDING.yml

@@ -1,4 +0,0 @@
-# One username per supported platform and one custom link
-patreon: matrixdotorg
-liberapay: matrixdotorg
-custom: https://paypal.me/matrixdotorg

.github/ISSUE_TEMPLATE/SUPPORT_REQUEST.md

@@ -4,7 +4,6 @@ about: I need support for Synapse
 
 ---
 
-Please don't file github issues asking for support.
+# Please ask for support in [**#matrix:matrix.org**](https://matrix.to/#/#matrix:matrix.org)
 
-Instead, please join [`#synapse:matrix.org`](https://matrix.to/#/#synapse:matrix.org)
-(from a matrix.org account if necessary), and ask there.
+## Don't file an issue as a support request.

.github/SUPPORT.md

@@ -1,3 +1,3 @@
-[**#synapse:matrix.org**](https://matrix.to/#/#synapse:matrix.org) is the official support room for
-Synapse, and can be accessed by any client from https://matrix.org/docs/projects/try-matrix-now.html.
-Please ask for support there, rather than filing github issues.
+[**#matrix:matrix.org**](https://matrix.to/#/#matrix:matrix.org) is the official support room for Matrix, and can be accessed by any client from https://matrix.org/docs/projects/try-matrix-now.html 
+
+It can also be access via IRC bridge at irc://irc.freenode.net/matrix or on the web here: https://webchat.freenode.net/?channels=matrix

AUTHORS.rst

@@ -72,6 +72,3 @@ Jason Robinson <jasonr at matrix.org>
 
 Joseph Weston <joseph at weston.cloud>
  + Add admin API for querying HS version
-
-Benjamin Saunders <ben.e.saunders at gmail dot com>
- * Documentation improvements

CHANGES.md

@@ -1,497 +1,3 @@
-Synapse 1.1.0rc1 (2019-07-02)
-=============================
-
-As of v1.1.0, Synapse no longer supports Python 2, nor Postgres version 9.4.
-See the [upgrade notes](UPGRADE.rst#upgrading-to-v110) for more details.
-
-Features
---------
-
-- Added possibilty to disable local password authentication. Contributed by Daniel Hoffend. ([\#5092](https://github.com/matrix-org/synapse/issues/5092))
-- Add monthly active users to phonehome stats. ([\#5252](https://github.com/matrix-org/synapse/issues/5252))
-- Allow expired user to trigger renewal email sending manually. ([\#5363](https://github.com/matrix-org/synapse/issues/5363))
-- Statistics on forward extremities per room are now exposed via Prometheus. ([\#5384](https://github.com/matrix-org/synapse/issues/5384), [\#5458](https://github.com/matrix-org/synapse/issues/5458), [\#5461](https://github.com/matrix-org/synapse/issues/5461))
-- Add --no-daemonize option to run synapse in the foreground, per issue #4130. Contributed by Soham Gumaste. ([\#5412](https://github.com/matrix-org/synapse/issues/5412), [\#5587](https://github.com/matrix-org/synapse/issues/5587))
-- Fully support SAML2 authentication. Contributed by [Alexander Trost](https://github.com/galexrt) - thank you! ([\#5422](https://github.com/matrix-org/synapse/issues/5422))
-- Allow server admins to define implementations of extra rules for allowing or denying incoming events. ([\#5440](https://github.com/matrix-org/synapse/issues/5440), [\#5474](https://github.com/matrix-org/synapse/issues/5474), [\#5477](https://github.com/matrix-org/synapse/issues/5477))
-- Add support for handling pagination APIs on client reader worker. ([\#5505](https://github.com/matrix-org/synapse/issues/5505), [\#5513](https://github.com/matrix-org/synapse/issues/5513), [\#5531](https://github.com/matrix-org/synapse/issues/5531))
-- Improve help and cmdline option names for --generate-config options. ([\#5512](https://github.com/matrix-org/synapse/issues/5512))
-- Allow configuration of the path used for ACME account keys. ([\#5516](https://github.com/matrix-org/synapse/issues/5516), [\#5521](https://github.com/matrix-org/synapse/issues/5521), [\#5522](https://github.com/matrix-org/synapse/issues/5522))
-- Add --data-dir and --open-private-ports options. ([\#5524](https://github.com/matrix-org/synapse/issues/5524))
-- Split public rooms directory auth config in two settings, in order to manage client auth independently from the federation part of it. Obsoletes the "restrict_public_rooms_to_local_users" configuration setting. If "restrict_public_rooms_to_local_users" is set in the config, Synapse will act as if both new options are enabled, i.e. require authentication through the client API and deny federation requests. ([\#5534](https://github.com/matrix-org/synapse/issues/5534))
-- The minimum TLS version used for outgoing federation requests can now be set with `federation_client_minimum_tls_version`. ([\#5550](https://github.com/matrix-org/synapse/issues/5550))
-- Optimise devices changed query to not pull unnecessary rows from the database, reducing database load. ([\#5559](https://github.com/matrix-org/synapse/issues/5559))
-- Add new metrics for number of forward extremities being persisted and number of state groups involved in resolution. ([\#5476](https://github.com/matrix-org/synapse/issues/5476))
-
-Bugfixes
---------
-
-- Fix bug processing incoming events over federation if call to `/get_missing_events` fails. ([\#5042](https://github.com/matrix-org/synapse/issues/5042))
-- Prevent more than one room upgrade happening simultaneously on the same room. ([\#5051](https://github.com/matrix-org/synapse/issues/5051))
-- Fix a bug where running synapse_port_db would cause the account validity feature to fail because it didn't set the type of the email_sent column to boolean. ([\#5325](https://github.com/matrix-org/synapse/issues/5325))
-- Warn about disabling email-based password resets when a reset occurs, and remove warning when someone attempts a phone-based reset. ([\#5387](https://github.com/matrix-org/synapse/issues/5387))
-- Fix email notifications for unnamed rooms with multiple people. ([\#5388](https://github.com/matrix-org/synapse/issues/5388))
-- Fix exceptions in federation reader worker caused by attempting to renew attestations, which should only happen on master worker. ([\#5389](https://github.com/matrix-org/synapse/issues/5389))
-- Fix handling of failures fetching remote content to not log failures as exceptions. ([\#5390](https://github.com/matrix-org/synapse/issues/5390))
-- Fix a bug where deactivated users could receive renewal emails if the account validity feature is on. ([\#5394](https://github.com/matrix-org/synapse/issues/5394))
-- Fix missing invite state after exchanging 3PID invites over federaton. ([\#5464](https://github.com/matrix-org/synapse/issues/5464))
-- Fix intermittent exceptions on Apple hardware. Also fix bug that caused database activity times to be under-reported in log lines. ([\#5498](https://github.com/matrix-org/synapse/issues/5498))
-- Fix logging error when a tampered event is detected. ([\#5500](https://github.com/matrix-org/synapse/issues/5500))
-- Fix bug where clients could tight loop calling `/sync` for a period. ([\#5507](https://github.com/matrix-org/synapse/issues/5507))
-- Fix bug with `jinja2` preventing Synapse from starting. Users who had this problem should now simply need to run `pip install matrix-synapse`. ([\#5514](https://github.com/matrix-org/synapse/issues/5514))
-- Fix a regression where homeservers on private IP addresses were incorrectly blacklisted. ([\#5523](https://github.com/matrix-org/synapse/issues/5523))
-- Fixed m.login.jwt using unregistred user_id and added pyjwt>=1.6.4 as jwt conditional dependencies. Contributed by Pau Rodriguez-Estivill. ([\#5555](https://github.com/matrix-org/synapse/issues/5555), [\#5586](https://github.com/matrix-org/synapse/issues/5586))
-- Fix a bug that would cause invited users to receive several emails for a single 3PID invite in case the inviter is rate limited. ([\#5576](https://github.com/matrix-org/synapse/issues/5576))
-
-
-Updates to the Docker image
----------------------------
-- Add ability to change Docker containers [timezone](https://en.wikipedia.org/wiki/List_of_tz_database_time_zones) with the `TZ` variable. ([\#5383](https://github.com/matrix-org/synapse/issues/5383))
-- Update docker image to use Python 3.7. ([\#5546](https://github.com/matrix-org/synapse/issues/5546))
-- Deprecate the use of environment variables for configuration, and make the use of a static configuration the default. ([\#5561](https://github.com/matrix-org/synapse/issues/5561), [\#5562](https://github.com/matrix-org/synapse/issues/5562), [\#5566](https://github.com/matrix-org/synapse/issues/5566), [\#5567](https://github.com/matrix-org/synapse/issues/5567))
-- Increase default log level for docker image to INFO. It can still be changed by editing the generated log.config file. ([\#5547](https://github.com/matrix-org/synapse/issues/5547))
-- Send synapse logs to the docker logging system, by default. ([\#5565](https://github.com/matrix-org/synapse/issues/5565))
-- Open the non-TLS port by default. ([\#5568](https://github.com/matrix-org/synapse/issues/5568))
-- Fix failure to start under docker with SAML support enabled. ([\#5490](https://github.com/matrix-org/synapse/issues/5490))
-- Use a sensible location for data files when generating a config file. ([\#5563](https://github.com/matrix-org/synapse/issues/5563))
-
-
-Deprecations and Removals
--------------------------
-
-- Python 2.7 is no longer a supported platform. Synapse now requires Python 3.5+ to run. ([\#5425](https://github.com/matrix-org/synapse/issues/5425))
-- PostgreSQL 9.4 is no longer supported. Synapse requires Postgres 9.5+ or above for Postgres support. ([\#5448](https://github.com/matrix-org/synapse/issues/5448))
-- Remove support for cpu_affinity setting. ([\#5525](https://github.com/matrix-org/synapse/issues/5525))
-
-
-Improved Documentation
-----------------------
-- Improve README section on performance troubleshooting. ([\#4276](https://github.com/matrix-org/synapse/issues/4276))
-- Add information about how to install and run `black` on the codebase to code_style.rst. ([\#5537](https://github.com/matrix-org/synapse/issues/5537))
-- Improve install docs on choosing server_name. ([\#5558](https://github.com/matrix-org/synapse/issues/5558))
-
-
-Internal Changes
-----------------
-
-- Add logging to 3pid invite signature verification. ([\#5015](https://github.com/matrix-org/synapse/issues/5015))
-- Update example haproxy config to a more compatible setup. ([\#5313](https://github.com/matrix-org/synapse/issues/5313))
-- Track deactivated accounts in the database. ([\#5378](https://github.com/matrix-org/synapse/issues/5378), [\#5465](https://github.com/matrix-org/synapse/issues/5465), [\#5493](https://github.com/matrix-org/synapse/issues/5493))
-- Clean up code for sending federation EDUs. ([\#5381](https://github.com/matrix-org/synapse/issues/5381))
-- Add a sponsor button to the repo. ([\#5382](https://github.com/matrix-org/synapse/issues/5382), [\#5386](https://github.com/matrix-org/synapse/issues/5386))
-- Don't log non-200 responses from federation queries as exceptions. ([\#5383](https://github.com/matrix-org/synapse/issues/5383))
-- Update Python syntax in contrib/ to Python 3. ([\#5446](https://github.com/matrix-org/synapse/issues/5446))
-- Update federation_client dev script to support `.well-known` and work with python3. ([\#5447](https://github.com/matrix-org/synapse/issues/5447))
-- SyTest has been moved to Buildkite. ([\#5459](https://github.com/matrix-org/synapse/issues/5459))
-- Demo script now uses python3. ([\#5460](https://github.com/matrix-org/synapse/issues/5460))
-- Synapse can now handle RestServlets that return coroutines. ([\#5475](https://github.com/matrix-org/synapse/issues/5475), [\#5585](https://github.com/matrix-org/synapse/issues/5585))
-- The demo servers talk to each other again. ([\#5478](https://github.com/matrix-org/synapse/issues/5478))
-- Add an EXPERIMENTAL config option to try and periodically clean up extremities by sending dummy events. ([\#5480](https://github.com/matrix-org/synapse/issues/5480))
-- Synapse's codebase is now formatted by `black`. ([\#5482](https://github.com/matrix-org/synapse/issues/5482))
-- Some cleanups and sanity-checking in the CPU and database metrics. ([\#5499](https://github.com/matrix-org/synapse/issues/5499))
-- Improve email notification logging. ([\#5502](https://github.com/matrix-org/synapse/issues/5502))
-- Fix "Unexpected entry in 'full_schemas'" log warning. ([\#5509](https://github.com/matrix-org/synapse/issues/5509))
-- Improve logging when generating config files. ([\#5510](https://github.com/matrix-org/synapse/issues/5510))
-- Refactor and clean up Config parser for maintainability. ([\#5511](https://github.com/matrix-org/synapse/issues/5511))
-- Make the config clearer in that email.template_dir is relative to the Synapse's root directory, not the `synapse/` folder within it. ([\#5543](https://github.com/matrix-org/synapse/issues/5543))
-- Update v1.0.0 release changelog to include more information about changes to password resets. ([\#5545](https://github.com/matrix-org/synapse/issues/5545))
-- Remove non-functioning check_event_hash.py dev script. ([\#5548](https://github.com/matrix-org/synapse/issues/5548))
-- Synapse will now only allow TLS v1.2 connections when serving federation, if it terminates TLS. As Synapse's allowed ciphers were only able to be used in TLSv1.2 before, this does not change behaviour. ([\#5550](https://github.com/matrix-org/synapse/issues/5550))
-- Logging when running GC collection on generation 0 is now at the DEBUG level, not INFO. ([\#5557](https://github.com/matrix-org/synapse/issues/5557))
-- Reduce the amount of stuff we send in the docker context. ([\#5564](https://github.com/matrix-org/synapse/issues/5564))
-- Point the reverse links in the Purge History contrib scripts at the intended location. ([\#5570](https://github.com/matrix-org/synapse/issues/5570))
-
-
-Synapse 1.0.0 (2019-06-11)
-==========================
-
-Bugfixes
---------
-
-- Fix bug where attempting to send transactions with large number of EDUs can fail. ([\#5418](https://github.com/matrix-org/synapse/issues/5418))
-
-
-Improved Documentation
-----------------------
-
-- Expand the federation guide to include relevant content from the MSC1711 FAQ ([\#5419](https://github.com/matrix-org/synapse/issues/5419))
-
-
-Internal Changes
-----------------
-
-- Move password reset links to /_matrix/client/unstable namespace. ([\#5424](https://github.com/matrix-org/synapse/issues/5424))
-
-
-Synapse 1.0.0rc3 (2019-06-10)
-=============================
-
-Security: Fix authentication bug introduced in 1.0.0rc1. Please upgrade to rc3 immediately
-
-
-Synapse 1.0.0rc2 (2019-06-10)
-=============================
-
-Bugfixes
---------
-
-- Remove redundant warning about key server response validation. ([\#5392](https://github.com/matrix-org/synapse/issues/5392))
-- Fix bug where old keys stored in the database with a null valid until timestamp caused all verification requests for that key to fail. ([\#5415](https://github.com/matrix-org/synapse/issues/5415))
-- Fix excessive memory using with default `federation_verify_certificates: true` configuration. ([\#5417](https://github.com/matrix-org/synapse/issues/5417))
-
-
-Synapse 1.0.0rc1 (2019-06-07)
-=============================
-
-Features
---------
-
-- Synapse now more efficiently collates room statistics. ([\#4338](https://github.com/matrix-org/synapse/issues/4338), [\#5260](https://github.com/matrix-org/synapse/issues/5260), [\#5324](https://github.com/matrix-org/synapse/issues/5324))
-- Add experimental support for relations (aka reactions and edits). ([\#5220](https://github.com/matrix-org/synapse/issues/5220))
-- Ability to configure default room version. ([\#5223](https://github.com/matrix-org/synapse/issues/5223), [\#5249](https://github.com/matrix-org/synapse/issues/5249))
-- Allow configuring a range for the account validity startup job. ([\#5276](https://github.com/matrix-org/synapse/issues/5276))
-- CAS login will now hit the r0 API, not the deprecated v1 one. ([\#5286](https://github.com/matrix-org/synapse/issues/5286))
-- Validate federation server TLS certificates by default (implements [MSC1711](https://github.com/matrix-org/matrix-doc/blob/master/proposals/1711-x509-for-federation.md)). ([\#5359](https://github.com/matrix-org/synapse/issues/5359))
-- Update /_matrix/client/versions to reference support for r0.5.0. ([\#5360](https://github.com/matrix-org/synapse/issues/5360))
-- Add a script to generate new signing-key files. ([\#5361](https://github.com/matrix-org/synapse/issues/5361))
-- Update upgrade and installation guides ahead of 1.0. ([\#5371](https://github.com/matrix-org/synapse/issues/5371))
-- Replace the `perspectives` configuration section with `trusted_key_servers`, and make validating the signatures on responses optional (since TLS will do this job for us). ([\#5374](https://github.com/matrix-org/synapse/issues/5374))
-- Add ability to perform password reset via email without trusting the identity server. **As a result of this PR, password resets will now be disabled on the default configuration.**
-
-  Password reset emails are now sent from the homeserver by default, instead of the identity server. To enable this functionality, ensure `email` and `public_baseurl` config options are filled out.
-
-  If you would like to re-enable password resets being sent from the identity server (warning: this is dangerous! See [#5345](https://github.com/matrix-org/synapse/pull/5345)), set `email.trust_identity_server_for_password_resets` to true. ([\#5377](https://github.com/matrix-org/synapse/issues/5377))
-- Set default room version to v4. ([\#5379](https://github.com/matrix-org/synapse/issues/5379))
-
-
-Bugfixes
---------
-
-- Fixes client-server API not sending "m.heroes" to lazy-load /sync requests when a rooms name or its canonical alias are empty. Thanks to @dnaf for this work! ([\#5089](https://github.com/matrix-org/synapse/issues/5089))
-- Prevent federation device list updates breaking when processing multiple updates at once. ([\#5156](https://github.com/matrix-org/synapse/issues/5156))
-- Fix worker registration bug caused by ClientReaderSlavedStore being unable to see get_profileinfo. ([\#5200](https://github.com/matrix-org/synapse/issues/5200))
-- Fix race when backfilling in rooms with worker mode. ([\#5221](https://github.com/matrix-org/synapse/issues/5221))
-- Fix appservice timestamp massaging. ([\#5233](https://github.com/matrix-org/synapse/issues/5233))
-- Ensure that server_keys fetched via a notary server are correctly signed. ([\#5251](https://github.com/matrix-org/synapse/issues/5251))
-- Show the correct error when logging out and access token is missing. ([\#5256](https://github.com/matrix-org/synapse/issues/5256))
-- Fix error code when there is an invalid parameter on /_matrix/client/r0/publicRooms ([\#5257](https://github.com/matrix-org/synapse/issues/5257))
-- Fix error when downloading thumbnail with missing width/height parameter. ([\#5258](https://github.com/matrix-org/synapse/issues/5258))
-- Fix schema update for account validity. ([\#5268](https://github.com/matrix-org/synapse/issues/5268))
-- Fix bug where we leaked extremities when we soft failed events, leading to performance degradation. ([\#5274](https://github.com/matrix-org/synapse/issues/5274), [\#5278](https://github.com/matrix-org/synapse/issues/5278), [\#5291](https://github.com/matrix-org/synapse/issues/5291))
-- Fix "db txn 'update_presence' from sentinel context" log messages. ([\#5275](https://github.com/matrix-org/synapse/issues/5275))
-- Fix dropped logcontexts during high outbound traffic. ([\#5277](https://github.com/matrix-org/synapse/issues/5277))
-- Fix a bug where it is not possible to get events in the federation format with the request `GET /_matrix/client/r0/rooms/{roomId}/messages`. ([\#5293](https://github.com/matrix-org/synapse/issues/5293))
-- Fix performance problems with the rooms stats background update. ([\#5294](https://github.com/matrix-org/synapse/issues/5294))
-- Fix noisy 'no key for server' logs. ([\#5300](https://github.com/matrix-org/synapse/issues/5300))
-- Fix bug where a notary server would sometimes forget old keys. ([\#5307](https://github.com/matrix-org/synapse/issues/5307))
-- Prevent users from setting huge displaynames and avatar URLs. ([\#5309](https://github.com/matrix-org/synapse/issues/5309))
-- Fix handling of failures when processing incoming events where calling `/event_auth` on remote server fails. ([\#5317](https://github.com/matrix-org/synapse/issues/5317))
-- Ensure that we have an up-to-date copy of the signing key when validating incoming federation requests. ([\#5321](https://github.com/matrix-org/synapse/issues/5321))
-- Fix various problems which made the signing-key notary server time out for some requests. ([\#5333](https://github.com/matrix-org/synapse/issues/5333))
-- Fix bug which would make certain operations (such as room joins) block for 20 minutes while attemoting to fetch verification keys. ([\#5334](https://github.com/matrix-org/synapse/issues/5334))
-- Fix a bug where we could rapidly mark a server as unreachable even though it was only down for a few minutes. ([\#5335](https://github.com/matrix-org/synapse/issues/5335), [\#5340](https://github.com/matrix-org/synapse/issues/5340))
-- Fix a bug where account validity renewal emails could only be sent when email notifs were enabled. ([\#5341](https://github.com/matrix-org/synapse/issues/5341))
-- Fix failure when fetching batches of events during backfill, etc. ([\#5342](https://github.com/matrix-org/synapse/issues/5342))
-- Add a new room version where the timestamps on events are checked against the validity periods on signing keys. ([\#5348](https://github.com/matrix-org/synapse/issues/5348), [\#5354](https://github.com/matrix-org/synapse/issues/5354))
-- Fix room stats and presence background updates to correctly handle missing events. ([\#5352](https://github.com/matrix-org/synapse/issues/5352))
-- Include left members in room summaries' heroes. ([\#5355](https://github.com/matrix-org/synapse/issues/5355))
-- Fix `federation_custom_ca_list` configuration option. ([\#5362](https://github.com/matrix-org/synapse/issues/5362))
-- Fix missing logcontext warnings on shutdown. ([\#5369](https://github.com/matrix-org/synapse/issues/5369))
-
-
-Improved Documentation
-----------------------
-
-- Fix docs on resetting the user directory. ([\#5282](https://github.com/matrix-org/synapse/issues/5282))
-- Fix notes about ACME in the MSC1711 faq. ([\#5357](https://github.com/matrix-org/synapse/issues/5357))
-
-
-Internal Changes
-----------------
-
-- Synapse will now serve the experimental "room complexity" API endpoint. ([\#5216](https://github.com/matrix-org/synapse/issues/5216))
-- The base classes for the v1 and v2_alpha REST APIs have been unified. ([\#5226](https://github.com/matrix-org/synapse/issues/5226), [\#5328](https://github.com/matrix-org/synapse/issues/5328))
-- Simplifications and comments in do_auth. ([\#5227](https://github.com/matrix-org/synapse/issues/5227))
-- Remove urllib3 pin as requests 2.22.0 has been released supporting urllib3 1.25.2. ([\#5230](https://github.com/matrix-org/synapse/issues/5230))
-- Preparatory work for key-validity features. ([\#5232](https://github.com/matrix-org/synapse/issues/5232), [\#5234](https://github.com/matrix-org/synapse/issues/5234), [\#5235](https://github.com/matrix-org/synapse/issues/5235), [\#5236](https://github.com/matrix-org/synapse/issues/5236), [\#5237](https://github.com/matrix-org/synapse/issues/5237), [\#5244](https://github.com/matrix-org/synapse/issues/5244), [\#5250](https://github.com/matrix-org/synapse/issues/5250), [\#5296](https://github.com/matrix-org/synapse/issues/5296), [\#5299](https://github.com/matrix-org/synapse/issues/5299), [\#5343](https://github.com/matrix-org/synapse/issues/5343), [\#5347](https://github.com/matrix-org/synapse/issues/5347), [\#5356](https://github.com/matrix-org/synapse/issues/5356))
-- Specify the type of reCAPTCHA key to use. ([\#5283](https://github.com/matrix-org/synapse/issues/5283))
-- Improve sample config for monthly active user blocking. ([\#5284](https://github.com/matrix-org/synapse/issues/5284))
-- Remove spurious debug from MatrixFederationHttpClient.get_json. ([\#5287](https://github.com/matrix-org/synapse/issues/5287))
-- Improve logging for logcontext leaks. ([\#5288](https://github.com/matrix-org/synapse/issues/5288))
-- Clarify that the admin change password API logs the user out. ([\#5303](https://github.com/matrix-org/synapse/issues/5303))
-- New installs will now use the v54 full schema, rather than the full schema v14 and applying incremental updates to v54. ([\#5320](https://github.com/matrix-org/synapse/issues/5320))
-- Improve docstrings on MatrixFederationClient. ([\#5332](https://github.com/matrix-org/synapse/issues/5332))
-- Clean up FederationClient.get_events for clarity. ([\#5344](https://github.com/matrix-org/synapse/issues/5344))
-- Various improvements to debug logging. ([\#5353](https://github.com/matrix-org/synapse/issues/5353))
-- Don't run CI build checks until sample config check has passed. ([\#5370](https://github.com/matrix-org/synapse/issues/5370))
-- Automatically retry buildkite builds (max twice) when an agent is lost. ([\#5380](https://github.com/matrix-org/synapse/issues/5380))
-
-
-Synapse 0.99.5.2 (2019-05-30)
-=============================
-
-Bugfixes
---------
-
-- Fix bug where we leaked extremities when we soft failed events, leading to performance degradation. ([\#5274](https://github.com/matrix-org/synapse/issues/5274), [\#5278](https://github.com/matrix-org/synapse/issues/5278), [\#5291](https://github.com/matrix-org/synapse/issues/5291))
-
-
-Synapse 0.99.5.1 (2019-05-22)
-=============================
-
-0.99.5.1 supersedes 0.99.5 due to malformed debian changelog - no functional changes.
-
-Synapse 0.99.5 (2019-05-22)
-===========================
-
-No significant changes.
-
-
-Synapse 0.99.5rc1 (2019-05-21)
-==============================
-
-Features
---------
-
-- Add ability to blacklist IP ranges for the federation client. ([\#5043](https://github.com/matrix-org/synapse/issues/5043))
-- Ratelimiting configuration for clients sending messages and the federation server has been altered to match login ratelimiting. The old configuration names will continue working. Check the sample config for details of the new names. ([\#5181](https://github.com/matrix-org/synapse/issues/5181))
-- Drop support for the undocumented /_matrix/client/v2_alpha API prefix. ([\#5190](https://github.com/matrix-org/synapse/issues/5190))
-- Add an option to disable per-room profiles. ([\#5196](https://github.com/matrix-org/synapse/issues/5196))
-- Stick an expiration date to any registered user missing one at startup if account validity is enabled. ([\#5204](https://github.com/matrix-org/synapse/issues/5204))
-- Add experimental support for relations (aka reactions and edits). ([\#5209](https://github.com/matrix-org/synapse/issues/5209), [\#5211](https://github.com/matrix-org/synapse/issues/5211), [\#5203](https://github.com/matrix-org/synapse/issues/5203), [\#5212](https://github.com/matrix-org/synapse/issues/5212))
-- Add a room version 4 which uses a new event ID format, as per [MSC2002](https://github.com/matrix-org/matrix-doc/pull/2002). ([\#5210](https://github.com/matrix-org/synapse/issues/5210), [\#5217](https://github.com/matrix-org/synapse/issues/5217))
-
-
-Bugfixes
---------
-
-- Fix image orientation when generating thumbnails (needs pillow>=4.3.0). Contributed by Pau Rodriguez-Estivill. ([\#5039](https://github.com/matrix-org/synapse/issues/5039))
-- Exclude soft-failed events from forward-extremity candidates: fixes "No forward extremities left!" error. ([\#5146](https://github.com/matrix-org/synapse/issues/5146))
-- Re-order stages in registration flows such that msisdn and email verification are done last. ([\#5174](https://github.com/matrix-org/synapse/issues/5174))
-- Fix 3pid guest invites. ([\#5177](https://github.com/matrix-org/synapse/issues/5177))
-- Fix a bug where the register endpoint would fail with M_THREEPID_IN_USE instead of returning an account previously registered in the same session. ([\#5187](https://github.com/matrix-org/synapse/issues/5187))
-- Prevent registration for user ids that are too long to fit into a state key. Contributed by Reid Anderson. ([\#5198](https://github.com/matrix-org/synapse/issues/5198))
-- Fix incompatibility between ACME support and Python 3.5.2. ([\#5218](https://github.com/matrix-org/synapse/issues/5218))
-- Fix error handling for rooms whose versions are unknown. ([\#5219](https://github.com/matrix-org/synapse/issues/5219))
-
-
-Internal Changes
-----------------
-
-- Make /sync attempt to return device updates for both joined and invited users. Note that this doesn't currently work correctly due to other bugs. ([\#3484](https://github.com/matrix-org/synapse/issues/3484))
-- Update tests to consistently be configured via the same code that is used when loading from configuration files. ([\#5171](https://github.com/matrix-org/synapse/issues/5171), [\#5185](https://github.com/matrix-org/synapse/issues/5185))
-- Allow client event serialization to be async. ([\#5183](https://github.com/matrix-org/synapse/issues/5183))
-- Expose DataStore._get_events as get_events_as_list. ([\#5184](https://github.com/matrix-org/synapse/issues/5184))
-- Make generating SQL bounds for pagination generic. ([\#5191](https://github.com/matrix-org/synapse/issues/5191))
-- Stop telling people to install the optional dependencies by default. ([\#5197](https://github.com/matrix-org/synapse/issues/5197))
-
-
-Synapse 0.99.4 (2019-05-15)
-===========================
-
-No significant changes.
-
-
-Synapse 0.99.4rc1 (2019-05-13)
-==============================
-
-Features
---------
-
-- Add systemd-python to the optional dependencies to enable logging to the systemd journal. Install with `pip install matrix-synapse[systemd]`. ([\#4339](https://github.com/matrix-org/synapse/issues/4339))
-- Add a default .m.rule.tombstone push rule. ([\#4867](https://github.com/matrix-org/synapse/issues/4867))
-- Add ability for password provider modules to bind email addresses to users upon registration. ([\#4947](https://github.com/matrix-org/synapse/issues/4947))
-- Implementation of [MSC1711](https://github.com/matrix-org/matrix-doc/pull/1711) including config options for requiring valid TLS certificates for federation traffic, the ability to disable TLS validation for specific domains, and the ability to specify your own list of CA certificates. ([\#4967](https://github.com/matrix-org/synapse/issues/4967))
-- Remove presence list support as per MSC 1819. ([\#4989](https://github.com/matrix-org/synapse/issues/4989))
-- Reduce CPU usage starting pushers during start up. ([\#4991](https://github.com/matrix-org/synapse/issues/4991))
-- Add a delete group admin API. ([\#5002](https://github.com/matrix-org/synapse/issues/5002))
-- Add config option to block users from looking up 3PIDs. ([\#5010](https://github.com/matrix-org/synapse/issues/5010))
-- Add context to phonehome stats. ([\#5020](https://github.com/matrix-org/synapse/issues/5020))
-- Configure the example systemd units to have a log identifier of `matrix-synapse`
-  instead of the executable name, `python`.
-  Contributed by Christoph Müller. ([\#5023](https://github.com/matrix-org/synapse/issues/5023))
-- Add time-based account expiration. ([\#5027](https://github.com/matrix-org/synapse/issues/5027), [\#5047](https://github.com/matrix-org/synapse/issues/5047), [\#5073](https://github.com/matrix-org/synapse/issues/5073), [\#5116](https://github.com/matrix-org/synapse/issues/5116))
-- Add support for handling `/versions`, `/voip` and `/push_rules` client endpoints to client_reader worker. ([\#5063](https://github.com/matrix-org/synapse/issues/5063), [\#5065](https://github.com/matrix-org/synapse/issues/5065), [\#5070](https://github.com/matrix-org/synapse/issues/5070))
-- Add a configuration option to require authentication on /publicRooms and /profile endpoints. ([\#5083](https://github.com/matrix-org/synapse/issues/5083))
-- Move admin APIs to `/_synapse/admin/v1`. (The old paths are retained for backwards-compatibility, for now). ([\#5119](https://github.com/matrix-org/synapse/issues/5119))
-- Implement an admin API for sending server notices. Many thanks to @krombel who provided a foundation for this work. ([\#5121](https://github.com/matrix-org/synapse/issues/5121), [\#5142](https://github.com/matrix-org/synapse/issues/5142))
-
-
-Bugfixes
---------
-
-- Avoid redundant URL encoding of redirect URL for SSO login in the fallback login page. Fixes a regression introduced in [#4220](https://github.com/matrix-org/synapse/pull/4220). Contributed by Marcel Fabian Krüger ("[zaugin](https://github.com/zauguin)"). ([\#4555](https://github.com/matrix-org/synapse/issues/4555))
-- Fix bug where presence updates were sent to all servers in a room when a new server joined, rather than to just the new server. ([\#4942](https://github.com/matrix-org/synapse/issues/4942), [\#5103](https://github.com/matrix-org/synapse/issues/5103))
-- Fix sync bug which made accepting invites unreliable in worker-mode synapses. ([\#4955](https://github.com/matrix-org/synapse/issues/4955), [\#4956](https://github.com/matrix-org/synapse/issues/4956))
-- start.sh: Fix the --no-rate-limit option for messages and make it bypass rate limit on registration and login too. ([\#4981](https://github.com/matrix-org/synapse/issues/4981))
-- Transfer related groups on room upgrade. ([\#4990](https://github.com/matrix-org/synapse/issues/4990))
-- Prevent the ability to kick users from a room they aren't in. ([\#4999](https://github.com/matrix-org/synapse/issues/4999))
-- Fix issue #4596 so synapse_port_db script works with --curses option on Python 3. Contributed by Anders Jensen-Waud <anders@jensenwaud.com>. ([\#5003](https://github.com/matrix-org/synapse/issues/5003))
-- Clients timing out/disappearing while downloading from the media repository will now no longer log a spurious "Producer was not unregistered" message. ([\#5009](https://github.com/matrix-org/synapse/issues/5009))
-- Fix "cannot import name execute_batch" error with postgres. ([\#5032](https://github.com/matrix-org/synapse/issues/5032))
-- Fix disappearing exceptions in manhole. ([\#5035](https://github.com/matrix-org/synapse/issues/5035))
-- Workaround bug in twisted where attempting too many concurrent DNS requests could cause it to hang due to running out of file descriptors. ([\#5037](https://github.com/matrix-org/synapse/issues/5037))
-- Make sure we're not registering the same 3pid twice on registration. ([\#5071](https://github.com/matrix-org/synapse/issues/5071))
-- Don't crash on lack of expiry templates. ([\#5077](https://github.com/matrix-org/synapse/issues/5077))
-- Fix the ratelimiting on third party invites. ([\#5104](https://github.com/matrix-org/synapse/issues/5104))
-- Add some missing limitations to room alias creation. ([\#5124](https://github.com/matrix-org/synapse/issues/5124), [\#5128](https://github.com/matrix-org/synapse/issues/5128))
-- Limit the number of EDUs in transactions to 100 as expected by synapse. Thanks to @superboum for this work! ([\#5138](https://github.com/matrix-org/synapse/issues/5138))
-
-Internal Changes
-----------------
-
-- Add test to verify threepid auth check added in #4435. ([\#4474](https://github.com/matrix-org/synapse/issues/4474))
-- Fix/improve some docstrings in the replication code. ([\#4949](https://github.com/matrix-org/synapse/issues/4949))
-- Split synapse.replication.tcp.streams into smaller files. ([\#4953](https://github.com/matrix-org/synapse/issues/4953))
-- Refactor replication row generation/parsing. ([\#4954](https://github.com/matrix-org/synapse/issues/4954))
-- Run `black` to clean up formatting on `synapse/storage/roommember.py` and `synapse/storage/events.py`. ([\#4959](https://github.com/matrix-org/synapse/issues/4959))
-- Remove log line for password via the admin API. ([\#4965](https://github.com/matrix-org/synapse/issues/4965))
-- Fix typo in TLS filenames in docker/README.md. Also add the '-p' commandline option to the 'docker run' example. Contributed by Jurrie Overgoor. ([\#4968](https://github.com/matrix-org/synapse/issues/4968))
-- Refactor room version definitions. ([\#4969](https://github.com/matrix-org/synapse/issues/4969))
-- Reduce log level of .well-known/matrix/client responses. ([\#4972](https://github.com/matrix-org/synapse/issues/4972))
-- Add `config.signing_key_path` that can be read by `synapse.config` utility. ([\#4974](https://github.com/matrix-org/synapse/issues/4974))
-- Track which identity server is used when binding a threepid and use that for unbinding, as per MSC1915. ([\#4982](https://github.com/matrix-org/synapse/issues/4982))
-- Rewrite KeyringTestCase as a HomeserverTestCase. ([\#4985](https://github.com/matrix-org/synapse/issues/4985))
-- README updates: Corrected the default POSTGRES_USER. Added port forwarding hint in TLS section. ([\#4987](https://github.com/matrix-org/synapse/issues/4987))
-- Remove a number of unused tables from the database schema. ([\#4992](https://github.com/matrix-org/synapse/issues/4992), [\#5028](https://github.com/matrix-org/synapse/issues/5028), [\#5033](https://github.com/matrix-org/synapse/issues/5033))
-- Run `black` on the remainder of `synapse/storage/`. ([\#4996](https://github.com/matrix-org/synapse/issues/4996))
-- Fix grammar in get_current_users_in_room and give it a docstring. ([\#4998](https://github.com/matrix-org/synapse/issues/4998))
-- Clean up some code in the server-key Keyring. ([\#5001](https://github.com/matrix-org/synapse/issues/5001))
-- Convert SYNAPSE_NO_TLS Docker variable to boolean for user friendliness. Contributed by Gabriel Eckerson. ([\#5005](https://github.com/matrix-org/synapse/issues/5005))
-- Refactor synapse.storage._base._simple_select_list_paginate. ([\#5007](https://github.com/matrix-org/synapse/issues/5007))
-- Store the notary server name correctly in server_keys_json. ([\#5024](https://github.com/matrix-org/synapse/issues/5024))
-- Rewrite Datastore.get_server_verify_keys to reduce the number of database transactions. ([\#5030](https://github.com/matrix-org/synapse/issues/5030))
-- Remove extraneous period from copyright headers. ([\#5046](https://github.com/matrix-org/synapse/issues/5046))
-- Update documentation for where to get Synapse packages. ([\#5067](https://github.com/matrix-org/synapse/issues/5067))
-- Add workarounds for pep-517 install errors. ([\#5098](https://github.com/matrix-org/synapse/issues/5098))
-- Improve logging when event-signature checks fail. ([\#5100](https://github.com/matrix-org/synapse/issues/5100))
-- Factor out an "assert_requester_is_admin" function. ([\#5120](https://github.com/matrix-org/synapse/issues/5120))
-- Remove the requirement to authenticate for /admin/server_version. ([\#5122](https://github.com/matrix-org/synapse/issues/5122))
-- Prevent an exception from being raised in a IResolutionReceiver and use a more generic error message for blacklisted URL previews. ([\#5155](https://github.com/matrix-org/synapse/issues/5155))
-- Run `black` on the tests directory. ([\#5170](https://github.com/matrix-org/synapse/issues/5170))
-- Fix CI after new release of isort. ([\#5179](https://github.com/matrix-org/synapse/issues/5179))
-- Fix bogus imports in unit tests. ([\#5154](https://github.com/matrix-org/synapse/issues/5154))
-
-
-Synapse 0.99.3.2 (2019-05-03)
-=============================
-
-Internal Changes
-----------------
-
-- Ensure that we have `urllib3` <1.25, to resolve incompatibility with `requests`. ([\#5135](https://github.com/matrix-org/synapse/issues/5135))
-
-
-Synapse 0.99.3.1 (2019-05-03)
-=============================
-
-Security update
----------------
-
-This release includes two security fixes:
-
-- Switch to using a cryptographically-secure random number generator for token strings, ensuring they cannot be predicted by an attacker. Thanks to @opnsec for identifying and responsibly disclosing this issue! ([\#5133](https://github.com/matrix-org/synapse/issues/5133))
-- Blacklist 0.0.0.0 and :: by default for URL previews. Thanks to @opnsec for identifying and responsibly disclosing this issue too! ([\#5134](https://github.com/matrix-org/synapse/issues/5134))
-
-Synapse 0.99.3 (2019-04-01)
-===========================
-
-No significant changes.
-
-
-Synapse 0.99.3rc1 (2019-03-27)
-==============================
-
-Features
---------
-
-- The user directory has been rewritten to make it faster, with less chance of falling behind on a large server. ([\#4537](https://github.com/matrix-org/synapse/issues/4537), [\#4846](https://github.com/matrix-org/synapse/issues/4846), [\#4864](https://github.com/matrix-org/synapse/issues/4864), [\#4887](https://github.com/matrix-org/synapse/issues/4887), [\#4900](https://github.com/matrix-org/synapse/issues/4900), [\#4944](https://github.com/matrix-org/synapse/issues/4944))
-- Add configurable rate limiting to the /register endpoint. ([\#4735](https://github.com/matrix-org/synapse/issues/4735), [\#4804](https://github.com/matrix-org/synapse/issues/4804))
-- Move server key queries to federation reader. ([\#4757](https://github.com/matrix-org/synapse/issues/4757))
-- Add support for /account/3pid REST endpoint to client_reader worker. ([\#4759](https://github.com/matrix-org/synapse/issues/4759))
-- Add an endpoint to the admin API for querying the server version. Contributed by Joseph Weston. ([\#4772](https://github.com/matrix-org/synapse/issues/4772))
-- Include a default configuration file in the 'docs' directory. ([\#4791](https://github.com/matrix-org/synapse/issues/4791), [\#4801](https://github.com/matrix-org/synapse/issues/4801))
-- Synapse is now permissive about trailing slashes on some of its federation endpoints, allowing zero or more to be present. ([\#4793](https://github.com/matrix-org/synapse/issues/4793))
-- Add support for /keys/query and /keys/changes REST endpoints to client_reader worker. ([\#4796](https://github.com/matrix-org/synapse/issues/4796))
-- Add checks to incoming events over federation for events evading auth (aka "soft fail"). ([\#4814](https://github.com/matrix-org/synapse/issues/4814))
-- Add configurable rate limiting to the /login endpoint. ([\#4821](https://github.com/matrix-org/synapse/issues/4821), [\#4865](https://github.com/matrix-org/synapse/issues/4865))
-- Remove trailing slashes from certain outbound federation requests. Retry if receiving a 404. Context: #3622. ([\#4840](https://github.com/matrix-org/synapse/issues/4840))
-- Allow passing --daemonize flags to workers in the same way as with master. ([\#4853](https://github.com/matrix-org/synapse/issues/4853))
-- Batch up outgoing read-receipts to reduce federation traffic. ([\#4890](https://github.com/matrix-org/synapse/issues/4890), [\#4927](https://github.com/matrix-org/synapse/issues/4927))
-- Add option to disable searching the user directory. ([\#4895](https://github.com/matrix-org/synapse/issues/4895))
-- Add option to disable searching of local and remote public room lists. ([\#4896](https://github.com/matrix-org/synapse/issues/4896))
-- Add ability for password providers to login/register a user via 3PID (email, phone). ([\#4931](https://github.com/matrix-org/synapse/issues/4931))
-
-
-Bugfixes
---------
-
-- Fix a bug where media with spaces in the name would get a corrupted name. ([\#2090](https://github.com/matrix-org/synapse/issues/2090))
-- Fix attempting to paginate in rooms where server cannot see any events, to avoid unnecessarily pulling in lots of redacted events. ([\#4699](https://github.com/matrix-org/synapse/issues/4699))
-- 'event_id' is now a required parameter in federated state requests, as per the matrix spec. ([\#4740](https://github.com/matrix-org/synapse/issues/4740))
-- Fix tightloop over connecting to replication server. ([\#4749](https://github.com/matrix-org/synapse/issues/4749))
-- Fix parsing of Content-Disposition headers on remote media requests and URL previews. ([\#4763](https://github.com/matrix-org/synapse/issues/4763))
-- Fix incorrect log about not persisting duplicate state event. ([\#4776](https://github.com/matrix-org/synapse/issues/4776))
-- Fix v4v6 option in HAProxy example config. Contributed by Flakebi. ([\#4790](https://github.com/matrix-org/synapse/issues/4790))
-- Handle batch updates in worker replication protocol. ([\#4792](https://github.com/matrix-org/synapse/issues/4792))
-- Fix bug where we didn't correctly throttle sending of USER_IP commands over replication. ([\#4818](https://github.com/matrix-org/synapse/issues/4818))
-- Fix potential race in handling missing updates in device list updates. ([\#4829](https://github.com/matrix-org/synapse/issues/4829))
-- Fix bug where synapse expected an un-specced `prev_state` field on state events. ([\#4837](https://github.com/matrix-org/synapse/issues/4837))
-- Transfer a user's notification settings (push rules) on room upgrade. ([\#4838](https://github.com/matrix-org/synapse/issues/4838))
-- fix test_auto_create_auto_join_where_no_consent. ([\#4886](https://github.com/matrix-org/synapse/issues/4886))
-- Fix a bug where hs_disabled_message was sometimes not correctly enforced. ([\#4888](https://github.com/matrix-org/synapse/issues/4888))
-- Fix bug in shutdown room admin API where it would fail if a user in the room hadn't consented to the privacy policy. ([\#4904](https://github.com/matrix-org/synapse/issues/4904))
-- Fix bug where blocked world-readable rooms were still peekable. ([\#4908](https://github.com/matrix-org/synapse/issues/4908))
-
-
-Internal Changes
-----------------
-
-- Add a systemd setup that supports synapse workers. Contributed by Luca Corbatto. ([\#4662](https://github.com/matrix-org/synapse/issues/4662))
-- Change from TravisCI to Buildkite for CI. ([\#4752](https://github.com/matrix-org/synapse/issues/4752))
-- When presence is disabled don't send over replication. ([\#4757](https://github.com/matrix-org/synapse/issues/4757))
-- Minor docstring fixes for MatrixFederationAgent. ([\#4765](https://github.com/matrix-org/synapse/issues/4765))
-- Optimise EDU transmission for the federation_sender worker. ([\#4770](https://github.com/matrix-org/synapse/issues/4770))
-- Update test_typing to use HomeserverTestCase. ([\#4771](https://github.com/matrix-org/synapse/issues/4771))
-- Update URLs for riot.im icons and logos in the default notification templates. ([\#4779](https://github.com/matrix-org/synapse/issues/4779))
-- Removed unnecessary $ from some federation endpoint path regexes. ([\#4794](https://github.com/matrix-org/synapse/issues/4794))
-- Remove link to deleted title in README. ([\#4795](https://github.com/matrix-org/synapse/issues/4795))
-- Clean up read-receipt handling. ([\#4797](https://github.com/matrix-org/synapse/issues/4797))
-- Add some debug about processing read receipts. ([\#4798](https://github.com/matrix-org/synapse/issues/4798))
-- Clean up some replication code. ([\#4799](https://github.com/matrix-org/synapse/issues/4799))
-- Add some docstrings. ([\#4815](https://github.com/matrix-org/synapse/issues/4815))
-- Add debug logger to try and track down #4422. ([\#4816](https://github.com/matrix-org/synapse/issues/4816))
-- Make shutdown API send explanation message to room after users have been forced joined. ([\#4817](https://github.com/matrix-org/synapse/issues/4817))
-- Update example_log_config.yaml. ([\#4820](https://github.com/matrix-org/synapse/issues/4820))
-- Document the `generate` option for the docker image. ([\#4824](https://github.com/matrix-org/synapse/issues/4824))
-- Fix check-newsfragment for debian-only changes. ([\#4825](https://github.com/matrix-org/synapse/issues/4825))
-- Add some debug logging for device list updates to help with #4828. ([\#4828](https://github.com/matrix-org/synapse/issues/4828))
-- Improve federation documentation, specifically .well-known support. Many thanks to @vaab. ([\#4832](https://github.com/matrix-org/synapse/issues/4832))
-- Disable captcha registration by default in unit tests. ([\#4839](https://github.com/matrix-org/synapse/issues/4839))
-- Add stuff back to the .gitignore. ([\#4843](https://github.com/matrix-org/synapse/issues/4843))
-- Clarify what registration_shared_secret allows for. ([\#4844](https://github.com/matrix-org/synapse/issues/4844))
-- Correctly log expected errors when fetching server keys. ([\#4847](https://github.com/matrix-org/synapse/issues/4847))
-- Update install docs to explicitly state a full-chain (not just the top-level) TLS certificate must be provided to Synapse. This caused some people's Synapse ports to appear correct in a browser but still (rightfully so) upset the federation tester. ([\#4849](https://github.com/matrix-org/synapse/issues/4849))
-- Move client read-receipt processing to federation sender worker. ([\#4852](https://github.com/matrix-org/synapse/issues/4852))
-- Refactor federation TransactionQueue. ([\#4855](https://github.com/matrix-org/synapse/issues/4855))
-- Comment out most options in the generated config. ([\#4863](https://github.com/matrix-org/synapse/issues/4863))
-- Fix yaml library warnings by using safe_load. ([\#4869](https://github.com/matrix-org/synapse/issues/4869))
-- Update Apache setup to remove location syntax. Thanks to @cwmke! ([\#4870](https://github.com/matrix-org/synapse/issues/4870))
-- Reinstate test case that runs unit tests against oldest supported dependencies. ([\#4879](https://github.com/matrix-org/synapse/issues/4879))
-- Update link to federation docs. ([\#4881](https://github.com/matrix-org/synapse/issues/4881))
-- fix test_auto_create_auto_join_where_no_consent. ([\#4886](https://github.com/matrix-org/synapse/issues/4886))
-- Use a regular HomeServerConfig object for unit tests rater than a Mock. ([\#4889](https://github.com/matrix-org/synapse/issues/4889))
-- Add some notes about tuning postgres for larger deployments. ([\#4895](https://github.com/matrix-org/synapse/issues/4895))
-- Add a config option for torture-testing worker replication. ([\#4902](https://github.com/matrix-org/synapse/issues/4902))
-- Log requests which are simulated by the unit tests. ([\#4905](https://github.com/matrix-org/synapse/issues/4905))
-- Allow newsfragments to end with exclamation marks. Exciting! ([\#4912](https://github.com/matrix-org/synapse/issues/4912))
-- Refactor some more tests to use HomeserverTestCase. ([\#4913](https://github.com/matrix-org/synapse/issues/4913))
-- Refactor out the state deltas portion of the user directory store and handler. ([\#4917](https://github.com/matrix-org/synapse/issues/4917))
-- Fix nginx example in ACME doc. ([\#4923](https://github.com/matrix-org/synapse/issues/4923))
-- Use an explicit dbname for postgres connections in the tests. ([\#4928](https://github.com/matrix-org/synapse/issues/4928))
-- Fix `ClientReplicationStreamProtocol.__str__()`. ([\#4929](https://github.com/matrix-org/synapse/issues/4929))
-
-
 Synapse 0.99.2 (2019-03-01)
 ===========================
 

CONTRIBUTING.rst

@@ -30,20 +30,21 @@ use github's pull request workflow to review the contribution, and either ask
 you to make any refinements needed or merge it and make them ourselves. The
 changes will then land on master when we next do a release.
 
-We use `CircleCI <https://circleci.com/gh/matrix-org>`_ and `Buildkite
-<https://buildkite.com/matrix-dot-org/synapse>`_ for continuous integration.
-Buildkite builds need to be authorised by a maintainer. If your change breaks
-the build, this will be shown in GitHub, so please keep an eye on the pull
-request for feedback.
+We use `CircleCI <https://circleci.com/gh/matrix-org>`_ and `Travis CI
+<https://travis-ci.org/matrix-org/synapse>`_ for continuous integration. All
+pull requests to synapse get automatically tested by Travis and CircleCI.
+If your change breaks the build, this will be shown in GitHub, so please
+keep an eye on the pull request for feedback.
 
 To run unit tests in a local development environment, you can use:
 
-- ``tox -e py35`` (requires tox to be installed by ``pip install tox``)
-  for SQLite-backed Synapse on Python 3.5.
+- ``tox -e py27`` (requires tox to be installed by ``pip install tox``) for
+  SQLite-backed Synapse on Python 2.7.
+- ``tox -e py35`` for SQLite-backed Synapse on Python 3.5.
 - ``tox -e py36`` for SQLite-backed Synapse on Python 3.6.
-- ``tox -e py36-postgres`` for PostgreSQL-backed Synapse on Python 3.6
+- ``tox -e py27-postgres`` for PostgreSQL-backed Synapse on Python 2.7
   (requires a running local PostgreSQL with access to create databases).
-- ``./test_postgresql.sh`` for PostgreSQL-backed Synapse on Python 3.5
+- ``./test_postgresql.sh`` for PostgreSQL-backed Synapse on Python 2.7
   (requires Docker). Entirely self-contained, recommended if you don't want to
   set up PostgreSQL yourself.
 

INSTALL.md

@@ -1,31 +1,13 @@
-- [Choosing your server name](#choosing-your-server-name)
-- [Installing Synapse](#installing-synapse)
-  - [Installing from source](#installing-from-source)
-    - [Platform-Specific Instructions](#platform-specific-instructions)
-    - [Troubleshooting Installation](#troubleshooting-installation)
-  - [Prebuilt packages](#prebuilt-packages)
-- [Setting up Synapse](#setting-up-synapse)
-  - [TLS certificates](#tls-certificates)
-  - [Email](#email)
-  - [Registering a user](#registering-a-user)
-  - [Setting up a TURN server](#setting-up-a-turn-server)
-  - [URL previews](#url-previews)
-
-# Choosing your server name
-
-It is important to choose the name for your server before you install Synapse,
-because it cannot be changed later.
-
-The server name determines the "domain" part of user-ids for users on your
-server: these will all be of the format `@user:my.domain.name`. It also
-determines how other matrix servers will reach yours for federation.
-
-For a test configuration, set this to the hostname of your server. For a more
-production-ready setup, you will probably want to specify your domain
-(`example.com`) rather than a matrix-specific hostname here (in the same way
-that your email address is probably `user@example.com` rather than
-`user@email.example.com`) - but doing so may require more advanced setup: see
-[Setting up Federation](docs/federate.md).
+* [Installing Synapse](#installing-synapse)
+  * [Installing from source](#installing-from-source)
+    * [Platform-Specific Instructions](#platform-specific-instructions)
+    * [Troubleshooting Installation](#troubleshooting-installation)
+  * [Prebuilt packages](#prebuilt-packages)
+* [Setting up Synapse](#setting-up-synapse)
+  * [TLS certificates](#tls-certificates)
+  * [Registering a user](#registering-a-user)
+  * [Setting up a TURN server](#setting-up-a-turn-server)
+  * [URL previews](#url-previews)
 
 # Installing Synapse
 
@@ -53,7 +35,7 @@ virtualenv -p python3 ~/synapse/env
 source ~/synapse/env/bin/activate
 pip install --upgrade pip
 pip install --upgrade setuptools
-pip install matrix-synapse
+pip install matrix-synapse[all]
 ```
 
 This will download Synapse from [PyPI](https://pypi.org/project/matrix-synapse)
@@ -66,7 +48,7 @@ update flag:
 
 ```
 source ~/synapse/env/bin/activate
-pip install -U matrix-synapse
+pip install -U matrix-synapse[all]
 ```
 
 Before you can start Synapse, you will need to generate a configuration
@@ -81,7 +63,16 @@ python -m synapse.app.homeserver \
     --report-stats=[yes|no]
 ```
 
-... substituting an appropriate value for `--server-name`.
+... substituting an appropriate value for `--server-name`. The server name
+determines the "domain" part of user-ids for users on your server: these will
+all be of the format `@user:my.domain.name`. It also determines how other
+matrix servers will reach yours for Federation. For a test configuration,
+set this to the hostname of your server. For a more production-ready setup, you
+will probably want to specify your domain (`example.com`) rather than a
+matrix-specific hostname here (in the same way that your email address is
+probably `user@example.com` rather than `user@email.example.com`) - but
+doing so may require more advanced setup: see [Setting up Federation](docs/federate.md).
+Beware that the server name cannot be changed later.
 
 This command will generate you a config file that you can then customise, but it will
 also generate a set of keys for you. These keys will allow your Home Server to
@@ -94,6 +85,9 @@ different. See the
 [spec](https://matrix.org/docs/spec/server_server/latest.html#retrieving-server-keys)
 for more information on key management.)
 
+You will need to give Synapse a TLS certficate before it will start - see [TLS
+certificates](#tls-certificates).
+
 To actually run your new homeserver, pick a working directory for Synapse to
 run (e.g. `~/synapse`), and::
 
@@ -263,29 +257,18 @@ https://github.com/spantaleev/matrix-docker-ansible-deploy
 #### Matrix.org packages
 
 Matrix.org provides Debian/Ubuntu packages of the latest stable version of
-Synapse via https://packages.matrix.org/debian/. They are available for Debian
-9 (Stretch), Ubuntu 16.04 (Xenial), and later. To use them:
+Synapse via https://matrix.org/packages/debian/. To use them:
 
 ```
-sudo apt install -y lsb-release wget apt-transport-https
-sudo wget -O /usr/share/keyrings/matrix-org-archive-keyring.gpg https://packages.matrix.org/debian/matrix-org-archive-keyring.gpg
-echo "deb [signed-by=/usr/share/keyrings/matrix-org-archive-keyring.gpg] https://packages.matrix.org/debian/ $(lsb_release -cs) main" |
+sudo apt install -y lsb-release curl apt-transport-https
+echo "deb https://matrix.org/packages/debian `lsb_release -cs` main" |
     sudo tee /etc/apt/sources.list.d/matrix-org.list
+curl "https://matrix.org/packages/debian/repo-key.asc" |
+    sudo apt-key add -
 sudo apt update
 sudo apt install matrix-synapse-py3
 ```
 
-**Note**: if you followed a previous version of these instructions which
-recommended using `apt-key add` to add an old key from
-`https://matrix.org/packages/debian/`, you should note that this key has been
-revoked. You should remove the old key with `sudo apt-key remove
-C35EB17E1EAE708E6603A9B3AD0592FE47F0DF61`, and follow the above instructions to
-update your configuration.
-
-The fingerprint of the repository signing key (as shown by `gpg
-/usr/share/keyrings/matrix-org-archive-keyring.gpg`) is
-`AAF9AE843A7584B5A3E4CD2BCF45A512DE2DA058`.
-
 #### Downstream Debian/Ubuntu packages
 
 For `buster` and `sid`, Synapse is available in the Debian repositories and
@@ -400,22 +383,8 @@ To configure Synapse to expose an HTTPS port, you will need to edit
   instance, if using certbot, use `fullchain.pem` as your certificate, not
   `cert.pem`).
 
-For a more detailed guide to configuring your server for federation, see
-[federate.md](docs/federate.md)
-
-
-## Email
-
-It is desirable for Synapse to have the capability to send email. For example,
-this is required to support the 'password reset' feature.
-
-To configure an SMTP server for Synapse, modify the configuration section
-headed ``email``, and be sure to have at least the ``smtp_host``, ``smtp_port``
-and ``notif_from`` fields filled out. You may also need to set ``smtp_user``,
-``smtp_pass``, and ``require_transport_security``.
-
-If Synapse is not configured with an SMTP server, password reset via email will
- be disabled by default.
+For those of you upgrading your TLS certificate in readiness for Synapse 1.0,
+please take a look at `our guide <docs/MSC1711_certificates_FAQ.md#configuring-certificates-for-compatibility-with-synapse-100>`_.
 
 ## Registering a user
 

MANIFEST.in

@@ -9,19 +9,14 @@ include demo/*.py
 include demo/*.sh
 
 recursive-include synapse/storage/schema *.sql
-recursive-include synapse/storage/schema *.sql.postgres
-recursive-include synapse/storage/schema *.sql.sqlite
 recursive-include synapse/storage/schema *.py
-recursive-include synapse/storage/schema *.txt
 
 recursive-include docs *
 recursive-include scripts *
 recursive-include scripts-dev *
 recursive-include synapse *.pyi
+recursive-include tests *.pem
 recursive-include tests *.py
-include tests/http/ca.crt
-include tests/http/ca.key
-include tests/http/server.key
 
 recursive-include synapse/res *
 recursive-include synapse/static *.css
@@ -48,3 +43,8 @@ prune .buildkite
 
 exclude jenkins*
 recursive-exclude jenkins *.sh
+
+# FIXME: we shouldn't have these templates here
+recursive-include res/templates-dinsic *.css
+recursive-include res/templates-dinsic *.html
+recursive-include res/templates-dinsic *.txt

README.rst

@@ -173,7 +173,7 @@ Synapse offers two database engines:
  * `PostgreSQL <https://www.postgresql.org>`_
 
 By default Synapse uses SQLite in and doing so trades performance for convenience.
-SQLite is only recommended in Synapse for testing purposes or for servers with
+SQLite is only recommended in Synapse for testing purposes or for servers with 
 light workloads.
 
 Almost all installations should opt to use PostreSQL. Advantages include:
@@ -272,7 +272,7 @@ to install using pip and a virtualenv::
 
     virtualenv -p python3 env
     source env/bin/activate
-    python -m pip install --no-pep-517 -e .[all]
+    python -m pip install -e .[all]
 
 This will run a process of downloading and installing all the needed
 dependencies into a virtual env.
@@ -340,11 +340,8 @@ log lines and looking for any 'Processed request' lines which take more than
 a few seconds to execute. Please let us know at #synapse:matrix.org if
 you see this failure mode so we can help debug it, however.
 
-Help!! Synapse is slow and eats all my RAM/CPU!
------------------------------------------------
-
-First, ensure you are running the latest version of Synapse, using Python 3
-with a PostgreSQL database.
+Help!! Synapse eats all my RAM!
+-------------------------------
 
 Synapse's architecture is quite RAM hungry currently - we deliberately
 cache a lot of recent room data and metadata in RAM in order to speed up
@@ -355,29 +352,14 @@ variable. The default is 0.5, which can be decreased to reduce RAM usage
 in memory constrained enviroments, or increased if performance starts to
 degrade.
 
-However, degraded performance due to a low cache factor, common on
-machines with slow disks, often leads to explosions in memory use due
-backlogged requests. In this case, reducing the cache factor will make
-things worse. Instead, try increasing it drastically. 2.0 is a good
-starting value.
-
 Using `libjemalloc <http://jemalloc.net/>`_ can also yield a significant
-improvement in overall memory use, and especially in terms of giving back
-RAM to the OS. To use it, the library must simply be put in the
-LD_PRELOAD environment variable when launching Synapse. On Debian, this
-can be done by installing the ``libjemalloc1`` package and adding this
-line to ``/etc/default/matrix-synapse``::
+improvement in overall amount, and especially in terms of giving back RAM
+to the OS. To use it, the library must simply be put in the LD_PRELOAD
+environment variable when launching Synapse. On Debian, this can be done
+by installing the ``libjemalloc1`` package and adding this line to
+``/etc/default/matrix-synapse``::
 
     LD_PRELOAD=/usr/lib/x86_64-linux-gnu/libjemalloc.so.1
 
 This can make a significant difference on Python 2.7 - it's unclear how
 much of an improvement it provides on Python 3.x.
-
-If you're encountering high CPU use by the Synapse process itself, you
-may be affected by a bug with presence tracking that leads to a
-massive excess of outgoing federation requests (see `discussion
-<https://github.com/matrix-org/synapse/issues/3971>`_). If metrics
-indicate that your server is also issuing far more outgoing federation
-requests than can be accounted for by your users' activity, this is a
-likely cause. The misbehavior can be worked around by setting
-``use_presence: false`` in the Synapse config file.

UPGRADE.rst

@@ -49,82 +49,6 @@ returned by the Client-Server API:
     # configured on port 443.
     curl -kv https://<host.name>/_matrix/client/versions 2>&1 | grep "Server:"
 
-Upgrading to v1.1.0
-===================
-
-Synapse v1.1.0 removes support for older Python and PostgreSQL versions, as
-outlined in `our deprecation notice <https://matrix.org/blog/2019/04/08/synapse-deprecating-postgres-9-4-and-python-2-x>`_.
-
-Minimum Python Version
-----------------------
-
-Synapse v1.1.0 has a minimum Python requirement of Python 3.5. Python 3.6 or
-Python 3.7 are recommended as they have improved internal string handling,
-significantly reducing memory usage.
-
-If you use current versions of the Matrix.org-distributed Debian packages or
-Docker images, action is not required.
-
-If you install Synapse in a Python virtual environment, please see "Upgrading to
-v0.34.0" for notes on setting up a new virtualenv under Python 3.
-
-Minimum PostgreSQL Version
---------------------------
-
-If using PostgreSQL under Synapse, you will need to use PostgreSQL 9.5 or above.
-Please see the
-`PostgreSQL documentation <https://www.postgresql.org/docs/11/upgrading.html>`_
-for more details on upgrading your database.
-
-Upgrading to v1.0
-=================
-
-Validation of TLS certificates
-------------------------------
-
-Synapse v1.0 is the first release to enforce
-validation of TLS certificates for the federation API. It is therefore
-essential that your certificates are correctly configured. See the `FAQ
-<docs/MSC1711_certificates_FAQ.md>`_ for more information.
-
-Note, v1.0 installations will also no longer be able to federate with servers
-that have not correctly configured their certificates.
-
-In rare cases, it may be desirable to disable certificate checking: for
-example, it might be essential to be able to federate with a given legacy
-server in a closed federation. This can be done in one of two ways:-
-
-* Configure the global switch ``federation_verify_certificates`` to ``false``.
-* Configure a whitelist of server domains to trust via ``federation_certificate_verification_whitelist``.
-
-See the `sample configuration file <docs/sample_config.yaml>`_
-for more details on these settings.
-
-Email
------
-When a user requests a password reset, Synapse will send an email to the
-user to confirm the request.
-
-Previous versions of Synapse delegated the job of sending this email to an
-identity server. If the identity server was somehow malicious or became
-compromised, it would be theoretically possible to hijack an account through
-this means.
-
-Therefore, by default, Synapse v1.0 will send the confirmation email itself. If
-Synapse is not configured with an SMTP server, password reset via email will be
-disabled.
-
-To configure an SMTP server for Synapse, modify the configuration section
-headed ``email``, and be sure to have at least the ``smtp_host``, ``smtp_port``
-and ``notif_from`` fields filled out. You may also need to set ``smtp_user``,
-``smtp_pass``, and ``require_transport_security``.
-
-If you are absolutely certain that you wish to continue using an identity
-server for password resets, set ``trust_identity_server_for_password_resets`` to ``true``.
-
-See the `sample configuration file <docs/sample_config.yaml>`_
-for more details on these settings.
-
 Upgrading to v0.99.0
 ====================
 

changelog.d/2090.bugfix

@@ -0,0 +1 @@
+Fix a bug where media with spaces in the name would get a corrupted name.

changelog.d/4537.feature

@@ -0,0 +1 @@
+The user directory has been rewritten to make it faster, with less chance of falling behind on a large server.

changelog.d/4662.misc

@@ -0,0 +1 @@
+Add a systemd setup that supports synapse workers. Contributed by Luca Corbatto.

changelog.d/4699.bugfix

@@ -0,0 +1 @@
+Fix attempting to paginate in rooms where server cannot see any events, to avoid unnecessarily pulling in lots of redacted events.

changelog.d/4735.feature

@@ -0,0 +1 @@
+Add configurable rate limiting to the /register endpoint.

changelog.d/4740.bugfix

@@ -0,0 +1 @@
+'event_id' is now a required parameter in federated state requests, as per the matrix spec.

changelog.d/4749.bugfix

@@ -0,0 +1 @@
+Fix tightloop over connecting to replication server.

changelog.d/4752.misc

@@ -0,0 +1 @@
+Change from TravisCI to Buildkite for CI.

changelog.d/4757.feature

@@ -0,0 +1 @@
+Move server key queries to federation reader.

changelog.d/4757.misc

@@ -0,0 +1 @@
+When presence is disabled don't send over replication.

changelog.d/4759.feature

@@ -0,0 +1 @@
+Add support for /account/3pid REST endpoint to client_reader worker.

changelog.d/4763.bugfix

@@ -0,0 +1 @@
+Fix parsing of Content-Disposition headers on remote media requests and URL previews.

changelog.d/4765.misc

@@ -0,0 +1 @@
+Minor docstring fixes for MatrixFederationAgent.

changelog.d/4770.misc

@@ -0,0 +1 @@
+Optimise EDU transmission for the federation_sender worker.

changelog.d/4771.misc

@@ -0,0 +1 @@
+Update test_typing to use HomeserverTestCase.

changelog.d/4772.feature

@@ -0,0 +1 @@
+Add an endpoint to the admin API for querying the server version. Contributed by Joseph Weston.

changelog.d/4776.bugfix

@@ -0,0 +1 @@
+Fix incorrect log about not persisting duplicate state event.

changelog.d/4779.misc

@@ -0,0 +1 @@
+Update URLs for riot.im icons and logos in the default notification templates.

changelog.d/4790.bugfix

@@ -0,0 +1 @@
+Fix v4v6 option in HAProxy example config. Contributed by Flakebi.

changelog.d/4791.feature

@@ -0,0 +1 @@
+Include a default configuration file in the 'docs' directory.

changelog.d/4792.bugfix

@@ -0,0 +1 @@
+Handle batch updates in worker replication protocol.

changelog.d/4794.misc

@@ -0,0 +1 @@
+Removed unnecessary $ from some federation endpoint path regexes.

changelog.d/4795.misc

@@ -0,0 +1 @@
+Remove link to deleted title in README.

changelog.d/4796.feature

@@ -0,0 +1 @@
+Add support for /keys/query and /keys/changes REST endpoints to client_reader worker.

changelog.d/4797.misc

@@ -0,0 +1 @@
+Clean up read-receipt handling.

changelog.d/4798.misc

@@ -0,0 +1 @@
+Add some debug about processing read receipts.

changelog.d/4799.misc

@@ -0,0 +1 @@
+Clean up some replication code.

changelog.d/4801.feature

@@ -0,0 +1 @@
+Include a default configuration file in the 'docs' directory.

changelog.d/4804.feature

@@ -0,0 +1 @@
+Add configurable rate limiting to the /register endpoint.

changelog.d/4814.feature

@@ -0,0 +1 @@
+Add checks to incoming events over federation for events evading auth (aka "soft fail").

changelog.d/4815.misc

@@ -0,0 +1 @@
+Add some docstrings.

changelog.d/4816.misc

@@ -0,0 +1 @@
+Add debug logger to try and track down #4422.

changelog.d/4817.misc

@@ -0,0 +1 @@
+Make shutdown API send explanation message to room after users have been forced joined.

changelog.d/4818.bugfix

@@ -0,0 +1 @@
+Fix bug where we didn't correctly throttle sending of USER_IP commands over replication.

changelog.d/4820.misc

@@ -0,0 +1 @@
+Update example_log_config.yaml.

changelog.d/4821.feature

@@ -0,0 +1 @@
+Add configurable rate limiting to the /login endpoint.

changelog.d/4824.misc

@@ -0,0 +1 @@
+Document the `generate` option for the docker image.

changelog.d/4825.misc

@@ -0,0 +1 @@
+Fix check-newsfragment for debian-only changes.

changelog.d/4828.misc

@@ -0,0 +1 @@
+Add some debug logging for device list updates to help with #4828.

changelog.d/4829.bugfix

@@ -0,0 +1 @@
+Fix potential race in handling missing updates in device list updates.

changelog.d/4832.misc

@@ -0,0 +1 @@
+Improve federation documentation, specifically .well-known support. Many thanks to @vaab.

changelog.d/4837.bugfix

@@ -0,0 +1 @@
+Fix bug where synapse expected an un-specced `prev_state` field on state events.

changelog.d/4838.bugfix

@@ -0,0 +1 @@
+Transfer a user's notification settings (push rules) on room upgrade.

changelog.d/4839.misc

@@ -0,0 +1 @@
+Disable captcha registration by default in unit tests.

changelog.d/4843.misc

@@ -0,0 +1 @@
+Add stuff back to the .gitignore.

changelog.d/4844.misc

@@ -0,0 +1 @@
+Clarify what registration_shared_secret allows for.

changelog.d/4846.feature

@@ -0,0 +1 @@
+The user directory has been rewritten to make it faster, with less chance of falling behind on a large server.

changelog.d/4847.misc

@@ -0,0 +1 @@
+Correctly log expected errors when fetching server keys.

changelog.d/4849.misc

@@ -0,0 +1 @@
+Update install docs to explicitly state a full-chain (not just the top-level) TLS certificate must be provided to Synapse. This caused some people's Synapse ports to appear correct in a browser but still (rightfully so) upset the federation tester.

changelog.d/4852.misc

@@ -0,0 +1 @@
+ Move client read-receipt processing to federation sender worker.

changelog.d/4853.feature

@@ -0,0 +1 @@
+Allow passing --daemonize flags to workers in the same way as with master.

changelog.d/4855.misc

@@ -0,0 +1 @@
+Refactor federation TransactionQueue.

changelog.d/4863.misc

@@ -0,0 +1 @@
+Comment out most options in the generated config.

changelog.d/4864.feature

@@ -0,0 +1 @@
+The user directory has been rewritten to make it faster, with less chance of falling behind on a large server.

changelog.d/4865.feature

@@ -0,0 +1 @@
+Add configurable rate limiting to the /login endpoint.

changelog.d/4879.misc

@@ -0,0 +1 @@
+Reinstate test case that runs unit tests against oldest supported dependencies.

changelog.d/4881.misc

@@ -0,0 +1 @@
+Update link to federation docs.

changelog.d/4886.bugfix

@@ -0,0 +1 @@
+fix test_auto_create_auto_join_where_no_consent.

changelog.d/4886.misc

@@ -0,0 +1 @@
+fix test_auto_create_auto_join_where_no_consent.

changelog.d/4887.feature

@@ -0,0 +1 @@
+The user directory has been rewritten to make it faster, with less chance of falling behind on a large server. 

changelog.d/4888.bugfix

@@ -0,0 +1,2 @@
+Fix a bug where hs_disabled_message was sometimes not correctly enforced.
+

changelog.d/4889.misc

@@ -0,0 +1 @@
+Use a regular HomeServerConfig object for unit tests rater than a Mock.

changelog.d/4890.feature

@@ -0,0 +1 @@
+Batch up outgoing read-receipts to reduce federation traffic.

changelog.d/4895.feature

@@ -0,0 +1 @@
+Add option to disable searching the user directory.

changelog.d/4895.misc

@@ -0,0 +1 @@
+Add some notes about tuning postgres for larger deployments.

changelog.d/4896.feature

@@ -0,0 +1 @@
+Add option to disable searching of local and remote public room lists.

changelog.d/4900.feature

@@ -0,0 +1 @@
+The user directory has been rewritten to make it faster, with less chance of falling behind on a large server.

changelog.d/4902.misc

@@ -0,0 +1 @@
+Add a config option for torture-testing worker replication.

changelog.d/4904.bugfix

@@ -0,0 +1 @@
+Fix bug in shutdown room admin API where it would fail if a user in the room hadn't consented to the privacy policy.

changelog.d/4905.misc

@@ -0,0 +1 @@
+Log requests which are simulated by the unit tests.

changelog.d/5552.misc

@@ -1 +0,0 @@
-Update github templates.

changelog.d/5596.bugfix

@@ -1 +0,0 @@
-Removed the `SYNAPSE_SMTP_*` docker container environment variables. Using these environment variables prevented the docker container from starting in Synapse v1.0, even though they didn't actually allow any functionality anyway. Users are advised to remove `SYNAPSE_SMTP_HOST`, `SYNAPSE_SMTP_PORT`, `SYNAPSE_SMTP_USER`, `SYNAPSE_SMTP_PASSWORD` and `SYNAPSE_SMTP_FROM` environment variables from their docker run commands.

contrib/cmdclient/console.py

@@ -15,7 +15,6 @@
 # limitations under the License.
 
 """ Starts a synapse client console. """
-from __future__ import print_function
 
 from twisted.internet import reactor, defer, threads
 from http import TwistedHttpClient
@@ -37,8 +36,9 @@ from signedjson.sign import verify_signed_json, SignatureVerifyException
 
 CONFIG_JSON = "cmdclient_config.json"
 
-TRUSTED_ID_SERVERS = ["localhost:8001"]
-
+TRUSTED_ID_SERVERS = [
+    'localhost:8001'
+]
 
 class SynapseCmd(cmd.Cmd):
 
@@ -58,7 +58,7 @@ class SynapseCmd(cmd.Cmd):
             "token": token,
             "verbose": "on",
             "complete_usernames": "on",
-            "send_delivery_receipts": "on",
+            "send_delivery_receipts": "on"
         }
         self.path_prefix = "/_matrix/client/api/v1"
         self.event_stream_token = "END"
@@ -109,7 +109,7 @@ class SynapseCmd(cmd.Cmd):
         by using $. E.g. 'config roomid room1' then 'raw get /rooms/$roomid'.
         """
         if len(line) == 0:
-            print(json.dumps(self.config, indent=4))
+            print json.dumps(self.config, indent=4)
             return
 
         try:
@@ -119,11 +119,12 @@ class SynapseCmd(cmd.Cmd):
             config_rules = [  # key, valid_values
                 ("verbose", ["on", "off"]),
                 ("complete_usernames", ["on", "off"]),
-                ("send_delivery_receipts", ["on", "off"]),
+                ("send_delivery_receipts", ["on", "off"])
             ]
             for key, valid_vals in config_rules:
                 if key == args["key"] and args["val"] not in valid_vals:
-                    print("%s value must be one of %s" % (args["key"], valid_vals))
+                    print "%s value must be one of %s" % (args["key"],
+                                                          valid_vals)
                     return
 
             # toggle the http client verbosity
@@ -132,11 +133,11 @@ class SynapseCmd(cmd.Cmd):
 
             # assign the new config
             self.config[args["key"]] = args["val"]
-            print(json.dumps(self.config, indent=4))
+            print json.dumps(self.config, indent=4)
 
             save_config(self.config)
         except Exception as e:
-            print(e)
+            print e
 
     def do_register(self, line):
         """Registers for a new account: "register <userid> <noupdate>"
@@ -152,32 +153,33 @@ class SynapseCmd(cmd.Cmd):
             pwd = getpass.getpass("Type a password for this user: ")
             pwd2 = getpass.getpass("Retype the password: ")
             if pwd != pwd2 or len(pwd) == 0:
-                print("Password mismatch.")
+                print "Password mismatch."
                 pwd = None
             else:
                 password = pwd
 
-        body = {"type": "m.login.password"}
+        body = {
+            "type": "m.login.password"
+        }
         if "userid" in args:
             body["user"] = args["userid"]
         if password:
             body["password"] = password
 
-        reactor.callFromThread(self._do_register, body, "noupdate" not in args)
+        reactor.callFromThread(self._do_register, body,
+                               "noupdate" not in args)
 
     @defer.inlineCallbacks
     def _do_register(self, data, update_config):
         # check the registration flows
         url = self._url() + "/register"
         json_res = yield self.http_client.do_request("GET", url)
-        print(json.dumps(json_res, indent=4))
+        print json.dumps(json_res, indent=4)
 
         passwordFlow = None
         for flow in json_res["flows"]:
-            if flow["type"] == "m.login.recaptcha" or (
-                "stages" in flow and "m.login.recaptcha" in flow["stages"]
-            ):
-                print("Unable to register: Home server requires captcha.")
+            if flow["type"] == "m.login.recaptcha" or ("stages" in flow and "m.login.recaptcha" in flow["stages"]):
+                print "Unable to register: Home server requires captcha."
                 return
             if flow["type"] == "m.login.password" and "stages" not in flow:
                 passwordFlow = flow
@@ -187,7 +189,7 @@ class SynapseCmd(cmd.Cmd):
             return
 
         json_res = yield self.http_client.do_request("POST", url, data=data)
-        print(json.dumps(json_res, indent=4))
+        print json.dumps(json_res, indent=4)
         if update_config and "user_id" in json_res:
             self.config["user"] = json_res["user_id"]
             self.config["token"] = json_res["access_token"]
@@ -199,7 +201,9 @@ class SynapseCmd(cmd.Cmd):
         """
         try:
             args = self._parse(line, ["user_id"], force_keys=True)
-            can_login = threads.blockingCallFromThread(reactor, self._check_can_login)
+            can_login = threads.blockingCallFromThread(
+                reactor,
+                self._check_can_login)
             if can_login:
                 p = getpass.getpass("Enter your password: ")
                 user = args["user_id"]
@@ -207,25 +211,29 @@ class SynapseCmd(cmd.Cmd):
                     domain = self._domain()
                     if domain:
                         user = "@" + user + ":" + domain
-
+                
                 reactor.callFromThread(self._do_login, user, p)
-                # print " got %s " % p
+                #print " got %s " % p
         except Exception as e:
-            print(e)
+            print e
 
     @defer.inlineCallbacks
     def _do_login(self, user, password):
         path = "/login"
-        data = {"user": user, "password": password, "type": "m.login.password"}
+        data = {
+            "user": user,
+            "password": password,
+            "type": "m.login.password"
+        }
         url = self._url() + path
         json_res = yield self.http_client.do_request("POST", url, data=data)
-        print(json_res)
+        print json_res
 
         if "access_token" in json_res:
             self.config["user"] = user
             self.config["token"] = json_res["access_token"]
             save_config(self.config)
-            print("Login successful.")
+            print "Login successful."
 
     @defer.inlineCallbacks
     def _check_can_login(self):
@@ -234,19 +242,18 @@ class SynapseCmd(cmd.Cmd):
         # submitting!
         url = self._url() + path
         json_res = yield self.http_client.do_request("GET", url)
-        print(json_res)
+        print json_res
 
         if "flows" not in json_res:
-            print("Failed to find any login flows.")
+            print "Failed to find any login flows."
             defer.returnValue(False)
 
-        flow = json_res["flows"][0]  # assume first is the one we want.
-        if "type" not in flow or "m.login.password" != flow["type"] or "stages" in flow:
+        flow = json_res["flows"][0] # assume first is the one we want.
+        if ("type" not in flow or "m.login.password" != flow["type"] or
+                "stages" in flow):
             fallback_url = self._url() + "/login/fallback"
-            print(
-                "Unable to login via the command line client. Please visit "
-                "%s to login." % fallback_url
-            )
+            print ("Unable to login via the command line client. Please visit "
+                "%s to login." % fallback_url)
             defer.returnValue(False)
         defer.returnValue(True)
 
@@ -256,33 +263,21 @@ class SynapseCmd(cmd.Cmd):
         <clientSecret> A string of characters generated when requesting an email that you'll supply in subsequent calls to identify yourself
         <sendAttempt> The number of times the user has requested an email. Leave this the same between requests to retry the request at the transport level. Increment it to request that the email be sent again.
         """
-        args = self._parse(line, ["address", "clientSecret", "sendAttempt"])
+        args = self._parse(line, ['address', 'clientSecret', 'sendAttempt'])
 
-        postArgs = {
-            "email": args["address"],
-            "clientSecret": args["clientSecret"],
-            "sendAttempt": args["sendAttempt"],
-        }
+        postArgs = {'email': args['address'], 'clientSecret': args['clientSecret'], 'sendAttempt': args['sendAttempt']}
 
         reactor.callFromThread(self._do_emailrequest, postArgs)
 
     @defer.inlineCallbacks
     def _do_emailrequest(self, args):
-        url = (
-            self._identityServerUrl()
-            + "/_matrix/identity/api/v1/validate/email/requestToken"
-        )
+        url = self._identityServerUrl()+"/_matrix/identity/api/v1/validate/email/requestToken"
 
-        json_res = yield self.http_client.do_request(
-            "POST",
-            url,
-            data=urllib.urlencode(args),
-            jsonreq=False,
-            headers={"Content-Type": ["application/x-www-form-urlencoded"]},
-        )
-        print(json_res)
-        if "sid" in json_res:
-            print("Token sent. Your session ID is %s" % (json_res["sid"]))
+        json_res = yield self.http_client.do_request("POST", url, data=urllib.urlencode(args), jsonreq=False,
+                                                     headers={'Content-Type': ['application/x-www-form-urlencoded']})
+        print json_res
+        if 'sid' in json_res:
+            print "Token sent. Your session ID is %s" % (json_res['sid'])
 
     def do_emailvalidate(self, line):
         """Validate and associate a third party ID
@@ -290,56 +285,39 @@ class SynapseCmd(cmd.Cmd):
         <token> The token sent to your third party identifier address
         <clientSecret> The same clientSecret you supplied in requestToken
         """
-        args = self._parse(line, ["sid", "token", "clientSecret"])
+        args = self._parse(line, ['sid', 'token', 'clientSecret'])
 
-        postArgs = {
-            "sid": args["sid"],
-            "token": args["token"],
-            "clientSecret": args["clientSecret"],
-        }
+        postArgs = { 'sid' : args['sid'], 'token' : args['token'], 'clientSecret': args['clientSecret'] }
 
         reactor.callFromThread(self._do_emailvalidate, postArgs)
 
     @defer.inlineCallbacks
     def _do_emailvalidate(self, args):
-        url = (
-            self._identityServerUrl()
-            + "/_matrix/identity/api/v1/validate/email/submitToken"
-        )
+        url = self._identityServerUrl()+"/_matrix/identity/api/v1/validate/email/submitToken"
 
-        json_res = yield self.http_client.do_request(
-            "POST",
-            url,
-            data=urllib.urlencode(args),
-            jsonreq=False,
-            headers={"Content-Type": ["application/x-www-form-urlencoded"]},
-        )
-        print(json_res)
+        json_res = yield self.http_client.do_request("POST", url, data=urllib.urlencode(args), jsonreq=False,
+                                                     headers={'Content-Type': ['application/x-www-form-urlencoded']})
+        print json_res
 
     def do_3pidbind(self, line):
         """Validate and associate a third party ID
         <sid> The session ID (sid) given to you in the response to requestToken
         <clientSecret> The same clientSecret you supplied in requestToken
         """
-        args = self._parse(line, ["sid", "clientSecret"])
+        args = self._parse(line, ['sid', 'clientSecret'])
 
-        postArgs = {"sid": args["sid"], "clientSecret": args["clientSecret"]}
-        postArgs["mxid"] = self.config["user"]
+        postArgs = { 'sid' : args['sid'], 'clientSecret': args['clientSecret'] }
+        postArgs['mxid'] = self.config["user"]
 
         reactor.callFromThread(self._do_3pidbind, postArgs)
 
     @defer.inlineCallbacks
     def _do_3pidbind(self, args):
-        url = self._identityServerUrl() + "/_matrix/identity/api/v1/3pid/bind"
+        url = self._identityServerUrl()+"/_matrix/identity/api/v1/3pid/bind"
 
-        json_res = yield self.http_client.do_request(
-            "POST",
-            url,
-            data=urllib.urlencode(args),
-            jsonreq=False,
-            headers={"Content-Type": ["application/x-www-form-urlencoded"]},
-        )
-        print(json_res)
+        json_res = yield self.http_client.do_request("POST", url, data=urllib.urlencode(args), jsonreq=False,
+                                                     headers={'Content-Type': ['application/x-www-form-urlencoded']})
+        print json_res
 
     def do_join(self, line):
         """Joins a room: "join <roomid>" """
@@ -347,7 +325,7 @@ class SynapseCmd(cmd.Cmd):
             args = self._parse(line, ["roomid"], force_keys=True)
             self._do_membership_change(args["roomid"], "join", self._usr())
         except Exception as e:
-            print(e)
+            print e
 
     def do_joinalias(self, line):
         try:
@@ -355,7 +333,7 @@ class SynapseCmd(cmd.Cmd):
             path = "/join/%s" % urllib.quote(args["roomname"])
             reactor.callFromThread(self._run_and_pprint, "POST", path, {})
         except Exception as e:
-            print(e)
+            print e
 
     def do_topic(self, line):
         """"topic [set|get] <roomid> [<newtopic>]"
@@ -365,24 +343,26 @@ class SynapseCmd(cmd.Cmd):
         try:
             args = self._parse(line, ["action", "roomid", "topic"])
             if "action" not in args or "roomid" not in args:
-                print("Must specify set|get and a room ID.")
+                print "Must specify set|get and a room ID."
                 return
             if args["action"].lower() not in ["set", "get"]:
-                print("Must specify set|get, not %s" % args["action"])
+                print "Must specify set|get, not %s" % args["action"]
                 return
 
             path = "/rooms/%s/topic" % urllib.quote(args["roomid"])
 
             if args["action"].lower() == "set":
                 if "topic" not in args:
-                    print("Must specify a new topic.")
+                    print "Must specify a new topic."
                     return
-                body = {"topic": args["topic"]}
+                body = {
+                    "topic": args["topic"]
+                }
                 reactor.callFromThread(self._run_and_pprint, "PUT", path, body)
             elif args["action"].lower() == "get":
                 reactor.callFromThread(self._run_and_pprint, "GET", path)
         except Exception as e:
-            print(e)
+            print e
 
     def do_invite(self, line):
         """Invite a user to a room: "invite <userid> <roomid>" """
@@ -393,64 +373,49 @@ class SynapseCmd(cmd.Cmd):
 
             reactor.callFromThread(self._do_invite, args["roomid"], user_id)
         except Exception as e:
-            print(e)
+            print e
 
     @defer.inlineCallbacks
     def _do_invite(self, roomid, userstring):
-        if not userstring.startswith("@") and self._is_on("complete_usernames"):
-            url = self._identityServerUrl() + "/_matrix/identity/api/v1/lookup"
+        if (not userstring.startswith('@') and
+                    self._is_on("complete_usernames")):
+            url = self._identityServerUrl()+"/_matrix/identity/api/v1/lookup"
 
-            json_res = yield self.http_client.do_request(
-                "GET", url, qparams={"medium": "email", "address": userstring}
-            )
+            json_res = yield self.http_client.do_request("GET", url, qparams={'medium':'email','address':userstring})
 
             mxid = None
 
-            if "mxid" in json_res and "signatures" in json_res:
-                url = (
-                    self._identityServerUrl()
-                    + "/_matrix/identity/api/v1/pubkey/ed25519"
-                )
+            if 'mxid' in json_res and 'signatures' in json_res:
+                url = self._identityServerUrl()+"/_matrix/identity/api/v1/pubkey/ed25519"
 
                 pubKey = None
                 pubKeyObj = yield self.http_client.do_request("GET", url)
-                if "public_key" in pubKeyObj:
-                    pubKey = nacl.signing.VerifyKey(
-                        pubKeyObj["public_key"], encoder=nacl.encoding.HexEncoder
-                    )
+                if 'public_key' in pubKeyObj:
+                    pubKey = nacl.signing.VerifyKey(pubKeyObj['public_key'], encoder=nacl.encoding.HexEncoder)
                 else:
-                    print("No public key found in pubkey response!")
+                    print "No public key found in pubkey response!"
 
                 sigValid = False
 
                 if pubKey:
-                    for signame in json_res["signatures"]:
+                    for signame in json_res['signatures']:
                         if signame not in TRUSTED_ID_SERVERS:
-                            print(
-                                "Ignoring signature from untrusted server %s"
-                                % (signame)
-                            )
+                            print "Ignoring signature from untrusted server %s" % (signame)
                         else:
                             try:
                                 verify_signed_json(json_res, signame, pubKey)
                                 sigValid = True
-                                print(
-                                    "Mapping %s -> %s correctly signed by %s"
-                                    % (userstring, json_res["mxid"], signame)
-                                )
+                                print "Mapping %s -> %s correctly signed by %s" % (userstring, json_res['mxid'], signame)
                                 break
                             except SignatureVerifyException as e:
-                                print("Invalid signature from %s" % (signame))
-                                print(e)
+                                print "Invalid signature from %s" % (signame)
+                                print e
 
                 if sigValid:
-                    print("Resolved 3pid %s to %s" % (userstring, json_res["mxid"]))
-                    mxid = json_res["mxid"]
+                    print "Resolved 3pid %s to %s" % (userstring, json_res['mxid'])
+                    mxid = json_res['mxid']
                 else:
-                    print(
-                        "Got association for %s but couldn't verify signature"
-                        % (userstring)
-                    )
+                    print "Got association for %s but couldn't verify signature" % (userstring)
 
             if not mxid:
                 mxid = "@" + userstring + ":" + self._domain()
@@ -463,17 +428,18 @@ class SynapseCmd(cmd.Cmd):
             args = self._parse(line, ["roomid"], force_keys=True)
             self._do_membership_change(args["roomid"], "leave", self._usr())
         except Exception as e:
-            print(e)
+            print e
 
     def do_send(self, line):
         """Sends a message. "send <roomid> <body>" """
         args = self._parse(line, ["roomid", "body"])
         txn_id = "txn%s" % int(time.time())
-        path = "/rooms/%s/send/m.room.message/%s" % (
-            urllib.quote(args["roomid"]),
-            txn_id,
-        )
-        body_json = {"msgtype": "m.text", "body": args["body"]}
+        path = "/rooms/%s/send/m.room.message/%s" % (urllib.quote(args["roomid"]),
+                                             txn_id)
+        body_json = {
+            "msgtype": "m.text",
+            "body": args["body"]
+        }
         reactor.callFromThread(self._run_and_pprint, "PUT", path, body_json)
 
     def do_list(self, line):
@@ -487,10 +453,10 @@ class SynapseCmd(cmd.Cmd):
         """
         args = self._parse(line, ["type", "roomid", "qp"])
         if not "type" in args or not "roomid" in args:
-            print("Must specify type and room ID.")
+            print "Must specify type and room ID."
             return
         if args["type"] not in ["members", "messages"]:
-            print("Unrecognised type: %s" % args["type"])
+            print "Unrecognised type: %s" % args["type"]
             return
         room_id = args["roomid"]
         path = "/rooms/%s/%s" % (urllib.quote(room_id), args["type"])
@@ -502,10 +468,11 @@ class SynapseCmd(cmd.Cmd):
                     key_value = key_value_str.split("=")
                     qp[key_value[0]] = key_value[1]
                 except:
-                    print("Bad query param: %s" % key_value)
+                    print "Bad query param: %s" % key_value
                     return
 
-        reactor.callFromThread(self._run_and_pprint, "GET", path, query_params=qp)
+        reactor.callFromThread(self._run_and_pprint, "GET", path,
+                               query_params=qp)
 
     def do_create(self, line):
         """Creates a room.
@@ -541,22 +508,14 @@ class SynapseCmd(cmd.Cmd):
         args = self._parse(line, ["method", "path", "data"])
         # sanity check
         if "method" not in args or "path" not in args:
-            print("Must specify path and method.")
+            print "Must specify path and method."
             return
 
         args["method"] = args["method"].upper()
-        valid_methods = [
-            "PUT",
-            "GET",
-            "POST",
-            "DELETE",
-            "XPUT",
-            "XGET",
-            "XPOST",
-            "XDELETE",
-        ]
+        valid_methods = ["PUT", "GET", "POST", "DELETE",
+                         "XPUT", "XGET", "XPOST", "XDELETE"]
         if args["method"] not in valid_methods:
-            print("Unsupported method: %s" % args["method"])
+            print "Unsupported method: %s" % args["method"]
             return
 
         if "data" not in args:
@@ -565,7 +524,7 @@ class SynapseCmd(cmd.Cmd):
             try:
                 args["data"] = json.loads(args["data"])
             except Exception as e:
-                print("Data is not valid JSON. %s" % e)
+                print "Data is not valid JSON. %s" % e
                 return
 
         qp = {"access_token": self._tok()}
@@ -581,13 +540,10 @@ class SynapseCmd(cmd.Cmd):
             except:
                 pass
 
-        reactor.callFromThread(
-            self._run_and_pprint,
-            args["method"],
-            args["path"],
-            args["data"],
-            query_params=qp,
-        )
+        reactor.callFromThread(self._run_and_pprint, args["method"],
+                                                     args["path"],
+                                                     args["data"],
+                                                     query_params=qp)
 
     def do_stream(self, line):
         """Stream data from the server: "stream <longpoll timeout ms>" """
@@ -597,29 +553,26 @@ class SynapseCmd(cmd.Cmd):
             try:
                 timeout = int(args["timeout"])
             except ValueError:
-                print("Timeout must be in milliseconds.")
+                print "Timeout must be in milliseconds."
                 return
         reactor.callFromThread(self._do_event_stream, timeout)
 
     @defer.inlineCallbacks
     def _do_event_stream(self, timeout):
         res = yield self.http_client.get_json(
-            self._url() + "/events",
-            {
-                "access_token": self._tok(),
-                "timeout": str(timeout),
-                "from": self.event_stream_token,
-            },
-        )
-        print(json.dumps(res, indent=4))
+                self._url() + "/events",
+                {
+                    "access_token": self._tok(),
+                    "timeout": str(timeout),
+                    "from": self.event_stream_token
+                })
+        print json.dumps(res, indent=4)
 
         if "chunk" in res:
             for event in res["chunk"]:
-                if (
-                    event["type"] == "m.room.message"
-                    and self._is_on("send_delivery_receipts")
-                    and event["user_id"] != self._usr()
-                ):  # not sent by us
+                if (event["type"] == "m.room.message" and
+                        self._is_on("send_delivery_receipts") and
+                        event["user_id"] != self._usr()):  # not sent by us
                     self._send_receipt(event, "d")
 
         # update the position in the stram
@@ -627,28 +580,18 @@ class SynapseCmd(cmd.Cmd):
             self.event_stream_token = res["end"]
 
     def _send_receipt(self, event, feedback_type):
-        path = "/rooms/%s/messages/%s/%s/feedback/%s/%s" % (
-            urllib.quote(event["room_id"]),
-            event["user_id"],
-            event["msg_id"],
-            self._usr(),
-            feedback_type,
-        )
+        path = ("/rooms/%s/messages/%s/%s/feedback/%s/%s" %
+               (urllib.quote(event["room_id"]), event["user_id"], event["msg_id"],
+                self._usr(), feedback_type))
         data = {}
-        reactor.callFromThread(
-            self._run_and_pprint,
-            "PUT",
-            path,
-            data=data,
-            alt_text="Sent receipt for %s" % event["msg_id"],
-        )
+        reactor.callFromThread(self._run_and_pprint, "PUT", path, data=data,
+                               alt_text="Sent receipt for %s" % event["msg_id"])
 
     def _do_membership_change(self, roomid, membership, userid):
-        path = "/rooms/%s/state/m.room.member/%s" % (
-            urllib.quote(roomid),
-            urllib.quote(userid),
-        )
-        data = {"membership": membership}
+        path = "/rooms/%s/state/m.room.member/%s" % (urllib.quote(roomid), urllib.quote(userid))
+        data = {
+            "membership": membership
+        }
         reactor.callFromThread(self._run_and_pprint, "PUT", path, data=data)
 
     def do_displayname(self, line):
@@ -701,20 +644,15 @@ class SynapseCmd(cmd.Cmd):
         for i, arg in enumerate(line_args):
             for config_key in self.config:
                 if ("$" + config_key) in arg:
-                    arg = arg.replace("$" + config_key, self.config[config_key])
+                    arg = arg.replace("$" + config_key,
+                                      self.config[config_key])
             line_args[i] = arg
 
         return dict(zip(keys, line_args))
 
     @defer.inlineCallbacks
-    def _run_and_pprint(
-        self,
-        method,
-        path,
-        data=None,
-        query_params={"access_token": None},
-        alt_text=None,
-    ):
+    def _run_and_pprint(self, method, path, data=None,
+                        query_params={"access_token": None}, alt_text=None):
         """ Runs an HTTP request and pretty prints the output.
 
         Args:
@@ -727,31 +665,31 @@ class SynapseCmd(cmd.Cmd):
         if "access_token" in query_params:
             query_params["access_token"] = self._tok()
 
-        json_res = yield self.http_client.do_request(
-            method, url, data=data, qparams=query_params
-        )
+        json_res = yield self.http_client.do_request(method, url,
+                                                    data=data,
+                                                    qparams=query_params)
         if alt_text:
-            print(alt_text)
+            print alt_text
         else:
-            print(json.dumps(json_res, indent=4))
+            print json.dumps(json_res, indent=4)
 
 
 def save_config(config):
-    with open(CONFIG_JSON, "w") as out:
+    with open(CONFIG_JSON, 'w') as out:
         json.dump(config, out)
 
 
 def main(server_url, identity_server_url, username, token, config_path):
-    print("Synapse command line client")
-    print("===========================")
-    print("Server: %s" % server_url)
-    print("Type 'help' to get started.")
-    print("Close this console with CTRL+C then CTRL+D.")
+    print "Synapse command line client"
+    print "==========================="
+    print "Server: %s" % server_url
+    print "Type 'help' to get started."
+    print "Close this console with CTRL+C then CTRL+D."
     if not username or not token:
-        print("-  'register <username>' - Register an account")
-        print("-  'stream' - Connect to the event stream")
-        print("-  'create <roomid>' - Create a room")
-        print("-  'send <roomid> <message>' - Send a message")
+        print "-  'register <username>' - Register an account"
+        print "-  'stream' - Connect to the event stream"
+        print "-  'create <roomid>' - Create a room"
+        print "-  'send <roomid> <message>' - Send a message"
     http_client = TwistedHttpClient()
 
     # the command line client
@@ -761,13 +699,13 @@ def main(server_url, identity_server_url, username, token, config_path):
     global CONFIG_JSON
     CONFIG_JSON = config_path  # bit cheeky, but just overwrite the global
     try:
-        with open(config_path, "r") as config:
+        with open(config_path, 'r') as config:
             syn_cmd.config = json.load(config)
             try:
                 http_client.verbose = "on" == syn_cmd.config["verbose"]
             except:
                 pass
-            print("Loaded config from %s" % config_path)
+            print "Loaded config from %s" % config_path
     except:
         pass
 
@@ -778,37 +716,27 @@ def main(server_url, identity_server_url, username, token, config_path):
     reactor.run()
 
 
-if __name__ == "__main__":
+if __name__ == '__main__':
     parser = argparse.ArgumentParser("Starts a synapse client.")
     parser.add_argument(
-        "-s",
-        "--server",
-        dest="server",
-        default="http://localhost:8008",
-        help="The URL of the home server to talk to.",
-    )
+        "-s", "--server", dest="server", default="http://localhost:8008",
+        help="The URL of the home server to talk to.")
     parser.add_argument(
-        "-i",
-        "--identity-server",
-        dest="identityserver",
-        default="http://localhost:8090",
-        help="The URL of the identity server to talk to.",
-    )
+        "-i", "--identity-server", dest="identityserver", default="http://localhost:8090",
+        help="The URL of the identity server to talk to.")
     parser.add_argument(
-        "-u", "--username", dest="username", help="Your username on the server."
-    )
-    parser.add_argument("-t", "--token", dest="token", help="Your access token.")
+        "-u", "--username", dest="username",
+        help="Your username on the server.")
     parser.add_argument(
-        "-c",
-        "--config",
-        dest="config",
-        default=CONFIG_JSON,
-        help="The location of the config.json file to read from.",
-    )
+        "-t", "--token", dest="token",
+        help="Your access token.")
+    parser.add_argument(
+        "-c", "--config", dest="config", default=CONFIG_JSON,
+        help="The location of the config.json file to read from.")
     args = parser.parse_args()
 
     if not args.server:
-        print("You must supply a server URL to communicate with.")
+        print "You must supply a server URL to communicate with."
         parser.print_help()
         sys.exit(1)
 

contrib/cmdclient/http.py

@@ -13,7 +13,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from __future__ import print_function
 from twisted.web.client import Agent, readBody
 from twisted.web.http_headers import Headers
 from twisted.internet import defer, reactor
@@ -73,7 +72,9 @@ class TwistedHttpClient(HttpClient):
     @defer.inlineCallbacks
     def put_json(self, url, data):
         response = yield self._create_put_request(
-            url, data, headers_dict={"Content-Type": ["application/json"]}
+            url,
+            data,
+            headers_dict={"Content-Type": ["application/json"]}
         )
         body = yield readBody(response)
         defer.returnValue((response.code, body))
@@ -93,34 +94,40 @@ class TwistedHttpClient(HttpClient):
         """
 
         if "Content-Type" not in headers_dict:
-            raise defer.error(RuntimeError("Must include Content-Type header for PUTs"))
+            raise defer.error(
+                RuntimeError("Must include Content-Type header for PUTs"))
 
         return self._create_request(
-            "PUT", url, producer=_JsonProducer(json_data), headers_dict=headers_dict
+            "PUT",
+            url,
+            producer=_JsonProducer(json_data),
+            headers_dict=headers_dict
         )
 
     def _create_get_request(self, url, headers_dict={}):
         """ Wrapper of _create_request to issue a GET request
         """
-        return self._create_request("GET", url, headers_dict=headers_dict)
+        return self._create_request(
+            "GET",
+            url,
+            headers_dict=headers_dict
+        )
 
     @defer.inlineCallbacks
-    def do_request(
-        self, method, url, data=None, qparams=None, jsonreq=True, headers={}
-    ):
+    def do_request(self, method, url, data=None, qparams=None, jsonreq=True, headers={}):
         if qparams:
             url = "%s?%s" % (url, urllib.urlencode(qparams, True))
 
         if jsonreq:
             prod = _JsonProducer(data)
-            headers["Content-Type"] = ["application/json"]
+            headers['Content-Type'] = ["application/json"];
         else:
             prod = _RawProducer(data)
 
         if method in ["POST", "PUT"]:
-            response = yield self._create_request(
-                method, url, producer=prod, headers_dict=headers
-            )
+            response = yield self._create_request(method, url,
+                    producer=prod,
+                    headers_dict=headers)
         else:
             response = yield self._create_request(method, url)
 
@@ -134,24 +141,27 @@ class TwistedHttpClient(HttpClient):
         headers_dict["User-Agent"] = ["Synapse Cmd Client"]
 
         retries_left = 5
-        print("%s to %s with headers %s" % (method, url, headers_dict))
+        print "%s to %s with headers %s" % (method, url, headers_dict)
         if self.verbose and producer:
             if "password" in producer.data:
                 temp = producer.data["password"]
                 producer.data["password"] = "[REDACTED]"
-                print(json.dumps(producer.data, indent=4))
+                print json.dumps(producer.data, indent=4)
                 producer.data["password"] = temp
             else:
-                print(json.dumps(producer.data, indent=4))
+                print json.dumps(producer.data, indent=4)
 
         while True:
             try:
                 response = yield self.agent.request(
-                    method, url.encode("UTF8"), Headers(headers_dict), producer
+                    method,
+                    url.encode("UTF8"),
+                    Headers(headers_dict),
+                    producer
                 )
                 break
             except Exception as e:
-                print("uh oh: %s" % e)
+                print "uh oh: %s" % e
                 if retries_left:
                     yield self.sleep(2 ** (5 - retries_left))
                     retries_left -= 1
@@ -159,8 +169,8 @@ class TwistedHttpClient(HttpClient):
                     raise e
 
         if self.verbose:
-            print("Status %s %s" % (response.code, response.phrase))
-            print(pformat(list(response.headers.getAllRawHeaders())))
+            print "Status %s %s" % (response.code, response.phrase)
+            print pformat(list(response.headers.getAllRawHeaders()))
         defer.returnValue(response)
 
     def sleep(self, seconds):
@@ -168,7 +178,6 @@ class TwistedHttpClient(HttpClient):
         reactor.callLater(seconds, d.callback, seconds)
         return d
 
-
 class _RawProducer(object):
     def __init__(self, data):
         self.data = data
@@ -185,11 +194,9 @@ class _RawProducer(object):
     def stopProducing(self):
         pass
 
-
 class _JsonProducer(object):
     """ Used by the twisted http client to create the HTTP body from json
     """
-
     def __init__(self, jsn):
         self.data = jsn
         self.body = json.dumps(jsn).encode("utf8")

contrib/docker/README.md

@@ -1,9 +1,5 @@
 # Synapse Docker
 
-FIXME: this is out-of-date as of
-https://github.com/matrix-org/synapse/issues/5518. Contributions to bring it up
-to date would be welcome.
-
 ### Automated configuration
 
 It is recommended that you use Docker Compose to run your containers, including

contrib/experiments/cursesio.py

@@ -19,13 +19,13 @@ from curses.ascii import isprint
 from twisted.internet import reactor
 
 
-class CursesStdIO:
+class CursesStdIO():
     def __init__(self, stdscr, callback=None):
         self.statusText = "Synapse test app -"
-        self.searchText = ""
+        self.searchText = ''
         self.stdscr = stdscr
 
-        self.logLine = ""
+        self.logLine = ''
 
         self.callback = callback
 
@@ -71,7 +71,8 @@ class CursesStdIO:
         i = 0
         index = len(self.lines) - 1
         while i < (self.rows - 3) and index >= 0:
-            self.stdscr.addstr(self.rows - 3 - i, 0, self.lines[index], curses.A_NORMAL)
+            self.stdscr.addstr(self.rows - 3 - i, 0, self.lines[index],
+                               curses.A_NORMAL)
             i = i + 1
             index = index - 1
 
@@ -84,13 +85,15 @@ class CursesStdIO:
             raise RuntimeError("TextTooLongError")
 
         self.stdscr.addstr(
-            self.rows - 2, 0, text + " " * (self.cols - len(text)), curses.A_STANDOUT
-        )
+            self.rows - 2, 0,
+            text + ' ' * (self.cols - len(text)),
+            curses.A_STANDOUT)
 
     def printLogLine(self, text):
         self.stdscr.addstr(
-            0, 0, text + " " * (self.cols - len(text)), curses.A_STANDOUT
-        )
+            0, 0,
+            text + ' ' * (self.cols - len(text)),
+            curses.A_STANDOUT)
 
     def doRead(self):
         """ Input is ready! """
@@ -102,7 +105,7 @@ class CursesStdIO:
 
         elif c == curses.KEY_ENTER or c == 10:
             text = self.searchText
-            self.searchText = ""
+            self.searchText = ''
 
             self.print_line(">> %s" % text)
 
@@ -119,13 +122,11 @@ class CursesStdIO:
                 return
             self.searchText = self.searchText + chr(c)
 
-        self.stdscr.addstr(
-            self.rows - 1,
-            0,
-            self.searchText + (" " * (self.cols - len(self.searchText) - 2)),
-        )
+        self.stdscr.addstr(self.rows - 1, 0,
+                           self.searchText + (' ' * (
+                           self.cols - len(self.searchText) - 2)))
 
-        self.paintStatus(self.statusText + " %d" % len(self.searchText))
+        self.paintStatus(self.statusText + ' %d' % len(self.searchText))
         self.stdscr.move(self.rows - 1, len(self.searchText))
         self.stdscr.refresh()
 
@@ -142,6 +143,7 @@ class CursesStdIO:
 
 
 class Callback(object):
+
     def __init__(self, stdio):
         self.stdio = stdio
 
@@ -150,7 +152,7 @@ class Callback(object):
 
 
 def main(stdscr):
-    screen = CursesStdIO(stdscr)  # create Screen object
+    screen = CursesStdIO(stdscr)   # create Screen object
 
     callback = Callback(screen)
 
@@ -162,5 +164,5 @@ def main(stdscr):
     screen.close()
 
 
-if __name__ == "__main__":
+if __name__ == '__main__':
     curses.wrapper(main)

contrib/experiments/test_messaging.py

@@ -28,7 +28,9 @@ Currently assumes the local address is localhost:<port>
 """
 
 
-from synapse.federation import ReplicationHandler
+from synapse.federation import (
+    ReplicationHandler
+)
 
 from synapse.federation.units import Pdu
 
@@ -36,7 +38,7 @@ from synapse.util import origin_from_ucid
 
 from synapse.app.homeserver import SynapseHomeServer
 
-# from synapse.util.logutils import log_function
+#from synapse.util.logutils import log_function
 
 from twisted.internet import reactor, defer
 from twisted.python import log
@@ -81,7 +83,7 @@ class InputOutput(object):
                 room_name, = m.groups()
                 self.print_line("%s joining %s" % (self.user, room_name))
                 self.server.join_room(room_name, self.user, self.user)
-                # self.print_line("OK.")
+                #self.print_line("OK.")
                 return
 
             m = re.match("^invite (\S+) (\S+)$", line)
@@ -90,7 +92,7 @@ class InputOutput(object):
                 room_name, invitee = m.groups()
                 self.print_line("%s invited to %s" % (invitee, room_name))
                 self.server.invite_to_room(room_name, self.user, invitee)
-                # self.print_line("OK.")
+                #self.print_line("OK.")
                 return
 
             m = re.match("^send (\S+) (.*)$", line)
@@ -99,7 +101,7 @@ class InputOutput(object):
                 room_name, body = m.groups()
                 self.print_line("%s send to %s" % (self.user, room_name))
                 self.server.send_message(room_name, self.user, body)
-                # self.print_line("OK.")
+                #self.print_line("OK.")
                 return
 
             m = re.match("^backfill (\S+)$", line)
@@ -123,6 +125,7 @@ class InputOutput(object):
 
 
 class IOLoggerHandler(logging.Handler):
+
     def __init__(self, io):
         logging.Handler.__init__(self)
         self.io = io
@@ -139,7 +142,6 @@ class Room(object):
     """ Used to store (in memory) the current membership state of a room, and
     which home servers we should send PDUs associated with the room to.
     """
-
     def __init__(self, room_name):
         self.room_name = room_name
         self.invited = set()
@@ -173,7 +175,6 @@ class HomeServer(ReplicationHandler):
     """ A very basic home server implentation that allows people to join a
     room and then invite other people.
     """
-
     def __init__(self, server_name, replication_layer, output):
         self.server_name = server_name
         self.replication_layer = replication_layer
@@ -196,27 +197,26 @@ class HomeServer(ReplicationHandler):
             elif pdu.content["membership"] == "invite":
                 self._on_invite(pdu.origin, pdu.context, pdu.state_key)
         else:
-            self.output.print_line(
-                "#%s (unrec) %s = %s"
-                % (pdu.context, pdu.pdu_type, json.dumps(pdu.content))
+            self.output.print_line("#%s (unrec) %s = %s" %
+                (pdu.context, pdu.pdu_type, json.dumps(pdu.content))
             )
 
-    # def on_state_change(self, pdu):
-    ##self.output.print_line("#%s (state) %s *** %s" %
-    ##(pdu.context, pdu.state_key, pdu.pdu_type)
-    ##)
+    #def on_state_change(self, pdu):
+        ##self.output.print_line("#%s (state) %s *** %s" %
+                ##(pdu.context, pdu.state_key, pdu.pdu_type)
+            ##)
 
-    # if "joinee" in pdu.content:
-    # self._on_join(pdu.context, pdu.content["joinee"])
-    # elif "invitee" in pdu.content:
-    # self._on_invite(pdu.origin, pdu.context, pdu.content["invitee"])
+        #if "joinee" in pdu.content:
+            #self._on_join(pdu.context, pdu.content["joinee"])
+        #elif "invitee" in pdu.content:
+            #self._on_invite(pdu.origin, pdu.context, pdu.content["invitee"])
 
     def _on_message(self, pdu):
         """ We received a message
         """
-        self.output.print_line(
-            "#%s %s %s" % (pdu.context, pdu.content["sender"], pdu.content["body"])
-        )
+        self.output.print_line("#%s %s %s" %
+                (pdu.context, pdu.content["sender"], pdu.content["body"])
+            )
 
     def _on_join(self, context, joinee):
         """ Someone has joined a room, either a remote user or a local user
@@ -224,7 +224,9 @@ class HomeServer(ReplicationHandler):
         room = self._get_or_create_room(context)
         room.add_participant(joinee)
 
-        self.output.print_line("#%s %s %s" % (context, joinee, "*** JOINED"))
+        self.output.print_line("#%s %s %s" %
+                (context, joinee, "*** JOINED")
+            )
 
     def _on_invite(self, origin, context, invitee):
         """ Someone has been invited
@@ -232,7 +234,9 @@ class HomeServer(ReplicationHandler):
         room = self._get_or_create_room(context)
         room.add_invited(invitee)
 
-        self.output.print_line("#%s %s %s" % (context, invitee, "*** INVITED"))
+        self.output.print_line("#%s %s %s" %
+                (context, invitee, "*** INVITED")
+            )
 
         if not room.have_got_metadata and origin is not self.server_name:
             logger.debug("Get room state")
@@ -268,14 +272,14 @@ class HomeServer(ReplicationHandler):
 
         try:
             pdu = Pdu.create_new(
-                context=room_name,
-                pdu_type="sy.room.member",
-                is_state=True,
-                state_key=joinee,
-                content={"membership": "join"},
-                origin=self.server_name,
-                destinations=destinations,
-            )
+                    context=room_name,
+                    pdu_type="sy.room.member",
+                    is_state=True,
+                    state_key=joinee,
+                    content={"membership": "join"},
+                    origin=self.server_name,
+                    destinations=destinations,
+                )
             yield self.replication_layer.send_pdu(pdu)
         except Exception as e:
             logger.exception(e)
@@ -314,21 +318,21 @@ class HomeServer(ReplicationHandler):
         return self.replication_layer.backfill(dest, room_name, limit)
 
     def _get_room_remote_servers(self, room_name):
-        return [i for i in self.joined_rooms.setdefault(room_name).servers]
+        return [i for i in self.joined_rooms.setdefault(room_name,).servers]
 
     def _get_or_create_room(self, room_name):
         return self.joined_rooms.setdefault(room_name, Room(room_name))
 
     def get_servers_for_context(self, context):
         return defer.succeed(
-            self.joined_rooms.setdefault(context, Room(context)).servers
-        )
+                self.joined_rooms.setdefault(context, Room(context)).servers
+            )
 
 
 def main(stdscr):
     parser = argparse.ArgumentParser()
-    parser.add_argument("user", type=str)
-    parser.add_argument("-v", "--verbose", action="count")
+    parser.add_argument('user', type=str)
+    parser.add_argument('-v', '--verbose', action='count')
     args = parser.parse_args()
 
     user = args.user
@@ -338,9 +342,8 @@ def main(stdscr):
 
     root_logger = logging.getLogger()
 
-    formatter = logging.Formatter(
-        "%(asctime)s - %(name)s - %(lineno)d - " "%(levelname)s - %(message)s"
-    )
+    formatter = logging.Formatter('%(asctime)s - %(name)s - %(lineno)d - '
+            '%(levelname)s - %(message)s')
     if not os.path.exists("logs"):
         os.makedirs("logs")
     fh = logging.FileHandler("logs/%s" % user)

contrib/graph/graph.py

@@ -1,5 +1,3 @@
-from __future__ import print_function
-
 # Copyright 2014-2016 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -50,7 +48,7 @@ def make_graph(pdus, room, filename_prefix):
             c = colors.pop()
             color_map[o] = c
         except:
-            print("Run out of colours!")
+            print "Run out of colours!"
             color_map[o] = "black"
 
     graph = pydot.Dot(graph_name="Test")
@@ -59,9 +57,9 @@ def make_graph(pdus, room, filename_prefix):
         name = make_name(pdu.get("pdu_id"), pdu.get("origin"))
         pdu_map[name] = pdu
 
-        t = datetime.datetime.fromtimestamp(float(pdu["ts"]) / 1000).strftime(
-            "%Y-%m-%d %H:%M:%S,%f"
-        )
+        t = datetime.datetime.fromtimestamp(
+            float(pdu["ts"]) / 1000
+        ).strftime('%Y-%m-%d %H:%M:%S,%f')
 
         label = (
             "<"
@@ -81,7 +79,11 @@ def make_graph(pdus, room, filename_prefix):
             "depth": pdu.get("depth"),
         }
 
-        node = pydot.Node(name=name, label=label, color=color_map[pdu.get("origin")])
+        node = pydot.Node(
+            name=name,
+            label=label,
+            color=color_map[pdu.get("origin")]
+        )
         node_map[name] = node
         graph.add_node(node)
 
@@ -91,7 +93,7 @@ def make_graph(pdus, room, filename_prefix):
             end_name = make_name(i, o)
 
             if end_name not in node_map:
-                print("%s not in nodes" % end_name)
+                print "%s not in nodes" % end_name
                 continue
 
             edge = pydot.Edge(node_map[start_name], node_map[end_name])
@@ -105,13 +107,14 @@ def make_graph(pdus, room, filename_prefix):
 
             if prev_state_name in node_map:
                 state_edge = pydot.Edge(
-                    node_map[start_name], node_map[prev_state_name], style="dotted"
+                    node_map[start_name], node_map[prev_state_name],
+                    style='dotted'
                 )
                 graph.add_edge(state_edge)
 
-    graph.write("%s.dot" % filename_prefix, format="raw", prog="dot")
-    #    graph.write_png("%s.png" % filename_prefix, prog='dot')
-    graph.write_svg("%s.svg" % filename_prefix, prog="dot")
+    graph.write('%s.dot' % filename_prefix, format='raw', prog='dot')
+#    graph.write_png("%s.png" % filename_prefix, prog='dot')
+    graph.write_svg("%s.svg" % filename_prefix, prog='dot')
 
 
 def get_pdus(host, room):
@@ -127,14 +130,15 @@ def get_pdus(host, room):
 if __name__ == "__main__":
     parser = argparse.ArgumentParser(
         description="Generate a PDU graph for a given room by talking "
-        "to the given homeserver to get the list of PDUs. \n"
-        "Requires pydot."
+                    "to the given homeserver to get the list of PDUs. \n"
+                    "Requires pydot."
     )
     parser.add_argument(
-        "-p", "--prefix", dest="prefix", help="String to prefix output files with"
+        "-p", "--prefix", dest="prefix",
+        help="String to prefix output files with"
     )
-    parser.add_argument("host")
-    parser.add_argument("room")
+    parser.add_argument('host')
+    parser.add_argument('room')
 
     args = parser.parse_args()
 

contrib/graph/graph2.py

@@ -36,7 +36,10 @@ def make_graph(db_name, room_id, file_prefix, limit):
     args = [room_id]
 
     if limit:
-        sql += " ORDER BY topological_ordering DESC, stream_ordering DESC " "LIMIT ?"
+        sql += (
+            " ORDER BY topological_ordering DESC, stream_ordering DESC "
+            "LIMIT ?"
+        )
 
         args.append(limit)
 
@@ -53,8 +56,9 @@ def make_graph(db_name, room_id, file_prefix, limit):
 
     for event in events:
         c = conn.execute(
-            "SELECT state_group FROM event_to_state_groups " "WHERE event_id = ?",
-            (event.event_id,),
+            "SELECT state_group FROM event_to_state_groups "
+            "WHERE event_id = ?",
+            (event.event_id,)
         )
 
         res = c.fetchone()
@@ -65,7 +69,7 @@ def make_graph(db_name, room_id, file_prefix, limit):
 
         t = datetime.datetime.fromtimestamp(
             float(event.origin_server_ts) / 1000
-        ).strftime("%Y-%m-%d %H:%M:%S,%f")
+        ).strftime('%Y-%m-%d %H:%M:%S,%f')
 
         content = json.dumps(unfreeze(event.get_dict()["content"]))
 
@@ -89,7 +93,10 @@ def make_graph(db_name, room_id, file_prefix, limit):
             "state_group": state_group,
         }
 
-        node = pydot.Node(name=event.event_id, label=label)
+        node = pydot.Node(
+            name=event.event_id,
+            label=label,
+        )
 
         node_map[event.event_id] = node
         graph.add_node(node)
@@ -99,7 +106,10 @@ def make_graph(db_name, room_id, file_prefix, limit):
             try:
                 end_node = node_map[prev_id]
             except:
-                end_node = pydot.Node(name=prev_id, label="<<b>%s</b>>" % (prev_id,))
+                end_node = pydot.Node(
+                    name=prev_id,
+                    label="<<b>%s</b>>" % (prev_id,),
+                )
 
                 node_map[prev_id] = end_node
                 graph.add_node(end_node)
@@ -111,33 +121,36 @@ def make_graph(db_name, room_id, file_prefix, limit):
         if len(event_ids) <= 1:
             continue
 
-        cluster = pydot.Cluster(str(group), label="<State Group: %s>" % (str(group),))
+        cluster = pydot.Cluster(
+            str(group),
+            label="<State Group: %s>" % (str(group),)
+        )
 
         for event_id in event_ids:
             cluster.add_node(node_map[event_id])
 
         graph.add_subgraph(cluster)
 
-    graph.write("%s.dot" % file_prefix, format="raw", prog="dot")
-    graph.write_svg("%s.svg" % file_prefix, prog="dot")
-
+    graph.write('%s.dot' % file_prefix, format='raw', prog='dot')
+    graph.write_svg("%s.svg" % file_prefix, prog='dot')
 
 if __name__ == "__main__":
     parser = argparse.ArgumentParser(
         description="Generate a PDU graph for a given room by talking "
-        "to the given homeserver to get the list of PDUs. \n"
-        "Requires pydot."
+                    "to the given homeserver to get the list of PDUs. \n"
+                    "Requires pydot."
     )
     parser.add_argument(
-        "-p",
-        "--prefix",
-        dest="prefix",
+        "-p", "--prefix", dest="prefix",
         help="String to prefix output files with",
-        default="graph_output",
+        default="graph_output"
     )
-    parser.add_argument("-l", "--limit", help="Only retrieve the last N events.")
-    parser.add_argument("db")
-    parser.add_argument("room")
+    parser.add_argument(
+        "-l", "--limit",
+        help="Only retrieve the last N events.",
+    )
+    parser.add_argument('db')
+    parser.add_argument('room')
 
     args = parser.parse_args()
 

contrib/graph/graph3.py

@@ -1,5 +1,3 @@
-from __future__ import print_function
-
 # Copyright 2016 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -28,22 +26,22 @@ from six import string_types
 
 
 def make_graph(file_name, room_id, file_prefix, limit):
-    print("Reading lines")
+    print "Reading lines"
     with open(file_name) as f:
         lines = f.readlines()
 
-    print("Read lines")
+    print "Read lines"
 
     events = [FrozenEvent(json.loads(line)) for line in lines]
 
-    print("Loaded events.")
+    print "Loaded events."
 
     events.sort(key=lambda e: e.depth)
 
-    print("Sorted events")
+    print "Sorted events"
 
     if limit:
-        events = events[-int(limit) :]
+        events = events[-int(limit):]
 
     node_map = {}
 
@@ -52,12 +50,12 @@ def make_graph(file_name, room_id, file_prefix, limit):
     for event in events:
         t = datetime.datetime.fromtimestamp(
             float(event.origin_server_ts) / 1000
-        ).strftime("%Y-%m-%d %H:%M:%S,%f")
+        ).strftime('%Y-%m-%d %H:%M:%S,%f')
 
         content = json.dumps(unfreeze(event.get_dict()["content"]), indent=4)
         content = content.replace("\n", "<br/>\n")
 
-        print(content)
+        print content
         content = []
         for key, value in unfreeze(event.get_dict()["content"]).items():
             if value is None:
@@ -68,16 +66,15 @@ def make_graph(file_name, room_id, file_prefix, limit):
                 value = json.dumps(value)
 
             content.append(
-                "<b>%s</b>: %s,"
-                % (
-                    cgi.escape(key, quote=True).encode("ascii", "xmlcharrefreplace"),
-                    cgi.escape(value, quote=True).encode("ascii", "xmlcharrefreplace"),
+                "<b>%s</b>: %s," % (
+                    cgi.escape(key, quote=True).encode("ascii", 'xmlcharrefreplace'),
+                    cgi.escape(value, quote=True).encode("ascii", 'xmlcharrefreplace'),
                 )
             )
 
         content = "<br/>\n".join(content)
 
-        print(content)
+        print content
 
         label = (
             "<"
@@ -97,19 +94,25 @@ def make_graph(file_name, room_id, file_prefix, limit):
             "depth": event.depth,
         }
 
-        node = pydot.Node(name=event.event_id, label=label)
+        node = pydot.Node(
+            name=event.event_id,
+            label=label,
+        )
 
         node_map[event.event_id] = node
         graph.add_node(node)
 
-    print("Created Nodes")
+    print "Created Nodes"
 
     for event in events:
         for prev_id, _ in event.prev_events:
             try:
                 end_node = node_map[prev_id]
             except:
-                end_node = pydot.Node(name=prev_id, label="<<b>%s</b>>" % (prev_id,))
+                end_node = pydot.Node(
+                    name=prev_id,
+                    label="<<b>%s</b>>" % (prev_id,),
+                )
 
                 node_map[prev_id] = end_node
                 graph.add_node(end_node)
@@ -117,33 +120,33 @@ def make_graph(file_name, room_id, file_prefix, limit):
             edge = pydot.Edge(node_map[event.event_id], end_node)
             graph.add_edge(edge)
 
-    print("Created edges")
+    print "Created edges"
 
-    graph.write("%s.dot" % file_prefix, format="raw", prog="dot")
+    graph.write('%s.dot' % file_prefix, format='raw', prog='dot')
 
-    print("Created Dot")
+    print "Created Dot"
 
-    graph.write_svg("%s.svg" % file_prefix, prog="dot")
-
-    print("Created svg")
+    graph.write_svg("%s.svg" % file_prefix, prog='dot')
 
+    print "Created svg"
 
 if __name__ == "__main__":
     parser = argparse.ArgumentParser(
         description="Generate a PDU graph for a given room by reading "
-        "from a file with line deliminated events. \n"
-        "Requires pydot."
+                    "from a file with line deliminated events. \n"
+                    "Requires pydot."
     )
     parser.add_argument(
-        "-p",
-        "--prefix",
-        dest="prefix",
+        "-p", "--prefix", dest="prefix",
         help="String to prefix output files with",
-        default="graph_output",
+        default="graph_output"
     )
-    parser.add_argument("-l", "--limit", help="Only retrieve the last N events.")
-    parser.add_argument("event_file")
-    parser.add_argument("room")
+    parser.add_argument(
+        "-l", "--limit",
+        help="Only retrieve the last N events.",
+    )
+    parser.add_argument('event_file')
+    parser.add_argument('room')
 
     args = parser.parse_args()