changelog

It appears as though Python only checks to see if the key exists in a dictionary, not necessarily for a useful value. This means that when clients submit (valid) requests with `m.relates_to: null` and Synapse later reads it, it gets a None reference error on access.

This is the easier route than guarding all the places where it could be None.

.buildkite/docker-compose.py27.pg94.yaml

@@ -0,0 +1,21 @@
+version: '3.1'
+
+services:
+
+  postgres:
+    image: postgres:9.4
+    environment:
+      POSTGRES_PASSWORD: postgres
+
+  testenv:
+    image: python:2.7
+    depends_on:
+      - postgres
+    env_file: .env
+    environment:
+      SYNAPSE_POSTGRES_HOST: postgres
+      SYNAPSE_POSTGRES_USER: postgres
+      SYNAPSE_POSTGRES_PASSWORD: postgres
+    working_dir: /app
+    volumes:
+      - ..:/app

.buildkite/docker-compose.py27.pg95.yaml

@@ -0,0 +1,21 @@
+version: '3.1'
+
+services:
+
+  postgres:
+    image: postgres:9.5
+    environment:
+      POSTGRES_PASSWORD: postgres
+
+  testenv:
+    image: python:2.7
+    depends_on:
+      - postgres
+    env_file: .env
+    environment:
+      SYNAPSE_POSTGRES_HOST: postgres
+      SYNAPSE_POSTGRES_USER: postgres
+      SYNAPSE_POSTGRES_PASSWORD: postgres
+    working_dir: /app
+    volumes:
+      - ..:/app

.buildkite/docker-compose.py35.pg94.yaml

@@ -0,0 +1,21 @@
+version: '3.1'
+
+services:
+
+  postgres:
+    image: postgres:9.4
+    environment:
+      POSTGRES_PASSWORD: postgres
+
+  testenv:
+    image: python:3.5
+    depends_on:
+      - postgres
+    env_file: .env
+    environment:
+      SYNAPSE_POSTGRES_HOST: postgres
+      SYNAPSE_POSTGRES_USER: postgres
+      SYNAPSE_POSTGRES_PASSWORD: postgres
+    working_dir: /app
+    volumes:
+      - ..:/app

.buildkite/format_tap.py

@@ -1,33 +0,0 @@
-import sys
-from tap.parser import Parser
-from tap.line import Result, Unknown, Diagnostic
-
-out = ["### TAP Output for " + sys.argv[2]]
-
-p = Parser()
-
-in_error = False
-
-for line in p.parse_file(sys.argv[1]):
-    if isinstance(line, Result):
-        if in_error:
-            out.append("")
-            out.append("</pre></code></details>")
-            out.append("")
-            out.append("----")
-            out.append("")
-        in_error = False
-
-        if not line.ok and not line.todo:
-            in_error = True
-
-            out.append("FAILURE Test #%d: ``%s``" % (line.number, line.description))
-            out.append("")
-            out.append("<details><summary>Show log</summary><code><pre>")
-
-    elif isinstance(line, Diagnostic) and in_error:
-        out.append(line.text)
-
-if out:
-    for line in out[:-3]:
-        print(line)

.buildkite/pipeline.yml

@@ -2,7 +2,6 @@ env:
   CODECOV_TOKEN: "2dd7eb9b-0eda-45fe-a47c-9b5ac040045f"
 
 steps:
-
   - command:
       - "python -m pip install tox"
       - "tox -e pep8"
@@ -37,6 +36,8 @@ steps:
           image: "python:3.6"
           propagate-environment: true
 
+  - wait
+
   - command:
       - "python -m pip install tox"
       - "tox -e check-sampleconfig"
@@ -45,25 +46,16 @@ steps:
       - docker#v3.0.1:
           image: "python:3.6"
 
-  - wait
-
-
   - command:
       - "python -m pip install tox"
-      - "tox -e py35-old,codecov"
-    label: ":python: 3.5 / SQLite / Old Deps"
+      - "tox -e py27,codecov"
+    label: ":python: 2.7 / SQLite"
     env:
       TRIAL_FLAGS: "-j 2"
     plugins:
       - docker#v3.0.1:
-          image: "python:3.5"
+          image: "python:2.7"
           propagate-environment: true
-    retry:
-      automatic:
-        - exit_status: -1
-          limit: 2
-        - exit_status: 2
-          limit: 2
 
   - command:
       - "python -m pip install tox"
@@ -75,12 +67,6 @@ steps:
       - docker#v3.0.1:
           image: "python:3.5"
           propagate-environment: true
-    retry:
-      automatic:
-        - exit_status: -1
-          limit: 2
-        - exit_status: 2
-          limit: 2
 
   - command:
       - "python -m pip install tox"
@@ -92,12 +78,6 @@ steps:
       - docker#v3.0.1:
           image: "python:3.6"
           propagate-environment: true
-    retry:
-      automatic:
-        - exit_status: -1
-          limit: 2
-        - exit_status: 2
-          limit: 2
 
   - command:
       - "python -m pip install tox"
@@ -109,12 +89,50 @@ steps:
       - docker#v3.0.1:
           image: "python:3.7"
           propagate-environment: true
-    retry:
-      automatic:
-        - exit_status: -1
-          limit: 2
-        - exit_status: 2
-          limit: 2
+
+  - command:
+      - "python -m pip install tox"
+      - "tox -e py27-old,codecov"
+    label: ":python: 2.7 / SQLite / Old Deps"
+    env:
+      TRIAL_FLAGS: "-j 2"
+    plugins:
+      - docker#v3.0.1:
+          image: "python:2.7"
+          propagate-environment: true
+
+  - label: ":python: 2.7 / :postgres: 9.4"
+    env:
+      TRIAL_FLAGS: "-j 4"
+    command:
+      - "bash -c 'python -m pip install tox && python -m tox -e py27-postgres,codecov'"
+    plugins:
+      - docker-compose#v2.1.0:
+          run: testenv
+          config:
+            - .buildkite/docker-compose.py27.pg94.yaml
+
+  - label: ":python: 2.7 / :postgres: 9.5"
+    env:
+      TRIAL_FLAGS: "-j 4"
+    command:
+      - "bash -c 'python -m pip install tox && python -m tox -e py27-postgres,codecov'"
+    plugins:
+      - docker-compose#v2.1.0:
+          run: testenv
+          config:
+            - .buildkite/docker-compose.py27.pg95.yaml
+
+  - label: ":python: 3.5 / :postgres: 9.4"
+    env:
+      TRIAL_FLAGS: "-j 4"
+    command:
+      - "bash -c 'python -m pip install tox && python -m tox -e py35-postgres,codecov'"
+    plugins:
+      - docker-compose#v2.1.0:
+          run: testenv
+          config:
+            - .buildkite/docker-compose.py35.pg94.yaml
 
   - label: ":python: 3.5 / :postgres: 9.5"
     env:
@@ -126,12 +144,6 @@ steps:
           run: testenv
           config:
             - .buildkite/docker-compose.py35.pg95.yaml
-    retry:
-      automatic:
-        - exit_status: -1
-          limit: 2
-        - exit_status: 2
-          limit: 2
 
   - label: ":python: 3.7 / :postgres: 9.5"
     env:
@@ -143,12 +155,6 @@ steps:
           run: testenv
           config:
             - .buildkite/docker-compose.py37.pg95.yaml
-    retry:
-      automatic:
-        - exit_status: -1
-          limit: 2
-        - exit_status: 2
-          limit: 2
 
   - label: ":python: 3.7 / :postgres: 11"
     env:
@@ -160,67 +166,3 @@ steps:
           run: testenv
           config:
             - .buildkite/docker-compose.py37.pg11.yaml
-    retry:
-      automatic:
-        - exit_status: -1
-          limit: 2
-        - exit_status: 2
-          limit: 2
-
-
-  - label: "SyTest - :python: 3.5 / SQLite / Monolith"
-    agents:
-      queue: "medium"
-    command:
-      - "bash .buildkite/merge_base_branch.sh"
-      - "bash .buildkite/synapse_sytest.sh"
-    plugins:
-      - docker#v3.0.1:
-          image: "matrixdotorg/sytest-synapse:py35"
-          propagate-environment: true
-    retry:
-      automatic:
-        - exit_status: -1
-          limit: 2
-        - exit_status: 2
-          limit: 2
-
-  - label: "SyTest - :python: 3.5 / :postgres: 9.6 / Monolith"
-    agents:
-      queue: "medium"
-    env:
-      POSTGRES: "1"
-    command:
-      - "bash .buildkite/merge_base_branch.sh"
-      - "bash .buildkite/synapse_sytest.sh"
-    plugins:
-      - docker#v3.0.1:
-          image: "matrixdotorg/sytest-synapse:py35"
-          propagate-environment: true
-    retry:
-      automatic:
-        - exit_status: -1
-          limit: 2
-        - exit_status: 2
-          limit: 2
-
-  - label: "SyTest - :python: 3.5 / :postgres: 9.6 / Workers"
-    agents:
-      queue: "medium"
-    env:
-      POSTGRES: "1"
-      WORKERS: "1"
-    command:
-      - "bash .buildkite/merge_base_branch.sh"
-      - "bash .buildkite/synapse_sytest.sh"
-    plugins:
-      - docker#v3.0.1:
-          image: "matrixdotorg/sytest-synapse:py35"
-          propagate-environment: true
-    soft_fail: true
-    retry:
-      automatic:
-        - exit_status: -1
-          limit: 2
-        - exit_status: 2
-          limit: 2

.buildkite/synapse_sytest.sh

@@ -1,145 +0,0 @@
-#!/bin/bash
-#
-# Fetch sytest, and then run the tests for synapse. The entrypoint for the
-# sytest-synapse docker images.
-
-set -ex
-
-if [ -n "$BUILDKITE" ]
-then
-    SYNAPSE_DIR=`pwd`
-else
-    SYNAPSE_DIR="/src"
-fi
-
-# Attempt to find a sytest to use.
-# If /sytest exists, it means that a SyTest checkout has been mounted into the Docker image.
-if [ -d "/sytest" ]; then
-    # If the user has mounted in a SyTest checkout, use that.
-    echo "Using local sytests..."
-
-    # create ourselves a working directory and dos2unix some scripts therein
-    mkdir -p /work/jenkins
-    for i in install-deps.pl run-tests.pl tap-to-junit-xml.pl jenkins/prep_sytest_for_postgres.sh; do
-        dos2unix -n "/sytest/$i" "/work/$i"
-    done
-    ln -sf /sytest/tests /work
-    ln -sf /sytest/keys /work
-    SYTEST_LIB="/sytest/lib"
-else
-    if [ -n "BUILDKITE_BRANCH" ]
-    then
-        branch_name=$BUILDKITE_BRANCH
-    else
-        # Otherwise, try and find out what the branch that the Synapse checkout is using. Fall back to develop if it's not a branch.
-        branch_name="$(git --git-dir=/src/.git symbolic-ref HEAD 2>/dev/null)" || branch_name="develop"
-    fi
-
-    # Try and fetch the branch
-    echo "Trying to get same-named sytest branch..."
-    wget -q https://github.com/matrix-org/sytest/archive/$branch_name.tar.gz -O sytest.tar.gz || {
-        # Probably a 404, fall back to develop
-        echo "Using develop instead..."
-        wget -q https://github.com/matrix-org/sytest/archive/develop.tar.gz -O sytest.tar.gz
-    }
-
-    mkdir -p /work
-    tar -C /work --strip-components=1 -xf sytest.tar.gz
-    SYTEST_LIB="/work/lib"
-fi
-
-cd /work
-
-# PostgreSQL setup
-if [ -n "$POSTGRES" ]
-then
-    export PGUSER=postgres
-    export POSTGRES_DB_1=pg1
-    export POSTGRES_DB_2=pg2
-
-    # Start the database
-    su -c 'eatmydata /usr/lib/postgresql/9.6/bin/pg_ctl -w -D /var/lib/postgresql/data start' postgres
-
-    # Use the Jenkins script to write out the configuration for a PostgreSQL using Synapse
-    jenkins/prep_sytest_for_postgres.sh
-
-    # Make the test databases for the two Synapse servers that will be spun up
-    su -c 'psql -c "CREATE DATABASE pg1;"' postgres
-    su -c 'psql -c "CREATE DATABASE pg2;"' postgres
-
-fi
-
-if [ -n "$OFFLINE" ]; then
-    # if we're in offline mode, just put synapse into the virtualenv, and
-    # hope that the deps are up-to-date.
-    #
-    # (`pip install -e` likes to reinstall setuptools even if it's already installed,
-    # so we just run setup.py explicitly.)
-    #
-    (cd $SYNAPSE_DIR && /venv/bin/python setup.py -q develop)
-else
-    # We've already created the virtualenv, but lets double check we have all
-    # deps.
-    /venv/bin/pip install -q --upgrade --no-cache-dir -e $SYNAPSE_DIR
-    /venv/bin/pip install -q --upgrade --no-cache-dir \
-        lxml psycopg2 coverage codecov tap.py
-
-    # Make sure all Perl deps are installed -- this is done in the docker build
-    # so will only install packages added since the last Docker build
-    ./install-deps.pl
-fi
-
-
-# Run the tests
->&2 echo "+++ Running tests"
-
-RUN_TESTS=(
-    perl -I "$SYTEST_LIB" ./run-tests.pl --python=/venv/bin/python --synapse-directory=$SYNAPSE_DIR --coverage -O tap --all
-)
-
-TEST_STATUS=0
-
-if [ -n "$WORKERS" ]; then
-    RUN_TESTS+=(-I Synapse::ViaHaproxy --dendron-binary=/pydron.py)
-else
-    RUN_TESTS+=(-I Synapse)
-fi
-
-"${RUN_TESTS[@]}" "$@" > results.tap || TEST_STATUS=$?
-
-if [ $TEST_STATUS -ne 0 ]; then
-    >&2 echo -e "run-tests \e[31mFAILED\e[0m: exit code $TEST_STATUS"
-else
-    >&2 echo -e "run-tests \e[32mPASSED\e[0m"
-fi
-
->&2 echo "--- Copying assets"
-
-# Copy out the logs
-mkdir -p /logs
-cp results.tap /logs/results.tap
-rsync --ignore-missing-args  --min-size=1B -av server-0 server-1 /logs --include "*/" --include="*.log.*" --include="*.log" --exclude="*"
-
-# Upload coverage to codecov and upload files, if running on Buildkite
-if [ -n "$BUILDKITE" ]
-then
-    /venv/bin/coverage combine || true
-    /venv/bin/coverage xml || true
-    /venv/bin/codecov -X gcov -f coverage.xml
-
-    wget -O buildkite.tar.gz https://github.com/buildkite/agent/releases/download/v3.13.0/buildkite-agent-linux-amd64-3.13.0.tar.gz
-    tar xvf buildkite.tar.gz
-    chmod +x ./buildkite-agent
-
-    # Upload the files
-    ./buildkite-agent artifact upload "/logs/**/*.log*"
-    ./buildkite-agent artifact upload "/logs/results.tap"
-
-    if [ $TEST_STATUS -ne 0 ]; then
-        # Annotate, if failure
-        /venv/bin/python $SYNAPSE_DIR/.buildkite/format_tap.py /logs/results.tap "$BUILDKITE_LABEL" | ./buildkite-agent annotate --style="error" --context="$BUILDKITE_LABEL"
-    fi
-fi
-
-
-exit $TEST_STATUS

.circleci/config.yml

@@ -4,23 +4,160 @@ jobs:
     machine: true
     steps:
       - checkout
+      - run: docker build -f docker/Dockerfile --label gitsha1=${CIRCLE_SHA1} -t matrixdotorg/synapse:${CIRCLE_TAG}-py2 .
       - run: docker build -f docker/Dockerfile --label gitsha1=${CIRCLE_SHA1} -t matrixdotorg/synapse:${CIRCLE_TAG} -t matrixdotorg/synapse:${CIRCLE_TAG}-py3 --build-arg PYTHON_VERSION=3.6 .
       - run: docker login --username $DOCKER_HUB_USERNAME --password $DOCKER_HUB_PASSWORD
       - run: docker push matrixdotorg/synapse:${CIRCLE_TAG}
+      - run: docker push matrixdotorg/synapse:${CIRCLE_TAG}-py2
       - run: docker push matrixdotorg/synapse:${CIRCLE_TAG}-py3
   dockerhubuploadlatest:
     machine: true
     steps:
       - checkout
+      - run: docker build -f docker/Dockerfile --label gitsha1=${CIRCLE_SHA1} -t matrixdotorg/synapse:latest-py2 .
       - run: docker build -f docker/Dockerfile --label gitsha1=${CIRCLE_SHA1} -t matrixdotorg/synapse:latest -t matrixdotorg/synapse:latest-py3 --build-arg PYTHON_VERSION=3.6 .
       - run: docker login --username $DOCKER_HUB_USERNAME --password $DOCKER_HUB_PASSWORD
       - run: docker push matrixdotorg/synapse:latest
+      - run: docker push matrixdotorg/synapse:latest-py2
       - run: docker push matrixdotorg/synapse:latest-py3
+  sytestpy2:
+    docker:
+      - image: matrixdotorg/sytest-synapsepy2
+    working_directory: /src
+    steps:
+      - checkout
+      - run: /synapse_sytest.sh
+      - store_artifacts:
+          path: /logs
+          destination: logs
+      - store_test_results:
+          path: /logs
+  sytestpy2postgres:
+    docker:
+      - image: matrixdotorg/sytest-synapsepy2
+    working_directory: /src
+    steps:
+      - checkout
+      - run: POSTGRES=1 /synapse_sytest.sh
+      - store_artifacts:
+          path: /logs
+          destination: logs
+      - store_test_results:
+          path: /logs
+  sytestpy2merged:
+    docker:
+      - image: matrixdotorg/sytest-synapsepy2
+    working_directory: /src
+    steps:
+      - checkout
+      - run: bash .circleci/merge_base_branch.sh
+      - run: /synapse_sytest.sh
+      - store_artifacts:
+          path: /logs
+          destination: logs
+      - store_test_results:
+          path: /logs
+  sytestpy2postgresmerged:
+    docker:
+      - image: matrixdotorg/sytest-synapsepy2
+    working_directory: /src
+    steps:
+      - checkout
+      - run: bash .circleci/merge_base_branch.sh
+      - run: POSTGRES=1 /synapse_sytest.sh
+      - store_artifacts:
+          path: /logs
+          destination: logs
+      - store_test_results:
+          path: /logs
+
+  sytestpy3:
+    docker:
+      - image: matrixdotorg/sytest-synapsepy3
+    working_directory: /src
+    steps:
+      - checkout
+      - run: /synapse_sytest.sh
+      - store_artifacts:
+          path: /logs
+          destination: logs
+      - store_test_results:
+          path: /logs
+  sytestpy3postgres:
+    docker:
+      - image: matrixdotorg/sytest-synapsepy3
+    working_directory: /src
+    steps:
+      - checkout
+      - run: POSTGRES=1 /synapse_sytest.sh
+      - store_artifacts:
+          path: /logs
+          destination: logs
+      - store_test_results:
+          path: /logs
+  sytestpy3merged:
+    docker:
+      - image: matrixdotorg/sytest-synapsepy3
+    working_directory: /src
+    steps:
+      - checkout
+      - run: bash .circleci/merge_base_branch.sh
+      - run: /synapse_sytest.sh
+      - store_artifacts:
+          path: /logs
+          destination: logs
+      - store_test_results:
+          path: /logs
+  sytestpy3postgresmerged:
+    docker:
+      - image: matrixdotorg/sytest-synapsepy3
+    working_directory: /src
+    steps:
+      - checkout
+      - run: bash .circleci/merge_base_branch.sh
+      - run: POSTGRES=1 /synapse_sytest.sh
+      - store_artifacts:
+          path: /logs
+          destination: logs
+      - store_test_results:
+          path: /logs
 
 workflows:
   version: 2
   build:
     jobs:
+      - sytestpy2:
+          filters:
+            branches:
+              only: /develop|master|release-.*/
+      - sytestpy2postgres:
+          filters:
+            branches:
+              only: /develop|master|release-.*/
+      - sytestpy3:
+          filters:
+            branches:
+              only: /develop|master|release-.*/
+      - sytestpy3postgres:
+          filters:
+            branches:
+              only: /develop|master|release-.*/
+      - sytestpy2merged:
+          filters:
+            branches:
+              ignore: /develop|master|release-.*/
+      - sytestpy2postgresmerged:
+          filters:
+            branches:
+              ignore: /develop|master|release-.*/
+      - sytestpy3merged:
+          filters:
+            branches:
+              ignore: /develop|master|release-.*/
+      - sytestpy3postgresmerged:
+          filters:
+            branches:
+              ignore: /develop|master|release-.*/
       - dockerhubuploadrelease:
           filters:
             tags:

.circleci/merge_base_branch.sh

@@ -1,21 +1,22 @@
 #!/usr/bin/env bash
 
-set -ex
+set -e
 
-if [[ "$BUILDKITE_BRANCH" =~ ^(develop|master|dinsic|shhs|release-.*)$ ]]; then
-    echo "Not merging forward, as this is a release branch"
-    exit 0
-fi
+# CircleCI doesn't give CIRCLE_PR_NUMBER in the environment for non-forked PRs. Wonderful.
+# In this case, we just need to do some ~shell magic~ to strip it out of the PULL_REQUEST URL.
+echo 'export CIRCLE_PR_NUMBER="${CIRCLE_PR_NUMBER:-${CIRCLE_PULL_REQUEST##*/}}"' >> $BASH_ENV
+source $BASH_ENV
 
-if [[ -z $BUILDKITE_PULL_REQUEST_BASE_BRANCH ]]; then
-    echo "Not a pull request, or hasn't had a PR opened yet..."
+if [[ -z "${CIRCLE_PR_NUMBER}" ]]
+then
+    echo "Can't figure out what the PR number is! Assuming merge target is develop."
 
     # It probably hasn't had a PR opened yet. Since all PRs land on develop, we
     # can probably assume it's based on it and will be merged into it.
     GITBASE="develop"
 else
     # Get the reference, using the GitHub API
-    GITBASE=$BUILDKITE_PULL_REQUEST_BASE_BRANCH
+    GITBASE=`wget -O- https://api.github.com/repos/matrix-org/synapse/pulls/${CIRCLE_PR_NUMBER} | jq -r '.base.ref'`
 fi
 
 # Show what we are before

.github/FUNDING.yml

@@ -1,4 +0,0 @@
-# One username per supported platform and one custom link
-patreon: matrixdotorg
-liberapay: matrixdotorg
-custom: https://paypal.me/matrixdotorg

CHANGES.md

@@ -1,137 +1,8 @@
-Synapse 1.0.0 (2019-06-11)
-==========================
-
-Bugfixes
---------
-
-- Fix bug where attempting to send transactions with large number of EDUs can fail. ([\#5418](https://github.com/matrix-org/synapse/issues/5418))
-
-
-Improved Documentation
-----------------------
-
-- Expand the federation guide to include relevant content from the MSC1711 FAQ ([\#5419](https://github.com/matrix-org/synapse/issues/5419))
-
-
-Internal Changes
-----------------
-
-- Move password reset links to /_matrix/client/unstable namespace. ([\#5424](https://github.com/matrix-org/synapse/issues/5424))
-
-
-Synapse 1.0.0rc3 (2019-06-10)
-=============================
-
-Security: Fix authentication bug introduced in 1.0.0rc1. Please upgrade to rc3 immediately
-
-
-Synapse 1.0.0rc2 (2019-06-10)
-=============================
-
-Bugfixes
---------
-
-- Remove redundant warning about key server response validation. ([\#5392](https://github.com/matrix-org/synapse/issues/5392))
-- Fix bug where old keys stored in the database with a null valid until timestamp caused all verification requests for that key to fail. ([\#5415](https://github.com/matrix-org/synapse/issues/5415))
-- Fix excessive memory using with default `federation_verify_certificates: true` configuration. ([\#5417](https://github.com/matrix-org/synapse/issues/5417))
-
-
-Synapse 1.0.0rc1 (2019-06-07)
-=============================
-
-Features
---------
-
-- Synapse now more efficiently collates room statistics. ([\#4338](https://github.com/matrix-org/synapse/issues/4338), [\#5260](https://github.com/matrix-org/synapse/issues/5260), [\#5324](https://github.com/matrix-org/synapse/issues/5324))
-- Add experimental support for relations (aka reactions and edits). ([\#5220](https://github.com/matrix-org/synapse/issues/5220))
-- Ability to configure default room version. ([\#5223](https://github.com/matrix-org/synapse/issues/5223), [\#5249](https://github.com/matrix-org/synapse/issues/5249))
-- Allow configuring a range for the account validity startup job. ([\#5276](https://github.com/matrix-org/synapse/issues/5276))
-- CAS login will now hit the r0 API, not the deprecated v1 one. ([\#5286](https://github.com/matrix-org/synapse/issues/5286))
-- Validate federation server TLS certificates by default (implements [MSC1711](https://github.com/matrix-org/matrix-doc/blob/master/proposals/1711-x509-for-federation.md)). ([\#5359](https://github.com/matrix-org/synapse/issues/5359))
-- Update /_matrix/client/versions to reference support for r0.5.0. ([\#5360](https://github.com/matrix-org/synapse/issues/5360))
-- Add a script to generate new signing-key files. ([\#5361](https://github.com/matrix-org/synapse/issues/5361))
-- Update upgrade and installation guides ahead of 1.0. ([\#5371](https://github.com/matrix-org/synapse/issues/5371))
-- Replace the `perspectives` configuration section with `trusted_key_servers`, and make validating the signatures on responses optional (since TLS will do this job for us). ([\#5374](https://github.com/matrix-org/synapse/issues/5374))
-- Add ability to perform password reset via email without trusting the identity server. ([\#5377](https://github.com/matrix-org/synapse/issues/5377))
-- Set default room version to v4. ([\#5379](https://github.com/matrix-org/synapse/issues/5379))
-
-
-Bugfixes
---------
-
-- Fixes client-server API not sending "m.heroes" to lazy-load /sync requests when a rooms name or its canonical alias are empty. Thanks to @dnaf for this work! ([\#5089](https://github.com/matrix-org/synapse/issues/5089))
-- Prevent federation device list updates breaking when processing multiple updates at once. ([\#5156](https://github.com/matrix-org/synapse/issues/5156))
-- Fix worker registration bug caused by ClientReaderSlavedStore being unable to see get_profileinfo. ([\#5200](https://github.com/matrix-org/synapse/issues/5200))
-- Fix race when backfilling in rooms with worker mode. ([\#5221](https://github.com/matrix-org/synapse/issues/5221))
-- Fix appservice timestamp massaging. ([\#5233](https://github.com/matrix-org/synapse/issues/5233))
-- Ensure that server_keys fetched via a notary server are correctly signed. ([\#5251](https://github.com/matrix-org/synapse/issues/5251))
-- Show the correct error when logging out and access token is missing. ([\#5256](https://github.com/matrix-org/synapse/issues/5256))
-- Fix error code when there is an invalid parameter on /_matrix/client/r0/publicRooms ([\#5257](https://github.com/matrix-org/synapse/issues/5257))
-- Fix error when downloading thumbnail with missing width/height parameter. ([\#5258](https://github.com/matrix-org/synapse/issues/5258))
-- Fix schema update for account validity. ([\#5268](https://github.com/matrix-org/synapse/issues/5268))
-- Fix bug where we leaked extremities when we soft failed events, leading to performance degradation. ([\#5274](https://github.com/matrix-org/synapse/issues/5274), [\#5278](https://github.com/matrix-org/synapse/issues/5278), [\#5291](https://github.com/matrix-org/synapse/issues/5291))
-- Fix "db txn 'update_presence' from sentinel context" log messages. ([\#5275](https://github.com/matrix-org/synapse/issues/5275))
-- Fix dropped logcontexts during high outbound traffic. ([\#5277](https://github.com/matrix-org/synapse/issues/5277))
-- Fix a bug where it is not possible to get events in the federation format with the request `GET /_matrix/client/r0/rooms/{roomId}/messages`. ([\#5293](https://github.com/matrix-org/synapse/issues/5293))
-- Fix performance problems with the rooms stats background update. ([\#5294](https://github.com/matrix-org/synapse/issues/5294))
-- Fix noisy 'no key for server' logs. ([\#5300](https://github.com/matrix-org/synapse/issues/5300))
-- Fix bug where a notary server would sometimes forget old keys. ([\#5307](https://github.com/matrix-org/synapse/issues/5307))
-- Prevent users from setting huge displaynames and avatar URLs. ([\#5309](https://github.com/matrix-org/synapse/issues/5309))
-- Fix handling of failures when processing incoming events where calling `/event_auth` on remote server fails. ([\#5317](https://github.com/matrix-org/synapse/issues/5317))
-- Ensure that we have an up-to-date copy of the signing key when validating incoming federation requests. ([\#5321](https://github.com/matrix-org/synapse/issues/5321))
-- Fix various problems which made the signing-key notary server time out for some requests. ([\#5333](https://github.com/matrix-org/synapse/issues/5333))
-- Fix bug which would make certain operations (such as room joins) block for 20 minutes while attemoting to fetch verification keys. ([\#5334](https://github.com/matrix-org/synapse/issues/5334))
-- Fix a bug where we could rapidly mark a server as unreachable even though it was only down for a few minutes. ([\#5335](https://github.com/matrix-org/synapse/issues/5335), [\#5340](https://github.com/matrix-org/synapse/issues/5340))
-- Fix a bug where account validity renewal emails could only be sent when email notifs were enabled. ([\#5341](https://github.com/matrix-org/synapse/issues/5341))
-- Fix failure when fetching batches of events during backfill, etc. ([\#5342](https://github.com/matrix-org/synapse/issues/5342))
-- Add a new room version where the timestamps on events are checked against the validity periods on signing keys. ([\#5348](https://github.com/matrix-org/synapse/issues/5348), [\#5354](https://github.com/matrix-org/synapse/issues/5354))
-- Fix room stats and presence background updates to correctly handle missing events. ([\#5352](https://github.com/matrix-org/synapse/issues/5352))
-- Include left members in room summaries' heroes. ([\#5355](https://github.com/matrix-org/synapse/issues/5355))
-- Fix `federation_custom_ca_list` configuration option. ([\#5362](https://github.com/matrix-org/synapse/issues/5362))
-- Fix missing logcontext warnings on shutdown. ([\#5369](https://github.com/matrix-org/synapse/issues/5369))
-
-
-Improved Documentation
-----------------------
-
-- Fix docs on resetting the user directory. ([\#5282](https://github.com/matrix-org/synapse/issues/5282))
-- Fix notes about ACME in the MSC1711 faq. ([\#5357](https://github.com/matrix-org/synapse/issues/5357))
-
-
-Internal Changes
-----------------
-
-- Synapse will now serve the experimental "room complexity" API endpoint. ([\#5216](https://github.com/matrix-org/synapse/issues/5216))
-- The base classes for the v1 and v2_alpha REST APIs have been unified. ([\#5226](https://github.com/matrix-org/synapse/issues/5226), [\#5328](https://github.com/matrix-org/synapse/issues/5328))
-- Simplifications and comments in do_auth. ([\#5227](https://github.com/matrix-org/synapse/issues/5227))
-- Remove urllib3 pin as requests 2.22.0 has been released supporting urllib3 1.25.2. ([\#5230](https://github.com/matrix-org/synapse/issues/5230))
-- Preparatory work for key-validity features. ([\#5232](https://github.com/matrix-org/synapse/issues/5232), [\#5234](https://github.com/matrix-org/synapse/issues/5234), [\#5235](https://github.com/matrix-org/synapse/issues/5235), [\#5236](https://github.com/matrix-org/synapse/issues/5236), [\#5237](https://github.com/matrix-org/synapse/issues/5237), [\#5244](https://github.com/matrix-org/synapse/issues/5244), [\#5250](https://github.com/matrix-org/synapse/issues/5250), [\#5296](https://github.com/matrix-org/synapse/issues/5296), [\#5299](https://github.com/matrix-org/synapse/issues/5299), [\#5343](https://github.com/matrix-org/synapse/issues/5343), [\#5347](https://github.com/matrix-org/synapse/issues/5347), [\#5356](https://github.com/matrix-org/synapse/issues/5356))
-- Specify the type of reCAPTCHA key to use. ([\#5283](https://github.com/matrix-org/synapse/issues/5283))
-- Improve sample config for monthly active user blocking. ([\#5284](https://github.com/matrix-org/synapse/issues/5284))
-- Remove spurious debug from MatrixFederationHttpClient.get_json. ([\#5287](https://github.com/matrix-org/synapse/issues/5287))
-- Improve logging for logcontext leaks. ([\#5288](https://github.com/matrix-org/synapse/issues/5288))
-- Clarify that the admin change password API logs the user out. ([\#5303](https://github.com/matrix-org/synapse/issues/5303))
-- New installs will now use the v54 full schema, rather than the full schema v14 and applying incremental updates to v54. ([\#5320](https://github.com/matrix-org/synapse/issues/5320))
-- Improve docstrings on MatrixFederationClient. ([\#5332](https://github.com/matrix-org/synapse/issues/5332))
-- Clean up FederationClient.get_events for clarity. ([\#5344](https://github.com/matrix-org/synapse/issues/5344))
-- Various improvements to debug logging. ([\#5353](https://github.com/matrix-org/synapse/issues/5353))
-- Don't run CI build checks until sample config check has passed. ([\#5370](https://github.com/matrix-org/synapse/issues/5370))
-- Automatically retry buildkite builds (max twice) when an agent is lost. ([\#5380](https://github.com/matrix-org/synapse/issues/5380))
-
-
-Synapse 0.99.5.2 (2019-05-30)
-=============================
-
-Bugfixes
---------
-
-- Fix bug where we leaked extremities when we soft failed events, leading to performance degradation. ([\#5274](https://github.com/matrix-org/synapse/issues/5274), [\#5278](https://github.com/matrix-org/synapse/issues/5278), [\#5291](https://github.com/matrix-org/synapse/issues/5291))
-
-
 Synapse 0.99.5.1 (2019-05-22)
 =============================
 
-0.99.5.1 supersedes 0.99.5 due to malformed debian changelog - no functional changes.
+No significant changes.
+
 
 Synapse 0.99.5 (2019-05-22)
 ===========================

CONTRIBUTING.rst

@@ -30,20 +30,21 @@ use github's pull request workflow to review the contribution, and either ask
 you to make any refinements needed or merge it and make them ourselves. The
 changes will then land on master when we next do a release.
 
-We use `CircleCI <https://circleci.com/gh/matrix-org>`_ and `Buildkite
-<https://buildkite.com/matrix-dot-org/synapse>`_ for continuous integration.
-Buildkite builds need to be authorised by a maintainer. If your change breaks
-the build, this will be shown in GitHub, so please keep an eye on the pull
-request for feedback.
+We use `CircleCI <https://circleci.com/gh/matrix-org>`_ and `Travis CI
+<https://travis-ci.org/matrix-org/synapse>`_ for continuous integration. All
+pull requests to synapse get automatically tested by Travis and CircleCI.
+If your change breaks the build, this will be shown in GitHub, so please
+keep an eye on the pull request for feedback.
 
 To run unit tests in a local development environment, you can use:
 
-- ``tox -e py35`` (requires tox to be installed by ``pip install tox``)
-  for SQLite-backed Synapse on Python 3.5.
+- ``tox -e py27`` (requires tox to be installed by ``pip install tox``) for
+  SQLite-backed Synapse on Python 2.7.
+- ``tox -e py35`` for SQLite-backed Synapse on Python 3.5.
 - ``tox -e py36`` for SQLite-backed Synapse on Python 3.6.
-- ``tox -e py36-postgres`` for PostgreSQL-backed Synapse on Python 3.6
+- ``tox -e py27-postgres`` for PostgreSQL-backed Synapse on Python 2.7
   (requires a running local PostgreSQL with access to create databases).
-- ``./test_postgresql.sh`` for PostgreSQL-backed Synapse on Python 3.5
+- ``./test_postgresql.sh`` for PostgreSQL-backed Synapse on Python 2.7
   (requires Docker). Entirely self-contained, recommended if you don't want to
   set up PostgreSQL yourself.
 

INSTALL.md

@@ -1,14 +1,13 @@
-- [Installing Synapse](#installing-synapse)
-  - [Installing from source](#installing-from-source)
-    - [Platform-Specific Instructions](#platform-specific-instructions)
-    - [Troubleshooting Installation](#troubleshooting-installation)
-  - [Prebuilt packages](#prebuilt-packages)
-- [Setting up Synapse](#setting-up-synapse)
-  - [TLS certificates](#tls-certificates)
-  - [Email](#email)
-  - [Registering a user](#registering-a-user)
-  - [Setting up a TURN server](#setting-up-a-turn-server)
-  - [URL previews](#url-previews)
+* [Installing Synapse](#installing-synapse)
+  * [Installing from source](#installing-from-source)
+    * [Platform-Specific Instructions](#platform-specific-instructions)
+    * [Troubleshooting Installation](#troubleshooting-installation)
+  * [Prebuilt packages](#prebuilt-packages)
+* [Setting up Synapse](#setting-up-synapse)
+  * [TLS certificates](#tls-certificates)
+  * [Registering a user](#registering-a-user)
+  * [Setting up a TURN server](#setting-up-a-turn-server)
+  * [URL previews](#url-previews)
 
 # Installing Synapse
 
@@ -395,22 +394,8 @@ To configure Synapse to expose an HTTPS port, you will need to edit
   instance, if using certbot, use `fullchain.pem` as your certificate, not
   `cert.pem`).
 
-For a more detailed guide to configuring your server for federation, see
-[federate.md](docs/federate.md)
-
-
-## Email
-
-It is desirable for Synapse to have the capability to send email. For example,
-this is required to support the 'password reset' feature.
-
-To configure an SMTP server for Synapse, modify the configuration section
-headed ``email``, and be sure to have at least the ``smtp_host``, ``smtp_port``
-and ``notif_from`` fields filled out. You may also need to set ``smtp_user``,
-``smtp_pass``, and ``require_transport_security``.
-
-If Synapse is not configured with an SMTP server, password reset via email will
- be disabled by default.
+For those of you upgrading your TLS certificate in readiness for Synapse 1.0,
+please take a look at [our guide](docs/MSC1711_certificates_FAQ.md#configuring-certificates-for-compatibility-with-synapse-100).
 
 ## Registering a user
 

MANIFEST.in

@@ -9,19 +9,14 @@ include demo/*.py
 include demo/*.sh
 
 recursive-include synapse/storage/schema *.sql
-recursive-include synapse/storage/schema *.sql.postgres
-recursive-include synapse/storage/schema *.sql.sqlite
 recursive-include synapse/storage/schema *.py
-recursive-include synapse/storage/schema *.txt
 
 recursive-include docs *
 recursive-include scripts *
 recursive-include scripts-dev *
 recursive-include synapse *.pyi
+recursive-include tests *.pem
 recursive-include tests *.py
-include tests/http/ca.crt
-include tests/http/ca.key
-include tests/http/server.key
 
 recursive-include synapse/res *
 recursive-include synapse/static *.css

UPGRADE.rst

@@ -49,82 +49,6 @@ returned by the Client-Server API:
     # configured on port 443.
     curl -kv https://<host.name>/_matrix/client/versions 2>&1 | grep "Server:"
 
-Upgrading to v1.1
-=================
-
-Synapse 1.1 removes support for older Python and PostgreSQL versions, as
-outlined in `our deprecation notice <https://matrix.org/blog/2019/04/08/synapse-deprecating-postgres-9-4-and-python-2-x>`_.
-
-Minimum Python Version
-----------------------
-
-Synapse v1.1 has a minimum Python requirement of Python 3.5. Python 3.6 or
-Python 3.7 are recommended as they have improved internal string handling,
-significantly reducing memory usage.
-
-If you use current versions of the Matrix.org-distributed Debian packages or
-Docker images, action is not required.
-
-If you install Synapse in a Python virtual environment, please see "Upgrading to
-v0.34.0" for notes on setting up a new virtualenv under Python 3.
-
-Minimum PostgreSQL Version
---------------------------
-
-If using PostgreSQL under Synapse, you will need to use PostgreSQL 9.5 or above.
-Please see the
-`PostgreSQL documentation <https://www.postgresql.org/docs/11/upgrading.html>`_
-for more details on upgrading your database.
-
-Upgrading to v1.0
-=================
-
-Validation of TLS certificates
-------------------------------
-
-Synapse v1.0 is the first release to enforce
-validation of TLS certificates for the federation API. It is therefore
-essential that your certificates are correctly configured. See the `FAQ
-<docs/MSC1711_certificates_FAQ.md>`_ for more information.
-
-Note, v1.0 installations will also no longer be able to federate with servers
-that have not correctly configured their certificates.
-
-In rare cases, it may be desirable to disable certificate checking: for
-example, it might be essential to be able to federate with a given legacy
-server in a closed federation. This can be done in one of two ways:-
-
-* Configure the global switch ``federation_verify_certificates`` to ``false``.
-* Configure a whitelist of server domains to trust via ``federation_certificate_verification_whitelist``.
-
-See the `sample configuration file <docs/sample_config.yaml>`_
-for more details on these settings.
-
-Email
------
-When a user requests a password reset, Synapse will send an email to the
-user to confirm the request.
-
-Previous versions of Synapse delegated the job of sending this email to an
-identity server. If the identity server was somehow malicious or became
-compromised, it would be theoretically possible to hijack an account through
-this means.
-
-Therefore, by default, Synapse v1.0 will send the confirmation email itself. If
-Synapse is not configured with an SMTP server, password reset via email will be
-disabled.
-
-To configure an SMTP server for Synapse, modify the configuration section
-headed ``email``, and be sure to have at least the ``smtp_host``, ``smtp_port``
-and ``notif_from`` fields filled out. You may also need to set ``smtp_user``,
-``smtp_pass``, and ``require_transport_security``.
-
-If you are absolutely certain that you wish to continue using an identity
-server for password resets, set ``trust_identity_server_for_password_resets`` to ``true``.
-
-See the `sample configuration file <docs/sample_config.yaml>`_
-for more details on these settings.
-
 Upgrading to v0.99.0
 ====================
 

changelog.d/4338.feature

@@ -0,0 +1 @@
+Synapse now more efficiently collates room statistics.

changelog.d/5015.misc

@@ -1 +0,0 @@
-Add logging to 3pid invite signature verification.

changelog.d/5200.bugfix

@@ -0,0 +1 @@
+Fix worker registration bug caused by ClientReaderSlavedStore being unable to see get_profileinfo.

changelog.d/5230.misc

@@ -0,0 +1 @@
+Remove urllib3 pin as requests 2.22.0 has been released supporting urllib3 1.25.2.

changelog.d/5232.misc

@@ -0,0 +1 @@
+Run black on synapse.crypto.keyring.

changelog.d/5239.bugfix

@@ -0,0 +1 @@
+Fix 500 Internal Server Error when sending an event with `m.relates_to: null`.

changelog.d/5252.feature

@@ -1 +0,0 @@
-Add monthly active users to phonehome stats.

changelog.d/5325.bugfix

@@ -1 +0,0 @@
-Fix a bug where running synapse_port_db would cause the account validity feature to fail because it didn't set the type of the email_sent column to boolean.

changelog.d/5363.feature

@@ -1 +0,0 @@
-Allow expired user to trigger renewal email sending manually.

changelog.d/5378.misc

@@ -1 +0,0 @@
-Track deactivated accounts in the database.

changelog.d/5381.misc

@@ -1 +0,0 @@
-Clean up code for sending federation EDUs.

changelog.d/5382.misc

@@ -1 +0,0 @@
-Add a sponsor button to the repo.

changelog.d/5383.misc

@@ -1 +0,0 @@
-Don't log non-200 responses from federation queries as exceptions.

changelog.d/5384.feature

@@ -1 +0,0 @@
-Statistics on forward extremities per room are now exposed via Prometheus.

changelog.d/5386.misc

@@ -1 +0,0 @@
-Add a sponsor button to the repo.

changelog.d/5387.bugfix

@@ -1 +0,0 @@
-Warn about disabling email-based password resets when a reset occurs, and remove warning when someone attempts a phone-based reset.

changelog.d/5388.bugfix

@@ -1 +0,0 @@
-Fix email notifications for unnamed rooms with multiple people.

changelog.d/5389.bugfix

@@ -1 +0,0 @@
-Fix exceptions in federation reader worker caused by attempting to renew attestations, which should only happen on master worker.

changelog.d/5390.bugfix

@@ -1 +0,0 @@
-Fix handling of failures fetching remote content to not log failures as exceptions.

changelog.d/5394.bugfix

@@ -1 +0,0 @@
-Fix a bug where deactivated users could receive renewal emails if the account validity feature is on.

changelog.d/5412.feature

@@ -1 +0,0 @@
-Add --no-daemonize option to run synapse in the foreground, per issue #4130. Contributed by Soham Gumaste.

changelog.d/5425.removal

@@ -1 +0,0 @@
-Python 2.7 is no longer a supported platform. Synapse now requires Python 3.5+ to run.

changelog.d/5440.feature

@@ -1 +0,0 @@
-Allow server admins to define implementations of extra rules for allowing or denying incoming events.

changelog.d/5446.misc

@@ -1 +0,0 @@
-Update Python syntax in contrib/ to Python 3. 

changelog.d/5447.misc

@@ -1 +0,0 @@
-Update federation_client dev script to support `.well-known` and work with python3.

changelog.d/5448.removal

@@ -1 +0,0 @@
-PostgreSQL 9.4 is no longer supported. Synapse requires Postgres 9.5+ or above for Postgres support.

changelog.d/5458.feature

@@ -1 +0,0 @@
-Statistics on forward extremities per room are now exposed via Prometheus.

changelog.d/5459.misc

@@ -1 +0,0 @@
-SyTest has been moved to Buildkite.

changelog.d/5460.misc

@@ -1 +0,0 @@
-Demo script now uses python3.

changelog.d/5461.feature

@@ -1 +0,0 @@
-Statistics on forward extremities per room are now exposed via Prometheus.

changelog.d/5464.bugfix

@@ -1 +0,0 @@
-Fix missing invite state after exchanging 3PID invites over federaton.

changelog.d/5465.misc

@@ -1,2 +0,0 @@
-Track deactivated accounts in the database.
-

changelog.d/5474.feature

@@ -1 +0,0 @@
-Allow server admins to define implementations of extra rules for allowing or denying incoming events.

changelog.d/5477.feature

@@ -1 +0,0 @@
-Allow server admins to define implementations of extra rules for allowing or denying incoming events.

changelog.d/5478.misc

@@ -1 +0,0 @@
-The demo servers talk to each other again.

contrib/cmdclient/console.py

@@ -15,7 +15,6 @@
 # limitations under the License.
 
 """ Starts a synapse client console. """
-from __future__ import print_function
 
 from twisted.internet import reactor, defer, threads
 from http import TwistedHttpClient
@@ -110,7 +109,7 @@ class SynapseCmd(cmd.Cmd):
         by using $. E.g. 'config roomid room1' then 'raw get /rooms/$roomid'.
         """
         if len(line) == 0:
-            print(json.dumps(self.config, indent=4))
+            print json.dumps(self.config, indent=4)
             return
 
         try:
@@ -124,8 +123,8 @@ class SynapseCmd(cmd.Cmd):
             ]
             for key, valid_vals in config_rules:
                 if key == args["key"] and args["val"] not in valid_vals:
-                    print("%s value must be one of %s" % (args["key"],
-                                                          valid_vals))
+                    print "%s value must be one of %s" % (args["key"],
+                                                          valid_vals)
                     return
 
             # toggle the http client verbosity
@@ -134,11 +133,11 @@ class SynapseCmd(cmd.Cmd):
 
             # assign the new config
             self.config[args["key"]] = args["val"]
-            print(json.dumps(self.config, indent=4))
+            print json.dumps(self.config, indent=4)
 
             save_config(self.config)
         except Exception as e:
-            print(e)
+            print e
 
     def do_register(self, line):
         """Registers for a new account: "register <userid> <noupdate>"
@@ -154,7 +153,7 @@ class SynapseCmd(cmd.Cmd):
             pwd = getpass.getpass("Type a password for this user: ")
             pwd2 = getpass.getpass("Retype the password: ")
             if pwd != pwd2 or len(pwd) == 0:
-                print("Password mismatch.")
+                print "Password mismatch."
                 pwd = None
             else:
                 password = pwd
@@ -175,12 +174,12 @@ class SynapseCmd(cmd.Cmd):
         # check the registration flows
         url = self._url() + "/register"
         json_res = yield self.http_client.do_request("GET", url)
-        print(json.dumps(json_res, indent=4))
+        print json.dumps(json_res, indent=4)
 
         passwordFlow = None
         for flow in json_res["flows"]:
             if flow["type"] == "m.login.recaptcha" or ("stages" in flow and "m.login.recaptcha" in flow["stages"]):
-                print("Unable to register: Home server requires captcha.")
+                print "Unable to register: Home server requires captcha."
                 return
             if flow["type"] == "m.login.password" and "stages" not in flow:
                 passwordFlow = flow
@@ -190,7 +189,7 @@ class SynapseCmd(cmd.Cmd):
             return
 
         json_res = yield self.http_client.do_request("POST", url, data=data)
-        print(json.dumps(json_res, indent=4))
+        print json.dumps(json_res, indent=4)
         if update_config and "user_id" in json_res:
             self.config["user"] = json_res["user_id"]
             self.config["token"] = json_res["access_token"]
@@ -216,7 +215,7 @@ class SynapseCmd(cmd.Cmd):
                 reactor.callFromThread(self._do_login, user, p)
                 #print " got %s " % p
         except Exception as e:
-            print(e)
+            print e
 
     @defer.inlineCallbacks
     def _do_login(self, user, password):
@@ -228,13 +227,13 @@ class SynapseCmd(cmd.Cmd):
         }
         url = self._url() + path
         json_res = yield self.http_client.do_request("POST", url, data=data)
-        print(json_res)
+        print json_res
 
         if "access_token" in json_res:
             self.config["user"] = user
             self.config["token"] = json_res["access_token"]
             save_config(self.config)
-            print("Login successful.")
+            print "Login successful."
 
     @defer.inlineCallbacks
     def _check_can_login(self):
@@ -243,10 +242,10 @@ class SynapseCmd(cmd.Cmd):
         # submitting!
         url = self._url() + path
         json_res = yield self.http_client.do_request("GET", url)
-        print(json_res)
+        print json_res
 
         if "flows" not in json_res:
-            print("Failed to find any login flows.")
+            print "Failed to find any login flows."
             defer.returnValue(False)
 
         flow = json_res["flows"][0] # assume first is the one we want.
@@ -276,9 +275,9 @@ class SynapseCmd(cmd.Cmd):
 
         json_res = yield self.http_client.do_request("POST", url, data=urllib.urlencode(args), jsonreq=False,
                                                      headers={'Content-Type': ['application/x-www-form-urlencoded']})
-        print(json_res)
+        print json_res
         if 'sid' in json_res:
-            print("Token sent. Your session ID is %s" % (json_res['sid']))
+            print "Token sent. Your session ID is %s" % (json_res['sid'])
 
     def do_emailvalidate(self, line):
         """Validate and associate a third party ID
@@ -298,7 +297,7 @@ class SynapseCmd(cmd.Cmd):
 
         json_res = yield self.http_client.do_request("POST", url, data=urllib.urlencode(args), jsonreq=False,
                                                      headers={'Content-Type': ['application/x-www-form-urlencoded']})
-        print(json_res)
+        print json_res
 
     def do_3pidbind(self, line):
         """Validate and associate a third party ID
@@ -318,7 +317,7 @@ class SynapseCmd(cmd.Cmd):
 
         json_res = yield self.http_client.do_request("POST", url, data=urllib.urlencode(args), jsonreq=False,
                                                      headers={'Content-Type': ['application/x-www-form-urlencoded']})
-        print(json_res)
+        print json_res
 
     def do_join(self, line):
         """Joins a room: "join <roomid>" """
@@ -326,7 +325,7 @@ class SynapseCmd(cmd.Cmd):
             args = self._parse(line, ["roomid"], force_keys=True)
             self._do_membership_change(args["roomid"], "join", self._usr())
         except Exception as e:
-            print(e)
+            print e
 
     def do_joinalias(self, line):
         try:
@@ -334,7 +333,7 @@ class SynapseCmd(cmd.Cmd):
             path = "/join/%s" % urllib.quote(args["roomname"])
             reactor.callFromThread(self._run_and_pprint, "POST", path, {})
         except Exception as e:
-            print(e)
+            print e
 
     def do_topic(self, line):
         """"topic [set|get] <roomid> [<newtopic>]"
@@ -344,17 +343,17 @@ class SynapseCmd(cmd.Cmd):
         try:
             args = self._parse(line, ["action", "roomid", "topic"])
             if "action" not in args or "roomid" not in args:
-                print("Must specify set|get and a room ID.")
+                print "Must specify set|get and a room ID."
                 return
             if args["action"].lower() not in ["set", "get"]:
-                print("Must specify set|get, not %s" % args["action"])
+                print "Must specify set|get, not %s" % args["action"]
                 return
 
             path = "/rooms/%s/topic" % urllib.quote(args["roomid"])
 
             if args["action"].lower() == "set":
                 if "topic" not in args:
-                    print("Must specify a new topic.")
+                    print "Must specify a new topic."
                     return
                 body = {
                     "topic": args["topic"]
@@ -363,7 +362,7 @@ class SynapseCmd(cmd.Cmd):
             elif args["action"].lower() == "get":
                 reactor.callFromThread(self._run_and_pprint, "GET", path)
         except Exception as e:
-            print(e)
+            print e
 
     def do_invite(self, line):
         """Invite a user to a room: "invite <userid> <roomid>" """
@@ -374,7 +373,7 @@ class SynapseCmd(cmd.Cmd):
 
             reactor.callFromThread(self._do_invite, args["roomid"], user_id)
         except Exception as e:
-            print(e)
+            print e
 
     @defer.inlineCallbacks
     def _do_invite(self, roomid, userstring):
@@ -394,29 +393,29 @@ class SynapseCmd(cmd.Cmd):
                 if 'public_key' in pubKeyObj:
                     pubKey = nacl.signing.VerifyKey(pubKeyObj['public_key'], encoder=nacl.encoding.HexEncoder)
                 else:
-                    print("No public key found in pubkey response!")
+                    print "No public key found in pubkey response!"
 
                 sigValid = False
 
                 if pubKey:
                     for signame in json_res['signatures']:
                         if signame not in TRUSTED_ID_SERVERS:
-                            print("Ignoring signature from untrusted server %s" % (signame))
+                            print "Ignoring signature from untrusted server %s" % (signame)
                         else:
                             try:
                                 verify_signed_json(json_res, signame, pubKey)
                                 sigValid = True
-                                print("Mapping %s -> %s correctly signed by %s" % (userstring, json_res['mxid'], signame))
+                                print "Mapping %s -> %s correctly signed by %s" % (userstring, json_res['mxid'], signame)
                                 break
                             except SignatureVerifyException as e:
-                                print("Invalid signature from %s" % (signame))
-                                print(e)
+                                print "Invalid signature from %s" % (signame)
+                                print e
 
                 if sigValid:
-                    print("Resolved 3pid %s to %s" % (userstring, json_res['mxid']))
+                    print "Resolved 3pid %s to %s" % (userstring, json_res['mxid'])
                     mxid = json_res['mxid']
                 else:
-                    print("Got association for %s but couldn't verify signature" % (userstring))
+                    print "Got association for %s but couldn't verify signature" % (userstring)
 
             if not mxid:
                 mxid = "@" + userstring + ":" + self._domain()
@@ -429,7 +428,7 @@ class SynapseCmd(cmd.Cmd):
             args = self._parse(line, ["roomid"], force_keys=True)
             self._do_membership_change(args["roomid"], "leave", self._usr())
         except Exception as e:
-            print(e)
+            print e
 
     def do_send(self, line):
         """Sends a message. "send <roomid> <body>" """
@@ -454,10 +453,10 @@ class SynapseCmd(cmd.Cmd):
         """
         args = self._parse(line, ["type", "roomid", "qp"])
         if not "type" in args or not "roomid" in args:
-            print("Must specify type and room ID.")
+            print "Must specify type and room ID."
             return
         if args["type"] not in ["members", "messages"]:
-            print("Unrecognised type: %s" % args["type"])
+            print "Unrecognised type: %s" % args["type"]
             return
         room_id = args["roomid"]
         path = "/rooms/%s/%s" % (urllib.quote(room_id), args["type"])
@@ -469,7 +468,7 @@ class SynapseCmd(cmd.Cmd):
                     key_value = key_value_str.split("=")
                     qp[key_value[0]] = key_value[1]
                 except:
-                    print("Bad query param: %s" % key_value)
+                    print "Bad query param: %s" % key_value
                     return
 
         reactor.callFromThread(self._run_and_pprint, "GET", path,
@@ -509,14 +508,14 @@ class SynapseCmd(cmd.Cmd):
         args = self._parse(line, ["method", "path", "data"])
         # sanity check
         if "method" not in args or "path" not in args:
-            print("Must specify path and method.")
+            print "Must specify path and method."
             return
 
         args["method"] = args["method"].upper()
         valid_methods = ["PUT", "GET", "POST", "DELETE",
                          "XPUT", "XGET", "XPOST", "XDELETE"]
         if args["method"] not in valid_methods:
-            print("Unsupported method: %s" % args["method"])
+            print "Unsupported method: %s" % args["method"]
             return
 
         if "data" not in args:
@@ -525,7 +524,7 @@ class SynapseCmd(cmd.Cmd):
             try:
                 args["data"] = json.loads(args["data"])
             except Exception as e:
-                print("Data is not valid JSON. %s" % e)
+                print "Data is not valid JSON. %s" % e
                 return
 
         qp = {"access_token": self._tok()}
@@ -554,7 +553,7 @@ class SynapseCmd(cmd.Cmd):
             try:
                 timeout = int(args["timeout"])
             except ValueError:
-                print("Timeout must be in milliseconds.")
+                print "Timeout must be in milliseconds."
                 return
         reactor.callFromThread(self._do_event_stream, timeout)
 
@@ -567,7 +566,7 @@ class SynapseCmd(cmd.Cmd):
                     "timeout": str(timeout),
                     "from": self.event_stream_token
                 })
-        print(json.dumps(res, indent=4))
+        print json.dumps(res, indent=4)
 
         if "chunk" in res:
             for event in res["chunk"]:
@@ -670,9 +669,9 @@ class SynapseCmd(cmd.Cmd):
                                                     data=data,
                                                     qparams=query_params)
         if alt_text:
-            print(alt_text)
+            print alt_text
         else:
-            print(json.dumps(json_res, indent=4))
+            print json.dumps(json_res, indent=4)
 
 
 def save_config(config):
@@ -681,16 +680,16 @@ def save_config(config):
 
 
 def main(server_url, identity_server_url, username, token, config_path):
-    print("Synapse command line client")
-    print("===========================")
-    print("Server: %s" % server_url)
-    print("Type 'help' to get started.")
-    print("Close this console with CTRL+C then CTRL+D.")
+    print "Synapse command line client"
+    print "==========================="
+    print "Server: %s" % server_url
+    print "Type 'help' to get started."
+    print "Close this console with CTRL+C then CTRL+D."
     if not username or not token:
-        print("-  'register <username>' - Register an account")
-        print("-  'stream' - Connect to the event stream")
-        print("-  'create <roomid>' - Create a room")
-        print("-  'send <roomid> <message>' - Send a message")
+        print "-  'register <username>' - Register an account"
+        print "-  'stream' - Connect to the event stream"
+        print "-  'create <roomid>' - Create a room"
+        print "-  'send <roomid> <message>' - Send a message"
     http_client = TwistedHttpClient()
 
     # the command line client
@@ -706,7 +705,7 @@ def main(server_url, identity_server_url, username, token, config_path):
                 http_client.verbose = "on" == syn_cmd.config["verbose"]
             except:
                 pass
-            print("Loaded config from %s" % config_path)
+            print "Loaded config from %s" % config_path
     except:
         pass
 
@@ -737,7 +736,7 @@ if __name__ == '__main__':
     args = parser.parse_args()
 
     if not args.server:
-        print("You must supply a server URL to communicate with.")
+        print "You must supply a server URL to communicate with."
         parser.print_help()
         sys.exit(1)
 

contrib/cmdclient/http.py

@@ -13,7 +13,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from __future__ import print_function
 from twisted.web.client import Agent, readBody
 from twisted.web.http_headers import Headers
 from twisted.internet import defer, reactor
@@ -142,15 +141,15 @@ class TwistedHttpClient(HttpClient):
         headers_dict["User-Agent"] = ["Synapse Cmd Client"]
 
         retries_left = 5
-        print("%s to %s with headers %s" % (method, url, headers_dict))
+        print "%s to %s with headers %s" % (method, url, headers_dict)
         if self.verbose and producer:
             if "password" in producer.data:
                 temp = producer.data["password"]
                 producer.data["password"] = "[REDACTED]"
-                print(json.dumps(producer.data, indent=4))
+                print json.dumps(producer.data, indent=4)
                 producer.data["password"] = temp
             else:
-                print(json.dumps(producer.data, indent=4))
+                print json.dumps(producer.data, indent=4)
 
         while True:
             try:
@@ -162,7 +161,7 @@ class TwistedHttpClient(HttpClient):
                 )
                 break
             except Exception as e:
-                print("uh oh: %s" % e)
+                print "uh oh: %s" % e
                 if retries_left:
                     yield self.sleep(2 ** (5 - retries_left))
                     retries_left -= 1
@@ -170,8 +169,8 @@ class TwistedHttpClient(HttpClient):
                     raise e
 
         if self.verbose:
-            print("Status %s %s" % (response.code, response.phrase))
-            print(pformat(list(response.headers.getAllRawHeaders())))
+            print "Status %s %s" % (response.code, response.phrase)
+            print pformat(list(response.headers.getAllRawHeaders()))
         defer.returnValue(response)
 
     def sleep(self, seconds):

contrib/graph/graph.py

@@ -1,4 +1,3 @@
-from __future__ import print_function
 # Copyright 2014-2016 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -49,7 +48,7 @@ def make_graph(pdus, room, filename_prefix):
             c = colors.pop()
             color_map[o] = c
         except:
-            print("Run out of colours!")
+            print "Run out of colours!"
             color_map[o] = "black"
 
     graph = pydot.Dot(graph_name="Test")
@@ -94,7 +93,7 @@ def make_graph(pdus, room, filename_prefix):
             end_name = make_name(i, o)
 
             if end_name not in node_map:
-                print("%s not in nodes" % end_name)
+                print "%s not in nodes" % end_name
                 continue
 
             edge = pydot.Edge(node_map[start_name], node_map[end_name])

contrib/graph/graph3.py

@@ -1,4 +1,3 @@
-from __future__ import print_function
 # Copyright 2016 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -27,19 +26,19 @@ from six import string_types
 
 
 def make_graph(file_name, room_id, file_prefix, limit):
-    print("Reading lines")
+    print "Reading lines"
     with open(file_name) as f:
         lines = f.readlines()
 
-    print("Read lines")
+    print "Read lines"
 
     events = [FrozenEvent(json.loads(line)) for line in lines]
 
-    print("Loaded events.")
+    print "Loaded events."
 
     events.sort(key=lambda e: e.depth)
 
-    print("Sorted events")
+    print "Sorted events"
 
     if limit:
         events = events[-int(limit):]
@@ -56,7 +55,7 @@ def make_graph(file_name, room_id, file_prefix, limit):
         content = json.dumps(unfreeze(event.get_dict()["content"]), indent=4)
         content = content.replace("\n", "<br/>\n")
 
-        print(content)
+        print content
         content = []
         for key, value in unfreeze(event.get_dict()["content"]).items():
             if value is None:
@@ -75,7 +74,7 @@ def make_graph(file_name, room_id, file_prefix, limit):
 
         content = "<br/>\n".join(content)
 
-        print(content)
+        print content
 
         label = (
             "<"
@@ -103,7 +102,7 @@ def make_graph(file_name, room_id, file_prefix, limit):
         node_map[event.event_id] = node
         graph.add_node(node)
 
-    print("Created Nodes")
+    print "Created Nodes"
 
     for event in events:
         for prev_id, _ in event.prev_events:
@@ -121,15 +120,15 @@ def make_graph(file_name, room_id, file_prefix, limit):
             edge = pydot.Edge(node_map[event.event_id], end_node)
             graph.add_edge(edge)
 
-    print("Created edges")
+    print "Created edges"
 
     graph.write('%s.dot' % file_prefix, format='raw', prog='dot')
 
-    print("Created Dot")
+    print "Created Dot"
 
     graph.write_svg("%s.svg" % file_prefix, prog='dot')
 
-    print("Created svg")
+    print "Created svg"
 
 if __name__ == "__main__":
     parser = argparse.ArgumentParser(

contrib/jitsimeetbridge/jitsimeetbridge.py

@@ -8,9 +8,8 @@ we set the remote SDP at which point the stream ends. Our video never gets to
 the bridge.
 
 Requires:
-npm install jquery jsdom
+npm install jquery jsdom 
 """
-from __future__ import print_function
 
 import gevent
 import grequests
@@ -52,7 +51,7 @@ class TrivialMatrixClient:
             req = grequests.get(url)
             resps = grequests.map([req])
             obj = json.loads(resps[0].content)
-            print("incoming from matrix",obj)
+            print "incoming from matrix",obj
             if 'end' not in obj:
                 continue
             self.token = obj['end']
@@ -61,22 +60,22 @@ class TrivialMatrixClient:
 
     def joinRoom(self, roomId):
         url = MATRIXBASE+'rooms/'+roomId+'/join?access_token='+self.access_token
-        print(url)
+        print url
         headers={ 'Content-Type': 'application/json' }
         req = grequests.post(url, headers=headers, data='{}')
         resps = grequests.map([req])
         obj = json.loads(resps[0].content)
-        print("response: ",obj)
+        print "response: ",obj
 
     def sendEvent(self, roomId, evType, event):
         url = MATRIXBASE+'rooms/'+roomId+'/send/'+evType+'?access_token='+self.access_token
-        print(url)
-        print(json.dumps(event))
+        print url
+        print json.dumps(event)
         headers={ 'Content-Type': 'application/json' }
         req = grequests.post(url, headers=headers, data=json.dumps(event))
         resps = grequests.map([req])
         obj = json.loads(resps[0].content)
-        print("response: ",obj)
+        print "response: ",obj
 
 
 
@@ -86,31 +85,31 @@ xmppClients = {}
 def matrixLoop():
     while True:
         ev = matrixCli.getEvent()
-        print(ev)
+        print ev
         if ev['type'] == 'm.room.member':
-            print('membership event')
+            print 'membership event'
             if ev['membership'] == 'invite' and ev['state_key'] == MYUSERNAME:
                 roomId = ev['room_id']
-                print("joining room %s" % (roomId))
+                print "joining room %s" % (roomId)
                 matrixCli.joinRoom(roomId)
         elif ev['type'] == 'm.room.message':
             if ev['room_id'] in xmppClients:
-                print("already have a bridge for that user, ignoring")
+                print "already have a bridge for that user, ignoring"
                 continue
-            print("got message, connecting")
+            print "got message, connecting"
             xmppClients[ev['room_id']] = TrivialXmppClient(ev['room_id'], ev['user_id'])
             gevent.spawn(xmppClients[ev['room_id']].xmppLoop)
         elif ev['type'] == 'm.call.invite':
-            print("Incoming call")
+            print "Incoming call"
             #sdp = ev['content']['offer']['sdp']
             #print "sdp: %s" % (sdp)
             #xmppClients[ev['room_id']] = TrivialXmppClient(ev['room_id'], ev['user_id'])
             #gevent.spawn(xmppClients[ev['room_id']].xmppLoop)
         elif ev['type'] == 'm.call.answer':
-            print("Call answered")
+            print "Call answered"
             sdp = ev['content']['answer']['sdp']
             if ev['room_id'] not in xmppClients:
-                print("We didn't have a call for that room")
+                print "We didn't have a call for that room"
                 continue
             # should probably check call ID too
             xmppCli = xmppClients[ev['room_id']]
@@ -147,7 +146,7 @@ class TrivialXmppClient:
         return obj
 
     def sendAnswer(self, answer):
-        print("sdp from matrix client",answer)
+        print "sdp from matrix client",answer
         p = subprocess.Popen(['node', 'unjingle/unjingle.js', '--sdp'], stdin=subprocess.PIPE, stdout=subprocess.PIPE)
         jingle, out_err = p.communicate(answer)
         jingle = jingle % {
@@ -157,28 +156,28 @@ class TrivialXmppClient:
             'responder': self.jid,
             'sid': self.callsid
         }
-        print("answer jingle from sdp",jingle)
+        print "answer jingle from sdp",jingle
         res = self.sendIq(jingle)
-        print("reply from answer: ",res)
+        print "reply from answer: ",res
 
         self.ssrcs = {}
         jingleSoup = BeautifulSoup(jingle)
         for cont in jingleSoup.iq.jingle.findAll('content'):
             if cont.description:
                 self.ssrcs[cont['name']] = cont.description['ssrc']
-        print("my ssrcs:",self.ssrcs)
+        print "my ssrcs:",self.ssrcs
 
         gevent.joinall([
                 gevent.spawn(self.advertiseSsrcs)
         ])
 
     def advertiseSsrcs(self):
-        time.sleep(7)
-        print("SSRC spammer started")
+                time.sleep(7)
+        print "SSRC spammer started"
         while self.running:
             ssrcMsg = "<presence to='%(tojid)s' xmlns='jabber:client'><x xmlns='http://jabber.org/protocol/muc'/><c xmlns='http://jabber.org/protocol/caps' hash='sha-1' node='http://jitsi.org/jitsimeet' ver='0WkSdhFnAUxrz4ImQQLdB80GFlE='/><nick xmlns='http://jabber.org/protocol/nick'>%(nick)s</nick><stats xmlns='http://jitsi.org/jitmeet/stats'><stat name='bitrate_download' value='175'/><stat name='bitrate_upload' value='176'/><stat name='packetLoss_total' value='0'/><stat name='packetLoss_download' value='0'/><stat name='packetLoss_upload' value='0'/></stats><media xmlns='http://estos.de/ns/mjs'><source type='audio' ssrc='%(assrc)s' direction='sendre'/><source type='video' ssrc='%(vssrc)s' direction='sendre'/></media></presence>" % { 'tojid': "%s@%s/%s" % (ROOMNAME, ROOMDOMAIN, self.shortJid), 'nick': self.userId, 'assrc': self.ssrcs['audio'], 'vssrc': self.ssrcs['video'] }
             res = self.sendIq(ssrcMsg)
-            print("reply from ssrc announce: ",res)
+            print "reply from ssrc announce: ",res
             time.sleep(10)
 
 
@@ -187,19 +186,19 @@ class TrivialXmppClient:
         self.matrixCallId = time.time()
         res = self.xmppPoke("<body rid='%s' xmlns='http://jabber.org/protocol/httpbind' to='%s' xml:lang='en' wait='60' hold='1' content='text/xml; charset=utf-8' ver='1.6' xmpp:version='1.0' xmlns:xmpp='urn:xmpp:xbosh'/>" % (self.nextRid(), HOST))
 
-        print(res)
+        print res
         self.sid = res.body['sid']
-        print("sid %s" % (self.sid))
+        print "sid %s" % (self.sid)
 
         res = self.sendIq("<auth xmlns='urn:ietf:params:xml:ns:xmpp-sasl' mechanism='ANONYMOUS'/>")
 
         res = self.xmppPoke("<body rid='%s' xmlns='http://jabber.org/protocol/httpbind' sid='%s' to='%s' xml:lang='en' xmpp:restart='true' xmlns:xmpp='urn:xmpp:xbosh'/>" % (self.nextRid(), self.sid, HOST))
 
         res = self.sendIq("<iq type='set' id='_bind_auth_2' xmlns='jabber:client'><bind xmlns='urn:ietf:params:xml:ns:xmpp-bind'/></iq>")
-        print(res)
+        print res
 
         self.jid = res.body.iq.bind.jid.string
-        print("jid: %s" % (self.jid))
+        print "jid: %s" % (self.jid)
         self.shortJid = self.jid.split('-')[0]
 
         res = self.sendIq("<iq type='set' id='_session_auth_2' xmlns='jabber:client'><session xmlns='urn:ietf:params:xml:ns:xmpp-session'/></iq>")
@@ -218,13 +217,13 @@ class TrivialXmppClient:
             if p.c and p.c.nick:
                 u['nick'] = p.c.nick.string
             self.muc['users'].append(u)
-        print("muc: ",self.muc)
+        print "muc: ",self.muc
 
         # wait for stuff
         while True:
-            print("waiting...")
+            print "waiting..."
             res = self.sendIq("")
-            print("got from stream: ",res)
+            print "got from stream: ",res
             if res.body.iq:
                 jingles = res.body.iq.findAll('jingle')
                 if len(jingles):
@@ -233,15 +232,15 @@ class TrivialXmppClient:
             elif 'type' in res.body and res.body['type'] == 'terminate':
                 self.running = False
                 del xmppClients[self.matrixRoom]
-                return
+                        return
 
     def handleInvite(self, jingle):
         self.initiator = jingle['initiator']
         self.callsid = jingle['sid']
         p = subprocess.Popen(['node', 'unjingle/unjingle.js', '--jingle'], stdin=subprocess.PIPE, stdout=subprocess.PIPE)
-        print("raw jingle invite",str(jingle))
+        print "raw jingle invite",str(jingle)
         sdp, out_err = p.communicate(str(jingle))
-        print("transformed remote offer sdp",sdp)
+        print "transformed remote offer sdp",sdp
         inviteEvent = {
             'offer': {
                 'type': 'offer',
@@ -253,7 +252,7 @@ class TrivialXmppClient:
         }
         matrixCli.sendEvent(self.matrixRoom, 'm.call.invite', inviteEvent)
 
-matrixCli = TrivialMatrixClient(ACCESS_TOKEN)  # Undefined name
+matrixCli = TrivialMatrixClient(ACCESS_TOKEN)
 
 gevent.joinall([
     gevent.spawn(matrixLoop)

contrib/scripts/kick_users.py

@@ -1,16 +1,10 @@
 #!/usr/bin/env python
-from __future__ import print_function
 from argparse import ArgumentParser
 import json
 import requests
 import sys
 import urllib
 
-try:
-    raw_input
-except NameError:  # Python 3
-    raw_input = input
-
 def _mkurl(template, kws):
     for key in kws:
         template = template.replace(key, kws[key])
@@ -19,7 +13,7 @@ def _mkurl(template, kws):
 def main(hs, room_id, access_token, user_id_prefix, why):
     if not why:
         why = "Automated kick."
-    print("Kicking members on %s in room %s matching %s" % (hs, room_id, user_id_prefix))
+    print "Kicking members on %s in room %s matching %s" % (hs, room_id, user_id_prefix)
     room_state_url = _mkurl(
         "$HS/_matrix/client/api/v1/rooms/$ROOM/state?access_token=$TOKEN",
         {
@@ -28,13 +22,13 @@ def main(hs, room_id, access_token, user_id_prefix, why):
             "$TOKEN": access_token
         }
     )
-    print("Getting room state => %s" % room_state_url)
+    print "Getting room state => %s" % room_state_url
     res = requests.get(room_state_url)
-    print("HTTP %s" % res.status_code)
+    print "HTTP %s" % res.status_code
     state_events = res.json()
     if "error" in state_events:
-        print("FATAL")
-        print(state_events)
+        print "FATAL"
+        print state_events
         return
 
     kick_list = []
@@ -50,15 +44,15 @@ def main(hs, room_id, access_token, user_id_prefix, why):
             kick_list.append(event["state_key"])
 
     if len(kick_list) == 0:
-        print("No user IDs match the prefix '%s'" % user_id_prefix)
+        print "No user IDs match the prefix '%s'" % user_id_prefix
         return
 
-    print("The following user IDs will be kicked from %s" % room_name)
+    print "The following user IDs will be kicked from %s" % room_name
     for uid in kick_list:
-        print(uid)
+        print uid
     doit = raw_input("Continue? [Y]es\n")
     if len(doit) > 0 and doit.lower() == 'y':
-        print("Kicking members...")
+        print "Kicking members..."
         # encode them all
         kick_list = [urllib.quote(uid) for uid in kick_list]
         for uid in kick_list:
@@ -75,14 +69,14 @@ def main(hs, room_id, access_token, user_id_prefix, why):
                 "membership": "leave",
                 "reason": why
             }
-            print("Kicking %s" % uid)
+            print "Kicking %s" % uid
             res = requests.put(kick_url, data=json.dumps(kick_body))
             if res.status_code != 200:
-                print("ERROR: HTTP %s" % res.status_code)
+                print "ERROR: HTTP %s" % res.status_code
             if res.json().get("error"):
-                print("ERROR: JSON %s" % res.json())
-
-
+                print "ERROR: JSON %s" % res.json()
+            
+    
 
 if __name__ == "__main__":
     parser = ArgumentParser("Kick members in a room matching a certain user ID prefix.")

debian/changelog

@@ -1,15 +1,3 @@
-matrix-synapse-py3 (1.0.0) stable; urgency=medium
-
-  * New synapse release 1.0.0.
-
- -- Synapse Packaging team <packages@matrix.org>  Tue, 11 Jun 2019 17:09:53 +0100
-
-matrix-synapse-py3 (0.99.5.2) stable; urgency=medium
-
-  * New synapse release 0.99.5.2.
-
- -- Synapse Packaging team <packages@matrix.org>  Thu, 30 May 2019 16:28:07 +0100
-
 matrix-synapse-py3 (0.99.5.1) stable; urgency=medium
 
   * New synapse release 0.99.5.1.

demo/README

@@ -1,13 +1,9 @@
-DO NOT USE THESE DEMO SERVERS IN PRODUCTION
-
 Requires you to have done:
     python setup.py develop
 
 
-The demo start.sh will start three synapse servers on ports 8080, 8081 and 8082, with host names localhost:$port. This can be easily changed to `hostname`:$port in start.sh if required.
-
-To enable the servers to communicate untrusted ssl certs are used. In order to do this the servers do not check the certs
-and are configured in a highly insecure way. Do not use these configuration files in production.
+The demo start.sh will start three synapse servers on ports 8080, 8081 and 8082, with host names localhost:$port. This can be easily changed to `hostname`:$port in start.sh if required. 
+It will also start a web server on port 8000 pointed at the webclient.
 
 stop.sh will stop the synapse servers and the webclient.
 

demo/start.sh

@@ -21,76 +21,14 @@ for port in 8080 8081 8082; do
     pushd demo/$port
 
     #rm $DIR/etc/$port.config
-    python3 -m synapse.app.homeserver \
+    python -m synapse.app.homeserver \
         --generate-config \
         -H "localhost:$https_port" \
         --config-path "$DIR/etc/$port.config" \
         --report-stats no
 
-    if ! grep -F "Customisation made by demo/start.sh" -q  $DIR/etc/$port.config; then
-        printf '\n\n# Customisation made by demo/start.sh\n' >> $DIR/etc/$port.config
-        
-        echo 'enable_registration: true' >> $DIR/etc/$port.config
-
-        # Warning, this heredoc depends on the interaction of tabs and spaces. Please don't
-        # accidentaly bork me with your fancy settings.
-		listeners=$(cat <<-PORTLISTENERS
-		# Configure server to listen on both $https_port and $port
-		# This overides some of the default settings above
-		listeners:
-		  - port: $https_port
-		    type: http
-		    tls: true
-		    resources:
-		      - names: [client, federation]
-		
-		  - port: $port
-		    tls: false
-		    bind_addresses: ['::1', '127.0.0.1']
-		    type: http
-		    x_forwarded: true
-		    resources:
-		      - names: [client, federation]
-		        compress: false
-		PORTLISTENERS
-		)
-        echo "${listeners}" >> $DIR/etc/$port.config
-
-        # Disable tls for the servers
-        printf '\n\n# Disable tls on the servers.' >> $DIR/etc/$port.config
-        echo '# DO NOT USE IN PRODUCTION' >> $DIR/etc/$port.config
-        echo 'use_insecure_ssl_client_just_for_testing_do_not_use: true' >> $DIR/etc/$port.config
-        echo 'federation_verify_certificates: false' >> $DIR/etc/$port.config
-
-        # Set tls paths
-        echo "tls_certificate_path: \"$DIR/etc/localhost:$https_port.tls.crt\"" >> $DIR/etc/$port.config
-        echo "tls_private_key_path: \"$DIR/etc/localhost:$https_port.tls.key\"" >> $DIR/etc/$port.config
-
-        # Generate tls keys
-        openssl req -x509 -newkey rsa:4096 -keyout $DIR/etc/localhost\:$https_port.tls.key -out $DIR/etc/localhost\:$https_port.tls.crt -days 365 -nodes -subj "/O=matrix"
-        
-        # Ignore keys from the trusted keys server
-        echo '# Ignore keys from the trusted keys server' >> $DIR/etc/$port.config
-        echo 'trusted_key_servers:' >> $DIR/etc/$port.config
-        echo '  - server_name: "matrix.org"' >> $DIR/etc/$port.config
-        echo '    accept_keys_insecurely: true' >> $DIR/etc/$port.config
-
-        # Reduce the blacklist
-        blacklist=$(cat <<-BLACK
-		# Set the blacklist so that it doesn't include 127.0.0.1
-		federation_ip_range_blacklist:
-		  - '10.0.0.0/8'
-		  - '172.16.0.0/12'
-		  - '192.168.0.0/16'
-		  - '100.64.0.0/10'
-		  - '169.254.0.0/16'
-		  - '::1/128'
-		  - 'fe80::/64'
-		  - 'fc00::/7'
-		BLACK
-		)
-        echo "${blacklist}" >> $DIR/etc/$port.config
-    fi
+    printf '\n\n# Customisation made by demo/start.sh\n' >> $DIR/etc/$port.config
+    echo 'enable_registration: true' >> $DIR/etc/$port.config
 
     # Check script parameters
     if [ $# -eq 1 ]; then
@@ -117,7 +55,7 @@ for port in 8080 8081 8082; do
         echo "report_stats: false" >> $DIR/etc/$port.config
     fi
 
-    python3 -m synapse.app.homeserver \
+    python -m synapse.app.homeserver \
         --config-path "$DIR/etc/$port.config" \
         -D \
         -vv \

docker/Dockerfile-pgtests

@@ -3,10 +3,10 @@
 FROM matrixdotorg/sytest:latest
 
 # The Sytest image doesn't come with python, so install that
-RUN apt-get update && apt-get -qq install -y python3 python3-dev python3-pip
+RUN apt-get -qq install -y python python-dev python-pip
 
 # We need tox to run the tests in run_pg_tests.sh
-RUN python3 -m pip install tox
+RUN pip install tox
 
 ADD run_pg_tests.sh /pg_tests.sh
 ENTRYPOINT /pg_tests.sh

docker/README.md

@@ -14,7 +14,7 @@ This image is designed to run either with an automatically generated
 configuration file or with a custom configuration that requires manual editing.
 
 An easy way to make use of this image is via docker-compose. See the
-[contrib/docker](https://github.com/matrix-org/synapse/tree/master/contrib/docker) section of the synapse project for
+[contrib/docker](../contrib/docker) section of the synapse project for
 examples.
 
 ### Without Compose (harder)

docker/run_pg_tests.sh

@@ -17,4 +17,4 @@ su -c '/usr/lib/postgresql/9.6/bin/pg_ctl -w -D /var/lib/postgresql/data start'
 # Run the tests
 cd /src
 export TRIAL_FLAGS="-j 4"
-tox --workdir=/tmp -e py35-postgres
+tox --workdir=/tmp -e py27-postgres

docs/CAPTCHA_SETUP.rst

@@ -7,7 +7,6 @@ Requires a public/private key pair from:
 
 https://developers.google.com/recaptcha/
 
-Must be a reCAPTCHA v2 key using the "I'm not a robot" Checkbox option
 
 Setting ReCaptcha Keys
 ----------------------

docs/MSC1711_certificates_FAQ.md

@@ -1,22 +1,5 @@
 # MSC1711 Certificates FAQ
 
-## Historical Note
-This document was originally written to guide server admins through the upgrade
-path towards Synapse 1.0. Specifically,
-[MSC1711](https://github.com/matrix-org/matrix-doc/blob/master/proposals/1711-x509-for-federation.md)
-required that all servers present valid TLS certificates on their federation
-API. Admins were encouraged to achieve compliance from version 0.99.0 (released
-in February 2019) ahead of version 1.0 (released June 2019) enforcing the
-certificate checks.
-
-Much of what follows is now outdated since most admins will have already
-upgraded, however it may be of use to those with old installs returning to the
-project.
-
-If you are setting up a server from scratch you almost certainly should look at
-the [installation guide](INSTALL.md) instead.
-
-## Introduction
 The goal of Synapse 0.99.0 is to act as a stepping stone to Synapse 1.0.0. It
 supports the r0.1 release of the server to server specification, but is
 compatible with both the legacy Matrix federation behaviour (pre-r0.1) as well
@@ -85,14 +68,16 @@ Admins should upgrade and configure a valid CA cert. Homeservers that require a
 .well-known entry (see below), should retain their SRV record and use it
 alongside their .well-known record.
 
-**10th June 2019  - Synapse 1.0.0 is released**
+**>= 5th March 2019  - Synapse 1.0.0 is released**
 
-1.0.0 is scheduled for release on 10th June. In
+1.0.0 will land no sooner than 1 month after 0.99.0, leaving server admins one
+month after 5th February to upgrade to 0.99.0 and deploy their certificates. In
 accordance with the the [S2S spec](https://matrix.org/docs/spec/server_server/r0.1.0.html)
 1.0.0 will enforce certificate validity. This means that any homeserver without a
 valid certificate after this point will no longer be able to federate with
 1.0.0 servers.
 
+
 ## Configuring certificates for compatibility with Synapse 1.0.0
 
 ### If you do not currently have an SRV record
@@ -160,11 +145,12 @@ You can do this with a `.well-known` file as follows:
  1. Keep the SRV record in place - it is needed for backwards compatibility
     with Synapse 0.34 and earlier.
 
-  2. Give Synapse a certificate corresponding to the target domain
-    (`customer.example.net` in the above example). You can either use Synapse's
-    built-in [ACME support](./ACME.md) for this (via the `domain` parameter in
-    the `acme` section), or acquire a certificate yourself and give it to
-    Synapse via `tls_certificate_path` and `tls_private_key_path`.
+ 2. Give synapse a certificate corresponding to the target domain
+    (`customer.example.net` in the above example). Currently Synapse's ACME
+    support [does not support
+    this](https://github.com/matrix-org/synapse/issues/4552), so you will have
+    to acquire a certificate yourself and give it to Synapse via
+    `tls_certificate_path` and `tls_private_key_path`.
 
  3. Restart Synapse to ensure the new certificate is loaded.
 

docs/admin_api/user_admin_api.rst

@@ -69,7 +69,7 @@ An empty body may be passed for backwards compatibility.
 Reset password
 ==============
 
-Changes the password of another user. This will automatically log the user out of all their devices.
+Changes the password of another user.
 
 The api is::
 

docs/federate.md

@@ -14,9 +14,9 @@ up and will work provided you set the ``server_name`` to match your
 machine's public DNS hostname, and provide Synapse with a TLS certificate
 which is valid for your ``server_name``.
 
-Once federation has been configured, you should be able to join a room over
-federation. A good place to start is ``#synapse:matrix.org`` - a room for
-Synapse admins.
+Once you have completed the steps necessary to federate, you should be able to 
+join a room via federation. (A good place to start is ``#synapse:matrix.org`` - a 
+room for Synapse admins.)
 
 
 ## Delegation
@@ -98,77 +98,6 @@ _matrix._tcp.<server_name>``. In our example, we would expect this:
 Note that the target of a SRV record cannot be an alias (CNAME record): it has to point
 directly to the server hosting the synapse instance.
 
-### Delegation FAQ
-#### When do I need a SRV record or .well-known URI?
-
-If your homeserver listens on the default federation port (8448), and your
-`server_name` points to the host that your homeserver runs on, you do not need an SRV
-record or `.well-known/matrix/server` URI.
-
-For instance, if you registered `example.com` and pointed its DNS A record at a
-fresh server, you could install Synapse on that host,
-giving it a `server_name` of `example.com`, and once [ACME](acme.md) support is enabled,
-it would automatically generate a valid TLS certificate for you via Let's Encrypt
-and no SRV record or .well-known URI would be needed.
-
-This is the common case, although you can add an SRV record or
-`.well-known/matrix/server` URI for completeness if you wish.
-
-**However**, if your server does not listen on port 8448, or if your `server_name`
-does not point to the host that your homeserver runs on, you will need to let
-other servers know how to find it. The way to do this is via .well-known or an
-SRV record.
-
-#### I have created a .well-known URI. Do I still need an SRV record?
-
-As of Synapse 0.99, Synapse will first check for the existence of a .well-known
-URI and follow any delegation it suggests. It will only then check for the
-existence of an SRV record.
-
-That means that the SRV record will often be redundant. However, you should
-remember that there may still be older versions of Synapse in the federation
-which do not understand .well-known URIs, so if you removed your SRV record
-you would no longer be able to federate with them.
-
-It is therefore best to leave the SRV record in place for now. Synapse 0.34 and
-earlier will follow the SRV record (and not care about the invalid
-certificate). Synapse 0.99 and later will follow the .well-known URI, with the
-correct certificate chain.
-
-#### Can I manage my own certificates rather than having Synapse renew certificates itself?
-
-Yes, you are welcome to manage your certificates yourself. Synapse will only
-attempt to obtain certificates from Let's Encrypt if you configure it to do
-so.The only requirement is that there is a valid TLS cert present for
-federation end points.
-
-#### Do you still recommend against using a reverse proxy on the federation port?
-
-We no longer actively recommend against using a reverse proxy. Many admins will
-find it easier to direct federation traffic to a reverse proxy and manage their
-own TLS certificates, and this is a supported configuration.
-
-See [reverse_proxy.rst](reverse_proxy.rst) for information on setting up a
-reverse proxy.
-
-#### Do I still need to give my TLS certificates to Synapse if I am using a reverse proxy?
-
-Practically speaking, this is no longer necessary.
-
-If you are using a reverse proxy for all of your TLS traffic, then you can set
-`no_tls: True` in the Synapse config. In that case, the only reason Synapse
-needs the certificate is to populate a legacy `tls_fingerprints` field in the
-federation API. This is ignored by Synapse 0.99.0 and later, and the only time
-pre-0.99 Synapses will check it is when attempting to fetch the server keys -
-and generally this is delegated via `matrix.org`, which will be running a modern
-version of Synapse.
-
-#### Do I need the same certificate for the client and federation port?
-
-No. There is nothing stopping you from using different certificates,
-particularly if you are using a reverse proxy. However, Synapse will use the
-same certificate on any ports where TLS is configured.
-
 ## Troubleshooting
 
 You can use the [federation tester](

docs/postgres.rst

@@ -1,7 +1,7 @@
 Using Postgres
 --------------
 
-Postgres version 9.5 or later is known to work.
+Postgres version 9.4 or later is known to work.
 
 Install postgres client libraries
 =================================
@@ -16,7 +16,7 @@ a postgres database.
 * For other pre-built packages, please consult the documentation from the
   relevant package.
 
-* If you installed synapse `in a virtualenv
+* If you installed synapse `in a virtualenv 
   <../INSTALL.md#installing-from-source>`_, you can install the library with::
 
       ~/synapse/env/bin/pip install matrix-synapse[postgres]

docs/sample_config.yaml

@@ -83,16 +83,6 @@ pid_file: DATADIR/homeserver.pid
 #
 #restrict_public_rooms_to_local_users: true
 
-# The default room version for newly created rooms.
-#
-# Known room versions are listed here:
-# https://matrix.org/docs/spec/#complete-list-of-room-versions
-#
-# For example, for room version 1, default_room_version should be set
-# to "1".
-#
-#default_room_version: "4"
-
 # The GC threshold parameters to pass to `gc.set_threshold`, if defined
 #
 #gc_thresholds: [700, 10, 10]
@@ -261,22 +251,6 @@ listeners:
 
 # Monthly Active User Blocking
 #
-# Used in cases where the admin or server owner wants to limit to the
-# number of monthly active users.
-#
-# 'limit_usage_by_mau' disables/enables monthly active user blocking. When
-# anabled and a limit is reached the server returns a 'ResourceLimitError'
-# with error type Codes.RESOURCE_LIMIT_EXCEEDED
-#
-# 'max_mau_value' is the hard limit of monthly active users above which
-# the server will start blocking user actions.
-#
-# 'mau_trial_days' is a means to add a grace period for active users. It
-# means that users must be active for this number of days before they
-# can be considered active and guards against the case where lots of users
-# sign up in a short space of time never to return after their initial
-# session.
-#
 #limit_usage_by_mau: False
 #max_mau_value: 50
 #mau_trial_days: 2
@@ -329,12 +303,12 @@ listeners:
 #
 #tls_private_key_path: "CONFDIR/SERVERNAME.tls.key"
 
-# Whether to verify TLS server certificates for outbound federation requests.
+# Whether to verify TLS certificates when sending federation traffic.
 #
-# Defaults to `true`. To disable certificate verification, uncomment the
-# following line.
+# This currently defaults to `false`, however this will change in
+# Synapse 1.0 when valid federation certificates will be required.
 #
-#federation_verify_certificates: false
+#federation_verify_certificates: true
 
 # Skip federation certificate verification on the following whitelist
 # of domains.
@@ -779,9 +753,7 @@ uploads_path: "DATADIR/uploads"
 # This means that, if a validity period is set, and Synapse is restarted (it will
 # then derive an expiration date from the current validity period), and some time
 # after that the validity period changes and Synapse is restarted, the users'
-# expiration dates won't be updated unless their account is manually renewed. This
-# date will be randomly selected within a range [now + period - d ; now + period],
-# where d is equal to 10% of the validity period.
+# expiration dates won't be updated unless their account is manually renewed.
 #
 #account_validity:
 #  enabled: True
@@ -952,43 +924,12 @@ signing_key_path: "CONFDIR/SERVERNAME.signing.key"
 
 # The trusted servers to download signing keys from.
 #
-# When we need to fetch a signing key, each server is tried in parallel.
-#
-# Normally, the connection to the key server is validated via TLS certificates.
-# Additional security can be provided by configuring a `verify key`, which
-# will make synapse check that the response is signed by that key.
-#
-# This setting supercedes an older setting named `perspectives`. The old format
-# is still supported for backwards-compatibility, but it is deprecated.
-#
-# Options for each entry in the list include:
-#
-#    server_name: the name of the server. required.
-#
-#    verify_keys: an optional map from key id to base64-encoded public key.
-#       If specified, we will check that the response is signed by at least
-#       one of the given keys.
-#
-#    accept_keys_insecurely: a boolean. Normally, if `verify_keys` is unset,
-#       and federation_verify_certificates is not `true`, synapse will refuse
-#       to start, because this would allow anyone who can spoof DNS responses
-#       to masquerade as the trusted key server. If you know what you are doing
-#       and are sure that your network environment provides a secure connection
-#       to the key server, you can set this to `true` to override this
-#       behaviour.
-#
-# An example configuration might look like:
-#
-#trusted_key_servers:
-#  - server_name: "my_trusted_server.example.com"
-#    verify_keys:
-#      "ed25519:auto": "abcdefghijklmnopqrstuvwxyzabcdefghijklmopqr"
-#  - server_name: "my_other_trusted_server.example.com"
-#
-# The default configuration is:
-#
-#trusted_key_servers:
-#  - server_name: "matrix.org"
+#perspectives:
+#  servers:
+#    "matrix.org":
+#      verify_keys:
+#        "ed25519:auto":
+#          key: "Noi6WqcDj0QmPxCNQqgezwTlBKrfqehY1u2FyWP9uYw"
 
 
 # Enable SAML2 for registration and login. Uses pysaml2.
@@ -1065,8 +1006,10 @@ password_config:
 
 
 
-# Enable sending emails for password resets, notification events or
-# account expiry notices
+# Enable sending emails for notification events or expiry notices
+# Defining a custom URL for Riot is only needed if email notifications
+# should contain links to a self-hosted installation of Riot; when set
+# the "app_name" setting is ignored.
 #
 # If your SMTP server requires authentication, the optional smtp_user &
 # smtp_pass variables should be used
@@ -1074,64 +1017,22 @@ password_config:
 #email:
 #   enable_notifs: false
 #   smtp_host: "localhost"
-#   smtp_port: 25 # SSL: 465, STARTTLS: 587
+#   smtp_port: 25
 #   smtp_user: "exampleusername"
 #   smtp_pass: "examplepassword"
 #   require_transport_security: False
 #   notif_from: "Your Friendly %(app)s Home Server <noreply@example.com>"
 #   app_name: Matrix
-#
-#   # Enable email notifications by default
-#   notif_for_new_users: True
-#
-#   # Defining a custom URL for Riot is only needed if email notifications
-#   # should contain links to a self-hosted installation of Riot; when set
-#   # the "app_name" setting is ignored
-#   riot_base_url: "http://localhost/riot"
-#
-#   # Enable sending password reset emails via the configured, trusted
-#   # identity servers
-#   #
-#   # IMPORTANT! This will give a malicious or overtaken identity server
-#   # the ability to reset passwords for your users! Make absolutely sure
-#   # that you want to do this! It is strongly recommended that password
-#   # reset emails be sent by the homeserver instead
-#   #
-#   # If this option is set to false and SMTP options have not been
-#   # configured, resetting user passwords via email will be disabled
-#   #trust_identity_server_for_password_resets: false
-#
-#   # Configure the time that a validation email or text message code
-#   # will expire after sending
-#   #
-#   # This is currently used for password resets
-#   #validation_token_lifetime: 1h
-#
-#   # Template directory. All template files should be stored within this
-#   # directory
-#   #
+#   # if template_dir is unset, uses the example templates that are part of
+#   # the Synapse distribution.
 #   #template_dir: res/templates
-#
-#   # Templates for email notifications
-#   #
 #   notif_template_html: notif_mail.html
 #   notif_template_text: notif_mail.txt
-#
-#   # Templates for account expiry notices
-#   #
+#   # Templates for account expiry notices.
 #   expiry_template_html: notice_expiry.html
 #   expiry_template_text: notice_expiry.txt
-#
-#   # Templates for password reset emails sent by the homeserver
-#   #
-#   #password_reset_template_html: password_reset.html
-#   #password_reset_template_text: password_reset.txt
-#
-#   # Templates for password reset success and failure pages that a user
-#   # will see after attempting to reset their password
-#   #
-#   #password_reset_template_success_html: password_reset_success.html
-#   #password_reset_template_failure_html: password_reset_failure.html
+#   notif_for_new_users: True
+#   riot_base_url: "http://localhost/riot"
 
 
 #password_providers:
@@ -1192,9 +1093,9 @@ password_config:
 #
 # 'search_all_users' defines whether to search all users visible to your HS
 # when searching the user directory, rather than limiting to users visible
-# in public rooms.  Defaults to false.  If you set it True, you'll have to
-# rebuild the user_directory search indexes, see
-# https://github.com/matrix-org/synapse/blob/master/docs/user_directory.md
+# in public rooms.  Defaults to false.  If you set it True, you'll have to run
+# UPDATE user_directory_stream_pos SET stream_id = NULL;
+# on your database to tell it to rebuild the user_directory search indexes.
 #
 #user_directory:
 #  enabled: true
@@ -1351,16 +1252,3 @@ password_config:
 #    alias: "*"
 #    room_id: "*"
 #    action: allow
-
-
-# Server admins can define a Python module that implements extra rules for
-# allowing or denying incoming events. In order to work, this module needs to
-# override the methods defined in synapse/events/third_party_rules.py.
-#
-# This feature is designed to be used in closed federations only, where each
-# participating server enforces the same rules.
-#
-#third_party_event_rules:
-#  module: "my_custom_project.SuperRulesSet"
-#  config:
-#    example_option: 'things'

docs/user_directory.md

@@ -7,7 +7,11 @@ who are present in a publicly viewable room present on the server.
 
 The directory info is stored in various tables, which can (typically after
 DB corruption) get stale or out of sync.  If this happens, for now the
-solution to fix it is to execute the SQL here
-https://github.com/matrix-org/synapse/blob/master/synapse/storage/schema/delta/53/user_dir_populate.sql
-and then restart synapse. This should then start a background task to
+quickest solution to fix it is:
+
+```
+UPDATE user_directory_stream_pos SET stream_id = NULL;
+```
+
+and restart the synapse, which should then start a background task to
 flush the current tables and regenerate the directory.

scripts-dev/federation_client.py

@@ -21,8 +21,7 @@ import argparse
 import base64
 import json
 import sys
-
-from six.moves.urllib import parse as urlparse
+from urlparse import urlparse, urlunparse
 
 import nacl.signing
 import requests
@@ -146,7 +145,7 @@ def request_json(method, origin_name, origin_key, destination, path, content):
 
     for key, sig in signed_json["signatures"][origin_name].items():
         header = "X-Matrix origin=%s,key=\"%s\",sig=\"%s\"" % (origin_name, key, sig)
-        authorization_headers.append(header.encode("ascii"))
+        authorization_headers.append(bytes(header))
         print("Authorization: %s" % header, file=sys.stderr)
 
     dest = "matrix://%s%s" % (destination, path)
@@ -251,7 +250,7 @@ def read_args_from_config(args):
 
 class MatrixConnectionAdapter(HTTPAdapter):
     @staticmethod
-    def lookup(s, skip_well_known=False):
+    def lookup(s):
         if s[-1] == ']':
             # ipv6 literal (with no port)
             return s, 8448
@@ -264,51 +263,19 @@ class MatrixConnectionAdapter(HTTPAdapter):
                 raise ValueError("Invalid host:port '%s'" % s)
             return out[0], port
 
-        # try a .well-known lookup
-        if not skip_well_known:
-            well_known = MatrixConnectionAdapter.get_well_known(s)
-            if well_known:
-                return MatrixConnectionAdapter.lookup(
-                    well_known, skip_well_known=True
-                )
-
         try:
             srv = srvlookup.lookup("matrix", "tcp", s)[0]
             return srv.host, srv.port
         except Exception:
             return s, 8448
 
-    @staticmethod
-    def get_well_known(server_name):
-        uri = "https://%s/.well-known/matrix/server" % (server_name, )
-        print("fetching %s" % (uri, ), file=sys.stderr)
-
-        try:
-            resp = requests.get(uri)
-            if resp.status_code != 200:
-                print("%s gave %i" % (uri, resp.status_code), file=sys.stderr)
-                return None
-
-            parsed_well_known = resp.json()
-            if not isinstance(parsed_well_known, dict):
-                raise Exception("not a dict")
-            if "m.server" not in parsed_well_known:
-                raise Exception("Missing key 'm.server'")
-            new_name = parsed_well_known['m.server']
-            print("well-known lookup gave %s" % (new_name, ), file=sys.stderr)
-            return new_name
-
-        except Exception as e:
-            print("Invalid response from %s: %s" % (uri, e, ), file=sys.stderr)
-        return None
-
     def get_connection(self, url, proxies=None):
-        parsed = urlparse.urlparse(url)
+        parsed = urlparse(url)
 
         (host, port) = self.lookup(parsed.netloc)
         netloc = "%s:%d" % (host, port)
         print("Connecting to %s" % (netloc,), file=sys.stderr)
-        url = urlparse.urlunparse(
+        url = urlunparse(
             ("https", netloc, parsed.path, parsed.params, parsed.query, parsed.fragment)
         )
         return super(MatrixConnectionAdapter, self).get_connection(url, proxies)

scripts-dev/list_url_patterns.py

@@ -20,7 +20,9 @@ class CallVisitor(ast.NodeVisitor):
         else:
             return
 
-        if name == "client_patterns":
+        if name == "client_path_patterns":
+            PATTERNS_V1.append(node.args[0].s)
+        elif name == "client_v2_patterns":
             PATTERNS_V2.append(node.args[0].s)
 
 

scripts/generate_signing_key.py

@@ -1,37 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-# Copyright 2019 The Matrix.org Foundation C.I.C.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-import argparse
-import sys
-
-from signedjson.key import generate_signing_key, write_signing_keys
-
-from synapse.util.stringutils import random_string
-
-if __name__ == "__main__":
-    parser = argparse.ArgumentParser()
-
-    parser.add_argument(
-        "-o", "--output_file",
-
-        type=argparse.FileType('w'),
-        default=sys.stdout,
-        help="Where to write the output to",
-    )
-    args = parser.parse_args()
-
-    key_id = "a_" + random_string(4)
-    key = generate_signing_key(key_id),
-    write_signing_keys(args.output_file, key)

scripts/synapse_port_db

@@ -54,7 +54,6 @@ BOOLEAN_COLUMNS = {
     "group_roles": ["is_public"],
     "local_group_membership": ["is_publicised", "is_admin"],
     "e2e_room_keys": ["is_verified"],
-    "account_validity": ["email_sent"],
 }
 
 

setup.py

@@ -102,16 +102,6 @@ setup(
     include_package_data=True,
     zip_safe=False,
     long_description=long_description,
-    python_requires='~=3.5',
-    classifiers=[
-        'Development Status :: 5 - Production/Stable',
-        'Topic :: Communications :: Chat',
-        'License :: OSI Approved :: Apache Software License',
-        'Programming Language :: Python :: 3 :: Only',
-        'Programming Language :: Python :: 3.5',
-        'Programming Language :: Python :: 3.6',
-        'Programming Language :: Python :: 3.7',
-    ],
     scripts=["synctl"] + glob.glob("scripts/*"),
     cmdclass={'test': TestCommand},
 )

synapse/__init__.py

@@ -17,13 +17,6 @@
 """ This is a reference implementation of a Matrix home server.
 """
 
-import sys
-
-# Check that we're not running on an unsupported Python version.
-if sys.version_info < (3, 5):
-    print("Synapse requires Python 3.5 or above.")
-    sys.exit(1)
-
 try:
     from twisted.internet import protocol
     from twisted.internet.protocol import Factory
@@ -34,4 +27,4 @@ try:
 except ImportError:
     pass
 
-__version__ = "1.0.0"
+__version__ = "0.99.5.1"

synapse/api/auth.py

@@ -184,22 +184,11 @@ class Auth(object):
         return event_auth.get_public_keys(invite_event)
 
     @defer.inlineCallbacks
-    def get_user_by_req(
-        self,
-        request,
-        allow_guest=False,
-        rights="access",
-        allow_expired=False,
-    ):
+    def get_user_by_req(self, request, allow_guest=False, rights="access"):
         """ Get a registered user's ID.
 
         Args:
             request - An HTTP request with an access_token query parameter.
-            allow_expired - Whether to allow the request through even if the account is
-                expired. If true, Synapse will still require an access token to be
-                provided but won't check if the account it belongs to has expired. This
-                works thanks to /login delivering access tokens regardless of accounts'
-                expiration.
         Returns:
             defer.Deferred: resolves to a ``synapse.types.Requester`` object
         Raises:
@@ -240,7 +229,7 @@ class Auth(object):
             is_guest = user_info["is_guest"]
 
             # Deny the request if the user account has expired.
-            if self._account_validity.enabled and not allow_expired:
+            if self._account_validity.enabled:
                 user_id = user.to_string()
                 expiration_ts = yield self.store.get_expiration_ts_for_user(user_id)
                 if expiration_ts is not None and self.clock.time_msec() >= expiration_ts:

synapse/api/errors.py

@@ -339,15 +339,6 @@ class UnsupportedRoomVersionError(SynapseError):
         )
 
 
-class ThreepidValidationError(SynapseError):
-    """An error raised when there was a problem authorising an event."""
-
-    def __init__(self, *args, **kwargs):
-        if "errcode" not in kwargs:
-            kwargs["errcode"] = Codes.FORBIDDEN
-        super(ThreepidValidationError, self).__init__(*args, **kwargs)
-
-
 class IncompatibleRoomVersionError(SynapseError):
     """A server is trying to join a room whose version it does not support.
 

synapse/api/room_versions.py

@@ -50,7 +50,6 @@ class RoomVersion(object):
     disposition = attr.ib()     # str; one of the RoomDispositions
     event_format = attr.ib()    # int; one of the EventFormatVersions
     state_res = attr.ib()       # int; one of the StateResolutionVersions
-    enforce_key_validity = attr.ib()  # bool
 
 
 class RoomVersions(object):
@@ -59,44 +58,43 @@ class RoomVersions(object):
         RoomDisposition.STABLE,
         EventFormatVersions.V1,
         StateResolutionVersions.V1,
-        enforce_key_validity=False,
+    )
+    STATE_V2_TEST = RoomVersion(
+        "state-v2-test",
+        RoomDisposition.UNSTABLE,
+        EventFormatVersions.V1,
+        StateResolutionVersions.V2,
     )
     V2 = RoomVersion(
         "2",
         RoomDisposition.STABLE,
         EventFormatVersions.V1,
         StateResolutionVersions.V2,
-        enforce_key_validity=False,
     )
     V3 = RoomVersion(
         "3",
         RoomDisposition.STABLE,
         EventFormatVersions.V2,
         StateResolutionVersions.V2,
-        enforce_key_validity=False,
     )
     V4 = RoomVersion(
         "4",
         RoomDisposition.STABLE,
         EventFormatVersions.V3,
         StateResolutionVersions.V2,
-        enforce_key_validity=False,
-    )
-    V5 = RoomVersion(
-        "5",
-        RoomDisposition.STABLE,
-        EventFormatVersions.V3,
-        StateResolutionVersions.V2,
-        enforce_key_validity=True,
     )
 
 
+# the version we will give rooms which are created on this server
+DEFAULT_ROOM_VERSION = RoomVersions.V1
+
+
 KNOWN_ROOM_VERSIONS = {
     v.identifier: v for v in (
         RoomVersions.V1,
         RoomVersions.V2,
         RoomVersions.V3,
+        RoomVersions.STATE_V2_TEST,
         RoomVersions.V4,
-        RoomVersions.V5,
     )
 }   # type: dict[str, RoomVersion]

synapse/api/urls.py

@@ -26,7 +26,6 @@ CLIENT_API_PREFIX = "/_matrix/client"
 FEDERATION_PREFIX = "/_matrix/federation"
 FEDERATION_V1_PREFIX = FEDERATION_PREFIX + "/v1"
 FEDERATION_V2_PREFIX = FEDERATION_PREFIX + "/v2"
-FEDERATION_UNSTABLE_PREFIX = FEDERATION_PREFIX + "/unstable"
 STATIC_PREFIX = "/_matrix/static"
 WEB_CLIENT_PREFIX = "/_matrix/client"
 CONTENT_REPO_PREFIX = "/_matrix/content"

synapse/app/_base.py

@@ -344,21 +344,15 @@ class _LimitedHostnameResolver(object):
 
     def resolveHostName(self, resolutionReceiver, hostName, portNumber=0,
                         addressTypes=None, transportSemantics='TCP'):
+        # Note this is happening deep within the reactor, so we don't need to
+        # worry about log contexts.
+
         # We need this function to return `resolutionReceiver` so we do all the
         # actual logic involving deferreds in a separate function.
-
-        # even though this is happening within the depths of twisted, we need to drop
-        # our logcontext before starting _resolve, otherwise: (a) _resolve will drop
-        # the logcontext if it returns an incomplete deferred; (b) _resolve will
-        # call the resolutionReceiver *with* a logcontext, which it won't be expecting.
-        with PreserveLoggingContext():
-            self._resolve(
-                resolutionReceiver,
-                hostName,
-                portNumber,
-                addressTypes,
-                transportSemantics,
-            )
+        self._resolve(
+            resolutionReceiver, hostName, portNumber,
+            addressTypes, transportSemantics,
+        )
 
         return resolutionReceiver
 

synapse/app/frontend_proxy.py

@@ -37,7 +37,8 @@ from synapse.replication.slave.storage.client_ips import SlavedClientIpStore
 from synapse.replication.slave.storage.devices import SlavedDeviceStore
 from synapse.replication.slave.storage.registration import SlavedRegistrationStore
 from synapse.replication.tcp.client import ReplicationClientHandler
-from synapse.rest.client.v2_alpha._base import client_patterns
+from synapse.rest.client.v1.base import ClientV1RestServlet, client_path_patterns
+from synapse.rest.client.v2_alpha._base import client_v2_patterns
 from synapse.server import HomeServer
 from synapse.storage.engines import create_engine
 from synapse.util.httpresourcetree import create_resource_tree
@@ -48,11 +49,11 @@ from synapse.util.versionstring import get_version_string
 logger = logging.getLogger("synapse.app.frontend_proxy")
 
 
-class PresenceStatusStubServlet(RestServlet):
-    PATTERNS = client_patterns("/presence/(?P<user_id>[^/]*)/status")
+class PresenceStatusStubServlet(ClientV1RestServlet):
+    PATTERNS = client_path_patterns("/presence/(?P<user_id>[^/]*)/status")
 
     def __init__(self, hs):
-        super(PresenceStatusStubServlet, self).__init__()
+        super(PresenceStatusStubServlet, self).__init__(hs)
         self.http_client = hs.get_simple_http_client()
         self.auth = hs.get_auth()
         self.main_uri = hs.config.worker_main_http_uri
@@ -83,7 +84,7 @@ class PresenceStatusStubServlet(RestServlet):
 
 
 class KeyUploadServlet(RestServlet):
-    PATTERNS = client_patterns("/keys/upload(/(?P<device_id>[^/]+))?$")
+    PATTERNS = client_v2_patterns("/keys/upload(/(?P<device_id>[^/]+))?$")
 
     def __init__(self, hs):
         """

synapse/app/homeserver.py

@@ -540,7 +540,6 @@ def run(hs):
         stats["total_room_count"] = room_count
 
         stats["daily_active_users"] = yield hs.get_datastore().count_daily_users()
-        stats["monthly_active_users"] = yield hs.get_datastore().count_monthly_users()
         stats["daily_active_rooms"] = yield hs.get_datastore().count_daily_active_rooms()
         stats["daily_messages"] = yield hs.get_datastore().count_daily_messages()
 

synapse/config/emailconfig.py

@@ -1,7 +1,5 @@
 # -*- coding: utf-8 -*-
-# Copyright 2015-2016 OpenMarket Ltd
-# Copyright 2017-2018 New Vector Ltd
-# Copyright 2019 The Matrix.org Foundation C.I.C.
+# Copyright 2015, 2016 OpenMarket Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -19,87 +17,24 @@ from __future__ import print_function
 
 # This file can't be called email.py because if it is, we cannot:
 import email.utils
+import logging
 import os
 
 import pkg_resources
 
 from ._base import Config, ConfigError
 
+logger = logging.getLogger(__name__)
+
 
 class EmailConfig(Config):
     def read_config(self, config):
-        # TODO: We should separate better the email configuration from the notification
-        # and account validity config.
-
         self.email_enable_notifs = False
 
         email_config = config.get("email", {})
-
-        self.email_smtp_host = email_config.get("smtp_host", None)
-        self.email_smtp_port = email_config.get("smtp_port", None)
-        self.email_smtp_user = email_config.get("smtp_user", None)
-        self.email_smtp_pass = email_config.get("smtp_pass", None)
-        self.require_transport_security = email_config.get(
-            "require_transport_security", False
-        )
-        if "app_name" in email_config:
-            self.email_app_name = email_config["app_name"]
-        else:
-            self.email_app_name = "Matrix"
-
-        # TODO: Rename notif_from to something more generic, or have a separate
-        # from for password resets, message notifications, etc?
-        # Currently the email section is a bit bogged down with settings for
-        # multiple functions. Would be good to split it out into separate
-        # sections and only put the common ones under email:
-        self.email_notif_from = email_config.get("notif_from", None)
-        if self.email_notif_from is not None:
-            # make sure it's valid
-            parsed = email.utils.parseaddr(self.email_notif_from)
-            if parsed[1] == '':
-                raise RuntimeError("Invalid notif_from address")
-
-        template_dir = email_config.get("template_dir")
-        # we need an absolute path, because we change directory after starting (and
-        # we don't yet know what auxilliary templates like mail.css we will need).
-        # (Note that loading as package_resources with jinja.PackageLoader doesn't
-        # work for the same reason.)
-        if not template_dir:
-            template_dir = pkg_resources.resource_filename(
-                'synapse', 'res/templates'
-            )
-
-        self.email_template_dir = os.path.abspath(template_dir)
-
         self.email_enable_notifs = email_config.get("enable_notifs", False)
-        account_validity_renewal_enabled = config.get(
-            "account_validity", {},
-        ).get("renew_at")
 
-        email_trust_identity_server_for_password_resets = email_config.get(
-            "trust_identity_server_for_password_resets", False,
-        )
-        self.email_password_reset_behaviour = (
-            "remote" if email_trust_identity_server_for_password_resets else "local"
-        )
-        self.password_resets_were_disabled_due_to_email_config = False
-        if self.email_password_reset_behaviour == "local" and email_config == {}:
-            # We cannot warn the user this has happened here
-            # Instead do so when a user attempts to reset their password
-            self.password_resets_were_disabled_due_to_email_config = True
-
-            self.email_password_reset_behaviour = "off"
-
-        # Get lifetime of a validation token in milliseconds
-        self.email_validation_token_lifetime = self.parse_duration(
-            email_config.get("validation_token_lifetime", "1h")
-        )
-
-        if (
-            self.email_enable_notifs
-            or account_validity_renewal_enabled
-            or self.email_password_reset_behaviour == "local"
-        ):
+        if self.email_enable_notifs:
             # make sure we can import the required deps
             import jinja2
             import bleach
@@ -107,68 +42,6 @@ class EmailConfig(Config):
             jinja2
             bleach
 
-        if self.email_password_reset_behaviour == "local":
-            required = [
-                "smtp_host",
-                "smtp_port",
-                "notif_from",
-            ]
-
-            missing = []
-            for k in required:
-                if k not in email_config:
-                    missing.append(k)
-
-            if (len(missing) > 0):
-                raise RuntimeError(
-                    "email.password_reset_behaviour is set to 'local' "
-                    "but required keys are missing: %s" %
-                    (", ".join(["email." + k for k in missing]),)
-                )
-
-            # Templates for password reset emails
-            self.email_password_reset_template_html = email_config.get(
-                "password_reset_template_html", "password_reset.html",
-            )
-            self.email_password_reset_template_text = email_config.get(
-                "password_reset_template_text", "password_reset.txt",
-            )
-            self.email_password_reset_failure_template = email_config.get(
-                "password_reset_failure_template", "password_reset_failure.html",
-            )
-            # This template does not support any replaceable variables, so we will
-            # read it from the disk once during setup
-            email_password_reset_success_template = email_config.get(
-                "password_reset_success_template", "password_reset_success.html",
-            )
-
-            # Check templates exist
-            for f in [self.email_password_reset_template_html,
-                      self.email_password_reset_template_text,
-                      self.email_password_reset_failure_template,
-                      email_password_reset_success_template]:
-                p = os.path.join(self.email_template_dir, f)
-                if not os.path.isfile(p):
-                    raise ConfigError("Unable to find template file %s" % (p, ))
-
-            # Retrieve content of web templates
-            filepath = os.path.join(
-                self.email_template_dir,
-                email_password_reset_success_template,
-            )
-            self.email_password_reset_success_html_content = self.read_file(
-                filepath,
-                "email.password_reset_template_success_html",
-            )
-
-            if config.get("public_baseurl") is None:
-                raise RuntimeError(
-                    "email.password_reset_behaviour is set to 'local' but no "
-                    "public_baseurl is set. This is necessary to generate password "
-                    "reset links"
-                )
-
-        if self.email_enable_notifs:
             required = [
                 "smtp_host",
                 "smtp_port",
@@ -193,22 +66,11 @@ class EmailConfig(Config):
                     "email.enable_notifs is True but no public_baseurl is set"
                 )
 
+            self.email_smtp_host = email_config["smtp_host"]
+            self.email_smtp_port = email_config["smtp_port"]
+            self.email_notif_from = email_config["notif_from"]
             self.email_notif_template_html = email_config["notif_template_html"]
             self.email_notif_template_text = email_config["notif_template_text"]
-
-            for f in self.email_notif_template_text, self.email_notif_template_html:
-                p = os.path.join(self.email_template_dir, f)
-                if not os.path.isfile(p):
-                    raise ConfigError("Unable to find email template file %s" % (p, ))
-
-            self.email_notif_for_new_users = email_config.get(
-                "notif_for_new_users", True
-            )
-            self.email_riot_base_url = email_config.get(
-                "riot_base_url", None
-            )
-
-        if account_validity_renewal_enabled:
             self.email_expiry_template_html = email_config.get(
                 "expiry_template_html", "notice_expiry.html",
             )
@@ -216,15 +78,58 @@ class EmailConfig(Config):
                 "expiry_template_text", "notice_expiry.txt",
             )
 
-            for f in self.email_expiry_template_text, self.email_expiry_template_html:
-                p = os.path.join(self.email_template_dir, f)
+            template_dir = email_config.get("template_dir")
+            # we need an absolute path, because we change directory after starting (and
+            # we don't yet know what auxilliary templates like mail.css we will need).
+            # (Note that loading as package_resources with jinja.PackageLoader doesn't
+            # work for the same reason.)
+            if not template_dir:
+                template_dir = pkg_resources.resource_filename(
+                    'synapse', 'res/templates'
+                )
+            template_dir = os.path.abspath(template_dir)
+
+            for f in self.email_notif_template_text, self.email_notif_template_html:
+                p = os.path.join(template_dir, f)
                 if not os.path.isfile(p):
                     raise ConfigError("Unable to find email template file %s" % (p, ))
+            self.email_template_dir = template_dir
+
+            self.email_notif_for_new_users = email_config.get(
+                "notif_for_new_users", True
+            )
+            self.email_riot_base_url = email_config.get(
+                "riot_base_url", None
+            )
+            self.email_smtp_user = email_config.get(
+                "smtp_user", None
+            )
+            self.email_smtp_pass = email_config.get(
+                "smtp_pass", None
+            )
+            self.require_transport_security = email_config.get(
+                "require_transport_security", False
+            )
+            if "app_name" in email_config:
+                self.email_app_name = email_config["app_name"]
+            else:
+                self.email_app_name = "Matrix"
+
+            # make sure it's valid
+            parsed = email.utils.parseaddr(self.email_notif_from)
+            if parsed[1] == '':
+                raise RuntimeError("Invalid notif_from address")
+        else:
+            self.email_enable_notifs = False
+            # Not much point setting defaults for the rest: it would be an
+            # error for them to be used.
 
     def default_config(self, config_dir_path, server_name, **kwargs):
         return """
-        # Enable sending emails for password resets, notification events or
-        # account expiry notices
+        # Enable sending emails for notification events or expiry notices
+        # Defining a custom URL for Riot is only needed if email notifications
+        # should contain links to a self-hosted installation of Riot; when set
+        # the "app_name" setting is ignored.
         #
         # If your SMTP server requires authentication, the optional smtp_user &
         # smtp_pass variables should be used
@@ -232,62 +137,20 @@ class EmailConfig(Config):
         #email:
         #   enable_notifs: false
         #   smtp_host: "localhost"
-        #   smtp_port: 25 # SSL: 465, STARTTLS: 587
+        #   smtp_port: 25
         #   smtp_user: "exampleusername"
         #   smtp_pass: "examplepassword"
         #   require_transport_security: False
         #   notif_from: "Your Friendly %(app)s Home Server <noreply@example.com>"
         #   app_name: Matrix
-        #
-        #   # Enable email notifications by default
-        #   notif_for_new_users: True
-        #
-        #   # Defining a custom URL for Riot is only needed if email notifications
-        #   # should contain links to a self-hosted installation of Riot; when set
-        #   # the "app_name" setting is ignored
-        #   riot_base_url: "http://localhost/riot"
-        #
-        #   # Enable sending password reset emails via the configured, trusted
-        #   # identity servers
-        #   #
-        #   # IMPORTANT! This will give a malicious or overtaken identity server
-        #   # the ability to reset passwords for your users! Make absolutely sure
-        #   # that you want to do this! It is strongly recommended that password
-        #   # reset emails be sent by the homeserver instead
-        #   #
-        #   # If this option is set to false and SMTP options have not been
-        #   # configured, resetting user passwords via email will be disabled
-        #   #trust_identity_server_for_password_resets: false
-        #
-        #   # Configure the time that a validation email or text message code
-        #   # will expire after sending
-        #   #
-        #   # This is currently used for password resets
-        #   #validation_token_lifetime: 1h
-        #
-        #   # Template directory. All template files should be stored within this
-        #   # directory
-        #   #
+        #   # if template_dir is unset, uses the example templates that are part of
+        #   # the Synapse distribution.
         #   #template_dir: res/templates
-        #
-        #   # Templates for email notifications
-        #   #
         #   notif_template_html: notif_mail.html
         #   notif_template_text: notif_mail.txt
-        #
-        #   # Templates for account expiry notices
-        #   #
+        #   # Templates for account expiry notices.
         #   expiry_template_html: notice_expiry.html
         #   expiry_template_text: notice_expiry.txt
-        #
-        #   # Templates for password reset emails sent by the homeserver
-        #   #
-        #   #password_reset_template_html: password_reset.html
-        #   #password_reset_template_text: password_reset.txt
-        #
-        #   # Templates for password reset success and failure pages that a user
-        #   # will see after attempting to reset their password
-        #   #
-        #   #password_reset_template_success_html: password_reset_success.html
-        #   #password_reset_template_failure_html: password_reset_failure.html
+        #   notif_for_new_users: True
+        #   riot_base_url: "http://localhost/riot"
         """

synapse/config/homeserver.py

@@ -38,7 +38,6 @@ from .server import ServerConfig
 from .server_notices_config import ServerNoticesConfig
 from .spam_checker import SpamCheckerConfig
 from .stats import StatsConfig
-from .third_party_event_rules import ThirdPartyRulesConfig
 from .tls import TlsConfig
 from .user_directory import UserDirectoryConfig
 from .voip import VoipConfig
@@ -74,6 +73,5 @@ class HomeServerConfig(
     StatsConfig,
     ServerNoticesConfig,
     RoomDirectoryConfig,
-    ThirdPartyRulesConfig,
 ):
     pass

synapse/config/key.py

@@ -1,6 +1,5 @@
 # -*- coding: utf-8 -*-
 # Copyright 2015, 2016 OpenMarket Ltd
-# Copyright 2019 The Matrix.org Foundation C.I.C.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -18,8 +17,6 @@ import hashlib
 import logging
 import os
 
-import attr
-import jsonschema
 from signedjson.key import (
     NACL_ED25519,
     decode_signing_key_base64,
@@ -35,36 +32,11 @@ from synapse.util.stringutils import random_string, random_string_with_symbols
 
 from ._base import Config, ConfigError
 
-INSECURE_NOTARY_ERROR = """\
-Your server is configured to accept key server responses without signature
-validation or TLS certificate validation. This is likely to be very insecure. If
-you are *sure* you want to do this, set 'accept_keys_insecurely' on the
-keyserver configuration."""
-
-RELYING_ON_MATRIX_KEY_ERROR = """\
-Your server is configured to accept key server responses without TLS certificate
-validation, and which are only signed by the old (possibly compromised)
-matrix.org signing key 'ed25519:auto'. This likely isn't what you want to do,
-and you should enable 'federation_verify_certificates' in your configuration.
-
-If you are *sure* you want to do this, set 'accept_keys_insecurely' on the
-trusted_key_server configuration."""
-
-
 logger = logging.getLogger(__name__)
 
 
-@attr.s
-class TrustedKeyServer(object):
-    # string: name of the server.
-    server_name = attr.ib()
-
-    # dict[str,VerifyKey]|None: map from key id to key object, or None to disable
-    # signature verification.
-    verify_keys = attr.ib(default=None)
-
-
 class KeyConfig(Config):
+
     def read_config(self, config):
         # the signing key can be specified inline or in a separate file
         if "signing_key" in config:
@@ -77,27 +49,16 @@ class KeyConfig(Config):
             config.get("old_signing_keys", {})
         )
         self.key_refresh_interval = self.parse_duration(
-            config.get("key_refresh_interval", "1d")
+            config.get("key_refresh_interval", "1d"),
         )
-
-        # if neither trusted_key_servers nor perspectives are given, use the default.
-        if "perspectives" not in config and "trusted_key_servers" not in config:
-            key_servers = [{"server_name": "matrix.org"}]
-        else:
-            key_servers = config.get("trusted_key_servers", [])
-
-            if not isinstance(key_servers, list):
-                raise ConfigError(
-                    "trusted_key_servers, if given, must be a list, not a %s"
-                    % (type(key_servers).__name__,)
-                )
-
-            # merge the 'perspectives' config into the 'trusted_key_servers' config.
-            key_servers.extend(_perspectives_to_key_servers(config))
-
-        # list of TrustedKeyServer objects
-        self.key_servers = list(
-            _parse_key_servers(key_servers, self.federation_verify_certificates)
+        self.perspectives = self.read_perspectives(
+            config.get("perspectives", {}).get("servers", {
+                "matrix.org": {"verify_keys": {
+                    "ed25519:auto": {
+                        "key": "Noi6WqcDj0QmPxCNQqgezwTlBKrfqehY1u2FyWP9uYw",
+                    }
+                }}
+            })
         )
 
         self.macaroon_secret_key = config.get(
@@ -117,9 +78,8 @@ class KeyConfig(Config):
         # falsification of values
         self.form_secret = config.get("form_secret", None)
 
-    def default_config(
-        self, config_dir_path, server_name, generate_secrets=False, **kwargs
-    ):
+    def default_config(self, config_dir_path, server_name, generate_secrets=False,
+                       **kwargs):
         base_key_name = os.path.join(config_dir_path, server_name)
 
         if generate_secrets:
@@ -131,8 +91,7 @@ class KeyConfig(Config):
             macaroon_secret_key = "# macaroon_secret_key: <PRIVATE STRING>"
             form_secret = "# form_secret: <PRIVATE STRING>"
 
-        return (
-            """\
+        return """\
         # a secret which is used to sign access tokens. If none is specified,
         # the registration_shared_secret is used, if one is given; otherwise,
         # a secret key is derived from the signing key.
@@ -174,53 +133,33 @@ class KeyConfig(Config):
 
         # The trusted servers to download signing keys from.
         #
-        # When we need to fetch a signing key, each server is tried in parallel.
-        #
-        # Normally, the connection to the key server is validated via TLS certificates.
-        # Additional security can be provided by configuring a `verify key`, which
-        # will make synapse check that the response is signed by that key.
-        #
-        # This setting supercedes an older setting named `perspectives`. The old format
-        # is still supported for backwards-compatibility, but it is deprecated.
-        #
-        # Options for each entry in the list include:
-        #
-        #    server_name: the name of the server. required.
-        #
-        #    verify_keys: an optional map from key id to base64-encoded public key.
-        #       If specified, we will check that the response is signed by at least
-        #       one of the given keys.
-        #
-        #    accept_keys_insecurely: a boolean. Normally, if `verify_keys` is unset,
-        #       and federation_verify_certificates is not `true`, synapse will refuse
-        #       to start, because this would allow anyone who can spoof DNS responses
-        #       to masquerade as the trusted key server. If you know what you are doing
-        #       and are sure that your network environment provides a secure connection
-        #       to the key server, you can set this to `true` to override this
-        #       behaviour.
-        #
-        # An example configuration might look like:
-        #
-        #trusted_key_servers:
-        #  - server_name: "my_trusted_server.example.com"
-        #    verify_keys:
-        #      "ed25519:auto": "abcdefghijklmnopqrstuvwxyzabcdefghijklmopqr"
-        #  - server_name: "my_other_trusted_server.example.com"
-        #
-        # The default configuration is:
-        #
-        #trusted_key_servers:
-        #  - server_name: "matrix.org"
-        """
-            % locals()
-        )
+        #perspectives:
+        #  servers:
+        #    "matrix.org":
+        #      verify_keys:
+        #        "ed25519:auto":
+        #          key: "Noi6WqcDj0QmPxCNQqgezwTlBKrfqehY1u2FyWP9uYw"
+        """ % locals()
+
+    def read_perspectives(self, perspectives_servers):
+        servers = {}
+        for server_name, server_config in perspectives_servers.items():
+            for key_id, key_data in server_config["verify_keys"].items():
+                if is_signing_algorithm_supported(key_id):
+                    key_base64 = key_data["key"]
+                    key_bytes = decode_base64(key_base64)
+                    verify_key = decode_verify_key_bytes(key_id, key_bytes)
+                    servers.setdefault(server_name, {})[key_id] = verify_key
+        return servers
 
     def read_signing_key(self, signing_key_path):
         signing_keys = self.read_file(signing_key_path, "signing_key")
         try:
             return read_signing_keys(signing_keys.splitlines(True))
         except Exception as e:
-            raise ConfigError("Error reading signing_key: %s" % (str(e)))
+            raise ConfigError(
+                "Error reading signing_key: %s" % (str(e))
+            )
 
     def read_old_signing_keys(self, old_signing_keys):
         keys = {}
@@ -243,7 +182,9 @@ class KeyConfig(Config):
         if not self.path_exists(signing_key_path):
             with open(signing_key_path, "w") as signing_key_file:
                 key_id = "a_" + random_string(4)
-                write_signing_keys(signing_key_file, (generate_signing_key(key_id),))
+                write_signing_keys(
+                    signing_key_file, (generate_signing_key(key_id),),
+                )
         else:
             signing_keys = self.read_file(signing_key_path, "signing_key")
             if len(signing_keys.split("\n")[0].split()) == 1:
@@ -253,116 +194,6 @@ class KeyConfig(Config):
                     NACL_ED25519, key_id, signing_keys.split("\n")[0]
                 )
                 with open(signing_key_path, "w") as signing_key_file:
-                    write_signing_keys(signing_key_file, (key,))
-
-
-def _perspectives_to_key_servers(config):
-    """Convert old-style 'perspectives' configs into new-style 'trusted_key_servers'
-
-    Returns an iterable of entries to add to trusted_key_servers.
-    """
-
-    # 'perspectives' looks like:
-    #
-    # {
-    #     "servers": {
-    #         "matrix.org": {
-    #             "verify_keys": {
-    #                 "ed25519:auto": {
-    #                     "key": "Noi6WqcDj0QmPxCNQqgezwTlBKrfqehY1u2FyWP9uYw"
-    #                 }
-    #             }
-    #         }
-    #     }
-    # }
-    #
-    # 'trusted_keys' looks like:
-    #
-    # [
-    #     {
-    #         "server_name": "matrix.org",
-    #         "verify_keys": {
-    #             "ed25519:auto": "Noi6WqcDj0QmPxCNQqgezwTlBKrfqehY1u2FyWP9uYw",
-    #         }
-    #     }
-    # ]
-
-    perspectives_servers = config.get("perspectives", {}).get("servers", {})
-
-    for server_name, server_opts in perspectives_servers.items():
-        trusted_key_server_entry = {"server_name": server_name}
-        verify_keys = server_opts.get("verify_keys")
-        if verify_keys is not None:
-            trusted_key_server_entry["verify_keys"] = {
-                key_id: key_data["key"] for key_id, key_data in verify_keys.items()
-            }
-        yield trusted_key_server_entry
-
-
-TRUSTED_KEY_SERVERS_SCHEMA = {
-    "$schema": "http://json-schema.org/draft-04/schema#",
-    "description": "schema for the trusted_key_servers setting",
-    "type": "array",
-    "items": {
-        "type": "object",
-        "properties": {
-            "server_name": {"type": "string"},
-            "verify_keys": {
-                "type": "object",
-                # each key must be a base64 string
-                "additionalProperties": {"type": "string"},
-            },
-        },
-        "required": ["server_name"],
-    },
-}
-
-
-def _parse_key_servers(key_servers, federation_verify_certificates):
-    try:
-        jsonschema.validate(key_servers, TRUSTED_KEY_SERVERS_SCHEMA)
-    except jsonschema.ValidationError as e:
-        raise ConfigError("Unable to parse 'trusted_key_servers': " + e.message)
-
-    for server in key_servers:
-        server_name = server["server_name"]
-        result = TrustedKeyServer(server_name=server_name)
-
-        verify_keys = server.get("verify_keys")
-        if verify_keys is not None:
-            result.verify_keys = {}
-            for key_id, key_base64 in verify_keys.items():
-                if not is_signing_algorithm_supported(key_id):
-                    raise ConfigError(
-                        "Unsupported signing algorithm on key %s for server %s in "
-                        "trusted_key_servers" % (key_id, server_name)
+                    write_signing_keys(
+                        signing_key_file, (key,),
                     )
-                try:
-                    key_bytes = decode_base64(key_base64)
-                    verify_key = decode_verify_key_bytes(key_id, key_bytes)
-                except Exception as e:
-                    raise ConfigError(
-                        "Unable to parse key %s for server %s in "
-                        "trusted_key_servers: %s" % (key_id, server_name, e)
-                    )
-
-                result.verify_keys[key_id] = verify_key
-
-        if (
-            not federation_verify_certificates and
-            not server.get("accept_keys_insecurely")
-        ):
-            _assert_keyserver_has_verify_keys(result)
-
-        yield result
-
-
-def _assert_keyserver_has_verify_keys(trusted_key_server):
-    if not trusted_key_server.verify_keys:
-        raise ConfigError(INSECURE_NOTARY_ERROR)
-
-    # also check that they are not blindly checking the old matrix.org key
-    if trusted_key_server.server_name == "matrix.org" and any(
-        key_id == "ed25519:auto" for key_id in trusted_key_server.verify_keys
-    ):
-        raise ConfigError(RELYING_ON_MATRIX_KEY_ERROR)

synapse/config/registration.py

@@ -39,8 +39,6 @@ class AccountValidityConfig(Config):
             else:
                 self.renew_email_subject = "Renew your %(app)s account"
 
-            self.startup_job_max_delta = self.period * 10. / 100.
-
         if self.renew_by_email_enabled and "public_baseurl" not in synapse_config:
             raise ConfigError("Can't send renewal emails without 'public_baseurl'")
 
@@ -131,9 +129,7 @@ class RegistrationConfig(Config):
         # This means that, if a validity period is set, and Synapse is restarted (it will
         # then derive an expiration date from the current validity period), and some time
         # after that the validity period changes and Synapse is restarted, the users'
-        # expiration dates won't be updated unless their account is manually renewed. This
-        # date will be randomly selected within a range [now + period - d ; now + period],
-        # where d is equal to 10%% of the validity period.
+        # expiration dates won't be updated unless their account is manually renewed.
         #
         #account_validity:
         #  enabled: True

synapse/config/server.py

@@ -20,7 +20,6 @@ import os.path
 
 from netaddr import IPSet
 
-from synapse.api.room_versions import KNOWN_ROOM_VERSIONS
 from synapse.http.endpoint import parse_and_validate_server_name
 from synapse.python_dependencies import DependencyException, check_requirements
 
@@ -36,8 +35,6 @@ logger = logging.Logger(__name__)
 # in the list.
 DEFAULT_BIND_ADDRESSES = ['::', '0.0.0.0']
 
-DEFAULT_ROOM_VERSION = "4"
-
 
 class ServerConfig(Config):
 
@@ -91,22 +88,6 @@ class ServerConfig(Config):
             "restrict_public_rooms_to_local_users", False,
         )
 
-        default_room_version = config.get(
-            "default_room_version", DEFAULT_ROOM_VERSION,
-        )
-
-        # Ensure room version is a str
-        default_room_version = str(default_room_version)
-
-        if default_room_version not in KNOWN_ROOM_VERSIONS:
-            raise ConfigError(
-                "Unknown default_room_version: %s, known room versions: %s" %
-                (default_room_version, list(KNOWN_ROOM_VERSIONS.keys()))
-            )
-
-        # Get the actual room version object rather than just the identifier
-        self.default_room_version = KNOWN_ROOM_VERSIONS[default_room_version]
-
         # whether to enable search. If disabled, new entries will not be inserted
         # into the search tables and they will not be indexed. Users will receive
         # errors when attempting to search for messages.
@@ -329,10 +310,6 @@ class ServerConfig(Config):
             unsecure_port = 8008
 
         pid_file = os.path.join(data_dir_path, "homeserver.pid")
-
-        # Bring DEFAULT_ROOM_VERSION into the local-scope for use in the
-        # default config string
-        default_room_version = DEFAULT_ROOM_VERSION
         return """\
         ## Server ##
 
@@ -407,16 +384,6 @@ class ServerConfig(Config):
         #
         #restrict_public_rooms_to_local_users: true
 
-        # The default room version for newly created rooms.
-        #
-        # Known room versions are listed here:
-        # https://matrix.org/docs/spec/#complete-list-of-room-versions
-        #
-        # For example, for room version 1, default_room_version should be set
-        # to "1".
-        #
-        #default_room_version: "%(default_room_version)s"
-
         # The GC threshold parameters to pass to `gc.set_threshold`, if defined
         #
         #gc_thresholds: [700, 10, 10]
@@ -585,22 +552,6 @@ class ServerConfig(Config):
 
         # Monthly Active User Blocking
         #
-        # Used in cases where the admin or server owner wants to limit to the
-        # number of monthly active users.
-        #
-        # 'limit_usage_by_mau' disables/enables monthly active user blocking. When
-        # anabled and a limit is reached the server returns a 'ResourceLimitError'
-        # with error type Codes.RESOURCE_LIMIT_EXCEEDED
-        #
-        # 'max_mau_value' is the hard limit of monthly active users above which
-        # the server will start blocking user actions.
-        #
-        # 'mau_trial_days' is a means to add a grace period for active users. It
-        # means that users must be active for this number of days before they
-        # can be considered active and guards against the case where lots of users
-        # sign up in a short space of time never to return after their initial
-        # session.
-        #
         #limit_usage_by_mau: False
         #max_mau_value: 50
         #mau_trial_days: 2

synapse/config/third_party_event_rules.py

@@ -1,42 +0,0 @@
-# -*- coding: utf-8 -*-
-# Copyright 2019 The Matrix.org Foundation C.I.C.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-from synapse.util.module_loader import load_module
-
-from ._base import Config
-
-
-class ThirdPartyRulesConfig(Config):
-    def read_config(self, config):
-        self.third_party_event_rules = None
-
-        provider = config.get("third_party_event_rules", None)
-        if provider is not None:
-            self.third_party_event_rules = load_module(provider)
-
-    def default_config(self, **kwargs):
-        return """\
-        # Server admins can define a Python module that implements extra rules for
-        # allowing or denying incoming events. In order to work, this module needs to
-        # override the methods defined in synapse/events/third_party_rules.py.
-        #
-        # This feature is designed to be used in closed federations only, where each
-        # participating server enforces the same rules.
-        #
-        #third_party_event_rules:
-        #  module: "my_custom_project.SuperRulesSet"
-        #  config:
-        #    example_option: 'things'
-        """

synapse/config/tls.py

@@ -74,7 +74,7 @@ class TlsConfig(Config):
 
         # Whether to verify certificates on outbound federation traffic
         self.federation_verify_certificates = config.get(
-            "federation_verify_certificates", True,
+            "federation_verify_certificates", False,
         )
 
         # Whitelist of domains to not verify certificates for
@@ -107,7 +107,7 @@ class TlsConfig(Config):
             certs = []
             for ca_file in custom_ca_list:
                 logger.debug("Reading custom CA certificate file: %s", ca_file)
-                content = self.read_file(ca_file, "federation_custom_ca_list")
+                content = self.read_file(ca_file)
 
                 # Parse the CA certificates
                 try:
@@ -241,12 +241,12 @@ class TlsConfig(Config):
         #
         #tls_private_key_path: "%(tls_private_key_path)s"
 
-        # Whether to verify TLS server certificates for outbound federation requests.
+        # Whether to verify TLS certificates when sending federation traffic.
         #
-        # Defaults to `true`. To disable certificate verification, uncomment the
-        # following line.
+        # This currently defaults to `false`, however this will change in
+        # Synapse 1.0 when valid federation certificates will be required.
         #
-        #federation_verify_certificates: false
+        #federation_verify_certificates: true
 
         # Skip federation certificate verification on the following whitelist
         # of domains.

synapse/config/user_directory.py

@@ -43,9 +43,9 @@ class UserDirectoryConfig(Config):
         #
         # 'search_all_users' defines whether to search all users visible to your HS
         # when searching the user directory, rather than limiting to users visible
-        # in public rooms.  Defaults to false.  If you set it True, you'll have to
-        # rebuild the user_directory search indexes, see
-        # https://github.com/matrix-org/synapse/blob/master/docs/user_directory.md
+        # in public rooms.  Defaults to false.  If you set it True, you'll have to run
+        # UPDATE user_directory_stream_pos SET stream_id = NULL;
+        # on your database to tell it to rebuild the user_directory search indexes.
         #
         #user_directory:
         #  enabled: true

synapse/crypto/context_factory.py

@@ -15,13 +15,10 @@
 
 import logging
 
-import idna
-from service_identity import VerificationError
-from service_identity.pyopenssl import verify_hostname, verify_ip_address
 from zope.interface import implementer
 
 from OpenSSL import SSL, crypto
-from twisted.internet._sslverify import _defaultCurveName
+from twisted.internet._sslverify import ClientTLSOptions, _defaultCurveName
 from twisted.internet.abstract import isIPAddress, isIPv6Address
 from twisted.internet.interfaces import IOpenSSLClientConnectionCreator
 from twisted.internet.ssl import CertificateOptions, ContextFactory, platformTrust
@@ -59,19 +56,79 @@ class ServerContextFactory(ContextFactory):
         return self._context
 
 
-class ClientTLSOptionsFactory(object):
-    """Factory for Twisted SSLClientConnectionCreators that are used to make connections
-    to remote servers for federation.
-
-    Uses one of two OpenSSL context objects for all connections, depending on whether
-    we should do SSL certificate verification.
-
-    get_options decides whether we should do SSL certificate verification and
-    constructs an SSLClientConnectionCreator factory accordingly.
+def _idnaBytes(text):
     """
+    Convert some text typed by a human into some ASCII bytes. This is a
+    copy of twisted.internet._idna._idnaBytes. For documentation, see the
+    twisted documentation.
+    """
+    try:
+        import idna
+    except ImportError:
+        return text.encode("idna")
+    else:
+        return idna.encode(text)
+
+
+def _tolerateErrors(wrapped):
+    """
+    Wrap up an info_callback for pyOpenSSL so that if something goes wrong
+    the error is immediately logged and the connection is dropped if possible.
+    This is a copy of twisted.internet._sslverify._tolerateErrors. For
+    documentation, see the twisted documentation.
+    """
+
+    def infoCallback(connection, where, ret):
+        try:
+            return wrapped(connection, where, ret)
+        except:  # noqa: E722, taken from the twisted implementation
+            f = Failure()
+            logger.exception("Error during info_callback")
+            connection.get_app_data().failVerification(f)
+
+    return infoCallback
+
+
+@implementer(IOpenSSLClientConnectionCreator)
+class ClientTLSOptionsNoVerify(object):
+    """
+    Client creator for TLS without certificate identity verification. This is a
+    copy of twisted.internet._sslverify.ClientTLSOptions with the identity
+    verification left out. For documentation, see the twisted documentation.
+    """
+
+    def __init__(self, hostname, ctx):
+        self._ctx = ctx
+
+        if isIPAddress(hostname) or isIPv6Address(hostname):
+            self._hostnameBytes = hostname.encode('ascii')
+            self._sendSNI = False
+        else:
+            self._hostnameBytes = _idnaBytes(hostname)
+            self._sendSNI = True
+
+        ctx.set_info_callback(_tolerateErrors(self._identityVerifyingInfoCallback))
+
+    def clientConnectionForTLS(self, tlsProtocol):
+        context = self._ctx
+        connection = SSL.Connection(context, None)
+        connection.set_app_data(tlsProtocol)
+        return connection
+
+    def _identityVerifyingInfoCallback(self, connection, where, ret):
+        # Literal IPv4 and IPv6 addresses are not permitted
+        # as host names according to the RFCs
+        if where & SSL.SSL_CB_HANDSHAKE_START and self._sendSNI:
+            connection.set_tlsext_host_name(self._hostnameBytes)
+
+
+class ClientTLSOptionsFactory(object):
+    """Factory for Twisted ClientTLSOptions that are used to make connections
+    to remote servers for federation."""
 
     def __init__(self, config):
         self._config = config
+        self._options_noverify = CertificateOptions()
 
         # Check if we're using a custom list of a CA certificates
         trust_root = config.federation_ca_trust_root
@@ -79,13 +136,11 @@ class ClientTLSOptionsFactory(object):
             # Use CA root certs provided by OpenSSL
             trust_root = platformTrust()
 
-        self._verify_ssl_context = CertificateOptions(trustRoot=trust_root).getContext()
-        self._verify_ssl_context.set_info_callback(self._context_info_cb)
-
-        self._no_verify_ssl_context = CertificateOptions().getContext()
-        self._no_verify_ssl_context.set_info_callback(self._context_info_cb)
+        self._options_verify = CertificateOptions(trustRoot=trust_root)
 
     def get_options(self, host):
+        # Use _makeContext so that we get a fresh OpenSSL CTX each time.
+
         # Check if certificate verification has been enabled
         should_verify = self._config.federation_verify_certificates
 
@@ -96,93 +151,6 @@ class ClientTLSOptionsFactory(object):
                     should_verify = False
                     break
 
-        ssl_context = (
-            self._verify_ssl_context if should_verify else self._no_verify_ssl_context
-        )
-
-        return SSLClientConnectionCreator(host, ssl_context, should_verify)
-
-    @staticmethod
-    def _context_info_cb(ssl_connection, where, ret):
-        """The 'information callback' for our openssl context object."""
-        # we assume that the app_data on the connection object has been set to
-        # a TLSMemoryBIOProtocol object. (This is done by SSLClientConnectionCreator)
-        tls_protocol = ssl_connection.get_app_data()
-        try:
-            # ... we further assume that SSLClientConnectionCreator has set the
-            # '_synapse_tls_verifier' attribute to a ConnectionVerifier object.
-            tls_protocol._synapse_tls_verifier.verify_context_info_cb(
-                ssl_connection, where
-            )
-        except:  # noqa: E722, taken from the twisted implementation
-            logger.exception("Error during info_callback")
-            f = Failure()
-            tls_protocol.failVerification(f)
-
-
-@implementer(IOpenSSLClientConnectionCreator)
-class SSLClientConnectionCreator(object):
-    """Creates openssl connection objects for client connections.
-
-    Replaces twisted.internet.ssl.ClientTLSOptions
-    """
-
-    def __init__(self, hostname, ctx, verify_certs):
-        self._ctx = ctx
-        self._verifier = ConnectionVerifier(hostname, verify_certs)
-
-    def clientConnectionForTLS(self, tls_protocol):
-        context = self._ctx
-        connection = SSL.Connection(context, None)
-
-        # as per twisted.internet.ssl.ClientTLSOptions, we set the application
-        # data to our TLSMemoryBIOProtocol...
-        connection.set_app_data(tls_protocol)
-
-        # ... and we also gut-wrench a '_synapse_tls_verifier' attribute into the
-        # tls_protocol so that the SSL context's info callback has something to
-        # call to do the cert verification.
-        setattr(tls_protocol, "_synapse_tls_verifier", self._verifier)
-        return connection
-
-
-class ConnectionVerifier(object):
-    """Set the SNI, and do cert verification
-
-    This is a thing which is attached to the TLSMemoryBIOProtocol, and is called by
-    the ssl context's info callback.
-    """
-
-    # This code is based on twisted.internet.ssl.ClientTLSOptions.
-
-    def __init__(self, hostname, verify_certs):
-        self._verify_certs = verify_certs
-
-        if isIPAddress(hostname) or isIPv6Address(hostname):
-            self._hostnameBytes = hostname.encode("ascii")
-            self._is_ip_address = True
-        else:
-            # twisted's ClientTLSOptions falls back to the stdlib impl here if
-            # idna is not installed, but points out that lacks support for
-            # IDNA2008 (http://bugs.python.org/issue17305).
-            #
-            # We can rely on having idna.
-            self._hostnameBytes = idna.encode(hostname)
-            self._is_ip_address = False
-
-        self._hostnameASCII = self._hostnameBytes.decode("ascii")
-
-    def verify_context_info_cb(self, ssl_connection, where):
-        if where & SSL.SSL_CB_HANDSHAKE_START and not self._is_ip_address:
-            ssl_connection.set_tlsext_host_name(self._hostnameBytes)
-
-        if where & SSL.SSL_CB_HANDSHAKE_DONE and self._verify_certs:
-            try:
-                if self._is_ip_address:
-                    verify_ip_address(ssl_connection, self._hostnameASCII)
-                else:
-                    verify_hostname(ssl_connection, self._hostnameASCII)
-            except VerificationError:
-                f = Failure()
-                tls_protocol = ssl_connection.get_app_data()
-                tls_protocol.failVerification(f)
+        if should_verify:
+            return ClientTLSOptions(host, self._options_verify._makeContext())
+        return ClientTLSOptionsNoVerify(host, self._options_noverify._makeContext())

synapse/crypto/event_signing.py

@@ -31,11 +31,7 @@ logger = logging.getLogger(__name__)
 def check_event_content_hash(event, hash_algorithm=hashlib.sha256):
     """Check whether the hash for this PDU matches the contents"""
     name, expected_hash = compute_content_hash(event.get_pdu_json(), hash_algorithm)
-    logger.debug(
-        "Verifying content hash on %s (expecting: %s)",
-        event.event_id,
-        encode_base64(expected_hash),
-    )
+    logger.debug("Expecting hash: %s", encode_base64(expected_hash))
 
     # some malformed events lack a 'hashes'. Protect against it being missing
     # or a weird type by basically treating it the same as an unhashed event.

synapse/events/builder.py

@@ -76,7 +76,6 @@ class EventBuilder(object):
     # someone tries to get them when they don't exist.
     _state_key = attr.ib(default=None)
     _redacts = attr.ib(default=None)
-    _origin_server_ts = attr.ib(default=None)
 
     internal_metadata = attr.ib(default=attr.Factory(lambda: _EventInternalMetadata({})))
 
@@ -143,9 +142,6 @@ class EventBuilder(object):
         if self._redacts is not None:
             event_dict["redacts"] = self._redacts
 
-        if self._origin_server_ts is not None:
-            event_dict["origin_server_ts"] = self._origin_server_ts
-
         defer.returnValue(
             create_local_event_from_event_dict(
                 clock=self._clock,
@@ -213,7 +209,6 @@ class EventBuilderFactory(object):
             content=key_values.get("content", {}),
             unsigned=key_values.get("unsigned", {}),
             redacts=key_values.get("redacts", None),
-            origin_server_ts=key_values.get("origin_server_ts", None),
         )
 
 
@@ -250,7 +245,7 @@ def create_local_event_from_event_dict(clock, hostname, signing_key,
         event_dict["event_id"] = _create_event_id(clock, hostname)
 
     event_dict["origin"] = hostname
-    event_dict.setdefault("origin_server_ts", time_now)
+    event_dict["origin_server_ts"] = time_now
 
     event_dict.setdefault("unsigned", {})
     age = event_dict["unsigned"].pop("age", 0)

synapse/events/third_party_rules.py

@@ -1,114 +0,0 @@
-# -*- coding: utf-8 -*-
-# Copyright 2019 The Matrix.org Foundation C.I.C.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-from twisted.internet import defer
-
-
-class ThirdPartyEventRules(object):
-    """Allows server admins to provide a Python module implementing an extra
-    set of rules to apply when processing events.
-
-    This is designed to help admins of closed federations with enforcing custom
-    behaviours.
-    """
-
-    def __init__(self, hs):
-        self.third_party_rules = None
-
-        self.store = hs.get_datastore()
-
-        module = None
-        config = None
-        if hs.config.third_party_event_rules:
-            module, config = hs.config.third_party_event_rules
-
-        if module is not None:
-            self.third_party_rules = module(
-                config=config,
-                http_client=hs.get_simple_http_client(),
-            )
-
-    @defer.inlineCallbacks
-    def check_event_allowed(self, event, context):
-        """Check if a provided event should be allowed in the given context.
-
-        Args:
-            event (synapse.events.EventBase): The event to be checked.
-            context (synapse.events.snapshot.EventContext): The context of the event.
-
-        Returns:
-            defer.Deferred[bool]: True if the event should be allowed, False if not.
-        """
-        if self.third_party_rules is None:
-            defer.returnValue(True)
-
-        prev_state_ids = yield context.get_prev_state_ids(self.store)
-
-        # Retrieve the state events from the database.
-        state_events = {}
-        for key, event_id in prev_state_ids.items():
-            state_events[key] = yield self.store.get_event(event_id, allow_none=True)
-
-        ret = yield self.third_party_rules.check_event_allowed(event, state_events)
-        defer.returnValue(ret)
-
-    @defer.inlineCallbacks
-    def on_create_room(self, requester, config, is_requester_admin):
-        """Intercept requests to create room to allow, deny or update the
-        request config.
-
-        Args:
-            requester (Requester)
-            config (dict): The creation config from the client.
-            is_requester_admin (bool): If the requester is an admin
-
-        Returns:
-            defer.Deferred
-        """
-
-        if self.third_party_rules is None:
-            return
-
-        yield self.third_party_rules.on_create_room(
-            requester, config, is_requester_admin
-        )
-
-    @defer.inlineCallbacks
-    def check_threepid_can_be_invited(self, medium, address, room_id):
-        """Check if a provided 3PID can be invited in the given room.
-
-        Args:
-            medium (str): The 3PID's medium.
-            address (str): The 3PID's address.
-            room_id (str): The room we want to invite the threepid to.
-
-        Returns:
-            defer.Deferred[bool], True if the 3PID can be invited, False if not.
-        """
-
-        if self.third_party_rules is None:
-            defer.returnValue(True)
-
-        state_ids = yield self.store.get_filtered_current_state_ids(room_id)
-        room_state_events = yield self.store.get_events(state_ids.values())
-
-        state_events = {}
-        for key, event_id in state_ids.items():
-            state_events[key] = room_state_events[event_id]
-
-        ret = yield self.third_party_rules.check_threepid_can_be_invited(
-            medium, address, state_events,
-        )
-        defer.returnValue(ret)

synapse/events/utils.py

@@ -330,13 +330,12 @@ class EventClientSerializer(object):
         )
 
     @defer.inlineCallbacks
-    def serialize_event(self, event, time_now, bundle_aggregations=True, **kwargs):
+    def serialize_event(self, event, time_now, **kwargs):
         """Serializes a single event.
 
         Args:
             event (EventBase)
             time_now (int): The current time in milliseconds
-            bundle_aggregations (bool): Whether to bundle in related events
             **kwargs: Arguments to pass to `serialize_event`
 
         Returns:
@@ -351,7 +350,7 @@ class EventClientSerializer(object):
 
         # If MSC1849 is enabled then we need to look if thre are any relations
         # we need to bundle in with the event
-        if self.experimental_msc1849_support_enabled and bundle_aggregations:
+        if self.experimental_msc1849_support_enabled:
             annotations = yield self.store.get_aggregation_groups_for_event(
                 event_id,
             )

synapse/federation/federation_base.py

@@ -223,6 +223,9 @@ def _check_sigs_on_pdus(keyring, room_version, pdus):
            the signatures are valid, or fail (with a SynapseError) if not.
     """
 
+    # (currently this is written assuming the v1 room structure; we'll probably want a
+    # separate function for checking v2 rooms)
+
     # we want to check that the event is signed by:
     #
     # (a) the sender's server
@@ -254,10 +257,6 @@ def _check_sigs_on_pdus(keyring, room_version, pdus):
         for p in pdus
     ]
 
-    v = KNOWN_ROOM_VERSIONS.get(room_version)
-    if not v:
-        raise RuntimeError("Unrecognized room version %s" % (room_version,))
-
     # First we check that the sender event is signed by the sender's domain
     # (except if its a 3pid invite, in which case it may be sent by any server)
     pdus_to_check_sender = [
@@ -265,17 +264,10 @@ def _check_sigs_on_pdus(keyring, room_version, pdus):
         if not _is_invite_via_3pid(p.pdu)
     ]
 
-    more_deferreds = keyring.verify_json_objects_for_server(
-        [
-            (
-                p.sender_domain,
-                p.redacted_pdu_json,
-                p.pdu.origin_server_ts if v.enforce_key_validity else 0,
-                p.pdu.event_id,
-            )
-            for p in pdus_to_check_sender
-        ]
-    )
+    more_deferreds = keyring.verify_json_objects_for_server([
+        (p.sender_domain, p.redacted_pdu_json)
+        for p in pdus_to_check_sender
+    ])
 
     def sender_err(e, pdu_to_check):
         errmsg = "event id %s: unable to verify signature for sender %s: %s" % (
@@ -295,23 +287,20 @@ def _check_sigs_on_pdus(keyring, room_version, pdus):
     # event id's domain (normally only the case for joins/leaves), and add additional
     # checks. Only do this if the room version has a concept of event ID domain
     # (ie, the room version uses old-style non-hash event IDs).
+    v = KNOWN_ROOM_VERSIONS.get(room_version)
+    if not v:
+        raise RuntimeError("Unrecognized room version %s" % (room_version,))
+
     if v.event_format == EventFormatVersions.V1:
         pdus_to_check_event_id = [
             p for p in pdus_to_check
             if p.sender_domain != get_domain_from_id(p.pdu.event_id)
         ]
 
-        more_deferreds = keyring.verify_json_objects_for_server(
-            [
-                (
-                    get_domain_from_id(p.pdu.event_id),
-                    p.redacted_pdu_json,
-                    p.pdu.origin_server_ts if v.enforce_key_validity else 0,
-                    p.pdu.event_id,
-                )
-                for p in pdus_to_check_event_id
-            ]
-        )
+        more_deferreds = keyring.verify_json_objects_for_server([
+            (get_domain_from_id(p.pdu.event_id), p.redacted_pdu_json)
+            for p in pdus_to_check_event_id
+        ])
 
         def event_err(e, pdu_to_check):
             errmsg = (

synapse/federation/federation_client.py

@@ -17,6 +17,7 @@
 import copy
 import itertools
 import logging
+import random
 
 from six.moves import range
 
@@ -232,8 +233,7 @@ class FederationClient(FederationBase):
                 moving to the next destination. None indicates no timeout.
 
         Returns:
-            Deferred: Results in the requested PDU, or None if we were unable to find
-               it.
+            Deferred: Results in the requested PDU.
         """
 
         # TODO: Rate limit the number of times we try and get the same event.
@@ -258,12 +258,7 @@ class FederationClient(FederationBase):
                     destination, event_id, timeout=timeout,
                 )
 
-                logger.debug(
-                    "retrieved event id %s from %s: %r",
-                    event_id,
-                    destination,
-                    transaction_data,
-                )
+                logger.debug("transaction_data %r", transaction_data)
 
                 pdu_list = [
                     event_from_pdu_json(p, format_ver, outlier=outlier)
@@ -285,7 +280,6 @@ class FederationClient(FederationBase):
                     "Failed to get PDU %s from %s because %s",
                     event_id, destination, e,
                 )
-                continue
             except NotRetryingDestination as e:
                 logger.info(str(e))
                 continue
@@ -332,16 +326,12 @@ class FederationClient(FederationBase):
             state_event_ids = result["pdu_ids"]
             auth_event_ids = result.get("auth_chain_ids", [])
 
-            fetched_events, failed_to_fetch = yield self.get_events_from_store_or_dest(
-                destination, room_id, set(state_event_ids + auth_event_ids)
+            fetched_events, failed_to_fetch = yield self.get_events(
+                [destination], room_id, set(state_event_ids + auth_event_ids)
             )
 
             if failed_to_fetch:
-                logger.warning(
-                    "Failed to fetch missing state/auth events for %s: %s",
-                    room_id,
-                    failed_to_fetch
-                )
+                logger.warn("Failed to get %r", failed_to_fetch)
 
             event_map = {
                 ev.event_id: ev for ev in fetched_events
@@ -407,20 +397,27 @@ class FederationClient(FederationBase):
         defer.returnValue((signed_pdus, signed_auth))
 
     @defer.inlineCallbacks
-    def get_events_from_store_or_dest(self, destination, room_id, event_ids):
-        """Fetch events from a remote destination, checking if we already have them.
+    def get_events(self, destinations, room_id, event_ids, return_local=True):
+        """Fetch events from some remote destinations, checking if we already
+        have them.
 
         Args:
-            destination (str)
+            destinations (list)
             room_id (str)
             event_ids (list)
+            return_local (bool): Whether to include events we already have in
+                the DB in the returned list of events
 
         Returns:
             Deferred: A deferred resolving to a 2-tuple where the first is a list of
             events and the second is a list of event ids that we failed to fetch.
         """
-        seen_events = yield self.store.get_events(event_ids, allow_rejected=True)
-        signed_events = list(seen_events.values())
+        if return_local:
+            seen_events = yield self.store.get_events(event_ids, allow_rejected=True)
+            signed_events = list(seen_events.values())
+        else:
+            seen_events = yield self.store.have_seen_events(event_ids)
+            signed_events = []
 
         failed_to_fetch = set()
 
@@ -431,11 +428,10 @@ class FederationClient(FederationBase):
         if not missing_events:
             defer.returnValue((signed_events, failed_to_fetch))
 
-        logger.debug(
-            "Fetching unknown state/auth events %s for room %s",
-            missing_events,
-            event_ids,
-        )
+        def random_server_list():
+            srvs = list(destinations)
+            random.shuffle(srvs)
+            return srvs
 
         room_version = yield self.store.get_room_version(room_id)
 
@@ -447,7 +443,7 @@ class FederationClient(FederationBase):
             deferreds = [
                 run_in_background(
                     self.get_pdu,
-                    destinations=[destination],
+                    destinations=random_server_list(),
                     event_id=e_id,
                     room_version=room_version,
                 )

synapse/federation/sender/per_destination_queue.py

@@ -189,21 +189,11 @@ class PerDestinationQueue(object):
 
             pending_pdus = []
             while True:
-                # We have to keep 2 free slots for presence and rr_edus
-                limit = MAX_EDUS_PER_TRANSACTION - 2
-
-                device_update_edus, dev_list_id = (
-                    yield self._get_device_update_edus(limit)
+                device_message_edus, device_stream_id, dev_list_id = (
+                    # We have to keep 2 free slots for presence and rr_edus
+                    yield self._get_new_device_messages(MAX_EDUS_PER_TRANSACTION - 2)
                 )
 
-                limit -= len(device_update_edus)
-
-                to_device_edus, device_stream_id = (
-                    yield self._get_to_device_message_edus(limit)
-                )
-
-                pending_edus = device_update_edus + to_device_edus
-
                 # BEGIN CRITICAL SECTION
                 #
                 # In order to avoid a race condition, we need to make sure that
@@ -218,6 +208,10 @@ class PerDestinationQueue(object):
                 # We can only include at most 50 PDUs per transactions
                 pending_pdus, self._pending_pdus = pending_pdus[:50], pending_pdus[50:]
 
+                pending_edus = []
+
+                # We can only include at most 100 EDUs per transactions
+                # rr_edus and pending_presence take at most one slot each
                 pending_edus.extend(self._get_rr_edus(force_flush=False))
                 pending_presence = self._pending_presence
                 self._pending_presence = {}
@@ -238,6 +232,7 @@ class PerDestinationQueue(object):
                         )
                     )
 
+                pending_edus.extend(device_message_edus)
                 pending_edus.extend(
                     self._pop_pending_edus(MAX_EDUS_PER_TRANSACTION - len(pending_edus))
                 )
@@ -277,13 +272,10 @@ class PerDestinationQueue(object):
                         sent_edus_by_type.labels(edu.edu_type).inc()
                     # Remove the acknowledged device messages from the database
                     # Only bother if we actually sent some device messages
-                    if to_device_edus:
+                    if device_message_edus:
                         yield self._store.delete_device_msgs_for_remote(
                             self._destination, device_stream_id
                         )
-
-                    # also mark the device updates as sent
-                    if device_update_edus:
                         logger.info(
                             "Marking as sent %r %r", self._destination, dev_list_id
                         )
@@ -355,12 +347,11 @@ class PerDestinationQueue(object):
         return pending_edus
 
     @defer.inlineCallbacks
-    def _get_device_update_edus(self, limit):
+    def _get_new_device_messages(self, limit):
         last_device_list = self._last_device_list_stream_id
-
-        # Retrieve list of new device updates to send to the destination
+        # Will return at most 20 entries
         now_stream_id, results = yield self._store.get_devices_by_remote(
-            self._destination, last_device_list, limit=limit,
+            self._destination, last_device_list
         )
         edus = [
             Edu(
@@ -374,19 +365,15 @@ class PerDestinationQueue(object):
 
         assert len(edus) <= limit, "get_devices_by_remote returned too many EDUs"
 
-        defer.returnValue((edus, now_stream_id))
-
-    @defer.inlineCallbacks
-    def _get_to_device_message_edus(self, limit):
         last_device_stream_id = self._last_device_stream_id
         to_device_stream_id = self._store.get_to_device_stream_token()
         contents, stream_id = yield self._store.get_new_device_msgs_for_remote(
             self._destination,
             last_device_stream_id,
             to_device_stream_id,
-            limit,
+            limit - len(edus),
         )
-        edus = [
+        edus.extend(
             Edu(
                 origin=self._server_name,
                 destination=self._destination,
@@ -394,6 +381,6 @@ class PerDestinationQueue(object):
                 content=content,
             )
             for content in contents
-        ]
+        )
 
-        defer.returnValue((edus, stream_id))
+        defer.returnValue((edus, stream_id, now_stream_id))

synapse/federation/transport/server.py

@@ -23,11 +23,7 @@ from twisted.internet import defer
 import synapse
 from synapse.api.errors import Codes, FederationDeniedError, SynapseError
 from synapse.api.room_versions import RoomVersions
-from synapse.api.urls import (
-    FEDERATION_UNSTABLE_PREFIX,
-    FEDERATION_V1_PREFIX,
-    FEDERATION_V2_PREFIX,
-)
+from synapse.api.urls import FEDERATION_V1_PREFIX, FEDERATION_V2_PREFIX
 from synapse.http.endpoint import parse_and_validate_server_name
 from synapse.http.server import JsonResource
 from synapse.http.servlet import (
@@ -94,7 +90,6 @@ class NoAuthenticationError(AuthenticationError):
 
 class Authenticator(object):
     def __init__(self, hs):
-        self._clock = hs.get_clock()
         self.keyring = hs.get_keyring()
         self.server_name = hs.hostname
         self.store = hs.get_datastore()
@@ -103,7 +98,6 @@ class Authenticator(object):
     # A method just so we can pass 'self' as the authenticator to the Servlets
     @defer.inlineCallbacks
     def authenticate_request(self, request, content):
-        now = self._clock.time_msec()
         json_request = {
             "method": request.method.decode('ascii'),
             "uri": request.uri.decode('ascii'),
@@ -140,9 +134,7 @@ class Authenticator(object):
                 401, "Missing Authorization headers", Codes.UNAUTHORIZED,
             )
 
-        yield self.keyring.verify_json_for_server(
-            origin, json_request, now, "Incoming request"
-        )
+        yield self.keyring.verify_json_for_server(origin, json_request)
 
         logger.info("Request from %s", origin)
         request.authenticated_entity = origin
@@ -1312,30 +1304,6 @@ class FederationGroupsSettingJoinPolicyServlet(BaseFederationServlet):
         defer.returnValue((200, new_content))
 
 
-class RoomComplexityServlet(BaseFederationServlet):
-    """
-    Indicates to other servers how complex (and therefore likely
-    resource-intensive) a public room this server knows about is.
-    """
-    PATH = "/rooms/(?P<room_id>[^/]*)/complexity"
-    PREFIX = FEDERATION_UNSTABLE_PREFIX
-
-    @defer.inlineCallbacks
-    def on_GET(self, origin, content, query, room_id):
-
-        store = self.handler.hs.get_datastore()
-
-        is_public = yield store.is_room_world_readable_or_publicly_joinable(
-            room_id
-        )
-
-        if not is_public:
-            raise SynapseError(404, "Room not found", errcode=Codes.INVALID_PARAM)
-
-        complexity = yield store.get_room_complexity(room_id)
-        defer.returnValue((200, complexity))
-
-
 FEDERATION_SERVLET_CLASSES = (
     FederationSendServlet,
     FederationEventServlet,
@@ -1359,7 +1327,6 @@ FEDERATION_SERVLET_CLASSES = (
     FederationThirdPartyInviteExchangeServlet,
     On3pidBindServlet,
     FederationVersionServlet,
-    RoomComplexityServlet,
 )
 
 OPENID_SERVLET_CLASSES = (

synapse/groups/attestations.py

@@ -42,7 +42,7 @@ from signedjson.sign import sign_json
 
 from twisted.internet import defer
 
-from synapse.api.errors import HttpResponseException, RequestSendFailed, SynapseError
+from synapse.api.errors import RequestSendFailed, SynapseError
 from synapse.metrics.background_process_metrics import run_as_background_process
 from synapse.types import get_domain_from_id
 from synapse.util.logcontext import run_in_background
@@ -97,13 +97,10 @@ class GroupAttestationSigning(object):
 
         # TODO: We also want to check that *new* attestations that people give
         # us to store are valid for at least a little while.
-        now = self.clock.time_msec()
-        if valid_until_ms < now:
+        if valid_until_ms < self.clock.time_msec():
             raise SynapseError(400, "Attestation expired")
 
-        yield self.keyring.verify_json_for_server(
-            server_name, attestation, now, "Group attestation"
-        )
+        yield self.keyring.verify_json_for_server(server_name, attestation)
 
     def create_attestation(self, group_id, user_id):
         """Create an attestation for the group_id and user_id with default
@@ -132,10 +129,9 @@ class GroupAttestionRenewer(object):
         self.is_mine_id = hs.is_mine_id
         self.attestations = hs.get_groups_attestation_signing()
 
-        if not hs.config.worker_app:
-            self._renew_attestations_loop = self.clock.looping_call(
-                self._start_renew_attestations, 30 * 60 * 1000,
-            )
+        self._renew_attestations_loop = self.clock.looping_call(
+            self._start_renew_attestations, 30 * 60 * 1000,
+        )
 
     @defer.inlineCallbacks
     def on_renew_attestation(self, group_id, user_id, content):
@@ -195,7 +191,7 @@ class GroupAttestionRenewer(object):
                 yield self.store.update_attestation_renewal(
                     group_id, user_id, attestation
                 )
-            except (RequestSendFailed, HttpResponseException) as e:
+            except RequestSendFailed as e:
                 logger.warning(
                     "Failed to renew attestation of %r in %r: %s",
                     user_id, group_id, e,